diff --git a/tutorial/examples/Autotools/0.package.py b/tutorial/examples/Autotools/0.package.py index 070f9b85c4..b1ff000231 100644 --- a/tutorial/examples/Autotools/0.package.py +++ b/tutorial/examples/Autotools/0.package.py @@ -1,19 +1,19 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. +# Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * +from spack_repo.builtin.build_systems.autotools import AutotoolsPackage +from spack.package import * class Mpileaks(AutotoolsPackage): - """Tool to detect and report leaked MPI objects like MPI_Requests and - MPI_Datatypes.""" + """Tool to detect and report MPI objects like MPI_Requests and + MPI_Datatypes.""" - homepage = "https://github.com/hpc/mpileaks" - url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz" + homepage = "https://github.com/LLNL/mpileaks" + url = "https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz" - version('1.0', '8838c574b39202a57d7c2d68692718aa') + version("1.0", sha256="2e34cc4505556d1c1f085758e26f2f8eea0972db9382f051b2dcfb1d7d9e1825") depends_on("mpi") depends_on("adept-utils") @@ -25,3 +25,4 @@ def install(self, spec, prefix): "--with-callpath=" + spec['callpath'].prefix) make() make("install") + diff --git a/tutorial/examples/Autotools/1.package.py b/tutorial/examples/Autotools/1.package.py index 983f76434c..535eb9ec11 100644 --- a/tutorial/examples/Autotools/1.package.py +++ b/tutorial/examples/Autotools/1.package.py @@ -30,3 +30,4 @@ def configure_args(self): args.extend([f'--with-stack-start-c={stackstart}', f'--with-stack-start-fortran={stackstart}']) return args + diff --git a/tutorial/examples/Cmake/0.package.py b/tutorial/examples/Cmake/0.package.py index cc8d16756e..6e6e23bd61 100644 --- a/tutorial/examples/Cmake/0.package.py +++ b/tutorial/examples/Cmake/0.package.py @@ -1,8 +1,10 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. +# Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# ---------------------------------------------------------------------------- +# If you submit this package back to Spack as a pull request, +# please first remove this boilerplate and all FIXME comments. # # This is a template package file for Spack. We've put "FIXME" # next to all the things you'll want to change. Once you've handled @@ -15,23 +17,37 @@ # spack edit callpath # # See the Spack documentation for more information on packaging. -# If you submit this package back to Spack as a pull request, -# please first remove this boilerplate and all FIXME comments. -# -from spack import * +# ---------------------------------------------------------------------------- + +from spack.package import * +from spack_repo.builtin.build_systems.cmake import CMakePackage class Callpath(CMakePackage): """FIXME: Put a proper description of your package here.""" # FIXME: Add a proper url for your package's homepage here. - homepage = "http://www.example.com" - url = "https://github.com/llnl/callpath/archive/v1.0.1.tar.gz" + homepage = "https://www.example.com" + url = "https://github.com/llnl/callpath/archive/v1.0.3.tar.gz" + + # FIXME: Add a list of GitHub accounts to + # notify when the package is updated. + # maintainers("github_user1", "github_user2") + + # FIXME: Add the SPDX identifier of the project's license below. + # See https://spdx.org/licenses/ for a list. Upon manually verifying + # the license, set checked_by to your Github username. + license("UNKNOWN", checked_by="github_user1") + + version( + "1.0.3", + sha256="a7ddba34de8387a8cb2af9c46bf24e5d307fb196e6dd433707641219c8b4af3e", + ) - version('1.0.3', 'c89089b3f1c1ba47b09b8508a574294a') + depends_on("cxx", type="build") # FIXME: Add dependencies if required. - # depends_on('foo') + # depends_on("foo") def cmake_args(self): # FIXME: Add arguments other than diff --git a/tutorial/examples/Cmake/1.package.py b/tutorial/examples/Cmake/1.package.py index bb6558008e..b7ca4f0280 100644 --- a/tutorial/examples/Cmake/1.package.py +++ b/tutorial/examples/Cmake/1.package.py @@ -1,20 +1,24 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. +# Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * +from spack.package import * +from spack_repo.builtin.build_systems.cmake import CMakePackage class Callpath(CMakePackage): """Library for representing callpaths consistently in - distributed-memory performance tools.""" + distributed-memory performance tools.""" homepage = "https://github.com/llnl/callpath" - url = "https://github.com/llnl/callpath/archive/v1.0.3.tar.gz" + url = "https://github.com/llnl/callpath/archive/v1.0.3.tar.gz" - version('1.0.3', 'c89089b3f1c1ba47b09b8508a574294a') + version( + "1.0.3", + sha256="a7ddba34de8387a8cb2af9c46bf24e5d307fb196e6dd433707641219c8b4af3e", + ) + depends_on("cxx", type="build") depends_on("elf", type="link") depends_on("libdwarf") depends_on("dyninst") diff --git a/tutorial/examples/Cmake/2.package.py b/tutorial/examples/Cmake/2.package.py index ba91061bfe..99751d45e0 100644 --- a/tutorial/examples/Cmake/2.package.py +++ b/tutorial/examples/Cmake/2.package.py @@ -1,20 +1,24 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. +# Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * +from spack.package import * +from spack_repo.builtin.build_systems.cmake import CMakePackage class Callpath(CMakePackage): """Library for representing callpaths consistently in - distributed-memory performance tools.""" + distributed-memory performance tools.""" homepage = "https://github.com/llnl/callpath" - url = "https://github.com/llnl/callpath/archive/v1.0.3.tar.gz" + url = "https://github.com/llnl/callpath/archive/v1.0.3.tar.gz" - version('1.0.3', 'c89089b3f1c1ba47b09b8508a574294a') + version( + "1.0.3", + sha256="a7ddba34de8387a8cb2af9c46bf24e5d307fb196e6dd433707641219c8b4af3e", + ) + depends_on("cxx", type="build") depends_on("elf", type="link") depends_on("libdwarf") depends_on("dyninst") diff --git a/tutorial/examples/PyPackage/0.package.py b/tutorial/examples/PyPackage/0.package.py index 90160ef875..34db8bfcbe 100644 --- a/tutorial/examples/PyPackage/0.package.py +++ b/tutorial/examples/PyPackage/0.package.py @@ -1,41 +1,69 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. +# Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# ---------------------------------------------------------------------------- +# If you submit this package back to Spack as a pull request, +# please first remove this boilerplate and all FIXME comments. # # This is a template package file for Spack. We've put "FIXME" # next to all the things you'll want to change. Once you've handled # them, you can save this file and test your package like this: # -# spack install py-pandas +# spack install py-requests # # You can edit this file again by typing: # -# spack edit py-pandas +# spack edit py-requests # # See the Spack documentation for more information on packaging. -# If you submit this package back to Spack as a pull request, -# please first remove this boilerplate and all FIXME comments. -# -from spack import * +# ---------------------------------------------------------------------------- + +from spack.package import * +from spack_repo.builtin.build_systems.python import PythonPackage -class PyPandas(PythonPackage): +class PyRequests(PythonPackage): """FIXME: Put a proper description of your package here.""" # FIXME: Add a proper url for your package's homepage here. - homepage = "http://www.example.com" - url = "https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz" + homepage = "https://www.example.com" + pypi = "requests/requests-2.32.3.tar.gz" + + # FIXME: Add a list of GitHub accounts to + # notify when the package is updated. + # maintainers("github_user1", "github_user2") + + # FIXME: Add the SPDX identifier of the project's license below. + # See https://spdx.org/licenses/ for a list. Upon manually verifying + # the license, set checked_by to your Github username. + license("UNKNOWN", checked_by="github_user1") + + version( + "2.32.3", + sha256="55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", + ) + + # FIXME: Only add the python/pip/wheel dependencies if you need specific versions + # or need to change the dependency type. Generic python/pip/wheel dependencies are + # added implicity by the PythonPackage base class. + # depends_on("python@2.X:2.Y,3.Z:", type=("build", "run")) + # depends_on("py-pip@X.Y:", type="build") + # depends_on("py-wheel@X.Y:", type="build") + + # FIXME: Add a build backend, usually defined in pyproject.toml. If no such file + # exists, use setuptools. + # depends_on("py-setuptools", type="build") + # depends_on("py-hatchling", type="build") + # depends_on("py-flit-core", type="build") + # depends_on("py-poetry-core", type="build") - version('0.19.0', 'bc9bb7188e510b5d44fbdd249698a2c3') + # FIXME: Add additional dependencies if required. + # depends_on("py-foo", type=("build", "run")) - # FIXME: Add dependencies if required. - # depends_on('py-setuptools', type='build') - # depends_on('py-foo', type=('build', 'run')) + def config_settings(self, spec, prefix): + # FIXME: Add configuration settings to be passed to the build backend + # FIXME: If not needed, delete this function + settings = {} + return settings - def build_args(self, spec, prefix): - # FIXME: Add arguments other than --prefix - # FIXME: If not needed delete this function - args = [] - return args diff --git a/tutorial/examples/PyPackage/1.package.py b/tutorial/examples/PyPackage/1.package.py index b023d300c1..921f1101d8 100644 --- a/tutorial/examples/PyPackage/1.package.py +++ b/tutorial/examples/PyPackage/1.package.py @@ -1,32 +1,23 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. +# Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * - - -class PyPandas(PythonPackage): - """pandas is a Python package providing fast, flexible, and expressive - data structures designed to make working with relational or - labeled data both easy and intuitive. It aims to be the - fundamental high-level building block for doing practical, real - world data analysis in Python. Additionally, it has the broader - goal of becoming the most powerful and flexible open source data - analysis / manipulation tool available in any language. - """ - homepage = "http://pandas.pydata.org/" - url = "https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz" - - version('0.19.0', 'bc9bb7188e510b5d44fbdd249698a2c3') - version('0.18.0', 'f143762cd7a59815e348adf4308d2cf6') - version('0.16.1', 'fac4f25748f9610a3e00e765474bdea8') - version('0.16.0', 'bfe311f05dc0c351f8955fbd1e296e73') - - depends_on('py-dateutil', type=('build', 'run')) - depends_on('py-numpy', type=('build', 'run')) - depends_on('py-setuptools', type='build') - depends_on('py-cython', type='build') - depends_on('py-pytz', type=('build', 'run')) - depends_on('py-numexpr', type=('build', 'run')) - depends_on('py-bottleneck', type=('build', 'run')) +from spack.package import * +from spack_repo.builtin.build_systems.python import PythonPackage + + +class PyRequests(PythonPackage): + """Python HTTP for Humans.""" + + homepage = "https://requests.readthedocs.io" + pypi = "requests/requests-2.32.3.tar.gz" + + version("2.32.3", sha256="55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760") + + depends_on("py-setuptools", type="build") + + depends_on("py-charset-normalizer", type=("build", "run")) + depends_on("py-idna", type=("build", "run")) + depends_on("py-urllib3", type=("build", "run")) + depends_on("py-certifi", type=("build", "run")) + diff --git a/tutorial_buildsystems.rst b/tutorial_buildsystems.rst index d17bfbb95e..23e356cf11 100644 --- a/tutorial_buildsystems.rst +++ b/tutorial_buildsystems.rst @@ -10,12 +10,21 @@ Spack Package Build Systems =========================== -You may begin to notice after writing a couple of package template files that a pattern emerges for some packages. -For example, you may find yourself writing an ``install()`` method that invokes: ``configure``, ``cmake``, ``make``, ``make install``. -You may also find yourself writing ``"prefix=" + prefix`` as an argument to ``configure`` or ``cmake``. -Rather than having you repeat these lines for all packages, Spack has classes that can take care of these patterns. -In addition, these package files allow for finer-grained control of these build systems. -In this section, we will describe each build system and give examples on how these can be used to simplify packaging. +After writing a few package template files, certain recurring patterns often become apparent. +For example, an ``install()`` method may frequently include the following steps: + +- ``configure`` +- ``cmake`` +- ``make`` +- ``make install`` + +It's also common to pass arguments such as ``"prefix=" + prefix`` to ``configure`` or ``cmake``. + +To avoid repeating this logic across packages, Spack provides specialized build system base classes that encapsulate these common patterns. +These classes help reduce boilerplate while still offering fine-grained control over the build process when needed. + +In this section, we'll describe several of these build systems and show how they can be used to simplify and streamline package creation. + ----------------------- Package Class Hierarchy @@ -39,26 +48,29 @@ Package Class Hierarchy PackageBase -> PythonPackage [dir=back] } -The above diagram gives a high-level view of the class hierarchy and how each package relates. +The diagram above provides a high-level view of the class hierarchy and how each package class relates to the others. Each build system specific class inherits from the ``PackageBase`` superclass. -The bulk of the common work is done in this superclass which includes fetching, extracting to a staging directory and the install process. -Each subclass then adds additional build-system-specific functionality. -In the following sections, we will go over examples of how to utilize each subclass and see how powerful these abstractions are when packaging. + +The bulk of the common functionality, such as fetching sources, extracting them into a staging directory, and managing the install process, is implemented in the superclass. +Each subclass then extends this base with build system specific behavior and logic. + +In the following sections, we'll explore examples of how to use these subclasses in practice and demonstrate how powerful these abstractions can be when writing package definitions. + ----------------- Package ----------------- We've already seen examples of using the generic ``Package`` class in our walkthrough for writing package files, so we won't be spending much time with it here. -Briefly, the Package class allows for arbitrary control over the build process, whereas subclasses rely on certain patterns (e.g. ``configure`` ``make`` ``make install``) to be useful. -The generic ``Package`` class is particularly useful for packages that have a non-conventional build process, as it allows the packager to use Spack's helper functions to customize the building and installing of a package fully. +Briefly, the ``Package`` class allows for arbitrary control over the build process, whereas subclasses rely on certain patterns (e.g. ``configure`` ``make`` ``make install``) to be useful. +The generic ``Package`` class is particularly useful for packages that have an unconventional build process, as it allows the packager to use Spack's helper functions to customize the building and installing of a package. ------------------- Autotools ------------------- -As we have seen earlier, packages using ``Autotools`` use ``configure``, ``make`` and ``make install`` commands to execute the build and install process. -In our ``Package`` class, your typical build incantation will consist of the following: +As we've seen earlier, ``Autotools`` packages use ``configure``, ``make`` and ``make install`` commands to execute the build and install process. +In our ``Package`` class, typical build steps would consist of the following: .. code-block:: python @@ -67,7 +79,7 @@ In our ``Package`` class, your typical build incantation will consist of the fol make() make("install") -You'll see that this looks similar to what we wrote in our packaging tutorial. +We'll see that this looks similar to what we wrote in our packaging tutorial. The ``AutotoolsPackage`` subclass aims to simplify writing package files for Autotools-based software and provides convenient methods to manipulate each of the different phases for an ``Autotools`` build system. @@ -87,7 +99,7 @@ Let's take a quick look at some of the internals of the ``Autotools`` class: $ spack edit --build-system autotools -This will open the ``AutotoolsPackage`` file in your text editor. +This will open the ``AutotoolsPackage`` file in the text editor. .. note:: The examples showing code for these classes are abridged to avoid having @@ -101,8 +113,8 @@ This will open the ``AutotoolsPackage`` file in your text editor. Important to note are the highlighted lines. -These properties allow the packager to set what build targets and install targets they want for their package. -If, for example, we wanted to add as our build target ``foo`` then we can append to our ``build_targets`` list: +These properties allow the packager to set build and install targets for their package. +If we wanted to add ``foo`` as our build target, then we can append to our ``build_targets`` list. .. code-block:: python @@ -114,24 +126,24 @@ Which is similar to invoking ``make`` in our Package make("foo") -This is useful if we have packages that ignore environment variables and need a command-line argument. +This is useful if we have packages that ignore environment variables and need a command line argument. Another thing to take note of is in the ``configure()`` method in ``AutotoolsPackage``. -Here we see that the ``--prefix`` argument is already included since it is a common pattern amongst packages using ``Autotools``. +Here we see that the ``--prefix`` argument is already included since it is a common pattern among ``Autotools`` packages. Therefore, we typically only need to override the ``configure_args()`` method to return a list of additional arguments. The ``configure()`` method will then append these to the standard arguments. Packagers also have the option to run ``autoreconf`` in case a package needs to update the build system and generate a new ``configure``. -However, for the most part this will be unnecessary. +However, this is typically not necessary. -Let's look at the ``mpileaks`` package.py file that we worked on earlier: +Let's look at the ``mpileaks`` package.py file we worked on earlier: .. code-block:: console $ spack edit mpileaks -Notice that mpileaks was originally written as a generic ``Package`` but uses the ``Autotools`` build system. -Although this package is acceptable, let's covert it to an ``AutotoolsPackage`` to simplify it further. +Notice that mpileaks was originally written as a generic ``Package``, but uses the ``Autotools`` build system. +While this would build successfully, let's use the ``AutotoolsPackage`` class to simplify it further. .. literalinclude:: tutorial/examples/Autotools/0.package.py :language: python @@ -141,7 +153,7 @@ Although this package is acceptable, let's covert it to an ``AutotoolsPackage`` We first inherit from the ``AutotoolsPackage`` class. -Although we could keep the ``install()`` method, most of it can be handled by the ``AutotoolsPackage`` base class. +We could keep the ``install()`` method, but most of it can be handled by the ``AutotoolsPackage`` base class. In fact, the only thing that needs to be overridden is ``configure_args()``. .. literalinclude:: tutorial/examples/Autotools/1.package.py @@ -149,17 +161,17 @@ In fact, the only thing that needs to be overridden is ``configure_args()``. :emphasize-lines: 25,26,27,28,29,30,31,32 :linenos: -Since Spack's ``AutotoolsPackage`` takes care of setting the prefix for us, we can exclude that as an argument to ``configure``. +Since Spack's ``AutotoolsPackage`` sets the prefix, we can exclude that as an argument to ``configure``. Our package file looks simpler, and we don't need to worry about whether we have properly included ``configure`` and ``make``. -This version of the ``mpileaks`` package installs the same as the previous, but the ``AutotoolsPackage`` class lets us do it with a cleaner looking package file. +This version of the ``mpileaks`` package installs the same as the generic package class, but the ``AutotoolsPackage`` class allows us to leverage powerful abstractions, resulting in a simpler recipe. ----------------- Makefile ----------------- -Packages that utilize ``Make`` or a ``Makefile`` usually require you to edit a ``Makefile`` to set up platform and compiler-specific variables. -These packages are handled by the ``MakefilePackage`` subclass which provides convenience methods to help write these types of packages. +Packages that use ``make`` often require editing the Makefile to configure platform or compiler-specific variables. +These packages are handled by the ``MakefilePackage`` subclass, which provides convenience methods to write package definitions. A ``MakefilePackage`` build has three phases that can be overridden by the packager: @@ -167,9 +179,9 @@ A ``MakefilePackage`` build has three phases that can be overridden by the packa 2. ``build()`` 3. ``install()`` -Packagers then have the ability to control how a ``Makefile`` is edited, and what targets to include for the build phase or install phase. +Packagers then have the ability to control how a ``Makefile`` is edited, and the targets to include for the build or install phase. -Let's also take a look inside the ``MakefilePackage`` class: +Let's take a look inside the ``MakefilePackage`` class: .. code-block:: console @@ -180,11 +192,11 @@ Take note of the following: .. literalinclude:: _spack_root/lib/spack/spack/build_systems/makefile.py :language: python - :emphasize-lines: 60,64,69 + :emphasize-lines: 59,63,68 :lines: 40-111 :linenos: -Similar to ``Autotools``, ``MakefilePackage`` class has properties that can be set by the packager. +Similar to ``Autotools``, the ``MakefilePackage`` class has properties that can be set by the packager. We can also override the different methods highlighted. @@ -210,34 +222,36 @@ Let's try to recreate the Bowtie_ package: ==> Created template for bowtie package ==> Created package file: /Users/mamelara/spack/var/spack/repos/builtin/packages/bowtie/package.py -Once the fetching is completed, Spack will open up your text editor in the usual fashion and create a template of a ``MakefilePackage`` package.py. +Once the fetching is complete, Spack will open a text editor in the usual fashion and create a template of a ``MakefilePackage`` package.py. .. literalinclude:: tutorial/examples/Makefile/0.package.py :language: python :linenos: -Spack was successfully able to detect that ``Bowtie`` uses ``Makefiles``. -Let's add in the rest of our details for our package: +Spack successfully detected that ``Bowtie`` uses ``Makefiles``. +Let's add in the rest of the package's details: .. literalinclude:: tutorial/examples/Makefile/1.package.py :language: python :emphasize-lines: 10,11,13,14,18,20 :linenos: -As we mentioned earlier, most packages using a ``Makefile`` have hardcoded variables that must be edited. -These variables are fine if you happen to not care about setup or types of compilers used, but Spack is designed to work with any compiler. -The ``MakefilePackage`` subclass makes it easy to edit these ``Makefiles`` by having an ``edit()`` method that can be overridden. +As previously mentioned, most packages that use a ``Makefile`` include hardcoded variables that must be edited. +While this setup may be sufficient for basic use cases, it is often inflexible, especially when different compilers or build configurations are required. +Spack is designed to support a wide range of compilers and platforms, and the ``MakefilePackage`` subclass helps accommodate that flexibility. + +The ``MakefilePackage`` class simplifies the process of editing ``Makefiles`` through its overridable ``edit()`` method, which provides a hook for making in-place changes before the build begins. -Let's take a look at the default ``Makefile`` that ``Bowtie`` provides. -If we look inside, we see that ``CC`` and ``CXX`` point to our GNU compiler: +As an example, consider the default ``Makefile`` provided with ``Bowtie``. +Inspecting its contents reveals that ``CC`` and ``CXX`` are hardcoded to the GNU compilers: .. code-block:: console $ spack stage bowtie .. note:: - As usual make sure you have shell support activated with Spack: - ``source /path/to/spack/share/spack/setup-env.sh`` + Ensure Spack's shell support is active before running staging or build commands: + ``source /path/to/spack/share/spack/setup-env.sh`` .. code-block:: console @@ -263,7 +277,7 @@ To fix this, we need to use the ``edit()`` method to modify the ``Makefile``. Here we use a ``FileFilter`` object (a Spack utility) to edit our ``Makefile``. It takes a regular expression to find lines (e.g., assignments to ``CC`` and ``CXX``) and then replaces them with values derived from Spack's build environment (e.g., ``self.compiler.cc`` and ``self.compiler.cxx``). -This allows us to build ``Bowtie`` with whatever compiler we specify through Spack's spec syntax. +This allows us to build ``Bowtie`` with the compiler specified via Spack's spec syntax. Let's change the build and install phases of our package: @@ -272,8 +286,8 @@ Let's change the build and install phases of our package: :emphasize-lines: 28,29,30,31,32,35,36 :linenos: -Here we demonstrate another strategy that we can use to manipulate our package's build. -We can provide command-line arguments to ``make()``. +Here we demonstrate another strategy we can use to manipulate our package's build. +We can provide command line arguments to ``make()``. Since ``Bowtie`` can use ``tbb`` we can either add ``NO_TBB=1`` as a argument to prevent ``tbb`` support, or we can invoke ``make`` with no arguments if TBB is desired and found by its build system. ``Bowtie`` requires our ``install_target`` to provide a path to the install directory. @@ -287,37 +301,50 @@ Let's look at a couple of other examples and go through them: Some packages allow environment variables to be set and will honor them. Packages that use ``?=`` for assignment in their ``Makefile`` can be set using environment variables. -In our ``esmf`` example we set two environment variables in our ``edit()`` method: +In our ``esmf`` example we set two environment variables in our ``setup_build_environment()`` method: .. code-block:: python - def edit(self, spec, prefix): + def setup_build_environment(self, env: EnvironmentModifications) -> None: + spec = self.spec + # Installation instructions can be found at: + # http://www.earthsystemmodeling.org/esmf_releases/last_built/ESMF_usrdoc/node9.html + + # Unset any environment variables that may influence the installation. for var in os.environ: - if var.startswith('ESMF_'): - os.environ.pop(var) + if var.startswith("ESMF_"): + env.unset(var) # More code ... - if self.compiler.name == 'gcc': - os.environ['ESMF_COMPILER'] = 'gfortran' - elif self.compiler.name == 'intel': - os.environ['ESMF_COMPILER'] = 'intel' - elif self.compiler.name == 'clang': - os.environ['ESMF_COMPILER'] = 'gfortranclang' - elif self.compiler.name == 'nag': - os.environ['ESMF_COMPILER'] = 'nag' - elif self.compiler.name == 'pgi': - os.environ['ESMF_COMPILER'] = 'pgi' + if spec["fortran"].name == "gcc" and spec["c"].name == "gcc": + gfortran_major_version = int(spec["fortran"].version[0]) + env.set("ESMF_COMPILER", "gfortran") + elif self.pkg.compiler.name == "intel" or self.pkg.compiler.name == "oneapi": + env.set("ESMF_COMPILER", "intel") + elif spec["fortran"].name == "gcc" and spec["c"].name in ["clang", "apple-clang"]: + gfortran_major_version = int(spec["fortran"].version[0]) + env.set("ESMF_COMPILER", "gfortranclang") + elif spec["fortran"].name == "llvm": + env.set("ESMF_COMPILER", "llvm") + elif self.pkg.compiler.name == "nag": + env.set("ESMF_COMPILER", "nag") + elif self.pkg.compiler.name == "nvhpc": + env.set("ESMF_COMPILER", "nvhpc") + elif self.pkg.compiler.name == "cce": + env.set("ESMF_COMPILER", "cce") + elif self.pkg.compiler.name == "aocc": + env.set("ESMF_COMPILER", "aocc") else: - msg = "The compiler you are building with, " - msg += "'{0}', is not supported by ESMF." - raise InstallError(msg.format(self.compiler.name)) + msg = "The compiler you are building with, " + msg += '"{0}", is not supported by ESMF.' + raise InstallError(msg.format(self.pkg.compiler.name)) -As you may have noticed, we didn't really write anything to the ``Makefile`` but rather we set environment variables that will override variables set in the ``Makefile``. +In this example, nothing was written directly to the Makefile. Instead, environment variables were set to override those defined within the Makefile. Some packages include a configuration file that sets certain compiler variables, platform specific variables, and the location of dependencies or libraries. -If the file is simple and only requires a couple of changes, we can replace those entries with our ``FileFilter`` object. -If the configuration involves complex changes, we can write a new configuration file from scratch within the ``edit()`` method. +If the file is simple and only requires a couple changes, we can replace those entries with our ``FileFilter`` object. +If the changes will be more complex, we can write a new configuration file from scratch within the ``edit()`` method. Let's look at an example of this in the ``elk`` package: @@ -414,9 +441,9 @@ By the end of the ``edit()`` method, we write the contents of our dictionary to CMake --------------- -CMake_ is another common build system that has been gaining popularity. -It works in a similar manner to ``Autotools`` but with differences in variable names, the number of configuration options available, and the handling of shared libraries. -Typical build incantations look like this: +CMake_ is another popular build system. +It works in a similar manner to ``Autotools`` but with differences in variable names, configuration options, and handling of shared libraries. +Typical build steps look like this: .. _CMake: https://cmake.org @@ -427,7 +454,7 @@ Typical build incantations look like this: make() make("install") -As you can see from the example above, it's very similar to invoking ``configure`` and ``make`` in an ``Autotools`` build system. +As shown in the example above, the process is very similar to invoking ``configure`` and ``make`` in an ``Autotools`` build system. However, the variable names and options differ. Most options in CMake are prefixed with a ``'-D'`` flag to indicate a configuration setting. @@ -437,9 +464,9 @@ In the ``CMakePackage`` class, we can override the following build phases: 2. ``build()`` 3. ``install()`` -The ``CMakePackage`` class also provides sensible defaults, so often we only need to override ``cmake_args()`` to pass package-specific options. +The ``CMakePackage`` class provides sensible defaults, so we only need to override ``cmake_args()`` to pass package-specific options. -Let's look at these defaults in the ``CMakePackage`` class in the ``_std_args()`` method: +Let's look at these defaults in the ``_std_args()`` method of the ``CMakePackage`` class. .. code-block:: console @@ -454,13 +481,13 @@ Let's look at these defaults in the ``CMakePackage`` class in the ``_std_args()` Some ``CMake`` packages use different generators. Spack is able to support Unix-Makefile_ generators as well as Ninja_ generators. -.. _Unix-Makefile: https://cmake.org/cmake/help/v3.4/generator/Unix%20Makefiles.html -.. _Ninja: https://cmake.org/cmake/help/v3.4/generator/Ninja.html +.. _Unix-Makefile: https://cmake.org/cmake/help/latest/generator/Unix%20Makefiles.html +.. _Ninja: https://cmake.org/cmake/help/latest/generator/Ninja.html If no generator is specified, Spack will default to ``Unix Makefiles``. -Next we setup the build type. -In ``CMake`` you can specify the build type that you want. +Next, we'll set up the build type. +In ``CMake``, we can specify the build type. Options include: 1. ``empty`` @@ -469,24 +496,22 @@ Options include: 4. ``RelWithDebInfo`` 5. ``MinSizeRel`` -With these options you can specify whether you want your executable to have the debug version only, release version or the release with debug information. Release executables tend to be more optimized than Debug versions. In Spack, we set the default as `Release` unless otherwise specified through a variant (e.g., ``build_type=Debug``). -Spack then automatically sets up the ``-DCMAKE_INSTALL_PREFIX`` path, appends the build type (defaulting to ``RelWithDebInfo``), and enables a verbose ``Makefile`` output by default. - -Next we add the ``rpaths`` to ``-DCMAKE_INSTALL_RPATH:STRING``. +Spack then automatically sets the ``-DCMAKE_INSTALL_PREFIX`` path, appends the build type (defaulting to ``RelWithDebInfo``), and enables a verbose ``Makefile`` output by default. +Next, we add the ``rpaths`` to ``-DCMAKE_INSTALL_RPATH:STRING``. -Finally we add to ``-DCMAKE_PREFIX_PATH:STRING`` the locations of all our dependencies so that ``CMake`` can find them. +Finally, we add the locations of our dependencies to ``-DCMAKE_PREFIX_PATH:STRING`` so ``CMake`` can find them. -In the end our ``cmake`` line will look like this (example is ``xrootd``): +The Spack-generated ``cmake`` line will look like this (example is ``xrootd``): .. code-block:: console $ cmake $HOME/spack/var/spack/stage/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk/xrootd-4.6.0 -G Unix Makefiles -DCMAKE_INSTALL_PREFIX:PATH=$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_FIND_FRAMEWORK:STRING=LAST -DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=FALSE -DCMAKE_INSTALL_RPATH:STRING=$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk/lib:$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk/lib64 -DCMAKE_PREFIX_PATH:STRING=$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/cmake-3.9.4-hally3vnbzydiwl3skxcxcbzsscaasx5 -We can see now how ``CMake`` takes care of a lot of the boilerplate code that would have to be otherwise typed in. +We'll now explore how ``CMake`` takes care of the boilerplate code that would otherwise be manually written. Let's try to recreate callpath_: @@ -496,21 +521,11 @@ Let's try to recreate callpath_: $ spack create -f https://github.com/llnl/callpath/archive/v1.0.3.tar.gz ==> This looks like a URL for callpath - ==> Found 4 versions of callpath: - - 1.0.3 https://github.com/LLNL/callpath/archive/v1.0.3.tar.gz - 1.0.2 https://github.com/LLNL/callpath/archive/v1.0.2.tar.gz - 1.0.1 https://github.com/LLNL/callpath/archive/v1.0.1.tar.gz - 1.0 https://github.com/LLNL/callpath/archive/v1.0.tar.gz - - ==> How many would you like to checksum? (default is 1, q to abort) 1 - ==> Downloading... - ==> Fetching https://github.com/LLNL/callpath/archive/v1.0.3.tar.gz - ######################################################################## 100.0% - ==> Checksummed 1 version of callpath + ==> Fetching https://github.com/llnl/callpath/archive/v1.0.3.tar.gz + [100%] 47.00 KB @ 916.6 KB/s ==> This package looks like it uses the cmake build system ==> Created template for callpath package - ==> Created package file: /Users/mamelara/spack/var/spack/repos/builtin/packages/callpath/package.py + ==> Created package file: /home/spack/spack/var/spack/repos/spack_repo/builtin/packages/callpath/package.py which then produces the following template: @@ -519,30 +534,27 @@ which then produces the following template: :language: python :linenos: -Again we fill in the details: +We'll fill in the details: .. literalinclude:: tutorial/examples/Cmake/1.package.py :language: python :linenos: - :emphasize-lines: 9,13,14,18,19,20,21,22,23 - -As mentioned earlier, Spack's ``CMakePackage`` uses sensible defaults to reduce boilerplate and simplify writing package files for ``CMake``-based software. + :emphasize-lines: 9,13,14,22,23,24,25,26,27 -In ``callpath``, we want to control options like ``CALLPATH_WALKER`` or add specific compiler flags. -We can return these options from ``cmake_args()`` like so: +In ``callpath``, we want to control options like ``CALLPATH_WALKER`` and add compiler flags. +We can return these options from ``cmake_args()``: .. literalinclude:: tutorial/examples/Cmake/2.package.py :language: python :linenos: - :emphasize-lines: 26,30,31 + :emphasize-lines: 30,34 -Now we can control our build options using ``cmake_args()``. -If defaults are sufficient enough for the package, we can leave this method out. +We can now control our build options using ``cmake_args()``. If defaults are sufficient for this package, this method can be left out. ``CMakePackage`` classes allow for control of other features in the build system. -For example, you can specify the path to the "out of source" build directory and also point to the root of the ``CMakeLists.txt`` file if it is placed in a non-standard location. +For example, it is possible to specify the path to an out-of-source build directory and indicate the location of the ``CMakeLists.txt`` file if it resides in a non-standard location. -A good example of a package that has its ``CMakeLists.txt`` file located at a different location is found in ``spades``. +``spades`` has its ``CMakeLists.txt`` file located outside of the standard path. .. code-block:: console @@ -552,17 +564,15 @@ A good example of a package that has its ``CMakeLists.txt`` file located at a di root_cmakelists_dir = "src" -Here ``root_cmakelists_dir`` will tell Spack where to find the location of ``CMakeLists.txt``. -In this example, it is located a directory level below in the ``src`` directory. +``root_cmakelists_dir`` will tell Spack where to find the location of ``CMakeLists.txt``. -Some ``CMake`` packages also require the ``install`` phase to be overridden. -For example, let's take a look at ``sniffles``. +Some packages, like ``sniffles``, require the ``install`` phase to be overridden. .. code-block:: console $ spack edit sniffles -In the ``install()`` method, we have to manually install our targets so we override the ``install()`` method to do it for us: +In the ``install()`` method, the targets are manually installed, so we override the method to do it for us: .. code-block:: python @@ -580,109 +590,91 @@ PythonPackage -------------- Python extensions and modules are built differently from source than most applications. -These modules are usually installed using the following line: - -.. code-block:: console - - $ pip install . +These modules are usually installed by running ``pip install .``. +Package definitions for Python packages can be written using the ``Package`` class, but it contains many methods that are not useful for this use case. -We can write package files for Python packages using the ``Package`` class, but the class brings with it a lot of methods that are useless for Python packages. -Instead, Spack has a ``PythonPackage`` subclass that allows packagers of Python modules to be able to invoke ``pip``. +Spack provides a ``PythonPackage`` subclass to allow for easier installation of Python modules. -We will write a package file for Pandas_: +We'll write a package file for requests_: -.. _pandas: https://pandas.pydata.org +.. _requests: https://requests.readthedocs.io/en/latest/ .. code-block:: console - $ spack create -f https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz - ==> This looks like a URL for pandas - ==> Warning: Spack was unable to fetch url list due to a certificate verification problem. You can try running spack -k, which will not check SSL certificates. Use this at your own risk. - ==> Found 1 version of pandas: + $ spack create -f https://pypi.io/packages/source/p/requests/requests-2.32.3.tar.gz + ==> This looks like a URL for requests + ==> Selected 152 versions + 2.32.3 https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz#sha256=55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 + .. - 0.19.0 https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz + ==> Enter number of versions to take, or use a command: + [c]hecksum [e]dit [f]ilter [a]sk each [n]ew only [r]estart [q]uit + action> 1 + ==> Selected 1 of 152 versions + .. + + ==> Enter number of versions to take, or use a command: + [c]hecksum [e]dit [f]ilter [a]sk each [n]ew only [r]estart [q]uit + action> c + .. - ==> How many would you like to checksum? (default is 1, q to abort) 1 - ==> Downloading... - ==> Fetching https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz - ######################################################################## 100.0% - ==> Checksummed 1 version of pandas ==> This package looks like it uses the python build system - ==> Changing package name from pandas to py-pandas - ==> Created template for py-pandas package - ==> Created package file: /Users/mamelara/spack/var/spack/repos/builtin/packages/py-pandas/package.py + ==> Changing package name from requests to py-requests + ==> Created template for py-requests package + ==> Created package file: /home/spack/spack/var/spack/repos/spack_repo/builtin/packages/py_requests/package.py -And we are left with the following template: + +Spack generates the following template: .. literalinclude:: tutorial/examples/PyPackage/0.package.py :language: python :linenos: -As you can see this is not any different than any package template that we have written. -We have the choice of providing build options or using the sensible defaults. +This is similar to other package templates; we have the choice to provide build options or use sensible defaults. -Luckily for us, there is no need to provide build args. +Next, we'll need to find the dependencies for ``requests``. +Dependencies are usually listed in ``pyproject.toml``, ``setup.py``, ``setup.cfg``, or ``requirements.txt``. -Next we need to find the dependencies of a package. -Dependencies are usually listed in ``setup.py``. -You can find the dependencies by searching for ``install_requires`` keyword in that file. -Here it is for ``Pandas``: +Here's the relevant block for ``requests``: .. code-block:: python + # setup.cfg - # ... code - if sys.version_info[0] >= 3: - - setuptools_kwargs = { - 'zip_safe': False, - 'install_requires': ['python-dateutil >= 2', - 'pytz >= 2011k', - 'numpy >= %s' % min_numpy_ver], - 'setup_requires': ['numpy >= %s' % min_numpy_ver], - } - if not _have_setuptools: - sys.exit("need setuptools/distribute for Py3k" - "\n$ pip install distribute") + requires-dist = + certifi>=2017.4.17 + charset_normalizer>=2,<4 + idna>=2.5,<4 + urllib3>=1.21.1,<3 - # ... more code - -You can find a more comprehensive list at the Pandas documentation_. - -.. _documentation: https://pandas.pydata.org/pandas-docs/stable/install.html - - -By reading the documentation and ``setup.py`` we found that ``Pandas`` depends on ``python-dateutil``, ``pytz``, and ``numpy``, ``numexpr``, and finally ``bottleneck``. - -Here is the completed ``Pandas`` script: +We now have our project dependencies! It's important to declare them for Python packages, as Spack won't be able to properly load your package without this information. .. literalinclude:: tutorial/examples/PyPackage/1.package.py :language: python :linenos: -It is quite important to declare all the dependencies of a Python package. -Spack can "activate" Python packages to prevent the user from having to load each dependency module explicitly. -If a dependency is missed, Spack will be unable to properly activate the package and it will cause an issue. -To learn more about extensions go to `spack extensions `_. - -From this example, you can see that building Python modules is made easy through the ``PythonPackage`` class. +For more information about leveraging ``PythonPackage``, see the `docs `_. ------------------- Other Build Systems ------------------- -Although we won't get in depth with any of the other build systems that Spack supports, it is worth mentioning that Spack does provide subclasses for the following build systems: +While we won't go in depth on the other build systems that Spack supports, it's worth noting that Spack provides support for many specialized build systems beyond the ones covered in this tutorial. + +Some examples include: + +1. `RPackage `_ +2. `MesonPackage `_ +3. `PerlPackage `_ +4. `CUDAPackage `_ +5. `ROCmPackage `_ -1. ``IntelPackage`` -2. ``SconsPackage`` -3. ``WafPackage`` -4. ``RPackage`` -5. ``PerlPackage`` -6. ``QMakePackage`` +...and `many more `_! +Each of these build system classes provides abstractions to simplify and standardize the process of writing package recipes. +They help manage common build logic and reduce duplication across packages in the same ecosystem. -Each of these classes have their own abstractions to help assist in writing package files. -For whatever doesn't fit nicely into the other build systems, you can use the ``Package`` class. +For packages that don't align well with any specific build system, Spack also provides a generic ``Package`` base class that gives full control over the build process. -Hopefully by now you can see how we aim to make packaging simple and robust through these classes. -If you want to learn more about these build systems, check out `Implementing the installation procedure `_ in the Packaging Guide. +By now, we've seen how Spack aims to make packaging both simple and robust through its build system abstractions. +To learn more, refer to the `Overview of the installation procedure `_ in the Packaging Guide.