CMake: Tarball Support

This speeds up the configure step from 20.1sec to 6.5sec by just downloading a shallow copy of AMReX & pybind11.
AMReX-Codes · Oct 9, 2024 · d068371 · d068371
1 parent 7d23493
commit d068371
Show file tree

Hide file tree

Showing 5 changed files with 90 additions and 38 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -80,6 +80,9 @@ option(pyAMReX_INSTALL
     "Enable install targets for pyAMReX"
     ON
 )
+option(pyAMReX_SUPERBUILD
+    "Download & build extra dependencies"
+    ON)
 
 # change the default build type to Release (or RelWithDebInfo) instead of Debug
 set_default_build_type("Release")

diff --git a/README.md b/README.md
@@ -175,8 +175,9 @@ If you are using the pip-driven install, selected [AMReX CMake options](https://
 | `AMREX_SRC`                  | *None*                                     | Absolute path to AMReX source directory (preferred if set)   |
 | `AMREX_REPO`                 | `https://github.com/AMReX-Codes/amrex.git` | Repository URI to pull and build AMReX from                  |
 | `AMREX_BRANCH`               | `development`                              | Repository branch for `AMREX_REPO`                           |
-| `AMREX_INTERNAL`             | **ON**/OFF                                 | Needs a pre-installed AMReX library if set to `OFF`          |
-| `PYBIND11_INTERNAL`          | **ON**/OFF                                 | Needs a pre-installed pybind11 library if set to `OFF`       |
+| `AMREX_INTERNAL`             | `SUPERBUILD`                               | Needs a pre-installed AMReX library if set to `OFF`          |
+| `PYBIND11_INTERNAL`          | `SUPERBUILD`                               | Needs a pre-installed pybind11 library if set to `OFF`       |
+| `SUPERBUILD`                 | **ON**/OFF                                 | Download & build extra dependencies                          |
 | `CMAKE_BUILD_PARALLEL_LEVEL` | 2                                          | Number of parallel build threads                             |
 | `PYAMREX_LIBDIR`             | *None*                                     | If set, search for pre-built a pyAMReX library               |
 | `PYAMREX_CCACHE`             | **ON**/OFF                                 | Search and use CCache to speed up rebuilds                   |
@@ -192,12 +193,13 @@ Furthermore, pyAMReX adds a few selected CMake build options:
 | `pyAMReX_CCACHE`             | **ON**/OFF                                 | Search and use CCache to speed up rebuilds                    |
 | `pyAMReX_IPO`                | **ON**/OFF                                 | Compile with interprocedural/link optimization (IPO/LTO)      |
 | `pyAMReX_INSTALL`            | **ON**/OFF                                 | Enable install targets for pyAMReX                            |
+| `pyAMReX_SUPERBUILD`         | **ON**/OFF                                 | Download & build extra dependencies                           |
 | `pyAMReX_amrex_src`          | *None*                                     | Absolute path to AMReX source directory (preferred if set)    |
-| `pyAMReX_amrex_internal`     | **ON**/OFF                                 | Needs a pre-installed AMReX library if set to `OFF`           |
+| `pyAMReX_amrex_internal`     | `pyAMReX_SUPERBUILD`                       | Needs a pre-installed AMReX library if set to `OFF`           |
 | `pyAMReX_amrex_repo`         | `https://github.com/AMReX-Codes/amrex.git` | Repository URI to pull and build AMReX from                   |
 | `pyAMReX_amrex_branch`       | `development`                              | Repository branch for `pyAMReX_amrex_repo`                    |
 | `pyAMReX_pybind11_src`       | *None*                                     | Absolute path to pybind11 source directory (preferred if set) |
-| `pyAMReX_pybind11_internal`  | **ON**/OFF                                 | Needs a pre-installed pybind11 library if set to `OFF`        |
+| `pyAMReX_pybind11_internal`  | `pyAMReX_SUPERBUILD`                       | Needs a pre-installed pybind11 library if set to `OFF`        |
 | `pyAMReX_pybind11_repo`      | `https://github.com/pybind/pybind11.git`   | Repository URI to pull and build pybind11 from                |
 | `pyAMReX_pybind11_branch`    | `v2.12.0`                                  | Repository branch for `pyAMReX_pybind11_repo`                 |
 | `Python_EXECUTABLE`          | (newest found)                             | Path to Python executable                                     |

diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake
@@ -1,20 +1,25 @@
 macro(find_amrex)
     if(TARGET AMReX::amrex)
         message(STATUS "AMReX::amrex target already imported")
-    elseif(pyAMReX_amrex_src)
-        message(STATUS "Compiling local AMReX ...")
-        message(STATUS "AMReX source path: ${pyAMReX_amrex_src}")
-        if(NOT IS_DIRECTORY ${pyAMReX_amrex_src})
-            message(FATAL_ERROR "Specified directory pyAMReX_amrex_src='${pyAMReX_amrex_src}' does not exist!")
-        endif()
     elseif(pyAMReX_amrex_internal)
-        message(STATUS "Downloading AMReX ...")
-        message(STATUS "AMReX repository: ${pyAMReX_amrex_repo} (${pyAMReX_amrex_branch})")
-        include(FetchContent)
+        if(pyAMReX_amrex_src)
+            message(STATUS "Compiling local AMReX ...")
+            message(STATUS "AMReX source path: ${pyAMReX_amrex_src}")
+            if(NOT IS_DIRECTORY ${pyAMReX_amrex_src})
+                message(FATAL_ERROR "Specified directory pyAMReX_amrex_src='${pyAMReX_amrex_src}' does not exist!")
+            endif()
+        elseif(pyAMReX_amrex_tar)
+            message(STATUS "Downloading AMReX ...")
+            message(STATUS "AMReX source: ${pyAMReX_amrex_tar}")
+        elseif(pyAMReX_amrex_branch)
+            message(STATUS "Downloading AMReX ...")
+            message(STATUS "AMReX repository: ${pyAMReX_amrex_repo} (${pyAMReX_amrex_branch})")
+            include(FetchContent)
+        endif()
     endif()
     if(TARGET AMReX::amrex)
         # nothing to do, target already exists in the superbuild
-    elseif(pyAMReX_amrex_internal OR pyAMReX_amrex_src)
+    elseif(pyAMReX_amrex_internal)
         set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
 
         # see https://amrex-codes.github.io/amrex/docs_html/BuildingAMReX.html#customization-options
@@ -45,11 +50,20 @@ macro(find_amrex)
             if(AMReX_GPU_BACKEND STREQUAL CUDA)
                 enable_language(CUDA)
             endif()
-            FetchContent_Declare(fetchedamrex
-                GIT_REPOSITORY ${pyAMReX_amrex_repo}
-                GIT_TAG        ${pyAMReX_amrex_branch}
-                BUILD_IN_SOURCE 0
-            )
+            include(FetchContent)
+            if(pyAMReX_amrex_tar)
+                FetchContent_Declare(fetchedamrex
+                    URL             ${pyAMReX_amrex_tar}
+                    URL_HASH        ${pyAMReX_amrex_tar_hash}
+                    BUILD_IN_SOURCE OFF
+                )
+            else()
+                FetchContent_Declare(fetchedamrex
+                    GIT_REPOSITORY ${pyAMReX_amrex_repo}
+                    GIT_TAG        ${pyAMReX_amrex_branch}
+                    BUILD_IN_SOURCE 0
+                )
+            endif()
             FetchContent_MakeAvailable(fetchedamrex)
             list(APPEND CMAKE_MODULE_PATH "${fetchedamrex_SOURCE_DIR}/Tools/CMake")
 
@@ -76,13 +90,22 @@ macro(find_amrex)
     endif()
 endmacro()
 
+option(pyAMReX_amrex_internal "Download & build AMReX" ${pyAMReX_SUPERBUILD})
+
 # local source-tree
 set(pyAMReX_amrex_src ""
     CACHE PATH
     "Local path to AMReX source directory (preferred if set)")
 
+# tarball fetcher
+set(pyAMReX_amrex_tar "https://github.com/AMReX-Codes/amrex/archive/24.10.zip"
+    CACHE STRING
+    "Remote tarball link to pull and build AMReX from if(pyAMReX_amrex_internal)")
+set(pyAMReX_amrex_tar_hash "SHA256=245232a7218f4352892e00b42db9b510f2e66f9b4a33e89bb5fa8c956a18d55c"
+    CACHE STRING
+    "Hash checksum of the tarball of AMReX if(pyAMReX_amrex_internal)")
+
 # Git fetcher
-option(pyAMReX_amrex_internal "Download & build AMReX" ON)
 set(pyAMReX_amrex_repo "https://github.com/AMReX-Codes/amrex.git"
     CACHE STRING
     "Repository URI to pull and build AMReX from if(pyAMReX_amrex_internal)")

diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake
@@ -1,30 +1,44 @@
 function(find_pybind11)
     if(TARGET pybind11::module)
         message(STATUS "pybind11::module target already imported")
-    elseif(pyAMReX_pybind11_src)
-        message(STATUS "Compiling local pybind11 ...")
-        message(STATUS "pybind11 source path: ${pyAMReX_pybind11_src}")
-        if(NOT IS_DIRECTORY ${pyAMReX_pybind11_src})
-            message(FATAL_ERROR "Specified directory pyAMReX_pybind11_src='${pyAMReX_pybind11_src}' does not exist!")
-        endif()
     elseif(pyAMReX_pybind11_internal)
-        message(STATUS "Downloading pybind11 ...")
-        message(STATUS "pybind11 repository: ${pyAMReX_pybind11_repo} (${pyAMReX_pybind11_branch})")
-        include(FetchContent)
+        if(pyAMReX_pybind11_src)
+            message(STATUS "Compiling local pybind11 ...")
+            message(STATUS "pybind11 source path: ${pyAMReX_pybind11_src}")
+            if(NOT IS_DIRECTORY ${pyAMReX_pybind11_src})
+                message(FATAL_ERROR "Specified directory pyAMReX_pybind11_src='${pyAMReX_pybind11_src}' does not exist!")
+            endif()
+        elseif(pyAMReX_pybind11_tar)
+            message(STATUS "Downloading pybind11 ...")
+            message(STATUS "pybind11 source: ${pyAMReX_pybind11_tar}")
+        elseif(pyAMReX_pybind11_branch)
+            message(STATUS "Downloading pybind11 ...")
+            message(STATUS "pybind11 repository: ${pyAMReX_pybind11_repo} (${pyAMReX_pybind11_branch})")
+            include(FetchContent)
+        endif()
     endif()
     if(TARGET pybind11::module)
         # nothing to do, target already exists in the superbuild
-    elseif(pyAMReX_pybind11_internal OR pyAMReX_pybind11_src)
+    elseif(pyAMReX_pybind11_internal)
         set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
 
         if(pyAMReX_pybind11_src)
             add_subdirectory(${pyAMReX_pybind11_src} _deps/localpybind11-build/)
         else()
-            FetchContent_Declare(fetchedpybind11
-                GIT_REPOSITORY ${pyAMReX_pybind11_repo}
-                GIT_TAG        ${pyAMReX_pybind11_branch}
-                BUILD_IN_SOURCE 0
-            )
+            include(FetchContent)
+            if(pyAMReX_pybind11_tar)
+                FetchContent_Declare(fetchedpybind11
+                    URL             ${pyAMReX_pybind11_tar}
+                    URL_HASH        ${pyAMReX_pybind11_tar_hash}
+                    BUILD_IN_SOURCE OFF
+                )
+            else()
+                FetchContent_Declare(fetchedpybind11
+                    GIT_REPOSITORY ${pyAMReX_pybind11_repo}
+                    GIT_TAG        ${pyAMReX_pybind11_branch}
+                    BUILD_IN_SOURCE 0
+                )
+            endif()
             FetchContent_MakeAvailable(fetchedpybind11)
 
             # advanced fetch options
@@ -41,13 +55,22 @@ function(find_pybind11)
     endif()
 endfunction()
 
+option(pyAMReX_pybind11_internal "Download & build pybind11" ${pyAMReX_SUPERBUILD})
+
 # local source-tree
 set(pyAMReX_pybind11_src ""
     CACHE PATH
     "Local path to pybind11 source directory (preferred if set)")
 
+# tarball fetcher
+set(pyAMReX_pybind11_tar "https://github.com/pybind/pybind11/archive/refs/tags/v2.13.6.tar.gz"
+    CACHE STRING
+    "Remote tarball link to pull and build pybind11 from if(pyAMReX_pybind11_internal)")
+set(pyAMReX_pybind11_tar_hash "SHA256=e08cb87f4773da97fa7b5f035de8763abc656d87d5773e62f6da0587d1f0ec20"
+    CACHE STRING
+    "Hash checksum of the tarball of pybind11 if(pyAMReX_pybind11_internal)")
+
 # Git fetcher
-option(pyAMReX_pybind11_internal "Download & build pybind11" ON)
 set(pyAMReX_pybind11_repo "https://github.com/pybind/pybind11.git"
     CACHE STRING
     "Repository URI to pull and build pybind11 from if(pyAMReX_pybind11_internal)")

diff --git a/setup.py b/setup.py
@@ -171,11 +171,12 @@ def build_extension(self, ext):
 AMReX_SPACEDIM = os.environ.get("AMREX_SPACEDIM", "1;2;3")
 BUILD_SHARED_LIBS = os.environ.get("AMREX_BUILD_SHARED_LIBS", "OFF")
 # CMake dependency control (developers & package managers)
+Superbuild = os.environ.get("SUPERBUILD", "ON")
 AMReX_src = os.environ.get("AMREX_SRC")
-AMReX_internal = os.environ.get("AMREX_INTERNAL", "ON")
+AMReX_internal = os.environ.get("AMREX_INTERNAL", Superbuild)
 AMReX_repo = os.environ.get("AMREX_REPO")
 AMReX_branch = os.environ.get("AMREX_BRANCH")
-pybind11_internal = os.environ.get("PYBIND11_INTERNAL", "ON")
+pybind11_internal = os.environ.get("PYBIND11_INTERNAL", Superbuild)
 
 # https://cmake.org/cmake/help/v3.0/command/if.html
 if AMReX_MPI.upper() in ["1", "ON", "TRUE", "YES"]: