[py] Update pypi dependency installation scripts

.github/workflows/pypi.yml

@@ -20,10 +20,6 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-tags: true
-
-      - name: Get work dir
-        id: work-dir
-        run: echo "dir=$(pwd)" >> "$GITHUB_OUTPUT"
 
       - uses: actions/setup-python@v5
         with:
@@ -48,21 +44,26 @@ jobs:
           path: ${{ steps.cibuildwheel-cache.outputs.dir }}
           key: cibuildwheel-cache-${{ matrix.os }}
 
+      - name: Cache test files 
+        id: cache-test-files
+        uses: actions/cache@v4
+        with:
+          path: ${{ github.workspace }}/bpcells-pytest-data-cache
+          key: test-files-cache-${{ matrix.os }}
+
       - name: Cache dependency libs
         id: cache-libs
         uses: actions/cache@v4
         with:
-          path: ${{ steps.work-dir.outputs.dir }}/lib-cache
-          key: lib-cache-${{ matrix.os }}
+          path: ${{ github.workspace }}/lib-cache
+          key: lib-cache-v2-${{ matrix.os }}
 
-      # From: https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-actions-cache
-      - name: Export GitHub Actions cache environment variables for vcpkg
-        uses: actions/github-script@v7
-        if: ${{ matrix.os }} == 'windows-latest'
-        with:
-          script: |
-            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
-            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+      - name: Install dependency libs (Windows)
+        if: steps.cache-libs.outputs.cache-hit != 'true' && runner.os == 'Windows'
+        run: |
+          vcpkg install hdf5 eigen3 highway[contrib] zlib
+          vcpkg export --raw hdf5 eigen3 highway zlib --output-dir=${{ github.workspace }} --output=vcpkg-export
+          XCOPY /I /E ${{ github.workspace }}\vcpkg-export\installed\x64-windows ${{ github.workspace }}\lib-cache
 
       - name: Build wheels
         uses: pypa/cibuildwheel@v2.19.1
@@ -71,44 +72,35 @@ jobs:
         env:
           CIBW_ENVIRONMENT_LINUX: >-
             CXX="ccache g++"
-            LIB_CACHE="${{ steps.work-dir.outputs.dir }}/lib-cache"
-          CIBW_BEFORE_ALL_LINUX: >
-            LIB_CACHE=/host/$LIB_CACHE bash {package}/scripts/install_ccache.sh &&
-            LIB_CACHE=/host/$LIB_CACHE bash {package}/scripts/install_eigen.sh &&
-            LIB_CACHE=/host/$LIB_CACHE bash {package}/scripts/install_hdf5.sh &&
-            LIB_CACHE=/host/$LIB_CACHE bash {package}/scripts/install_highway.sh
+          CIBW_BEFORE_ALL_LINUX: >-
+            LIB_CACHE=/host/$LIB_CACHE bash {package}/scripts/install_ccache_linux.sh &&
+            python {package}/scripts/install_deps.py /host/${{ github.workspace }}/lib-cache
 
           # We need to set MACOSX_DEPLOYMENT_TARGET="10.15" on macos-13 because otherwise
           # it tries to target a MacOS verision before std::filesystem is available (10.9 currently)
           CIBW_ENVIRONMENT_MACOS: >-
             CXX="ccache g++"
-            LIB_CACHE="${{ steps.work-dir.outputs.dir }}/lib-cache"
-            CPATH="$(pwd)/lib-root/include"
-            LIBRARY_PATH="$(pwd)/lib-root/lib:$(pwd)/lib-root/lib64"
+            CPATH="${{ github.workspace }}/lib-cache/include"
+            LIBRARY_PATH="${{ github.workspace }}/lib-cache/lib:${{ github.workspace }}/lib-cache/lib64"
             ${{ matrix.os == 'macos-13' && 'MACOSX_DEPLOYMENT_TARGET="10.15"' || '' }}
-          CIBW_BEFORE_ALL_MACOS: >
+          CIBW_BEFORE_ALL_MACOS: >-
             brew install ccache &&
-            mkdir -p $LIB_CACHE &&
-            bash {package}/scripts/install_eigen.sh $(pwd)/lib-root &&
-            bash {package}/scripts/install_hdf5.sh $(pwd)/lib-root &&
-            bash {package}/scripts/install_highway.sh $(pwd)/lib-root
+            python {package}/scripts/install_deps.py ${{ github.workspace }}/lib-cache
           
           # VCPKG_BINARY_SOURCES from: https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-actions-cache
           CIBW_ENVIRONMENT_WINDOWS: >- 
-            VCPKG_BINARY_SOURCES="clear;x-gha,readwrite" 
-            LIBRARY_PATH="C:\\vcpkg\\installed\\x64-windows\\lib"
-            CPATH="C:\\vcpkg\\installed\\x64-windows\\include"
-            BPCELLS_PYTEST_DATA_CACHE="C:\\Users\\runneradmin\\AppData\\Local\\Temp\\bpcells-pytest-data-cache"
+            LIBRARY_PATH="${{ github.workspace }}\\lib-cache\\lib"
+            CPATH="${{ github.workspace }}\\lib-cache\\include"
+            BPCELLS_PYTEST_DATA_CACHE="${{ github.workspace }}\\bpcells-pytest-data-cache"
           # As of 8/26/2024, a delvewheel dependency version bump (pefile) broke delvewheel, so pin these 
           CIBW_BEFORE_ALL_WINDOWS: >-
-            vcpkg install hdf5 eigen3 highway[contrib] zlib && 
             pip install delvewheel==1.8.0 pefile==2023.2.7
           # See https://github.com/adang1345/delvewheel/issues/54 for explanation of `--add-path C:/Windows/System32`
           CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >-
             delvewheel repair --add-path C:/vcpkg/installed/x64-windows/bin --add-path C:/Windows/System32 --wheel-dir {dest_dir} {wheel} 
 
           CIBW_TEST_REQUIRES: pytest h5py anndata
-          CIBW_TEST_COMMAND: BPCELLS_PYTEST_DATA_CACHE=/tmp/bpcells-pytest-data-cache pytest {package}/tests
+          CIBW_TEST_COMMAND: BPCELLS_PYTEST_DATA_CACHE="${{ github.workspace }}/bpcells-pytest-data-cache" pytest {package}/tests
           # Data cache folder is already set
           CIBW_TEST_COMMAND_WINDOWS: pytest {package}/tests
 

python/DEVELOP.md

@@ -1,33 +1,61 @@
 ## Building from source
 
-1. Install required C++ dependencies, setting any required environment variables for
-   compilation:
-    - Eigen3:
-        - Typically available via package managers, e.g. `apt-get install libeigen3-dev`
-        - Installation path must be passed via standard environment variables, e.g. 
-        `export CPATH="${CPATH:+$CPATH:}/usr/include/eigen3"` (note that typical installations
-        prefix with the eigen3 directory, necessitating this CPATH modification)
-    - HDF5:
-        - Typically available via package managers, e.g. `apt-get install libhdf5-dev`
-        - Any non-standard installation paths must be passed via the standard environment variables `CPATH` for include and `LIBRARY_PATH` for lib, 
-        e.g. `export CPATH="${CPATH:+$CPATH:}/usr/include/hdf5/serial"` and `export LIBRARY_PATH=${LIBRARY_PATH:+$LIBRARY_PATH:}/usr/lib/hdf5/serial`
-    - Highway:
-        - Available [here](https://github.com/google/highway/), and also for Debian available via
-        `apt-get install libhwy-dev`. Requires cmake to build from source
-        - `bash ./scripts/install_highway_cmake.sh $(pwd)/highway` will fetch source and install, taking one optional
-        command-line argument to pass installation prefix
-        - Non-standard installation paths can be specified by setting `CPATH` and `LIBRARY_PATH` as with HDF5, e.g. `export CPATH=${CPATH:+$CPATH:}$(pwd)/highway/include; export LIBRARY_PATH=${LIBRARY_PATH:+$LIBRARY_PATH:}$(pwd)/highway/lib`
-2. Run `pip install .`
-
-On Linux/Mac machines the `scripts/install_{eigen,hdf5,highway}.sh` will perform builds from
-source of the respective libraries. These scripts accept one optional parameter with the 
-root directory to install into (default `/usr/local`). They also can optionally use the
-`LIB_CACHE` environment variable to help cache outputs across CI jobs.
+The core dependencies required are: Eigen3, HDF5, and [Highway](https://github.com/google/highway/).
+
+### Setup on Linux/Mac
+
+Build dependencies from source:
+```shell
+mkdir -p build/deps-root
+python scripts/install_deps.py build/deps-root
+```
+
+Then install via pip with appropriate environment variables set:
+```shell
+export CPATH="$(pwd)/build/deps-root/include"
+export LIBRARY_PATH="$(pwd)/build/deps-root/lib:$(pwd)/build/deps-root/lib64"
+
+pip install .
+```
+
+After having built the dependencies once, just running the `export` and `pip` commands is sufficient to re-build.
+
+Re-building can also be sped up by installing `ccache` (`brew install ccache`, `sudo apt install ccache`, etc.), then setting:
+```shell
+export CXX="ccache g++"
+export CC="ccache gcc"
+```
+
+### Setup on Windows
+
+Recommend using vcpkg to install dependencies. ([Installation instructions](https://learn.microsoft.com/en-us/vcpkg/get_started/get-started?pivots=shell-cmd#1---set-up-vcpkg))
+
+```shell
+vcpkg install hdf5 eigen3 highway[contrib] zlib 
+```
+
+Then set environment variables and run pip (assuming vcpkg root is at `C:\vcpkg`)
+
+PowerShell
+```powershell
+$env:CPATH = 'C:\vcpkg\installed\x64-windows\include'
+$env:LIBRARY_PATH = 'C:\vcpkg\installed\x64-windows\lib'
+
+pip install .
+```
+
+Command Prompt
+```shell
+set CPATH=C:\vcpkg\installed\x64-windows\include
+set LIBRARY_PATH=C:\vcpkg\installed\x64-windows\lib
+
+pip install .
+```
 
 ## Running tests
 BPCells uses pytest, so just running `pytest` will run man unit tests.
 Certain slow-running tests that require downloading data will only run if
-the environment variable `BPCELLS_PYTEST_DATA` is set, pointing to the
+the environment variable `BPCELLS_PYTEST_DATA_CACHE` is set, pointing to the
 directory that should be used to cache file downloads.
 
 ### Test dependencies
@@ -50,16 +78,9 @@ The exact right setup will depend on your OS, but this works well for Ubuntu
 ```bash
 layout python3
 
-# Eigen3 installation path (depends on your OS setup; this works for Ubuntu/Debian)
-export CPATH="${CPATH:+$CPATH:}/usr/include/eigen3"
-
-# HDF5 installation path (depends on your OS setup; this works for Ubuntu/Debian)
-export CPATH="${CPATH:+$CPATH:}/usr/include/hdf5/serial"
-export LIBRARY_PATH="${LIBRARY_PATH:+$LIBRARY_PATH:}/usr/lib/x86_64-linux-gnu/hdf5/serial"
-
-# Export highway lib path for building
-export CPATH="${CPATH:+$CPATH:}$(pwd)/highway/include"
-export LIBRARY_PATH="${LIBRARY_PATH:+$LIBRARY_PATH:}$(pwd)/highway/lib"
+# Export path to the pre-built dependencies
+export CPATH="$(pwd)/build/deps-root/include"
+export LIBRARY_PATH="$(pwd)/build/deps-root/lib:$(pwd)/build/deps-root/lib64"
 
 # Use ccache to speed up re-compilation cycles
 export CXX="ccache g++"

python/scripts/install_deps.py

@@ -0,0 +1,25 @@
+
+import tempfile
+from pathlib import Path
+import sys
+import subprocess
+
+# Run the install script for each dependency. Do builds in a temporary directory
+
+file_dir = Path(__file__).parent.resolve()
+
+temp_dir = tempfile.TemporaryDirectory()
+
+if len(sys.argv) > 1:
+    sys.argv[1] = Path(sys.argv[1]).resolve()
+
+def run_script(script_name):
+    args = ["bash", file_dir / script_name]
+    if len(sys.argv) > 1:
+        args.append(sys.argv[1])
+    subprocess.run(args, check=True, stdout=sys.stdout, stderr=sys.stderr, cwd=temp_dir.name)
+
+print("Running builds in ", temp_dir.name)
+run_script("install_eigen.sh")
+run_script("install_hdf5.sh")
+run_script("install_highway.sh")

python/scripts/install_eigen.sh

@@ -1,4 +1,4 @@
-set -euo
+set -euo pipefail
 set -x
 
 
@@ -8,26 +8,11 @@ else
     INSTALL_DIR="/usr/local"
 fi
 
-# Check for cached files if LIB_CACHE is set.
-# (This is useful for CI builds to not re-build dependencies from scratch)
-if [ ! -z "${LIB_CACHE+x}" ] && [ -f "${LIB_CACHE}/eigen/include/Eigen/Core" ]; then
-    echo "Copying Eigen from cache"
-    mkdir -p "$INSTALL_DIR/include"
-    cp -r "${LIB_CACHE}/eigen/include/Eigen" "$INSTALL_DIR/include"
-    exit 0;
-fi
-
 curl -L https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz > eigen.tar.gz
 tar -xzf eigen.tar.gz
 rm eigen.tar.gz
 
 mkdir -p "$INSTALL_DIR"/include
 cp -r eigen-3.4.0/Eigen $INSTALL_DIR/include
 
-# Put files in cache
-if [ ! -z "${LIB_CACHE+x}" ]; then
-    mkdir -p "${LIB_CACHE}/eigen/include"
-    cp -r "$INSTALL_DIR/include/Eigen" "${LIB_CACHE}/eigen/include/"
-fi
-
 rm -rf eigen-3.4.0

python/scripts/install_hdf5.sh

@@ -1,4 +1,4 @@
-set -euo
+set -euo pipefail
 set -x
 
 
@@ -8,23 +8,6 @@ else
     INSTALL_DIR="/usr/local"
 fi
 
-# Check for cached files if LIB_CACHE is set.
-# (This is useful for CI builds to not re-build dependencies from scratch)
-if [ ! -z "${LIB_CACHE+x}" ] && [ -f "${LIB_CACHE}/hdf5/lib/libhdf5.so" ]; then
-    echo "Copying hdf5 from cache"
-    mkdir -p "$INSTALL_DIR"/include "$INSTALL_DIR"/lib 
-    cp "${LIB_CACHE}"/hdf5/include/* "$INSTALL_DIR"/include
-    cp "${LIB_CACHE}"/hdf5/lib/* "$INSTALL_DIR"/lib 
-    exit 0;
-fi
-
-# Cache is empty, so fill cache
-if [ ! -z "${LIB_CACHE+x}" ]; then
-    MAIN_INSTALL_DIR="$INSTALL_DIR"
-    INSTALL_DIR="$LIB_CACHE/hdf5"
-    mkdir -p "$INSTALL_DIR"
-fi
-
 curl -L https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5_1.14.4.3.tar.gz > hdf5.tar.gz
 tar -xzf hdf5.tar.gz
 rm hdf5.tar.gz
@@ -35,11 +18,4 @@ make -j4
 make install
 cd ..
 
-# Fill cache
-if [ ! -z "${LIB_CACHE+x}" ]; then
-    mkdir -p "$MAIN_INSTALL_DIR"/include "$MAIN_INSTALL_DIR"/lib
-    cp "${LIB_CACHE}"/hdf5/include/* "$MAIN_INSTALL_DIR"/include
-    cp "${LIB_CACHE}"/hdf5/lib/* "$MAIN_INSTALL_DIR"/lib 
-fi
-
 rm -rf hdf5-hdf5_1.14.4.3 

python/scripts/install_highway.sh

@@ -1,4 +1,4 @@
-set -euo
+set -euo pipefail
 set -x
 
 
@@ -13,23 +13,6 @@ else
     INSTALL_DIR="/usr/local"
 fi
 
-# Check for cached files if LIB_CACHE is set.
-# (This is useful for CI builds to not re-build dependencies from scratch)
-if [ ! -z "${LIB_CACHE+x}" ] && [ -f "${LIB_CACHE}/hwy/lib64/libhwy.a" ]; then
-    echo "Copying hwy from cache"
-    mkdir -p "$INSTALL_DIR"/include "$INSTALL_DIR"/lib
-    cp -r "${LIB_CACHE}"/hwy/include/* "$INSTALL_DIR"/include
-    cp "${LIB_CACHE}"/hwy/lib*/libhwy.a "$INSTALL_DIR"/lib 
-    exit 0;
-fi
-
-# Cache is empty, so fill cache
-if [ ! -z "${LIB_CACHE+x}" ]; then
-    MAIN_INSTALL_DIR="$INSTALL_DIR"
-    INSTALL_DIR="$LIB_CACHE/hwy"
-    mkdir -p "$INSTALL_DIR"
-fi
-
 curl -L https://github.com/google/highway/archive/refs/tags/1.1.0.tar.gz > highway.tar.gz
 tar -xzf highway.tar.gz
 rm highway.tar.gz
@@ -52,13 +35,4 @@ cd ../../
 mkdir -p "$INSTALL_DIR/include/hwy/contrib/math"
 cp -r highway-1.1.0/hwy/contrib/math/math-inl.h "$INSTALL_DIR/include/hwy/contrib/math"
 
-# Fill cache
-if [ ! -z "${LIB_CACHE+x}" ]; then
-    mkdir -p "$MAIN_INSTALL_DIR"/include "$MAIN_INSTALL_DIR"/lib
-    cp -r "${LIB_CACHE}"/hwy/include/* "$MAIN_INSTALL_DIR"/include
-
-    # Can start in lib or lib64
-    cp "${LIB_CACHE}"/hwy/lib*/libhwy.a "$MAIN_INSTALL_DIR"/lib 
-fi
-
 rm -rf highway-1.1.0