diff --git a/.github/actions/setup-caches/action.yml b/.github/actions/setup-caches/action.yml index afb304a0..ad8926ef 100644 --- a/.github/actions/setup-caches/action.yml +++ b/.github/actions/setup-caches/action.yml @@ -18,10 +18,13 @@ runs: # Setup CCache - name: Setup ccache uses: hendrikmuhs/ccache-action@v1.2.12 + with: + key: ccache-${{ runner.os }}-${{ runner.arch }} + append-timestamp: false # avoid excess caching - name: Setup ccache in shell shell: bash - run: echo 'PATH=/usr/lib/ccache:'"$PATH" >> $GITHUB_ENV + run: echo 'PATH=/usr/lib/ccache:/usr/local/opt/ccache/libexec:'"$PATH" >> $GITHUB_ENV if: ${{ runner.os != 'Windows' }} # TODO windows @@ -31,10 +34,10 @@ runs: uses: actions/cache@v4 with: path: | - ~/Library/Caches/Homebrew/boost--* - ~/Library/Caches/Homebrew/downloads/*--boost-* - key: brew-${{ hashFiles('CMakeLists.txt') }} - restore-keys: brew- + ~/Library/Caches/Homebrew/* + ~/Library/Caches/Homebrew/downloads/* + key: brew-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }} + restore-keys: brew-${{ runner.os }}-${{ runner.arch }}- if: ${{ runner.os == 'macOS' }} ################ @@ -42,18 +45,18 @@ runs: - name: Setup vcpkg cache in shell shell: bash run: | - mkdir -p ~/vcpkg_cache - mkdir -p ~/vcpkg_download_cache + mkdir -p $HOME/vcpkg_cache + mkdir -p $HOME/vcpkg_download_cache echo "VCPKG_DEFAULT_BINARY_CACHE=$HOME/vcpkg_cache" >> $GITHUB_ENV echo "VCPKG_DOWNLOADS=$HOME/vcpkg_download_cache" >> $GITHUB_ENV if: ${{ runner.os != 'Windows' }} - - name: Setup vcpkg cache (Linux) + - name: Setup vcpkg cache uses: actions/cache@v4 with: path: | - ~/vcpkg_cache - ~/vcpkg_download_cache - key: vcpkg-${{ runner.os }}-${{ hashFiles('vcpkg.json') }} - restore-keys: vcpkg-${{ runner.os }} + $HOME/vcpkg_cache + $HOME/vcpkg_download_cache + key: vcpkg-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('vcpkg.json') }} + restore-keys: vcpkg-${{ runner.os }}-${{ runner.arch }}- if: ${{ runner.os != 'Windows' }} diff --git a/.github/actions/setup-dependencies/action.yml b/.github/actions/setup-dependencies/action.yml index cf1dba53..e08d822c 100644 --- a/.github/actions/setup-dependencies/action.yml +++ b/.github/actions/setup-dependencies/action.yml @@ -21,14 +21,14 @@ runs: - name: Install python dependencies shell: bash run: make requirements - if: ${{ runner.os == 'Linux' }} # skip on linux, in docker + if: ${{ runner.os == 'Linux' }} ################ # Linux # NOTE: skip for manylinux image # - name: Linux init steps # shell: bash # run: make dependencies-vcpkg - # if: ${{ runner.os == 'Linux' }} # skip + # if: ${{ runner.os == 'Linux' }} # skip ################ # Mac diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml index 4aab7121..ff94cd9c 100644 --- a/.github/actions/setup-python/action.yml +++ b/.github/actions/setup-python/action.yml @@ -19,16 +19,17 @@ runs: uses: actions/cache@v4 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }} + key: pip-${{ runner.os }}-${{ hashFiles('pyproject.toml') }} restore-keys: | - ${{ runner.os }}-pip- + pip-${{ runner.os }} - name: Set up Python ${{ inputs.version }} uses: actions/setup-python@v5 with: python-version: ${{ inputs.version }} - cache: "pip" - cache-dependency-path: 'pyproject.toml' + # TODO consider re-adding this and/or removing the pip cache + # cache: "pip" + # cache-dependency-path: 'pyproject.toml' - name: Install cibuildwheel shell: bash diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e29e4da0..ec4dcf47 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,9 +4,6 @@ on: push: branches: - main - - develop - - tkp/gh - - tkp/internal tags: - v* paths-ignore: @@ -52,7 +49,7 @@ jobs: # github.event.head_commit.message otherwise on pull # requests) initialize: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: COMMIT_MESSAGE: ${{ steps.setup.outputs.COMMIT_MSG }} @@ -125,7 +122,7 @@ jobs: strategy: matrix: os: - - ubuntu-20.04 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md + - ubuntu-22.04 python-version: - 3.9 @@ -167,11 +164,10 @@ jobs: strategy: matrix: os: - - ubuntu-20.04 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md - - macos-11 # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11-Readme.md - # - windows-2022 # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2022-Readme.md - # arm64 image - # - macos-13-xlarge # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-Readme.md + - ubuntu-22.04 # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md + - macos-12 # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md + - macos-14 # https://github.com/actions/runner-images/blob/main/images/macos/macos-14-arm64-Readme.md + # - windows-2022 # https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md python-version: - "3.8" - "3.9" @@ -218,18 +214,31 @@ jobs: ############################################## # Things to exclude if not a full matrix run # ############################################## + # minimize CI resource usage + - is-full-run: false + os: ubuntu-22.04 + python-version: "3.8" # windows is slow, so dont build unless its a full run - is-full-run: false os: windows-2022 # avoid unnecessary use of mac resources - is-full-run: false - os: macos-11 + os: macos-12 + python-version: "3.8" + - is-full-run: false + os: macos-12 + python-version: "3.9" + - is-full-run: false + os: macos-12 + python-version: "3.10" + - is-full-run: false + os: macos-14 python-version: "3.8" - is-full-run: false - os: macos-11 + os: macos-14 python-version: "3.9" - is-full-run: false - os: macos-11 + os: macos-14 python-version: "3.10" runs-on: ${{ matrix.os }} @@ -255,6 +264,7 @@ jobs: uses: ./.github/actions/setup-dependencies with: cibuildwheel: '${{ matrix.cibuildwheel }}' + if: ${{ runner.os == 'macOS' }} ######## # Linux @@ -262,17 +272,29 @@ jobs: run: make dist-py-cibw env: CIBW_BUILD: "${{ matrix.cibuildwheel }}-manylinux*" - CIBW_ENVIRONMENT_LINUX: CSP_MANYLINUX="ON" CCACHE_DIR="/host/home/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="/host/home/runner/vcpkg_cache" VCPKG_DOWNLOADS="/host/home/runner/vcpkg_download_cache" + CIBW_ENVIRONMENT_LINUX: CSP_MANYLINUX="ON" CCACHE_DIR="/host/home/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="/host${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="/host${{ env.VCPKG_DOWNLOADS }}" + CIBW_BUILD_VERBOSITY: 3 if: ${{ runner.os == 'Linux' }} ######## # Macos - - name: Python Build Steps (Macos) + - name: Python Build Steps (Macos x86) run: make dist-py-cibw env: CIBW_BUILD: "${{ matrix.cibuildwheel }}-macos*" - CIBW_ENVIRONMENT_MACOS: CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="/Users/runner/vcpkg_cache" VCPKG_DOWNLOADS="/Users/runner/vcpkg_download_cache" CC="/usr/local/bin/gcc-13" CXX="/usr/local/bin/g++-13" - if: ${{ runner.os == 'macOS' }} + CIBW_ENVIRONMENT_MACOS: CC="/usr/local/bin/gcc-13" CXX="/usr/local/bin/g++-13" CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="${{ env.VCPKG_DOWNLOADS }}" + CIBW_ARCHS_MACOS: x86_64 + CIBW_BUILD_VERBOSITY: 3 + if: ${{ matrix.os == 'macos-12' }} + + - name: Python Build Steps (Macos arm) + run: make dist-py-cibw + env: + CIBW_BUILD: "${{ matrix.cibuildwheel }}-macos*" + CIBW_ENVIRONMENT_MACOS: PATH="/opt/homebrew/opt/bison/bin/:$PATH" CC="/opt/homebrew/bin/gcc-13" CXX="/opt/homebrew/bin/g++-13" LDFLAGS="-Wl,-ld_classic" CCACHE_DIR="/Users/runner/work/csp/csp/.ccache" VCPKG_DEFAULT_BINARY_CACHE="${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" VCPKG_DOWNLOADS="${{ env.VCPKG_DOWNLOADS }}" + CIBW_ARCHS_MACOS: arm64 + CIBW_BUILD_VERBOSITY: 3 + if: ${{ matrix.os == 'macos-14' }} ########## # Windows @@ -288,7 +310,7 @@ jobs: - name: Upload Wheel uses: actions/upload-artifact@v4 with: - name: csp-dist-${{ runner.os }}-${{ matrix.python-version }} + name: csp-dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }} path: wheelhouse/*.whl ############################# @@ -310,7 +332,7 @@ jobs: strategy: matrix: os: - - ubuntu-20.04 + - ubuntu-22.04 python-version: - 3.9 @@ -361,10 +383,10 @@ jobs: strategy: matrix: os: - - ubuntu-20.04 - - macos-11 + - ubuntu-22.04 + - macos-12 + - macos-14 # - windows-2022 - # - macos-13-xlarge # arm64 image python-version: - 3.8 - 3.9 @@ -382,7 +404,10 @@ jobs: # Exclude macOS builds for now - is-full-run: false - os: macos-11 + os: macos-12 + + - is-full-run: false + os: macos-14 # Exclude Python 3.8, 3.10, 3.11 builds - is-full-run: false @@ -425,7 +450,7 @@ jobs: - name: Download wheel uses: actions/download-artifact@v4 with: - name: csp-dist-${{ runner.os }}-${{ matrix.python-version }} + name: csp-dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }} - name: Install wheel (Linux) run: python -m pip install -U *manylinux2014*.whl --target . @@ -469,7 +494,7 @@ jobs: # strategy: # matrix: # os: - # - ubuntu-20.04 + # - ubuntu-22.04 # python-version: # - 3.9 diff --git a/Makefile b/Makefile index 4e483ddd..2ac56254 100644 --- a/Makefile +++ b/Makefile @@ -154,11 +154,11 @@ clean: ## clean the repository .PHONY: dependencies-mac dependencies-debian dependencies-fedora dependencies-vcpkg dependencies-win dependencies-mac: ## install dependencies for mac - HOMEBREW_NO_AUTO_UPDATE=1 brew install bison cmake flex # gcc@12 + HOMEBREW_NO_AUTO_UPDATE=1 brew install bison cmake flex make ninja # gcc@13 + brew unlink bison flex && brew link --force bison flex dependencies-debian: ## install dependencies for linux apt-get install -y automake bison cmake curl flex ninja-build tar unzip zip - # libabsl-dev libarrow-dev libparquet-dev libthrift-dev dependencies-fedora: ## install dependencies for linux yum install -y automake bison cmake curl flex perl-IPC-Cmd tar unzip zip diff --git a/pyproject.toml b/pyproject.toml index e62fe041..09cbf971 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,15 +77,15 @@ test-requires = [ ] [tool.cibuildwheel.linux] -before-all ="make dependencies-fedora && make dependencies-vcpkg" +before-all ="make dependencies-fedora" environment = {CSP_MANYLINUX="ON"} repair-wheel-command = "auditwheel -v show {wheel} && LD_LIBRARY_PATH=/project/csp/lib auditwheel -v repair -w {dest_dir} {wheel}" skip = "*i686 musllinux*" [tool.cibuildwheel.macos] -before-all ="make dependencies-mac && make dependencies-vcpkg" +before-all ="make dependencies-mac" archs = "x86_64" # NOTE: we use gcc and we cannot cross compile for now -environment = {CFLAGS="-I/usr/local/include -L/usr/local/lib", CXXFLAGS="-I/usr/local/include -L/usr/local/lib", LDFLAGS="-L/usr/local/lib"} +environment = {} [tool.cibuildwheel.windows] archs = "AMD64"