Improve compilation times #1153
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (C) 2023 Roberto Rossini (roberros@uio.no) | |
| # SPDX-License-Identifier: MIT | |
| name: Windows CI | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - ".github/workflows/windows-ci.yml" | |
| - "cmake/**" | |
| - "examples/**" | |
| - "src/**" | |
| - "test/integration/**" | |
| - "test/units/**" | |
| - "CMakeLists.txt" | |
| - "conanfile.py" | |
| tags: | |
| - "v*.*.*" | |
| pull_request: | |
| paths: | |
| - ".github/workflows/windows-ci.yml" | |
| - "cmake/**" | |
| - "examples/**" | |
| - "src/**" | |
| - "test/integration/**" | |
| - "test/units/**" | |
| - "CMakeLists.txt" | |
| - "conanfile.py" | |
| # https://stackoverflow.com/a/72408109 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CONAN_HOME: "${{ github.workspace }}\\.conan2" | |
| HICTK_CI: "1" | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| matrix-factory: | |
| name: Generate job matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-result.outputs.result }} | |
| steps: | |
| - uses: actions/github-script@v7 | |
| id: set-result | |
| with: | |
| script: | | |
| // Documentation | |
| // https://docs.github.com/en/actions/learn-github-actions/contexts#fromjson | |
| // https://github.com/actions/runner/issues/982#issuecomment-809360765 | |
| var includes = [] | |
| includes.push({ compiler_name: 'msvc', os: 'windows-2022', conan: '2.4.*', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
| includes.push({ compiler_name: 'msvc', os: 'windows-2022', conan: '2.4.*', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) | |
| return { include: includes } | |
| cache-test-dataset: | |
| uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main | |
| name: Cache test dataset | |
| build-project: | |
| needs: matrix-factory | |
| runs-on: windows-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Generate requirements.txt for pip | |
| run: | | |
| echo 'conan==${{ matrix.conan }}' > requirements.txt | |
| echo 'cmake==${{ matrix.cmake }}' >> requirements.txt | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| - name: Detect number available CPUs | |
| run: | | |
| ncpus=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())') | |
| echo "CMAKE_BUILD_PARALLEL_LEVEL=$ncpus" >> $GITHUB_ENV | |
| echo "CTEST_PARALLEL_LEVEL=$ncpus" >> $GITHUB_ENV | |
| - name: Install deps | |
| run: pip install -r requirements.txt | |
| - name: Generate cache key | |
| id: cache-key | |
| run: | | |
| set -u | |
| os="${{ matrix.os }}" | |
| compiler="${{ matrix.compiler_name }}" | |
| build_type="${{ matrix.build_type }}" | |
| conanfile_hash="${{ hashFiles('conanfile.py') }}" | |
| conan_key_prefix="conan-$os-$compiler-$conanfile_hash-$build_type" | |
| echo "conan-key=$conan_key_prefix" | tee -a $GITHUB_OUTPUT | |
| - name: Restore Conan cache | |
| id: cache-conan | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ steps.cache-key.outputs.conan-key }} | |
| path: ${{ env.CONAN_HOME }}\p | |
| - name: Configure Conan | |
| run: | | |
| conan profile detect --force | |
| conan_profile="$(conan profile path default)" | |
| sed -i 's/build_type=Release/build_type=${{ matrix.build_type }}/' "$conan_profile" | |
| sed -i 's/compiler\.cppstd=.*/compiler.cppstd=17/' "$conan_profile" | |
| - name: Clean Conan cache (pre-build) | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| run: | | |
| conan cache clean "*" --build | |
| conan cache clean "*" --download | |
| conan cache clean "*" --source | |
| conan remove --confirm "*" | |
| - name: Compile build dependencies | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| run: | | |
| conan install . \ | |
| --build=missing \ | |
| --build="b2/*" \ | |
| --build="catch2/*" \ | |
| -pr:h default \ | |
| -pr:b default \ | |
| -s:h "compiler=${{ matrix.compiler_name }}" \ | |
| -s:h "build_type=${{ matrix.build_type }}" \ | |
| -s:h "compiler.runtime_type=${{ matrix.build_type }}" \ | |
| -s:h compiler.cppstd=17 \ | |
| -s:b "compiler=${{ matrix.compiler_name }}" \ | |
| -s:b "build_type=${{ matrix.build_type }}" \ | |
| -s:b "compiler.runtime_type=${{ matrix.build_type }}" \ | |
| -s:b compiler.cppstd=17 | |
| - name: Install build dependencies | |
| run: | | |
| conan install . \ | |
| -pr:h default \ | |
| -pr:b default \ | |
| -s:h "compiler=${{ matrix.compiler_name }}" \ | |
| -s:h "build_type=${{ matrix.build_type }}" \ | |
| -s:h "compiler.runtime_type=${{ matrix.build_type }}" \ | |
| -s:h compiler.cppstd=17 \ | |
| -s:b "compiler=${{ matrix.compiler_name }}" \ | |
| -s:b "build_type=${{ matrix.build_type }}" \ | |
| -s:b "compiler.runtime_type=${{ matrix.build_type }}" \ | |
| -s:b compiler.cppstd=17 \ | |
| --output-folder=build | |
| - name: Clean Conan cache (post-build) | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| run: | | |
| conan cache clean "*" --build | |
| conan cache clean "*" --download | |
| conan cache clean "*" --source | |
| - name: Save Conan cache | |
| uses: actions/cache/save@v4 | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| with: | |
| key: ${{ steps.cache-key.outputs.conan-key }} | |
| path: ${{ env.CONAN_HOME }}\p | |
| env: | |
| ZSTD_CLEVEL: 19 | |
| - name: Configure project | |
| run: | | |
| cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ | |
| -DCMAKE_PREFIX_PATH="${{ github.workspace }}/build" \ | |
| -DENABLE_DEVELOPER_MODE=${{ matrix.developer_mode }} \ | |
| -DHICTK_ENABLE_TESTING=ON \ | |
| -DHICTK_BUILD_EXAMPLES=ON \ | |
| -DHICTK_DOWNLOAD_TEST_DATASET=OFF \ | |
| -DHICTK_ENABLE_GIT_VERSION_TRACKING=OFF \ | |
| -DOPT_ENABLE_CLANG_TIDY=OFF \ | |
| -DOPT_ENABLE_CPPCHECK=OFF \ | |
| -DCMAKE_INSTALL_PREFIX=dest \ | |
| -S "${{ github.workspace }}" \ | |
| -B "${{ github.workspace }}/build" | |
| - name: Build project | |
| run: cmake --build build --config ${{ matrix.build_type }} | |
| - name: Package binaries | |
| run: | | |
| cmake --install build | |
| tar -cf - -C dest/ bin | zstd -T0 -13 -o binaries.tar.zst | |
| - name: Package unit tests | |
| run: | | |
| rm -r build/src | |
| tar -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst | |
| - name: Upload unit tests | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: "unit-tests-${{ matrix.os }}-\ | |
| ${{ matrix.compiler_name }}-\ | |
| ${{ matrix.build_type }}-\ | |
| ${{ matrix.developer_mode }}" | |
| path: unit-tests.tar.zst | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - name: Upload binaries | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: "binaries-${{ matrix.os }}-\ | |
| ${{ matrix.compiler_name }}-\ | |
| ${{ matrix.build_type }}-\ | |
| ${{ matrix.developer_mode }}" | |
| path: binaries.tar.zst | |
| if-no-files-found: error | |
| retention-days: 1 | |
| run-unit-tests: | |
| name: Run unit tests | |
| needs: [matrix-factory, cache-test-dataset, build-project] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Restore test dataset | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ needs.cache-test-dataset.outputs.cache-key }} | |
| path: test/data/hictk_test_data.tar.zst | |
| fail-on-cache-miss: true | |
| enableCrossOsArchive: true | |
| - name: Download unit tests artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: "unit-tests-${{ matrix.os }}-\ | |
| ${{ matrix.compiler_name }}-\ | |
| ${{ matrix.build_type }}-\ | |
| ${{ matrix.developer_mode }}" | |
| - name: Extract binaries test dataset | |
| run: | | |
| tar -xf unit-tests.tar.zst | |
| tar -xf test/data/hictk_test_data.tar.zst | |
| - name: Generate requirements.txt for pip | |
| run: | | |
| echo 'cmake==${{ matrix.cmake }}' > requirements.txt | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache: "pip" | |
| - name: Install test dependencies | |
| run: pip install -r requirements.txt | |
| - name: Run unit tests (Debug) | |
| if: matrix.build_type == 'Debug' | |
| run: | | |
| skip_labels=( | |
| '.*dataset large read\/write.*' | |
| '.*HiCFileWriter.*' | |
| '.*VectorOfAtomicDecimals.*' | |
| ) | |
| pattern="$(IFS='|' ; echo "${skip_labels[*]}")" | |
| ctest --test-dir build \ | |
| --schedule-random \ | |
| --output-on-failure \ | |
| --no-tests=error \ | |
| --timeout 360 \ | |
| --exclude-regex "$pattern" |& | |
| tail -n 1000 | |
| - name: Run unit tests (Release) | |
| if: matrix.build_type == 'Release' | |
| run: | | |
| ctest --test-dir build \ | |
| --schedule-random \ | |
| --output-on-failure \ | |
| --no-tests=error \ | |
| --timeout 120 |& | |
| tail -n 1000 | |
| run-integration-tests: | |
| name: Run integration tests | |
| needs: [matrix-factory, cache-test-dataset, build-project] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| cache-dependency-path: "test/integration/pyproject.toml" | |
| cache: "pip" | |
| - name: Restore test dataset | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ needs.cache-test-dataset.outputs.cache-key }} | |
| path: test/data/hictk_test_data.tar.zst | |
| fail-on-cache-miss: true | |
| enableCrossOsArchive: true | |
| - name: Download binaries artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: "binaries-${{ matrix.os }}-\ | |
| ${{ matrix.compiler_name }}-\ | |
| ${{ matrix.build_type }}-\ | |
| ${{ matrix.developer_mode }}" | |
| - name: Extract binaries test dataset | |
| run: | | |
| tar -xf binaries.tar.zst | |
| tar -xf test/data/hictk_test_data.tar.zst | |
| - name: Install test suite | |
| run: | | |
| python.exe -m venv venv --upgrade | |
| venv/Scripts/pip.exe install test/integration | |
| - name: Detect number available CPUs | |
| run: | | |
| ncpus=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())') | |
| echo "NPROC=$ncpus" | tee -a $GITHUB_ENV | |
| - name: Test hictk balance | |
| run: | | |
| venv/Scripts/hictk_integration_suite.exe \ | |
| bin/hictk.exe \ | |
| test/integration/config.toml \ | |
| --data-dir test/data \ | |
| --threads "$NPROC" \ | |
| --result-file integration-test-report.balance.json \ | |
| --suites=balance | |
| - name: Test hictk convert | |
| run: | | |
| venv/Scripts/hictk_integration_suite.exe \ | |
| bin/hictk.exe \ | |
| test/integration/config.toml \ | |
| --data-dir test/data \ | |
| --threads "$NPROC" \ | |
| --result-file integration-test-report.convert.json \ | |
| --suites=convert | |
| - name: Test hictk dump | |
| run: | | |
| venv/Scripts/hictk_integration_suite.exe \ | |
| bin/hictk.exe \ | |
| test/integration/config.toml \ | |
| --data-dir test/data \ | |
| --threads "$NPROC" \ | |
| --result-file integration-test-report.dump.json \ | |
| --suites=dump | |
| - name: Test hictk fix-mcool | |
| run: | | |
| venv/Scripts/hictk_integration_suite.exe \ | |
| bin/hictk.exe \ | |
| test/integration/config.toml \ | |
| --data-dir test/data \ | |
| --threads "$NPROC" \ | |
| --result-file integration-test-report.fix-mcool.json \ | |
| --suites=fix-mcool | |
| - name: Test hictk load | |
| run: | | |
| venv/Scripts/hictk_integration_suite.exe \ | |
| bin/hictk.exe \ | |
| test/integration/config.toml \ | |
| --data-dir test/data \ | |
| --threads "$NPROC" \ | |
| --result-file integration-test-report.load.json \ | |
| --suites=load | |
| - name: Test hictk merge | |
| run: | | |
| venv/Scripts/hictk_integration_suite.exe \ | |
| bin/hictk.exe \ | |
| test/integration/config.toml \ | |
| --data-dir test/data \ | |
| --threads "$NPROC" \ | |
| --result-file integration-test-report.merge.json \ | |
| --suites=merge | |
| - name: Test hictk metadata | |
| run: | | |
| venv/Scripts/hictk_integration_suite.exe \ | |
| bin/hictk.exe \ | |
| test/integration/config.toml \ | |
| --data-dir test/data \ | |
| --threads "$NPROC" \ | |
| --result-file integration-test-report.metadata.json \ | |
| --suites=metadata | |
| - name: Test hictk rename-chromosomes | |
| run: | | |
| venv/Scripts/hictk_integration_suite.exe \ | |
| bin/hictk.exe \ | |
| test/integration/config.toml \ | |
| --data-dir test/data \ | |
| --threads "$NPROC" \ | |
| --result-file integration-test-report.rename-chromosomes.json \ | |
| --suites=rename-chromosomes | |
| - name: Test hictk validate | |
| run: | | |
| venv/Scripts/hictk_integration_suite.exe \ | |
| bin/hictk.exe \ | |
| test/integration/config.toml \ | |
| --data-dir test/data \ | |
| --threads "$NPROC" \ | |
| --result-file integration-test-report.validate.json \ | |
| --suites=validate | |
| - name: Test hictk zoomify | |
| run: | | |
| venv/Scripts/hictk_integration_suite.exe \ | |
| bin/hictk.exe \ | |
| test/integration/config.toml \ | |
| --data-dir test/data \ | |
| --threads "$NPROC" \ | |
| --result-file integration-test-report.zoomify.json \ | |
| --suites=zoomify | |
| windows-ci-status-check: | |
| name: Status Check (Windows CI) | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - matrix-factory | |
| - cache-test-dataset | |
| - build-project | |
| - run-unit-tests | |
| - run-integration-tests | |
| steps: | |
| - name: Collect job results | |
| if: | | |
| needs.matrix-factory.result != 'success' || | |
| needs.cache-test-dataset.result != 'success' || | |
| needs.build-project.result != 'success' || | |
| needs.run-unit-tests.result != 'success' || | |
| needs.run-integration-tests.result != 'success' | |
| run: exit 1 |