Update the docs #543
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (C) 2023 Roberto Rossini (roberros@uio.no) | |
| # SPDX-License-Identifier: MIT | |
| name: Run fuzzy tests | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - ".github/workflows/fuzzy-testing.yml" | |
| - "cmake/**" | |
| - "src/**" | |
| - "test/fuzzer/**" | |
| - "CMakeLists.txt" | |
| - "conanfile.py" | |
| tags: | |
| - "v*.*.*" | |
| pull_request: | |
| paths: | |
| - ".github/workflows/fuzzy-testing.yml" | |
| - "cmake/**" | |
| - "src/**" | |
| - "test/fuzzer/**" | |
| - "CMakeLists.txt" | |
| - "conanfile.py" | |
| schedule: | |
| # Run weekly | |
| - cron: "15 3 * * 0" | |
| workflow_dispatch: | |
| inputs: | |
| duration: | |
| description: "Test duration in seconds" | |
| required: true | |
| default: "600" | |
| type: string | |
| resolution: | |
| description: "Matrix resolution to use for testing" | |
| required: true | |
| default: "random" | |
| type: string | |
| # https://stackoverflow.com/a/72408109 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| build-project: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - { | |
| dataset: "4DNFIYECESRC", | |
| format: "cool", | |
| normalization: "NONE", | |
| bin-type: "fixed", | |
| } | |
| - { | |
| dataset: "4DNFIYECESRC", | |
| format: "cool", | |
| normalization: "weight", | |
| bin-type: "fixed", | |
| } | |
| - { | |
| dataset: "4DNFIYECESRC", | |
| format: "cool", | |
| normalization: "VC", | |
| bin-type: "fixed", | |
| } | |
| - { | |
| dataset: "4DNFIYECESRC", | |
| format: "cool", | |
| normalization: "NONE", | |
| bin-type: "variable", | |
| } | |
| - { | |
| dataset: "4DNFIYECESRC", | |
| format: "hic8", | |
| normalization: "NONE", | |
| bin-type: "fixed", | |
| } | |
| - { | |
| dataset: "4DNFIYECESRC", | |
| format: "hic8", | |
| normalization: "KR", | |
| bin-type: "fixed", | |
| } | |
| - { | |
| dataset: "4DNFIYECESRC", | |
| format: "hic9", | |
| normalization: "NONE", | |
| bin-type: "fixed", | |
| } | |
| - { | |
| dataset: "4DNFIYECESRC", | |
| format: "hic9", | |
| normalization: "SCALE", | |
| bin-type: "fixed", | |
| } | |
| - { | |
| dataset: "4DNFIYECESRC", | |
| format: "hic9", | |
| normalization: "VC", | |
| bin-type: "fixed", | |
| } | |
| container: | |
| image: ghcr.io/paulsengroup/ci-docker-images/ubuntu-24.04-cxx-clang-18 | |
| options: "--user=root" | |
| env: | |
| CCACHE_DIR: "/opt/ccache-cache" | |
| CCACHE_COMPILERCHECK: "content" | |
| CCACHE_COMPRESSLEVEL: "1" | |
| CCACHE_MAXSIZE: "100M" | |
| CONAN_HOME: "/opt/conan/" | |
| HICTK_CI: "1" | |
| outputs: | |
| ccache-old-cache-key: ${{ steps.cache-ccache.outputs.cache-matched-key }} | |
| steps: | |
| - name: Clone hictk | |
| uses: actions/checkout@v4 | |
| - name: Fix permissions | |
| run: | | |
| chown -R $(id -u):$(id -g) $PWD | |
| - name: Generate cache key | |
| id: cache-key | |
| run: | | |
| conanfile_hash="${{ hashFiles('conanfile.py') }}" | |
| # This can be used by to always update a cache entry (useful e.g. for ccache) | |
| current_date="$(date '+%s')" | |
| ccache_key_prefix="ccache-fuzzy-testing-$GITHUB_REF-$conanfile_hash" | |
| echo "conan-key=fuzzy-testing-$conanfile_hash" | tee -a "$GITHUB_OUTPUT" | |
| echo "ccache-key=${ccache_key_prefix}-${current_date}" | tee -a "$GITHUB_OUTPUT" | |
| echo "ccache-restore-key=$ccache_key_prefix" | tee -a "$GITHUB_OUTPUT" | |
| - name: Install Python | |
| run: | | |
| apt-get update | |
| apt-get install -y python3.12 python3.12-dev | |
| - name: Restore Conan cache | |
| id: cache-conan | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ steps.cache-key.outputs.conan-key }} | |
| path: ${{ env.CONAN_HOME }}/p | |
| - name: Clean Conan cache (pre-build) | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| run: | | |
| conan cache clean "*" --build | |
| conan cache clean "*" --download | |
| conan cache clean "*" --source | |
| conan remove --confirm "*" | |
| - name: Install build dependencies | |
| run: | | |
| conan install . \ | |
| --build=missing \ | |
| -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ | |
| -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ | |
| -s build_type=Release \ | |
| -s compiler.libcxx=libstdc++11 \ | |
| -s compiler.cppstd=17 \ | |
| --output-folder=build | |
| - name: Clean Conan cache (post-build) | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| run: | | |
| conan cache clean "*" --build | |
| conan cache clean "*" --download | |
| conan cache clean "*" --source | |
| - name: Save Conan cache | |
| uses: actions/cache/save@v4 | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| with: | |
| key: ${{ steps.cache-key.outputs.conan-key }} | |
| path: ${{ env.CONAN_HOME }}/p | |
| env: | |
| ZSTD_CLEVEL: 19 | |
| - name: Configure project | |
| run: | | |
| cmake -DCMAKE_BUILD_TYPE=Release \ | |
| -DCMAKE_PREFIX_PATH="$PWD/build" \ | |
| -DENABLE_DEVELOPER_MODE=ON \ | |
| -DOPT_ENABLE_CLANG_TIDY=OFF \ | |
| -DOPT_ENABLE_CPPCHECK=OFF \ | |
| -DHICTK_ENABLE_TESTING=ON \ | |
| -DHICTK_ENABLE_FUZZY_TESTING=ON \ | |
| -DHICTK_BUILD_EXAMPLES=OFF \ | |
| -DHICTK_DOWNLOAD_TEST_DATASET=OFF \ | |
| -S . \ | |
| -B build | |
| - name: Restore Ccache folder | |
| id: cache-ccache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: ${{ steps.cache-key.outputs.ccache-restore-key }} | |
| path: ${{ env.CCACHE_DIR }} | |
| - name: Reset Ccache stats | |
| run: ccache --zero-stats | |
| - name: Build hictk_fuzzer | |
| run: cmake --build build -j $(nproc) -t hictk_fuzzer | |
| - name: Print Ccache statistics (pre-cleanup) | |
| run: | | |
| ccache --show-stats \ | |
| --show-compression \ | |
| --verbose | |
| - name: Cleanup Ccache folder | |
| run: | | |
| ccache --evict-older-than=14400s # 4h | |
| ccache --recompress=19 --recompress-threads="$(nproc)" | |
| ccache --cleanup | |
| - name: Print Ccache statistics (post-cleanup) | |
| run: | | |
| ccache --show-stats \ | |
| --show-compression \ | |
| --verbose | |
| - name: Save Ccache folder | |
| uses: actions/cache/save@v4 | |
| with: | |
| key: ${{ steps.cache-key.outputs.ccache-key }} | |
| path: ${{ env.CCACHE_DIR }} | |
| env: | |
| ZSTD_CLEVEL: 1 | |
| - name: Install test dependencies | |
| run: | | |
| pip install --no-cache-dir 'cooler==0.10.*' 'numpy<2' | |
| - name: Detect CI type | |
| id: ci-type | |
| run: | | |
| if git log --format=%B -n 1 ${{ github.event.after }} | grep -qF '[ci full]'; then | |
| echo "type=full" | tee -a "$GITHUB_OUTPUT" | |
| else | |
| echo "type=short" | tee -a "$GITHUB_OUTPUT" | |
| fi | |
| - name: Prepare for test | |
| id: test-params | |
| run: | | |
| duration=120 | |
| if [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then | |
| duration='${{ inputs.duration }}' | |
| elif [[ '${{ steps.ci-type.outputs.type }}' == 'full' ]]; then | |
| duration=3600 | |
| fi | |
| resolution=50000 | |
| if [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then | |
| if [[ '${{ inputs.resolution }}' == 'random' ]]; then | |
| resolution="$( | |
| python3 -c 'import random; import sys; print(random.choice([int(x) for x in sys.argv[1:]]))' \ | |
| 1000 5000 10000 50000 100000 500000 | |
| )" | |
| else | |
| resolution='${{ inputs.resolution }}' | |
| fi | |
| fi | |
| if [[ '${{ matrix.bin-type }}' == variable ]]; then | |
| resolution=0 | |
| fi | |
| 2>&1 echo "Duration: ${duration}" | |
| 2>&1 echo "Resolution: ${resolution}" | |
| echo "duration=$duration" | tee -a "$GITHUB_OUTPUT" | |
| echo "resolution=$resolution" | tee -a "$GITHUB_OUTPUT" | |
| - name: Download test datasets | |
| run: | | |
| test/fuzzer/scripts/download_test_datasets.py \ | |
| test/fuzzer/test_files.json \ | |
| . \ | |
| --format cool "${{ matrix.format }}" \ | |
| --resolution "${{ steps.test-params.outputs.resolution }}" \ | |
| --dataset "${{ matrix.dataset }}" \ | |
| --nproc 2 | |
| - name: Run test (df) | |
| run: | | |
| build/test/fuzzer/src/hictk_fuzzer fuzz \ | |
| --resolution ${{ steps.test-params.outputs.resolution }} \ | |
| --duration '${{ steps.test-params.outputs.duration }}' \ | |
| --normalization ${{ matrix.normalization }} \ | |
| --nproc $(nproc) \ | |
| --format df \ | |
| *".${{ matrix.format }}" \ | |
| *.cool | |
| - name: Run test (dense) | |
| run: | | |
| build/test/fuzzer/src/hictk_fuzzer fuzz \ | |
| --resolution ${{ steps.test-params.outputs.resolution }} \ | |
| --duration '${{ steps.test-params.outputs.duration }}' \ | |
| --normalization ${{ matrix.normalization }} \ | |
| --nproc $(nproc) \ | |
| --format dense \ | |
| *".${{ matrix.format }}" \ | |
| *.cool | |
| - name: Run test (sparse) | |
| run: | | |
| build/test/fuzzer/src/hictk_fuzzer fuzz \ | |
| --resolution ${{ steps.test-params.outputs.resolution }} \ | |
| --duration '${{ steps.test-params.outputs.duration }}' \ | |
| --normalization ${{ matrix.normalization }} \ | |
| --nproc $(nproc) \ | |
| --format sparse \ | |
| *".${{ matrix.format }}" \ | |
| *.cool | |
| clean-stale-cache: | |
| needs: [build-project] | |
| uses: paulsengroup/hictk/.github/workflows/evict-gha-cache.yml@main | |
| name: Clean stale Ccache cache | |
| permissions: | |
| actions: write | |
| if: needs.build-project.outputs.ccache-old-cache-key != '' | |
| with: | |
| cache-key: "${{ needs.build-project.outputs.ccache-old-cache-key }}" | |
| fuzzy-testing-status-check: | |
| name: Status Check (fuzzy-testing) | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-project | |
| steps: | |
| - name: Collect job results | |
| if: needs.build-project.result != 'success' | |
| run: exit 1 |