Fix ccache cache eviction logic when using GHA job matrix #1258

Workflow file for this run

.github/workflows/macos-ci.yml at 4150ae8

	# Copyright (C) 2022 Roberto Rossini <roberros@uio.no>
	# SPDX-License-Identifier: MIT

	name: MacOS CI

	on:
	push:
	branches: [main]
	paths:
	- ".github/workflows/build-conan-deps.yml"
	- ".github/workflows/cache-test-dataset.yml"
	- ".github/workflows/evict-gha-cache.yml"
	- ".github/workflows/macos-ci.yml"
	- "cmake/**"
	- "examples/**"
	- "src/**"
	- "test/integration/**"
	- "test/units/**"
	- "CMakeLists.txt"
	- "conanfile.py"

	pull_request:
	paths:
	- ".github/workflows/build-conan-deps.yml"
	- ".github/workflows/cache-test-dataset.yml"
	- ".github/workflows/evict-gha-cache.yml"
	- ".github/workflows/macos-ci.yml"
	- "cmake/**"
	- "examples/**"
	- "src/**"
	- "test/integration/**"
	- "test/units/**"
	- "CMakeLists.txt"
	- "conanfile.py"

	# https://stackoverflow.com/a/72408109
	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	env:
	CCACHE_DIR: "${{ github.workspace }}/ccache-cache"
	CCACHE_COMPILERCHECK: "content"
	CCACHE_COMPRESSLEVEL: "1"
	CCACHE_MAXSIZE: "250M"
	CONAN_HOME: "${{ github.workspace }}/.conan2"
	HICTK_CI: "1"
	HOMEBREW_NO_AUTO_UPDATE: "1"

	defaults:
	run:
	shell: bash

	jobs:
	matrix-factory:
	name: Generate job matrix
	runs-on: ubuntu-latest
	outputs:
	matrix: ${{ steps.set-result.outputs.result }}
	steps:
	- uses: actions/github-script@v7
	id: set-result
	with:
	script: \|
	// Documentation
	// https://docs.github.com/en/actions/learn-github-actions/contexts#fromjson
	// https://github.com/actions/runner/issues/982#issuecomment-809360765

	var includes = []

	includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-13', cmake: '3.30.*', build_type: 'Release', developer_mode: 'OFF' })
	includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-14', cmake: '3.30.*', build_type: 'Release', developer_mode: 'OFF' })
	includes.push({ compiler_name: 'apple-clang', compiler_version: '16', os: 'macos-15', cmake: '3.30.*', build_type: 'Release', developer_mode: 'OFF' })
	includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-13', cmake: '3.30.*', build_type: 'Debug', developer_mode: 'OFF' })
	includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-14', cmake: '3.30.*', build_type: 'Debug', developer_mode: 'OFF' })
	includes.push({ compiler_name: 'apple-clang', compiler_version: '16', os: 'macos-15', cmake: '3.30.*', build_type: 'Debug', developer_mode: 'OFF' })

	return { include: includes }

	cache-test-dataset:
	name: Cache test dataset
	uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main

	build-conan-deps-x86:
	name: Build Conan deps
	uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main
	with:
	os: macos-13

	build-conan-deps-arm64:
	name: Build Conan deps
	uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main
	with:
	os: macos-14

	build-project:
	name: Build project
	needs:
	- matrix-factory
	- build-conan-deps-x86
	- build-conan-deps-arm64
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }}

	env:
	CCACHE_DIR: "${{ github.workspace }}/ccache-cache"
	CCACHE_COMPILERCHECK: "content"
	CCACHE_COMPRESSLEVEL: "1"
	CCACHE_MAXSIZE: "250M"
	CONAN_HOME: "${{ github.workspace }}/.conan2"
	HOMEBREW_NO_AUTO_UPDATE: "1"

	steps:
	- uses: actions/checkout@v4

	- name: Generate requirements.txt for pip
	run: echo 'cmake==${{ matrix.cmake }}' > requirements.txt

	- uses: actions/setup-python@v5
	with:
	python-version: "3.12"
	cache: "pip"

	- name: Detect number available CPUs
	run: \|
	ncpus=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())')

	echo "NPROC=$ncpus" >> $GITHUB_ENV
	echo "CMAKE_BUILD_PARALLEL_LEVEL=$ncpus" >> $GITHUB_ENV
	echo "CTEST_PARALLEL_LEVEL=$ncpus" >> $GITHUB_ENV

	- name: Install build deps
	run: \|
	pip install -r requirements.txt
	brew install ccache

	- name: Generate cache key
	id: cache-key
	run: \|
	set -u

	os="${{ matrix.os }}"
	compiler="${{ matrix.compiler_name }}"
	compiler_version="${{ matrix.compiler_version }}"
	build_type="${{ matrix.build_type }}"

	conanfile_hash="${{ hashFiles('conanfile.py') }}"

	# This can be used by to always update a cache entry (useful e.g. for ccache)
	current_date="$(date '+%s')"

	ccache_key_prefix="ccache-$os-$compiler-$compiler_version-$conanfile_hash-$build_type"

	echo "ccache-key=${ccache_key_prefix}-$GITHUB_REF-${current_date}" \| tee -a $GITHUB_OUTPUT
	echo "ccache-restore-key-1=$ccache_key_prefix-$GITHUB_REF" \| tee -a $GITHUB_OUTPUT
	echo "ccache-restore-key-2=$ccache_key_prefix" \| tee -a $GITHUB_OUTPUT

	- name: Restore Conan cache (x86)
	if: matrix.os == 'macos-13'
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.build-conan-deps-x86.outputs.conan-key }}
	path: ${{ env.CONAN_HOME }}/p
	fail-on-cache-miss: true

	- name: Restore Conan cache (arm64)
	if: matrix.os == 'macos-14' \|\| matrix.os == 'macos-15'
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.build-conan-deps-arm64.outputs.conan-key }}
	path: ${{ env.CONAN_HOME }}/p
	fail-on-cache-miss: true

	- name: Restore CMake configs (x86; Debug)
	if: matrix.build_type == 'Debug' && matrix.os == 'macos-13'
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.build-conan-deps-x86.outputs.cmake-prefix-debug-key }}
	path: /tmp/cmake-prefix-dbg.tar
	fail-on-cache-miss: true

	- name: Restore CMake configs (x86; Release)
	if: matrix.build_type == 'Release' && matrix.os == 'macos-13'
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.build-conan-deps-x86.outputs.cmake-prefix-release-key }}
	path: /tmp/cmake-prefix-rel.tar
	fail-on-cache-miss: true

	- name: Restore CMake configs (arm64; Debug)
	if: matrix.build_type == 'Debug' && (matrix.os == 'macos-14' \|\| matrix.os == 'macos-15')
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.build-conan-deps-arm64.outputs.cmake-prefix-debug-key }}
	path: /tmp/cmake-prefix-dbg.tar
	fail-on-cache-miss: true

	- name: Restore CMake configs (arm64; Release)
	if: matrix.build_type == 'Release' && (matrix.os == 'macos-14' \|\| matrix.os == 'macos-15')
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.build-conan-deps-arm64.outputs.cmake-prefix-release-key }}
	path: /tmp/cmake-prefix-rel.tar
	fail-on-cache-miss: true

	- name: Extract CMake configs
	run: \|
	mkdir conan-env
	tar -xf /tmp/cmake-prefix-*.tar -C conan-env/ --strip-components=1

	- name: Configure project
	run: \|
	cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
	-DCMAKE_PREFIX_PATH="$PWD/conan-env/" \
	-DENABLE_DEVELOPER_MODE=${{ matrix.developer_mode }} \
	-DHICTK_ENABLE_TESTING=ON \
	-DHICTK_BUILD_EXAMPLES=ON \
	-DOPT_ENABLE_CLANG_TIDY=OFF \
	-DOPT_ENABLE_CPPCHECK=OFF \
	-DHICTK_DOWNLOAD_TEST_DATASET=OFF \
	-DHICTK_ENABLE_GIT_VERSION_TRACKING=OFF \
	-DCMAKE_INSTALL_PREFIX=dest \
	-S "${{ github.workspace }}" \
	-B "${{ github.workspace }}/build"

	- name: Restore Ccache folder
	id: cache-ccache
	uses: actions/cache/restore@v4
	with:
	key: ${{ steps.cache-key.outputs.ccache-restore-key-1 }}
	restore-keys: ${{ steps.cache-key.outputs.ccache-restore-key-2 }}
	path: ${{ env.CCACHE_DIR }}

	- name: Reset Ccache stats
	run: ccache --zero-stats

	- name: Build project
	run: cmake --build ${{ github.workspace }}/build

	- name: Package binaries
	run: \|
	cmake --install build

	gtar -cf - -C dest/ bin \| zstd -T0 -13 -o binaries.tar.zst

	- name: Package unit tests
	run: \|
	rm -r build/src
	gtar --exclude='*.o' -cf - build/ \| zstd -T0 -13 -o unit-tests.tar.zst

	- name: Upload unit tests
	uses: actions/upload-artifact@v4
	with:
	name: "unit-tests-${{ matrix.os }}-\
	${{ matrix.compiler_name }}-\
	${{ matrix.compiler_version }}-\
	${{ matrix.build_type }}-\
	${{ matrix.developer_mode }}"
	path: unit-tests.tar.zst
	if-no-files-found: error
	retention-days: 1

	- name: Upload binaries
	uses: actions/upload-artifact@v4
	with:
	name: "binaries-${{ matrix.os }}-\
	${{ matrix.compiler_name }}-\
	${{ matrix.compiler_version }}-\
	${{ matrix.build_type }}-\
	${{ matrix.developer_mode }}"
	path: binaries.tar.zst
	if-no-files-found: error
	retention-days: 1

	- name: Print Ccache statistics (pre-cleanup)
	run: \|
	ccache --show-stats \
	--show-compression \
	--verbose

	- name: Cleanup Ccache folder
	run: \|
	ccache --evict-older-than=14400s # 4h
	ccache --recompress=19 --recompress-threads="$NPROC"
	ccache --cleanup

	- name: Print Ccache statistics (post-cleanup)
	run: \|
	ccache --show-stats \
	--show-compression \
	--verbose

	- name: Save Ccache folder
	uses: actions/cache/save@v4
	with:
	key: ${{ steps.cache-key.outputs.ccache-key }}
	path: ${{ env.CCACHE_DIR }}
	env:
	ZSTD_CLEVEL: 1

	- name: Generate list of stale cache entries
	id: stale-cache
	if: steps.cache-ccache.outputs.cache-matched-key != ''
	run: \|
	fname='stale-cache-${{ matrix.os }}-${{ matrix.compiler_name }}-${{ matrix.compiler_version }}-${{ matrix.build_type }}-${{ matrix.developer_mode }}.txt'
	echo '${{ steps.cache-ccache.outputs.cache-matched-key }}' > "$fname"

	echo "name=$fname" \| tee -a "$GITHUB_OUTPUT"

	- name: Upload stale cache entries
	if: steps.cache-ccache.outputs.cache-matched-key != ''
	uses: actions/upload-artifact@v4
	with:
	name: ${{ steps.stale-cache.outputs.name }}
	path: "${{ steps.stale-cache.outputs.name }}"
	if-no-files-found: error
	retention-days: 1

	clean-stale-cache:
	needs: [build-project]
	name: Evict stale cache entries
	runs-on: ubuntu-latest
	permissions:
	actions: write

	steps:
	- name: Download artifacts
	uses: actions/download-artifact@v4
	continue-on-error: true
	with:
	pattern: stale-cache-*
	merge-multiple: true

	- name: Evict cache entries
	continue-on-error: true
	run: \|
	set -x
	while read entry; do
	if ! grep -q '/heads/main' <(echo "$entry"); then
	gh cache delete --repo '${{ github.repository }}' "$entry"
	fi
	done < <(cat stale-cache*.txt \| grep -v '^$')
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	run-unit-tests:
	name: Run unit tests
	needs: [matrix-factory, cache-test-dataset, build-project]
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }}

	env:
	HICTK_CI: "1"

	steps:
	- name: Checkout repo
	uses: actions/checkout@v4

	- name: Restore test dataset
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.cache-test-dataset.outputs.cache-key }}
	path: test/data/hictk_test_data.tar.zst
	fail-on-cache-miss: true

	- name: Download unit tests artifact
	uses: actions/download-artifact@v4
	with:
	name: "unit-tests-${{ matrix.os }}-\
	${{ matrix.compiler_name }}-\
	${{ matrix.compiler_version }}-\
	${{ matrix.build_type }}-\
	${{ matrix.developer_mode }}"

	- name: Extract binaries test dataset
	run: \|
	gtar -xf unit-tests.tar.zst
	gtar -xf test/data/hictk_test_data.tar.zst

	- name: Generate requirements.txt for pip
	run: \|
	echo 'cmake==${{ matrix.cmake }}' > requirements.txt

	- uses: actions/setup-python@v5
	with:
	python-version: "3.12"
	cache: "pip"

	- name: Install test dependencies
	run: pip install -r requirements.txt

	- name: Run unit tests
	run: \|
	ctest --test-dir build/ \
	--schedule-random \
	--output-on-failure \
	--no-tests=error \
	--timeout 400 2>&1 \|
	head -n 1000

	run-integration-tests:
	name: Run integration tests
	needs: [matrix-factory, cache-test-dataset, build-project]
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }}

	env:
	HICTK_CI: "1"

	steps:
	- name: Checkout repo
	uses: actions/checkout@v4

	- uses: actions/setup-python@v5
	with:
	python-version: "3.12"
	cache-dependency-path: "test/integration/pyproject.toml"
	cache: "pip"

	- name: Restore test dataset
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.cache-test-dataset.outputs.cache-key }}
	path: test/data/hictk_test_data.tar.zst
	fail-on-cache-miss: true

	- name: Download binaries artifact
	uses: actions/download-artifact@v4
	with:
	name: "binaries-${{ matrix.os }}-\
	${{ matrix.compiler_name }}-\
	${{ matrix.compiler_version }}-\
	${{ matrix.build_type }}-\
	${{ matrix.developer_mode }}"

	- name: Extract binaries test dataset
	run: \|
	gtar -xf binaries.tar.zst
	gtar -xf test/data/hictk_test_data.tar.zst

	- name: Install test suite
	run: \|
	python3.12 -m venv venv --upgrade
	venv/bin/pip install test/integration

	- name: Detect number available CPUs
	run: \|
	ncpus=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())')

	echo "NPROC=$ncpus" \| tee -a $GITHUB_ENV

	- name: Test hictk balance
	run: \|
	venv/bin/hictk_integration_suite \
	bin/hictk \
	test/integration/config.toml \
	--data-dir test/data \
	--threads "$NPROC" \
	--result-file integration-test-report.balance.json \
	--suites=balance

	- name: Test hictk convert
	run: \|
	venv/bin/hictk_integration_suite \
	bin/hictk \
	test/integration/config.toml \
	--data-dir test/data \
	--threads "$NPROC" \
	--result-file integration-test-report.convert.json \
	--suites=convert

	- name: Test hictk dump
	run: \|
	venv/bin/hictk_integration_suite \
	bin/hictk \
	test/integration/config.toml \
	--data-dir test/data \
	--threads "$NPROC" \
	--result-file integration-test-report.dump.json \
	--suites=dump

	- name: Test hictk fix-mcool
	run: \|
	venv/bin/hictk_integration_suite \
	bin/hictk \
	test/integration/config.toml \
	--data-dir test/data \
	--threads "$NPROC" \
	--result-file integration-test-report.fix-mcool.json \
	--suites=fix-mcool

	- name: Test hictk load
	run: \|
	venv/bin/hictk_integration_suite \
	bin/hictk \
	test/integration/config.toml \
	--data-dir test/data \
	--threads "$NPROC" \
	--result-file integration-test-report.load.json \
	--suites=load

	- name: Test hictk merge
	run: \|
	venv/bin/hictk_integration_suite \
	bin/hictk \
	test/integration/config.toml \
	--data-dir test/data \
	--threads "$NPROC" \
	--result-file integration-test-report.merge.json \
	--suites=merge

	- name: Test hictk metadata
	run: \|
	venv/bin/hictk_integration_suite \
	bin/hictk \
	test/integration/config.toml \
	--data-dir test/data \
	--threads "$NPROC" \
	--result-file integration-test-report.metadata.json \
	--suites=metadata

	- name: Test hictk rename-chromosomes
	run: \|
	venv/bin/hictk_integration_suite \
	bin/hictk \
	test/integration/config.toml \
	--data-dir test/data \
	--threads "$NPROC" \
	--result-file integration-test-report.rename-chromosomes.json \
	--suites=rename-chromosomes

	- name: Test hictk validate
	run: \|
	venv/bin/hictk_integration_suite \
	bin/hictk \
	test/integration/config.toml \
	--data-dir test/data \
	--threads "$NPROC" \
	--result-file integration-test-report.validate.json \
	--suites=validate

	- name: Test hictk zoomify
	run: \|
	venv/bin/hictk_integration_suite \
	bin/hictk \
	test/integration/config.toml \
	--data-dir test/data \
	--threads "$NPROC" \
	--result-file integration-test-report.zoomify.json \
	--suites=zoomify

	macos-ci-status-check:
	name: Status Check (MacOS CI)
	if: ${{ always() }}
	runs-on: ubuntu-latest
	needs:
	- build-project
	- run-unit-tests
	- run-integration-tests

	steps:
	- name: Collect job results
	if: \|
	needs.build-project.result != 'success' \|\|
	needs.run-unit-tests.result != 'success' \|\|
	needs.run-integration-tests.result != 'success'
	run: exit 1

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Fix ccache cache eviction logic when using GHA job matrix #1258

Workflow file

Fix ccache cache eviction logic when using GHA job matrix #1258

Jobs

Run details

Workflow file for this run