Skip to content

Commit

Permalink
test.yml: use image cache with parameters created from template
Browse files Browse the repository at this point in the history
Use `actions/cache@v4` with following params:
path: ".download/by-url-sha256/$(echo $location | sha256sum | cut -d' ' -f1)"
key: "digest-$digest"
enableCrossOsArchive: true

To make the image cache cross-platform, the cache directory is specified using a relative path from the working directory, and the platform-specific cache directories are accessed via symbolic links.

This should reduce the cache size used by the CI.

Signed-off-by: Norio Nomura <norio.nomura@gmail.com>
  • Loading branch information
norio-nomura committed Jul 25, 2024
1 parent 3075bd9 commit 2e32b6f
Show file tree
Hide file tree
Showing 3 changed files with 100 additions and 27 deletions.
79 changes: 79 additions & 0 deletions .github/actions/setup_cache_for_template/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
name: 'setup cache for template'
description: 'setup cache for template'
inputs:
arch:
description: arch to setup cache for
required: false
template:
description: template yaml file
required: true
runs:
using: "composite"
steps:
- name: "detect platform for download directory"
id: detect-platform
run: |
if [[ "$(uname)" == "Darwin" ]]; then
download_dir=~/Library/Caches/lima/download
else
download_dir=~/.cache/lima/download
fi
echo "download-dir=$download_dir" >> "$GITHUB_OUTPUT"
shell: bash
- name: "create cache parameters from template"
if: always()
id: cache-params-from-template
run: |
set -eux
arch="${{ inputs.arch }}"
template="${{ inputs.template }}"
test -f "$template" || exit 1
# detect arch from template if not provided
arch="${arch:-$(yq '.arch // ""' "$template")}"
arch="${arch:-$(uname -m)}"
# normalize arch. amd64 -> x86_64, arm64 -> aarch64
case "$arch" in
amd64) arch=x86_64 ;;
arm64) arch=aarch64 ;;
esac
# extract digest and location from template using arch
digest="$(yq ".images | map(select(.arch == \"$arch\")) | .[0].digest // \"\"" "$template")"
location="$(yq ".images | map(select(.arch == \"$arch\")) | .[0].location // \"\"" "$template")"
test -n "$location" || exit 1
# path to cache
if command -v sha256sum > /dev/null; then
sha256="$(echo -n "$location" | sha256sum | cut -d' ' -f1)"
elif command -v shasum > /dev/null; then
sha256="$(echo -n "$location" | shasum -a 256 | cut -d' ' -f1)"
else
echo "sha256sum or shasum not found" >&2
exit 1
fi
echo "path=.download/by-url-sha256/$sha256" >> "$GITHUB_OUTPUT"
# key for cache
key="${digest:+image-$digest}"
# fallback to os and hash of template file if digest not found
key="${key:-${{ runner.os }}-${{ hashFiles(inputs.template) }}}"
echo "key=$key" >> "$GITHUB_OUTPUT"
shell: bash

- name: "Cache ${{ steps.cache-params-from-template.outputs.path }}"
# avoid using `~` in path that will be expanded to platform specific home directory
uses: actions/cache@v4
with:
path: ${{ steps.cache-params-from-template.outputs.path }}
key: ${{ steps.cache-params-from-template.outputs.key }}
enableCrossOsArchive: true

- name: "Create symbolic link named ${{ steps.detect-platform.outputs.download-dir }} pointing to .download"
run: |
set -eux
[ -d .download ] || mkdir -p .download
path_to_cache=${{ steps.detect-platform.outputs.download-dir }}
mkdir -p $(dirname $path_to_cache)
ln -sfn $PWD/.download $path_to_cache
shell: bash
46 changes: 20 additions & 26 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -157,12 +157,10 @@ jobs:
- uses: actions/setup-go@v5
with:
go-version: 1.22.x
- name: Cache ~/Library/Caches/lima/download
uses: actions/cache@v4
- name: Cache image used by default.yaml
uses: ./.github/actions/setup_cache_for_template
with:
path: ~/Library/Caches/lima/download
# hashFiles do not seem to support symlinks
key: ${{ runner.os }}-${{ hashFiles('templates/default.yaml') }}
template: templates/default.yaml
- name: Unit tests
run: go test -v ./...
- name: Make
Expand Down Expand Up @@ -225,15 +223,14 @@ jobs:
- uses: actions/setup-go@v5
with:
go-version: 1.22.x
- id: path_for_hashFiles
# It seems that `hashFiles` cannot use `..` as a path component, so generate a normalized path here.
run: echo "NORMALIZED=$(realpath --relative-to=$PWD examples/${{ matrix.template }})" >> "$GITHUB_OUTPUT"
- uses: actions/cache@v4
- name: normalize template path
id: normalize_template_path
# `hashFiles` cannot use `..` as a path component, so generate a normalized path here.
run: echo "NORMALIZED=$(realpath templates/${{ matrix.template }})" >> "$GITHUB_OUTPUT"
- name: Cache image used by ${{ steps.normalize_template_path.outputs.NORMALIZED }}
uses: ./.github/actions/setup_cache_for_template
with:
path: ~/.cache/lima/download
# hashFiles do not seem to support symlinks
# TODO: more fine-grained cache
key: ${{ runner.os }}-${{ hashFiles(steps.path_for_hashFiles.outputs.NORMALIZED) }}
template: ${{ steps.normalize_template_path.outputs.NORMALIZED }}
- name: Make
run: make
- name: Install
Expand Down Expand Up @@ -322,12 +319,10 @@ jobs:
- uses: actions/setup-go@v5
with:
go-version: 1.22.x
- name: Cache ~/Library/Caches/lima/download
uses: actions/cache@v4
- name: Cache image used by vmnet.yaml
uses: ./.github/actions/setup_cache_for_template
with:
path: ~/Library/Caches/lima/download
# hashFiles do not seem to support symlinks
key: ${{ runner.os }}-${{ hashFiles('examples/vmnet.yaml') }}
template: templates/vmnet.yaml
- name: Make
run: make
- name: Install
Expand Down Expand Up @@ -404,15 +399,14 @@ jobs:
- uses: actions/setup-go@v5
with:
go-version: 1.22.x
- id: path_for_hashFiles
# It seems that `hashFiles` cannot use `..` as a path component, so generate a normalized path here.
run: echo "NORMALIZED=$(realpath examples/${{ matrix.template }})" >> "$GITHUB_OUTPUT"
- name: Cache ~/Library/Caches/lima/download
uses: actions/cache@v4
- name: normalize template path
id: normalize_template_path
# `hashFiles` cannot use `..` as a path component, so generate a normalized path here.
run: echo "NORMALIZED=$(realpath templates/${{ matrix.template }})" >> "$GITHUB_OUTPUT"
- name: Cache image used by ${{ steps.normalize_template_path.outputs.NORMALIZED }}
uses: ./.github/actions/setup_cache_for_template
with:
path: ~/Library/Caches/lima/download
# hashFiles do not seem to support symlinks
key: ${{ runner.os }}-${{ hashFiles(steps.path_for_hashFiles.outputs.NORMALIZED) }}
template: ${{ steps.normalize_template_path.outputs.NORMALIZED }}
- name: Make
run: make
- name: Install
Expand Down
2 changes: 1 addition & 1 deletion hack/debug-cache.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ cache_dir="${HOME}/Library/Caches"
if [ "$(uname -s)" != "Darwin" ]; then
cache_dir="${HOME}/.cache"
fi
if [ ! -e "${cache_dir}/lima" ]; then
if [ ! -e "${cache_dir}/lima/download/by-url-sha256" ]; then
echo "No cache"
exit 0
fi
Expand Down

0 comments on commit 2e32b6f

Please sign in to comment.