Skip to content

Commit

Permalink
Merge pull request #1137 from zama-ai/fix-actionlint
Browse files Browse the repository at this point in the history
chore(ci): Fix actionlint
  • Loading branch information
BourgerieQuentin authored Nov 14, 2024
2 parents 20d03d4 + 4a7b2f1 commit 1b2d1d2
Show file tree
Hide file tree
Showing 17 changed files with 128 additions and 110 deletions.
5 changes: 5 additions & 0 deletions .github/actionlint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
self-hosted-runner:
labels:
- distributed-ci
- aws-mac1-metal
- aws-mac2-metal
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
name: Setup rust

description: Setup rust for optimizer

runs:
using: "composite"
steps:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
name: Setup rust toolchain for concrete-cpu

description: Setup rust toolchain for concrete-cpu

runs:
using: "composite"
steps:
Expand Down
12 changes: 7 additions & 5 deletions .github/workflows/concrete_compiler_benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,17 @@ jobs:
- name: Ouput benchmark details
id: benchmark-details
run: |
echo "bench_date=$(date --iso-8601=seconds)" >> "$GITHUB_OUTPUT"
echo "commit_date=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" >> "$GITHUB_OUTPUT"
echo "commit_hash=$(git describe --tags --dirty)" >> "$GITHUB_OUTPUT"
{
echo "bench_date=$(date --iso-8601=seconds)";
echo "commit_date=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})";
echo "commit_hash=$(git describe --tags --dirty)";
} >> "$GITHUB_OUTPUT"
- name: Set up home
# "Install rust" step require root user to have a HOME directory which is not set.
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu
- name: Build compiler benchmarks
run: |
set -e
Expand Down Expand Up @@ -126,7 +128,7 @@ jobs:
shell: bash
run: |
echo "Computing HMac on downloaded artifact"
SIGNATURE="$(slab/scripts/hmac_calculator.sh $parsed_benchmark_results.json '${{ secrets.JOB_SECRET }}')"
SIGNATURE="$(slab/scripts/hmac_calculator.sh parsed_benchmark_results.json ${{ secrets.JOB_SECRET }})"
echo "Sending results to Slab..."
curl -v -k \
-H "Content-Type: application/json" \
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/concrete_compiler_test_cpu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -127,10 +127,10 @@ jobs:
for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done;
- name: Enable complete tests on push to main
if: github.ref == 'refs/heads/main'
run: echo "MINIMAL_TESTS=OFF" >> ${GITHUB_ENV}
run: echo "MINIMAL_TESTS=OFF" >> "${GITHUB_ENV}"
- name: Enable minimal tests otherwise
if: github.ref != 'refs/heads/main'
run: echo "MINIMAL_TESTS=ON" >> ${GITHUB_ENV}
run: echo "MINIMAL_TESTS=ON" >> "${GITHUB_ENV}"
- name: Run compiler tests
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
with:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ jobs:
echo "HOME=/shared" >> "${GITHUB_ENV}"
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu

- name: Build end-to-end distributed test
run: |
Expand Down
28 changes: 15 additions & 13 deletions .github/workflows/concrete_compiler_test_macos_cpu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,14 @@ jobs:
submodules: recursive
fetch-depth: 0
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu
- name: Set python variables
run: |
set -e
echo "PIP=${{ format('pip{0}', matrix.python-version) }}" >> "${GITHUB_ENV}"
echo "PYTHON=${{ format('python{0}', matrix.python-version) }}" >> "${GITHUB_ENV}"
echo "PYTHON_EXEC=$(which ${{ format('python{0}', matrix.python-version) }})" >> "${GITHUB_ENV}"
{
echo "PIP=${{ format('pip{0}', matrix.python-version) }}"
echo "PYTHON=${{ format('python{0}', matrix.python-version) }}"
echo "PYTHON_EXEC=$(which ${{ format('python{0}', matrix.python-version) }})"
} >> "${GITHUB_ENV}"
- name: Install dependencies
run: |
brew install ninja ccache
Expand All @@ -56,30 +57,31 @@ jobs:
cd compilers/concrete-compiler/compiler
echo "Debug: ccache statistics (prior to the build):"
ccache -s
make Python3_EXECUTABLE=$PYTHON_EXEC all
make Python3_EXECUTABLE="${PYTHON_EXEC}" all
echo "Debug: ccache statistics (after the build):"
ccache -s
- name: Enable complete tests on push to main
if: github.ref == 'refs/heads/main'
run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV
run: echo "MINIMAL_TESTS=OFF" >> "${GITHUB_ENV}"
- name: Enable minimal tests otherwise
if: github.ref != 'refs/heads/main'
run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV
run: echo "MINIMAL_TESTS=ON" >> "${GITHUB_ENV}"
- name: Create keyset cache directory
run: |
export KEY_CACHE_DIRECTORY=$(mktemp -d)/KeySetCache
echo "KEY_CACHE_DIRECTORY=$KEY_CACHE_DIRECTORY" >> "${GITHUB_ENV}"
mkdir $KEY_CACHE_DIRECTORY
KEY_CACHE_DIRECTORY=$(mktemp -d)/KeySetCache
export KEY_CACHE_DIRECTORY
echo "KEY_CACHE_DIRECTORY=${KEY_CACHE_DIRECTORY}" >> "${GITHUB_ENV}"
mkdir "${KEY_CACHE_DIRECTORY}"
- name: Test
run: |
set -e
cd compilers/concrete-compiler/compiler
export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF
make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE=$PYTHON_EXEC run-tests
make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE="${PYTHON_EXEC}" run-tests
- name: Cleanup host
if: success() || failure()
run: |
rm -rf $KEY_CACHE_DIRECTORY
rm -rf "${KEY_CACHE_DIRECTORY}"
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/concrete_ml_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,11 +101,11 @@ jobs:
- name: ML Tests
run: |
export HOME="/home/ubuntu"
export CONCRETE_PYTHON_WHEEL=$(pwd)/frontends/concrete-python/dist/*manylinux*.whl
CONCRETE_PYTHON_WHEEL="$(pwd)/frontends/concrete-python/dist/*manylinux*.whl"
apt update
apt install git git-lfs -y
pip install poetry==1.7.1
./ci/scripts/test_cml.sh --use-wheel $CONCRETE_PYTHON_WHEEL --verbose
./ci/scripts/test_cml.sh --use-wheel "${CONCRETE_PYTHON_WHEEL}" --verbose
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/concrete_optimizer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup
uses: ./.github/workflows/optimizer_setup
uses: ./.github/actions/optimizer_setup
- name: Formatting
run: |
cargo --version
Expand All @@ -62,7 +62,7 @@ jobs:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup
uses: ./.github/workflows/optimizer_setup
uses: ./.github/actions/optimizer_setup
- name: Run benchmark
run: |
cd compilers/concrete-optimizer
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/concrete_python_benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,7 @@ jobs:
set -e
rm -rf /build/*
export PYTHON=${{ format('python{0}', matrix.python-version) }}
echo "Using $PYTHON"
export PYTHON=python
cd /concrete/frontends/concrete-python
make PYTHON=$PYTHON venv
Expand Down
12 changes: 7 additions & 5 deletions .github/workflows/concrete_python_finalize_release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }}
run: |
mkdir wheels
aws s3 cp s3://${S3_BUCKET_NAME}/cpu/concrete-python/ ./wheels/ --recursive --exclude "*" --include "concrete_python-${{ inputs.version }}-*"
aws s3 cp "s3://${S3_BUCKET_NAME}/cpu/concrete-python/" ./wheels/ --recursive --exclude "*" --include "concrete_python-${{ inputs.version }}-*"
echo "============== Downloaded wheels ==============="
ls -la ./wheels/
- name: Push wheels to public PyPI (public)
Expand All @@ -37,7 +37,7 @@ jobs:
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "test-linux-x86 (${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
SLACK_MESSAGE: "publish-to-pypi finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

publish-to-dockerhub:
runs-on: ubuntu-latest
Expand All @@ -50,9 +50,11 @@ jobs:
- name: Get version from tag
run: |
# remove leading 'v' and '-' from tag
export VERSION=`echo ${{ inputs.tag }} | sed "s/^v*//g" | sed "s/-//g"`
echo "VERSION=$VERSION" >> "${GITHUB_ENV}"
echo "NAME_TAG=${{ env.DOCKER_IMAGE_NAME }}:v$VERSION" >> "${GITHUB_ENV}"
VERSION=$(echo ${{ inputs.version }} | sed "s/^v*//g" | sed "s/-//g")
{
echo "VERSION=$VERSION"
echo "NAME_TAG=${{ env.DOCKER_IMAGE_NAME }}:v$VERSION"
} >> "${GITHUB_ENV}"
- name: Build image
run: |
mkdir empty_context
Expand Down
84 changes: 42 additions & 42 deletions .github/workflows/concrete_python_release_cpu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,13 @@ jobs:
run: |
NIGHTLY_VERSION=$(date +"%Y.%m.%d")
NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d")
LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o`
LATEST_RELEASE_VERSION=$(git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o)
echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt
git tag nightly-$NIGHTLY_VERSION || true
git push origin nightly-$NIGHTLY_VERSION || true
git tag "nightly-${NIGHTLY_VERSION}" || true
git push origin "nightly-${NIGHTLY_VERSION}" || true
- name: Set release version (public)
if: ${{ env.RELEASE_TYPE == 'public' }}
run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt
run: echo "__version__ = \"$(git describe --tags --abbrev=0 | grep -e '[0-9].*' -o)\"" >| frontends/concrete-python/version.txt
- name: Expose release version from Python
run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py
- name: Build wheel
Expand Down Expand Up @@ -154,23 +154,22 @@ jobs:
run: |
brew install ninja ccache
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu
- name: Set release version (nightly)
if: ${{ env.RELEASE_TYPE == 'nightly' }}
run: |
NIGHTLY_VERSION=$(date +"%Y.%m.%d")
NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d")
LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o`
echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt
NIGHTLY_VERSION=$(date +"%Y%m%d")
LATEST_RELEASE_VERSION=$(git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o)
echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION}\"" >| frontends/concrete-python/version.txt
- name: Set release version (public)
if: ${{ env.RELEASE_TYPE == 'public' }}
run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt
run: echo "__version__ = \"$(git describe --tags --abbrev=0 | grep -e '[0-9].*' -o)\"" >| frontends/concrete-python/version.txt
- name: Expose release version from Python
run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py
- name: Build wheel
run: |
export CONCRETE_PYTHON=$(pwd)/frontends/concrete-python
export CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler
CONCRETE_PYTHON=$(pwd)/frontends/concrete-python
CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler
export COMPILER_BUILD_DIRECTORY=$CONCRETE_COMPILER/build
export PYTHON=${{ format('python{0}', matrix.python-version) }}
echo "Using $PYTHON"
Expand All @@ -183,17 +182,17 @@ jobs:
$PYTHON -m venv .venv && . .venv/bin/activate
# Install requirements
pip install -r $CONCRETE_PYTHON/requirements.txt
pip install -r $CONCRETE_PYTHON/requirements.dev.txt
pip install -r "${CONCRETE_PYTHON}"/requirements.txt
pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt
# Build python bindings of concrete compiler
cd $CONCRETE_COMPILER
cd "${CONCRETE_COMPILER}" || exit
echo "Debug: ccache statistics (prior to the build):" && ccache -s
make Python3_EXECUTABLE=$(which python) python-bindings
make Python3_EXECUTABLE="$(which python)" python-bindings
echo "Debug: ccache statistics (after the build):" && ccache -s
# Build wheel
cd $CONCRETE_PYTHON
cd "${CONCRETE_PYTHON}" || exit
rm -rf dist
mkdir -p dist
pip wheel -v --no-deps -w dist .
Expand Down Expand Up @@ -228,7 +227,7 @@ jobs:
merge-multiple: true
- name: generate hash
id: hash
run: cd frontends/concrete-python/dist && echo "hash=$(sha256sum *.whl | base64 -w0)" >> $GITHUB_OUTPUT
run: cd frontends/concrete-python/dist && echo "hash=$(sha256sum ./*.whl | base64 -w0)" >> "${GITHUB_OUTPUT}"

provenance:
needs: [hash]
Expand Down Expand Up @@ -260,12 +259,13 @@ jobs:
- name: create draft release
if: ${{ env.RELEASE_TYPE == 'public'}}
run: |
export TAG=$(git describe --tags --abbrev=0)
echo $TAG
TAG=$(git describe --tags --abbrev=0)
export TAG
echo "${TAG}"
gh release create --draft --repo ${{ github.repository }} \
--verify-tag $TAG \
--title $TAG \
wheels/* *.intoto.jsonl/*
--verify-tag "${TAG}" \
--title "${TAG}" \
wheels/* ./*.intoto.jsonl/*
env:
GH_TOKEN: ${{ github.token }}
- name: Upload wheels to S3
Expand All @@ -279,7 +279,7 @@ jobs:
run: |
pip install boto3 bigtree
# upload wheels
aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/cpu/concrete-python
aws s3 sync ./wheels/ "s3://${S3_BUCKET_NAME}/cpu/concrete-python"
# update indexes and invalidate cloudfront cache
python .github/workflows/scripts/s3_update_html_indexes.py
- name: Slack Notification
Expand Down Expand Up @@ -323,24 +323,24 @@ jobs:
CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python
# Initialize an empty test environment
cd $(mktemp -d)
cd "$(mktemp -d)"
python -m venv .testenv && source .testenv/bin/activate
# Install the concrete-python wheel
pip install $WHEEL_DIR/*.whl
pip install "${WHEEL_DIR}"/*.whl
# Install extra requirements for tests
sudo apt update -y
sudo apt install -y graphviz libgraphviz-dev
pip install -r $CONCRETE_PYTHON/requirements.extra-full.txt
pip install -r $CONCRETE_PYTHON/requirements.dev.txt
pip install -r "${CONCRETE_PYTHON}"/requirements.extra-full.txt
pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt
# TODO - check for version
# Copy test files
cp -R $CONCRETE_PYTHON/tests .
cp -R $CONCRETE_PYTHON/pytest.ini .
cp $CONCRETE_PYTHON/Makefile .
cp -R "${CONCRETE_PYTHON}"/tests .
cp -R "${CONCRETE_PYTHON}"/pytest.ini .
cp "${CONCRETE_PYTHON}"/Makefile .
# Running tests
make tfhers-utils
Expand Down Expand Up @@ -402,27 +402,27 @@ jobs:
PYTHON=${{ format('python{0}', matrix.python-version) }}
# Initialize an empty test environment
export TEST_TMP_DIR=$(mktemp -d)
echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}"
cd $TEST_TMP_DIR
TEST_TMP_DIR=$(mktemp -d)
echo "TEST_TMP_DIR=${TEST_TMP_DIR}" >> "${GITHUB_ENV}"
cd "${TEST_TMP_DIR}" || exit
# Activate virtual environment
$PYTHON -m venv .testenv && source .testenv/bin/activate
# Install extra requirements for tests
pip install $WHEEL_DIR/*macos*.whl
pip install -r $CONCRETE_PYTHON/requirements.dev.txt
pip install "${WHEEL_DIR}"/*macos*.whl
pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt
# MacOS x86 have conflict between our OpenMP library, and one from torch
# we fix it by using a single one (from torch)
# see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8
export SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON_VERSION/site-packages/
find $SITE_PACKAGES -not \( -path $SITE_PACKAGES/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1 ln -f -s $SITE_PACKAGES/concrete/.dylibs/libomp.dylib
SITE_PACKAGES=$(pwd)/.testenv/lib/$PYTHON_VERSION/site-packages/
find . "${SITE_PACKAGES}" \( -not \( -path "${SITE_PACKAGES}"/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' \) -exec -n 1 ln -f -s "${SITE_PACKAGES}"/concrete/.dylibs/libomp.dylib {} \;
# Copy test files
cp -R $CONCRETE_PYTHON/tests .
cp -R $CONCRETE_PYTHON/pytest.ini .
cp $CONCRETE_PYTHON/Makefile .
cp -R "${CONCRETE_PYTHON}"/tests .
cp -R "${CONCRETE_PYTHON}"/pytest.ini .
cp "${CONCRETE_PYTHON}"/Makefile .
# Running tests
make tfhers-utils
Expand All @@ -431,7 +431,7 @@ jobs:
- name: Cleanup host
if: success() || failure()
run: |
rm -rf $TEST_TMP_DIR
rm -rf "${TEST_TMP_DIR}"
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
Expand Down
Loading

0 comments on commit 1b2d1d2

Please sign in to comment.