Skip to content
Test Suite

Add hf datasets integration #280

Sign in to view logs

Add hf datasets integration

Add hf datasets integration #280

Workflow file for this run

.github/workflows/test_suite.yml at d99b4b8
name: Test Suite
on:
push:
branches:
- main
- develop
pull_request:
types:
- opened
- reopened
- synchronize
env:
CACHE_NUMBER: 4 # increase to reset cache manually
CONDA_ENV_FILE: 'env.yaml'
CONDA_ENV_NAME: 'project-test'
COOKIECUTTER_PROJECT_NAME: 'project-test'
HUGGING_FACE_HUB_TOKEN: ${{secrets.HUGGING_FACE_HUB_TOKEN}}
jobs:
build:
strategy:
fail-fast: false
matrix:
python-version: ['3.11']
include:
- os: ubuntu-20.04
label: linux-64
prefix: /usr/share/miniconda3/envs/
# - os: macos-latest
# label: osx-64
# prefix: /Users/runner/miniconda3/envs/$CONDA_ENV_NAME
# - os: windows-latest
# label: win-64
# prefix: C:\Miniconda3\envs\$CONDA_ENV_NAME
name: ${{ matrix.label }}-py${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
steps:
- name: Parametrize conda env name
run: echo "PY_CONDA_ENV_NAME=${{ env.CONDA_ENV_NAME }}-${{ matrix.python-version }}" >> $GITHUB_ENV
- name: echo conda env name
run: echo ${{ env.PY_CONDA_ENV_NAME }}
- name: Parametrize conda prefix
run: echo "PY_PREFIX=${{ matrix.prefix }}${{ env.PY_CONDA_ENV_NAME }}" >> $GITHUB_ENV
- name: echo conda prefix
run: echo ${{ env.PY_PREFIX }}
- name: Define generated project files paths
run: |
echo "PROJECT_SETUPCFG_FILE=${{ env.COOKIECUTTER_PROJECT_NAME }}/setup.cfg" >> $GITHUB_ENV
echo "PROJECT_CONDAENV_FILE=${{ env.COOKIECUTTER_PROJECT_NAME }}/${{ env.CONDA_ENV_FILE }}" >> $GITHUB_ENV
echo "PROJECT_PRECOMMIT_FILE=${{ env.COOKIECUTTER_PROJECT_NAME }}/.pre-commit-config.yaml" >> $GITHUB_ENV
- uses: actions/checkout@v2
# COOKIECUTTER GENERATION
- name: Set up Python
uses: actions/setup-python@v1
- name: Install Dependencies
shell: bash -l {0}
run: |
pip install cookiecutter
- name: Generate Repo
shell: bash -l {0}
run: |
echo -e 'n\nn\nn\n' | cookiecutter . --no-input project_name=${{ env.COOKIECUTTER_PROJECT_NAME }}
- name: Init git into generated repo
shell: bash -l {0}
run: |
git config --global user.name ci-bot
git config --global user.email ci-bot@ci.com
git init
git add --all
git commit -m "Initial commit"
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}
- name: Define cache key postfix
run: |
echo "CACHE_KEY_POSTFIX=${{ matrix.label }}-${{ matrix.python-version }}-${{ env.CACHE_NUMBER }}-${{ env.PY_CONDA_ENV_NAME }}-${{ hashFiles(env.PROJECT_CONDAENV_FILE) }}-${{ hashFiles(env.PROJECT_SETUPCFG_FILE) }}" >> $GITHUB_ENV
- name: Echo cache keys
run: |
echo ${{ env.PROJECT_SETUPCFG_FILE }}
echo ${{ env.PROJECT_CONDAENV_FILE }}
echo ${{ env.PROJECT_PRECOMMIT_FILE }}
echo ${{ env.CACHE_KEY_POSTFIX }}
# GENERATED PROJECT CI/CD
# Remove the python version pin from the env.yml which could be inconsistent
- name: Remove explicit python version from the environment
shell: bash -l {0}
run: |
sed -Ei '/^\s*-?\s*python\s*([#=].*)?$/d' ${{ env.CONDA_ENV_FILE }}
cat ${{ env.CONDA_ENV_FILE }}
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}
# Install torch cpu-only
- name: Install torch cpu only
shell: bash -l {0}
run: |
sed -i '/nvidia\|cuda/d' ${{ env.CONDA_ENV_FILE }}
cat ${{ env.CONDA_ENV_FILE }}
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}
- name: Setup Mambaforge
uses: conda-incubator/setup-miniconda@v2
with:
miniforge-variant: Mambaforge
miniforge-version: latest
activate-environment: ${{ env.PY_CONDA_ENV_NAME }}
python-version: ${{ matrix.python-version }}
use-mamba: true
- uses: actions/cache@v2
name: Conda cache
with:
path: ${{ env.PY_PREFIX }}
key: conda-${{ env.CACHE_KEY_POSTFIX }}
id: conda_cache
- uses: actions/cache@v2
name: Pip cache
with:
path: ~/.cache/pip
key: pip-${{ env.CACHE_KEY_POSTFIX }}
- uses: actions/cache@v2
name: Pre-commit cache
with:
path: ~/.cache/pre-commit
key: pre-commit-${{ hashFiles(env.PROJECT_PRECOMMIT_FILE) }}-conda-${{ env.CACHE_KEY_POSTFIX }}
# Ensure the hack for the python version worked
- name: Ensure we have the right Python
shell: bash -l {0}
run: |
echo "Installed Python: $(python --version)"
echo "Expected: ${{ matrix.python-version }}"
python --version | grep "Python ${{ matrix.python-version }}"
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}
# https://stackoverflow.com/questions/70520120/attributeerror-module-setuptools-distutils-has-no-attribute-version
# https://github.com/pytorch/pytorch/pull/69904
- name: Downgrade setuptools due to a but in PyTorch 1.10.1
shell: bash -l {0}
run: |
pip install setuptools==59.5.0 --upgrade
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}
- name: Update conda environment
run: mamba env update -n ${{ env.PY_CONDA_ENV_NAME }} -f ${{ env.CONDA_ENV_FILE }}
if: steps.conda_cache.outputs.cache-hit != 'true'
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}
# Update pip env whether or not there was a conda cache hit
- name: Update pip environment
shell: bash -l {0}
run: pip install -e ".[dev]"
if: steps.conda_cache.outputs.cache-hit == 'true'
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}
- run: pip3 list
shell: bash -l {0}
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}
- run: mamba info
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}
- run: mamba list
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}
# Ensure the hack for the python version worked
- name: Ensure we have the right Python
shell: bash -l {0}
run: |
echo "Installed Python: $(python --version)"
echo "Expected: ${{ matrix.python-version }}"
python --version | grep "Python ${{ matrix.python-version }}"
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}
- name: Run pre-commits
shell: bash -l {0}
run: |
pre-commit install
pre-commit run -v --all-files --show-diff-on-failure
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}
- name: Test with pytest
shell: bash -l {0}
run: |
pytest -v
working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }}