Update ci_runner.yml #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | ||
| workflow_call: | ||
| inputs: | ||
| python-version: | ||
| required: true | ||
| type: string | ||
| secrets: | ||
| CASJOBS_USERID: | ||
| description: 'CASJOBS user ID' | ||
| required: false | ||
| CASJOBS_PW: | ||
| description: 'CASJOBS password' | ||
| required: false | ||
| #permissions: | ||
| # id-token: write | ||
| jobs: | ||
| gather-notebooks: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| matrix: ${{ steps.set-matrix.outputs.matrix }} | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| ## routine to gather only the changed notebook files and supply them to the matrix | ||
| - name: changed-files | ||
| id: get-changed-files | ||
| uses: tj-actions/changed-files@v36 | ||
| with: | ||
| separator: "," | ||
| files: | | ||
| **/*.ipynb | ||
| ## convert the list of files to an array and push them into the matrix as a json object | ||
| - name: set-matrix | ||
| id: set-matrix | ||
| run: | | ||
| IFS=',' read -r -a array <<< "${{steps.get-changed-files.outputs.all_changed_files}}" | ||
| echo "{$array}" | ||
| echo "matrix=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${array[@]}")" >> $GITHUB_OUTPUT | ||
| notebook-execution: | ||
| needs: gather-notebooks | ||
| environment: ci_env | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| contents: write | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| notebooks: ${{ fromJson(needs.gather-notebooks.outputs.matrix) }} | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Set up Python ${{ inputs.python-version }} | ||
| uses: actions/setup-python@v4 ## needed for caching | ||
| with: | ||
| python-version: ${{ inputs.python-version }} | ||
| cache: 'pip' | ||
| - name: Add conda to system path | ||
| run: | | ||
| # $CONDA is an environment variable pointing to the root of the miniconda directory | ||
| echo $CONDA/bin >> $GITHUB_PATH | ||
| - name: Install dependencies | ||
| run: | | ||
| ## Install the local requirements file | ||
| echo DEBUG --- | ||
| pwd ## print working directory | ||
| echo "Path to req's: $(dirname ${{ matrix.notebooks }})/requirements.txt" | ||
| ls $(dirname ${{ matrix.notebooks }}) | ||
| echo --- | ||
| if [ -f $(dirname "${{ matrix.notebooks }}")/pre-requirements.sh ]; then | ||
| chmod +x $(dirname "${{ matrix.notebooks }}")/pre-requirements.sh | ||
| ./$(dirname "${{ matrix.notebooks }}")/pre-requirements.sh | ||
| fi | ||
| if [ -f pre-requirements.txt ]; then | ||
| pip install -r pre-requirements.txt | ||
| fi | ||
| if [ -f $(dirname "${{ matrix.notebooks }}")/pre-installl.sh ]; then | ||
| chmod +x $(dirname "${{ matrix.notebooks }}")/pre-install.sh | ||
| ./$(dirname "${{ matrix.notebooks }}")/pre-install.sh | ||
| fi | ||
| #pip install -r $(dirname "${{ matrix.notebooks }}")/requirements.txt | ||
| pip install -r $(dirname ${{ matrix.notebooks }})/requirements.txt | ||
| pip install pytest | ||
| pip install nbval | ||
| pip install nbconvert | ||
| pip install bandit | ||
| - name: Security testing with Bandit | ||
| run: | | ||
| bandit "${{ matrix.notebooks }}" | ||
| - name: Execute notebooks | ||
| id: execute | ||
| run: | | ||
| #export CASJOBS_PW="$CI_CASJOBS_PW" | ||
| #export CASJOBS_USERID="$CI_CASJOBS_USERID" | ||
| jupyter nbconvert --to notebook --execute --inplace ${{ matrix.notebooks }} | ||
| - name: Validate notebooks | ||
| run: | | ||
| jupyter nbconvert --clear-output --inplace "${{ matrix.notebooks }}" | ||
| pytest --nbval "${{ matrix.notebooks }}" | ||
| - name: Archive executed notebooks | ||
| run: | | ||
| git config pull.rebase false | ||
| git config user.name "GitHub Actions" | ||
| git config user.email "actions@github.com" | ||
| # Check if branch exists and checkout or create | ||
| git show-ref --verify --quiet refs/heads/gh-storage | ||
| if [ $? -ne 0 ]; then | ||
| git checkout -b gh-storage | ||
| else | ||
| git checkout gh-storage | ||
| fi | ||
| git add ${{ matrix.notebooks }} | ||
| if [[ $(git status --porcelain) ]]; then | ||
| git commit -m 'Added executed notebook ${{ matrix.notebooks }}' | ||
| else | ||
| echo "No changes to commit" | ||
| exit 0 | ||
| fi | ||
| ATTEMPTS_LEFT=3 | ||
| SLEEP_TIME=5 | ||
| until [ $ATTEMPTS_LEFT -lt 1 ] | ||
| do | ||
| # Fetch and reset instead of pulling | ||
| git fetch origin gh-storage | ||
| git reset --hard origin/gh-storage | ||
| git push origin gh-storage 2>&1 | tee output.log | ||
| if grep -q "Updates were rejected because the remote contains work" output.log; then | ||
| echo "Push failed due to conflicting remote changes. Remaining attempts: $ATTEMPTS_LEFT" | ||
| let "ATTEMPTS_LEFT--" | ||
| sleep $SLEEP_TIME | ||
| SLEEP_TIME=$(($SLEEP_TIME * 2)) | ||
| else | ||
| break | ||
| fi | ||
| done | ||
