ci_runner.yml

on:
  workflow_call:
    inputs:  
      python-version:
        required: true
        type: string
    secrets:
      CASJOBS_USERID:
        description: 'CASJOBS user ID'
        required: false
      CASJOBS_PW:
        description: 'CASJOBS password'
        required: false

#permissions:
#  id-token: write

  
jobs:
  gather-notebooks:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
    - uses: actions/checkout@v3
    ## routine to gather only the changed notebook files and supply them to the matrix
    - name: changed-files
      id: get-changed-files
      uses: tj-actions/changed-files@v36
      with:
        separator: ","
        files: |
          **/*.ipynb
    ## convert the list of files to an array and push them into the matrix as a json object      
    - name: set-matrix
      id: set-matrix
      run: |
        IFS=',' read -r -a array <<< "${{steps.get-changed-files.outputs.all_changed_files}}"
        echo "{$array}"
        echo "matrix=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${array[@]}")" >> $GITHUB_OUTPUT
  notebook-execution:
    needs: gather-notebooks
    environment: ci_env
    runs-on: ubuntu-latest
    permissions:
      contents: write
    strategy:
        fail-fast: false
        matrix:
            notebooks: ${{ fromJson(needs.gather-notebooks.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v3
        with:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Set up Python ${{ inputs.python-version }}
        uses: actions/setup-python@v4 ## needed for caching
        with:
          python-version: ${{ inputs.python-version }}
          cache: 'pip'
      - name: Add conda to system path
        run: |
          # $CONDA is an environment variable pointing to the root of the miniconda directory
          echo $CONDA/bin >> $GITHUB_PATH
      - name: Install dependencies
        run: |
          ## Install the local requirements file
          echo DEBUG ---
          pwd ## print working directory
          echo "Path to req's: $(dirname ${{ matrix.notebooks }})/requirements.txt"
          ls $(dirname ${{ matrix.notebooks }})
          echo ---
          if [ -f $(dirname "${{ matrix.notebooks }}")/pre-requirements.sh ]; then
            chmod +x $(dirname "${{ matrix.notebooks }}")/pre-requirements.sh
            ./$(dirname "${{ matrix.notebooks }}")/pre-requirements.sh
          fi
          if [ -f pre-requirements.txt ]; then
            pip install -r pre-requirements.txt
          fi
          if [ -f $(dirname "${{ matrix.notebooks }}")/pre-installl.sh ]; then
            chmod +x $(dirname "${{ matrix.notebooks }}")/pre-install.sh
            ./$(dirname "${{ matrix.notebooks }}")/pre-install.sh
          fi
          #pip install -r $(dirname "${{ matrix.notebooks }}")/requirements.txt
          pip install -r $(dirname ${{ matrix.notebooks }})/requirements.txt
          pip install pytest
          pip install nbval
          pip install nbconvert
          pip install bandit
          
      - name: Security testing with Bandit
        run: | 
          bandit "${{ matrix.notebooks }}"
          
      - name: Execute notebooks
        id: execute
        run: |
          #export CASJOBS_PW="$CI_CASJOBS_PW"
          #export CASJOBS_USERID="$CI_CASJOBS_USERID"
          jupyter nbconvert --to notebook --execute --inplace ${{ matrix.notebooks }}
          
      - name: Validate notebooks 
        run: |
         jupyter nbconvert --clear-output --inplace "${{ matrix.notebooks }}" 
         pytest --nbval "${{ matrix.notebooks }}" 
         
      - name: Archive executed notebooks
        run: |
          git config pull.rebase false
          git config user.name "GitHub Actions"
          git config user.email "actions@github.com"
      
          # Check if branch exists and checkout or create
          git show-ref --verify --quiet refs/heads/gh-storage
          if [ $? -ne 0 ]; then
              git checkout -b gh-storage
          else
              git checkout gh-storage
          fi
      
          git add ${{ matrix.notebooks }}
      
          if [[ $(git status --porcelain) ]]; then
              git commit -m 'Added executed notebook ${{ matrix.notebooks }}'
          else
              echo "No changes to commit"
              exit 0
          fi
      
          ATTEMPTS_LEFT=3
          SLEEP_TIME=5
          until [ $ATTEMPTS_LEFT -lt 1 ]
          do
            # Fetch and reset instead of pulling
            git fetch origin gh-storage
            git reset --hard origin/gh-storage
            git push origin gh-storage 2>&1 | tee output.log
            
            if grep -q "Updates were rejected because the remote contains work" output.log; then
              echo "Push failed due to conflicting remote changes. Remaining attempts: $ATTEMPTS_LEFT"
              let "ATTEMPTS_LEFT--"
              sleep $SLEEP_TIME
              SLEEP_TIME=$(($SLEEP_TIME * 2))
            else
              break
            fi
    done