Update ci_build_merge_manual.yml #4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Manual Full Archive Execution/HTML Gen/Deply | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| python-version: | ||
| required: true | ||
| type: string | ||
| secrets: | ||
| CASJOBS_USERID: | ||
| description: 'CASJOBS user ID' | ||
| required: false | ||
| CASJOBS_PW: | ||
| description: 'CASJOBS password' | ||
| required: false | ||
| env: | ||
| CASJOBS_PW: ${{ secrets.CASJOBS_PW }} | ||
| CASJOBS_USERID: ${{ secrets.CASJOBS_USERID }} | ||
| jobs: | ||
| find_notebooks: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| notebook_list: ${{ steps.find.outputs.notebook_list }} | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v3 | ||
| - name: Find notebook files | ||
| id: find | ||
| run: | | ||
| notebook_files=$(find notebooks -name "*.ipynb" | jq -R -s -c 'split("\n")[:-1]') | ||
| #echo "::set-output name=notebook_list::$notebook_files" | ||
| echo "notebook_list=$notebook_files" >>$GITHUB_OUTPUT | ||
| execute_notebooks: | ||
| needs: find_notebooks | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| executed_notebooks: ${{ steps.execute.outputs.executed_notebooks }} | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| notebook: ${{fromJson(needs.find_notebooks.outputs.notebook_list)}} | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v3 | ||
| - name: Set up Python ${{ inputs.python-version }} | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: ${{ inputs.python-version }} | ||
| - name: Install dependencies | ||
| continue-on-error: true | ||
| run: | | ||
| export PYDEVD_DISABLE_FILE_VALIDATION=1 | ||
| export SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL=True | ||
| python -m pip install --upgrade pip | ||
| pip install pytest | ||
| pip install nbval | ||
| pip install nbconvert | ||
| echo "Path to req's: $(dirname ${{ matrix.notebook }})/requirements.txt" | ||
| ls $(dirname ${{ matrix.notebook }}) | ||
| echo --- | ||
| if [ -f $(dirname "${{ matrix.notebook }}")/pre-installl.sh ]; then | ||
| chmod +x $(dirname "${{ matrix.notebook }}")/pre-install.sh | ||
| ./$(dirname "${{ matrix.notebook }}")/pre-install.sh | ||
| fi | ||
| if [ -f $(dirname "${{ matrix.notebook }}")/pre-requirements.sh ]; then | ||
| chmod +x $(dirname "${{ matrix.notebook }}")/pre-requirements.sh | ||
| ./$(dirname "${{ matrix.notebook }}")/pre-requirements.sh | ||
| fi | ||
| if [ -f pre-requirements.txt ]; then | ||
| pip install -r pre-requirements.txt | ||
| fi | ||
| pip install -r $(dirname ${{ matrix.notebook }})/requirements.txt | ||
| - name: Execute notebook | ||
| id: execute | ||
| run: | | ||
| export PYDEVD_DISABLE_FILE_VALIDATION=1 | ||
| export SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL=True | ||
| NOTEBOOK="${{ matrix.notebook }}" | ||
| if [ "$NOTEBOOK" != "notebooks/preimaging/preimaging_01_mirage.ipynb" ]; then | ||
| if jupyter nbconvert --to notebook --execute --inplace "$NOTEBOOK"; then | ||
| echo "Notebook executed successfully: $NOTEBOOK" | ||
| else | ||
| echo "Notebook execution failed, inserting failure message: $NOTEBOOK" | ||
| python .github/scripts/insert_failure_message.py "$NOTEBOOK" | ||
| fi | ||
| git config --global user.name "GitHub Actions" | ||
| git config --global user.email "actions@github.com" | ||
| UNIQUE_BRANCH="storage-${{ github.run_id }}-$(echo "$NOTEBOOK" | sed 's/\//_/g')" | ||
| git checkout -b "$UNIQUE_BRANCH" | ||
| git add "$NOTEBOOK" | ||
| git commit -m "Added executed notebook $NOTEBOOK" | ||
| git push origin "$UNIQUE_BRANCH" | ||
| fi | ||
| replace_in_storage: | ||
| needs: execute_notebooks | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout gh-storage branch | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| ref: gh-storage | ||
| - name: Set up Git config | ||
| run: | | ||
| git config pull.rebase false | ||
| git config --global user.name "GitHub Actions" | ||
| git config --global user.email "actions@github.com" | ||
| - name: Replace files with the executed notebooks | ||
| run: | | ||
| git fetch --all | ||
| BRANCHES=$(git branch -r | grep 'storage-' | sed 's/origin\///') | ||
| for BRANCH in $BRANCHES; do | ||
| # Extract the notebook path from the branch name | ||
| NOTEBOOK=$(echo $BRANCH | sed 's/storage-[0-9]*-//; s/_/\//g') | ||
| echo "Replacing file in gh-storage with the one from $BRANCH" | ||
| git fetch origin $BRANCH | ||
| git checkout "origin/$BRANCH" -- "$NOTEBOOK" | ||
| # If the notebook exists and was checked out, commit it to gh-storage | ||
| if [ -f "$NOTEBOOK" ]; then | ||
| git add "$NOTEBOOK" | ||
| git commit -m "Update executed notebook from $BRANCH" -- "$NOTEBOOK" | ||
| # Push the change to the gh-storage branch | ||
| git push origin gh-storage | ||
| else | ||
| echo "Notebook file $NOTEBOOK was not found in $BRANCH." | ||
| fi | ||
| done | ||
| - name: Delete unique branches | ||
| run: | | ||
| git fetch --prune | ||
| BRANCHES=$(git branch -r | grep 'storage-' | sed 's/origin\///') | ||
| for BRANCH in $BRANCHES; do | ||
| git push origin --delete $BRANCH | ||
| done | ||
| generate_html: | ||
| runs-on: ubuntu-latest | ||
| needs: replace_in_storage | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v3 | ||
| - name: Set up Python ${{ inputs.python-version }} | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: ${{ inputs.python-version }} | ||
| - name: Install dependencies | ||
| run: | | ||
| pip install jupyter-book | ||
| - name: Build HTML | ||
| run: | | ||
| git fetch | ||
| git checkout origin/gh-storage -- notebooks/ | ||
| jupyter-book build . | ||
| # Push the book's HTML to github-pages | ||
| - name: GitHub Pages action | ||
| uses: peaceiris/actions-gh-pages@v3 | ||
| with: | ||
| github_token: ${{ secrets.GITHUB_TOKEN }} | ||
| publish_dir: ./_build/html | ||
