FederatedRuntime Workflow for CI Pipeline - 301 Watermarking notebook run #5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #--------------------------------------------------------------------------- | |
| # Workflow to run 301_MNIST_Watermarking notebook | |
| # Authors - Noopur, Payal Chaurasiya | |
| #--------------------------------------------------------------------------- | |
| name: Federated Runtime 301 MNIST Watermarking | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| jobs: | |
| run_notebook: | |
| if: github.event.pull_request.draft == false | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Checkout OpenFL repository | |
| uses: actions/checkout@v4.1.1 | |
| with: | |
| fetch-depth: 2 # needed for detecting changes | |
| submodules: "true" | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: "3.10" | |
| - name: Install Jupyter Lab Package | |
| run: pip install jupyterlab | |
| - name: Install dependencies # Without this step, fx command will not work | |
| id: install_dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install . | |
| - name: Activate and run the director in background | |
| run: | | |
| ps -ef| grep -e "director start" -e "envoy start" | |
| cd openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking | |
| fx experimental activate | |
| cd director | |
| nohup ./start_director.sh &> director.log & | |
| - name: Run the envoy Bangalore in background | |
| run: | | |
| cd openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking | |
| cd Bangalore | |
| nohup ./start_envoy.sh Bangalore Bangalore_config.yaml &> bangalore.log & | |
| - name: Run the envoy Chandler in background | |
| run: | | |
| cd openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking | |
| cd Chandler | |
| nohup ./start_envoy.sh Chandler Chandler_config.yaml &> chandler.log & | |
| - name: Check if envoys are connected to the director | |
| run: | | |
| cd openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking | |
| timeout=60 | |
| connected=false | |
| while [ $timeout -gt 0 ]; do | |
| if grep "Bangalore is connected to the director" ./Bangalore/bangalore.log && grep "Chandler is connected to the director" ./Chandler/chandler.log; then | |
| connected=true | |
| break | |
| else | |
| echo "Waiting for both envoys to connect to the director" | |
| fi | |
| sleep 5 | |
| timeout=$((timeout-5)) | |
| done | |
| if [ $connected = false ]; then | |
| echo "Envoys are not connected to the director in 60 seconds" | |
| echo "| Failure Reason |" >> $GITHUB_STEP_SUMMARY | |
| echo "| -------------- |" >> $GITHUB_STEP_SUMMARY | |
| echo "Envoys are not connected to the director in 60 seconds" >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| fi | |
| - name: Run Notebook | |
| run: | | |
| ps -ef| grep -e "director start" -e "envoy start" | |
| cd openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/workspace | |
| jupyter nbconvert --execute --to notebook ./MNIST_Watermarking.ipynb | |
| echo "Notebook run completed" | |
| - name: Check if the experiment was successful using logs | |
| run: | | |
| cd openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking | |
| if grep "End of Federation reached." ./Bangalore/bangalore.log; then | |
| echo "Process for Bangalore envoy is successful" | |
| else | |
| echo "Process for Bangalore envoy failed" | |
| fi | |
| if grep "End of Federation reached." ./Chandler/chandler.log; then | |
| echo "Process for Chandler envoy is successful" | |
| else | |
| echo "Process for Chandler envoy failed" | |
| fi | |
| if grep "Experiment FederatedFlow_MNIST_Watermarking was finished successfully" ./director/director.log; then | |
| echo "Notebook run was successful" | |
| else | |
| echo "Notebook run failed" | |
| fi | |
| - name: Tar files | |
| if: ${{ always() }} # collect artifacts regardless of failures | |
| run: | | |
| tar -cvf notebook.tar \ | |
| ./openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/workspace/MNIST_Watermarking.nbconvert.ipynb \ | |
| ./openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/workspace/MNIST_Watermarking.ipynb \ | |
| ./openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/director/director.log \ | |
| ./openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Bangalore/bangalore.log \ | |
| ./openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/Chandler/chandler.log \ | |
| --ignore-failed-read | |
| echo "TAR file created" | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} # collect artifacts regardless of failures | |
| with: | |
| name: federated_runtime_301_watermarking_${{ github.run_id }} | |
| path: notebook.tar |