[LAB-723] Update CI to use Bacalhau 1.1.2 #264
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # This workflow will build a golang project and runs CI tests | |
| name: CI | |
| on: | |
| push: | |
| branches: ["main"] | |
| paths-ignore: | |
| - 'docs/**' | |
| - 'infrastructure/**' | |
| pull_request: | |
| branches: ["main"] | |
| paths-ignore: | |
| - 'docs/**' | |
| - 'infrastructure/**' | |
| # Only run one at a time | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| go: | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-22.04 | |
| # - macos-13 | |
| runs-on: ${{ matrix.os }} | |
| environment: ci | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: 1.20.3 | |
| - name: Install dependencies | |
| run: go mod download | |
| - name: Build | |
| run: go build | |
| - name: upload plex binary to be used by other jobs | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: plex-binary-${{matrix.os}} | |
| path: ./plex | |
| - name: Test | |
| run: go test ./... -v | |
| # # Only downloading cli for macos | |
| # ci-setup-compose: | |
| # strategy: | |
| # matrix: | |
| # os: [macos-13] | |
| # runs-on: ${{ matrix.os }} | |
| # environment: ci | |
| # steps: | |
| # - name: Download docker compose | |
| # run: | | |
| # # Download docker-compose plugin | |
| # curl -sSL https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-darwin-aarch64 -o docker-compose | |
| # - name: upload docker-compose plugin to be used later | |
| # uses: actions/upload-artifact@v3 | |
| # with: | |
| # name: docker-compose-plugin-${{matrix.os}} | |
| # path: ./docker-compose | |
| ci-setup-ipfs: | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-22.04 | |
| # - macos-13 | |
| runs-on: ${{ matrix.os }} | |
| environment: ci | |
| steps: | |
| - name: Download and extract ipfs binary tarball | |
| run: | | |
| # Download ipfs binary | |
| # Download docker-compose plugin | |
| if [[ "$OSTYPE" == "linux-gnu"* ]]; then | |
| curl -sSL https://github.com/ipfs/kubo/releases/download/v0.18.0/kubo_v0.18.0_linux-amd64.tar.gz -o kubo.tgz | |
| elif [[ "$OSTYPE" == "darwin"* ]]; then | |
| curl -sSL https://github.com/ipfs/kubo/releases/download/v0.18.0/kubo_v0.18.0_darwin-arm64.tar.gz -o kubo.tgz | |
| fi | |
| tar -zxvf kubo.tgz | |
| - name: upload ipfs binary to be used later | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ipfs-binary-${{matrix.os}} | |
| path: ./kubo/ipfs | |
| # Download Bacalhau CLI for troubleshooting | |
| ci-setup-bacalhau: | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-22.04 | |
| # - macos-13 | |
| bacalhau_version: | |
| - 1.1.2 | |
| runs-on: ${{ matrix.os }} | |
| environment: ci | |
| steps: | |
| - name: Download bacalhau | |
| run: | | |
| # Download bacalhau plugin | |
| if [[ "$OSTYPE" == "linux-gnu"* ]]; then | |
| curl -sSL https://github.com/bacalhau-project/bacalhau/releases/download/v${{ matrix.bacalhau_version }}/bacalhau_v${{ matrix.bacalhau_version }}_linux_amd64.tar.gz -o bacalhau.tgz | |
| elif [[ "$OSTYPE" == "darwin"* ]]; then | |
| curl -sSL https://github.com/bacalhau-project/bacalhau/releases/download/v${{ matrix.bacalhau_version }}/bacalhau_v${{ matrix.bacalhau_version }}_darwin_arm64.tar.gz -o bacalhau.tgz | |
| fi | |
| tar -zxvf bacalhau.tgz | |
| - name: upload bacalhau plugin to be used later | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: bacalhau-binary-${{matrix.os}}-${{matrix.bacalhau_version}} | |
| path: ./bacalhau | |
| ci-public: | |
| needs: | |
| - go | |
| # - ci-setup-compose | |
| - ci-setup-bacalhau | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-22.04 | |
| # - macos-13 | |
| bacalhau_version: | |
| - 1.1.2 | |
| runs-on: ${{ matrix.os }} | |
| environment: ci | |
| env: | |
| BACALHAU_VERSION: ${{matrix.bacalhau_version}} | |
| # Setting it at workflow level to be used by all the steps | |
| BACALHAU_API_HOST: "127.0.0.1" | |
| LOG_LEVEL: trace | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: download plex binary | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: plex-binary-${{matrix.os}} | |
| - name: download bacalhau binary | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: bacalhau-binary-${{matrix.os}}-${{matrix.bacalhau_version}} | |
| - name: Setup docker (missing on MacOS) | |
| if: runner.os == 'macos' | |
| run: | | |
| brew install colima docker docker-compose | |
| colima start | |
| # For testcontainers to find the Colima socket | |
| # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running | |
| sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock | |
| mkdir -p ~/.docker/cli-plugins | |
| ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose | |
| docker version | |
| docker compose version | |
| - name: Setup docker compose plugin | |
| if: runner.os != 'macos' | |
| run: | | |
| # Output version info | |
| docker version | |
| docker compose version | |
| - name: docker compose build | |
| run: | | |
| # Build in parallel | |
| docker compose build --parallel | |
| env: | |
| NEXT_PUBLIC_PRIVY_APP_ID: ${{ secrets.NEXT_PUBLIC_PRIVY_APP_ID }} | |
| - name: Bring up the stack | |
| run: | | |
| # Setup docker compose | |
| docker compose -f docker-compose.yml up -d --wait | |
| env: | |
| NEXT_PUBLIC_PRIVY_APP_ID: ${{ secrets.NEXT_PUBLIC_PRIVY_APP_ID }} | |
| - name: Run docker compose ps | |
| run: | | |
| # Inspect number of running containers | |
| docker compose ps | |
| - name: Run Equibind | |
| run: | | |
| # Execute permission | |
| chmod +x ./bacalhau ./plex | |
| # Dump BACALHAU_API_HOST | |
| echo "BACALHAU_API_HOST set to ${BACALHAU_API_HOST}" | |
| # Create bacalhau config file | |
| ./bacalhau version | |
| # Pull equibind image before running the job to get better idea of job run | |
| docker pull $(cat tools/equibind-ci.json | jq -r '.dockerPull') | |
| # Get node list | |
| ./bacalhau node list | |
| # Run the job | |
| ./plex init -t tools/equibind-ci.json -i '{"protein": ["testdata/binding/abl/7n9g.pdb"], "small_molecule": ["testdata/binding/abl/ZINC000003986735.sdf"]}' --scatteringMethod=dotProduct --autoRun=true -a test -a ci | tee -a plex_run_output.log | |
| result_dir=$(cat plex_run_output.log | grep 'Finished processing, results written to' | sed -n 's/^.*Finished processing, results written to //p' | sed 's/\/io.json//') | |
| # chdir for outputs | |
| cd "$result_dir/entry-0/outputs" | |
| if [ "$(find . -name '*docked.sdf' | grep 'docked.sdf')" == "" ]; then | |
| echo "No docked files found" | |
| exit 1 | |
| else | |
| echo "Docked files found:" | |
| find . -name '*docked.sdf' | grep 'docked.sdf' | |
| fi | |
| - name: Run bacalhau describe | |
| # run always even when | |
| if: always() | |
| run: | | |
| JOB_ID=$(./bacalhau list --output json | jq -r '.[0].State.JobID') | |
| ./bacalhau describe ${JOB_ID} | |
| - name: Run Gateway Integration Tests | |
| uses: ./.github/actions/gateway-tests | |
| - name: Run docker compose logs | |
| # run always even when | |
| if: always() | |
| run: | | |
| docker compose logs | |
| - name: Run docker logs to get additional logs | |
| # run always even when | |
| if: always() | |
| run: | | |
| docker ps -a | |
| for container in $(docker ps -qa); do echo ${container};docker logs ${container};done | |
| - name: upload outputs | |
| # run always even when | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ci-public-output-${{matrix.os}}-${{matrix.bacalhau_version}} | |
| path: | | |
| plex_run_output.log | |
| job-* | |
| jobs/ | |
| ci-private: | |
| needs: | |
| - go | |
| # - ci-setup-compose | |
| - ci-setup-ipfs | |
| - ci-setup-bacalhau | |
| strategy: | |
| matrix: | |
| os: | |
| - ubuntu-22.04 | |
| # - macos-13 | |
| bacalhau_version: | |
| - 1.1.2 | |
| runs-on: ${{ matrix.os }} | |
| environment: ci | |
| env: | |
| BACALHAU_VERSION: ${{matrix.bacalhau_version}} | |
| # Setting it at workflow level to be used by all the steps | |
| BACALHAU_API_HOST: "127.0.0.1" | |
| LOG_LEVEL: trace | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: download plex binary | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: plex-binary-${{matrix.os}} | |
| - name: download ipfs binary | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ipfs-binary-${{matrix.os}} | |
| - name: download bacalhau binary | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: bacalhau-binary-${{matrix.os}}-${{matrix.bacalhau_version}} | |
| - name: Setup docker (missing on MacOS) | |
| if: runner.os == 'macos' | |
| run: | | |
| brew install colima docker docker-compose | |
| colima start | |
| # For testcontainers to find the Colima socket | |
| # https://github.com/abiosoft/colima/blob/main/docs/FAQ.md#cannot-connect-to-the-docker-daemon-at-unixvarrundockersock-is-the-docker-daemon-running | |
| sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock | |
| mkdir -p ~/.docker/cli-plugins | |
| ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose | |
| docker version | |
| docker compose version | |
| - name: Setup docker compose plugin | |
| if: runner.os != 'macos' | |
| run: | | |
| # Output version info | |
| docker version | |
| docker compose version | |
| - name: docker compose build | |
| run: | | |
| # Build in parallel | |
| docker compose build --parallel | |
| env: | |
| NEXT_PUBLIC_PRIVY_APP_ID: ${{ secrets.NEXT_PUBLIC_PRIVY_APP_ID }} | |
| - name: Bring up the stack | |
| run: | | |
| # Setup docker compose private | |
| docker compose -f docker-compose.yml -f docker-compose.private.yml up -d --wait | |
| env: | |
| NEXT_PUBLIC_PRIVY_APP_ID: ${{ secrets.NEXT_PUBLIC_PRIVY_APP_ID }} | |
| - name: Run docker compose ps | |
| run: | | |
| # Inspect number of running containers | |
| docker compose ps | |
| - name: Run Equibind | |
| # turn back on once baclahau docker in docker job launcher is fixed | |
| run: | | |
| set -x | |
| # Execute permission | |
| chmod +x ./bacalhau ./plex ./ipfs | |
| # Dump BACALHAU_API_HOST | |
| echo "BACALHAU_API_HOST set to ${BACALHAU_API_HOST}" | |
| # Create bacalhau config file | |
| ./bacalhau version | |
| # Pull equibind image before running the job to get better idea of job run | |
| docker pull $(cat tools/equibind-ci.json | jq -r '.dockerPull') | |
| # Get node list | |
| ./bacalhau node list | |
| # Run tests against it | |
| # using temp directory for ipfs stuff | |
| export IPFS_PATH=$(mktemp -d) | |
| # Initialize IPFS repo | |
| ./ipfs init -e | |
| # Copy over swarm key and config | |
| cp -rav $(pwd)/docker/ipfs_data/* "${IPFS_PATH}/" | |
| export BACALHAU_SERVE_IPFS_PATH="${IPFS_PATH}" | |
| export BACALHAU_IPFS_SWARM_ADDRESSES="/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWLpoHJCGxxKozRaUK1e1m2ocyVPB9dzbsU2cydujYBCD7" | |
| export BACALHAU_NODE_IPFS_SWARMADDRESSES="/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWLpoHJCGxxKozRaUK1e1m2ocyVPB9dzbsU2cydujYBCD7" | |
| # Run the job | |
| ./plex init -t tools/equibind-ci.json -i '{"protein": ["testdata/binding/abl/7n9g.pdb"], "small_molecule": ["testdata/binding/abl/ZINC000003986735.sdf"]}' --scatteringMethod=dotProduct --autoRun=true -a test -a ci | tee -a plex_run_output.log | |
| result_dir=$(cat plex_run_output.log | grep 'Finished processing, results written to' | sed -n 's/^.*Finished processing, results written to //p' | sed 's/\/io.json//') | |
| cd "$result_dir/entry-0/outputs" | |
| ls -ltraR | |
| cat exitCode stderr stderr log.txt || true | |
| if [ "$(find . -name '*docked.sdf' | grep 'docked.sdf')" == "" ]; then | |
| echo "No docked files found" | |
| exit 1 | |
| else | |
| echo "Docked files found:" | |
| find . -name '*docked.sdf' | grep 'docked.sdf' | |
| fi | |
| - name: Run bacalhau describe | |
| # run always even when | |
| if: always() | |
| run: | | |
| JOB_ID=$(./bacalhau list --output json | jq -r '.[0].State.JobID') | |
| ./bacalhau describe ${JOB_ID} | |
| - name: Run Gateway Integration Tests | |
| uses: ./.github/actions/gateway-tests | |
| - name: Run docker compose logs | |
| # run always even when | |
| if: always() | |
| run: | | |
| docker compose logs | |
| - name: Run docker logs to get additional logs | |
| # run always even when | |
| if: always() | |
| run: | | |
| docker ps -a | |
| for container in $(docker ps -qa); do echo ${container};docker logs ${container};done | |
| - name: upload outputs | |
| # run always even when | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ci-private-output-${{matrix.os}}-${{matrix.bacalhau_version}} | |
| path: | | |
| plex_run_output.log | |
| job-* | |
| jobs/ |