cypress-end-to-end-tests.yml

name: cypress-end-to-end-tests
on:
  pull_request:
    paths-ignore:
      - "**.md"
  push:
    branches: [main]
    paths-ignore:
      - "**.md"
env:
  HYDRA_FULL_ERROR: 1
  CYPRESS_HYDRA_FULL_ERROR: 1
jobs:
  # This job is made to setup path filtering, learn more about it here: https://github.com/facebookresearch/Mephisto/pull/857
  changes:
    runs-on: ubuntu-latest
    # Set job outputs to values from filters step below
    outputs:
      simple_static_task: ${{ steps.filter.outputs.simple_static_task }}
      static_react_task: ${{ steps.filter.outputs.static_react_task }}
      static_react_task_with_tips: ${{ steps.filter.outputs.static_react_task_with_tips }}
      mnist: ${{ steps.filter.outputs.mnist }}
      template: ${{ steps.filter.outputs.template }}
      toxicity_detection: ${{ steps.filter.outputs.toxicity_detection }}
      abstractions: ${{ steps.filter.outputs.abstractions }}
      data_model: ${{ steps.filter.outputs.data_model }}
      operations: ${{ steps.filter.outputs.operations }}
      tools: ${{ steps.filter.outputs.tools }}
      mephisto-task: ${{ steps.filter.outputs.mephisto-task }}
      mephisto-worker-addons: ${{ steps.filter.outputs.mephisto-worker-addons }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 2
      # Workaround to avoid Post Use step failures. See: https://github.com/actions/setup-node/issues/317
      - run: mkdir -p /home/runner/work/Mephisto/Mephisto/.yarn/cache
        continue-on-error: true
      - uses: actions/setup-node@v3
        with:
          cache: "yarn"
          node-version: 16
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            simple_static_task:
              - 'examples/simple_static_task/**'
            static_react_task:
              - 'examples/static_react_task/**'
            static_react_task_with_tips:
              - 'examples/static_react_task_with_tips/**'
            mnist:
              - 'examples/remote_procedure/mnist/**'
            template:
              - 'examples/remote_procedure/template/**'
            toxicity_detection:
              - 'examples/remote_procedure/toxicity_detection/**'
            abstractions:
              - 'mephisto/abstractions/**'
            data_model:
              - 'mephisto/data_model/**'
            operations:
              - 'mephisto/operations/**'
            tools:
              - 'mephisto/tools/**'
            mephisto-task:
              - 'packages/mephisto-task/src/**'
            mephisto-worker-addons:
              - 'packages/mephisto-worker-addons/src/**'

  # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/881
  simple_static_task:
    needs: changes
    if: ${{ (needs.changes.outputs.simple_static_task == 'true') || (needs.changes.outputs.mephisto-task == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
    runs-on: ubuntu-latest
    steps:
      - name: 🔀 Checking out repo
        uses: actions/checkout@v2

      - name: 🐍 Installing python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: 🪨 Setup Node
        uses: actions/setup-node@v3
        with:
          node-version: 16.16.0

      - name: 🤖 Install Mephisto
        run: pip install -e .

      - name: 🖋 Create data directory
        run: mkdir -p ~/mephisto/data

      - name: 📂 Set the data directory
        run: mephisto config core.main_data_directory ~/mephisto/data

      - name: 📦 Setting up mephisto-task package
        run: |
          cd packages/mephisto-task
          yarn install
          yarn build
          npm link

      # - name: Set env variables
      #   run: export HYDRA_FULL_ERROR=1 CYPRESS_HYDRA_FULL_ERROR=1

      # - name: Echo env variables
      #   run: echo "HYDRA_FULL_ERROR --> $HYDRA_FULL_ERROR"

      - name: ⌛️ Running cypress tests
        uses: cypress-io/github-action@v3.1.0
        with:
          build: npm i -D cypress@11
          install: false
          browser: chrome
          # inputs:
          #   HYDRA_FULL_ERROR: 1
          #   CYPRESS_HYDRA_FULL_ERROR: 1
          project: ./mephisto/abstractions/blueprints/static_html_task/source
          config-file: ./cypress.config.js
          start: "python ./examples/simple_static_task/static_test_script.py"
          wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
          headless: true

  # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/795
  static-react-task:
    needs: changes
    if: ${{ (needs.changes.outputs.static_react_task == 'true') || (needs.changes.outputs.mephisto-task == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
    runs-on: ubuntu-latest
    steps:
      - name: 🔀 Checking out repo
        uses: actions/checkout@v2

      - name: 🐍 Installing python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: 🪨 Setup Node
        uses: actions/setup-node@v3
        with:
          node-version: 16.16.0

      - name: 🤖 Install Mephisto
        run: pip install -e .

      - name: 🖋 Create data directory
        run: mkdir -p ~/mephisto/data

      - name: 📂 Set the data directory
        run: mephisto config core.main_data_directory ~/mephisto/data

      - name: 📦 Setting up mephisto-task package
        run: |
          cd packages/mephisto-task
          yarn install
          yarn build
          npm link

      - name: ⌛️ Running cypress tests
        uses: cypress-io/github-action@v3.1.0
        with:
          build: npm i -D cypress@11
          install: false
          browser: chrome
          project: ./examples/static_react_task/webapp
          config-file: ./cypress.config.js
          start: python examples/static_react_task/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh
          wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
          headless: true

  # Learn more about the remote_procedure_tests here: https://github.com/facebookresearch/Mephisto/pull/800
  remote_procedure_template:
    needs: changes
    if: ${{ (needs.changes.outputs.template == 'true') || (needs.changes.outputs.mephisto-task == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
    runs-on: ubuntu-latest
    steps:
      - name: 🔀 Checking out repo
        uses: actions/checkout@v2

      - name: 🐍 Installing python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: 🪨 Setup Node
        uses: actions/setup-node@v3
        with:
          node-version: 16.16.0

      - name: 🤖 Install Mephisto
        run: pip install -e .

      - name: 🖋 Create data directory
        run: mkdir -p ~/mephisto/data

      - name: 📂 Set the data directory
        run: mephisto config core.main_data_directory ~/mephisto/data

      - name: 📦 Setting up mephisto-task package
        run: |
          cd packages/mephisto-task
          yarn install
          yarn build
          npm link

      - name: ⌛️ Running cypress tests
        uses: cypress-io/github-action@v3.1.0
        with:
          build: npm i -D cypress@11
          install: false
          browser: chrome
          project: ./examples/remote_procedure/template/webapp
          config-file: ./cypress.config.js
          start: python examples/remote_procedure/template/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh
          wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
          headless: true

  remote_procedure_mnist:
    needs: changes
    if: ${{ (needs.changes.outputs.mnist == 'true') || (needs.changes.outputs.mephisto-task == 'true') }}
    runs-on: ubuntu-latest
    steps:
      - name: 🔀 Checking out repo
        uses: actions/checkout@v2

      - name: 🐍 Installing python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: 🪨 Setup Node
        uses: actions/setup-node@v3
        with:
          node-version: 16.16.0

      - name: 🤖 Install Mephisto
        run: |
          pip install -e .
          pip install torch pillow numpy

      - name: 🖋 Create data directory
        run: mkdir -p ~/mephisto/data

      - name: 📂 Set the data directory
        run: mephisto config core.main_data_directory ~/mephisto/data

      - name: 📦 Setting up mephisto-task package
        run: |
          cd packages/mephisto-task
          yarn install
          yarn build
          npm link

      - name: ⌛️ Running cypress tests
        uses: cypress-io/github-action@v3.1.0
        with:
          build: npm i -D cypress@11
          install: false
          browser: chrome
          project: ./examples/remote_procedure/mnist/webapp
          config-file: ./cypress.config.js
          start: python examples/remote_procedure/mnist/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh
          wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
          headless: true

  remote_procedure_toxicity_detection:
    needs: changes
    if: ${{ (needs.changes.outputs.toxicity_detection == 'true') || (needs.changes.outputs.mephisto-task == 'true') }}
    runs-on: ubuntu-latest
    steps:
      - name: 🔀 Checking out repo
        uses: actions/checkout@v2

      - name: 🐍 Installing python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: 🪨 Setup Node
        uses: actions/setup-node@v3
        with:
          node-version: 16.16.0

      - name: 🤖 Install Mephisto
        run: pip install -e .

      - name: 🖋 Create data directory
        run: mkdir -p ~/mephisto/data

      - name: 📂 Set the data directory
        run: mephisto config core.main_data_directory ~/mephisto/data

      - name: 📦 Setting up mephisto-task package
        run: |
          cd packages/mephisto-task
          yarn install
          yarn build
          npm link

      - name: ⌛️ Running cypress tests
        uses: cypress-io/github-action@v3.1.0
        with:
          build: npm i -D cypress@11
          install: false
          browser: chrome
          project: ./examples/remote_procedure/toxicity_detection/webapp
          config-file: ./cypress.config.js
          start: python examples/remote_procedure/toxicity_detection/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh
          wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
          headless: true

  # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/833
  static_react_task_with_tips:
    needs: changes
    if: ${{ (needs.changes.outputs.static_react_task_with_tips == 'true') || (needs.changes.outputs.mephisto-task == 'true') || (needs.changes.outputs.mephisto-worker-addons == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
    runs-on: ubuntu-latest
    steps:
      - name: 🔀 Checking out repo
        uses: actions/checkout@v2
      - name: 🐍 Installing python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: 🪨 Setup Node
        uses: actions/setup-node@v3
        with:
          node-version: 16.16.0

      - name: 🤖 Install Mephisto
        run: |
          pip install -e .
          yes | pip uninstall detoxify

      - name: 🖋 Create data directory
        run: mkdir -p ~/mephisto/data

      - name: 📂 Set the data directory
        run: mephisto config core.main_data_directory ~/mephisto/data

      - name: 📦 Setting up mephisto-task package
        run: |
          cd packages/mephisto-task
          yarn install
          yarn build
          npm link

      - name: 📦 Setting up mephisto-worker-addons package
        run: |
          cd packages/mephisto-worker-addons
          yarn install
          yarn build
          npm link

      - name: ⌛️ Running pre-submission cypress tests
        uses: cypress-io/github-action@v3.1.0
        with:
          build: npm i -D cypress@11
          install: false
          browser: chrome
          project: ./examples/static_react_task_with_tips/webapp
          config-file: ./cypress.config.js
          spec: ./examples/static_react_task_with_tips/webapp/cypress/e2e/pre_submission_tests/*
          start: python examples/static_react_task_with_tips/run_task.py mephisto.task.force_rebuild=true mephisto.task.post_install_script=link_packages.sh
          wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
          headless: true

      - name: 🔪 Killing the web server
        run: |
          lsof -nPi :3000
          lsof -i -P -n | grep LISTEN | grep python | awk '{print $2}'
          kill -INT $(lsof -i -P -n | grep LISTEN | grep python | awk '{print $2}')
          echo "killed 1"
          sleep 0.5
          kill -INT $(lsof -i -P -n | grep LISTEN | grep python | awk '{print $2}')
          echo "killed 2"
          lsof -i -P -n | grep LISTEN | grep node | awk '{print $2}'
          kill -INT $(lsof -i -P -n | grep LISTEN | grep node | awk '{print $2}')
          sleep 30
          echo "30 seconds passed"

      - name: 🥛 Expiring units
        run: |
          cd mephisto/scripts/local_db/gh_actions
          python expire_all_units.py

      - name: 📚 Accepting the first submitted tip, accepting the second submitted tip, and rejecting the last submitted tip
        run: |
          cd mephisto/scripts/local_db
          python review_tips_for_task.py << EOF
          react-static-task-with-tips
          a
          5
          The tip is very informative
          a
          0
          r
          EOF

      - name: 🔥 Removing the second accepted tip
        run: |
          cd mephisto/scripts/local_db
          python remove_accepted_tip.py << EOF
          react-static-task-with-tips
          n
          y
          EOF

      - name: 📖 Reviewing the accepted feedback
        run: |
          cd mephisto/scripts/local_db
          python review_feedback_for_task.py << EOF
          react-static-task-with-tips
          0
          n
          u
          y
          y
          EOF
          python review_feedback_for_task.py << EOF
          react-static-task-with-tips
          1
          n
          u
          y
          n
          EOF

      - name: ⌛️ Running post-submission cypress tests
        uses: cypress-io/github-action@v3.1.0
        with:
          build: npm i -D cypress@11
          install: false
          project: ./examples/static_react_task_with_tips/webapp
          config-file: cypress.config.js
          spec: ./examples/static_react_task_with_tips/webapp/cypress/e2e/post_submission_tests/*
          start: python examples/static_react_task_with_tips/run_task.py mephisto.task.post_install_script=link_packages.sh mephisto.task.force_rebuild=true
          wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1"
          browser: chrome
          headless: true