Create a reusable app for task review #995
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: cypress-end-to-end-tests | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - "**.md" | |
| push: | |
| branches: [main] | |
| paths-ignore: | |
| - "**.md" | |
| env: | |
| HYDRA_FULL_ERROR: 1 | |
| CYPRESS_HYDRA_FULL_ERROR: 1 | |
| jobs: | |
| # This job is made to setup path filtering, learn more about it here: https://github.com/facebookresearch/Mephisto/pull/857 | |
| changes: | |
| runs-on: ubuntu-latest | |
| # Set job outputs to values from filters step below | |
| outputs: | |
| simple_static_task: ${{ steps.filter.outputs.simple_static_task }} | |
| static_react_task: ${{ steps.filter.outputs.static_react_task }} | |
| static_react_task_with_tips: ${{ steps.filter.outputs.static_react_task_with_tips }} | |
| mnist: ${{ steps.filter.outputs.mnist }} | |
| template: ${{ steps.filter.outputs.template }} | |
| toxicity_detection: ${{ steps.filter.outputs.toxicity_detection }} | |
| abstractions: ${{ steps.filter.outputs.abstractions }} | |
| data_model: ${{ steps.filter.outputs.data_model }} | |
| operations: ${{ steps.filter.outputs.operations }} | |
| tools: ${{ steps.filter.outputs.tools }} | |
| mephisto-task: ${{ steps.filter.outputs.mephisto-task }} | |
| mephisto-worker-addons: ${{ steps.filter.outputs.mephisto-worker-addons }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 2 | |
| # Workaround to avoid Post Use step failures. See: https://github.com/actions/setup-node/issues/317 | |
| - run: mkdir -p /home/runner/work/Mephisto/Mephisto/.yarn/cache | |
| continue-on-error: true | |
| - uses: actions/setup-node@v3 | |
| with: | |
| cache: "yarn" | |
| node-version: 16 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| simple_static_task: | |
| - 'examples/simple_static_task/**' | |
| static_react_task: | |
| - 'examples/static_react_task/**' | |
| static_react_task_with_tips: | |
| - 'examples/static_react_task_with_tips/**' | |
| mnist: | |
| - 'examples/remote_procedure/mnist/**' | |
| template: | |
| - 'examples/remote_procedure/template/**' | |
| toxicity_detection: | |
| - 'examples/remote_procedure/toxicity_detection/**' | |
| abstractions: | |
| - 'mephisto/abstractions/**' | |
| data_model: | |
| - 'mephisto/data_model/**' | |
| operations: | |
| - 'mephisto/operations/**' | |
| tools: | |
| - 'mephisto/tools/**' | |
| mephisto-task: | |
| - 'packages/mephisto-task/src/**' | |
| mephisto-worker-addons: | |
| - 'packages/mephisto-worker-addons/src/**' | |
| # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/881 | |
| simple_static_task: | |
| needs: changes | |
| if: ${{ (needs.changes.outputs.simple_static_task == 'true') || (needs.changes.outputs.mephisto-task == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: 🔀 Checking out repo | |
| uses: actions/checkout@v2 | |
| - name: 🐍 Installing python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.8 | |
| - name: 🪨 Setup Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16.16.0 | |
| - name: 🤖 Install Mephisto | |
| run: pip install -e . | |
| - name: 🖋 Create data directory | |
| run: mkdir -p ~/mephisto/data | |
| - name: 📂 Set the data directory | |
| run: mephisto config core.main_data_directory ~/mephisto/data | |
| - name: 📦 Setting up mephisto-task package | |
| run: | | |
| cd packages/mephisto-task | |
| yarn install | |
| yarn build | |
| npm link | |
| # - name: Set env variables | |
| # run: export HYDRA_FULL_ERROR=1 CYPRESS_HYDRA_FULL_ERROR=1 | |
| # - name: Echo env variables | |
| # run: echo "HYDRA_FULL_ERROR --> $HYDRA_FULL_ERROR" | |
| - name: ⌛️ Running cypress tests | |
| uses: cypress-io/github-action@v3.1.0 | |
| with: | |
| build: npm i -D cypress@11 | |
| install: false | |
| browser: chrome | |
| # inputs: | |
| # HYDRA_FULL_ERROR: 1 | |
| # CYPRESS_HYDRA_FULL_ERROR: 1 | |
| project: ./mephisto/abstractions/blueprints/static_html_task/source | |
| config-file: ./cypress.config.js | |
| start: "python ./examples/simple_static_task/static_test_script.py" | |
| wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" | |
| headless: true | |
| # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/795 | |
| static-react-task: | |
| needs: changes | |
| if: ${{ (needs.changes.outputs.static_react_task == 'true') || (needs.changes.outputs.mephisto-task == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: 🔀 Checking out repo | |
| uses: actions/checkout@v2 | |
| - name: 🐍 Installing python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.8 | |
| - name: 🪨 Setup Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16.16.0 | |
| - name: 🤖 Install Mephisto | |
| run: pip install -e . | |
| - name: 🖋 Create data directory | |
| run: mkdir -p ~/mephisto/data | |
| - name: 📂 Set the data directory | |
| run: mephisto config core.main_data_directory ~/mephisto/data | |
| - name: 📦 Setting up mephisto-task package | |
| run: | | |
| cd packages/mephisto-task | |
| yarn install | |
| yarn build | |
| npm link | |
| - name: ⌛️ Running cypress tests | |
| uses: cypress-io/github-action@v3.1.0 | |
| with: | |
| build: npm i -D cypress@11 | |
| install: false | |
| browser: chrome | |
| project: ./examples/static_react_task/webapp | |
| config-file: ./cypress.config.js | |
| start: python examples/static_react_task/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh | |
| wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" | |
| headless: true | |
| # Learn more about the remote_procedure_tests here: https://github.com/facebookresearch/Mephisto/pull/800 | |
| remote_procedure_template: | |
| needs: changes | |
| if: ${{ (needs.changes.outputs.template == 'true') || (needs.changes.outputs.mephisto-task == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: 🔀 Checking out repo | |
| uses: actions/checkout@v2 | |
| - name: 🐍 Installing python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.8 | |
| - name: 🪨 Setup Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16.16.0 | |
| - name: 🤖 Install Mephisto | |
| run: pip install -e . | |
| - name: 🖋 Create data directory | |
| run: mkdir -p ~/mephisto/data | |
| - name: 📂 Set the data directory | |
| run: mephisto config core.main_data_directory ~/mephisto/data | |
| - name: 📦 Setting up mephisto-task package | |
| run: | | |
| cd packages/mephisto-task | |
| yarn install | |
| yarn build | |
| npm link | |
| - name: ⌛️ Running cypress tests | |
| uses: cypress-io/github-action@v3.1.0 | |
| with: | |
| build: npm i -D cypress@11 | |
| install: false | |
| browser: chrome | |
| project: ./examples/remote_procedure/template/webapp | |
| config-file: ./cypress.config.js | |
| start: python examples/remote_procedure/template/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh | |
| wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" | |
| headless: true | |
| remote_procedure_mnist: | |
| needs: changes | |
| if: ${{ (needs.changes.outputs.mnist == 'true') || (needs.changes.outputs.mephisto-task == 'true') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: 🔀 Checking out repo | |
| uses: actions/checkout@v2 | |
| - name: 🐍 Installing python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.8 | |
| - name: 🪨 Setup Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16.16.0 | |
| - name: 🤖 Install Mephisto | |
| run: | | |
| pip install -e . | |
| pip install torch pillow numpy | |
| - name: 🖋 Create data directory | |
| run: mkdir -p ~/mephisto/data | |
| - name: 📂 Set the data directory | |
| run: mephisto config core.main_data_directory ~/mephisto/data | |
| - name: 📦 Setting up mephisto-task package | |
| run: | | |
| cd packages/mephisto-task | |
| yarn install | |
| yarn build | |
| npm link | |
| - name: ⌛️ Running cypress tests | |
| uses: cypress-io/github-action@v3.1.0 | |
| with: | |
| build: npm i -D cypress@11 | |
| install: false | |
| browser: chrome | |
| project: ./examples/remote_procedure/mnist/webapp | |
| config-file: ./cypress.config.js | |
| start: python examples/remote_procedure/mnist/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh | |
| wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" | |
| headless: true | |
| remote_procedure_toxicity_detection: | |
| needs: changes | |
| if: ${{ (needs.changes.outputs.toxicity_detection == 'true') || (needs.changes.outputs.mephisto-task == 'true') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: 🔀 Checking out repo | |
| uses: actions/checkout@v2 | |
| - name: 🐍 Installing python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.8 | |
| - name: 🪨 Setup Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16.16.0 | |
| - name: 🤖 Install Mephisto | |
| run: pip install -e . | |
| - name: 🖋 Create data directory | |
| run: mkdir -p ~/mephisto/data | |
| - name: 📂 Set the data directory | |
| run: mephisto config core.main_data_directory ~/mephisto/data | |
| - name: 📦 Setting up mephisto-task package | |
| run: | | |
| cd packages/mephisto-task | |
| yarn install | |
| yarn build | |
| npm link | |
| - name: ⌛️ Running cypress tests | |
| uses: cypress-io/github-action@v3.1.0 | |
| with: | |
| build: npm i -D cypress@11 | |
| install: false | |
| browser: chrome | |
| project: ./examples/remote_procedure/toxicity_detection/webapp | |
| config-file: ./cypress.config.js | |
| start: python examples/remote_procedure/toxicity_detection/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh | |
| wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" | |
| headless: true | |
| # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/833 | |
| static_react_task_with_tips: | |
| needs: changes | |
| if: ${{ (needs.changes.outputs.static_react_task_with_tips == 'true') || (needs.changes.outputs.mephisto-task == 'true') || (needs.changes.outputs.mephisto-worker-addons == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: 🔀 Checking out repo | |
| uses: actions/checkout@v2 | |
| - name: 🐍 Installing python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.8 | |
| - name: 🪨 Setup Node | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16.16.0 | |
| - name: 🤖 Install Mephisto | |
| run: | | |
| pip install -e . | |
| yes | pip uninstall detoxify | |
| - name: 🖋 Create data directory | |
| run: mkdir -p ~/mephisto/data | |
| - name: 📂 Set the data directory | |
| run: mephisto config core.main_data_directory ~/mephisto/data | |
| - name: 📦 Setting up mephisto-task package | |
| run: | | |
| cd packages/mephisto-task | |
| yarn install | |
| yarn build | |
| npm link | |
| - name: 📦 Setting up mephisto-worker-addons package | |
| run: | | |
| cd packages/mephisto-worker-addons | |
| yarn install | |
| yarn build | |
| npm link | |
| - name: ⌛️ Running pre-submission cypress tests | |
| uses: cypress-io/github-action@v3.1.0 | |
| with: | |
| build: npm i -D cypress@11 | |
| install: false | |
| browser: chrome | |
| project: ./examples/static_react_task_with_tips/webapp | |
| config-file: ./cypress.config.js | |
| spec: ./examples/static_react_task_with_tips/webapp/cypress/e2e/pre_submission_tests/* | |
| start: python examples/static_react_task_with_tips/run_task.py mephisto.task.force_rebuild=true mephisto.task.post_install_script=link_packages.sh | |
| wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" | |
| headless: true | |
| - name: 🔪 Killing the web server | |
| run: | | |
| lsof -nPi :3000 | |
| lsof -i -P -n | grep LISTEN | grep python | awk '{print $2}' | |
| kill -INT $(lsof -i -P -n | grep LISTEN | grep python | awk '{print $2}') | |
| echo "killed 1" | |
| sleep 0.5 | |
| kill -INT $(lsof -i -P -n | grep LISTEN | grep python | awk '{print $2}') | |
| echo "killed 2" | |
| lsof -i -P -n | grep LISTEN | grep node | awk '{print $2}' | |
| kill -INT $(lsof -i -P -n | grep LISTEN | grep node | awk '{print $2}') | |
| sleep 30 | |
| echo "30 seconds passed" | |
| - name: 🥛 Expiring units | |
| run: | | |
| cd mephisto/scripts/local_db/gh_actions | |
| python expire_all_units.py | |
| - name: 📚 Accepting the first submitted tip, accepting the second submitted tip, and rejecting the last submitted tip | |
| run: | | |
| cd mephisto/scripts/local_db | |
| python review_tips_for_task.py << EOF | |
| react-static-task-with-tips | |
| a | |
| 5 | |
| The tip is very informative | |
| a | |
| 0 | |
| r | |
| EOF | |
| - name: 🔥 Removing the second accepted tip | |
| run: | | |
| cd mephisto/scripts/local_db | |
| python remove_accepted_tip.py << EOF | |
| react-static-task-with-tips | |
| n | |
| y | |
| EOF | |
| - name: 📖 Reviewing the accepted feedback | |
| run: | | |
| cd mephisto/scripts/local_db | |
| python review_feedback_for_task.py << EOF | |
| react-static-task-with-tips | |
| 0 | |
| n | |
| u | |
| y | |
| y | |
| EOF | |
| python review_feedback_for_task.py << EOF | |
| react-static-task-with-tips | |
| 1 | |
| n | |
| u | |
| y | |
| n | |
| EOF | |
| - name: ⌛️ Running post-submission cypress tests | |
| uses: cypress-io/github-action@v3.1.0 | |
| with: | |
| build: npm i -D cypress@11 | |
| install: false | |
| project: ./examples/static_react_task_with_tips/webapp | |
| config-file: cypress.config.js | |
| spec: ./examples/static_react_task_with_tips/webapp/cypress/e2e/post_submission_tests/* | |
| start: python examples/static_react_task_with_tips/run_task.py mephisto.task.post_install_script=link_packages.sh mephisto.task.force_rebuild=true | |
| wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" | |
| browser: chrome | |
| headless: true |