Skip to content

Merge pull request #1263 from facebookresearch/add-provider-doc-pages #1382

Merge pull request #1263 from facebookresearch/add-provider-doc-pages

Merge pull request #1263 from facebookresearch/add-provider-doc-pages #1382

name: cypress-end-to-end-tests
on:
pull_request:
paths-ignore:
- "**.md"
push:
branches: [main]
paths-ignore:
- "**.md"
env:
HYDRA_FULL_ERROR: 1
jobs:
# This job is made to setup path filtering, learn more about it here: https://github.com/facebookresearch/Mephisto/pull/857
changes:
runs-on: ubuntu-latest
# Set job outputs to values from filters step below
outputs:
simple_static_task: ${{ steps.filter.outputs.simple_static_task }}
static_react_task: ${{ steps.filter.outputs.static_react_task }}
static_react_task_with_worker_opinion: ${{ steps.filter.outputs.static_react_task_with_worker_opinion }}
elementary_remote_procedure: ${{ steps.filter.outputs.elementary_remote_procedure }}
interactive_image_generation: ${{ steps.filter.outputs.interactive_image_generation }}
mnist: ${{ steps.filter.outputs.mnist }}
toxicity_detection: ${{ steps.filter.outputs.toxicity_detection }}
abstractions: ${{ steps.filter.outputs.abstractions }}
data_model: ${{ steps.filter.outputs.data_model }}
operations: ${{ steps.filter.outputs.operations }}
tools: ${{ steps.filter.outputs.tools }}
mephisto-core: ${{ steps.filter.outputs.mephisto-core }}
mephisto-addons: ${{ steps.filter.outputs.mephisto-addons }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 2
# Workaround to avoid Post Use step failures. See: https://github.com/actions/setup-node/issues/317
- run: mkdir -p /home/runner/work/Mephisto/Mephisto/.yarn/cache
continue-on-error: true
- uses: actions/setup-node@v4
with:
cache: "yarn"
node-version: 16
- uses: dorny/paths-filter@v3
id: filter
with:
filters: |
video_annotator_demo:
- 'examples/video_annotator_demo/**'
form_composer_demo:
- 'examples/form_composer_demo/**'
simple_static_task:
- 'examples/simple_static_task/**'
static_react_task:
- 'examples/static_react_task/**'
static_react_task_with_worker_opinion:
- 'examples/static_react_task_with_worker_opinion/**'
elementary_remote_procedure:
- 'examples/remote_procedure/elementary_remote_procedure/**'
interactive_image_generation:
- 'examples/remote_procedure/interactive_image_generation/**'
mnist:
- 'examples/remote_procedure/mnist/**'
toxicity_detection:
- 'examples/remote_procedure/toxicity_detection/**'
abstractions:
- 'mephisto/abstractions/**'
data_model:
- 'mephisto/data_model/**'
operations:
- 'mephisto/operations/**'
tools:
- 'mephisto/tools/**'
mephisto-core:
- 'packages/mephisto-core/src/**'
mephisto-addons:
- 'packages/mephisto-addons/src/**'
# TODO: Add tests and enable
# form_composer_demo:
# needs: changes
# if: ${{ (needs.changes.outputs.form_composer_demo == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.mephisto-addons == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
# runs-on: ubuntu-latest
# steps:
# - name: 🔀 Checking out repo
# uses: actions/checkout@v2
#
# - name: 🐍 Installing python
# uses: actions/setup-python@v5
# with:
# python-version: 3.8
#
# - name: 🪨 Setup Node
# uses: actions/setup-node@v4
# with:
# node-version: 16.16.0
#
# - name: 🤖 Install Mephisto
# run: pip install -e .
#
# - name: 🖋 Create data directory
# run: mkdir -p ~/mephisto/data
#
# - name: 📂 Set the data directory
# run: mephisto config core.main_data_directory ~/mephisto/data
#
# - name: 📦 Setting up mephisto-core package
# run: |
# cd packages/mephisto-core
# yarn install
# yarn build
# npm link
#
# - name: 📦 Setting up mephisto-addons package
# run: |
# cd packages/mephisto-addons
# yarn install
# yarn build
# npm link
#
# - name: ⌛️ Running cypress tests
# uses: cypress-io/github-action@v6.7.1
# with:
# build: npm i -D cypress@11
# install: false
# browser: chrome
# project: ./examples/form_composer_demo/webapp
# config-file: ./cypress.config.js
# start: python examples/form_composer_demo/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
# wait-on: "http://localhost:3000/?worker_id=x&=1"
# headless: true
#
# video_annotator_demo:
# needs: changes
# if: ${{ (needs.changes.outputs.video_annotator_demo == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.mephisto-addons == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
# runs-on: ubuntu-latest
# steps:
# - name: 🔀 Checking out repo
# uses: actions/checkout@v2
#
# - name: 🐍 Installing python
# uses: actions/setup-python@v5
# with:
# python-version: 3.8
#
# - name: 🪨 Setup Node
# uses: actions/setup-node@v4
# with:
# node-version: 16.16.0
#
# - name: 🤖 Install Mephisto
# run: pip install -e .
#
# - name: 🖋 Create data directory
# run: mkdir -p ~/mephisto/data
#
# - name: 📂 Set the data directory
# run: mephisto config core.main_data_directory ~/mephisto/data
#
# - name: 📦 Setting up mephisto-core package
# run: |
# cd packages/mephisto-core
# yarn install
# yarn build
# npm link
#
# - name: 📦 Setting up mephisto-addons package
# run: |
# cd packages/mephisto-addons
# yarn install
# yarn build
# npm link
#
# - name: ⌛️ Running cypress tests
# uses: cypress-io/github-action@v6.7.1
# with:
# build: npm i -D cypress@11
# install: false
# browser: chrome
# project: ./examples/video_annotator_demo/webapp
# config-file: ./cypress.config.js
# start: python examples/video_annotator_demo/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
# wait-on: "http://localhost:3000/?worker_id=x&id=1"
# headless: true
# Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/881
simple_static_task:
needs: changes
if: ${{ (needs.changes.outputs.simple_static_task == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
runs-on: ubuntu-latest
steps:
- name: 🔀 Checking out repo
uses: actions/checkout@v2
- name: 🐍 Installing python
uses: actions/setup-python@v5
with:
python-version: 3.8
- name: 🪨 Setup Node
uses: actions/setup-node@v4
with:
node-version: 16.16.0
- name: 🤖 Install Mephisto
run: pip install -e .
- name: 🖋 Create data directory
run: mkdir -p ~/mephisto/data
- name: 📂 Set the data directory
run: mephisto config core.main_data_directory ~/mephisto/data
- name: 🚚 Create In-House provider
run: mephisto register inhouse
- name: 📦 Setting up mephisto-core package
run: |
cd packages/mephisto-core
yarn install
yarn build
npm link
- name: ⌛️ Running cypress tests
uses: cypress-io/github-action@v6.7.1
with:
build: npm i -D cypress@11
install: false
browser: chrome
project: ./mephisto/abstractions/blueprints/static_html_task/source
config-file: ./cypress.config.js
start: python ./examples/simple_static_task/run_task__local__inhouse.py
wait-on: "http://localhost:3000/?worker_id=x&id=1"
headless: true
# Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/795
static_react_task:
needs: changes
if: ${{ (needs.changes.outputs.static_react_task == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
runs-on: ubuntu-latest
steps:
- name: 🔀 Checking out repo
uses: actions/checkout@v2
- name: 🐍 Installing python
uses: actions/setup-python@v5
with:
python-version: 3.8
- name: 🪨 Setup Node
uses: actions/setup-node@v4
with:
node-version: 16.16.0
- name: 🤖 Install Mephisto
run: pip install -e .
- name: 🖋 Create data directory
run: mkdir -p ~/mephisto/data
- name: 📂 Set the data directory
run: mephisto config core.main_data_directory ~/mephisto/data
- name: 🚚 Create In-House provider
run: mephisto register inhouse
- name: 📦 Setting up mephisto-core package
run: |
cd packages/mephisto-core
yarn install
yarn build
npm link
- name: ⌛️ Running cypress tests
uses: cypress-io/github-action@v6.7.1
with:
build: npm i -D cypress@11
install: false
browser: chrome
project: ./examples/static_react_task/webapp
config-file: ./cypress.config.js
start: python examples/static_react_task/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
wait-on: "http://localhost:3000/?worker_id=x&id=1"
headless: true
# Learn more about the remote_procedure_tests here: https://github.com/facebookresearch/Mephisto/pull/800
remote_procedure_elementary_remote_procedure:
needs: changes
if: ${{ (needs.changes.outputs.elementary_remote_procedure == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
runs-on: ubuntu-latest
steps:
- name: 🔀 Checking out repo
uses: actions/checkout@v2
- name: 🐍 Installing python
uses: actions/setup-python@v5
with:
python-version: 3.8
- name: 🪨 Setup Node
uses: actions/setup-node@v4
with:
node-version: 16.16.0
- name: 🤖 Install Mephisto
run: pip install -e .
- name: 🖋 Create data directory
run: mkdir -p ~/mephisto/data
- name: 📂 Set the data directory
run: mephisto config core.main_data_directory ~/mephisto/data
- name: 🚚 Create In-House provider
run: mephisto register inhouse
- name: 📦 Setting up mephisto-core package
run: |
cd packages/mephisto-core
yarn install
yarn build
npm link
- name: ⌛️ Running cypress tests
uses: cypress-io/github-action@v6.7.1
with:
build: npm i -D cypress@11
install: false
browser: chrome
project: ./examples/remote_procedure/elementary_remote_procedure/webapp
config-file: ./cypress.config.js
start: python examples/remote_procedure/elementary_remote_procedure/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
wait-on: "http://localhost:3000/?worker_id=x&id=1"
headless: true
# TODO: Add tests and enable
# remote_procedure_interactive_image_generation:
# needs: changes
# if: ${{ (needs.changes.outputs.interactive_image_generation == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.mephisto-addons == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
# runs-on: ubuntu-latest
# steps:
# - name: 🔀 Checking out repo
# uses: actions/checkout@v2
#
# - name: 🐍 Installing python
# uses: actions/setup-python@v5
# with:
# python-version: 3.8
#
# - name: 🪨 Setup Node
# uses: actions/setup-node@v4
# with:
# node-version: 16.16.0
#
# - name: 🤖 Install Mephisto
# run: pip install -e .
#
# - name: 🖋 Create data directory
# run: mkdir -p ~/mephisto/data
#
# - name: 📂 Set the data directory
# run: mephisto config core.main_data_directory ~/mephisto/data
#
# - name: 📦 Setting up mephisto-core package
# run: |
# cd packages/mephisto-core
# yarn install
# yarn build
# npm link
#
# - name: 📦 Setting up mephisto-addons package
# run: |
# cd packages/mephisto-addons
# yarn install
# yarn build
# npm link
#
# - name: ⌛️ Running cypress tests
# uses: cypress-io/github-action@v6.7.1
# with:
# build: npm i -D cypress@11
# install: false
# browser: chrome
# project: ./examples/remote_procedure/interactive_image_generation/webapp
# config-file: ./cypress.config.js
# start: python examples/remote_procedure/interactive_image_generation/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
# wait-on: "http://localhost:3000/?worker_id=x&id=1"
# headless: true
remote_procedure_mnist:
needs: changes
if: ${{ (needs.changes.outputs.mnist == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
runs-on: ubuntu-latest
steps:
- name: 🔀 Checking out repo
uses: actions/checkout@v2
- name: 🐍 Installing python
uses: actions/setup-python@v5
with:
python-version: 3.8
- name: 🪨 Setup Node
uses: actions/setup-node@v4
with:
node-version: 16.16.0
- name: 🤖 Install Mephisto
run: |
pip install -e .
pip install torch pillow numpy
- name: 🖋 Create data directory
run: mkdir -p ~/mephisto/data
- name: 📂 Set the data directory
run: mephisto config core.main_data_directory ~/mephisto/data
- name: 🚚 Create In-House provider
run: mephisto register inhouse
- name: 📦 Setting up mephisto-core package
run: |
cd packages/mephisto-core
yarn install
yarn build
npm link
- name: ⌛️ Running cypress tests
uses: cypress-io/github-action@v6.7.1
with:
build: npm i -D cypress@11
install: false
browser: chrome
project: ./examples/remote_procedure/mnist/webapp
config-file: ./cypress.config.js
start: python examples/remote_procedure/mnist/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
wait-on: "http://localhost:3000/?worker_id=x&id=1"
headless: true
remote_procedure_toxicity_detection:
needs: changes
if: ${{ (needs.changes.outputs.toxicity_detection == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
runs-on: ubuntu-latest
steps:
- name: 🔀 Checking out repo
uses: actions/checkout@v2
- name: 🐍 Installing python
uses: actions/setup-python@v5
with:
python-version: 3.8
- name: 🪨 Setup Node
uses: actions/setup-node@v4
with:
node-version: 16.16.0
- name: 🤖 Install Mephisto
run: |
pip install -e .
pip install detoxify
- name: 🖋 Create data directory
run: mkdir -p ~/mephisto/data
- name: 📂 Set the data directory
run: mephisto config core.main_data_directory ~/mephisto/data
- name: 🚚 Create In-House provider
run: mephisto register inhouse
- name: 📦 Setting up mephisto-core package
run: |
cd packages/mephisto-core
yarn install
yarn build
npm link
- name: ⌛️ Running cypress tests
uses: cypress-io/github-action@v6.7.1
with:
build: npm i -D cypress@11
install: false
browser: chrome
project: ./examples/remote_procedure/toxicity_detection/webapp
config-file: ./cypress.config.js
start: python examples/remote_procedure/toxicity_detection/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
wait-on: "http://localhost:3000/?worker_id=x&id=1"
headless: true
# Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/833
static_react_task_with_worker_opinion:
needs: changes
if: ${{ (needs.changes.outputs.static_react_task_with_worker_opinion == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.mephisto-addons == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
runs-on: ubuntu-latest
steps:
- name: 🔀 Checking out repo
uses: actions/checkout@v2
- name: 🐍 Installing python
uses: actions/setup-python@v5
with:
python-version: 3.8
- name: 🪨 Setup Node
uses: actions/setup-node@v4
with:
node-version: 16.16.0
- name: 🤖 Install Mephisto
run: |
pip install -e .
yes | pip uninstall detoxify
- name: 🖋 Create data directory
run: mkdir -p ~/mephisto/data
- name: 📂 Set the data directory
run: mephisto config core.main_data_directory ~/mephisto/data
- name: 🚚 Create In-House provider
run: mephisto register inhouse
- name: 📦 Setting up mephisto-core package
run: |
cd packages/mephisto-core
yarn install
yarn build
npm link
- name: 📦 Setting up mephisto-addons package
run: |
cd packages/mephisto-addons
yarn install
yarn build
npm link
- name: ⌛️ Running pre-submission cypress tests
uses: cypress-io/github-action@v6.7.1
with:
build: npm i -D cypress@11
install: false
browser: chrome
project: ./examples/static_react_task_with_worker_opinion/webapp
config-file: ./cypress.config.js
spec: ./examples/static_react_task_with_worker_opinion/webapp/cypress/e2e/pre_submission_tests/*
start: python examples/static_react_task_with_worker_opinion/run_task__local__inhouse.py mephisto.task.force_rebuild=true mephisto.task.post_install_script=link_mephisto_task.sh
wait-on: "http://localhost:3000/?worker_id=x&id=1"
headless: true
- name: 🔪 Killing the web server
run: |
lsof -nPi :3000
lsof -i -P -n | grep LISTEN | grep python | awk '{print $2}'
kill -INT $(lsof -i -P -n | grep LISTEN | grep python | awk '{print $2}')
echo "killed 1"
sleep 0.5
kill -INT $(lsof -i -P -n | grep LISTEN | grep python | awk '{print $2}')
echo "killed 2"
lsof -i -P -n | grep LISTEN | grep node | awk '{print $2}'
kill -INT $(lsof -i -P -n | grep LISTEN | grep node | awk '{print $2}')
sleep 30
echo "30 seconds passed"
- name: 🥛 Expiring units
run: |
cd mephisto/scripts/local_db/gh_actions
python expire_all_units.py
# TODO: Fix tests for WorkerOpinion widget
# - name: 📚 Accepting the first submitted tip, accepting the second submitted tip, and rejecting the last submitted tip
# run: |
# cd mephisto/scripts/local_db
# python review_tips_for_task.py << EOF
# react-static-task-with-tips
# a
# 5
# The tip is very informative
# a
# 0
# r
# EOF
#
# - name: 🔥 Removing the second accepted tip
# run: |
# cd mephisto/scripts/local_db
# python remove_accepted_tip.py << EOF
# react-static-task-with-tips
# n
# y
# EOF
#
# - name: 📖 Reviewing the accepted feedback
# run: |
# cd mephisto/scripts/local_db
# python review_feedback_for_task.py << EOF
# react-static-task-with-tips
# 0
# n
# u
# y
# y
# EOF
# python review_feedback_for_task.py << EOF
# react-static-task-with-tips
# 1
# n
# u
# y
# n
# EOF
- name: ⌛️ Running post-submission cypress tests
uses: cypress-io/github-action@v6.7.1
with:
build: npm i -D cypress@11
install: false
project: ./examples/static_react_task_with_worker_opinion/webapp
config-file: cypress.config.js
spec: ./examples/static_react_task_with_worker_opinion/webapp/cypress/e2e/post_submission_tests/*
start: python examples/static_react_task_with_worker_opinion/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh mephisto.task.force_rebuild=true
wait-on: "http://localhost:3000/?worker_id=x&id=1"
browser: chrome
headless: true