Skip to content

Commit

Permalink
Add test matrix (#1308)
Browse files Browse the repository at this point in the history
* add simple test matrix

* add expected materics to test matrix

* update torch to 1.8.1 for test matrix

* add overnight flag

Co-authored-by: Jesse Swanson <js11133Wnyu.edu>
jeswan authored Apr 23, 2021

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent ee65662 commit 4d0f6a9
Showing 4 changed files with 45 additions and 5 deletions.
7 changes: 7 additions & 0 deletions conftest.py
Original file line number Diff line number Diff line change
@@ -8,6 +8,8 @@
def pytest_addoption(parser):
parser.addoption("--runslow", action="store_true", default=False, help="run slow tests")
parser.addoption("--rungpu", action="store_true", default=False, help="run gpu tests")
parser.addoption("--runovernight", action="store_true",
default=False, help="run overnight tests")


def pytest_configure(config):
@@ -26,3 +28,8 @@ def pytest_collection_modifyitems(config, items):
for item in items:
if "gpu" in item.keywords:
item.add_marker(skip_gpu)
if not config.getoption("--runovernight"):
skip_overnight = pytest.mark.skip(reason="need --runovernight option to run")
for item in items:
if "overnight" in item.keywords:
item.add_marker(skip_overnight)
4 changes: 2 additions & 2 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
-r requirements-no-torch.txt
torch>=1.5.0
torchvision==0.6.0
torch>=1.8.1
torchvision==0.9.1
4 changes: 2 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
@@ -73,10 +73,10 @@
"scipy == 1.4.1",
"sentencepiece == 0.1.86",
"tokenizers == 0.8.1.rc2",
"torch >= 1.5.0",
"torch >= 1.8.1",
"tqdm == 4.46.0",
"transformers == 3.1.0",
"torchvision == 0.6.0",
"torchvision == 0.9.1",
],
extras_require=extras,
python_requires=">=3.6.0",
35 changes: 34 additions & 1 deletion tests/proj/simple/test_runscript.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,18 @@
import os
import pytest
import torch
import math

import jiant.utils.python.io as py_io
from jiant.proj.simple import runscript as run
import jiant.scripts.download_data.runscript as downloader
import jiant.utils.torch_utils as torch_utils

EXPECTED_AGG_VAL_METRICS = {"bert-base-cased": {"rte": 0.5740072202166066, "commonsenseqa": 0.4258804258804259, "squad_v1": 29.071789929086883},
"roberta-base": {"rte": 0.49458483754512633, "commonsenseqa": 0.23013923013923013, "squad_v1": 48.222444172918955},
"xlm-roberta-base": {"rte": 0.4729241877256318, "commonsenseqa": 0.22686322686322685, "squad_v1": 10.30104037978786}}


@pytest.mark.gpu
@pytest.mark.parametrize("task_name", ["copa"])
@pytest.mark.parametrize("model_type", ["bert-base-cased"])
def test_simple_runscript(tmpdir, task_name, model_type):
@@ -34,6 +38,35 @@ def test_simple_runscript(tmpdir, task_name, model_type):
assert val_metrics["aggregated"] > 0


@pytest.mark.overnight
@pytest.mark.parametrize(("task_name", "train_examples_cap"), [("rte", 1024), ("commonsenseqa", 1024), ("squad_v1", 2048)])
@pytest.mark.parametrize("model_type", ["bert-base-cased", "roberta-base", "xlm-roberta-base"])
def test_simple_runscript(tmpdir, task_name, train_examples_cap, model_type):
RUN_NAME = f"{test_simple_runscript.__name__}_{task_name}_{model_type}"
data_dir = str(tmpdir.mkdir("data"))
exp_dir = str(tmpdir.mkdir("exp"))

torch.use_deterministic_algorithms(True)

downloader.download_data([task_name], data_dir)
args = run.RunConfiguration(
run_name=RUN_NAME,
exp_dir=exp_dir,
data_dir=data_dir,
hf_pretrained_model_name_or_path=model_type,
tasks=task_name,
train_examples_cap=train_examples_cap,
train_batch_size=32,
seed=42,
no_cuda=False,
)
run.run_simple(args)

val_metrics = py_io.read_json(os.path.join(exp_dir, "runs", RUN_NAME, "val_metrics.json"))
assert math.isclose(val_metrics["aggregated"], EXPECTED_AGG_VAL_METRICS[model_type][task_name])
torch.use_deterministic_algorithms(False)


@pytest.mark.gpu
@pytest.mark.parametrize("task_name", ["copa"])
@pytest.mark.parametrize("model_type", ["roberta-large"])

0 comments on commit 4d0f6a9

Please sign in to comment.