From f8d40cd7c9a403652092fb4a718c002a778c6f34 Mon Sep 17 00:00:00 2001 From: Stephen Roller Date: Sat, 17 Apr 2021 23:19:55 -0400 Subject: [PATCH 1/3] [tests] Speed up test_chunkteacher --- parlai/scripts/multiprocessing_train.py | 9 ++- tests/test_chunkteacher.py | 77 +++---------------------- 2 files changed, 16 insertions(+), 70 deletions(-) diff --git a/parlai/scripts/multiprocessing_train.py b/parlai/scripts/multiprocessing_train.py index 302bb416908..e6dad8a29d9 100644 --- a/parlai/scripts/multiprocessing_train.py +++ b/parlai/scripts/multiprocessing_train.py @@ -82,10 +82,15 @@ def setup_args(): class MultiProcessTrain(ParlaiScript): @classmethod def setup_args(cls): - return setup_args() + argparser = setup_args() + argparser.add_argument('--port', type=int, default=None) + return argparser def run(self): - port = random.randint(32000, 48000) + if self.opt['port'] is None: + port = random.randint(32000, 48000) + else: + port = self.opt['port'] return launch_and_train(self.opt, port) diff --git a/tests/test_chunkteacher.py b/tests/test_chunkteacher.py index f98f5fa7009..eb9930b5a1e 100644 --- a/tests/test_chunkteacher.py +++ b/tests/test_chunkteacher.py @@ -8,6 +8,7 @@ Test correctness of ChunkTeacher in a large number of settings. """ +import random from unittest import TestCase import os import parlai.utils.testing as testing_utils @@ -15,13 +16,13 @@ import parlai.scripts.multiprocessing_train as mp_train -class TestNumExamples(TestCase): +class _Abstract(TestCase): BASE_ARGS = { 'model': 'test_agents/counter', 'dict_file': 'zoo:unittest/transformer_generator2/model.dict', 'dict_tokenizer': 'space', 'truncate': 8, - 'num_epochs': 2, + 'max_train_steps': 10, 'datatype': 'train:stream', } @@ -47,14 +48,12 @@ def _run_mp(self, **kwargs): assert test_report['times_seen'] == 1 return valid_report, test_report - # Regular chunk teacher +class TestNumExamples(_Abstract): + # Regular chunk teacher def test_normal_bs1(self): self._run(task='integration_tests:chunky') - def test_normal_bs2(self): - self._run(task='integration_tests:chunky', batchsize=2) - def test_normal_bs3(self): self._run(task='integration_tests:chunky', batchsize=3) @@ -66,14 +65,6 @@ def test_normal_batchsort(self): task='integration_tests:chunky', batchsize=2, dynamic_batching='batchsort' ) - @testing_utils.skipUnlessGPU - def test_mp_normal_bs1(self): - self._run_mp(task='integration_tests:chunky') - - @testing_utils.skipUnlessGPU - def test_mp_normal_bs2(self): - self._run_mp(task='integration_tests:chunky', batchsize=2) - @testing_utils.skipUnlessGPU def test_mp_normal_bs3(self): self._run_mp(task='integration_tests:chunky', batchsize=3) @@ -84,20 +75,12 @@ def test_mp_normal_dynb(self): task='integration_tests:chunky', batchsize=2, dynamic_batching='full' ) - @testing_utils.skipUnlessGPU - def test_mp_normal_batchsort(self): - self._run_mp( - task='integration_tests:chunky', batchsize=2, dynamic_batching='batchsort' - ) +class TestSmallBuffer(_Abstract): # Small buffer - def test_small_buffer_bs1(self): self._run(task='integration_tests:chunky_small_buffer') - def test_small_buffer_bs2(self): - self._run(task='integration_tests:chunky_small_buffer', batchsize=2) - def test_small_buffer_bs3(self): self._run(task='integration_tests:chunky_small_buffer', batchsize=3) @@ -119,10 +102,6 @@ def test_small_buffer_batchsort(self): def test_mp_small_buffer_bs1(self): self._run_mp(task='integration_tests:chunky_small_buffer') - @testing_utils.skipUnlessGPU - def test_mp_small_buffer_bs2(self): - self._run_mp(task='integration_tests:chunky_small_buffer', batchsize=2) - @testing_utils.skipUnlessGPU def test_mp_small_buffer_bs3(self): self._run_mp(task='integration_tests:chunky_small_buffer', batchsize=3) @@ -143,14 +122,9 @@ def test_mp_small_buffer_batchsort(self): dynamic_batching='batchsort', ) - # Slow chunk - - def test_slow_bs1(self): - self._run(task='integration_tests:chunky_slow') - - def test_slow_bs2(self): - self._run(task='integration_tests:chunky_slow', batchsize=2) +class TestSlowChunk(_Abstract): + # Slow chunk def test_slow_bs3(self): self._run(task='integration_tests:chunky_slow', batchsize=3) @@ -159,39 +133,6 @@ def test_slow_dynb(self): task='integration_tests:chunky_slow', batchsize=2, dynamic_batching='full' ) - def test_slow_batchsort(self): - self._run( - task='integration_tests:chunky_slow', - batchsize=2, - dynamic_batching='batchsort', - ) - - @testing_utils.skipUnlessGPU - def test_mp_slow_bs1(self): - self._run_mp(task='integration_tests:chunky_slow') - - @testing_utils.skipUnlessGPU - def test_mp_slow_bs2(self): - self._run_mp(task='integration_tests:chunky_slow', batchsize=2) - - @testing_utils.skipUnlessGPU - def test_mp_slow_bs3(self): - self._run_mp(task='integration_tests:chunky_slow', batchsize=3) - - @testing_utils.skipUnlessGPU - def test_mp_slow_dynb(self): - self._run_mp( - task='integration_tests:chunky_slow', batchsize=2, dynamic_batching='full' - ) - - @testing_utils.skipUnlessGPU - def test_mp_slow_batchsort(self): - self._run_mp( - task='integration_tests:chunky_slow', - batchsize=2, - dynamic_batching='batchsort', - ) - class TestBackgroundPreprocessorNumExamples(TestNumExamples): BASE_ARGS = { @@ -199,7 +140,7 @@ class TestBackgroundPreprocessorNumExamples(TestNumExamples): 'dict_file': 'zoo:unittest/transformer_generator2/model.dict', 'dict_tokenizer': 'space', 'truncate': 8, - 'num_epochs': 2, + 'max_train_steps': 10, 'datatype': 'train:stream', 'num_workers': 4, } From 10af952d9d5fb50aac0d3aac2dde43a8be3aef6b Mon Sep 17 00:00:00 2001 From: Stephen Roller Date: Mon, 19 Apr 2021 14:36:40 -0400 Subject: [PATCH 2/3] Bring back test for jing. --- tests/test_chunkteacher.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_chunkteacher.py b/tests/test_chunkteacher.py index eb9930b5a1e..02739d0d79f 100644 --- a/tests/test_chunkteacher.py +++ b/tests/test_chunkteacher.py @@ -65,6 +65,10 @@ def test_normal_batchsort(self): task='integration_tests:chunky', batchsize=2, dynamic_batching='batchsort' ) + @testing_utils.skipUnlessGPU + def test_mp_normal_bs1(self): + self._run_mp(task='integration_tests:chunky', batchsize=1) + @testing_utils.skipUnlessGPU def test_mp_normal_bs3(self): self._run_mp(task='integration_tests:chunky', batchsize=3) From e451398a1773a658263e67c8eb1b43889559f424 Mon Sep 17 00:00:00 2001 From: Stephen Roller Date: Mon, 19 Apr 2021 14:36:50 -0400 Subject: [PATCH 3/3] Lint. --- tests/test_chunkteacher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_chunkteacher.py b/tests/test_chunkteacher.py index 02739d0d79f..8f85f156aa3 100644 --- a/tests/test_chunkteacher.py +++ b/tests/test_chunkteacher.py @@ -8,7 +8,6 @@ Test correctness of ChunkTeacher in a large number of settings. """ -import random from unittest import TestCase import os import parlai.utils.testing as testing_utils