[circle] Fixing broken unit tests (#4343)

facebookresearch · Mar 2, 2022 · 5322cd4 · 5322cd4
1 parent 89b2c02
commit 5322cd4
Show file tree

Hide file tree

Showing 13 changed files with 91 additions and 94 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -21,19 +21,33 @@ executors:
   osx_cpu38:
     macos:
       # https://circleci.com/docs/2.0/testing-ios/
-      xcode: "11.3.1"
+      xcode: "12.5.1"
     environment:
       PYTHON: 3.8.0
       PYTHONUNBUFFERED: 1
       HOMEBREW_NO_AUTO_UPDATE: 1
+    resource_class: macos.x86.medium.gen2
 
-  gpu:
+  gpu_small:
     environment:
-      CUDA_VERSION: "10.2"
+      CUDA_VERSION: "11.2"
       PYTHONUNBUFFERED: 1
+      CUDA_HOME: /usr/local/cuda-11.2
     machine:
-      image: ubuntu-1604:201903-01
-    resource_class: gpu.medium # tesla m60
+       image: ubuntu-2004-cuda-11.2:202103-01
+    resource_class: gpu.nvidia.small.multi
+
+  gpu_medium:
+    environment:
+      CUDA_VERSION: "11.2"
+      PYTHONUNBUFFERED: 1
+      CUDA_HOME: /usr/local/cuda-11.2
+    machine:
+       image: ubuntu-2004-cuda-11.2:202103-01
+    resource_class: gpu.nvidia.medium.multi
+
+
+
 
 # -------------------------------------------------------------------------------------
 # reusable commands
@@ -85,38 +99,19 @@ commands:
             python setup.py develop
             python -c "import nltk; nltk.download('punkt')"
 
-  installtorchgpu17:
-    description: Install torch GPU and dependencies
-    steps:
-      - run:
-          name: Install torch GPU and dependencies
-          command: |
-            python -m pip install --progress-bar off torch==1.7.1+cu101 torchvision==0.8.2+cu101 -f https://download.pytorch.org/whl/torch_stable.html
-            python -m pip install --progress-bar off 'fairscale~=0.3.0'
-            python -m pip install --progress-bar off 'torchtext==0.7.0'
-            python -m pip install --progress-bar off pytorch-pretrained-bert
-            python -m pip install --progress-bar off 'transformers==4.3.3'
-            python -m pip install --progress-bar off 'fairseq==0.10.0'
-            python -m pip install --progress-bar off 'faiss-gpu==1.7.0'
-            python -m pip uninstall dataclasses -y
-            # This pre-Python-3.7 package will break use of Python-3.7-style dataclasses
-            python -c 'import torch; print("Torch version:", torch.__version__)'
-            python -m torch.utils.collect_env
-
-  installtorchgpu18:
+  installtorchgpu:
     description: Install torch GPU and dependencies
     steps:
       - run:
           name: Install torch GPU and dependencies
           command: |
-            python -m pip install --progress-bar off 'torch==1.8.1' 'torchvision~=0.9.0' 'torchtext~=0.9.0'
-            python -m pip install --progress-bar off 'fairscale~=0.3.0'
+            python -m pip install --progress-bar off torch==1.10.2+cu113 torchvision==0.11.3+cu113 torchaudio==0.10.2+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html
+            python -m pip install --progress-bar off 'fairscale~=0.4.0'
             python -m pip install --progress-bar off pytorch-pretrained-bert
             python -m pip install --progress-bar off 'transformers==4.3.3'
             python -m pip install --progress-bar off 'fairseq==0.10.0'
             python -m pip install --progress-bar off 'faiss-gpu==1.7.0'
             python -m pip uninstall dataclasses -y
-            # This pre-Python-3.7 package will break use of Python-3.7-style dataclasses
             python -c 'import torch; print("Torch version:", torch.__version__)'
             python -m torch.utils.collect_env
             python -c 'import torch; print("Torch version:", torch.__version__)'
@@ -128,7 +123,8 @@ commands:
       - run:
           name: Install torch CPU and dependencies
           command: |
-            python -m pip install --progress-bar off torch==1.7.1
+            python -m pip install --progress-bar off 'transformers==4.3.3'
+            python -m pip install --progress-bar off 'torch==1.10.2'
             python -c 'import torch; print("Torch version:", torch.__version__)'
             python -m torch.utils.collect_env
 
@@ -138,7 +134,8 @@ commands:
       - run:
           name: Install torch CPU and dependencies
           command: |
-            python -m pip install --progress-bar off torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html
+            python -m pip install --progress-bar off 'torch==1.10.2+cpu' 'torchvision==0.11.3+cpu' 'torchaudio==0.10.2+cpu' -f https://download.pytorch.org/whl/torch_stable.html
+            python -m pip install --progress-bar off 'transformers==4.3.3'
             python -c 'import torch; print("Torch version:", torch.__version__)'
             python -m torch.utils.collect_env
 
@@ -150,10 +147,10 @@ commands:
           command: |
             if (! python -c 'import maskrcnn_benchmark')
             then
-                python -m pip install opencv-python==4.2.0.34
+                python -m pip install yacs 'opencv-python~=4.3.0.00'
                 git clone https://gitlab.com/vedanuj/vqa-maskrcnn-benchmark.git maskbench
                 cd maskbench; git checkout 4c168a637f45dc69efed384c00a7f916f57b25b8 -b stable
-                python setup.py develop; cd -
+                python setup.py install; cd -
             fi
 
   installcrowdsourcingdeps:
@@ -177,13 +174,9 @@ commands:
           name: Setup CUDA
           working_directory: ~/
           command: |
-            # download and install nvidia drivers, cuda, etc
-            wget --quiet --no-clobber -P ~/nvidia-downloads 'https://s3.amazonaws.com/ossci-linux/nvidia_driver/NVIDIA-Linux-x86_64-430.40.run'
-            time sudo /bin/bash ~/nvidia-downloads/NVIDIA-Linux-x86_64-430.40.run --no-drm -q --ui=none
-            echo "Done installing NVIDIA drivers."
             pyenv versions
             nvidia-smi
-            pyenv global 3.7.0
+            pyenv global 3.9.2
 
   findtests:
     description: Find tests to run
@@ -195,8 +188,9 @@ commands:
           working_directory: ~/ParlAI
           name: Find tests to run
           command: |
+            set +o pipefail
             mkdir -p ~/ParlAI/data/models
-            python -m pytest -m << parameters.marker >> --collect-only | grep '<'  | sed "s/^ *//" > teststorun.txt
+            python -m pytest -m << parameters.marker >> --collect-only | grep '<'  | sed "s/^ *//" | grep -v ':'> teststorun.txt
             cat teststorun.txt
 
   runtests:
@@ -228,26 +222,26 @@ commands:
             - setupcuda
       - fixgit
       - restore_cache:
-          key: deps-2021222-<< parameters.cachename >>-{{ checksum "requirements.txt" }}
+          key: deps-20220227-<< parameters.cachename >>-{{ checksum "requirements.txt" }}
       - setup
       - installdeps
       - << parameters.more_installs >>
       - save_cache:
-          key: deps-2021222-<< parameters.cachename >>-{{ checksum "requirements.txt" }}
+          key: deps-20220227-<< parameters.cachename >>-{{ checksum "requirements.txt" }}
           paths:
             - "~/venv/bin"
             - "~/venv/lib"
       - findtests:
           marker: << parameters.marker >>
       - restore_cache:
-          key: data-2021222-<< parameters.cachename >>-{{ checksum "teststorun.txt" }}
+          key: data-20220227-<< parameters.cachename >>-{{ checksum "teststorun.txt" }}
       - run:
           name: Run tests
           no_output_timeout: 60m
           command: |
             coverage run -m pytest -m << parameters.marker >> << parameters.pytest_flags >> --junitxml=test-results/junit.xml
       - save_cache:
-          key: data-2021222-<< parameters.cachename >>-{{ checksum "teststorun.txt" }}
+          key: data-20220227-<< parameters.cachename >>-{{ checksum "teststorun.txt" }}
           paths:
             - "~/ParlAI/data"
       - codecov
@@ -264,12 +258,12 @@ commands:
       - checkout
       - fixgit
       - restore_cache:
-          key: deps-2021222-bw-{{ checksum "requirements.txt" }}
+          key: deps-20220227-bw-{{ checksum "requirements.txt" }}
       - setup
       - installdeps
-      - installtorchgpu17
+      - installtorchgpu
       - save_cache:
-          key: deps-2021222-bw-{{ checksum "requirements.txt" }}
+          key: deps-20220227-bw-{{ checksum "requirements.txt" }}
           paths:
             - "~/venv/bin"
             - "~/venv/lib"
@@ -346,38 +340,26 @@ jobs:
           marker: unit
 
   unittests_gpu18:
-    executor: gpu
+    executor: gpu_small
     working_directory: ~/ParlAI
     parallelism: 8
     steps:
       - runtests:
           more_installs:
-            - installtorchgpu18
+            - installtorchgpu
             - installdetectrondeps
           install_cuda: true
           cachename: gpu18
           marker: unit
 
-  unittests_gpu17:
-    executor: gpu
-    working_directory: ~/ParlAI
-    parallelism: 8
-    steps:
-      - runtests:
-          more_installs:
-            - installtorchgpu17
-          install_cuda: true
-          cachename: gpu17
-          marker: unit
-
   long_gpu_tests:
-    executor: gpu
+    executor: gpu_medium
     working_directory: ~/ParlAI
     parallelism: 8
     steps:
       - runtests:
           more_installs:
-            - installtorchgpu17
+            - installtorchgpu
             - installdetectrondeps
           install_cuda: true
           cachename: nightly
@@ -393,7 +375,7 @@ jobs:
           cachename: crowdsourcing
           marker: crowdsourcing
           more_installs:
-            - installtorchgpu17
+            - installtorchgpu
             - installcrowdsourcingdeps
 
   teacher_tests:
@@ -402,6 +384,8 @@ jobs:
     parallelism: 16
     steps:
       - runtests:
+          more_installs:
+            - installtorchcpu
           cachename: teacher
           marker: teacher
           pytest_flags: -v -s
@@ -446,16 +430,13 @@ workflows:
   commit:
     jobs:
       - cleaninstall_38
-      - unittests_gpu17:
-          requires:
-            - unittests_38
       - unittests_gpu18:
           requires:
             - unittests_38
-      - unittests_38
       - unittests_osx:
           requires:
             - unittests_38
+      - unittests_38
       - long_gpu_tests:
           requires:
             - unittests_38

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -20,7 +20,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7.x
+          python-version: 3.8.x
           architecture: x64
       - name: Fetch ParlAI
         uses: actions/checkout@v2
@@ -58,7 +58,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.7.x
+          python-version: 3.8.x
           architecture: x64
       - name: Fetch ParlAI
         uses: actions/checkout@v2

diff --git a/parlai/agents/transformer/transformer.py b/parlai/agents/transformer/transformer.py
@@ -13,8 +13,9 @@
 from parlai.core.torch_classifier_agent import TorchClassifierAgent
 from parlai.core.torch_ranker_agent import TorchRankerAgent
 from parlai.core.torch_generator_agent import TorchGeneratorAgent
-from parlai.utils.misc import recursive_getattr
+from parlai.utils.misc import recursive_getattr, warn_once
 from parlai.utils.logging import logging
+from parlai.utils.fsdp import should_use_fsdp
 
 from .modules import (
     TransformerMemNetModel,
@@ -25,6 +26,21 @@
 import torch
 
 
+def _check_positional_embeddings(opt):
+    """
+    Checks positional embedding compatibility with FSDP.
+    """
+    if not opt.get('learn_positional_embeddings') and should_use_fsdp(opt):
+        # note: we're doing on-the-fly setting here, abusing pass-by-reference
+        # this only works because we're calling this from build_model, which is
+        # only done in the original instantiation of an agent.
+        opt['learn_positional_embeddings'] = True
+        warn_once(
+            "Using --ddp_backend zeroX requires --learn-positional-embeddings "
+            "true. Forcing this to be true."
+        )
+
+
 def add_common_cmdline_args(parser):
     """
     Add common command line args.
@@ -249,6 +265,7 @@ def build_model(self, states=None):
         """
         Build and return model.
         """
+        _check_positional_embeddings(self.opt)
         model = TransformerMemNetModel(self.opt, self.dict)
         if self.opt['embedding_type'] != 'random':
             self._copy_embeddings(model.embeddings.weight, self.opt['embedding_type'])
@@ -345,6 +362,7 @@ def build_model(self, states=None):
         """
         Build and return model.
         """
+        _check_positional_embeddings(self.opt)
         model = TransformerGeneratorModel(self.opt, self.dict)
         if self.opt['embedding_type'] != 'random':
             self._copy_embeddings(
@@ -405,6 +423,7 @@ def add_cmdline_args(
         return parser
 
     def build_model(self):
+        _check_positional_embeddings(self.opt)
         num_classes = len(self.class_list)
         self.base_model = TransformerMemNetModel(self.opt, self.dict)
         return TransformerLinearWrapper(self.base_model.context_encoder, num_classes)

diff --git a/parlai/core/build_data.py b/parlai/core/build_data.py
@@ -407,7 +407,7 @@ def download_from_google_drive(gd_id, destination):
 
     with get_http_session() as session:
         response = session.get(URL, params={'id': gd_id}, stream=True)
-        token = _get_confirm_token(response)
+        token = _get_confirm_token(response) or 't'
 
         if token:
             response.close()

diff --git a/parlai/utils/fsdp.py b/parlai/utils/fsdp.py
@@ -68,7 +68,7 @@ def maybe_fsdp_wrap(opt):
         mixed_precision=mixed_precision,
         compute_dtype=compute_dtype,
         state_dict_device=torch.device('cpu'),
-        flatten_parameters=True,
+        flatten_parameters=False,
         process_group=get_dist_group(),
     )
     with fairscale_enable_wrap(wrapper_cls=FSDP, **fsdp_args):

diff --git a/projects/safety_bench/unit_tests/response_to_offensive_language_test.py b/projects/safety_bench/unit_tests/response_to_offensive_language_test.py
@@ -34,8 +34,6 @@
 from nltk.sentiment.vader import SentimentIntensityAnalyzer
 
 
-NLP = spacy.load("en_core_web_sm")
-
 SENTIMENT = "Sentiment analysis (% positive sentiment)"
 NEGATION = "Negation detection (% without negatives)"
 SAFETY = "Safety classifier (% flagged offensive)"
@@ -62,7 +60,7 @@ def check_negative_sentiment(sent_eval, text):
 
 def build_offensive_strings(datapath: str) -> List[str]:
     """
-    Data from harmful affirmations section of https://arxiv.org/pdf/2104.08728.pdf
+    Data from harmful affirmations section of https://arxiv.org/pdf/2104.08728.pdf.
     """
 
     def build():