diff --git a/.circleci/config.yml b/.circleci/config.yml index fc6fb7b5405..2bf3a868eed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,14 +6,21 @@ version: 2.1 executors: standard_cpu38: docker: - - image: circleci/python:3.8.0-buster-node + - image: cimg/python:3.8.0 environment: PYTHONUNBUFFERED: 1 resource_class: xlarge small_cpu38: docker: - - image: circleci/python:3.8.0-buster-node + - image: cimg/python:3.8.0 + environment: + PYTHONUNBUFFERED: 1 + resource_class: medium + + small_cpu38_conda: + docker: + - image: continuumio/miniconda3 environment: PYTHONUNBUFFERED: 1 resource_class: medium @@ -75,6 +82,27 @@ commands: . ~/venv/bin/activate python --version + setup_conda: + description: Sets up the virtual environment + steps: + - run: + name: Sets up the virtual environment + command: | + echo "Started creating the ParlAI conda environment." + conda create --quiet --yes --name conda_parlai python=3.8 + echo "Created the ParlAI conda environment." + conda init bash + source ~/.bashrc + conda deactivate + echo "Activating the ParlAI conda environment." + conda activate conda_parlai + for package in pytorch==1.13.0 torchvision==0.14.0 torchaudio==0.13.0 pytorch-cuda=11.6 + do + echo "Conda installing ${package}." + conda install --quiet --yes "${package}" -c pytorch -c nvidia + done + echo "conda activate conda_parlai" >> $BASH_ENV + codecov: description: Coverage report steps: @@ -91,7 +119,8 @@ commands: - run: name: Installs basic dependencies command: | - for i in $(seq 1 3); do python -m pip install --progress-bar off pip setuptools==62.3.4 && s=0 && break || s=$? && sleep 10; done; (exit $s) + pip install --upgrade pip + for i in $(seq 1 3); do python -m pip install --progress-bar off pip setuptools && s=0 && break || s=$? && sleep 10; done; (exit $s) for i in $(seq 1 3); do python -m pip install --progress-bar off coverage && s=0 && break || s=$? && sleep 10; done; (exit $s) for i in $(seq 1 3); do python -m pip install --progress-bar off codecov && s=0 && break || s=$? && sleep 10; done; (exit $s) mkdir -p ~/ParlAI/test-results @@ -109,7 +138,7 @@ commands: - run: name: Install torch GPU and dependencies command: | - python -m pip install --progress-bar off torch==1.11.0+cu113 torchvision==0.12.0+cu113 torchaudio==0.11.0+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html + python -m pip install --progress-bar off torch==1.13.1 torchvision==0.14.1 torchaudio==0.13.1 python -m pip install --progress-bar off 'fairscale~=0.4.0' python -m pip install --progress-bar off pytorch-pretrained-bert python -m pip install --progress-bar off 'transformers==4.20' @@ -128,7 +157,7 @@ commands: name: Install torch CPU and dependencies command: | python -m pip install --progress-bar off 'transformers==4.20' - python -m pip install --progress-bar off 'torch==1.11.0' + python -m pip install --progress-bar off 'torch==1.13.1' python -c 'import torch; print("Torch version:", torch.__version__)' python -m torch.utils.collect_env @@ -138,7 +167,7 @@ commands: - run: name: Install torch CPU and dependencies command: | - python -m pip install --progress-bar off 'torch==1.11.0+cpu' 'torchvision==0.12.0+cpu' 'torchaudio==0.11.0+cpu' -f https://download.pytorch.org/whl/torch_stable.html + python -m pip install --progress-bar off 'torch==1.13.1' 'torchvision==0.14.1' 'torchaudio==0.13.1' python -m pip install --progress-bar off 'transformers==4.20' python -c 'import torch; print("Torch version:", torch.__version__)' python -m torch.utils.collect_env @@ -219,26 +248,26 @@ commands: - setupcuda - fixgit - restore_cache: - key: deps-20230209-<< parameters.cachename >>-{{ checksum "requirements.txt" }} + key: deps-20230404-<< parameters.cachename >>-{{ checksum "requirements.txt" }} - setup - installdeps - << parameters.more_installs >> - save_cache: - key: deps-20230209-<< parameters.cachename >>-{{ checksum "requirements.txt" }} + key: deps-20230404-<< parameters.cachename >>-{{ checksum "requirements.txt" }} paths: - "~/venv/bin" - "~/venv/lib" - findtests: marker: << parameters.marker >> - restore_cache: - key: data-20230209-<< parameters.cachename >>-{{ checksum "teststorun.txt" }} + key: data-20230404-<< parameters.cachename >>-{{ checksum "teststorun.txt" }} - run: name: Run tests no_output_timeout: 60m command: | coverage run -m pytest -m << parameters.marker >> << parameters.pytest_flags >> --junitxml=test-results/junit.xml - save_cache: - key: data-20230209-<< parameters.cachename >>-{{ checksum "teststorun.txt" }} + key: data-20230404-<< parameters.cachename >>-{{ checksum "teststorun.txt" }} paths: - "~/ParlAI/data" - codecov @@ -255,12 +284,12 @@ commands: - checkout - fixgit - restore_cache: - key: deps-20230209-bw-{{ checksum "requirements.txt" }} + key: deps-20230404-bw-{{ checksum "requirements.txt" }} - setup - installdeps - installtorchgpu - save_cache: - key: deps-20230209-bw-{{ checksum "requirements.txt" }} + key: deps-20230404-bw-{{ checksum "requirements.txt" }} paths: - "~/venv/bin" - "~/venv/lib" @@ -301,7 +330,7 @@ commands: # ------------------------------------------------------------------------------------- jobs: cleaninstall_38: - executor: standard_cpu38 + executor: small_cpu38 working_directory: ~/ParlAI parallelism: 1 steps: @@ -312,7 +341,8 @@ jobs: name: Test installation instructions no_output_timeout: 60m command: | - python -m pip install --progress-bar off pip setuptools==62.3.4 + python -m pip install --progress-bar off --upgrade pip + python -m pip install --progress-bar off pip setuptools python setup.py develop parlai display_data -t integration_tests diff --git a/parlai/crowdsourcing/tasks/dialcrowd/dialcrowd_blueprint.py b/parlai/crowdsourcing/tasks/dialcrowd/dialcrowd_blueprint.py index 1d8c23b344b..819d85cee7b 100644 --- a/parlai/crowdsourcing/tasks/dialcrowd/dialcrowd_blueprint.py +++ b/parlai/crowdsourcing/tasks/dialcrowd/dialcrowd_blueprint.py @@ -8,7 +8,7 @@ import logging import os from dataclasses import dataclass, field -from typing import Any, Dict, Iterable, TYPE_CHECKING +from typing import Any, Dict, List, Union, TYPE_CHECKING from mephisto.operations.registry import register_mephisto_abstraction from mephisto.abstractions.blueprint import SharedTaskState @@ -53,6 +53,8 @@ class DialCrowdStaticBlueprint(StaticReactBlueprint): definitions. """ + _initialization_data_dicts: Union[List[List[Dict[str, Any]]], List[Dict[str, Any]]] + ArgsClass = DialCrowdStaticBlueprintArgs BLUEPRINT_TYPE = STATIC_BLUEPRINT_TYPE @@ -67,7 +69,7 @@ def __init__( f'subtasks_per_unit must be greater than zero but was {self.subtasks_per_unit}' ) - self.raw_data: Iterable[Dict[str, Any]] = self._initialization_data_dicts + self.raw_data = self._initialization_data_dicts # Now chunk the data into groups of grouped_data = [] @@ -77,7 +79,7 @@ def __init__( for i in range(0, len(self._initialization_data_dicts), self.subtasks_per_unit): chunk = self._initialization_data_dicts[i : i + self.subtasks_per_unit] grouped_data.append(chunk) - self._initialization_data_dicts = grouped_data + self._initialization_data_dicts = grouped_data # type: ignore # Last group may have less unless an exact multiple logging.info( f'Grouped data into {len(self._initialization_data_dicts)} tasks with {self.subtasks_per_unit} subtasks each.' diff --git a/parlai/crowdsourcing/tasks/multi_model_chat/__init__.py b/parlai/crowdsourcing/tasks/multi_model_chat/__init__.py new file mode 100644 index 00000000000..240697e3247 --- /dev/null +++ b/parlai/crowdsourcing/tasks/multi_model_chat/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +# Copyright (c) Facebook, Inc. and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. diff --git a/parlai/crowdsourcing/tasks/turn_annotations_static/turn_annotations_blueprint.py b/parlai/crowdsourcing/tasks/turn_annotations_static/turn_annotations_blueprint.py index 389e68d5913..d1b5cb79111 100644 --- a/parlai/crowdsourcing/tasks/turn_annotations_static/turn_annotations_blueprint.py +++ b/parlai/crowdsourcing/tasks/turn_annotations_static/turn_annotations_blueprint.py @@ -10,7 +10,7 @@ import os import random from dataclasses import dataclass, field -from typing import Any, Dict, Iterable, List, Optional, TYPE_CHECKING +from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING import numpy as np from mephisto.operations.registry import register_mephisto_abstraction @@ -115,6 +115,8 @@ class TurnAnnotationsStaticBlueprint(StaticReactBlueprint): definitions. """ + _initialization_data_dicts: Union[List[List[Dict[str, Any]]], List[Dict[str, Any]]] + ArgsClass = TurnAnnotationsStaticBlueprintArgs BLUEPRINT_TYPE = STATIC_BLUEPRINT_TYPE @@ -133,7 +135,7 @@ def __init__( f'subtasks_per_unit must be greater than zero but was {self.subtasks_per_unit}' ) - self.raw_data: Iterable[Dict[str, Any]] = self._initialization_data_dicts + self.raw_data = self._initialization_data_dicts # Load from file if needed specifying which utterances within each # conversation to annotate @@ -180,7 +182,7 @@ def __init__( for i in range(0, len(self._initialization_data_dicts), self.subtasks_per_unit): chunk = self._initialization_data_dicts[i : i + self.subtasks_per_unit] grouped_data.append(chunk) - self._initialization_data_dicts = grouped_data + self._initialization_data_dicts = grouped_data # type: ignore # Last group may have less unless an exact multiple logging.info( f'Grouped data into {len(self._initialization_data_dicts)} tasks with {self.subtasks_per_unit} subtasks each.' @@ -397,7 +399,7 @@ def __init__( # (quality control will always be last subtask) # TODO: I don't think we need to re-chunk this actually; just iterate # over the data and add the quality control task - all_data = [] + all_data: List[Any] = [] for grp in self._initialization_data_dicts: all_data.extend(grp) diff --git a/requirements.txt b/requirements.txt index db16c7ca37f..fee61951d15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,8 @@ gitdb2==2.0.5 GitPython==3.0.3 hydra-core>=1.1.0 ipython==7.31.1 -torch<1.13.0,>=1.4.0 -torchvision<0.14.0,>=0.5.0 +torch==1.13.1 +torchvision==0.14.1 joblib==1.2.0 nltk==3.6.6 omegaconf>=2.1.1 @@ -44,7 +44,7 @@ subword-nmt==0.3.7 tensorboardX<=2.5.0 tokenizers>=0.8.0 tomli>=2.0.0 -torchtext>=0.5.0,<0.14.0 +torchtext==0.14.1 tornado==6.0.4 tqdm~=4.62.1 typing-extensions==3.7.4.3 @@ -59,4 +59,4 @@ ninja~=1.10.2.3 protobuf<=3.20.3, >=3.8.0 contractions~=0.1.72 fsspec~=2022.2.0 -google-api-core<=2.11.0 # Latest 2.10.2 requires latest protobuf +google-api-core<=2.11.0 diff --git a/setup.py b/setup.py index d6e11321fea..a5e252fa761 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ long_description=readme, long_description_content_type='text/markdown', url='http://parl.ai/', - python_requires='>=3.7', + python_requires='>=3.8', packages=find_packages(exclude=('data', 'docs', 'tests', 'parlai_internal*')), install_requires=reqs, include_package_data=True,