Fixing some dependency issues due to some pip updates (#325)

* Fixing some dependency issues due to some pip updates

* 'srl' option is removed.

* Try to ignore stubs.

* Ignore torch

* mypy ini not really working.

* Fix more type warnings.
1. Remove stubs and use Pytorch defaults
2. Update pylint to 2.6.0, fix a few more warnings raised by that.

* pylint and flake8

Co-authored-by: hector.liu <hector.liu@petuum.com>

.travis.yml

@@ -11,10 +11,11 @@ env:
   matrix:
     - TORCH_VER="1.5.0" TENSORFLOW_VER="1.15.0"
 install:
-  - pip install --upgrade pip
+  # Installing pip from master source due to https://github.com/pypa/pip/issues/9215
+  - pip install git+https://github.com/pypa/pip.git
   - pip install --progress-bar off torch==$TORCH_VER
   - pip install --progress-bar off tensorflow==$TENSORFLOW_VER
-  - pip install --progress-bar off .[nltk,ner,srl,txtgen,stanza,test,example,ir,wikipedia,spacy,sentiment,allennlp]
+  - pip install --progress-bar off .[nltk,ner,txtgen,stanza,test,example,ir,wikipedia,spacy,sentiment,allennlp]
   - git clone https://github.com/asyml/texar-pytorch.git
   - cd texar-pytorch
   - pip install --progress-bar off .
@@ -25,8 +26,8 @@ install:
   - rm -rf elasticsearch-7.4.2-linux-x86_64.tar.gz
   - elasticsearch-7.4.2/bin/elasticsearch -d
   - pip install faiss-cpu
-  - pip install pylint==2.5.2 flake8==3.8.2
-  - pip install mypy==0.780
+  - pip install pylint==2.6.0 flake8==3.8.2
+  - pip install mypy==0.790
   - pip install pytest==5.1.3
   - pip install coverage codecov
 script:

examples/content_rewriter/model/config_data_e2e_clean.py

@@ -1,5 +1,7 @@
 import os
 
+from typing import Dict, Any
+
 dataset_dir = 'model/e2e_data'  # 'e2ev14_demo'#'e2e_0512_max5'
 
 modes = ['train', 'val', 'test']
@@ -27,7 +29,7 @@
     'test': 1,  # eval_batch_size,
 }
 
-datas = {}
+datas: Dict[str, Any] = {}
 
 
 # pylint: disable=global-statement

examples/content_rewriter/model/copy_net.py

@@ -37,7 +37,7 @@ def with_same_shape(old, new):
         return nest.map_structure(
             with_same_shape,
             self,
-            super(CopyNetWrapperState, self)._replace(**kwargs))
+            super()._replace(**kwargs))
 
 
 class CopyNetWrapper(tf.nn.rnn_cell.RNNCell):
@@ -47,7 +47,7 @@ def __init__(
             coverity_dim=None, coverity_rnn_cell_hparams=None,
             disabled_vocab_size=0, eps=0.,
             reuse=tf.AUTO_REUSE, name=None):
-        super(CopyNetWrapper, self).__init__(name=name)
+        super().__init__(name=name)
 
         with tf.variable_scope("CopyNetWrapper", reuse=reuse):
             self._cell = cell

examples/wiki_parser/wiki_dump_parse.py

@@ -52,7 +52,7 @@ def __init__(self):
         self.article_count: int = 0
 
     def initialize(self, resources: Resources, configs: Config):
-        super(WikiArticleWriter, self).initialize(resources, configs)
+        super().initialize(resources, configs)
         self.article_count = 0
         self.article_index = open(
             os.path.join(self.configs.output_dir, 'article.idx'), 'w')
@@ -77,7 +77,7 @@ def _process(self, input_pack: DataPack):
 
         Returns:
         """
-        super(WikiArticleWriter, self)._process(input_pack)
+        super()._process(input_pack)
 
         out_path = self.sub_output_path(input_pack)
         if self.zip_pack:

forte/data/batchers.py

@@ -136,7 +136,7 @@ def default_configs(cls) -> Dict[str, Any]:
 
 class FixedSizeDataPackBatcher(ProcessingBatcher[DataPack]):
     def initialize(self, config: Config):
-        super(FixedSizeDataPackBatcher, self).initialize(config)
+        super().initialize(config)
         self.batch_size = config.batch_size
         self.batch_is_full = False
 

forte/data/ontology/code_generation_objects.py

@@ -320,7 +320,7 @@ class NonCompositeProperty(Property):
     def __init__(self, import_manager: ImportManager,
                  name: str, type_str: str, description: Optional[str] = None,
                  default_val: Any = None, self_ref: bool = False):
-        super(NonCompositeProperty, self).__init__(
+        super().__init__(
             import_manager, name, type_str, description, default_val)
 
         # Primitive type will use optional in type string, so we add the

forte/data/ontology/core.py

@@ -407,9 +407,10 @@ def __init__(self, parent_entry: ParentEntryType,
     def __setitem__(self, k: KeyType, v: ValueType) -> None:
         try:
             self.__data[k] = v.as_pointer(self.__parent_entry)
-        except AttributeError:
+        except AttributeError as e:
             raise AttributeError(
-                f"Item of the FDict must be of type entry, got {v.__class__}")
+                f"Item of the FDict must be of type entry, "
+                f"got {v.__class__}") from e
 
     def __delitem__(self, k: KeyType) -> None:
         del self.__data[k]

forte/data/ontology/ontology_code_generator.py

@@ -27,7 +27,7 @@
 from distutils import dir_util
 from pathlib import Path
 from types import ModuleType
-from typing import Dict, List, Optional, Tuple, Set, no_type_check, Any, cast
+from typing import Dict, List, Optional, Tuple, Set, no_type_check, Any
 
 import jsonschema
 import typed_ast.ast3 as ast
@@ -470,7 +470,7 @@ def visit_ontology_imports(
         except Exception as exception:
             if type(exception).__name__.split('.')[0] == jsonschema.__name__ \
                     and hasattr(exception, 'message'):
-                raise OntologySpecValidationError(cast(Any, exception).message)
+                raise OntologySpecValidationError() from exception
             raise
 
         return import_path, visited_paths, rec_visited_paths

forte/data/ontology/top.py

@@ -249,7 +249,7 @@ def get_members(self) -> List[Entry]:
 
 class MultiPackGeneric(MultiEntry, Entry):
     def __init__(self, pack: PackType):
-        super(MultiPackGeneric, self).__init__(pack=pack)
+        super().__init__(pack=pack)
 
 
 class MultiPackLink(MultiEntry, BaseLink):

forte/data/ontology/utils.py

@@ -32,7 +32,6 @@
 
 import jsonschema
 
-
 AUTO_GEN_SIGNATURE = '***automatically_generated***'
 
 
@@ -48,15 +47,15 @@ def copytree(src, dst, ignore_pattern_if_file_exists='*', preserve_mode=True,
 
     if not dry_run and not os.path.isdir(src):
         raise DistutilsFileError(
-              "cannot copy tree '%s': not a directory" % src)
+            "cannot copy tree '%s': not a directory" % src)
     try:
         names = os.listdir(src)
     except OSError as e:
         if dry_run:
             names = []
         else:
             raise DistutilsFileError(
-                  "error listing files in '%s': %s" % (src, e.strerror))
+                "error listing files in '%s': %s" % (src, e.strerror)) from e
 
     if not dry_run:
         mkpath(dst, verbose=verbose)
@@ -250,6 +249,7 @@ def is_generated(file_path):
         with open(file_path, 'r') as f:
             lines = f.readlines()
             return len(lines) > 0 and lines[0] == f'# {AUTO_GEN_SIGNATURE}\n'
+
     ext_files = []
     for root, _, files in os.walk(path):
         for file in files:

forte/models/srl/model.py

@@ -83,7 +83,7 @@ def __init__(self, word_vocab: tx.data.Vocab, char_vocab: tx.data.Vocab,
                              num_output_representations=1)
             self._elmo_char_ids_fn = batch_to_ids
         else:
-            self.elmo = None
+            self.elmo = None  # type: ignore
 
         # LSTM
         single_hidden_dim = self._hparams.contextualization_size

forte/models/srl/model_utils.py

@@ -239,13 +239,13 @@ def sum_list(xs: List[torch.Tensor]) -> torch.Tensor:
 
 # pylint: disable=unused-argument,function-redefined
 @overload
-def batch_gather(tensors: torch.Tensor,
-                 index: torch.LongTensor) -> torch.Tensor: ...
+def batch_gather(tensors: List[torch.Tensor],
+                 index: torch.LongTensor) -> List[torch.Tensor]: ...
 
 
 @overload
-def batch_gather(tensors: List[torch.Tensor],
-                 index: torch.LongTensor) -> List[torch.Tensor]: ...
+def batch_gather(tensors: torch.Tensor,
+                 index: torch.LongTensor) -> torch.Tensor: ...
 
 
 def batch_gather(tensors, index):

forte/processors/allennlp_processors.py

@@ -49,7 +49,7 @@ def initialize(self, resources: Resources, configs: Config):
         if configs.tag_formalism not in MODEL2URL:
             raise ProcessorConfigError('Incorrect value for tag_formalism')
         model_url = MODEL2URL[configs.tag_formalism]
-        self.predictor = Predictor.from_path(model_url)
+        self.predictor: Predictor = Predictor.from_path(model_url)
 
         if configs.overwrite_entries:
             logger.warning("`overwrite_entries` is set to True, this means "
@@ -104,7 +104,8 @@ def _process(self, input_pack: DataPack):
         self._process_existing_entries(input_pack)
 
         for sentence in input_pack.get(Sentence):
-            result = self.predictor.predict(sentence=sentence.text)
+            result = self.predictor.predict(  # type: ignore
+                sentence=sentence.text)
 
             if "tokenize" in self.configs.processors:
                 # creating new tokens and dependencies

forte/processors/base/batch_processor.py

@@ -63,9 +63,9 @@ def initialize(self, resources: Resources, configs: Optional[Config]):
             self.batcher.initialize(configs.batcher)
         except AttributeError as e:
             raise ProcessorConfigError(
-                e, "Error in handling batcher config, please provide the "
-                   "check the config to see if you have the key 'batcher'."
-            )
+                "Error in handling batcher config, please provide the "
+                "check the config to see if you have the key 'batcher'."
+            ) from e
 
     @staticmethod
     @abstractmethod

forte/processors/base/writers.py

@@ -92,7 +92,7 @@ def __init__(self):
         self.indent: Optional[int] = None
 
     def initialize(self, resources: Resources, configs: Config):
-        super(JsonPackWriter, self).initialize(resources, configs)
+        super().initialize(resources, configs)
 
         if not configs.output_dir:
             raise NotADirectoryError('Root output directory is not defined '

forte/processors/vocabulary_processor.py

@@ -109,8 +109,8 @@ def get_index(self, instance):
     def get_instance(self, index):
         try:
             return self.instances[index]
-        except IndexError:
-            raise IndexError("unknown index: %d" % index)
+        except IndexError as e:
+            raise IndexError("unknown index: %d" % index) from e
 
     def size(self):
         return len(self.instances)

mypy.ini

@@ -7,11 +7,11 @@ warn_redundant_casts = True
 no_implicit_optional = True
 follow_imports = silent
 ignore_missing_imports = True
-mypy_path = ./, ./stubs/
+mypy_path = ./
 allow_redefinition = True
 
 [mypy-torch]
 follow_imports = skip
 
 [mypy-numpy]
-follow_imports = skip
+follow_imports = skip

setup.py

@@ -39,8 +39,7 @@
     ],
     extras_require={
         'nltk': ['nltk==3.4.5'],
-        'ner': ['torch>=1.1.0', 'torchtext==0.4.0', 'tqdm==4.36.1'],
-        'srl': ['allennlp==1.0.0'],
+        'ner': ['torch>=1.1.0', 'torchtext==0.4.0', 'tqdm>=4.36.1'],
         'sentiment': ['vaderSentiment==3.2.1'],
         'txtgen': ['regex', 'tensorflow'],
         'stanza': ['stanza==1.0.1'],
@@ -49,8 +48,8 @@
         'wikipedia': ['rdflib==4.2.2'],
         'ir': ['faiss-cpu>=1.6.1', 'elasticsearch==7.5.1'],
         'spacy': ['spacy==2.3.0'],
-        'allennlp': ['allennlp==1.0.0', 'allennlp-models==1.0.0',
-                     'torch>=1.5.0,<1.6.0'],
+        'allennlp': ['allennlp==1.1.0', 'allennlp-models==1.1.0',
+                     'torch>=1.5.0'],
         'cliner': ['marisa-trie==0.7.4', 'scipy==1.5.0',
                    'scikit-learn==0.23.1', 'repoze.lru==0.7',
                    'tensorflow-gpu==1.12.0', 'python-crfsuite==0.9.7'],