[v0.2.3] Fixing mypy (facebookresearch#1023)

* Unpin habitat-lab requirements * Reorder dependencies * Fix mypy config * remove unnecessary mypy args * Update mypy version * Fix python lint * Fix pre-commit mypy further * fix mypy ignored errors * fixing mypy * fixing mypy * more accurate type:ignores * adding docs/ to exclude mypy * trying to fix python_lint * exlude arguments * exlude arguments * Container needs to be a string Co-authored-by: Aaron Gokaslan <aaronGokaslan@gmail.com> Co-authored-by: vincentpierre <vincentpierre@users.noreply.github.com>
SgtVincent · Dec 1, 2022 · 6d4ebc0 · 6d4ebc0
1 parent 557543a
commit 6d4ebc0
Show file tree

Hide file tree

Showing 45 changed files with 164 additions and 123 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -22,7 +22,7 @@ jobs:
           command: |
               pip install black --progress-bar off
               pip install "isort[pyproject]" numpy --progress-bar off
-              pip install mypy==0.950 --progress-bar off
+              pip install mypy==0.991 types-mock types-Pillow types-tqdm types-PyYAML --progress-bar off
               pip install -r habitat-lab/requirements.txt --progress-bar off
       - run:
           name: run black
@@ -39,10 +39,7 @@ jobs:
           name: run mypy
           command: |
               mypy --version
-              mypy habitat-lab/
-              mypy habitat-baselines/
-              mypy examples/
-              mypy test/
+              mypy --exclude="^docs/|setup.py$"
       - run:
           name: run assert no files in habitat and habitat_baselines
           command: |

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -69,20 +69,20 @@ repos:
         - flake8-simplify==0.18.2
 
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.950
+    rev: v0.991
     hooks:
     -   id: mypy
-        name: mypy-habitat-lab
-        files: habitat-lab
+        # https://github.com/python/mypy/issues/4008
+        exclude: "^docs/|setup.py$"
         additional_dependencies:
          - attrs>=19.1.0
-         - numpy
-    -   id: mypy
-        name: mypy-habitat-baselines
-        files: habitat-baselines
-        additional_dependencies:
-         - attrs>=19.1.0
-         - numpy
+         - hydra-core>=1.2.0
+         - numpy>=1.20.0
+         - omegaconf>=2.2.3
+         - types-mock
+         - types-Pillow
+         - types-tqdm
+         - types-PyYAML
 
 -   repo: https://github.com/kynan/nbstripout
     rev: 0.5.0

diff --git a/examples/interactive_play.py b/examples/interactive_play.py
@@ -49,6 +49,7 @@
 import os.path as osp
 import time
 from collections import defaultdict
+from typing import Any, Dict, List
 
 import magnum as mn
 import numpy as np
@@ -265,7 +266,7 @@ def get_input_vel_ctlr(
         joint_state = [float("%.3f" % x) for x in env._sim.robot.arm_joint_pos]
         logger.info(f"Robot arm joint state: {joint_state}")
 
-    args = {}
+    args: Dict[str, Any] = {}
     if base_action is not None and base_action_name in env.action_space.spaces:
         name = base_action_name
         args = {base_key: base_action}
@@ -390,7 +391,7 @@ def play_env(env, args, config):
     prev_time = time.time()
     all_obs = []
     total_reward = 0
-    all_arm_actions = []
+    all_arm_actions: List[float] = []
     agent_to_control = 0
 
     free_cam = FreeCamHelper()
@@ -509,7 +510,7 @@ def play_env(env, args, config):
             screen.blit(draw_obuse_ob, (0, 0))
             pygame.display.update()
         if args.save_obs:
-            all_obs.append(draw_ob)
+            all_obs.append(draw_ob)  # type: ignore[assignment]
 
         if not args.no_render:
             pygame.event.pump()
@@ -538,8 +539,8 @@ def play_env(env, args, config):
         return
 
     if args.save_obs:
-        all_obs = np.array(all_obs)
-        all_obs = np.transpose(all_obs, (0, 2, 1, 3))
+        all_obs = np.array(all_obs)  # type: ignore[assignment]
+        all_obs = np.transpose(all_obs, (0, 2, 1, 3))  # type: ignore[assignment]
         os.makedirs(SAVE_VIDEO_DIR, exist_ok=True)
         vut.make_video(
             np.expand_dims(all_obs, 1),

diff --git a/examples/tutorials/colabs/Habitat2_Quickstart.ipynb b/examples/tutorials/colabs/Habitat2_Quickstart.ipynb
@@ -73,7 +73,7 @@
     "\n",
     "    import PIL\n",
     "\n",
-    "    importlib.reload(PIL.TiffTags)\n",
+    "    importlib.reload(PIL.TiffTags)  # type:ignore\n",
     "\n",
     "import os\n",
     "\n",
@@ -113,7 +113,9 @@
     "# 'IFD'\", then restart the Colab runtime instance and rerun this cell and the previous cell.\n",
     "import PIL\n",
     "\n",
-    "importlib.reload(PIL.TiffTags)  # To potentially avoid PIL problem"
+    "importlib.reload(\n",
+    "    PIL.TiffTags  # type: ignore[attr-defined]\n",
+    ")  # To potentially avoid PIL problem"
    ]
   },
   {

diff --git a/examples/tutorials/colabs/Habitat_Lab.ipynb b/examples/tutorials/colabs/Habitat_Lab.ipynb
@@ -140,7 +140,7 @@
     "    )\n",
     "\n",
     "    try:\n",
-    "        env.close()\n",
+    "        env.close()  # type: ignore[has-type]\n",
     "    except NameError:\n",
     "        pass\n",
     "    env = habitat.Env(config=config)"

diff --git a/examples/tutorials/colabs/habitat2_gym_tutorial.ipynb b/examples/tutorials/colabs/habitat2_gym_tutorial.ipynb
@@ -43,7 +43,7 @@
     "\n",
     "    import PIL\n",
     "\n",
-    "    importlib.reload(PIL.TiffTags)\n",
+    "    importlib.reload(PIL.TiffTags)  # type: ignore[attr-defined]\n",
     "\n",
     "# Video rendering utility.\n",
     "from habitat_sim.utils import viz_utils as vut\n",

diff --git a/examples/tutorials/nb_python/Habitat2_Quickstart.py b/examples/tutorials/nb_python/Habitat2_Quickstart.py
@@ -66,7 +66,7 @@
 
     import PIL
 
-    importlib.reload(PIL.TiffTags)
+    importlib.reload(PIL.TiffTags)  # type:ignore
 
 import os
 
@@ -106,7 +106,9 @@ def insert_render_options(config):
 # 'IFD'", then restart the Colab runtime instance and rerun this cell and the previous cell.
 import PIL
 
-importlib.reload(PIL.TiffTags)  # To potentially avoid PIL problem
+importlib.reload(
+    PIL.TiffTags  # type: ignore[attr-defined]
+)  # To potentially avoid PIL problem
 
 
 # %% [markdown]

diff --git a/examples/tutorials/nb_python/Habitat_Lab.py b/examples/tutorials/nb_python/Habitat_Lab.py
@@ -121,7 +121,7 @@ def display_sample(
     )
 
     try:
-        env.close()
+        env.close()  # type: ignore[has-type]
     except NameError:
         pass
     env = habitat.Env(config=config)

diff --git a/examples/tutorials/nb_python/habitat2_gym_tutorial.py b/examples/tutorials/nb_python/habitat2_gym_tutorial.py
@@ -44,7 +44,7 @@
 
     import PIL
 
-    importlib.reload(PIL.TiffTags)
+    importlib.reload(PIL.TiffTags)  # type: ignore[attr-defined]
 
 # Video rendering utility.
 from habitat_sim.utils import viz_utils as vut

diff --git a/examples/visualization_examples.py b/examples/visualization_examples.py
@@ -22,16 +22,17 @@
 def example_pointnav_draw_target_birdseye_view():
     goal_radius = 0.5
     goal = NavigationGoal(position=[10, 0.25, 10], radius=goal_radius)
-    agent_position = np.array([0, 0.25, 0])
+    agent_position = [0, 0.25, 0]
     agent_rotation = -np.pi / 4
 
     dummy_episode = NavigationEpisode(
         goals=[goal],
         episode_id="dummy_id",
         scene_id="dummy_scene",
         start_position=agent_position,
-        start_rotation=agent_rotation,
+        start_rotation=agent_rotation,  # type: ignore[arg-type]
     )
+    agent_position = np.array(agent_position)
     target_image = maps.pointnav_draw_target_birdseye_view(
         agent_position,
         agent_rotation,
@@ -54,16 +55,17 @@ def example_pointnav_draw_target_birdseye_view_agent_on_border():
             if not np.bitwise_xor(x_edge == 0, y_edge == 0):
                 continue
             ii += 1
-            agent_position = np.array([7.8 * x_edge, 0.25, 7.8 * y_edge])
+            agent_position = [7.8 * x_edge, 0.25, 7.8 * y_edge]
             agent_rotation = np.pi / 2
 
             dummy_episode = NavigationEpisode(
                 goals=[goal],
                 episode_id="dummy_id",
                 scene_id="dummy_scene",
                 start_position=agent_position,
-                start_rotation=agent_rotation,
+                start_rotation=agent_rotation,  # type: ignore[arg-type]
             )
+            agent_position = np.array(agent_position)
             target_image = maps.pointnav_draw_target_birdseye_view(
                 agent_position,
                 agent_rotation,

diff --git a/examples/vln_reference_path_follower_example.py b/examples/vln_reference_path_follower_example.py
@@ -6,6 +6,7 @@
 
 import os
 import shutil
+from typing import List
 
 import numpy as np
 
@@ -77,7 +78,7 @@ def reference_path_example(mode):
                 shutil.rmtree(dirname)
             os.makedirs(dirname)
 
-            images = []
+            images: List[np.ndarray] = []
             steps = 0
             reference_path = env.habitat_env.current_episode.reference_path + [
                 env.habitat_env.current_episode.goals[0].position

diff --git a/habitat-baselines/habitat_baselines/agents/ppo_agents.py b/habitat-baselines/habitat_baselines/agents/ppo_agents.py
@@ -36,7 +36,7 @@ class PPOAgentConfig:
 
 
 def get_default_config() -> DictConfig:
-    return OmegaConf.create(PPOAgentConfig())
+    return OmegaConf.create(PPOAgentConfig())  # type: ignore[call-overload]
 
 
 class PPOAgent(Agent):

diff --git a/habitat-baselines/habitat_baselines/config/default_structured_configs.py b/habitat-baselines/habitat_baselines/config/default_structured_configs.py
@@ -83,7 +83,7 @@ class ActionDistributionConfig(HabitatBaselinesBaseConfig):
 
 @dataclass
 class ObsTransformConfig(HabitatBaselinesBaseConfig):
-    pass
+    type: str = MISSING
 
 
 @dataclass
@@ -92,7 +92,7 @@ class CenterCropperConfig(ObsTransformConfig):
     height: int = 256
     width: int = 256
     channels_last: bool = True
-    trans_keys: Tuple[str] = (
+    trans_keys: Tuple[str, ...] = (
         "rgb",
         "depth",
         "semantic",
@@ -112,7 +112,7 @@ class ResizeShortestEdgeConfig(ObsTransformConfig):
     type: str = "ResizeShortestEdge"
     size: int = 256
     channels_last: bool = True
-    trans_keys: Tuple[str] = (
+    trans_keys: Tuple[str, ...] = (
         "rgb",
         "depth",
         "semantic",
@@ -159,7 +159,7 @@ class Cube2FishConfig(ObsTransformConfig):
     height: int = 256
     width: int = 256
     fov: int = 180
-    params: Tuple[float] = (0.2, 0.2, 0.2)
+    params: Tuple[float, ...] = (0.2, 0.2, 0.2)
     sensor_uuids: List[str] = field(
         default_factory=lambda: [
             "BACK",

diff --git a/habitat-baselines/habitat_baselines/il/data/data.py b/habitat-baselines/habitat_baselines/il/data/data.py
@@ -3,7 +3,7 @@
 # LICENSE file in the root directory of this source tree.
 
 import os
-from typing import Callable, List, Tuple
+from typing import Any, Callable, Dict, List, Tuple
 
 import cv2
 import numpy as np
@@ -139,7 +139,7 @@ def __init__(
     def group_by_keys_(
         self,
         data,
-        keys: Callable[[str], Tuple[str]] = base_plus_ext,
+        keys: Callable[[str], Tuple[str, ...]] = base_plus_ext,
         lcase: bool = True,
         suffixes=None,
     ):
@@ -148,7 +148,7 @@ def group_by_keys_(
         keys: function that splits the key into key and extension (base_plus_ext)
         lcase: convert suffixes to lower case (Default value = True)
         """
-        current_sample = {}
+        current_sample: Dict[str, Any] = {}
         for fname, value in data:
             prefix, suffix = keys(fname)
             if prefix is None:
@@ -228,10 +228,10 @@ def save_frame_queue(
                 pos.position, pos.rotation
             )
             img = observation["rgb"]
-            idx = "{0:0=3d}".format(idx)
+            str_idx = "{0:0=3d}".format(idx)
             episode_id = "{0:0=4d}".format(int(episode_id))
             new_path = os.path.join(
-                self.frame_dataset_path, "{}.{}".format(episode_id, idx)
+                self.frame_dataset_path, "{}.{}".format(episode_id, str_idx)
             )
             cv2.imwrite(new_path + ".jpg", img[..., ::-1])
 

diff --git a/habitat-baselines/habitat_baselines/il/data/nav_data.py b/habitat-baselines/habitat_baselines/il/data/nav_data.py
@@ -3,7 +3,16 @@
 # LICENSE file in the root directory of this source tree.
 
 import os
-from typing import TYPE_CHECKING, Callable, Dict, Generator, List, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    List,
+    Tuple,
+    Union,
+)
 
 import numpy as np
 import torch
@@ -28,6 +37,8 @@
 if TYPE_CHECKING:
     from omegaconf import DictConfig
 
+    from habitat.task.nav import NavigationEpisode
+
 cv2 = try_cv2_import()
 
 
@@ -50,7 +61,7 @@ def __init__(
         """
         self.config = config.habitat
         self.env = env
-        self.episodes = self.env._dataset.episodes  # type:ignore
+        self.episodes: List[NavigationEpisode] = self.env._dataset.episodes
         self.max_controller_actions = max_controller_actions
         self.device = device
         self.sim = self.env.sim
@@ -315,7 +326,7 @@ def preprocess_actions(self) -> None:
     def group_by_keys_(
         self,
         data: Generator,
-        keys: Callable[[str], Tuple[str]] = base_plus_ext,
+        keys: Callable[[str], Tuple[str, ...]] = base_plus_ext,
         lcase: bool = True,
         suffixes=None,
     ):
@@ -324,7 +335,7 @@ def group_by_keys_(
         keys: function that splits the key into key and extension (base_plus_ext)
         lcase: convert suffixes to lower case (Default value = True)
         """
-        current_sample = {}
+        current_sample: Dict[str, Any] = {}
         for fname, value in data:
             prefix, suffix = keys(fname)
             if prefix is None:
@@ -403,10 +414,10 @@ def save_frame_queue(
                 pos.position, pos.rotation
             )
             img = observation["rgb"]
-            idx = "{0:0=3d}".format(idx)
+            str_idx = "{0:0=3d}".format(idx)
             episode_id = "{0:0=4d}".format(int(episode_id))
             new_path = os.path.join(
-                self.frame_dataset_path, "{}.{}".format(episode_id, idx)
+                self.frame_dataset_path, "{}.{}".format(episode_id, str_idx)
             )
             cv2.imwrite(new_path + ".jpg", img[..., ::-1])