Upgrade Python dependencies & use mypy strict mode (#1238)

* Upgrade Python dependencies

* Explicitly pin isort to avoid upstream issue with not skipping gitignore

* Update MkDocs configuration

* Enable mypy strict mode

certmgr/scripts/letsencrypt_cert.py

@@ -73,7 +73,7 @@ def create(self) -> None:
         is_letsencrypt_cert = False
         if self.nginx_cert_dir.is_symlink():
             link_target = os.readlink(self.nginx_cert_dir)
-            if link_target == self.cert_dir:
+            if link_target == str(self.cert_dir):
                 is_letsencrypt_cert = True
 
         # if we do not have a certificate from letsencrypt, then we:

certmgr/scripts/utils.py

@@ -67,7 +67,7 @@ def update_link(src: Path, dest: Path) -> None:
     """
     if dest.is_symlink():
         link_target = os.readlink(dest)
-        if link_target != src:
+        if link_target != str(src):
             dest.unlink()
             dest.symlink_to(src)
     elif not dest.exists():

deploy/bin/clean_repo.py

@@ -11,6 +11,8 @@
         aws configure
 """
 
+from __future__ import annotations
+
 import argparse
 import json
 import re
@@ -56,7 +58,7 @@ def parse_args() -> argparse.Namespace:
     return parser.parse_args()
 
 
-def run_aws_cmd(aws_cmd: List[str], verbose: bool = False) -> subprocess.CompletedProcess:
+def run_aws_cmd(aws_cmd: List[str], verbose: bool = False) -> subprocess.CompletedProcess[str]:
     """Run an AWS command and print the command and results if verbose is set."""
     if verbose:
         print(aws_cmd)

deploy/roles/combine_maintenance/files/aws_backup.py

@@ -16,17 +16,17 @@ def __init__(self, *, bucket: str, profile: str) -> None:
         self.profile = profile
         self.bucket = bucket
 
-    def push(self, src: Path, dest: str) -> subprocess.CompletedProcess:
+    def push(self, src: Path, dest: str) -> subprocess.CompletedProcess[str]:
         """Push a file to the AWS S3 bucket."""
         s3_uri = f"s3://{self.bucket}/{dest}"
         return run_cmd(["aws", "s3", "cp", str(src), s3_uri, "--profile", self.profile])
 
-    def pull(self, src: str, dest: Path) -> subprocess.CompletedProcess:
+    def pull(self, src: str, dest: Path) -> subprocess.CompletedProcess[str]:
         """Push a file to the AWS S3 bucket."""
         s3_uri = f"s3://{self.bucket}/{src}"
         return run_cmd(["aws", "s3", "cp", s3_uri, str(dest), "--profile", self.profile])
 
-    def list(self) -> subprocess.CompletedProcess:
+    def list(self) -> subprocess.CompletedProcess[str]:
         """List the objects in the S3 bucket."""
         return run_cmd(
             ["aws", "s3", "ls", f"s3://{self.bucket}", "--recursive", "--profile", self.profile]

deploy/roles/combine_maintenance/files/combine_app.py

@@ -29,7 +29,7 @@ class CombineApp:
 
     def __init__(self, compose_file_path: Path) -> None:
         """Initialize the CombineApp from the configuration file."""
-        if compose_file_path == "":
+        if str(compose_file_path) == "":
             self.compose_opts = []
         else:
             self.compose_opts = ["-f", str(compose_file_path)]
@@ -45,7 +45,7 @@ def exec(
         *,
         exec_opts: Optional[List[str]] = None,
         check_results: bool = True,
-    ) -> subprocess.CompletedProcess:
+    ) -> subprocess.CompletedProcess[str]:
         """
         Run a docker-compose 'exec' command in a Combine container.
 
@@ -109,11 +109,11 @@ def db_cmd(self, cmd: str) -> Optional[Dict[str, Any]]:
             return result_dict
         return None
 
-    def start(self, services: List[str]) -> subprocess.CompletedProcess:
+    def start(self, services: List[str]) -> subprocess.CompletedProcess[str]:
         """Start the specified combine service(s)."""
         return run_cmd(["docker-compose"] + self.compose_opts + ["start"] + services)
 
-    def stop(self, services: List[str]) -> subprocess.CompletedProcess:
+    def stop(self, services: List[str]) -> subprocess.CompletedProcess[str]:
         """Stop the specified combine service(s)."""
         return run_cmd(
             ["docker-compose"] + self.compose_opts + ["stop", "--timeout", "0"] + services
@@ -129,7 +129,7 @@ def get_project_id(self, project_name: str) -> Optional[str]:
             return None
 
         if len(results) == 1:
-            return results[0]["_id"]
+            return results[0]["_id"]  # type: ignore
         if len(results) > 1:
             print(f"More than one project is named {project_name}", file=sys.stderr)
             sys.exit(1)
@@ -141,10 +141,10 @@ def get_user_id(self, user: str) -> Optional[str]:
             f'db.UsersCollection.findOne({{ username: "{user}"}}, {{ username: 1 }})'
         )
         if results is not None:
-            return results["_id"]
+            return results["_id"]  # type: ignore
         results = self.db_cmd(
             f'db.UsersCollection.findOne({{ email: "{user}"}}, {{ username: 1 }})'
         )
         if results is not None:
-            return results["_id"]
+            return results["_id"]  # type: ignore
         return None

deploy/roles/combine_maintenance/files/maint_utils.py

@@ -7,7 +7,7 @@
 from typing import List
 
 
-def run_cmd(cmd: List[str], *, check_results: bool = True) -> subprocess.CompletedProcess:
+def run_cmd(cmd: List[str], *, check_results: bool = True) -> subprocess.CompletedProcess[str]:
     """Run a command with subprocess and catch any CalledProcessErrors."""
     try:
         return subprocess.run(

dev-requirements.in

@@ -2,14 +2,16 @@ tox
 
 # Type Checking.
 mypy
+types-requests
 
 # Linting.
 flake8
 pep8-naming
 
 # Formatting.
 black
-isort
+# Pin until this issue is fixed: https://github.com/PyCQA/isort/issues/1762
+isort==5.8.0
 
 # Documentation.
 mkdocs-material

dev-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile dev-requirements.in
@@ -8,72 +8,69 @@ appdirs==1.4.4
     # via
     #   black
     #   virtualenv
-black==21.5b1
+black==21.6b0
     # via -r dev-requirements.in
-click==8.0.0
+click==8.0.1
     # via
     #   black
     #   mkdocs
-    #   nltk
 colorama==0.4.4
     # via
     #   click
     #   tox
-distlib==0.3.1
+distlib==0.3.2
     # via virtualenv
 filelock==3.0.12
     # via
     #   tox
     #   virtualenv
-flake8-polyfill==1.0.2
-    # via pep8-naming
 flake8==3.9.2
     # via
     #   -r dev-requirements.in
     #   flake8-polyfill
-future==0.18.2
-    # via lunr
-humanfriendly==9.1
+flake8-polyfill==1.0.2
+    # via pep8-naming
+ghp-import==2.0.1
+    # via mkdocs
+humanfriendly==9.2
     # via -r dev-requirements.in
+importlib-metadata==4.6.0
+    # via mkdocs
 isort==5.8.0
     # via -r dev-requirements.in
 jinja2==2.11.3
     # via
     #   -r dev-requirements.in
     #   mkdocs
-joblib==1.0.1
-    # via nltk
-livereload==2.6.3
-    # via mkdocs
-lunr[languages]==0.5.8
-    # via mkdocs
 markdown==3.3.4
     # via
     #   mkdocs
     #   mkdocs-material
     #   pymdown-extensions
-markupsafe==2.0.0
+markupsafe==2.0.1
     # via jinja2
 mccabe==0.6.1
     # via flake8
-mkdocs-material-extensions==1.0.1
+mergedeep==1.3.4
+    # via mkdocs
+mkdocs==1.2.1
     # via mkdocs-material
-mkdocs-material==7.1.4
+mkdocs-material==7.1.9
     # via
     #   -r dev-requirements.in
     #   mkdocs-material-extensions
-mkdocs==1.1.2
+mkdocs-material-extensions==1.0.1
     # via mkdocs-material
+mypy==0.910
+    # via -r dev-requirements.in
 mypy-extensions==0.4.3
     # via
     #   black
     #   mypy
-mypy==0.812
-    # via -r dev-requirements.in
-nltk==3.6.2
-    # via lunr
 packaging==20.9
-    # via tox
+    # via
+    #   mkdocs
+    #   tox
 pathspec==0.8.1
     # via black
 pep8-naming==0.11.1
@@ -94,33 +91,35 @@ pyparsing==2.4.7
     # via packaging
 pyreadline==2.1
     # via humanfriendly
+python-dateutil==2.8.1
+    # via ghp-import
 pyyaml==5.4.1
-    # via mkdocs
-regex==2021.4.4
     # via
-    #   black
-    #   nltk
+    #   mkdocs
+    #   pyyaml-env-tag
+pyyaml-env-tag==0.1
+    # via mkdocs
+regex==2021.7.1
+    # via black
 six==1.16.0
     # via
-    #   livereload
-    #   lunr
+    #   python-dateutil
     #   tox
     #   virtualenv
 toml==0.10.2
     # via
     #   black
+    #   mypy
     #   tox
-tornado==6.1
-    # via
-    #   livereload
-    #   mkdocs
 tox==3.23.1
     # via -r dev-requirements.in
-tqdm==4.60.0
-    # via nltk
-typed-ast==1.4.3
-    # via mypy
+types-requests==2.25.0
+    # via -r dev-requirements.in
 typing-extensions==3.10.0.0
     # via mypy
-virtualenv==20.4.6
+virtualenv==20.4.7
     # via tox
+watchdog==2.1.3
+    # via mkdocs
+zipp==3.4.1
+    # via importlib-metadata

docs/user_guide/mkdocs.yml

@@ -1,4 +1,3 @@
-site_name: The Combine
 strict: true
 theme:
   name: material
@@ -11,6 +10,9 @@ theme:
   language: en
   # Disable pulling Google fonts from the Internet, to support offline networks.
   font: false
+site_name: The Combine
+site_url: ""
+use_directory_urls: false
 repo_url: https://github.com/sillsdev/TheCombine
 repo_name: sillsdev/TheCombine
 copyright: © 2019-2021 SIL International

pyproject.toml

@@ -1,3 +1,7 @@
+[tool.mypy]
+ignore_missing_imports = true
+strict = true
+
 [tool.black]
 line-length = 99
 target-version = ["py37", "py38", "py39"]