Mypy fixups

.github/workflows/ci.yml

@@ -54,49 +54,43 @@ jobs:
       FORCE_COLOR: 1
 
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-        cache: 'pip'
-    - name: Set up Node.js 16
-      uses: actions/setup-node@v3
-      with:
-        node-version: 16
-    - name: Install dependencies
-      run: |
-        python -m venv ~/env
-        source ~/env/bin/activate
-        python -m pip install --upgrade pip
-        python -m pip install --upgrade setuptools
-        pip install mypy
-        pip install bandit
-        sudo apt-get update
-        sudo apt-get install libsasl2-dev libldap2-dev xvfb
-    - name: Test
-      run: |
-        python -m venv ~/env
-        source ~/env/bin/activate
-        make test
-    - name: mypy
-      run: |
-        python -m venv ~/env
-        source ~/env/bin/activate
-        mypy --install-types --non-interactive .
-    - name: Test JS
-      run: |
-        python -m venv ~/env
-        source ~/env/bin/activate
-        bandit -r . -ll -ii -x lemur/tests/,docs
-        xvfb-run make test-js
-    - name: Coveralls GitHub Action
-      uses: coverallsapp/github-action@v2
-      with:
-        parallel: true
-        flag-name: Python ${{ matrix.python-version }} Postgres ${{ matrix.postgres-version }} OS ${{ matrix.os }}
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+      - name: Set up Node.js 16
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: Install dependencies
+        run: |
+          python -m venv ~/env
+          source ~/env/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade setuptools
+          pip install bandit
+          sudo apt-get update
+          sudo apt-get install libsasl2-dev libldap2-dev xvfb
+      - name: Test
+        run: |
+          python -m venv ~/env
+          source ~/env/bin/activate
+          make test
+      - name: Test JS
+        run: |
+          python -m venv ~/env
+          source ~/env/bin/activate
+          bandit -r . -ll -ii -x lemur/tests/,docs
+          xvfb-run make test-js
+      - name: Coveralls GitHub Action
+        uses: coverallsapp/github-action@v2
+        with:
+          parallel: true
+          flag-name: Python ${{ matrix.python-version }} Postgres ${{ matrix.postgres-version }} OS ${{ matrix.os }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   coveralls:
       name: Indicate completion to coveralls.io
       needs: build

Makefile

@@ -11,7 +11,7 @@ ifeq ($(USER), root)
 else
 	npm install
 endif
-	pip install "setuptools>=0.9.8"
+	pip install setuptools
 	# order matters here, base package must install first
 	pip install -e .
 	pip install -e "file://`pwd`#egg=lemur[dev]"
@@ -28,7 +28,7 @@ ifeq ($(USER), root)
 else
 	npm install
 endif
-	pip install "setuptools>=0.9.8"
+	pip install setuptools
 	# order matters here, base package must install first
 	pip install -e .
 	node_modules/.bin/gulp build
@@ -94,6 +94,7 @@ lint: lint-python lint-js
 lint-python:
 	@echo "--> Linting Python files"
 	PYFLAKES_NODOCTEST=1 flake8 lemur
+	mypy .
 	@echo ""
 
 lint-js:

lemur/authorities/schemas.py

@@ -6,11 +6,14 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from flask import current_app
-
 from marshmallow import fields, validates_schema, pre_load
 from marshmallow import validate
 from marshmallow.exceptions import ValidationError
 
+from lemur.common import validators, missing
+from lemur.common.fields import ArrowDateTime
+from lemur.common.schema import LemurInputSchema, LemurOutputSchema
+from lemur.constants import CERTIFICATE_KEY_TYPES
 from lemur.schemas import (
     PluginInputSchema,
     PluginOutputSchema,
@@ -19,11 +22,6 @@
     AssociatedRoleSchema,
 )
 from lemur.users.schemas import UserNestedOutputSchema
-from lemur.common.schema import LemurInputSchema, LemurOutputSchema
-from lemur.common import validators, missing
-
-from lemur.common.fields import ArrowDateTime
-from lemur.constants import CERTIFICATE_KEY_TYPES
 
 
 class AuthorityInputSchema(LemurInputSchema):
@@ -60,7 +58,8 @@ class AuthorityInputSchema(LemurInputSchema):
     parent = fields.Nested(AssociatedAuthoritySchema)
     signing_algorithm = fields.String(
         validate=validate.OneOf(["sha256WithRSA", "sha1WithRSA",
-                                 "sha256WithECDSA", "SHA384withECDSA", "SHA512withECDSA", "sha384WithECDSA", "sha512WithECDSA"]),
+                                 "sha256WithECDSA", "SHA384withECDSA", "SHA512withECDSA", "sha384WithECDSA",
+                                 "sha512WithECDSA"]),
         missing="sha256WithRSA",
     )
     key_type = fields.String(

lemur/common/fields.py

@@ -5,18 +5,16 @@
     :license: Apache, see LICENSE for more details.
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
-import arrow
-import warnings
 import ipaddress
-
-from flask import current_app
+import warnings
 from datetime import datetime as dt
 
+import arrow
 from cryptography import x509
-
+from flask import current_app
 from marshmallow import utils
-from marshmallow.fields import Field
 from marshmallow.exceptions import ValidationError
+from marshmallow.fields import Field
 
 from lemur.common import validators
 

lemur/domains/schemas.py

@@ -6,9 +6,11 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from marshmallow import fields
+
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
 from lemur.schemas import AssociatedCertificateSchema
 
+
 # from lemur.certificates.schemas import CertificateNestedOutputSchema
 
 

lemur/factory.py

@@ -14,11 +14,11 @@
 import os
 import socket
 import stat
+from importlib.metadata import entry_points
 from logging import Formatter, StreamHandler
 from logging.handlers import RotatingFileHandler
 
 import logmatic
-import pkg_resources
 import sentry_sdk
 from click import get_current_context
 from flask import Flask, current_app
@@ -256,12 +256,7 @@ def install_plugins(app):
     from lemur.plugins import plugins
     from lemur.plugins.base import register
 
-    # entry_points={
-    #    'lemur.plugins': [
-    #         'verisign = lemur_verisign.plugin:VerisignPlugin'
-    #     ],
-    # },
-    for ep in pkg_resources.iter_entry_points("lemur.plugins"):
+    for ep in entry_points().get("lemur.plugins", []):
         try:
             plugin = ep.load()
         except Exception:

lemur/notifications/schemas.py

@@ -6,6 +6,7 @@
 .. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
 """
 from marshmallow import fields, post_dump
+
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
 from lemur.schemas import (
     PluginInputSchema,

lemur/pending_certificates/schemas.py

@@ -1,9 +1,9 @@
 from marshmallow import fields, validates_schema, post_load
 from marshmallow.exceptions import ValidationError
 
-from lemur.common import utils, validators
 from lemur.authorities.schemas import AuthorityNestedOutputSchema
 from lemur.certificates.schemas import CertificateNestedOutputSchema
+from lemur.common import utils, validators
 from lemur.common.schema import LemurInputSchema, LemurOutputSchema
 from lemur.destinations.schemas import DestinationNestedOutputSchema
 from lemur.domains.schemas import DomainNestedOutputSchema

lemur/plugins/lemur_acme/acme_handlers.py

@@ -12,31 +12,29 @@
 .. moduleauthor:: Curtis Castrapel <ccastrapel@netflix.com>
 .. moduleauthor:: Mathias Petermann <mathias.petermann@projektfokus.ch>
 """
-from datetime import datetime, timezone, timedelta
 import json
 import time
+from datetime import datetime, timezone, timedelta
 
 import OpenSSL.crypto
-import josepy as jose
 import dns.resolver
+import josepy as jose
 from acme import challenges, errors, messages
 from acme.client import ClientV2, ClientNetwork
 from acme.errors import TimeoutError
 from acme.messages import Error as AcmeError, STATUS_VALID
 from certbot import crypto_util as acme_crypto_util
 from flask import current_app
+from retrying import retry
 from sentry_sdk import capture_exception
 
+from lemur.authorities import service as authorities_service
+from lemur.common.utils import data_encrypt, data_decrypt, is_json
 from lemur.common.utils import generate_private_key, key_to_alg
 from lemur.dns_providers import service as dns_provider_service
 from lemur.exceptions import InvalidAuthority, UnknownProvider, InvalidConfiguration
 from lemur.extensions import metrics
-
 from lemur.plugins.lemur_acme import cloudflare, dyn, route53, ultradns, powerdns, nsone
-from lemur.authorities import service as authorities_service
-from retrying import retry
-
-from lemur.common.utils import data_encrypt, data_decrypt, is_json
 
 
 class AuthorizationRecord(object):
@@ -149,6 +147,9 @@ def extract_cert_and_chain(self, fullchain_pem, alternative_fullchains_pem, pref
 
     @retry(stop_max_attempt_number=5, wait_fixed=5000)
     def setup_acme_client(self, authority):
+        return self.setup_acme_client_no_retry(authority)
+
+    def setup_acme_client_no_retry(self, authority):
         if not authority.options:
             raise InvalidAuthority("Invalid authority. Options not set")
         options = {}

lemur/plugins/lemur_acme/tests/test_acme_dns.py

@@ -1,16 +1,16 @@
 import unittest
+from unittest.mock import MagicMock
 from unittest.mock import patch, Mock
 
 import josepy as jose
-
 from acme.messages import STATUS_PENDING, STATUS_VALID
 from cryptography.x509 import DNSName
 from flask import Flask, current_app
+
+from lemur.common.utils import generate_private_key
 from lemur.plugins.lemur_acme import plugin
 from lemur.plugins.lemur_acme.acme_handlers import AuthorizationRecord
-from lemur.common.utils import generate_private_key
 from lemur.tests.conf import LEMUR_ENCRYPTION_KEYS
-from unittest.mock import MagicMock
 
 
 class TestAcmeDns(unittest.TestCase):
@@ -214,7 +214,7 @@ def test_setup_acme_client_fail(self):
         mock_authority = Mock()
         mock_authority.options = []
         with self.assertRaises(Exception):
-            self.acme.setup_acme_client(mock_authority)
+            self.acme.setup_acme_client_no_retry(mock_authority)
 
     @patch("lemur.plugins.lemur_acme.acme_handlers.jose.JWK.json_loads")
     @patch("lemur.plugins.lemur_acme.acme_handlers.ClientV2")

lemur/plugins/lemur_acme/tests/test_acme_handler.py

@@ -1,10 +1,10 @@
 import unittest
 from unittest.mock import patch, Mock
 
-from flask import Flask
 from cryptography.x509 import DNSName
-from lemur.plugins.lemur_acme import acme_handlers
+from flask import Flask
 
+from lemur.plugins.lemur_acme import acme_handlers
 from lemur.tests.vectors import (
     ACME_CHAIN_SHORT_STR,
     ACME_CHAIN_LONG_STR,
@@ -43,7 +43,7 @@ def test_setup_acme_client_fail(self):
         mock_authority = Mock()
         mock_authority.options = []
         with self.assertRaises(Exception):
-            self.acme.setup_acme_client(mock_authority)
+            self.acme.setup_acme_client_no_retry(mock_authority)
 
     def test_reuse_account_not_defined(self):
         mock_authority = Mock()

lemur/plugins/lemur_aws/tests/test_sns.py

@@ -5,8 +5,8 @@
 import boto3
 from moto import mock_sns, mock_sqs, mock_ses
 
-from lemur.notifications import service
 from lemur.certificates.schemas import certificate_notification_output_schema
+from lemur.notifications import service
 from lemur.plugins.lemur_aws.sns import format_message
 from lemur.plugins.lemur_aws.sns import publish
 from lemur.tests.factories import NotificationFactory, CertificateFactory

lemur/plugins/lemur_digicert/plugin.py

@@ -16,20 +16,21 @@
 import copy
 import ipaddress
 import json
+import sys
 from typing import Any, Dict, List
 
 import arrow
 import pem
 import requests
-import sys
 from cryptography import x509
 from flask import current_app, g
+from retrying import retry
+from urllib3.util.retry import Retry
+
 from lemur.common.utils import validate_conf, convert_pkcs7_bytes_to_pem
 from lemur.extensions import metrics
 from lemur.plugins import lemur_digicert as digicert
 from lemur.plugins.bases import IssuerPlugin, SourcePlugin
-from retrying import retry
-from requests.packages.urllib3.util.retry import Retry
 
 
 def log_status_code(r, *args, **kwargs):

lemur/plugins/lemur_entrust/plugin.py

@@ -1,16 +1,17 @@
-import arrow
-import requests
 import json
 import sys
+
+import arrow
+import requests
 from flask import current_app
 from retrying import retry
-from requests.packages.urllib3.util.retry import Retry
+from urllib3.util.retry import Retry
 
+from lemur.common.utils import validate_conf, get_key_type_from_certificate
 from lemur.constants import CRLReason
+from lemur.extensions import metrics
 from lemur.plugins import lemur_entrust as entrust
 from lemur.plugins.bases import IssuerPlugin, SourcePlugin
-from lemur.extensions import metrics
-from lemur.common.utils import validate_conf, get_key_type_from_certificate
 
 
 def log_status_code(r, *args, **kwargs):