linting

Signed-off-by: Niels Drost <codingdutchman@gmail.com>

kedro/extras/datasets/api/api_dataset.py

@@ -9,7 +9,7 @@
 from kedro.extras.datasets.api.auth_factory import create_authenticator
 from kedro.io.core import AbstractDataSet, DataSetError
 
-_DEFAULT_CREDENTIALS = {}
+_DEFAULT_CREDENTIALS: Dict[str, Any] = {}
 
 
 class APIDataSet(AbstractDataSet):
@@ -39,13 +39,14 @@ class APIDataSet(AbstractDataSet):
         >>> data = data_set.load()
     """
 
+    # pylint: disable=too-many-arguments
     def __init__(
         self,
         url: str,
         method: str = "GET",
-        auth_type: str = 'requests.auth.HTTPBasicAuth',
+        auth_type: str = "requests.auth.HTTPBasicAuth",
         load_args: Dict[str, Any] = None,
-        credentials: Dict[str, Any] = None
+        credentials: Dict[str, Any] = None,
     ) -> None:
         """Creates a new instance of ``APIDataSet`` to fetch data from an API endpoint.
 
@@ -71,7 +72,7 @@ def __init__(
             **(load_args or {}),
             "url": url,
             "method": method,
-            "auth": self._auth
+            "auth": self._auth,
         }
 
     def _describe(self) -> Dict[str, Any]:

kedro/extras/datasets/api/auth_factory.py

@@ -1,24 +1,37 @@
-import importlib
-
+"""``auth_factory`` creates `requests.auth.AuthBase` instances from Catalog configuration.
+"""
 from requests.auth import AuthBase
 
 from kedro.io import DataSetError
 from kedro.utils import load_obj
 
 
 def create_authenticator(class_type: str, **kwargs):
+    """
+    Args:
+        class_type: path to class that inherits from `requests.auth.AuthBase`.
+        **kwargs: constructor parameters for this class.
+
+    Returns:
+        An instance of the class that is provided.
+    Raises:
+        DataSetError: if class cannot be loaded or instantiated,
+        or class does not inherit from `requests.auth.AuthBase`
+    """
     try:
         class_obj = load_obj(class_type)
     except Exception as err:
         raise DataSetError(
-            f"The specified class path {class_type} for constructing an Auth object cannot be found."
+            f"The specified class path {class_type} "
+            f"for constructing an Auth object cannot be found."
         ) from err
 
     try:
         authenticator = class_obj(**kwargs)  # type: ignore
     except TypeError as err:
         raise DataSetError(
-            f"\n{err}.\nAuthenticator Object '{class_type}' must only contain arguments valid for the "
+            f"\n{err}.\nAuthenticator Object '{class_type}' "
+            f"must only contain arguments valid for the "
             f"constructor of '{class_obj.__module__}.{class_obj.__qualname__}'."
         ) from err
     except Exception as err:
@@ -31,4 +44,4 @@ def create_authenticator(class_type: str, **kwargs):
             raise DataSetError(
                 f"The requests library expects {class_type} to be an instance of AuthBase."
             )
-    return authenticator
+    return authenticator

tests/extras/datasets/api/conftest.py

@@ -5,4 +5,4 @@
 @pytest.fixture
 def requests_mocker():
     with requests_mock.Mocker() as mock:
-        yield mock
+        yield mock

tests/extras/datasets/api/test_api_auth.py

@@ -5,7 +5,12 @@
 from requests.auth import AuthBase
 
 from kedro.extras.datasets.api import APIDataSet
-from tests.extras.datasets.api.test_api_dataset import TEST_URL, TEST_METHOD, TEST_HEADERS, TEST_TEXT_RESPONSE_DATA
+from tests.extras.datasets.api.test_api_dataset import (
+    TEST_HEADERS,
+    TEST_METHOD,
+    TEST_TEXT_RESPONSE_DATA,
+    TEST_URL,
+)
 
 
 class AccessTokenAuth(AuthBase):
@@ -17,43 +22,63 @@ def __init__(self, token):
 
     def __call__(self, r):
         # modify and return the request
-        r.headers['Authorization'] = f'access_token {self.token}'
+        r.headers["Authorization"] = f"access_token {self.token}"
         return r
 
 
 def _basic_auth(username, password):
-    encoded = base64.b64encode(f"{username}:{password}".encode('latin-1'))
+    encoded = base64.b64encode(f"{username}:{password}".encode("latin-1"))
     return f"Basic {encoded.decode('latin-1')}"
 
 
 class TestApiAuth:
-
-    @pytest.mark.parametrize("auth_type,auth_cred,auth_header_key, auth_header_value", [
-        ("requests.auth.HTTPBasicAuth", { "username": "john", "password": "doe" }, "Authorization", _basic_auth('john', 'doe')),
-        ("requests.auth.HTTPProxyAuth", { "username": "john", "password": "doe" }, "Proxy-Authorization", _basic_auth('john', 'doe')),
-        ("tests.extras.datasets.api.test_api_auth.AccessTokenAuth", { "token": "abc" }, "Authorization", "access_token abc"),
-    ])
-    def test_auth_sequence(self, requests_mocker, auth_cred, auth_type, auth_header_key, auth_header_value):
+    @pytest.mark.parametrize(
+        "auth_type,auth_cred,auth_header_key, auth_header_value",
+        [
+            (
+                "requests.auth.HTTPBasicAuth",
+                {"username": "john", "password": "doe"},
+                "Authorization",
+                _basic_auth("john", "doe"),
+            ),
+            (
+                "requests.auth.HTTPProxyAuth",
+                {"username": "john", "password": "doe"},
+                "Proxy-Authorization",
+                _basic_auth("john", "doe"),
+            ),
+            (
+                "tests.extras.datasets.api.test_api_auth.AccessTokenAuth",
+                {"token": "abc"},
+                "Authorization",
+                "access_token abc",
+            ),
+        ],
+    )
+    def test_auth_sequence(
+        self, requests_mocker, auth_cred, auth_type, auth_header_key, auth_header_value
+    ):
         """
-            Tests to make sure request Authenticator instances can be created and configured with the right credentials.
-            The created authenticator is passed in with a request and headers are tested for the correct value.
+        Tests to make sure request Authenticator instances
+        can be created and configured with the right credentials.
+        The created authenticator is passed in with a request
+        and headers are tested for the correct value.
         """
         api_data_set = APIDataSet(
             url=TEST_URL,
             method=TEST_METHOD,
             auth_type=auth_type,
             load_args={"headers": TEST_HEADERS},
-            credentials=auth_cred
+            credentials=auth_cred,
         )
 
         requests_mocker.register_uri(
-            TEST_METHOD,
-            TEST_URL,
-            headers=TEST_HEADERS,
-            text=TEST_TEXT_RESPONSE_DATA
+            TEST_METHOD, TEST_URL, headers=TEST_HEADERS, text=TEST_TEXT_RESPONSE_DATA
         )
 
         response = api_data_set.load()
         assert isinstance(response, requests.Response)
         assert response.text == TEST_TEXT_RESPONSE_DATA
-        assert requests_mocker.last_request.headers[auth_header_key] == auth_header_value
+        assert (
+            requests_mocker.last_request.headers[auth_header_key] == auth_header_value
+        )

tests/extras/datasets/api/test_api_dataset.py

@@ -21,16 +21,13 @@
 
 
 class TestAPIDataSet:
-
     @pytest.mark.parametrize("method", POSSIBLE_METHODS)
     def test_successfully_load_with_response(self, requests_mocker, method):
         api_data_set = APIDataSet(
             url=TEST_URL, method=method, load_args={"params": TEST_PARAMS}
         )
         requests_mocker.register_uri(
-            method,
-            TEST_URL_WITH_PARAMS,
-            text=TEST_TEXT_RESPONSE_DATA
+            method, TEST_URL_WITH_PARAMS, text=TEST_TEXT_RESPONSE_DATA
         )
 
         response = api_data_set.load()
@@ -41,22 +38,18 @@ def test_headers_in_request(self, requests_mocker):
         api_data_set = APIDataSet(
             url=TEST_URL, method=TEST_METHOD, load_args={"headers": TEST_HEADERS}
         )
-        requests_mocker.register_uri(
-            TEST_METHOD,
-            TEST_URL,
-            headers={"pan": "cake"}
-        )
+        requests_mocker.register_uri(TEST_METHOD, TEST_URL, headers={"pan": "cake"})
 
         response = api_data_set.load()
 
-        assert response.request.headers['key'] == 'value'
-        assert response.headers['pan'] == 'cake'
+        assert response.request.headers["key"] == "value"
+        assert response.headers["pan"] == "cake"
 
     def test_successful_json_load_with_response(self, requests_mocker):
         api_data_set = APIDataSet(
             url=TEST_URL,
             method=TEST_METHOD,
-            load_args={"json": TEST_JSON_RESPONSE_DATA, "headers": TEST_HEADERS}
+            load_args={"json": TEST_JSON_RESPONSE_DATA, "headers": TEST_HEADERS},
         )
         requests_mocker.register_uri(
             TEST_METHOD,
@@ -71,7 +64,9 @@ def test_successful_json_load_with_response(self, requests_mocker):
 
     def test_http_error(self, requests_mocker):
         api_data_set = APIDataSet(
-            url=TEST_URL, method=TEST_METHOD, load_args={"params": TEST_PARAMS, "headers": TEST_HEADERS}
+            url=TEST_URL,
+            method=TEST_METHOD,
+            load_args={"params": TEST_PARAMS, "headers": TEST_HEADERS},
         )
         requests_mocker.register_uri(
             TEST_METHOD,
@@ -86,9 +81,13 @@ def test_http_error(self, requests_mocker):
 
     def test_socket_error(self, requests_mocker):
         api_data_set = APIDataSet(
-            url=TEST_URL, method=TEST_METHOD, load_args={"params": TEST_PARAMS, "headers": TEST_HEADERS}
+            url=TEST_URL,
+            method=TEST_METHOD,
+            load_args={"params": TEST_PARAMS, "headers": TEST_HEADERS},
+        )
+        requests_mocker.register_uri(
+            TEST_METHOD, TEST_URL_WITH_PARAMS, exc=socket.error
         )
-        requests_mocker.register_uri(TEST_METHOD, TEST_URL_WITH_PARAMS, exc=socket.error)
 
         with pytest.raises(DataSetError, match="Failed to connect"):
             api_data_set.load()
@@ -107,7 +106,9 @@ def test_exists_http_error(self, requests_mocker):
         ``exists()`` should not silently catch it.
         """
         api_data_set = APIDataSet(
-            url=TEST_URL, method=TEST_METHOD, load_args={"params": TEST_PARAMS, "headers": TEST_HEADERS }
+            url=TEST_URL,
+            method=TEST_METHOD,
+            load_args={"params": TEST_PARAMS, "headers": TEST_HEADERS},
         )
         requests_mocker.register_uri(
             TEST_METHOD,
@@ -125,7 +126,9 @@ def test_exists_ok(self, requests_mocker):
         ``exists()`` should return True
         """
         api_data_set = APIDataSet(
-            url=TEST_URL, method=TEST_METHOD, load_args={"params": TEST_PARAMS, "headers": TEST_HEADERS}
+            url=TEST_URL,
+            method=TEST_METHOD,
+            load_args={"params": TEST_PARAMS, "headers": TEST_HEADERS},
         )
         requests_mocker.register_uri(
             TEST_METHOD,
@@ -139,15 +142,14 @@ def test_exists_ok(self, requests_mocker):
     def test_certs(self, requests_mocker):
 
         api_data_set = APIDataSet(
-            url=TEST_URL, method=TEST_METHOD, load_args={ "cert": ('cert.pem', 'privkey.pem')}
-        )
-        requests_mocker.register_uri(
-            TEST_METHOD,
-            TEST_URL
+            url=TEST_URL,
+            method=TEST_METHOD,
+            load_args={"cert": ("cert.pem", "privkey.pem")},
         )
+        requests_mocker.register_uri(TEST_METHOD, TEST_URL)
 
         api_data_set.load()
-        assert requests_mocker.last_request.cert == ('cert.pem', 'privkey.pem')
+        assert requests_mocker.last_request.cert == ("cert.pem", "privkey.pem")
 
     def test_stream(self, requests_mocker):
         text = "I am being streamed."
@@ -156,40 +158,34 @@ def test_stream(self, requests_mocker):
             url=TEST_URL, method=TEST_METHOD, load_args={"stream": True}
         )
 
-        requests_mocker.register_uri(
-            TEST_METHOD,
-            TEST_URL,
-            text=text
-        )
+        requests_mocker.register_uri(TEST_METHOD, TEST_URL, text=text)
 
         response = api_data_set.load()
         assert isinstance(response, requests.Response)
         assert requests_mocker.last_request.stream
 
-        chunks = [chunk for chunk in response.iter_content(chunk_size=2, decode_unicode=True)]
-        assert chunks == ['I ', 'am', ' b', 'ei', 'ng', ' s', 'tr', 'ea', 'me', 'd.']
+        chunks = list(response.iter_content(chunk_size=2, decode_unicode=True))
+        assert chunks == ["I ", "am", " b", "ei", "ng", " s", "tr", "ea", "me", "d."]
 
     def test_proxy(self, requests_mocker):
         api_data_set = APIDataSet(
-            url="ftp://example.com/api/test", method=TEST_METHOD, load_args={"proxies": {"ftp": "ftp://127.0.0.1:3000"}}
+            url="ftp://example.com/api/test",
+            method=TEST_METHOD,
+            load_args={"proxies": {"ftp": "ftp://127.0.0.1:3000"}},
         )
         requests_mocker.register_uri(
             TEST_METHOD,
             "ftp://example.com/api/test",
         )
 
         api_data_set.load()
-        assert requests_mocker.last_request.proxies.get('ftp') == "ftp://127.0.0.1:3000"
+        assert requests_mocker.last_request.proxies.get("ftp") == "ftp://127.0.0.1:3000"
 
     def test_api_cookies(self, requests_mocker):
         api_data_set = APIDataSet(
-            url=TEST_URL, method=TEST_METHOD, load_args={"cookies":  {"pan": "cake"}}
-        )
-        requests_mocker.register_uri(
-            TEST_METHOD,
-            TEST_URL,
-            text='text'
+            url=TEST_URL, method=TEST_METHOD, load_args={"cookies": {"pan": "cake"}}
         )
+        requests_mocker.register_uri(TEST_METHOD, TEST_URL, text="text")
 
         api_data_set.load()
-        assert requests_mocker.last_request.headers['Cookie'] == "pan=cake"
+        assert requests_mocker.last_request.headers["Cookie"] == "pan=cake"