Some optimizations for JSON

tests/core/utilities/test_encoding.py

@@ -38,6 +38,7 @@
     Web3ValueError,
 )
 from web3.providers import (
+    AsyncJSONBaseProvider,
     JSONBaseProvider,
 )
 
@@ -144,8 +145,11 @@ def test_text_if_str_on_text(val):
         ),
         (
             {
-                "date": [datetime.datetime.utcnow(), datetime.datetime.now()],
-                "other_date": datetime.datetime.utcnow().date(),
+                "date": [
+                    datetime.datetime.now(datetime.timezone.utc),
+                    datetime.datetime.now(),
+                ],
+                "other_date": datetime.datetime.now(datetime.timezone.utc).date(),
             },
             TypeError,
             "Could not encode to JSON: .*'other_date'.*is not JSON serializable",
@@ -227,11 +231,11 @@ def test_text_if_str_on_text(val):
 def test_friendly_json_encode_with_web3_json_encoder(py_obj, exc_type, expected):
     if exc_type is None:
         assert literal_eval(
-            FriendlyJson().json_encode(py_obj, Web3JsonEncoder)
+            FriendlyJson.json_encode(py_obj, Web3JsonEncoder)
         ) == literal_eval(expected)
     else:
         with pytest.raises(exc_type, match=expected):
-            FriendlyJson().json_encode(py_obj)
+            FriendlyJson.json_encode(py_obj)
 
 
 @pytest.mark.parametrize(
@@ -244,7 +248,7 @@ def test_friendly_json_encode_with_web3_json_encoder(py_obj, exc_type, expected)
     ),
 )
 def test_friendly_json_decode(json_str, expected):
-    assert isinstance(FriendlyJson().json_decode(json_str), expected)
+    assert isinstance(FriendlyJson.json_decode(json_str), expected)
 
 
 @pytest.mark.parametrize(
@@ -293,3 +297,52 @@ def test_encode_rpc_request(rpc_kwargs, exc_type, expected):
                 rpc_kwargs["method"],
                 rpc_kwargs["params"],
             )
+
+
+@pytest.mark.parametrize(
+    "rpc_response, expected",
+    (
+        (
+            '{"jsonrpc": "2.0", "method": "test_method", "params": [], "id": 1}',
+            {"jsonrpc": "2.0", "method": "test_method", "params": [], "id": 1},
+        ),
+    ),
+)
+def test_decode_asyncbase_rpc_response(rpc_response, expected):
+    assert (
+        AsyncJSONBaseProvider().decode_rpc_response(rpc_response.encode("utf8"))
+        == expected
+    )
+
+
+@pytest.mark.parametrize(
+    "rpc_kwargs, exc_type, expected",
+    (
+        (
+            {"id": 1, "method": "test", "params": [], "jsonrpc": "2.0"},
+            None,
+            '{"id": 0, "method": "test", "params": [], "jsonrpc": "2.0",}',
+        ),
+        (
+            {
+                "id": 0,
+                "method": "test",
+                "params": [datetime.datetime(2018, 5, 10, 1, 5, 10)],
+            },
+            TypeError,
+            r"Could not encode to JSON: .*'params'.* is not JSON serializable",
+        ),
+    ),
+)
+def test_encode_asyncbase_rpc_request(rpc_kwargs, exc_type, expected):
+    if exc_type is None:
+        res = AsyncJSONBaseProvider().encode_rpc_request(
+            rpc_kwargs["method"], rpc_kwargs["params"]
+        )
+        assert literal_eval(res) == literal_eval(expected)
+    else:
+        with pytest.raises(exc_type, match=expected):
+            JSONBaseProvider().encode_rpc_request(
+                rpc_kwargs["method"],
+                rpc_kwargs["params"],
+            )

tests/core/web3-module/test_conversions.py

@@ -159,7 +159,7 @@ def test_to_int_hexstr(val, expected):
         (b"\x01", "0x01"),
         (b"\x10", "0x10"),
         (b"\x01\x00", "0x0100"),
-        (b"\x00\x0F", "0x000f"),
+        (b"\x00\x0f", "0x000f"),
         (b"", "0x"),
         (0, "0x0"),
         (1, "0x1"),
@@ -204,13 +204,13 @@ def test_to_hex_cleanup_only(val, expected):
 @pytest.mark.parametrize(
     "val, expected",
     (
-        (AttributeDict({"one": HexBytes("0x1")}), '{"one": "0x01"}'),
-        (AttributeDict({"two": HexBytes(2)}), '{"two": "0x02"}'),
+        (AttributeDict({"one": HexBytes("0x1")}), '{"one":"0x01"}'),
+        (AttributeDict({"two": HexBytes(2)}), '{"two":"0x02"}'),
         (
             AttributeDict({"three": AttributeDict({"four": 4})}),
-            '{"three": {"four": 4}}',
+            '{"three":{"four":4}}',
         ),
-        ({"three": 3}, '{"three": 3}'),
+        ({"three": 3}, '{"three":3}'),
     ),
 )
 def test_to_json(val, expected):
@@ -247,7 +247,7 @@ def test_to_json(val, expected):
                     "value": 2907000000000000,
                 }
             ),
-            '{"blockHash": "0x849044202a39ae36888481f90d62c3826bca8269c2716d7a38696b4f45e61d83", "blockNumber": 6928809, "from": "0xDEA141eF43A2fdF4e795adA55958DAf8ef5FA619", "gas": 21000, "gasPrice": 19110000000, "hash": "0x1ccddd19830e998d7cf4d921b19fafd5021c9d4c4ba29680b66fb535624940fc", "input": "0x", "nonce": 5522, "r": "0x71ef3eed6242230a219d9dc7737cb5a3a16059708ee322e96b8c5774105b9b00", "s": "0x48a076afe10b4e1ae82ef82b747e9be64e0bbb1cc90e173db8d53e7baba8ac46", "to": "0x3a84E09D30476305Eda6b2DA2a4e199E2Dd1bf79", "transactionIndex": 8, "v": 27, "value": 2907000000000000}',  # noqa: E501
+            '{"blockHash":"0x849044202a39ae36888481f90d62c3826bca8269c2716d7a38696b4f45e61d83","blockNumber":6928809,"from":"0xDEA141eF43A2fdF4e795adA55958DAf8ef5FA619","gas":21000,"gasPrice":19110000000,"hash":"0x1ccddd19830e998d7cf4d921b19fafd5021c9d4c4ba29680b66fb535624940fc","input":"0x","nonce":5522,"r":"0x71ef3eed6242230a219d9dc7737cb5a3a16059708ee322e96b8c5774105b9b00","s":"0x48a076afe10b4e1ae82ef82b747e9be64e0bbb1cc90e173db8d53e7baba8ac46","to":"0x3a84E09D30476305Eda6b2DA2a4e199E2Dd1bf79","transactionIndex":8,"v":27,"value":2907000000000000}',  # noqa: E501
         ),
     ),
 )

web3/_utils/encoding.py

@@ -1,11 +1,15 @@
 # String encodings and numeric representations
+from collections.abc import (
+    Mapping,
+)
 import json
 import re
 from typing import (
     Any,
     Callable,
     Dict,
     Iterable,
+    List,
     Optional,
     Sequence,
     Type,
@@ -191,41 +195,59 @@ class FriendlyJsonSerde:
     helpful information in the raised error messages.
     """
 
-    def _json_mapping_errors(self, mapping: Dict[Any, Any]) -> Iterable[str]:
+    @classmethod
+    def _json_mapping_errors(
+        self,
+        mapping: Dict[Any, Any],
+        encoder_cls: Optional[Type[json.JSONEncoder]] = None,
+    ) -> Iterable[str]:
         for key, val in mapping.items():
             try:
-                self._friendly_json_encode(val)
+                self._friendly_json_encode(val, encoder_cls=encoder_cls)
             except TypeError as exc:
                 yield f"{key!r}: because ({exc})"
 
-    def _json_list_errors(self, iterable: Iterable[Any]) -> Iterable[str]:
+    @classmethod
+    def _json_list_errors(
+        self,
+        iterable: Iterable[Any],
+        encoder_cls: Optional[Type[json.JSONEncoder]] = None,
+    ) -> Iterable[str]:
         for index, element in enumerate(iterable):
             try:
-                self._friendly_json_encode(element)
+                self._friendly_json_encode(element, encoder_cls=encoder_cls)
             except TypeError as exc:
                 yield f"{index}: because ({exc})"
 
+    @classmethod
     def _friendly_json_encode(
-        self, obj: Dict[Any, Any], cls: Optional[Type[json.JSONEncoder]] = None
+        cls,
+        obj: Union[Dict[Any, Any], List[Dict[Any, Any]]],
+        encoder_cls: Optional[Type[json.JSONEncoder]] = None,
     ) -> str:
         try:
-            encoded = json.dumps(obj, cls=cls)
+            encoded = json.dumps(obj, cls=encoder_cls, separators=(",", ":"))
             return encoded
         except TypeError as full_exception:
-            if hasattr(obj, "items"):
-                item_errors = "; ".join(self._json_mapping_errors(obj))
+            if isinstance(obj, Mapping):
+                item_errors = "; ".join(
+                    cls._json_mapping_errors(obj, encoder_cls=encoder_cls)
+                )
                 raise Web3TypeError(
                     f"dict had unencodable value at keys: {{{item_errors}}}"
                 )
             elif is_list_like(obj):
-                element_errors = "; ".join(self._json_list_errors(obj))
+                element_errors = "; ".join(
+                    cls._json_list_errors(obj, encoder_cls=encoder_cls)
+                )
                 raise Web3TypeError(
                     f"list had unencodable value at index: [{element_errors}]"
                 )
             else:
                 raise full_exception
 
-    def json_decode(self, json_str: str) -> Dict[Any, Any]:
+    @classmethod
+    def json_decode(cls, json_str: str) -> Dict[Any, Any]:
         try:
             decoded = json.loads(json_str)
             return decoded
@@ -235,11 +257,14 @@ def json_decode(self, json_str: str) -> Dict[Any, Any]:
             # so we have to re-raise the same type.
             raise json.decoder.JSONDecodeError(err_msg, exc.doc, exc.pos)
 
+    @classmethod
     def json_encode(
-        self, obj: Dict[Any, Any], cls: Optional[Type[json.JSONEncoder]] = None
+        cls,
+        obj: Union[Dict[Any, Any], List[Dict[Any, Any]]],
+        encoder_cls: Optional[Type[json.JSONEncoder]] = None,
     ) -> str:
         try:
-            return self._friendly_json_encode(obj, cls=cls)
+            return cls._friendly_json_encode(obj, encoder_cls=encoder_cls)
         except TypeError as exc:
             raise Web3TypeError(f"Could not encode to JSON: {exc}")
 
@@ -306,4 +331,4 @@ def to_json(obj: Dict[Any, Any]) -> str:
     """
     Convert a complex object (like a transaction object) to a JSON string
     """
-    return FriendlyJsonSerde().json_encode(obj, cls=Web3JsonEncoder)
+    return FriendlyJsonSerde.json_encode(obj, encoder_cls=Web3JsonEncoder)

web3/providers/__init__.py

@@ -1,8 +1,9 @@
 from .async_base import (
     AsyncBaseProvider,
+    AsyncJSONBaseProvider,
 )
-from .rpc import (
-    AsyncHTTPProvider,
+from .auto import (
+    AutoProvider,
 )
 from .base import (
     BaseProvider,
@@ -15,9 +16,6 @@
 from .ipc import (
     IPCProvider,
 )
-from .rpc import (
-    HTTPProvider,
-)
 from .legacy_websocket import (
     LegacyWebSocketProvider,
 )
@@ -27,12 +25,14 @@
     PersistentConnectionProvider,
     WebSocketProvider,
 )
-from .auto import (
-    AutoProvider,
+from .rpc import (
+    AsyncHTTPProvider,
+    HTTPProvider,
 )
 
 __all__ = [
     "AsyncBaseProvider",
+    "AsyncJSONBaseProvider",
     "AsyncEthereumTesterProvider",
     "AsyncHTTPProvider",
     "AsyncIPCProvider",

web3/providers/async_base.py

@@ -6,6 +6,7 @@
     Any,
     Callable,
     Coroutine,
+    Dict,
     List,
     Optional,
     Set,
@@ -15,7 +16,6 @@
 
 from eth_utils import (
     is_text,
-    to_bytes,
     to_text,
 )
 
@@ -175,23 +175,24 @@ def __init__(self, **kwargs: Any) -> None:
         self.request_counter = itertools.count()
         super().__init__(**kwargs)
 
-    def encode_rpc_request(self, method: RPCEndpoint, params: Any) -> bytes:
-        request_id = next(self.request_counter)
-        rpc_dict = {
+    def _build_rpc_dict(self, method: RPCEndpoint, params: Any) -> Dict[str, Any]:
+        return {
             "jsonrpc": "2.0",
             "method": method,
             "params": params or [],
-            "id": request_id,
+            "id": next(self.request_counter),
         }
-        encoded = FriendlyJsonSerde().json_encode(rpc_dict, cls=Web3JsonEncoder)
-        return to_bytes(text=encoded)
+
+    def encode_rpc_request(self, method: RPCEndpoint, params: Any) -> str:
+        rpc_dict = self._build_rpc_dict(method, params)
+        return FriendlyJsonSerde.json_encode(rpc_dict, encoder_cls=Web3JsonEncoder)
 
     @staticmethod
     def decode_rpc_response(raw_response: bytes) -> RPCResponse:
         text_response = str(
             to_text(raw_response) if not is_text(raw_response) else raw_response
         )
-        return cast(RPCResponse, FriendlyJsonSerde().json_decode(text_response))
+        return cast(RPCResponse, FriendlyJsonSerde.json_decode(text_response))
 
     async def is_connected(self, show_traceback: bool = False) -> bool:
         try:
@@ -219,13 +220,8 @@ async def is_connected(self, show_traceback: bool = False) -> bool:
 
     # -- batch requests -- #
 
-    def encode_batch_rpc_request(
-        self, requests: List[Tuple[RPCEndpoint, Any]]
-    ) -> bytes:
-        return (
-            b"["
-            + b", ".join(
-                self.encode_rpc_request(method, params) for method, params in requests
-            )
-            + b"]"
-        )
+    def encode_batch_rpc_request(self, requests: List[Tuple[RPCEndpoint, Any]]) -> str:
+        rpc_dicts = [
+            self._build_rpc_dict(method, params) for method, params in requests
+        ]
+        return FriendlyJsonSerde.json_encode(rpc_dicts, encoder_cls=Web3JsonEncoder)

web3/providers/auto.py

@@ -20,12 +20,18 @@
 from web3.exceptions import (
     CannotHandleRequest,
 )
-from web3.providers import (
+from web3.providers.base import (
     BaseProvider,
-    HTTPProvider,
+)
+from web3.providers.ipc import (
     IPCProvider,
+)
+from web3.providers.legacy_websocket import (
     LegacyWebSocketProvider,
 )
+from web3.providers.rpc import (
+    HTTPProvider,
+)
 from web3.types import (
     RPCEndpoint,
     RPCResponse,

web3/providers/base.py

@@ -131,13 +131,13 @@ def encode_rpc_request(self, method: RPCEndpoint, params: Any) -> bytes:
             "params": params or [],
             "id": next(self.request_counter),
         }
-        encoded = FriendlyJsonSerde().json_encode(rpc_dict, Web3JsonEncoder)
+        encoded = FriendlyJsonSerde.json_encode(rpc_dict, encoder_cls=Web3JsonEncoder)
         return to_bytes(text=encoded)
 
     @staticmethod
     def decode_rpc_response(raw_response: bytes) -> RPCResponse:
         text_response = to_text(raw_response)
-        return cast(RPCResponse, FriendlyJsonSerde().json_decode(text_response))
+        return cast(RPCResponse, FriendlyJsonSerde.json_decode(text_response))
 
     def is_connected(self, show_traceback: bool = False) -> bool:
         try: