Feature/general serde (#48)

* feat: add GeneralSerde for unified JSON serialization/deserialization

- Introduce GeneralSerde to handle both Pydantic BaseModel instances and regular JSON objects.
- Use model_dump_json for Pydantic models, falling back to json.dumps for non-Pydantic objects.
- Maintain consistency with existing serde implementations.

* remove trailing white space

* refactor: rename extract_io_type_hints to update_handler_io_with_type_hints for clarity

* fix: support JsonSerde in update_handler_io_with_type_hints and clean up GeneralSerde logging

* refactor: replace GeneralSerde with DefaultSerde across multiple files

- Updated imports and default serializer/deserializer in **context**, handler, object, **server_context**, service, and workflow modules.
- DefaultSerde provides a unified approach for JSON serialization/deserialization, ensuring compatibility with both Pydantic models and standard JSON objects.

* doc formating to respect just checks

* fix merge error

* fix Just trainling space

Author: ouatu-ro

python/restate/context.py

@@ -18,7 +18,7 @@
 from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVar, Union
 import typing
 from datetime import timedelta
-from restate.serde import JsonSerde, Serde
+from restate.serde import DefaultSerde, JsonSerde, Serde
 
 T = TypeVar('T')
 I = TypeVar('I')
@@ -120,7 +120,7 @@ def request(self) -> Request:
     def run(self,
             name: str,
             action: RunAction[T],
-            serde: Serde[T] = JsonSerde(),
+            serde: Serde[T] = DefaultSerde(),
             max_attempts: typing.Optional[int] = None,
             max_retry_duration: typing.Optional[timedelta] = None) -> RestateDurableFuture[T]:
         """

python/restate/handler.py

@@ -20,7 +20,7 @@
 from typing import Any, Callable, Awaitable, Dict, Generic, Literal, Optional, TypeVar
 
 from restate.exceptions import TerminalError
-from restate.serde import JsonSerde, Serde, PydanticJsonSerde
+from restate.serde import DefaultSerde, PydanticBaseModel, PydanticJsonSerde, Serde
 
 I = TypeVar('I')
 O = TypeVar('O')
@@ -29,22 +29,6 @@
 # we will use this symbol to store the handler in the function
 RESTATE_UNIQUE_HANDLER_SYMBOL = str(object())
 
-
-def try_import_pydantic_base_model():
-    """
-    Try to import PydanticBaseModel from Pydantic.
-    """
-    try:
-        from pydantic import BaseModel # type: ignore # pylint: disable=import-outside-toplevel
-        return BaseModel
-    except ImportError:
-        class Dummy: # pylint: disable=too-few-public-methods
-            """a dummy class to use when Pydantic is not available"""
-
-        return Dummy
-
-PydanticBaseModel = try_import_pydantic_base_model()
-
 @dataclass
 class ServiceTag:
     """
@@ -90,7 +74,7 @@ def is_pydantic(annotation) -> bool:
         return False
 
 
-def extract_io_type_hints(handler_io: HandlerIO[I, O], signature: Signature):
+def update_handler_io_with_type_hints(handler_io: HandlerIO[I, O], signature: Signature):
     """
     Augment handler_io with additional information about the input and output types.
 
@@ -105,15 +89,15 @@ def extract_io_type_hints(handler_io: HandlerIO[I, O], signature: Signature):
 
     if is_pydantic(annotation):
         handler_io.input_type.is_pydantic = True
-        if isinstance(handler_io.input_serde, JsonSerde): # type: ignore
+        if isinstance(handler_io.input_serde, DefaultSerde):  # type: ignore
             handler_io.input_serde = PydanticJsonSerde(annotation)
 
     annotation = signature.return_annotation
     handler_io.output_type = TypeHint(annotation=annotation, is_pydantic=False)
 
     if is_pydantic(annotation):
         handler_io.output_type.is_pydantic=True
-        if isinstance(handler_io.output_serde, JsonSerde): # type: ignore
+        if isinstance(handler_io.output_serde, DefaultSerde): # type: ignore
             handler_io.output_serde = PydanticJsonSerde(annotation)
 
 # pylint: disable=R0902
@@ -157,7 +141,7 @@ def make_handler(service_tag: ServiceTag,
         raise ValueError("Handler must have at least one parameter")
 
     arity = len(signature.parameters)
-    extract_io_type_hints(handler_io, signature)
+    update_handler_io_with_type_hints(handler_io, signature) # mutates handler_io
 
     handler = Handler[I, O](service_tag=service_tag,
                             handler_io=handler_io,

python/restate/object.py

@@ -17,7 +17,7 @@
 import inspect
 import typing
 
-from restate.serde import Serde, JsonSerde
+from restate.serde import Serde, DefaultSerde
 from restate.handler import Handler, HandlerIO, ServiceTag, make_handler
 
 I = typing.TypeVar('I')
@@ -60,8 +60,8 @@ def handler(self,
                 kind: typing.Optional[typing.Literal["exclusive", "shared"]] = "exclusive",
                 accept: str = "application/json",
                 content_type: str = "application/json",
-                input_serde: Serde[I] = JsonSerde[I](), # type: ignore
-                output_serde: Serde[O] = JsonSerde[O](), # type: ignore
+                input_serde: Serde[I] = DefaultSerde[I](), # type: ignore
+                output_serde: Serde[O] = DefaultSerde[O](), # type: ignore
                 metadata: typing.Optional[dict] = None) -> typing.Callable:
         """
         Decorator for defining a handler function.

python/restate/serde.py

@@ -13,6 +13,21 @@
 import json
 import typing
 
+def try_import_pydantic_base_model():
+    """
+    Try to import PydanticBaseModel from Pydantic.
+    """
+    try:
+        from pydantic import BaseModel # type: ignore # pylint: disable=import-outside-toplevel
+        return BaseModel
+    except ImportError:
+        class Dummy: # pylint: disable=too-few-public-methods
+            """a dummy class to use when Pydantic is not available"""
+
+        return Dummy
+
+PydanticBaseModel = try_import_pydantic_base_model()
+
 T = typing.TypeVar('T')
 I = typing.TypeVar('I')
 O = typing.TypeVar('O')
@@ -107,6 +122,65 @@ def serialize(self, obj: typing.Optional[I]) -> bytes:
 
         return bytes(json.dumps(obj), "utf-8")
 
+class DefaultSerde(Serde[I]):
+    """
+    The default serializer/deserializer used when no explicit type hints are provided.
+
+    Behavior:
+    - Serialization:
+        - If the object is an instance of Pydantic's `BaseModel`,
+            it uses `model_dump_json()` for serialization.
+        - Otherwise, it falls back to `json.dumps()`.
+    - Deserialization:
+        - Uses `json.loads()` to convert byte arrays into Python objects.
+        - Does **not** automatically reconstruct Pydantic models; 
+            deserialized objects remain as generic JSON structures (dicts, lists, etc.).
+
+    Serde Selection:
+    - When using the `@handler` decorator, if a function's type hints specify a Pydantic model, 
+      `PydanticJsonSerde` is automatically selected instead of `DefaultSerde`.
+    - `DefaultSerde` is only used if no explicit type hints are provided.
+
+    This serde ensures compatibility with both structured (Pydantic) and unstructured JSON data, 
+    while allowing automatic serde selection based on type hints.
+    """
+
+    def deserialize(self, buf: bytes) -> typing.Optional[I]:
+        """
+        Deserializes a byte array into a Python object.
+
+        Args:
+            buf (bytes): The byte array to deserialize.
+
+        Returns:
+            Optional[I]: The resulting Python object, or None if the input is empty.
+        """
+        if not buf:
+            return None
+        return json.loads(buf)
+
+    def serialize(self, obj: typing.Optional[I]) -> bytes:
+        """
+        Serializes a Python object into a byte array.
+        If the object is a Pydantic BaseModel, uses its model_dump_json method.
+
+        Args:
+            obj (Optional[I]): The Python object to serialize.
+
+        Returns:
+            bytes: The serialized byte array.
+        """
+
+        if obj is None:
+            return bytes()
+
+        if isinstance(obj, PydanticBaseModel):
+            # Use the Pydantic-specific serialization
+            return obj.model_dump_json().encode("utf-8")  # type: ignore[attr-defined]
+
+        # Fallback to standard JSON serialization
+        return json.dumps(obj).encode("utf-8")
+
 
 class PydanticJsonSerde(Serde[I]):
     """

python/restate/server_context.py

@@ -21,7 +21,7 @@
 from restate.context import DurablePromise, ObjectContext, Request, RestateDurableFuture, SendHandle
 from restate.exceptions import TerminalError
 from restate.handler import Handler, handler_from_callable, invoke_handler
-from restate.serde import BytesSerde, JsonSerde, Serde
+from restate.serde import BytesSerde, DefaultSerde, JsonSerde, Serde
 from restate.server_types import Receive, Send
 from restate.vm import Failure, Invocation, NotReady, SuspendedException, VMWrapper, RunRetryConfig # pylint: disable=line-too-long
 from restate.vm import DoProgressAnyCompleted, DoProgressCancelSignalReceived, DoProgressReadFromInput, DoProgressExecuteRun # pylint: disable=line-too-long
@@ -339,7 +339,7 @@ async def create_run_coroutine(self,
     def run(self,
                   name: str,
                   action: Callable[[], T] | Callable[[], Awaitable[T]],
-                  serde: Optional[Serde[T]] = JsonSerde(),
+                  serde: Optional[Serde[T]] = DefaultSerde(),
                   max_attempts: Optional[int] = None,
                   max_retry_duration: Optional[timedelta] = None) -> RestateDurableFuture[T]:
         assert serde is not None

python/restate/service.py

@@ -18,8 +18,8 @@
 import inspect
 import typing
 
-from restate.serde import Serde, JsonSerde
-from restate.handler import Handler, HandlerIO, ServiceTag, make_handler
+from restate.serde import Serde, DefaultSerde
+from .handler import Handler, HandlerIO, ServiceTag, make_handler
 
 I = typing.TypeVar('I')
 O = typing.TypeVar('O')
@@ -56,9 +56,10 @@ def handler(self,
                 name: typing.Optional[str] = None,
                 accept: str = "application/json",
                 content_type: str = "application/json",
-                input_serde: Serde[I] = JsonSerde[I](), # type: ignore
-                output_serde: Serde[O] = JsonSerde[O](), # type: ignore
-                metadata: typing.Optional[typing.Dict[str, str]] = None) -> typing.Callable:
+                input_serde: Serde[I] = DefaultSerde[I](), # type: ignore
+                output_serde: Serde[O] = DefaultSerde[O](), # type: ignore
+                metadata: typing.Optional[typing.Dict[str, str]] = None) -> typing.Callable: # type: ignore
+
         """
         Decorator for defining a handler function.
 

python/restate/workflow.py

@@ -18,7 +18,7 @@
 import inspect
 import typing
 
-from restate.serde import Serde, JsonSerde
+from restate.serde import DefaultSerde, Serde
 from restate.handler import Handler, HandlerIO, ServiceTag, make_handler
 
 I = typing.TypeVar('I')
@@ -59,8 +59,8 @@ def main(self,
             name: typing.Optional[str] = None,
             accept: str = "application/json",
             content_type: str = "application/json",
-            input_serde: Serde[I] = JsonSerde[I](), # type: ignore
-            output_serde: Serde[O] = JsonSerde[O](), # type: ignore
+            input_serde: Serde[I] = DefaultSerde[I](), # type: ignore
+            output_serde: Serde[O] = DefaultSerde[O](), # type: ignore
             metadata: typing.Optional[typing.Dict[str, str]] = None) -> typing.Callable: # type: ignore
         """Mark this handler as a workflow entry point"""
         return self._add_handler(name,
@@ -75,8 +75,8 @@ def handler(self,
                 name: typing.Optional[str] = None,
                 accept: str = "application/json",
                 content_type: str = "application/json",
-                input_serde: Serde[I] = JsonSerde[I](), # type: ignore
-                output_serde: Serde[O] = JsonSerde[O](), # type: ignore
+                input_serde: Serde[I] = DefaultSerde[I](), # type: ignore
+                output_serde: Serde[O] = DefaultSerde[O](), # type: ignore
                 metadata: typing.Optional[typing.Dict[str, str]] = None) -> typing.Callable:
         """
         Decorator for defining a handler function.
@@ -88,8 +88,8 @@ def _add_handler(self,
                 kind: typing.Literal["workflow", "shared", "exclusive"] = "shared",
                 accept: str = "application/json",
                 content_type: str = "application/json",
-                input_serde: Serde[I] = JsonSerde[I](), # type: ignore
-                output_serde: Serde[O] = JsonSerde[O](), # type: ignore
+                input_serde: Serde[I] = DefaultSerde[I](), # type: ignore
+                output_serde: Serde[O] = DefaultSerde[O](), # type: ignore
                 metadata: typing.Optional[typing.Dict[str, str]] = None) -> typing.Callable: # type: ignore
         """
         Decorator for defining a handler function.