Merge pull request #85 from epics-containers/pydantic2

Replace APISchema with Pydantic2

.vscode/launch.json

@@ -5,7 +5,7 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "Debug Unit Test",
+            "name": "Debug example test",
             "type": "python",
             "request": "launch",
             "justMyCode": false,

docs/conf.py

@@ -40,8 +40,6 @@
     "sphinx.ext.viewcode",
     # Adds the inheritance-diagram generation directive
     "sphinx.ext.inheritance_diagram",
-    # Makes autodoc understand apischema annotated classes/functions
-    "sphinx_apischema",
     # Add a copy button to each code block
     "sphinx_copybutton",
     # For the card element
@@ -67,16 +65,17 @@
     ("py:class", "typing_extensions.Literal"),
 ]
 
-# Both the class’ and the __init__ method’s docstring are concatenated and
-# inserted into the main body of the autoclass directive
-autoclass_content = "both"
+# Dont use the __init__ docstring because pydantic base classes cause sphinx
+# to generate a lot of warnings
+autoclass_content = "class"
 
 # Order the members by the order they appear in the source code
 autodoc_member_order = "bysource"
 
 # Don't inherit docstrings from baseclasses
 autodoc_inherit_docstrings = False
 
+
 # Output graphviz directive produced images in a scalable format
 graphviz_output_format = "svg"
 

examples/README.md

@@ -0,0 +1,38 @@
+Intro
+=====
+
+These example scripts are used to investigate an issue with error reporting in ibek.
+
+They also serve as a good minimal example of how to do object references within a pydantic 2 model.
+
+The incrementing numeric suffix represents a progression from the most simple possible example of a pydantic model with a reference to a more complex example that more closely resembles ibek's approach which dynamically creates the Entity classes.
+
+In the yaml subfolder is a support module yaml and IOC yaml that will make ibek load a very similar model to that described in these test scripts.
+
+Issue under investigation
+=========================
+
+The issue is that when an object refers to another object then the error reported is that the offending object's id cannot be found. This masks the underlying schema issue which is what should be reported first. The custom field validator created in make_entity_model seems to be throwing the error before the schema validation issue is reported.
+
+At present for the incorrect schema in entity e1 ibek reports:
+
+```
+KeyError: 'object one not found in []'
+```
+
+And test_refs4.py reports
+
+```
+Extra inputs are not permitted [type=extra_forbidden, input_value='bad argument', input_type=str]
+```
+
+The latter is the useful error that points you at the root cause.
+
+Resolution
+==========
+
+The simplest test_refs1.py has been updated to demo the issue (forgot that
+entity "one" already existed in model1!).
+
+I've posted a discussion on the subject here
+https://github.com/pydantic/pydantic/discussions/6731

examples/test_refs1.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+id_to_entity: Dict[str, Entity] = {}
+
+
+class Entity(BaseModel):
+    name: str = Field(..., description="The name of this entity")
+    value: str = Field(..., description="The value of this entity")
+    ref: Optional[str] = Field(
+        default=None, description="Reference another Entity name"
+    )
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")  # type: ignore
+    def add_ibek_attributes(cls, entity: Entity):
+        id_to_entity[entity.name] = entity
+
+        return entity
+
+    @field_validator("ref", mode="after")
+    def lookup_instance(cls, id):
+        try:
+            return id_to_entity[id]
+        except KeyError:
+            raise KeyError(f"object {id} not found in {list(id_to_entity)}")
+
+
+class Entities(BaseModel):
+    entities: List[Entity] = Field(..., description="The entities in this model")
+
+
+model1 = Entities(
+    **{
+        "entities": [
+            {"name": "one", "value": "OneValue"},
+            {"name": "two", "value": "TwoValue", "ref": "one"},
+        ]
+    }
+)
+
+# demonstrate that entity two has a reference to entity one
+assert model1.entities[1].ref.value == "OneValue"
+
+# this should throw an error because entity one_again has illegal arguments
+# BUT the error shown is:
+#    KeyError: "object one_again not found in ['one', 'two']"
+# which masks the underlying schema violation error that should look like:
+#    Extra inputs are not permitted [type=extra_forbidden, input_value='bad argument',
+model2 = Entities(
+    **{
+        "entities": [
+            {"name": "one_again", "value": "OneValue", "illegal": "bad argument"},
+            {"name": "two_again", "value": "TwoValue", "ref": "one_again"},
+        ]
+    }
+)

examples/test_refs2.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    create_model,
+    field_validator,
+    model_validator,
+)
+
+id_to_entity: Dict[str, Entity] = {}
+
+
+class Entity(BaseModel):
+    name: str = Field(..., description="The name of this entity")
+    value: str = Field(..., description="The value of this entity")
+    ref: Optional[str] = Field(
+        default=None, description="Reference another Entity name"
+    )
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")  # type: ignore
+    def add_ibek_attributes(cls, entity: Entity):
+        id_to_entity[entity.name] = entity
+
+        return entity
+
+
+@field_validator("ref", mode="after")
+def lookup_instance(cls, id):
+    try:
+        return id_to_entity[id]
+    except KeyError:
+        raise KeyError(f"object {id} not found in {list(id_to_entity)}")
+
+
+validators = {"Entity": lookup_instance}
+
+# add validator to the Entity class using create model
+Entity2 = create_model(
+    "Entity",
+    __validators__=validators,
+    __base__=Entity,
+)  # type: ignore
+
+args = {"entities": (List[Entity2], None)}
+Entities = create_model(
+    "Entities", **args, __config__=ConfigDict(extra="forbid")
+)  # type: ignore
+
+
+model1 = Entities(
+    **{
+        "entities": [
+            {"name": "one", "value": "OneValue"},
+            {"name": "two", "value": "TwoValue", "ref": "one"},
+        ]
+    }
+)
+
+# demonstrate that entity one has a reference to entity two
+assert model1.entities[1].ref.value == "OneValue"
+
+# this should throw an error because entity one has illegal arguments
+model2 = Entities(
+    **{
+        "entities": [
+            {"name": "one", "value": "OneValue", "illegal": "bad argument"},
+            {"name": "two", "value": "TwoValue", "ref": "one"},
+        ]
+    }
+)

examples/test_refs3.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+from typing import Dict, Literal, Optional, Sequence, Union
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    RootModel,
+    create_model,
+    field_validator,
+    model_validator,
+)
+
+id_to_entity: Dict[str, Entity] = {}
+
+
+class Entity(BaseModel):
+    type: str = Field(description="The type of this entity")
+    name: str = Field(..., description="The name of this entity")
+    value: str = Field(..., description="The value of this entity")
+    ref: Optional[str] = Field(
+        default=None, description="Reference another Entity name"
+    )
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")  # type: ignore
+    def add_ibek_attributes(cls, entity: Entity):
+        id_to_entity[entity.name] = entity
+
+        return entity
+
+
+class Entity1(Entity):
+    type: Literal["e1"] = Field(description="The type of this entity")
+
+
+class Entity2(Entity):
+    type: Literal["e2"] = Field(description="The type of this entity")
+
+
+@field_validator("ref", mode="after")
+def lookup_instance(cls, id):
+    try:
+        return id_to_entity[id]
+    except KeyError:
+        raise KeyError(f"object {id} not found in {list(id_to_entity)}")
+
+
+validators = {"Entity": lookup_instance}
+
+# add validator to the Entity classes using create model
+EntityOne = create_model(
+    "EntityOne",
+    __validators__=validators,
+    __base__=Entity1,
+)  # type: ignore
+
+EntityTwo = create_model(
+    "EntityTwo",
+    __validators__=validators,
+    __base__=Entity2,
+)  # type: ignore
+
+entity_models = (EntityOne, EntityTwo)
+
+
+class EntityModel(RootModel):
+    root: Union[entity_models] = Field(discriminator="type")  # type: ignore
+
+
+class Entities(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+    entities: Sequence[EntityModel] = Field(  # type: ignore
+        description="List of entities classes we want to create"
+    )
+
+
+model1 = Entities(
+    **{
+        "entities": [
+            {"type": "e1", "name": "one", "value": "OneValue"},
+            {"type": "e2", "name": "two", "value": "TwoValue", "ref": "one"},
+        ]
+    }
+)
+
+# demonstrate that entity one has a reference to entity two
+assert model1.entities[1].root.ref.value == "OneValue"
+
+# this should throw an error because entity one has illegal arguments
+model2 = Entities(
+    **{
+        "entities": [
+            {"type": "e2", "name": "two", "value": "TwoValue", "ref": "one"},
+            {
+                "type": "e1",
+                "name": "one",
+                "value": "OneValue",
+                "illegal": "bad argument",
+            },
+        ]
+    }
+)