Add ModelType base class and OneOf construct for type declarations

Signed-off-by: Jean Snyman <git@jsnyman.com>

keylime/models/base/__init__.py

@@ -1,8 +1,10 @@
-from sqlalchemy import Boolean, Integer, LargeBinary, String, Text
+from sqlalchemy import BigInteger, Boolean, Float, Integer, LargeBinary, SmallInteger, String, Text
 
 from keylime.models.base.basic_model import BasicModel
 from keylime.models.base.da import da_manager
 from keylime.models.base.db import db_manager
 from keylime.models.base.persistable_model import PersistableModel
+from keylime.models.base.type import ModelType
 from keylime.models.base.types.certificate import Certificate
 from keylime.models.base.types.dictionary import Dictionary
+from keylime.models.base.types.one_of import OneOf

keylime/models/base/basic_model.py

@@ -4,9 +4,6 @@
 from abc import ABC, abstractmethod
 from types import MappingProxyType
 
-from sqlalchemy.dialects.sqlite import dialect as sqlite_dialect
-from sqlalchemy.types import PickleType
-
 from keylime.models.base.errors import FieldValueInvalid, UndefinedField
 from keylime.models.base.field import ModelField
 
@@ -220,27 +217,15 @@ def change(self, name, value):
         # Reset the errors for the field to an empty list
         self._errors[name] = list()
 
-        # Get Field instance for name in order to obtain its type (TypeEngine object)
+        # Get Field instance for name in order to obtain its type (ModelType object)
         field = self.__class__.fields[name]
-        # Get processor which translates values of the given type to a format which can be stored in a DB
-        bind_processor = field.type.bind_processor(sqlite_dialect())
-        # Get processor which translates values retrieved by a DB query according to the field type
-        result_processor = field.type.result_processor(sqlite_dialect(), None)
 
         try:
-            # Process incoming value as if it were to be stored in a DB (if type requires inbound processing)
-            value = bind_processor(value) if bind_processor else value
-            # Process resulting value as if it were being retrieved from a DB (if type requires outbound processing)
-            value = result_processor(value) if result_processor else value
-            # Add value (processed according to the field type) to the model instance's collection of changes
-            self._changes[name] = value
+            # Attempt to cast incoming value to field's declared type
+            self._changes[name] = field.type.cast(value)
         except:
-            # If the above mock DB storage and retrieval fails, the incoming value is of an incorrect type for the field
-            if hasattr(field.type, "type_mismatch_msg") and not callable(getattr(field.type, "type_mismatch_msg")):
-                # Some custom types provide a special "invalid type" message
-                self._add_error(name, field.type.type_mismatch_msg)
-            else:
-                self._add_error(name, "is of an incorrect type")
+            # If above casting fails, produce a type mismatch message and add it the field's list of errors
+            self._add_error(name, field.type.generate_error_msg(value))
 
     def cast_changes(self, changes, permitted={}):
         for name, value in changes.items():

keylime/models/base/field.py

@@ -1,27 +1,30 @@
 from inspect import isclass
 
-from sqlalchemy.types import PickleType, TypeEngine
+from sqlalchemy.types import TypeEngine
 
 from keylime.models.base.errors import FieldDefinitionInvalid
+from keylime.models.base.type import ModelType
 
 
 class ModelField:
     _name: str
-    _type: TypeEngine
+    _type: ModelType
     _nullable: bool
 
     def __init__(self, name, type, nullable=False):
-        if isclass(type):
-            type = type()
-
         self._name = name
-        self._type = type
         self._nullable = nullable
 
-        if not isinstance(type, TypeEngine):
+        if isinstance(type, ModelType):
+            self._type = type
+        elif isclass(type) and issubclass(type, ModelType):
+            self._type = type()  # type: ignore
+        elif isinstance(type, TypeEngine) or (isclass(type) and issubclass(type, TypeEngine)):
+            self._type = ModelType(type)
+        else:
             raise FieldDefinitionInvalid(
-                f"field '{name}' cannot be defined with type '{type}' as this is not a SQLAlchemy datatype "
-                f"inheriting from 'sqlalchemy.types.TypeEngine'"
+                f"field '{name}' cannot be defined with type '{type}' as this is neither a ModelType subclass/instance "
+                f"nor a SQLAlchemy data type inheriting from 'sqlalchemy.types.TypeEngine'"
             )
 
     def __get__(self, obj, objtype=None):
@@ -45,12 +48,12 @@ def type(self):
         return self._type
 
     @property
-    def nullable(self):
-        return self._nullable
-
-    @property
-    def python_type(self):
+    def native_type(self):
         try:
-            return self.type.python_type
+            return self.type.native_type
         except:
             return None
+
+    @property
+    def nullable(self):
+        return self._nullable

keylime/models/base/persistable_model.py

@@ -3,12 +3,7 @@
 from sqlalchemy.orm import relationship
 
 from keylime.models.base import BasicModel
-from keylime.models.base.associations import (
-    AssociatedRecordSet,
-    BelongsToAssociation,
-    HasManyAssociation,
-    HasOneAssociation,
-)
+from keylime.models.base.associations import BelongsToAssociation, HasManyAssociation, HasOneAssociation
 from keylime.models.base.db import db_manager
 from keylime.models.base.errors import FieldValueInvalid, QueryInvalid, SchemaInvalid
 
@@ -133,9 +128,11 @@ def _new_field(cls, name, type, nullable=False, primary_key=False, column_args=(
         if not isinstance(column_args, tuple):
             column_args = (column_args,)
 
-        cls.__db_columns.append(Column(name, type, *column_args, nullable=nullable, primary_key=primary_key))
+        field = super()._new_field(name, type, nullable)
+        db_type = field.type.get_db_type(db_manager.engine.dialect)
+        cls.__db_columns.append(Column(name, db_type, *column_args, nullable=nullable, primary_key=primary_key))
 
-        return super()._new_field(name, type, nullable)
+        return field
 
     @classmethod
     def _field(cls, name, type, nullable=False, primary_key=False):
@@ -357,9 +354,9 @@ def __init__(self, data={}, process_associations=True):
     def _init_from_mapping(self, mapping_inst, process_associations):
         self._db_mapping_inst = mapping_inst
 
-        for name in self.__class__.fields.keys():
+        for name, field in self.__class__.fields.items():
             value = getattr(mapping_inst, name)
-            self.change(name, value)
+            self.change(name, field.type.db_load(value, db_manager.engine.dialect))
 
         if process_associations:
             for name, association in self.__class__.associations.items():
@@ -394,7 +391,9 @@ def commit_changes(self):
 
         for name, value in self._changes.items():
             self._record_values[name] = value
-            setattr(self._db_mapping_inst, name, value)
+
+            field = self.__class__.fields[name]
+            setattr(self._db_mapping_inst, name, field.type.db_dump(value, db_manager.engine.dialect))
 
         with db_manager.session_context() as session:
             session.add(self._db_mapping_inst)

keylime/models/base/type.py

@@ -0,0 +1,157 @@
+from decimal import Decimal
+from inspect import isclass
+from numbers import Real
+
+from sqlalchemy.types import TypeEngine
+
+
+class ModelType:
+    """The ModelType class and its subclasses enable type declarations for fields in model schemas. When a model
+    instance receives data from an external source, the incoming data is checked to be of the declared type for the
+    field. If not, under certain circumstances, it may be cast to the declared type automatically. Similarly, when data
+    is later read from that field to be used externally, it is prepared and formatted according to the context. The
+    logic for these checks and conversions are contained in ModelType and its subclasses.
+
+    Use of SQLAlchemy types
+    -----------------------
+
+    In many cases, when a field is declared, it is done so by specifying a SQLAlchemy type. For example, the ``"name"``
+    field below is given the type of `String` which inherits from SQLAlchemy's `TypeEngine` class:
+
+        def User(BasicModel):
+            def _schema(cls):
+                cls._field("name", String, nullable=True)
+                # (Any additional schema declarations...)
+
+    When this declaration is made, an instance of ModelType is transparently created by ``BasicModel`` and this is what
+    is used to perform data conversion for the field, if necessary. Internally, ModelType understands that the "native
+    type" for this field is `str`, i.e., that data for this field should be held in memory as a `str`. Any incoming data
+    which is not a `str` or `None` will result in an error being generated for the ``"name"`` field.
+
+    It is important to note the following caveats:
+
+    * Any parameters passed to the SQLAlchemy ``TypeEngine`` when the field is declared are ignored by ModelType. So, if
+    instead of the above declaration, ``"name"`` was declared with a type of ``String(50)``, the 50 character limit
+    would not be enforced. Instead, the author of the ``User`` model should use the validation methods in `BasicModel`
+    to impose a maximum length.
+
+    * The base ModelType class performs minimal implicit conversion of data, so when using SQLAlchemy types as above,
+    incoming data must typically already be in the correct "native type" or ``None``. Numeric SQLAlchemy types are a
+    notable exception and will allow strings to be accepted so long as they are convertible to the equivalent Python
+    type. Subclasses of ModelType (like ``Certificate``) often accept data in a variety of types and formats.
+
+    * Although the case for most, only SQLAlchemy types with the ``python_type`` property may used in the above manner.
+    When trying to use a SQLAlchemy type for which this is not the case, it is usually best to define a new custom type
+    (as described below).
+
+    Data lifecycle
+    --------------
+
+    Model data is processed according to the following diagram::
+
+                                      db_input   ┌──────────────┐     ┌──────────────┐
+                    ┌──────────────┐ ----------> │  SQLAlchemy  │ --> │   Database   │
+           input    │              │ <---------- │  TypeEngine  │ <-- │    Engine    │
+        ----------> │    Record    │  db_output  └──────────────┘     └──────────────┘
+                    | (instance of |
+        <---------- |    a model)  |  da_input   ┌─────────────────────┐
+           output   |              | ----------> │ Durable Attestation │
+                    └──────────────┘ <---------- │      Backend        │
+                                      da_output  └─────────────────────┘
+
+    When a field is set to a value (e.g., by calling ``record.change(field_name, value)``), an instance of ``ModelType``
+    or a subclass receives ``input`` as an argument to ``type.cast()``. This method has the task of converting ``input``
+    to the "native type" of the ``ModelType`` instance and the result is held in memory in the model instance.
+
+    If the field is contained within a ``PersistableModel``, it may be written to a database (DB) or durable
+    attestation (DA) backend. This is done by calling ``type.db_dump(value, dialect)`` or ``type.da_dump(value)`` which
+    each produce ``db_input`` and ``da_input`` in the diagram. When these are later retrieved from the database or DA
+    backend, they are ingested back into the model instance by calling ``type.db_load(value, dialect)`` or
+    ``type.da_load(value)`` to produce ``db_output`` and ``da_output`` respectively.
+
+    When a field is read from a model instance (to produce ``output`` in the diagram), this can happen in one of two
+    ways. If the field is accessed directly (``record.field_name``) or obtained from one of ``record.values``,
+    ``record.changes`` or ``record.record``, it is returned unchanged as it is stored in memory in the model instance.
+    If instead data in the field is to be prepared for external use outside the application, this is done by calling
+    ``type.render(value)``.
+
+    Custom types
+    ------------
+
+    A custom type can be created by subclassing ``ModelType`` and overriding a number of its various methods. You can
+    see examples of this in the ``Certificate`` and ``Dictionary`` classes found in ``keylime.models.base.types``.
+
+    Typically, you will wish to provide your own implementation of `cast` at minimum. You may also optionally override
+    each of ``db_dump``, ``da_dump``, ``db_load`` and ``da_load`` individually, but it is likely easier to just override
+    ``_dump`` and ``_load`` which are called by the default implementations of the various public "dump" and "load"
+    methods. In addition to these, you may also wish to override ``render`` if you need to prepare data in a particular
+    format on its way out of the application. Whichever of these data lifecycle methods you override, note that your
+    implementations must accept ``None`` as input (and, generally, return ``None`` in this case).
+
+    To customise the error messages which BasicModel produces in the event of a type mismatch, you have the option of
+    overriding ``generate_error_msg``. This method is called whenever a call to ``cast`` raises an error.
+
+    Finally, you will typically want to set the ``_type_engine`` attribute to the SQLAlchemy ``TypeEngine`` you wish to
+    use when persisting values of your custom type to the database (this is usually done in the ``__init__`` method). If
+    you wish to use different SQLAlchemy types depending on the database engine being used (the SQLAlchemy "dialect"),
+    you should instead set ``_type_engine`` to ``None`` and override the ``get_db_type`` method.
+    """
+
+    def __init__(self, type_engine):
+        if isclass(type_engine) and issubclass(type_engine, TypeEngine):
+            self._type_engine = type_engine()
+        elif isinstance(type_engine, TypeEngine):
+            self._type_engine = type_engine
+        else:
+            raise TypeError(f"{self.__class__.__name__} must be initialised with a 'TypeEngine' class/object")
+
+        try:
+            self._type_engine.python_type  # type: ignore
+        except NotImplementedError:
+            raise TypeError(f"{self._type_engine.__class__.__name__} does not define a 'python_type' property")
+
+    def cast(self, value):
+        if not value and isinstance(value, str):
+            value = None
+
+        if isinstance(value, str) and value.isnumeric() and issubclass(self.native_type, (Real, Decimal)):
+            value = self.native_type(value)  # type: ignore
+
+        if not isinstance(value, self.native_type) and value is not None:
+            raise TypeError(f"value '{value}' was expected to be of type '{self.native_type.__class__.__name__}'")
+
+        return value
+
+    def generate_error_msg(self, value):
+        return "is of an incorrect type"
+
+    def render(self, value):
+        value = self.cast(value)
+        return value
+
+    def _dump(self, value):
+        value = self.cast(value)
+        return value
+
+    def _load(self, value):
+        value = self.cast(value)
+        return value
+
+    def db_dump(self, value, dialect):
+        return self._dump(value)
+
+    def db_load(self, value, dialect):
+        return self._load(value)
+
+    def get_db_type(self, dialect):
+        return self._type_engine
+
+    def da_dump(self, value):
+        return self._dump(value)
+
+    def da_load(self, value):
+        return self._load(value)
+
+    @property
+    def native_type(self):
+        return self._type_engine.python_type  # type: ignore

keylime/models/base/types/__init__.py

@@ -1,2 +1,3 @@
 from keylime.models.base.types.certificate import Certificate
 from keylime.models.base.types.dictionary import Dictionary
+from keylime.models.base.types.one_of import OneOf