Merge pull request #20 from timheap/feature/serialization

 Support Django serialization framework, and dumpdata/loaddata

Author: mx-moth

postgres_composite_types/__init__.py

@@ -35,10 +35,12 @@
 """
 
 import inspect
+import json
 import logging
 import sys
 from collections import OrderedDict
 
+from django.core.exceptions import ValidationError
 from django.db import migrations, models
 from django.db.backends.postgresql.base import \
     DatabaseWrapper as PostgresDatabaseWrapper
@@ -111,6 +113,10 @@ class BaseField(models.Field):
 
     Meta = None
 
+    default_error_messages = {
+        'bad_json': "to_python() received a string that was not valid JSON",
+    }
+
     def db_type(self, connection):
         LOGGER.debug("db_type")
 
@@ -132,6 +138,42 @@ def formfield(self, **kwargs):  # pylint:disable=arguments-differ
 
         return super().formfield(**defaults)
 
+    def to_python(self, value):
+        """
+        Convert a value to the correct type for this field. Values from the
+        database will already be of the correct type, due to the the caster
+        registered with psycopg2. The field can also be serialized as a string
+        via value_to_string, where it is encoded as a JSON object.
+        """
+        # Composite types are serialized as JSON blobs. If BaseField.to_python
+        # is called with a string, assume it was produced by value_to_string
+        # and decode it
+        if isinstance(value, str):
+            try:
+                value = json.loads(value)
+            except ValueError:
+                raise ValidationError(
+                    self.error_messages['bad_json'],
+                    code='bad_json',
+                )
+            return self.Meta.model(**{
+                name: field.to_python(value.get(name))
+                for name, field in self.Meta.fields
+            })
+
+        return super().to_python(value)
+
+    def value_to_string(self, obj):
+        """
+        Serialize this as a JSON object {name: field.value_to_string(...)} for
+        each child field.
+        """
+        value = self.value_from_object(obj)
+        return json.dumps({
+            name: field.value_to_string(value)
+            for name, field in self.Meta.fields
+        })
+
 
 class BaseOperation(migrations.operations.base.Operation):
     """Base class for the DB operation that relates to this type."""

test_requirements.in

@@ -1,6 +1,5 @@
 nose
 -egit+https://github.com/django-nose/django-nose.git@master#egg=django-nose
-django_fake_model
 
 flake8
 pylint

tests/migrations/0001_initial.py

@@ -5,7 +5,7 @@
 
 from django.db import migrations
 
-from ..base import (
+from ..models import (
     Box,
     Card,
     DateRange,

tests/migrations/0002_models.py

@@ -0,0 +1,62 @@
+# Generated by Django 2.0.2 on 2018-03-04 23:12
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+import tests.models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('tests', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='DescriptorModel',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('field', tests.models.DescriptorTypeField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Hand',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('cards', django.contrib.postgres.fields.ArrayField(base_field=tests.models.CardField(), size=None)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Item',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=20)),
+                ('bounding_box', tests.models.BoxField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name='NamedDateRange',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.TextField()),
+                ('date_range', tests.models.DateRangeField()),
+            ],
+        ),
+        migrations.CreateModel(
+            name='OptionalModel',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('optional_field', tests.models.OptionalBitsField(blank=True, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='SimpleModel',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('test_field', tests.models.SimpleTypeField()),
+            ],
+        ),
+    ]

tests/base.py renamed to tests/models.py

@@ -1,14 +1,11 @@
 """Models and types for the tests"""
 from django.contrib.postgres.fields.array import ArrayField
 from django.db import models
-from django_fake_model.models import FakeModel
 
 from postgres_composite_types import CompositeType
 
 from .fields import TriplingIntegerField
 
-# pylint:disable=no-member
-
 
 class SimpleType(CompositeType):
     """A test type."""
@@ -21,14 +18,11 @@ class Meta:
     c = models.DateTimeField(verbose_name='A date')
 
 
-class SimpleModel(FakeModel):
+class SimpleModel(models.Model):
     """A test model."""
     # pylint:disable=invalid-name
     test_field = SimpleType.Field()
 
-    class Meta:
-        app_label = 'test'
-
 
 class OptionalBits(CompositeType):
     """A type with an optional field"""
@@ -39,7 +33,7 @@ class Meta:
         db_type = 'optional_type'
 
 
-class OptionalModel(FakeModel):
+class OptionalModel(models.Model):
     """A model with an optional composity type"""
     optional_field = OptionalBits.Field(null=True, blank=True)
 
@@ -53,7 +47,7 @@ class Meta:
     rank = models.CharField(max_length=2)
 
 
-class Hand(FakeModel):
+class Hand(models.Model):
     """A hand of cards."""
     cards = ArrayField(base_field=Card.Field())
 
@@ -79,17 +73,19 @@ class Meta:
     @property
     def bottom_left(self):
         """The bottom-left corner of the box."""
+        # pylint:disable=no-member
         return Point(x=self.top_left.x,
                      y=self.bottom_right.y)
 
     @property
     def top_right(self):
         """The top-right corner of the box."""
+        # pylint:disable=no-member
         return Point(x=self.bottom_right.x,
                      y=self.top_left.y)
 
 
-class Item(FakeModel):
+class Item(models.Model):
     """An item that exists somewhere on a cartesian plane."""
     name = models.CharField(max_length=20)
     bounding_box = Box.Field()
@@ -104,7 +100,7 @@ class Meta:
     end = models.DateTimeField()   # uses reserved keyword
 
 
-class NamedDateRange(FakeModel):
+class NamedDateRange(models.Model):
     """A date-range with a name"""
     name = models.TextField()
     date_range = DateRange.Field()

tests/test_field.py

@@ -1,18 +1,21 @@
 """Tests for composite field."""
 
 import datetime
+import json
 from unittest import mock
 
+from django.core import serializers
+from django.core.exceptions import ValidationError
 from django.db import connection
 from django.db.migrations.executor import MigrationExecutor
 from django.test import TestCase, TransactionTestCase
 from psycopg2.extensions import adapt
 
 from postgres_composite_types import composite_type_created
 
-from .base import (
-    DateRange, NamedDateRange, OptionalBits, OptionalModel, SimpleModel,
-    SimpleType)
+from .models import (
+    Box, DateRange, Item, NamedDateRange, OptionalBits, OptionalModel, Point,
+    SimpleModel, SimpleType)
 
 
 class TestMigrations(TransactionTestCase):
@@ -91,8 +94,6 @@ def test_migration_quoting(self):
         self.assertTrue(self.does_type_exist(DateRange._meta.db_type))
 
 
-@SimpleModel.fake_me
-@NamedDateRange.fake_me
 class FieldTests(TestCase):
     """Tests for composite field."""
 
@@ -101,7 +102,7 @@ def test_field_save_and_load(self):
         # pylint:disable=invalid-name
         t = SimpleType(a=1, b="β ☃", c=datetime.datetime(1985, 10, 26, 9, 0))
         m = SimpleModel(test_field=t)
-        m.save()  # pylint:disable=no-member
+        m.save()
 
         # Retrieve from DB
         m = SimpleModel.objects.get(id=1)
@@ -150,8 +151,50 @@ def test_adapted_sql(self):
             b"(1, 'b', '1985-10-26T09:00:00'::timestamp)::test_type",
             adapted.getquoted())
 
+    def test_serialize(self):
+        """
+        Check that composite values are correctly handled through Django's
+        serialize/deserialize helpers, used for dumpdata/loaddata.
+        """
+        old = Item(
+            name="table",
+            bounding_box=Box(top_left=Point(x=1, y=1),
+                             bottom_right=Point(x=4, y=2)))
+        out = serializers.serialize("json", [old])
+        new = next(serializers.deserialize("json", out)).object
+
+        self.assertEqual(old.bounding_box,
+                         new.bounding_box)
+
+    def test_to_python(self):
+        """
+        Test the Field.to_python() method interprets strings as JSON data.
+        """
+        start = datetime.datetime.now()
+        end = datetime.datetime.now() + datetime.timedelta(days=1)
+
+        field = NamedDateRange._meta.get_field('date_range')
+        out = field.to_python(json.dumps({
+            "start": start.isoformat(),
+            "end": end.isoformat(),
+        }))
+
+        self.assertEqual(out, DateRange(start=start, end=end))
+
+    def test_to_python_bad_json(self):
+        """
+        Test the Field.to_python() handles bad JSON data by raising
+        a ValidationError
+        """
+        field = NamedDateRange._meta.get_field('date_range')
+
+        with self.assertRaises(ValidationError) as context:
+            field.to_python("bogus JSON")
+
+        exception = context.exception
+        self.assertEqual(exception.code, 'bad_json')
+
 
-@OptionalModel.fake_me
 class TestOptionalFields(TestCase):
     """
     Test optional composite type fields, and optional fields on composite types
@@ -160,18 +203,18 @@ class TestOptionalFields(TestCase):
     def test_null_field_save_and_load(self):
         """Save and load a null composite field"""
         model = OptionalModel(optional_field=None)
-        model.save()  # pylint:disable=no-member
+        model.save()
 
-        model = OptionalModel.objects.get(id=1)
+        model = OptionalModel.objects.get()
         self.assertIsNone(model.optional_field)
 
     def test_null_subfield_save_and_load(self):
         """Save and load a null composite field"""
         model = OptionalModel(optional_field=OptionalBits(
             required='foo', optional=None))
-        model.save()  # pylint:disable=no-member
+        model.save()
 
-        model = OptionalModel.objects.get(id=1)
+        model = OptionalModel.objects.get()
         self.assertIsNotNone(model.optional_field)
         self.assertEqual(model.optional_field, OptionalBits(
             required='foo', optional=None))
@@ -183,7 +226,7 @@ def test_all_filled(self):
         """
         model = OptionalModel(optional_field=OptionalBits(
             required='foo', optional='bar'))
-        model.save()  # pylint:disable=no-member
+        model.save()
 
         model = OptionalModel.objects.get(id=1)
         self.assertIsNotNone(model.optional_field)

tests/test_more.py

@@ -5,11 +5,10 @@
 from django.db.migrations.writer import MigrationWriter
 from django.test import TestCase
 
-from .base import (
+from .models import (
     Box, Card, DescriptorModel, DescriptorType, Hand, Item, Point)
 
 
-@Hand.fake_me
 class TestArrayFields(TestCase):
     """
     Test ArrayFields combined with CompositeType.Fields
@@ -27,7 +26,7 @@ def test_saving_and_loading_array_field(self):
             Card('♡', 'J'),
             Card('♡', '10'),
         ])
-        hand.save()  # pylint:disable=no-member
+        hand.save()
 
         hand = Hand.objects.get()
         self.assertEqual(hand.cards, [
@@ -49,7 +48,7 @@ def test_querying_array_field_contains(self):
             Card('♡', 'J'),
             Card('♡', '10'),
         ])
-        hand.save()  # pylint:disable=no-member
+        hand.save()
 
         queen_of_hearts = Card('♡', 'Q')
         jack_of_spades = Card('♠', 'J')
@@ -71,7 +70,6 @@ def test_generate_migrations(self):
         self.assertIn(expected_deconstruction, text)
 
 
-@Item.fake_me
 class TestNestedCompositeTypes(TestCase):
     """
     Test CompositeTypes within CompositeTypes
@@ -84,7 +82,7 @@ def test_saving_and_loading_nested_composite_types(self):
         item = Item(name="table",
                     bounding_box=Box(top_left=Point(x=1, y=1),
                                      bottom_right=Point(x=4, y=2)))
-        item.save()  # pylint:disable=no-member
+        item.save()
 
         item = Item.objects.get()
         self.assertEqual(item.name, "table")