From 271b1449cd877e11bb7eaaaa3ab0891fe90cde50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dami=C3=A1n=20Nohales?= Date: Thu, 19 Nov 2015 17:57:32 -0300 Subject: [PATCH 1/6] Add default normalization rule --- cerberus/cerberus.py | 6 ++++++ cerberus/tests/tests.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/cerberus/cerberus.py b/cerberus/cerberus.py index c5912f5b..d1832a7f 100644 --- a/cerberus/cerberus.py +++ b/cerberus/cerberus.py @@ -454,6 +454,7 @@ def __normalize_mapping(self, mapping, schema): if self.purge_unknown: self._normalize_purge_unknown(mapping, schema) self._normalize_coerce(mapping, schema) + self._normalize_default(mapping, schema) self.__normalize_containers(mapping, schema) return mapping @@ -564,6 +565,11 @@ def _normalize_rename_handler(self, mapping, schema, field): mapping[new_name] = mapping[field] del mapping[field] + def _normalize_default(self, mapping, schema): + for field in tuple(schema): + if 'default' in schema[field] and field not in mapping: + mapping[field] = schema[field]['default'] + # # Validating def validate(self, document, schema=None, update=False, normalize=True): diff --git a/cerberus/tests/tests.py b/cerberus/tests/tests.py index 9179acda..c23481bf 100644 --- a/cerberus/tests/tests.py +++ b/cerberus/tests/tests.py @@ -1438,6 +1438,45 @@ def test_coercion_of_sequence_items(self): for x in result['a_list']: self.assertIsInstance(x, float) + def test_default_missing(self): + schema = {'foo': {'type': 'string'}, + 'bar': {'type': 'string', + 'default': 'bar_value'}} + + self.assertSuccess({'foo': 'foo_value'}, schema) + result = self.validator.document + self.assertDictEqual(result, {'foo': 'foo_value', 'bar': 'bar_value'}) + + def test_default_existent(self): + schema = {'foo': {'type': 'string'}, + 'bar': {'type': 'string', + 'default': 'bar_value'}} + + self.assertSuccess({'foo': 'foo_value', 'bar': 'non_default'}, schema) + result = self.validator.document + self.assertDictEqual(result, {'foo': 'foo_value', 'bar': 'non_default'}) + + def test_default_none(self): + schema = {'foo': {'type': 'string'}, + 'bar': {'type': 'string', + 'nullable': True, + 'default': 'bar_value'}} + + self.assertSuccess({'foo': 'foo_value', 'bar': None}, schema) + result = self.validator.document + self.assertDictEqual(result, {'foo': 'foo_value', 'bar': None}) + + def test_default_missing_in_subschema(self): + schema = {'thing': {'type': 'dict', + 'schema': {'foo': {'type': 'string'}, + 'bar': {'type': 'string', + 'default': 'bar_value'}}}} + + self.assertSuccess({'thing': {'foo': 'foo_value'}}, schema) + result = self.validator.document + self.assertDictEqual(result, {'thing': {'foo': 'foo_value', + 'bar': 'bar_value'}}) + class DefinitionSchema(TestBase): def test_validated_schema_cache(self): From 16852ceb41df76f77d6a3915c1b4fba7afd1c6f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dami=C3=A1n=20Nohales?= Date: Fri, 20 Nov 2015 10:05:19 -0300 Subject: [PATCH 2/6] Add documentation for default normalization rule --- docs/normalization-rules.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/normalization-rules.rst b/docs/normalization-rules.rst index 039a7f0a..9644f408 100644 --- a/docs/normalization-rules.rst +++ b/docs/normalization-rules.rst @@ -39,6 +39,24 @@ subdocuments like ``allow_unknown`` (see :ref:`allowing-the-unknown`). The defau .. versionadded:: 0.10 +Default Values +-------------- +You can set default values for missing fields in the document by using the ``default`` rule. + +.. doctest:: + + >>> v.schema = {'amount': {'type': 'integer'}, 'kind': {'type': 'string', 'default': 'purchase'}} + >>> v.normalized({'amount': 1}) + {'amount': 1, 'kind': 'purchase'} + + >>> v.normalized({'amount': 1, 'kind': 'other'}) + {'amount': 1, 'kind': 'other'} + + >>> v.normalized({'amount': 1, 'kind': None}) + {'amount': 1, 'kind': None} + +.. versionadded:: 0.10 + .. _type-coercion: Value Coercion From bdb28721b905c0fdc1c65efc6c9ae500ea79e9ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dami=C3=A1n=20Nohales?= Date: Fri, 20 Nov 2015 11:39:03 -0300 Subject: [PATCH 3/6] Add assertNormalizedEqual method This method allow to validate a document for a specific schema/validator and then compare the normalized document to another document in one single call. --- cerberus/tests/__init__.py | 8 +++ cerberus/tests/tests.py | 107 ++++++++++++++++++++----------------- 2 files changed, 65 insertions(+), 50 deletions(-) diff --git a/cerberus/tests/__init__.py b/cerberus/tests/__init__.py index 7df20fe5..1011c562 100644 --- a/cerberus/tests/__init__.py +++ b/cerberus/tests/__init__.py @@ -235,3 +235,11 @@ def assertChildErrors(self, *args, **kwargs): def assertBadType(self, field, data_type, value): self.assertFail({field: value}) self.assertError(field, (field, 'type'), errors.BAD_TYPE, data_type) + + def assertNormalizedEqual(self, document, expected, schema=None, + validator=None): + if validator is None: + validator = self.validator + + self.assertSuccess(document, schema, validator) + self.assertDictEqual(validator.document, expected) diff --git a/cerberus/tests/tests.py b/cerberus/tests/tests.py index c23481bf..77f1d4fe 100644 --- a/cerberus/tests/tests.py +++ b/cerberus/tests/tests.py @@ -1319,19 +1319,17 @@ def test_excludes_hashable(self): class TestNormalization(TestBase): def test_coerce(self): - schema = { - 'amount': {'coerce': int} - } - v = Validator(schema) - v.validate({'amount': '1'}) - self.assertEqual(v.document['amount'], 1) + schema = {'amount': {'coerce': int}} + document = {'amount': '1'} + expected = {'amount': 1} + self.assertNormalizedEqual(document, expected, schema) def test_coerce_in_subschema(self): schema = {'thing': {'type': 'dict', 'schema': {'amount': {'coerce': int}}}} - v = Validator(schema) - self.assertEqual(v.validated({'thing': {'amount': '2'}}) - ['thing']['amount'], 2) # noqa + document = {'thing': {'amount': '2'}} + expected = {'thing': {'amount': 2}} + self.assertNormalizedEqual(document, expected, schema) def test_coerce_not_destructive(self): schema = { @@ -1361,34 +1359,47 @@ def test_coerce_catches_TypeError(self): def test_coerce_unknown(self): schema = {'foo': {'schema': {}, 'allow_unknown': {'coerce': int}}} - v = Validator(schema) document = {'foo': {'bar': '0'}} - self.assertDictEqual(v.normalized(document), {'foo': {'bar': 0}}) + expected = {'foo': {'bar': 0}} + self.assertNormalizedEqual(document, expected, schema) def test_normalized(self): schema = {'amount': {'coerce': int}} - v = Validator(schema) - self.assertEqual(v.normalized({'amount': '2'})['amount'], 2) + document = {'amount': '2'} + expected = {'amount': 2} + self.assertNormalizedEqual(document, expected, schema) def test_rename(self): + schema = {'foo': {'rename': 'bar'}} document = {'foo': 0} - v = Validator({'foo': {'rename': 'bar'}}) - self.assertDictEqual(v.normalized(document), {'bar': 0}) + expected = {'bar': 0} + # We cannot use assertNormalizedEqual here since there is bug where + # Cerberus says that the renamed field is an unknown field: + # {'bar': 'unknown field'} + self.validator(document, schema, False) + self.assertDictEqual(self.validator.document, expected) def test_rename_handler(self): + validator = Validator(allow_unknown={'rename_handler': int}) + schema = {} document = {'0': 'foo'} - v = Validator({}, allow_unknown={'rename_handler': int}) - self.assertDictEqual(v.normalized(document), {0: 'foo'}) + expected = {0: 'foo'} + self.assertNormalizedEqual(document, expected, schema, validator) def test_purge_unknown(self): - v = Validator({'foo': {'type': 'string'}}, purge_unknown=True) - self.assertDictEqual(v.normalized({'bar': 'foo'}), {}) - v.purge_unknown = False - self.assertDictEqual(v.normalized({'bar': 'foo'}), {'bar': 'foo'}) - v.schema = {'foo': {'type': 'dict', - 'schema': {'foo': {'type': 'string'}}, - 'purge_unknown': True}} - self.assertDictEqual(v.normalized({'foo': {'bar': ''}}), {'foo': {}}) + validator = Validator(purge_unknown=True) + schema = {'foo': {'type': 'string'}} + document = {'bar': 'foo'} + expected = {} + self.assertNormalizedEqual(document, expected, schema, validator) + + def test_purge_unknown_in_subschema(self): + schema = {'foo': {'type': 'dict', + 'schema': {'foo': {'type': 'string'}}, + 'purge_unknown': True}} + document = {'foo': {'bar': ''}} + expected = {'foo': {}} + self.assertNormalizedEqual(document, expected, schema) def test_issue_147_complex(self): schema = {'revision': {'coerce': int}} @@ -1417,65 +1428,61 @@ def test_coerce_in_valueschema(self): schema = {'thing': {'type': 'dict', 'valueschema': {'coerce': int, 'type': 'integer'}}} - self.assertSuccess({'thing': {'amount': '2'}}, schema) - self.assertDictEqual(self.validator.document, {'thing': {'amount': 2}}) + document = {'thing': {'amount': '2'}} + expected = {'thing': {'amount': 2}} + self.assertNormalizedEqual(document, expected, schema) def test_coerce_in_propertyschema(self): # https://github.com/nicolaiarocci/cerberus/issues/155 schema = {'thing': {'type': 'dict', 'propertyschema': {'coerce': int, 'type': 'integer'}}} - self.assertSuccess({'thing': {'5': 'foo'}}, schema) - self.assertDictEqual(self.validator.document, {'thing': {5: 'foo'}}) + document = {'thing': {'5': 'foo'}} + expected = {'thing': {5: 'foo'}} + self.assertNormalizedEqual(document, expected, schema) def test_coercion_of_sequence_items(self): # https://github.com/nicolaiarocci/cerberus/issues/161 schema = {'a_list': {'type': 'list', 'schema': {'type': 'float', 'coerce': float}}} - self.assertSuccess({'a_list': [3, 4, 5]}, schema) - result = self.validator.document - self.assertListEqual(result['a_list'], [3.0, 4.0, 5.0]) - for x in result['a_list']: + document = {'a_list': [3, 4, 5]} + expected = {'a_list': [3.0, 4.0, 5.0]} + self.assertNormalizedEqual(document, expected, schema) + for x in self.validator.document['a_list']: self.assertIsInstance(x, float) def test_default_missing(self): schema = {'foo': {'type': 'string'}, 'bar': {'type': 'string', 'default': 'bar_value'}} - - self.assertSuccess({'foo': 'foo_value'}, schema) - result = self.validator.document - self.assertDictEqual(result, {'foo': 'foo_value', 'bar': 'bar_value'}) + document = {'foo': 'foo_value'} + expected = {'foo': 'foo_value', 'bar': 'bar_value'} + self.assertNormalizedEqual(document, expected, schema) def test_default_existent(self): schema = {'foo': {'type': 'string'}, 'bar': {'type': 'string', 'default': 'bar_value'}} - - self.assertSuccess({'foo': 'foo_value', 'bar': 'non_default'}, schema) - result = self.validator.document - self.assertDictEqual(result, {'foo': 'foo_value', 'bar': 'non_default'}) + document = {'foo': 'foo_value', 'bar': 'non_default'} + self.assertNormalizedEqual(document, document.copy(), schema) def test_default_none(self): schema = {'foo': {'type': 'string'}, 'bar': {'type': 'string', 'nullable': True, 'default': 'bar_value'}} - - self.assertSuccess({'foo': 'foo_value', 'bar': None}, schema) - result = self.validator.document - self.assertDictEqual(result, {'foo': 'foo_value', 'bar': None}) + document = {'foo': 'foo_value', 'bar': None} + self.assertNormalizedEqual(document, document.copy(), schema) def test_default_missing_in_subschema(self): schema = {'thing': {'type': 'dict', 'schema': {'foo': {'type': 'string'}, 'bar': {'type': 'string', 'default': 'bar_value'}}}} - - self.assertSuccess({'thing': {'foo': 'foo_value'}}, schema) - result = self.validator.document - self.assertDictEqual(result, {'thing': {'foo': 'foo_value', - 'bar': 'bar_value'}}) + document = {'thing': {'foo': 'foo_value'}} + expected = {'thing': {'foo': 'foo_value', + 'bar': 'bar_value'}} + self.assertNormalizedEqual(document, expected, schema) class DefinitionSchema(TestBase): From 0ce4e90004d0c68dee99907ddbb41f70401b7b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dami=C3=A1n=20Nohales?= Date: Sun, 22 Nov 2015 14:59:57 -0300 Subject: [PATCH 4/6] Rename assertNormalizedEqual to assertNormalized --- cerberus/tests/__init__.py | 3 +-- cerberus/tests/tests.py | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/cerberus/tests/__init__.py b/cerberus/tests/__init__.py index 1011c562..9772a17b 100644 --- a/cerberus/tests/__init__.py +++ b/cerberus/tests/__init__.py @@ -236,8 +236,7 @@ def assertBadType(self, field, data_type, value): self.assertFail({field: value}) self.assertError(field, (field, 'type'), errors.BAD_TYPE, data_type) - def assertNormalizedEqual(self, document, expected, schema=None, - validator=None): + def assertNormalized(self, document, expected, schema=None, validator=None): if validator is None: validator = self.validator diff --git a/cerberus/tests/tests.py b/cerberus/tests/tests.py index 77f1d4fe..a3372ade 100644 --- a/cerberus/tests/tests.py +++ b/cerberus/tests/tests.py @@ -1322,14 +1322,14 @@ def test_coerce(self): schema = {'amount': {'coerce': int}} document = {'amount': '1'} expected = {'amount': 1} - self.assertNormalizedEqual(document, expected, schema) + self.assertNormalized(document, expected, schema) def test_coerce_in_subschema(self): schema = {'thing': {'type': 'dict', 'schema': {'amount': {'coerce': int}}}} document = {'thing': {'amount': '2'}} expected = {'thing': {'amount': 2}} - self.assertNormalizedEqual(document, expected, schema) + self.assertNormalized(document, expected, schema) def test_coerce_not_destructive(self): schema = { @@ -1361,19 +1361,19 @@ def test_coerce_unknown(self): schema = {'foo': {'schema': {}, 'allow_unknown': {'coerce': int}}} document = {'foo': {'bar': '0'}} expected = {'foo': {'bar': 0}} - self.assertNormalizedEqual(document, expected, schema) + self.assertNormalized(document, expected, schema) def test_normalized(self): schema = {'amount': {'coerce': int}} document = {'amount': '2'} expected = {'amount': 2} - self.assertNormalizedEqual(document, expected, schema) + self.assertNormalized(document, expected, schema) def test_rename(self): schema = {'foo': {'rename': 'bar'}} document = {'foo': 0} expected = {'bar': 0} - # We cannot use assertNormalizedEqual here since there is bug where + # We cannot use assertNormalized here since there is bug where # Cerberus says that the renamed field is an unknown field: # {'bar': 'unknown field'} self.validator(document, schema, False) @@ -1384,14 +1384,14 @@ def test_rename_handler(self): schema = {} document = {'0': 'foo'} expected = {0: 'foo'} - self.assertNormalizedEqual(document, expected, schema, validator) + self.assertNormalized(document, expected, schema, validator) def test_purge_unknown(self): validator = Validator(purge_unknown=True) schema = {'foo': {'type': 'string'}} document = {'bar': 'foo'} expected = {} - self.assertNormalizedEqual(document, expected, schema, validator) + self.assertNormalized(document, expected, schema, validator) def test_purge_unknown_in_subschema(self): schema = {'foo': {'type': 'dict', @@ -1399,7 +1399,7 @@ def test_purge_unknown_in_subschema(self): 'purge_unknown': True}} document = {'foo': {'bar': ''}} expected = {'foo': {}} - self.assertNormalizedEqual(document, expected, schema) + self.assertNormalized(document, expected, schema) def test_issue_147_complex(self): schema = {'revision': {'coerce': int}} @@ -1430,7 +1430,7 @@ def test_coerce_in_valueschema(self): 'type': 'integer'}}} document = {'thing': {'amount': '2'}} expected = {'thing': {'amount': 2}} - self.assertNormalizedEqual(document, expected, schema) + self.assertNormalized(document, expected, schema) def test_coerce_in_propertyschema(self): # https://github.com/nicolaiarocci/cerberus/issues/155 @@ -1439,7 +1439,7 @@ def test_coerce_in_propertyschema(self): 'type': 'integer'}}} document = {'thing': {'5': 'foo'}} expected = {'thing': {5: 'foo'}} - self.assertNormalizedEqual(document, expected, schema) + self.assertNormalized(document, expected, schema) def test_coercion_of_sequence_items(self): # https://github.com/nicolaiarocci/cerberus/issues/161 @@ -1447,7 +1447,7 @@ def test_coercion_of_sequence_items(self): 'coerce': float}}} document = {'a_list': [3, 4, 5]} expected = {'a_list': [3.0, 4.0, 5.0]} - self.assertNormalizedEqual(document, expected, schema) + self.assertNormalized(document, expected, schema) for x in self.validator.document['a_list']: self.assertIsInstance(x, float) @@ -1457,14 +1457,14 @@ def test_default_missing(self): 'default': 'bar_value'}} document = {'foo': 'foo_value'} expected = {'foo': 'foo_value', 'bar': 'bar_value'} - self.assertNormalizedEqual(document, expected, schema) + self.assertNormalized(document, expected, schema) def test_default_existent(self): schema = {'foo': {'type': 'string'}, 'bar': {'type': 'string', 'default': 'bar_value'}} document = {'foo': 'foo_value', 'bar': 'non_default'} - self.assertNormalizedEqual(document, document.copy(), schema) + self.assertNormalized(document, document.copy(), schema) def test_default_none(self): schema = {'foo': {'type': 'string'}, @@ -1472,7 +1472,7 @@ def test_default_none(self): 'nullable': True, 'default': 'bar_value'}} document = {'foo': 'foo_value', 'bar': None} - self.assertNormalizedEqual(document, document.copy(), schema) + self.assertNormalized(document, document.copy(), schema) def test_default_missing_in_subschema(self): schema = {'thing': {'type': 'dict', @@ -1482,7 +1482,7 @@ def test_default_missing_in_subschema(self): document = {'thing': {'foo': 'foo_value'}} expected = {'thing': {'foo': 'foo_value', 'bar': 'bar_value'}} - self.assertNormalizedEqual(document, expected, schema) + self.assertNormalized(document, expected, schema) class DefinitionSchema(TestBase): From 21f88c796942aed61790cb4ff35f37ecb3097a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dami=C3=A1n=20Nohales?= Date: Sun, 22 Nov 2015 15:05:39 -0300 Subject: [PATCH 5/6] Add default values before coercing --- cerberus/cerberus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cerberus/cerberus.py b/cerberus/cerberus.py index d1832a7f..a09cf054 100644 --- a/cerberus/cerberus.py +++ b/cerberus/cerberus.py @@ -453,8 +453,8 @@ def __normalize_mapping(self, mapping, schema): self.__normalize_rename_fields(mapping, schema) if self.purge_unknown: self._normalize_purge_unknown(mapping, schema) - self._normalize_coerce(mapping, schema) self._normalize_default(mapping, schema) + self._normalize_coerce(mapping, schema) self.__normalize_containers(mapping, schema) return mapping From 71ccc39b7cc7b373e7501edb911a56f77f76f781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dami=C3=A1n=20Nohales?= Date: Sun, 22 Nov 2015 15:26:28 -0300 Subject: [PATCH 6/6] Override values with default for non-nullable fields --- cerberus/cerberus.py | 5 ++++- cerberus/tests/tests.py | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cerberus/cerberus.py b/cerberus/cerberus.py index a09cf054..9b3c45e5 100644 --- a/cerberus/cerberus.py +++ b/cerberus/cerberus.py @@ -567,7 +567,10 @@ def _normalize_rename_handler(self, mapping, schema, field): def _normalize_default(self, mapping, schema): for field in tuple(schema): - if 'default' in schema[field] and field not in mapping: + nullable = schema[field].get('nullable', False) + if 'default' in schema[field] and \ + (field not in mapping or + mapping[field] is None and not nullable): mapping[field] = schema[field]['default'] # # Validating diff --git a/cerberus/tests/tests.py b/cerberus/tests/tests.py index a3372ade..2a3d0e18 100644 --- a/cerberus/tests/tests.py +++ b/cerberus/tests/tests.py @@ -1466,7 +1466,7 @@ def test_default_existent(self): document = {'foo': 'foo_value', 'bar': 'non_default'} self.assertNormalized(document, document.copy(), schema) - def test_default_none(self): + def test_default_none_nullable(self): schema = {'foo': {'type': 'string'}, 'bar': {'type': 'string', 'nullable': True, @@ -1474,6 +1474,14 @@ def test_default_none(self): document = {'foo': 'foo_value', 'bar': None} self.assertNormalized(document, document.copy(), schema) + def test_default_none_nonnullable(self): + schema = {'foo': {'type': 'string'}, + 'bar': {'type': 'string', + 'nullable': False, + 'default': 'bar_value'}} + document = {'foo': 'foo_value', 'bar': 'bar_value'} + self.assertNormalized(document, document.copy(), schema) + def test_default_missing_in_subschema(self): schema = {'thing': {'type': 'dict', 'schema': {'foo': {'type': 'string'},