From 12f63bc3689e2a15376906b3a32b2e9220f7e45b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Rocha?= Date: Wed, 7 Feb 2024 22:14:55 -0300 Subject: [PATCH] Change format of content of _validation field (#425) The content of _validation field must to be the string representation of a Python dict instead defaultdict when SPIDERMON_VALIDATION_ADD_ERRORS_TO_ITEMS setting is True. --- docs/source/getting-started.rst | 3 +-- docs/source/item-validation.rst | 2 +- spidermon/contrib/scrapy/pipelines.py | 5 ++++- tests/contrib/validation/test_item_validation_pipeline.py | 2 ++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/source/getting-started.rst b/docs/source/getting-started.rst index ec51455d..5ba4e446 100644 --- a/docs/source/getting-started.rst +++ b/docs/source/getting-started.rst @@ -478,8 +478,7 @@ The resulted item will look like this: .. code-block:: js { - '_validation': defaultdict( - , {'author_url': ['Invalid URL']}), + '_validation': {'author_url': ['Invalid URL']}, 'author': 'Mark Twain', 'author_url': 'not_a_valid_url', 'quote': 'Never tell the truth to people who are not worthy of it.', diff --git a/docs/source/item-validation.rst b/docs/source/item-validation.rst index 8f3a0606..a9b7019c 100644 --- a/docs/source/item-validation.rst +++ b/docs/source/item-validation.rst @@ -81,7 +81,7 @@ You can change the name of the field by assigning a name to `SPIDERMON_VALIDATIO .. code-block:: js { - '_validation': defaultdict(, {'author_url': ['Invalid URL']}), + '_validation': {'author_url': ['Invalid URL']}, 'author': 'C.S. Lewis', 'author_url': 'invalid_url', 'quote': 'Some day you will be old enough to start reading fairy tales ' diff --git a/spidermon/contrib/scrapy/pipelines.py b/spidermon/contrib/scrapy/pipelines.py index d86fcdc6..5611c375 100644 --- a/spidermon/contrib/scrapy/pipelines.py +++ b/spidermon/contrib/scrapy/pipelines.py @@ -129,13 +129,16 @@ def find_validators(self, item): def _add_errors_to_item(self, item: ItemAdapter, errors: Dict[str, str]): errors_field_instance = get_nested_attribute(item, self.errors_field) + if errors_field_instance is None: errors_field_instance = defaultdict(list) - set_nested_attribute(item, self.errors_field, errors_field_instance) for field_name, messages in errors.items(): errors_field_instance[field_name] += messages + # change defaultdict to dict for errors_field_instance + set_nested_attribute(item, self.errors_field, dict(errors_field_instance)) + def _drop_item(self, item, errors): """ This method drops the item after detecting validation errors. Note diff --git a/tests/contrib/validation/test_item_validation_pipeline.py b/tests/contrib/validation/test_item_validation_pipeline.py index 8190a7c0..f134d8e4 100644 --- a/tests/contrib/validation/test_item_validation_pipeline.py +++ b/tests/contrib/validation/test_item_validation_pipeline.py @@ -1,3 +1,4 @@ +from collections import defaultdict from dataclasses import dataclass import pytest @@ -87,6 +88,7 @@ def test_jsonschema_validation(dummy_schema): result_item = pipeline.process_item(dict(item), None) assert item != result_item assert "_validation" in result_item + assert not isinstance(result_item["_validation"], defaultdict) assert result_item["_validation"]["foo"] == ["Missing required field"]