Automatically transform dynamodb types

boto3/dynamodb/transform.py

@@ -0,0 +1,239 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import copy
+from collections import Mapping, MutableSequence
+
+from boto3.dynamodb.types import TypeSerializer, TypeDeserializer
+from boto3.dynamodb.conditions import ConditionBase
+from boto3.dynamodb.conditions import ConditionExpressionBuilder
+
+
+def register_high_level_interface(base_classes, **kwargs):
+    base_classes.insert(0, DynamoDBHighLevelResource)
+
+
+def copy_dynamodb_params(params, **kwargs):
+    return copy.deepcopy(params)
+
+
+class DynamoDBHighLevelResource(object):
+    def __init__(self, *args, **kwargs):
+        super(DynamoDBHighLevelResource, self).__init__(*args, **kwargs)
+
+        # Apply handler that creates a copy of the user provided dynamodb
+        # item such that it can be modified.
+        self.meta.client.meta.events.register(
+            'provide-client-params.dynamodb',
+            copy_dynamodb_params,
+            unique_id='dynamodb-create-params-copy'
+        )
+
+        self._injector = TransformationInjector()
+        # Apply the handler that generates condition expressions including
+        # placeholders.
+        self.meta.client.meta.events.register(
+            'before-parameter-build.dynamodb',
+            self._injector.inject_condition_expressions,
+            unique_id='dynamodb-condition-expression')
+
+        # Apply the handler that serializes the request from python
+        # types to dynamodb types.
+        self.meta.client.meta.events.register(
+            'before-parameter-build.dynamodb',
+            self._injector.inject_attribute_value_input,
+            unique_id='dynamodb-attr-value-input')
+
+        # Apply the handler that deserializes the response from dynamodb
+        # types to python types.
+        self.meta.client.meta.events.register(
+            'after-call.dynamodb',
+            self._injector.inject_attribute_value_output,
+            unique_id='dynamodb-attr-value-output')
+
+
+class TransformationInjector(object):
+    """Injects the transformations into the user provided parameters."""
+    def __init__(self, transformer=None, condition_builder=None,
+                 serializer=None, deserializer=None):
+        self._transformer = transformer
+        if transformer is None:
+            self._transformer = ParameterTransformer()
+
+        self._condition_builder = condition_builder
+        if condition_builder is None:
+            self._condition_builder = ConditionExpressionBuilder()
+
+        self._serializer = serializer
+        if serializer is None:
+            self._serializer = TypeSerializer()
+
+        self._deserializer = deserializer
+        if deserializer is None:
+            self._deserializer = TypeDeserializer()
+
+    def inject_condition_expressions(self, params, model, **kwargs):
+        """Injects the condition expression transformation into the parameters
+
+        This injection includes transformations for ConditionExpression shapes
+        and KeyExpression shapes. It also handles any placeholder names and
+        values that are generated when transforming the condition expressions.
+        """
+        self._condition_builder.reset()
+        generated_names = {}
+        generated_values = {}
+
+        # Create and apply the Condition Expression transformation.
+        transformation = ConditionExpressionTransformation(
+            self._condition_builder,
+            placeholder_names=generated_names,
+            placeholder_values=generated_values,
+            is_key_condition=False
+        )
+        self._transformer.transform(
+            params, model.input_shape, transformation,
+            'ConditionExpression')
+
+        # Create and apply the Key Condition Expression transformation.
+        transformation = ConditionExpressionTransformation(
+            self._condition_builder,
+            placeholder_names=generated_names,
+            placeholder_values=generated_values,
+            is_key_condition=True
+        )
+        self._transformer.transform(
+            params, model.input_shape, transformation,
+            'KeyExpression')
+
+        expr_attr_names_input = 'ExpressionAttributeNames'
+        expr_attr_values_input = 'ExpressionAttributeValues'
+
+        # Now that all of the condition expression transformation are done,
+        # update the placeholder dictionaries in the request.
+        if expr_attr_names_input in params:
+            params[expr_attr_names_input].update(generated_names)
+        else:
+            if generated_names:
+                params[expr_attr_names_input] = generated_names
+
+        if expr_attr_values_input in params:
+            params[expr_attr_values_input].update(generated_values)
+        else:
+            if generated_values:
+                params[expr_attr_values_input] = generated_values
+
+    def inject_attribute_value_input(self, params, model, **kwargs):
+        """Injects DynamoDB serialization into parameter input"""
+        self._transformer.transform(
+            params, model.input_shape, self._serializer.serialize,
+            'AttributeValue')
+
+    def inject_attribute_value_output(self, parsed, model, **kwargs):
+        """Injects DynamoDB deserialization into responses"""
+        self._transformer.transform(
+            parsed, model.output_shape, self._deserializer.deserialize,
+            'AttributeValue')
+
+
+class ConditionExpressionTransformation(object):
+    """Provides a transformation for condition expressions
+
+    The ``ParameterTransformer`` class can call this class directly
+    to transform the condition expressions in the parameters provided.
+    """
+    def __init__(self, condition_builder, placeholder_names,
+                 placeholder_values, is_key_condition=False):
+        self._condition_builder = condition_builder
+        self._placeholder_names = placeholder_names
+        self._placeholder_values = placeholder_values
+        self._is_key_condition = is_key_condition
+
+    def __call__(self, value):
+        if isinstance(value, ConditionBase):
+            # Create a conditional expression string with placeholders
+            # for the provided condition.
+            built_expression = self._condition_builder.build_expression(
+                value, is_key_condition=self._is_key_condition)
+
+            self._placeholder_names.update(
+                built_expression.attribute_name_placeholders)
+            self._placeholder_values.update(
+                built_expression.attribute_value_placeholders)
+
+            return built_expression.condition_expression
+        # Use the user provided value if it is not a ConditonBase object.
+        return value
+
+
+class ParameterTransformer(object):
+    """Transforms the input to and output from botocore based on shape"""
+
+    def transform(self, params, model, transformation, target_shape):
+        """Transforms the dynamodb input to or output from botocore
+
+        It applies a specified transformation whenever a specific shape name
+        is encountered while traversing the parameters in the dictionary.
+
+        :param params: The parameters structure to transform.
+        :param model: The operation model.
+        :param transformation: The function to apply the parameter
+        :param target_shape: The name of the shape to apply the
+            transformation to
+        """
+        self._transform_parameters(
+            model, params, transformation, target_shape)
+
+    def _transform_parameters(self, model, params, transformation,
+                              target_shape):
+        type_name = model.type_name
+        if type_name in ['structure', 'map', 'list']:
+            getattr(self, '_transform_%s' % type_name)(
+                model, params, transformation, target_shape)
+
+    def _transform_structure(self, model, params, transformation,
+                             target_shape):
+        if not isinstance(params, Mapping):
+            return
+        for param in params:
+            if param in model.members:
+                member_model = model.members[param]
+                member_shape = member_model.name
+                if member_shape == target_shape:
+                    params[param] = transformation(params[param])
+                else:
+                    self._transform_parameters(
+                        member_model, params[param], transformation,
+                        target_shape)
+
+    def _transform_map(self, model, params, transformation, target_shape):
+        if not isinstance(params, Mapping):
+            return
+        value_model = model.value
+        value_shape = value_model.name
+        for key, value in params.items():
+            if value_shape == target_shape:
+                params[key] = transformation(value)
+            else:
+                self._transform_parameters(
+                    value_model, params[key], transformation, target_shape)
+
+    def _transform_list(self, model, params, transformation, target_shape):
+        if not isinstance(params, MutableSequence):
+            return
+        member_model = model.member
+        member_shape = member_model.name
+        for i, item in enumerate(params):
+            if member_shape == target_shape:
+                params[i] = transformation(item)
+            else:
+                self._transform_parameters(
+                    member_model, params[i], transformation, target_shape)

boto3/session.py

@@ -270,3 +270,8 @@ def _register_default_handlers(self):
             'creating-client-class.s3',
             boto3.utils.lazy_call(
                 'boto3.s3.inject.inject_s3_transfer_methods'))
+        self._session.register(
+            'creating-resource-class.dynamodb',
+            boto3.utils.lazy_call(
+                'boto3.dynamodb.transform.register_high_level_interface'),
+            unique_id='high-level-dynamodb')

tests/functional/test_dynamodb.py

@@ -0,0 +1,72 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import json
+from tests import unittest, mock
+
+from botocore.vendored import requests
+
+from boto3.session import Session
+from boto3.dynamodb.conditions import Attr
+
+
+class TestDynamoDB(unittest.TestCase):
+    def setUp(self):
+        self.http_response = requests.models.Response()
+        self.http_response.status_code = 200
+        self.parsed_response = {}
+        self.make_request_patch = mock.patch(
+            'botocore.endpoint.Endpoint.make_request')
+        self.make_request_mock = self.make_request_patch.start()
+        self.make_request_mock.return_value = (
+            self.http_response, self.parsed_response)
+        self.session = Session(
+            aws_access_key_id='dummy',
+            aws_secret_access_key='dummy',
+            region_name='us-east-1')
+
+    def tearDown(self):
+        self.make_request_patch.stop()
+
+    def test_resource(self):
+        dynamodb = self.session.resource('dynamodb')
+        table = dynamodb.Table('MyTable')
+        # Make sure it uses the high level interface
+        table.scan(FilterExpression=Attr('mykey').eq('myvalue'))
+        request = self.make_request_mock.call_args_list[0][0][1]
+        request_params = json.loads(request['body'])
+        self.assertEqual(
+            request_params,
+            {'TableName': 'MyTable',
+             'FilterExpression': '#n0 = :v0',
+             'ExpressionAttributeNames': {'#n0': 'mykey'},
+             'ExpressionAttributeValues': {':v0': {'S': 'myvalue'}}}
+        )
+
+    def test_client(self):
+        dynamodb = self.session.client('dynamodb')
+        # Make sure the client still uses the botocore level interface
+        dynamodb.scan(
+            TableName='MyTable',
+            FilterExpression='#n0 = :v0',
+            ExpressionAttributeNames={'#n0': 'mykey'},
+            ExpressionAttributeValues={':v0': {'S': 'myvalue'}}
+        )
+        request = self.make_request_mock.call_args_list[0][0][1]
+        request_params = json.loads(request['body'])
+        self.assertEqual(
+            request_params,
+            {'TableName': 'MyTable',
+             'FilterExpression': '#n0 = :v0',
+             'ExpressionAttributeNames': {'#n0': 'mykey'},
+             'ExpressionAttributeValues': {':v0': {'S': 'myvalue'}}}
+        )

tests/integration/__init__.py

@@ -0,0 +1,12 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.