diff --git a/docs/serialization.rst b/docs/serialization.rst index c5a2ae0d79..0683f31a0f 100644 --- a/docs/serialization.rst +++ b/docs/serialization.rst @@ -11,21 +11,27 @@ chapter. Hazelcast serializes all your objects before sending them to the server. The ``bool``, ``int``, ``long`` (for Python 2), ``float``, ``str``, -``unicode`` (for Python 2) and ``bytearray`` types are serialized -natively and you cannot override this behavior. The following table is -the conversion of types for the Java server side. - -========= ====================================== -Python Java -========= ====================================== -bool Boolean -int Byte, Short, Integer, Long, BigInteger -long Byte, Short, Integer, Long, BigInteger -float Float, Double -str String -unicode String -bytearray byte[] -========= ====================================== +``unicode`` (for Python 2), ``bytearray``, ``list`` ``datetime.date``, +``datetime.time``, ``datetime.datetime``, and ``decimal.Decimal`` types are +serialized natively and you cannot override this behavior. The following +table is the conversion of types for the Java server side. + +================= ================================================ +Python Java +================= ================================================ +bool Boolean +int Byte, Short, Integer, Long, java.math.BigInteger +long Byte, Short, Integer, Long, java.math.BigInteger +float Float, Double +str String +unicode String +bytearray byte[] +list java.util.ArrayList +datetime.date java.time.LocalDate +datetime.time java.time.LocalTime +datetime.datetime java.time.OffsetDateTime +decimal.Decimal java.math.BigDecimal +================= ================================================ .. Note:: A ``int`` or ``long`` type is serialized as ``Integer`` by diff --git a/hazelcast/serialization/serialization_const.py b/hazelcast/serialization/serialization_const.py index ba34f25cfe..56c5321f0c 100644 --- a/hazelcast/serialization/serialization_const.py +++ b/hazelcast/serialization/serialization_const.py @@ -29,11 +29,15 @@ # DEFAULT SERIALIZERS JAVA_DEFAULT_TYPE_CLASS = -24 -JAVA_DEFAULT_TYPE_DATE = -25 JAVA_DEFAULT_TYPE_BIG_INTEGER = -26 +JAVA_DEFAULT_TYPE_BIG_DECIMAL = -27 JAVA_DEFAULT_TYPE_ARRAY = -28 JAVA_DEFAULT_TYPE_ARRAY_LIST = -29 JAVA_DEFAULT_TYPE_LINKED_LIST = -30 +JAVA_DEFAULT_TYPE_LOCAL_DATE = -51 +JAVA_DEFAULT_TYPE_LOCAL_TIME = -52 +JAVA_DEFAULT_TYPE_LOCAL_DATE_TIME = -53 +JAVA_DEFAULT_TYPE_OFFSET_DATE_TIME = -54 JAVASCRIPT_JSON_SERIALIZATION_TYPE = -130 # ------------------------------------------------------------ diff --git a/hazelcast/serialization/serializer.py b/hazelcast/serialization/serializer.py index c55e5f19a6..a92f29646e 100644 --- a/hazelcast/serialization/serializer.py +++ b/hazelcast/serialization/serializer.py @@ -1,6 +1,5 @@ -import binascii -import time -from datetime import datetime +import datetime +import decimal from hazelcast import six from hazelcast.core import HazelcastJsonValue @@ -9,7 +8,7 @@ from hazelcast.serialization.base import HazelcastSerializationError from hazelcast.serialization.serialization_const import * from hazelcast.six.moves import range, cPickle -from hazelcast.util import UUIDUtil +from hazelcast.util import UUIDUtil, int_from_bytes, int_to_bytes, timezone if not six.PY2: long = int @@ -248,51 +247,42 @@ def get_type_id(self): # EXTENSIONS -class DateTimeSerializer(BaseSerializer): +class BigIntegerSerializer(BaseSerializer): def read(self, inp): - long_time = inp.read_long() - return datetime.fromtimestamp(long_time / 1000.0) + length = inp.read_int() + result = bytearray(length) + inp.read_into(result, 0, length) + return int_from_bytes(result) def write(self, out, obj): - long_time = long(time.mktime(obj.timetuple())) * 1000 - out.write_long(long_time) + out.write_byte_array(int_to_bytes(obj)) def get_type_id(self): - return JAVA_DEFAULT_TYPE_DATE + return JAVA_DEFAULT_TYPE_BIG_INTEGER -class BigIntegerSerializer(BaseSerializer): +class BigDecimalSerializer(BaseSerializer): def read(self, inp): length = inp.read_int() - if length == NULL_ARRAY_LENGTH: - return None result = bytearray(length) - if length > 0: - inp.read_into(result, 0, length) - if result[0] & 0x80: - neg = bytearray() - for c in result: - neg.append(c ^ 0xFF) - return -1 * int(binascii.hexlify(neg), 16) - 1 - return int(binascii.hexlify(result), 16) + inp.read_into(result, 0, length) + unscaled_value = int_from_bytes(result) + scale = inp.read_int() + sign = 0 if unscaled_value >= 0 else 1 + return decimal.Decimal( + (sign, tuple(int(digit) for digit in str(abs(unscaled_value))), -1 * scale) + ) def write(self, out, obj): - the_big_int = -obj - 1 if obj < 0 else obj - end_index = -1 if (type(obj) == long and six.PY2) else None - hex_str = hex(the_big_int)[2:end_index] - if len(hex_str) % 2 == 1: - prefix = "0" # "f" if obj < 0 else "0" - hex_str = prefix + hex_str - num_array = bytearray(binascii.unhexlify(bytearray(hex_str, encoding="utf-8"))) - if obj < 0: - neg = bytearray() - for c in num_array: - neg.append(c ^ 0xFF) - num_array = neg - out.write_byte_array(num_array) + sign, digits, exponent = obj.as_tuple() + unscaled_value = long("".join([str(digit) for digit in digits])) + if sign == 1: + unscaled_value = -1 * unscaled_value + out.write_byte_array(int_to_bytes(unscaled_value)) + out.write_int(-1 * exponent) def get_type_id(self): - return JAVA_DEFAULT_TYPE_BIG_INTEGER + return JAVA_DEFAULT_TYPE_BIG_DECIMAL class JavaClassSerializer(BaseSerializer): @@ -346,6 +336,97 @@ def get_type_id(self): return JAVA_DEFAULT_TYPE_LINKED_LIST +class LocalDateSerializer(BaseSerializer): + def read(self, inp): + return datetime.date( + inp.read_int(), + inp.read_byte(), + inp.read_byte(), + ) + + def write(self, out, obj): + out.write_int(obj.year) + out.write_byte(obj.month) + out.write_byte(obj.day) + + def get_type_id(self): + return JAVA_DEFAULT_TYPE_LOCAL_DATE + + +class LocalTimeSerializer(BaseSerializer): + def read(self, inp): + return datetime.time( + inp.read_byte(), + inp.read_byte(), + inp.read_byte(), + inp.read_int() // 1000, # server sends nanoseconds + ) + + def write(self, out, obj): + out.write_byte(obj.hour) + out.write_byte(obj.minute) + out.write_byte(obj.second) + out.write_int(obj.microsecond * 1000) # server expects nanoseconds + + def get_type_id(self): + return JAVA_DEFAULT_TYPE_LOCAL_TIME + + +class LocalDateTimeSerializer(BaseSerializer): + def read(self, inp): + return datetime.datetime( + inp.read_int(), + inp.read_byte(), + inp.read_byte(), + inp.read_byte(), + inp.read_byte(), + inp.read_byte(), + inp.read_int() // 1000, # server sends nanoseconds + ) + + # "write(self, out, obj)" is never called so not implemented here + + def get_type_id(self): + return JAVA_DEFAULT_TYPE_LOCAL_DATE_TIME + + +class OffsetDateTimeSerializer(BaseSerializer): + def read(self, inp): + return datetime.datetime( + inp.read_int(), + inp.read_byte(), + inp.read_byte(), + inp.read_byte(), + inp.read_byte(), + inp.read_byte(), + inp.read_int() // 1000, # server sends nanoseconds + timezone(datetime.timedelta(seconds=inp.read_int())), + ) + + def write(self, out, obj): + out.write_int(obj.year) + out.write_byte(obj.month) + out.write_byte(obj.day) + out.write_byte(obj.hour) + out.write_byte(obj.minute) + out.write_byte(obj.second) + out.write_int(obj.microsecond * 1000) # server expects nanoseconds + + timezone_info = obj.tzinfo + if not timezone_info: + out.write_int(0) + return + + utc_offset = timezone_info.utcoffset(None) + if utc_offset: + out.write_int(int(utc_offset.total_seconds())) + else: + out.write_int(0) + + def get_type_id(self): + return JAVA_DEFAULT_TYPE_OFFSET_DATE_TIME + + class PythonObjectSerializer(BaseSerializer): def read(self, inp): str = inp.read_string().encode() diff --git a/hazelcast/serialization/service.py b/hazelcast/serialization/service.py index ceea44493e..92c74b4838 100644 --- a/hazelcast/serialization/service.py +++ b/hazelcast/serialization/service.py @@ -1,3 +1,5 @@ +import datetime +import decimal import uuid from hazelcast import six @@ -101,12 +103,16 @@ def _register_constant_serializers(self): self._registry.register_constant_serializer(DoubleArraySerializer()) self._registry.register_constant_serializer(StringArraySerializer()) # EXTENSIONS - self._registry.register_constant_serializer(DateTimeSerializer(), datetime) self._registry.register_constant_serializer(BigIntegerSerializer()) + self._registry.register_constant_serializer(BigDecimalSerializer(), decimal.Decimal) self._registry.register_constant_serializer(JavaClassSerializer()) self._registry.register_constant_serializer(ArraySerializer()) self._registry.register_constant_serializer(ArrayListSerializer(), list) self._registry.register_constant_serializer(LinkedListSerializer()) + self._registry.register_constant_serializer(LocalDateSerializer(), datetime.date) + self._registry.register_constant_serializer(LocalTimeSerializer(), datetime.time) + self._registry.register_constant_serializer(LocalDateTimeSerializer()) + self._registry.register_constant_serializer(OffsetDateTimeSerializer(), datetime.datetime) self._registry.register_constant_serializer( HazelcastJsonValueSerializer(), HazelcastJsonValue ) diff --git a/hazelcast/util.py b/hazelcast/util.py index 903fc2921b..c43ecb0d25 100644 --- a/hazelcast/util.py +++ b/hazelcast/util.py @@ -359,20 +359,44 @@ def from_bits(most_significant_bits, least_significant_bits): if hasattr(int, "from_bytes"): - def int_from_bytes(buffer): - return int.from_bytes(buffer, "big", signed=True) + def int_from_bytes(buf): + return int.from_bytes(buf, "big", signed=True) else: # Compatibility with Python 2 - def int_from_bytes(buffer): - buffer = bytearray(buffer) - if buffer[0] & 0x80: + def int_from_bytes(buf): + buf = bytearray(buf) + if buf[0] & 0x80: neg = bytearray() - for c in buffer: - neg.append(~c) + for c in buf: + neg.append(c ^ 0xFF) return -1 * int(binascii.hexlify(neg), 16) - 1 - return int(binascii.hexlify(buffer), 16) + return int(binascii.hexlify(buf), 16) + + +if hasattr(int, "to_bytes"): + + def int_to_bytes(number): + # number of bytes to represent the number + width = (8 + (number + (number < 0)).bit_length()) // 8 + return number.to_bytes(length=width, byteorder="big", signed=True) + + +else: + # Compatibility with Python 2 + def int_to_bytes(number): + is_neg = number < 0 + number = -number - 1 if is_neg else number + # number of bytes to represent the number * 2, so that + # each byte is represented with 2 digit hex numbers. + width = ((8 + number.bit_length()) // 8) * 2 + fmt = "%%0%dx" % width + buf = bytearray(binascii.unhexlify(fmt % number)) + if is_neg: + for i in range(len(buf)): + buf[i] = buf[i] ^ 0xFF + return buf try: @@ -398,6 +422,15 @@ def tzname(self, dt): def dst(self, dt): return timedelta(0) + def __eq__(self, other): + return isinstance(other, FixedOffsetTimezone) and self._offset == other._offset + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self._offset) + timezone = FixedOffsetTimezone diff --git a/tests/integration/backward_compatible/serialization/serializers_test.py b/tests/integration/backward_compatible/serialization/serializers_test.py index 5620dba5ca..dfa8c46f41 100644 --- a/tests/integration/backward_compatible/serialization/serializers_test.py +++ b/tests/integration/backward_compatible/serialization/serializers_test.py @@ -1,5 +1,6 @@ # coding=utf-8 import datetime +import decimal import uuid from hazelcast import six, HazelcastClient @@ -8,7 +9,18 @@ from hazelcast.serialization import MAX_BYTE, MAX_SHORT, MAX_INT, MAX_LONG from tests.base import SingleMemberTestCase from tests.hzrc.ttypes import Lang -from tests.util import random_string +from tests.util import ( + random_string, + mark_client_version_at_most, + mark_client_version_at_least, + mark_server_version_at_least, +) + +try: + from hazelcast.util import timezone +except ImportError: + # Added in 4.2 version of the client + pass if not six.PY2: long = int @@ -157,6 +169,7 @@ def test_bytearray(self): self.assertEqual(value, response) def test_datetime(self): + mark_client_version_at_most(self, "5.0") value = datetime.datetime.now() self.map.set("key", value) self.assertEqual(value.timetuple(), self.map.get("key").timetuple()) @@ -203,6 +216,15 @@ def test_variable_integer(self): response = long(self.get_from_server()) self.assertEqual(value, response) + def test_decimal(self): + mark_client_version_at_least(self, "5.0") + decimal_value = "1234567890123456789012345678901234567890.987654321" + value = decimal.Decimal(decimal_value) + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = self.get_from_server() + self.assertEqual(decimal_value, response) + def test_list(self): value = [1, 2, 3] self.map.set("key", value) @@ -210,6 +232,33 @@ def test_list(self): response = self.get_from_server() self.assertEqual(value, list(map(int, response[1:-1].split(", ")))) + def test_datetime_date(self): + mark_client_version_at_least(self, "5.0") + mark_server_version_at_least(self, self.client, "5.0") + value = datetime.datetime.now().date() + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = self.get_from_server() + self.assertEqual(response, value.strftime("%Y-%m-%d")) + + def test_datetime_time(self): + mark_client_version_at_least(self, "5.0") + mark_server_version_at_least(self, self.client, "5.0") + value = datetime.datetime.now().time() + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = self.get_from_server() + self.assertEqual(response, value.strftime("%H:%M:%S.%f")) + + def test_datetime_datetime(self): + mark_client_version_at_least(self, "5.0") + mark_server_version_at_least(self, self.client, "5.0") + value = datetime.datetime.now(timezone(datetime.timedelta(seconds=1800))) + self.map.set("key", value) + self.assertEqual(value, self.map.get("key")) + response = self.get_from_server() + self.assertEqual(response, value.strftime("%Y-%m-%dT%H:%M:%S.%f+00:30")) + def test_bool_from_server(self): self.assertTrue(self.set_on_server("true")) self.assertEqual(True, self.map.get("key")) @@ -297,6 +346,7 @@ def test_string_array_from_server(self): self.assertEqual(["hey", six.u("1βšδΈ­πŸ’¦2πŸ˜­β€πŸ™†πŸ˜”5")], self.map.get("key")) def test_date_from_server(self): + mark_client_version_at_most(self, "5.0") self.assertTrue(self.set_on_server("new java.util.Date(100, 11, 15, 23, 59, 49)")) # server adds 1900 to year. Also, month is 0-based for server and 1-based for the client self.assertEqual(datetime.datetime(2000, 12, 15, 23, 59, 49), self.map.get("key")) @@ -322,6 +372,34 @@ def test_big_integer_from_server(self): ) self.assertEqual(-1234567890123456789012345678901234567890, self.map.get("key")) + def test_big_decimal_from_server(self): + mark_client_version_at_least(self, "5.0") + self.assertTrue(self.set_on_server('new java.math.BigDecimal("12.12")')) + self.assertEqual(decimal.Decimal("12.12"), self.map.get("key")) + + self.assertTrue(self.set_on_server('new java.math.BigDecimal("-13.13")')) + self.assertEqual(decimal.Decimal("-13.13"), self.map.get("key")) + + self.assertTrue( + self.set_on_server( + 'new java.math.BigDecimal("1234567890123456789012345678901234567890.123456789")' + ) + ) + self.assertEqual( + decimal.Decimal("1234567890123456789012345678901234567890.123456789"), + self.map.get("key"), + ) + + self.assertTrue( + self.set_on_server( + 'new java.math.BigDecimal("-1234567890123456789012345678901234567890.123456789")' + ) + ) + self.assertEqual( + decimal.Decimal("-1234567890123456789012345678901234567890.123456789"), + self.map.get("key"), + ) + def test_java_class_from_server(self): self.assertTrue(self.set_on_server("java.lang.String.class")) self.assertEqual("java.lang.String", self.map.get("key")) @@ -355,3 +433,39 @@ def test_linked_list_from_server(self): response = self.rc.executeOnController(self.cluster.id, script, Lang.JAVASCRIPT) self.assertTrue(response.success) self.assertEqual(["a", "b", "c"], self.map.get("key")) + + def test_local_date_from_server(self): + mark_client_version_at_least(self, "5.0") + mark_server_version_at_least(self, self.client, "5.0") + self.assertTrue(self.set_on_server("java.time.LocalDate.of(2000, 12, 15)")) + self.assertEqual(datetime.date(2000, 12, 15), self.map.get("key")) + + def test_local_time_from_server(self): + mark_client_version_at_least(self, "5.0") + mark_server_version_at_least(self, self.client, "5.0") + self.assertTrue(self.set_on_server("java.time.LocalTime.of(18, 3, 35)")) + self.assertEqual(datetime.time(18, 3, 35), self.map.get("key")) + + def test_local_date_time_from_server(self): + mark_client_version_at_least(self, "5.0") + mark_server_version_at_least(self, self.client, "5.0") + self.assertTrue( + self.set_on_server("java.time.LocalDateTime.of(2021, 8, 24, 0, 59, 55, 987654000)") + ) + self.assertEqual(datetime.datetime(2021, 8, 24, 0, 59, 55, 987654), self.map.get("key")) + + def test_offset_date_time_from_server(self): + mark_client_version_at_least(self, "5.0") + mark_server_version_at_least(self, self.client, "5.0") + self.assertTrue( + self.set_on_server( + "java.time.OffsetDateTime.of(2021, 8, 24, 0, 59, 55, 987654000, " + "java.time.ZoneOffset.ofTotalSeconds(2400))" + ) + ) + self.assertEqual( + datetime.datetime( + 2021, 8, 24, 0, 59, 55, 987654, timezone(datetime.timedelta(seconds=2400)) + ), + self.map.get("key"), + ) diff --git a/tests/unit/serialization/binary_compatibility/1.serialization.compatibility.binary b/tests/unit/serialization/binary_compatibility/1.serialization.compatibility.binary index 19789c6fe7..ca1da0dfe4 100644 Binary files a/tests/unit/serialization/binary_compatibility/1.serialization.compatibility.binary and b/tests/unit/serialization/binary_compatibility/1.serialization.compatibility.binary differ diff --git a/tests/unit/serialization/binary_compatibility/README.md b/tests/unit/serialization/binary_compatibility/README.md index 5e9e79722f..44d5e1faf8 100644 --- a/tests/unit/serialization/binary_compatibility/README.md +++ b/tests/unit/serialization/binary_compatibility/README.md @@ -7,6 +7,7 @@ However, we had to edit ``ReferenceObjects`` while generating the file because t such serializers for the Python client yet. The list of skipped items from ``ReferenceObjects`` are shown below. +* aDate * aSimpleMapEntry * aSimpleImmutableMapEntry * aBigDecimal @@ -33,8 +34,6 @@ are shown below. * synchronousQueue * linkedTransferQueue * partitionPredicate -* Aggregators (all of them) -* Projections (all of them) * Part of the linkedList because it contain items listed above * Part of the arrayList because it contain items listed above and below. Apart from these, we also removed number types other than anInt, because in the Python client we can specify a @@ -54,6 +53,7 @@ below. * ints * longs * strings +* aLocalDateTime * aClass * linkedList @@ -70,17 +70,23 @@ provided at the end of this file for future usages. ```java package com.hazelcast.nio.serialization.compatibility; +import com.hazelcast.aggregation.Aggregators; import com.hazelcast.internal.serialization.impl.HeapData; import com.hazelcast.internal.serialization.Data; import com.hazelcast.nio.serialization.Portable; +import com.hazelcast.projection.Projections; import com.hazelcast.query.Predicates; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.CharBuffer; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Calendar; -import java.util.Date; import java.util.LinkedList; import java.util.UUID; @@ -168,23 +174,31 @@ class ReferenceObjects { aCustomStreamSerializable, aCustomByteArraySerializable, aData); - static Date aDate; + static LocalDate aLocalDate; + static LocalTime aLocalTime; + static LocalDateTime aLocalDateTime; + static OffsetDateTime anOffsetDateTime; static { Calendar calendar = Calendar.getInstance(); calendar.set(1990, Calendar.FEBRUARY, 1, 0, 0, 0); calendar.set(Calendar.MILLISECOND, 0); calendar.set(Calendar.ZONE_OFFSET, 0); - aDate = calendar.getTime(); + aLocalDate = LocalDate.of(2021, 6, 28); + aLocalTime = LocalTime.of(11, 22, 41, 123456000); + aLocalDateTime = LocalDateTime.of(aLocalDate, aLocalTime); + anOffsetDateTime = OffsetDateTime.of(aLocalDateTime, ZoneOffset.ofHours(18)); } static BigInteger aBigInteger = new BigInteger("1314432323232411"); + static BigDecimal aBigDecimal = new BigDecimal("31231.12331"); static Class aClass = BigDecimal.class; static ArrayList nonNullList = new ArrayList(asList( aBoolean, aDouble, anInt, anSqlString, anInnerPortable, bytes, aCustomStreamSerializable, aCustomByteArraySerializable, - anIdentifiedDataSerializable, aPortable, aDate)); + anIdentifiedDataSerializable, aPortable, + aBigDecimal, aLocalDate, aLocalTime, anOffsetDateTime)); static ArrayList arrayList = new ArrayList(asList(aNullObject, nonNullList)); @@ -195,7 +209,7 @@ class ReferenceObjects { booleans, bytes, chars, doubles, shorts, floats, ints, longs, strings, aCustomStreamSerializable, aCustomByteArraySerializable, anIdentifiedDataSerializable, aPortable, - aDate, aBigInteger, aClass, + aLocalDate, aLocalTime, aLocalDateTime, anOffsetDateTime, aBigInteger, aBigDecimal, aClass, arrayList, linkedList, // predicates @@ -221,6 +235,30 @@ class ReferenceObjects { Predicates.greaterThan(anSqlString, anInt), Predicates.greaterEqual(anSqlString, anInt)), Predicates.instanceOf(aCustomStreamSerializable.getClass()), + + // Aggregators + Aggregators.distinct(anSqlString), + Aggregators.integerMax(anSqlString), + Aggregators.maxBy(anSqlString), + Aggregators.comparableMin(anSqlString), + Aggregators.minBy(anSqlString), + Aggregators.count(anSqlString), + Aggregators.numberAvg(anSqlString), + Aggregators.integerAvg(anSqlString), + Aggregators.longAvg(anSqlString), + Aggregators.doubleAvg(anSqlString), + Aggregators.integerSum(anSqlString), + Aggregators.longSum(anSqlString), + Aggregators.doubleSum(anSqlString), + Aggregators.fixedPointSum(anSqlString), + Aggregators.floatingPointSum(anSqlString), + Aggregators.bigDecimalSum(anSqlString), + + // projections + Projections.singleAttribute(anSqlString), + Projections.multiAttribute(anSqlString, anSqlString, anSqlString), + Projections.identity() + }; } ``` \ No newline at end of file diff --git a/tests/unit/serialization/binary_compatibility/reference_objects.py b/tests/unit/serialization/binary_compatibility/reference_objects.py index bad5503c34..85576c93c1 100644 --- a/tests/unit/serialization/binary_compatibility/reference_objects.py +++ b/tests/unit/serialization/binary_compatibility/reference_objects.py @@ -1,13 +1,14 @@ # coding=utf-8 import datetime +import decimal import re import uuid -from hazelcast import predicate +from hazelcast import predicate, aggregator, projection from hazelcast import six from hazelcast.serialization.api import Portable, IdentifiedDataSerializable from hazelcast.serialization.data import Data -from hazelcast.util import to_signed +from hazelcast.util import to_signed, timezone IDENTIFIED_DATA_SERIALIZABLE_FACTORY_ID = 1 PORTABLE_FACTORY_ID = 1 @@ -681,8 +682,14 @@ def __eq__(self, other): ], ) ), - "Date": datetime.datetime.fromtimestamp(633830400), + "LocalDate": datetime.date(2021, 6, 28), + "LocalTime": datetime.time(11, 22, 41, 123456), + "LocalDateTime": datetime.datetime(2021, 6, 28, 11, 22, 41, 123456), + "OffsetDateTime": datetime.datetime( + 2021, 6, 28, 11, 22, 41, 123456, timezone(datetime.timedelta(hours=18)) + ), "BigInteger": 1314432323232411, + "BigDecimal": decimal.Decimal("31231.12331"), "Class": _to_unicode("java.math.BigDecimal"), } @@ -764,7 +771,10 @@ def __eq__(self, other): _custom_byte_array_serializable, _identified, _portable, - REFERENCE_OBJECTS["Date"], + REFERENCE_OBJECTS["BigDecimal"], + REFERENCE_OBJECTS["LocalDate"], + REFERENCE_OBJECTS["LocalTime"], + REFERENCE_OBJECTS["OffsetDateTime"], ] REFERENCE_OBJECTS.update( @@ -808,6 +818,26 @@ def __eq__(self, other): "InstanceOfPredicate": predicate.instance_of( "com.hazelcast.nio.serialization.compatibility.CustomStreamSerializable" ), + "DistinctValuesAggregator": aggregator.distinct(_sql_string), + "MaxAggregator": aggregator.max_(_sql_string), + "MaxByAggregator": aggregator.max_by(_sql_string), + "MinAggregator": aggregator.min_(_sql_string), + "MinByAggregator": aggregator.min_by(_sql_string), + "CountAggregator": aggregator.count(_sql_string), + "NumberAverageAggregator": aggregator.number_avg(_sql_string), + "IntegerAverageAggregator": aggregator.int_avg(_sql_string), + "LongAverageAggregator": aggregator.long_avg(_sql_string), + "DoubleAverageAggregator": aggregator.double_avg(_sql_string), + "IntegerSumAggregator": aggregator.int_sum(_sql_string), + "LongSumAggregator": aggregator.long_sum(_sql_string), + "DoubleSumAggregator": aggregator.double_sum(_sql_string), + "FixedSumAggregator": aggregator.fixed_point_sum(_sql_string), + "FloatingPointSumAggregator": aggregator.floating_point_sum(_sql_string), + "SingleAttributeProjection": projection.single_attribute(_sql_string), + "MultiAttributeProjection": projection.multi_attribute( + _sql_string, _sql_string, _sql_string + ), + "IdentityProjection": projection.identity(), } ) @@ -823,6 +853,7 @@ def __eq__(self, other): "long[]", "String[]", "Class", + "LocalDateTime", "LinkedList", } @@ -831,7 +862,7 @@ def skip_on_serialize(object_type): return object_type in _SKIP_ON_SERIALIZE -_SKIP_ON_DESERIALIZE_PATTERN = re.compile(r"^.*Predicate$") +_SKIP_ON_DESERIALIZE_PATTERN = re.compile(r"^.*(Predicate|Aggregator|Projection)$") def skip_on_deserialize(object_type): diff --git a/tests/unit/serialization/serializers_test.py b/tests/unit/serialization/serializers_test.py index 183029a6a1..4ebbf6bdbe 100644 --- a/tests/unit/serialization/serializers_test.py +++ b/tests/unit/serialization/serializers_test.py @@ -1,5 +1,6 @@ # coding=utf-8 import datetime +import decimal import unittest import uuid @@ -10,6 +11,7 @@ from hazelcast.serialization import BE_INT from hazelcast.predicate import * from hazelcast.serialization.service import SerializationServiceV1 +from hazelcast.util import timezone class A(object): @@ -65,11 +67,29 @@ def test_hazelcast_json_value(self): def test_uuid(self): self.validate(uuid.uuid4()) - def test_datetime(self): - d = datetime.datetime.now() + def test_datetime_datetime(self): + d = datetime.datetime.now(tz=timezone(datetime.timedelta(hours=-15))) serialized = self.service.to_data(d) deserialized = self.service.to_object(serialized) - self.assertEqual(d.timetuple(), deserialized.timetuple()) + self.assertEqual(d, deserialized) + + def test_datetime_date(self): + d = datetime.datetime.now().date() + serialized = self.service.to_data(d) + deserialized = self.service.to_object(serialized) + self.assertEqual(d, deserialized) + + def test_datetime_time(self): + d = datetime.datetime.now().time() + serialized = self.service.to_data(d) + deserialized = self.service.to_object(serialized) + self.assertEqual(d, deserialized) + + def test_decimal(self): + d = decimal.Decimal("-123456.789") + serialized = self.service.to_data(d) + deserialized = self.service.to_object(serialized) + self.assertEqual(d, deserialized) def test_list(self): self.validate([1, 2.0, "a", None, bytearray("abc".encode()), [], [1, 2, 3]]) diff --git a/tests/unit/util_test.py b/tests/unit/util_test.py index 4ab77d9148..e41152b0a3 100644 --- a/tests/unit/util_test.py +++ b/tests/unit/util_test.py @@ -1,3 +1,5 @@ +from parameterized import parameterized + from hazelcast.config import ( IndexConfig, IndexUtil, @@ -5,7 +7,7 @@ QueryConstants, UniqueKeyTransformation, ) -from hazelcast.util import calculate_version +from hazelcast.util import calculate_version, int_from_bytes, int_to_bytes from unittest import TestCase @@ -94,3 +96,38 @@ def test_with_bitmap_indexes(self): bio["unique_key_transformation"], normalized.bitmap_index_options.unique_key_transformation, ) + + +# (int number, bytearray representation). +_CONVERSION_TEST_CASES = [ + (0, [0]), + (-1, [255]), + (127, [127]), + (-128, [128]), + (999, [3, 231]), + (-123456, [254, 29, 192]), + (2147483647, [127, 255, 255, 255]), + (-2147483648, [128, 0, 0, 0]), + (9223372036854775807, [127, 255, 255, 255, 255, 255, 255, 255]), + (-9223372036854775808, [128, 0, 0, 0, 0, 0, 0, 0]), + # fmt: off + ( + 23154266667777888899991234566543219999888877776666245132, + [0, 241, 189, 232, 38, 131, 251, 137, 60, 34, 23, 53, 163, 116, 208, 85, 14, 65, 40, 174, 120, 10, 56, 12] + ), + ( + -23154266667777888899991234566543219999888877776666245132, + [255, 14, 66, 23, 217, 124, 4, 118, 195, 221, 232, 202, 92, 139, 47, 170, 241, 190, 215, 81, 135, 245, 199, 244] + ) + # fmt: on +] + + +class IntConversionTest(TestCase): + @parameterized.expand(_CONVERSION_TEST_CASES) + def test_int_from_bytes(self, number, buf): + self.assertEqual(number, int_from_bytes(buf)) + + @parameterized.expand(_CONVERSION_TEST_CASES) + def test_int_to_bytes(self, number, buf): + self.assertEqual(bytearray(buf), int_to_bytes(number))