From 96a47b99daa835cb6438d5314db1b5b4b78d33de Mon Sep 17 00:00:00 2001 From: Arham Chopra Date: Wed, 10 Apr 2024 15:50:40 -0400 Subject: [PATCH] Fix to_json serialization for floats Signed-off-by: Arham Chopra --- cpp/csp/python/PyStructToJson.cpp | 46 +++++++++++++++++++++++++-- csp/tests/impl/test_struct.py | 52 +++++++++++++++++++++++++++++-- 2 files changed, 93 insertions(+), 5 deletions(-) diff --git a/cpp/csp/python/PyStructToJson.cpp b/cpp/csp/python/PyStructToJson.cpp index 1226326c..b7e10ef7 100644 --- a/cpp/csp/python/PyStructToJson.cpp +++ b/cpp/csp/python/PyStructToJson.cpp @@ -21,6 +21,32 @@ inline rapidjson::Value toJson( const T& val, const CspType& typ, rapidjson::Doc return rapidjson::Value( val ); } +// Helper function for parsing doubles +inline rapidjson::Value doubleToJson( const double& val, rapidjson::Document& doc ) +{ + // NOTE: Rapidjson adds support for this in a future release. Remove this when we upgrade rapidjson to a version + // after 07/16/2023 and use kWriteNanAndInfNullFlag in the writer. + // + // To be compatible with other JSON libraries, we cannot use the default approach that rapidjson has to + // serializing NaN, and (+/-)Infs. We need to manually convert them to NULLs. Rapidjson adds support for this + // in a future release. + if ( std::isnan( val ) || std::isinf( val ) ) + { + return rapidjson::Value(); + } + else + { + return rapidjson::Value( val ); + } +} + +// Helper function to convert doubles into json format recursively, by properly handlings NaNs, and Infs +template<> +inline rapidjson::Value toJson( const double& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) +{ + return doubleToJson( val, doc ); +} + // Helper function to convert Enums into json format recursively template<> inline rapidjson::Value toJson( const CspEnum& val, const CspType& typ, rapidjson::Document& doc, PyObject * callable ) @@ -183,7 +209,21 @@ rapidjson::Value pyDictKeyToName( PyObject * py_key, rapidjson::Document& doc ) else if( PyFloat_Check( py_key ) ) { auto key = PyFloat_AsDouble( py_key ); - val.SetString( std::to_string( key ), doc.GetAllocator() ); + auto json_obj = doubleToJson( key, doc ); + if ( json_obj.IsNull() ) + { + auto * str_obj = PyObject_Str( py_key ); + Py_ssize_t len = 0; + const char * str = PyUnicode_AsUTF8AndSize( str_obj, &len ); + CSP_THROW( ValueError, "Cannot serialize " + std::string( str ) + " to key in JSON" ); + } + else + { + // Convert to string + std::stringstream s; + s << key; + val.SetString( s.str(), doc.GetAllocator() ); + } } else { @@ -255,12 +295,12 @@ rapidjson::Value pyObjectToJson( PyObject * value, rapidjson::Document& doc, PyO } else if( PyFloat_Check( value ) ) { - return rapidjson::Value( fromPython( value ) ); + return doubleToJson( fromPython( value ), doc ); } else if( PyUnicode_Check( value ) ) { Py_ssize_t len; - auto str = PyUnicode_AsUTF8AndSize( value , &len ); + auto str = PyUnicode_AsUTF8AndSize( value, &len ); rapidjson::Value str_val; str_val.SetString( str, len, doc.GetAllocator() ); return str_val; diff --git a/csp/tests/impl/test_struct.py b/csp/tests/impl/test_struct.py index 02caacf8..c588478f 100644 --- a/csp/tests/impl/test_struct.py +++ b/csp/tests/impl/test_struct.py @@ -1282,6 +1282,18 @@ class MyStruct(csp.Struct): result_dict = {"b": False, "i": 456, "f": 1.73, "s": "789"} self.assertEqual(json.loads(test_struct.to_json()), result_dict) + test_struct = MyStruct(b=False, i=456, f=float("nan"), s="789") + result_dict = {"b": False, "i": 456, "f": None, "s": "789"} + self.assertEqual(json.loads(test_struct.to_json()), result_dict) + + test_struct = MyStruct(b=False, i=456, f=float("inf"), s="789") + result_dict = {"b": False, "i": 456, "f": None, "s": "789"} + self.assertEqual(json.loads(test_struct.to_json()), result_dict) + + test_struct = MyStruct(b=False, i=456, f=float("-inf"), s="789") + result_dict = {"b": False, "i": 456, "f": None, "s": "789"} + self.assertEqual(json.loads(test_struct.to_json()), result_dict) + def test_to_json_enums(self): from enum import Enum as PyEnum @@ -1434,8 +1446,13 @@ class MyStruct(csp.Struct): result_dict = {"i": 456, "l_any": l_l_i} self.assertEqual(json.loads(test_struct.to_json()), result_dict) - l_any = [[1, 2], "hello", [4, 3.2, [6, [7], (8, True, 10.5, (11, [12, False]))]]] - l_any_result = [[1, 2], "hello", [4, 3.2, [6, [7], [8, True, 10.5, [11, [12, False]]]]]] + l_any = [[1, float("nan")], [float("INFINITY"), float("-inf")]] + test_struct = MyStruct(i=456, l_any=l_any) + result_dict = {"i": 456, "l_any": [[1, None], [None, None]]} + self.assertEqual(json.loads(test_struct.to_json()), result_dict) + + l_any = [[1, 2], "hello", [4, 3.2, [6, [7], (8, True, 10.5, (11, [float("nan"), False]))]]] + l_any_result = [[1, 2], "hello", [4, 3.2, [6, [7], [8, True, 10.5, [11, [None, False]]]]]] test_struct = MyStruct(i=456, l_any=l_any) result_dict = {"i": 456, "l_any": l_any_result} self.assertEqual(json.loads(test_struct.to_json()), result_dict) @@ -1444,6 +1461,7 @@ def test_to_json_dict(self): class MyStruct(csp.Struct): i: int = 123 d_i: typing.Dict[int, int] + d_f: typing.Dict[float, int] d_dt: typing.Dict[str, datetime] d_d_s: typing.Dict[str, typing.Dict[str, str]] d_any: dict @@ -1458,6 +1476,12 @@ class MyStruct(csp.Struct): result_dict = {"i": 456, "d_i": d_i_res} self.assertEqual(json.loads(test_struct.to_json()), result_dict) + d_f = {1.2: 2, 2.3: 4, 3.4: 6, 4.5: 7} + d_f_res = {str(k): v for k, v in d_f.items()} + test_struct = MyStruct(i=456, d_f=d_f) + result_dict = {"i": 456, "d_f": d_f_res} + self.assertEqual(json.loads(test_struct.to_json()), result_dict) + dt = datetime.now(tz=pytz.utc) d_dt = {"d1": dt, "d2": dt} test_struct = MyStruct(i=456, d_dt=d_dt) @@ -1475,6 +1499,12 @@ class MyStruct(csp.Struct): result_dict = {"i": 456, "d_any": d_i_res} self.assertEqual(json.loads(test_struct.to_json()), result_dict) + d_f = {1.2: 2, 2.3: 4, 3.4: 6, 4.5: 7} + d_f_res = {str(k): v for k, v in d_f.items()} + test_struct = MyStruct(i=456, d_any=d_f) + result_dict = {"i": 456, "d_any": d_f_res} + self.assertEqual(json.loads(test_struct.to_json()), result_dict) + dt = datetime.now(tz=pytz.utc) d_dt = {"d1": dt, "d2": dt} test_struct = MyStruct(i=456, d_any=d_dt) @@ -1487,6 +1517,24 @@ class MyStruct(csp.Struct): result_dict = {"i": 456, "d_any": d_any_res} self.assertEqual(json.loads(test_struct.to_json()), result_dict) + d_f = {float("nan"): 2, 2.3: 4, 3.4: 6, 4.5: 7} + d_f_res = {str(k): v for k, v in d_f.items()} + test_struct = MyStruct(i=456, d_any=d_f) + with self.assertRaises(ValueError): + test_struct.to_json() + + d_f = {float("inf"): 2, 2.3: 4, 3.4: 6, 4.5: 7} + d_f_res = {str(k): v for k, v in d_f.items()} + test_struct = MyStruct(i=456, d_any=d_f) + with self.assertRaises(ValueError): + test_struct.to_json() + + d_f = {float("-inf"): 2, 2.3: 4, 3.4: 6, 4.5: 7} + d_f_res = {str(k): v for k, v in d_f.items()} + test_struct = MyStruct(i=456, d_any=d_f) + with self.assertRaises(ValueError): + test_struct.to_json() + def test_to_json_struct(self): class MySubSubStruct(csp.Struct): b: bool = True