From 25fd9f8f1b432b536ffa188d74308847a1ee1b30 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 19 Oct 2021 10:19:51 +0200 Subject: [PATCH 1/7] create branch --- pyaerocom/_lowlevel_helpers.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pyaerocom/_lowlevel_helpers.py b/pyaerocom/_lowlevel_helpers.py index cf1c588a6..82d6fc72f 100644 --- a/pyaerocom/_lowlevel_helpers.py +++ b/pyaerocom/_lowlevel_helpers.py @@ -8,6 +8,27 @@ from concurrent.futures import ThreadPoolExecutor from pyaerocom import print_log +def round_floats(dict, precision=5): + """simple helper routine to turn all floats of a dictionary a given precision + + """ + ret_dict = dict.copy() + + if isinstance(dict, float): + return round(ret_dict, 5) + if isinstance(dict, dict): + for key in dict: + ret_dict[key] = round_floats(dict[key]) + # return {k: round_floats(v) for k, v in dict.items()} + elif isinstance(dict, (list, tuple)): + return [round_floats(x) for x in dict] + else: + raise NotImplementedError + + if isinstance(dict, (list, tuple)): + return [round_floats(x) for x in dict] + return ret_dict + def read_json(file_path): """Read json file From 2cd254a15d01352a6431fdc3c94a753c9d95f7fa Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 19 Oct 2021 12:27:09 +0200 Subject: [PATCH 2/7] first working version --- pyaerocom/_lowlevel_helpers.py | 52 +++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/pyaerocom/_lowlevel_helpers.py b/pyaerocom/_lowlevel_helpers.py index 82d6fc72f..c548b3474 100644 --- a/pyaerocom/_lowlevel_helpers.py +++ b/pyaerocom/_lowlevel_helpers.py @@ -8,26 +8,46 @@ from concurrent.futures import ThreadPoolExecutor from pyaerocom import print_log -def round_floats(dict, precision=5): - """simple helper routine to turn all floats of a dictionary a given precision +def round_floats(in_data, precision=5): + """ + simple helper method to change all floats of a data structure to a given precision. + For nested structures, this method is called recursively to go through + all levels + + Parameters + ---------- + in_data : float, dict, tuple, list + data structure whose numbers should be limited in precision + + + Returns + ------- + in_data + all the floats in in_data with limited precision + tuples in the structure have been converted to lists to make them mutable """ - ret_dict = dict.copy() - - if isinstance(dict, float): - return round(ret_dict, 5) - if isinstance(dict, dict): - for key in dict: - ret_dict[key] = round_floats(dict[key]) - # return {k: round_floats(v) for k, v in dict.items()} - elif isinstance(dict, (list, tuple)): - return [round_floats(x) for x in dict] + + if isinstance(in_data, float): + return round(in_data, precision) + elif isinstance(in_data, dict): + # no list comprehension here since there might be mixed data structures + for key in in_data: + in_data[key] = round_floats(in_data[key], precision=precision) + + elif isinstance(in_data, (list, tuple)): + # if in_data is a tuple, convert that to a list so that it becomes mutable + if isinstance(in_data,tuple): + in_data = list(in_data) + # no list comprehension here since there might be mixed data structures + for idx, x in enumerate(in_data): + in_data[idx] = round_floats(x, precision=precision) + else: - raise NotImplementedError + # leave all types we have not recognised before alone + pass - if isinstance(dict, (list, tuple)): - return [round_floats(x) for x in dict] - return ret_dict + return in_data def read_json(file_path): """Read json file From f767736b82c6cb508643f3bbc4d3986cf7b83d4f Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 19 Oct 2021 12:44:13 +0200 Subject: [PATCH 3/7] added round_floats call write_json method --- pyaerocom/_lowlevel_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyaerocom/_lowlevel_helpers.py b/pyaerocom/_lowlevel_helpers.py index c548b3474..c157129c5 100644 --- a/pyaerocom/_lowlevel_helpers.py +++ b/pyaerocom/_lowlevel_helpers.py @@ -81,7 +81,7 @@ def write_json(data_dict, file_path, **kwargs): indent, ) """ with open(file_path, 'w') as f: - simplejson.dump(data_dict, f, **kwargs) + simplejson.dump(round_floats(data_dict), f, **kwargs) def check_make_json(fp, indent=4): """ From 853353d848a2073e32f8f7965f83f72f1cfbfd5f Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Tue, 19 Oct 2021 13:49:24 +0200 Subject: [PATCH 4/7] adjusted comments a bit --- pyaerocom/_lowlevel_helpers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyaerocom/_lowlevel_helpers.py b/pyaerocom/_lowlevel_helpers.py index c157129c5..aabd4c3ef 100644 --- a/pyaerocom/_lowlevel_helpers.py +++ b/pyaerocom/_lowlevel_helpers.py @@ -19,7 +19,6 @@ def round_floats(in_data, precision=5): in_data : float, dict, tuple, list data structure whose numbers should be limited in precision - Returns ------- in_data @@ -28,7 +27,7 @@ def round_floats(in_data, precision=5): """ - if isinstance(in_data, float): + if isinstance(in_data, float): # this also covers np.float types return round(in_data, precision) elif isinstance(in_data, dict): # no list comprehension here since there might be mixed data structures From 1b8c0c13b0cfe91e8ac1706a6bb368fc85a484e5 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 25 Oct 2021 14:07:33 +0200 Subject: [PATCH 5/7] restructured code; add other numpy float types --- pyaerocom/_lowlevel_helpers.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/pyaerocom/_lowlevel_helpers.py b/pyaerocom/_lowlevel_helpers.py index aabd4c3ef..2c7ee03ab 100644 --- a/pyaerocom/_lowlevel_helpers.py +++ b/pyaerocom/_lowlevel_helpers.py @@ -27,25 +27,17 @@ def round_floats(in_data, precision=5): """ - if isinstance(in_data, float): # this also covers np.float types - return round(in_data, precision) - elif isinstance(in_data, dict): - # no list comprehension here since there might be mixed data structures - for key in in_data: - in_data[key] = round_floats(in_data[key], precision=precision) - + if isinstance(in_data, (float, np.float32, np.float16, np.float128, np.float64, )): + # np.float64, is an aliase for the Python float, but is mentioned here for completeness + # note that round and np.round yield different results with the Python round being mathematically correct + # details are here: + # https://numpy.org/doc/stable/reference/generated/numpy.around.html#numpy.around + # use numpy around for now + return np.around(in_data, precision) elif isinstance(in_data, (list, tuple)): - # if in_data is a tuple, convert that to a list so that it becomes mutable - if isinstance(in_data,tuple): - in_data = list(in_data) - # no list comprehension here since there might be mixed data structures - for idx, x in enumerate(in_data): - in_data[idx] = round_floats(x, precision=precision) - - else: - # leave all types we have not recognised before alone - pass - + return [round_floats(v, precision=precision) for v in in_data] + elif isinstance(in_data, dict): + return {k:round_floats(v, precision=precision) for k, v in in_data.items()} return in_data def read_json(file_path): From 485d658a7e1d7e412ef4f840a43b63333e33d2c3 Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 25 Oct 2021 14:50:31 +0200 Subject: [PATCH 6/7] added tests --- tests/test__lowlevel_helpers.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test__lowlevel_helpers.py b/tests/test__lowlevel_helpers.py index 76d71f09e..cd229429f 100644 --- a/tests/test__lowlevel_helpers.py +++ b/tests/test__lowlevel_helpers.py @@ -4,6 +4,29 @@ import simplejson from pyaerocom import _lowlevel_helpers as mod from .conftest import does_not_raise_exception +import numpy as np + + +def test_round_floats(): + fl = float(1.12344567890) + assert mod.round_floats(fl, precision=5) == 1.12345 + fl_list = [np.float_(2.3456789), np.float32(3.456789012)] + tmp = mod.round_floats(fl_list, precision=3) + assert tmp[0] == 2.346 + assert tmp[1] == pytest.approx(3.457, 1e-3) + fl_tuple = (np.float128(4.567890123), np.float_(5.6789012345)) + tmp = mod.round_floats(fl_tuple, precision=5) + assert isinstance(tmp, list) + assert tmp[0] == pytest.approx(4.56789, 1e-5) + assert tmp[1] == 5.67890 + fl_dict = {'bla': np.float128(0.1234455667), 'blubb': int(1), 'ha': 'test'} + tmp = mod.round_floats(fl_dict, precision=5) + assert tmp['bla'] == pytest.approx(0.12345, 1e-5) + assert tmp['blubb'] == 1 + assert isinstance(tmp['blubb'], int) + assert isinstance(tmp['ha'], str) + + class Constrainer(mod.ConstrainedContainer): def __init__(self): From 5ee4b3e09e1820c8a9a5d6f887c05adaeb29e2bb Mon Sep 17 00:00:00 2001 From: Jan Jurgen Griesfeller Date: Mon, 25 Oct 2021 14:55:53 +0200 Subject: [PATCH 7/7] refined assert statements --- tests/test__lowlevel_helpers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test__lowlevel_helpers.py b/tests/test__lowlevel_helpers.py index cd229429f..4326ed614 100644 --- a/tests/test__lowlevel_helpers.py +++ b/tests/test__lowlevel_helpers.py @@ -12,13 +12,11 @@ def test_round_floats(): assert mod.round_floats(fl, precision=5) == 1.12345 fl_list = [np.float_(2.3456789), np.float32(3.456789012)] tmp = mod.round_floats(fl_list, precision=3) - assert tmp[0] == 2.346 - assert tmp[1] == pytest.approx(3.457, 1e-3) + assert tmp == [2.346, pytest.approx(3.457, 1e-3)] fl_tuple = (np.float128(4.567890123), np.float_(5.6789012345)) tmp = mod.round_floats(fl_tuple, precision=5) assert isinstance(tmp, list) - assert tmp[0] == pytest.approx(4.56789, 1e-5) - assert tmp[1] == 5.67890 + assert tmp == [pytest.approx(4.56789, 1e-5), 5.67890] fl_dict = {'bla': np.float128(0.1234455667), 'blubb': int(1), 'ha': 'test'} tmp = mod.round_floats(fl_dict, precision=5) assert tmp['bla'] == pytest.approx(0.12345, 1e-5)