make JSON files smaller by limiting the precision

pyaerocom/_lowlevel_helpers.py

@@ -8,6 +8,38 @@
 from concurrent.futures import ThreadPoolExecutor
 from pyaerocom import print_log
 
+def round_floats(in_data, precision=5):
+    """
+    simple helper method to change all floats of a data structure to a given precision.
+    For nested structures, this method is called recursively to go through
+    all levels
+
+    Parameters
+    ----------
+    in_data : float, dict, tuple, list
+        data structure whose numbers should be limited in precision
+
+    Returns
+    -------
+    in_data
+        all the floats in in_data with limited precision
+        tuples in the structure have been converted to lists to make them mutable
+
+    """
+
+    if isinstance(in_data, (float, np.float32, np.float16, np.float128, np.float64, )):
+        # np.float64, is an aliase for the Python float, but is mentioned here for completeness
+        # note that round and np.round yield different results with the Python round being mathematically correct
+        # details are here:
+        # https://numpy.org/doc/stable/reference/generated/numpy.around.html#numpy.around
+        # use numpy around for now
+        return np.around(in_data, precision)
+    elif isinstance(in_data, (list, tuple)):
+        return [round_floats(v, precision=precision) for v in in_data]
+    elif isinstance(in_data, dict):
+        return {k:round_floats(v, precision=precision) for k, v in in_data.items()}
+    return in_data
+
 def read_json(file_path):
     """Read json file
 
@@ -40,7 +72,7 @@ def write_json(data_dict, file_path, **kwargs):
         indent, )
     """
     with open(file_path, 'w') as f:
-        simplejson.dump(data_dict, f, **kwargs)
+        simplejson.dump(round_floats(data_dict), f, **kwargs)
 
 def check_make_json(fp, indent=4):
     """

tests/test__lowlevel_helpers.py

@@ -4,6 +4,29 @@
 import simplejson
 from pyaerocom import _lowlevel_helpers as mod
 from .conftest import does_not_raise_exception
+import numpy as np
+
+
+def test_round_floats():
+    fl = float(1.12344567890)
+    assert mod.round_floats(fl, precision=5) == 1.12345
+    fl_list = [np.float_(2.3456789), np.float32(3.456789012)]
+    tmp = mod.round_floats(fl_list, precision=3)
+    assert tmp[0] == 2.346
+    assert tmp[1] == pytest.approx(3.457, 1e-3)
+    fl_tuple = (np.float128(4.567890123), np.float_(5.6789012345))
+    tmp = mod.round_floats(fl_tuple, precision=5)
+    assert isinstance(tmp, list)
+    assert tmp[0] == pytest.approx(4.56789, 1e-5)
+    assert tmp[1] == 5.67890
+    fl_dict = {'bla': np.float128(0.1234455667), 'blubb': int(1), 'ha': 'test'}
+    tmp = mod.round_floats(fl_dict, precision=5)
+    assert tmp['bla'] == pytest.approx(0.12345, 1e-5)
+    assert tmp['blubb'] == 1
+    assert isinstance(tmp['blubb'], int)
+    assert isinstance(tmp['ha'], str)
+
+
 
 class Constrainer(mod.ConstrainedContainer):
     def __init__(self):