From 3f7449b40484913f696c9aa0b58a1e0595dead11 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 13 Jun 2023 14:21:58 +0200 Subject: [PATCH 001/332] dataarrays instead of datasets --- examples/3D_shipping_route.py | 2 +- examples/3D_shipping_route_str.py | 2 +- examples/4D_flight_path.py | 2 +- examples/country_slicing.py | 2 +- examples/cyclic_route_around_earth.py | 2 +- examples/read_me_example.py | 2 +- examples/slicing_all_ecmwf_countries.py | 2 +- examples/timeseries_example.py | 2 +- examples/wind_farms.py | 10 ++- performance/scalability_test.py | 2 +- performance/scalability_test_2.py | 42 ++++----- polytope/datacube/xarray.py | 12 ++- tests/test_cyclic_axis_over_negative_vals.py | 32 ++++--- tests/test_cyclic_axis_slicer_not_0.py | 32 ++++--- tests/test_cyclic_axis_slicing.py | 67 +++++++++------ tests/test_datacube_xarray.py | 11 +-- tests/test_float_type.py | 20 +++-- tests/test_hullslicer_engine.py | 1 - tests/test_slicer_era5.py | 2 +- tests/test_slicer_xarray.py | 7 +- tests/test_slicing_unsliceable_axis.py | 10 ++- tests/test_slicing_xarray_3D.py | 87 ++++++++++++------- tests/test_slicing_xarray_4D.py | 90 +++++++++++++------- 23 files changed, 273 insertions(+), 168 deletions(-) diff --git a/examples/3D_shipping_route.py b/examples/3D_shipping_route.py index cf3f84027..0e5f7cb04 100644 --- a/examples/3D_shipping_route.py +++ b/examples/3D_shipping_route.py @@ -14,7 +14,7 @@ class Test: def setup_method(self): - array = xr.open_dataset("./examples/data/winds.grib", engine="cfgrib") + array = xr.open_dataset("./examples/data/winds.grib", engine="cfgrib").u10 self.array = array self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/examples/3D_shipping_route_str.py b/examples/3D_shipping_route_str.py index f0af827f8..fc624d97f 100644 --- a/examples/3D_shipping_route_str.py +++ b/examples/3D_shipping_route_str.py @@ -13,7 +13,7 @@ class Test: def setup_method(self): - array = xr.open_dataset("./examples/data/output4.grib", engine="cfgrib") + array = xr.open_dataset("./examples/data/output4.grib", engine="cfgrib").u10 self.xarraydatacube = XArrayDatacube(array) self.array = array self.slicer = HullSlicer() diff --git a/examples/4D_flight_path.py b/examples/4D_flight_path.py index c7793476f..87db4720a 100644 --- a/examples/4D_flight_path.py +++ b/examples/4D_flight_path.py @@ -12,7 +12,7 @@ class Test: def setup_method(self): - array = xr.open_dataset("./examples/data/temp_model_levels.grib", engine="cfgrib") + array = xr.open_dataset("./examples/data/temp_model_levels.grib", engine="cfgrib").t options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) for dim in array.dims: diff --git a/examples/country_slicing.py b/examples/country_slicing.py index e81812559..481e7c995 100644 --- a/examples/country_slicing.py +++ b/examples/country_slicing.py @@ -12,7 +12,7 @@ class Test: def setup_method(self, method): - array = xr.open_dataset("./examples/data/output8.grib", engine="cfgrib") + array = xr.open_dataset("./examples/data/output8.grib", engine="cfgrib").t2m options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() diff --git a/examples/cyclic_route_around_earth.py b/examples/cyclic_route_around_earth.py index beacd10ce..404df875a 100644 --- a/examples/cyclic_route_around_earth.py +++ b/examples/cyclic_route_around_earth.py @@ -11,7 +11,7 @@ class Test: def setup_method(self, method): - array = xr.open_dataset("./examples/data/output8.grib", engine="cfgrib") + array = xr.open_dataset("./examples/data/output8.grib", engine="cfgrib").t2m options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() diff --git a/examples/read_me_example.py b/examples/read_me_example.py index bb1eeee11..db36664a2 100644 --- a/examples/read_me_example.py +++ b/examples/read_me_example.py @@ -4,7 +4,7 @@ from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select -array = xr.open_dataset("./examples/data/winds.grib", engine="cfgrib") +array = xr.open_dataset("./examples/data/winds.grib", engine="cfgrib").u10 options = {"longitude": {"Cyclic": [0, 360.0]}} diff --git a/examples/slicing_all_ecmwf_countries.py b/examples/slicing_all_ecmwf_countries.py index f6b756eb9..6038160ea 100644 --- a/examples/slicing_all_ecmwf_countries.py +++ b/examples/slicing_all_ecmwf_countries.py @@ -12,7 +12,7 @@ class Test: def setup_method(self, method): - array = xr.open_dataset(".examples/data/output8.grib", engine="cfgrib") + array = xr.open_dataset(".examples/data/output8.grib", engine="cfgrib").t2m options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() diff --git a/examples/timeseries_example.py b/examples/timeseries_example.py index 87f87bdde..876d0865f 100644 --- a/examples/timeseries_example.py +++ b/examples/timeseries_example.py @@ -13,7 +13,7 @@ class Test: def setup_method(self): - array = xr.open_dataset("./examples/data/timeseries_t2m.grib", engine="cfgrib") + array = xr.open_dataset("./examples/data/timeseries_t2m.grib", engine="cfgrib").t2m self.xarraydatacube = XArrayDatacube(array) for dim in array.dims: array = array.sortby(dim) diff --git a/examples/wind_farms.py b/examples/wind_farms.py index 631904925..45ff54c68 100644 --- a/examples/wind_farms.py +++ b/examples/wind_farms.py @@ -16,7 +16,7 @@ class Test: def setup_method(self): - array = xr.open_dataset("./examples/data/winds.grib", engine="cfgrib") + array = xr.open_dataset("./examples/data/winds.grib", engine="cfgrib").u10 self.array = array options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) @@ -73,10 +73,12 @@ def test_slice_shipping_route(self): long = long - 360 lats.append(lat) longs.append(long) - u10_idx = result.leaves[i].result["u10"] + # u10_idx = result.leaves[i].result["u10"] + u10_idx = result.leaves[i].result[1] wind_u = u10_idx - v10_idx = result.leaves[i].result["v10"] - wind_v = v10_idx + # v10_idx = result.leaves[i].result["v10"] + # wind_v = v10_idx + wind_v = 0 winds_u.append(wind_u) winds_v.append(wind_v) parameter_values.append(math.sqrt(wind_u**2 + wind_v**2)) diff --git a/performance/scalability_test.py b/performance/scalability_test.py index 186fd5616..98eeac9d1 100644 --- a/performance/scalability_test.py +++ b/performance/scalability_test.py @@ -11,7 +11,7 @@ class Test: def setup_method(self): - array = xr.open_dataset("../examples/data/temp_model_levels.grib", engine="cfgrib") + array = xr.open_dataset("../examples/data/temp_model_levels.grib", engine="cfgrib").t options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) for dim in array.dims: diff --git a/performance/scalability_test_2.py b/performance/scalability_test_2.py index de39adc40..9330fd60a 100644 --- a/performance/scalability_test_2.py +++ b/performance/scalability_test_2.py @@ -11,7 +11,7 @@ class Test: def setup_method(self): - array = xr.open_dataset("../examples/data/temp_model_levels.grib", engine="cfgrib") + array = xr.open_dataset("./examples/data/temp_model_levels.grib", engine="cfgrib").t options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) for dim in array.dims: @@ -82,23 +82,23 @@ def test_scalability_2D_v3(self): print(len(result.leaves)) print(time.time() - time_start) - def test_scalability_2D_v4(self): - union = Box(["latitude", "longitude"], [0 - 100, 0], [20 - 100, 36]) - for i in range(9): - box = Box(["latitude", "longitude"], [20 * (i + 1) - 100, 0], [20 * (i + 2) - 100, 36]) - union = Union(["latitude", "longitude"], union, box) - for j in range(9): - box = Box(["latitude", "longitude"], [0 - 100, 36 * (j + 1)], [20 - 100, 36 * (j + 2)]) - union = Union(["latitude", "longitude"], union, box) - for i in range(9): - for j in range(9): - box = Box( - ["latitude", "longitude"], [20 * (i + 1) - 100, 36 * (j + 1)], [20 * (i + 2) - 100, 36 * (j + 2)] - ) - union = Union(["latitude", "longitude"], union, box) - time_start = time.time() - print(time_start) - request = Request(union, Select("step", [np.timedelta64(0, "ns")]), Select("hybrid", [1])) - result = self.API.retrieve(request) - print(len(result.leaves)) - print(time.time() - time_start) + # def test_scalability_2D_v4(self): + # union = Box(["latitude", "longitude"], [0 - 100, 0], [20 - 100, 36]) + # for i in range(9): + # box = Box(["latitude", "longitude"], [20 * (i + 1) - 100, 0], [20 * (i + 2) - 100, 36]) + # union = Union(["latitude", "longitude"], union, box) + # for j in range(9): + # box = Box(["latitude", "longitude"], [0 - 100, 36 * (j + 1)], [20 - 100, 36 * (j + 2)]) + # union = Union(["latitude", "longitude"], union, box) + # for i in range(9): + # for j in range(9): + # box = Box( + # ["latitude", "longitude"], [20 * (i + 1) - 100, 36 * (j + 1)], [20 * (i + 2) - 100, 36 * (j + 2)] + # ) + # union = Union(["latitude", "longitude"], union, box) + # time_start = time.time() + # print(time_start) + # request = Request(union, Select("step", [np.timedelta64(0, "ns")]), Select("hybrid", [1])) + # result = self.API.retrieve(request) + # print(len(result.leaves)) + # print(time.time() - time_start) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index bc32d7705..52ee283a4 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -25,6 +25,7 @@ np.float64: FloatAxis(), np.str_: UnsliceableaAxis(), str: UnsliceableaAxis(), + np.object_ : UnsliceableaAxis() } @@ -69,9 +70,12 @@ def get(self, requests: IndexTree): # TODO: Here, once we flatten the path, we want to remap the values on the axis to fit the datacube... if len(path.items()) == len(self.dataarray.coords): subxarray = self.dataarray.sel(path, method="nearest") - data_variables = subxarray.data_vars - result_tuples = [(key, value) for key, value in data_variables.items()] - r.result = dict(result_tuples) + value = subxarray.item() + key = subxarray.name + # data_variables = subxarray.data_vars + # result_tuples = [(key, value) for key, value in data_variables.items()] + # r.result = dict(result_tuples) + r.result = (key, value) else: r.remove_branch() @@ -142,7 +146,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - subarray = self.dataarray.sel(path)[axis.name] + subarray = self.dataarray.sel(path, method='nearest')[axis.name] subarray_vals = subarray.values return index in subarray_vals diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index 3b0fe5743..a0ee1e1c6 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -21,6 +21,7 @@ def setup_method(self, method): "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1], }, ) + array = array.to_array() options = {"long": {"Cyclic": [-1.1, -0.1]}, "level": {"Cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() @@ -30,12 +31,13 @@ def setup_method(self, method): def test_cyclic_float_axis_across_seam(self): request = Request( - Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 20 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 0.8, 0.9, 1.0, @@ -60,12 +62,13 @@ def test_cyclic_float_axis_across_seam(self): def test_cyclic_float_axis_inside_cyclic_range(self): request = Request( - Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 16 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 0.0, 0.1, 0.2, @@ -86,21 +89,23 @@ def test_cyclic_float_axis_inside_cyclic_range(self): def test_cyclic_float_axis_above_axis_range(self): request = Request( - Box(["step", "long"], [0, 1.3], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 1.3], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] + assert [leaf.parent.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] def test_cyclic_float_axis_two_range_loops(self): request = Request( - Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 50 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 0.3, 0.4, 0.5, @@ -155,21 +160,24 @@ def test_cyclic_float_axis_two_range_loops(self): def test_cyclic_float_axis_below_axis_range(self): request = Request( - Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, -0.3] + assert [leaf.parent.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, + -0.3] def test_cyclic_float_axis_below_axis_range_crossing_seam(self): request = Request( - Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 22 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ -0.7, -0.6, -0.5, diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 3b94f1503..a8c2f6d47 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -21,6 +21,7 @@ def setup_method(self, method): "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1], }, ) + array = array.to_array() options = {"long": {"Cyclic": [-1.1, -0.1]}, "level": {"Cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() @@ -30,12 +31,13 @@ def setup_method(self, method): def test_cyclic_float_axis_across_seam(self): request = Request( - Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 20 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 0.8, 0.9, 1.0, @@ -60,12 +62,13 @@ def test_cyclic_float_axis_across_seam(self): def test_cyclic_float_axis_inside_cyclic_range(self): request = Request( - Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 16 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 0.0, 0.1, 0.2, @@ -86,21 +89,23 @@ def test_cyclic_float_axis_inside_cyclic_range(self): def test_cyclic_float_axis_above_axis_range(self): request = Request( - Box(["step", "long"], [0, 1.3], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 1.3], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] + assert [leaf.parent.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] def test_cyclic_float_axis_two_range_loops(self): request = Request( - Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 50 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 0.3, 0.4, 0.5, @@ -155,21 +160,24 @@ def test_cyclic_float_axis_two_range_loops(self): def test_cyclic_float_axis_below_axis_range(self): request = Request( - Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, -0.3] + assert [leaf.parent.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, + -0.3] def test_cyclic_float_axis_below_axis_range_crossing_seam(self): request = Request( - Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 22 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ -0.7, -0.6, -0.5, diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index fcd8bab47..9c8001464 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -21,6 +21,7 @@ def setup_method(self, method): "long": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], }, ) + array = array.to_array() options = {"long": {"Cyclic": [0, 1.0]}, "level": {"Cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() @@ -30,12 +31,13 @@ def setup_method(self, method): def test_cyclic_float_axis_across_seam(self): request = Request( - Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 20 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 0.8, 0.9, 1.0, @@ -60,12 +62,13 @@ def test_cyclic_float_axis_across_seam(self): def test_cyclic_float_axis_across_seam_repeated(self): request = Request( - Box(["step", "long"], [0, 0.0], [3, 1.0]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 0.0], [3, 1.0]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 22 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 0.0, 0.1, 0.2, @@ -92,12 +95,13 @@ def test_cyclic_float_axis_across_seam_repeated(self): def test_cyclic_float_axis_across_seam_repeated_twice(self): request = Request( - Box(["step", "long"], [0, 0.0], [3, 2.0]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 0.0], [3, 2.0]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 22 * 2 - 2 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 0.0, 0.1, 0.2, @@ -144,12 +148,13 @@ def test_cyclic_float_axis_across_seam_repeated_twice(self): def test_cyclic_float_axis_inside_cyclic_range(self): request = Request( - Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 16 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 0.0, 0.1, 0.2, @@ -170,21 +175,23 @@ def test_cyclic_float_axis_inside_cyclic_range(self): def test_cyclic_float_axis_above_axis_range(self): request = Request( - Box(["step", "long"], [0, 1.3], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 1.3], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] + assert [leaf.parent.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] def test_cyclic_float_axis_two_range_loops(self): request = Request( - Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 50 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 0.3, 0.4, 0.5, @@ -239,21 +246,24 @@ def test_cyclic_float_axis_two_range_loops(self): def test_cyclic_float_axis_below_axis_range(self): request = Request( - Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, -0.3] + assert [leaf.parent.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, + -0.3] def test_cyclic_float_axis_below_axis_range_crossing_seam(self): request = Request( - Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 22 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ -0.7, -0.6, -0.5, @@ -280,19 +290,21 @@ def test_cyclic_float_axis_below_axis_range_crossing_seam(self): def test_cyclic_float_axis_reversed(self): request = Request( - Box(["step", "long"], [0, 1.7], [3, 1.3]), Select("date", ["2000-01-01"]), Select("level", [128]) + Box(["step", "long"], [0, 1.7], [3, 1.3]), Select("date", ["2000-01-01"]), Select("level", [128]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] + assert [leaf.parent.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] def test_two_cyclic_axis_wrong_axis_order(self): - request = Request(Box(["step", "long", "level"], [0, 1.3, 131], [3, 1.7, 132]), Select("date", ["2000-01-01"])) + request = Request(Box(["step", "long", "level"], [0, 1.3, 131], [3, 1.7, 132]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 20 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 1.3, 1.4, 1.5, @@ -316,11 +328,12 @@ def test_two_cyclic_axis_wrong_axis_order(self): ] def test_two_cyclic_axis(self): - request = Request(Box(["step", "level", "long"], [0, 131, 1.3], [3, 132, 1.7]), Select("date", ["2000-01-01"])) + request = Request(Box(["step", "level", "long"], [0, 131, 1.3], [3, 132, 1.7]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 20 - assert [leaf.value for leaf in result.leaves] == [ + assert [leaf.parent.value for leaf in result.leaves] == [ 1.3, 1.4, 1.5, @@ -344,15 +357,17 @@ def test_two_cyclic_axis(self): ] def test_select_cyclic_float_axis_edge(self): - request = Request(Box(["step", "level"], [0, 3], [3, 5]), Select("date", ["2000-01-01"]), Select("long", [0])) + request = Request(Box(["step", "level"], [0, 3], [3, 5]), Select("date", ["2000-01-01"]), Select("long", [0]), + Select("variable", ["param"])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 6 - assert [leaf.value for leaf in result.leaves] == [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + assert [leaf.parent.value for leaf in result.leaves] == [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] def test_cyclic_int_axis(self): - request = Request(Box(["step", "level"], [0, 3], [3, 5]), Select("date", ["2000-01-01"]), Select("long", [0.1])) + request = Request(Box(["step", "level"], [0, 3], [3, 5]), Select("date", ["2000-01-01"]), Select("long", [0.1]), + Select("variable", ["param"])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 6 - assert [leaf.value for leaf in result.leaves] == [0.1, 0.1, 0.1, 0.1, 0.1, 0.1] + assert [leaf.parent.value for leaf in result.leaves] == [0.1, 0.1, 0.1, 0.1, 0.1, 0.1] diff --git a/tests/test_datacube_xarray.py b/tests/test_datacube_xarray.py index b18f91f95..52e61172f 100644 --- a/tests/test_datacube_xarray.py +++ b/tests/test_datacube_xarray.py @@ -16,21 +16,22 @@ def setup_method(self, method): def test_validate(self): dims = np.random.randn(1, 1, 1) array = xr.Dataset(data_vars=dict(param=(["x", "y", "z"], dims)), coords={"x": [1], "y": [1], "z": [1]}) + array = array.to_array() datacube = Datacube.create(array, options={}) datacube = Datacube.create(array, options={}) - datacube.validate(["x", "y", "z"]) - datacube.validate(["x", "z", "y"]) + datacube.validate(["x", "y", "z", "variable"]) + datacube.validate(["x", "z", "y", "variable"]) with pytest.raises(AxisNotFoundError): - datacube.validate(["x", "y", "z", "w"]) + datacube.validate(["x", "y", "z", "w", "variable"]) with pytest.raises(AxisNotFoundError): - datacube.validate(["w", "x", "y", "z"]) + datacube.validate(["w", "x", "y", "z", "variable"]) with pytest.raises(AxisOverdefinedError): - datacube.validate(["x", "x", "y", "z"]) + datacube.validate(["x", "x", "y", "z", "variable"]) def test_create(self): # Create a dataarray with 3 labelled axes using different index types diff --git a/tests/test_float_type.py b/tests/test_float_type.py index 4b42ce42d..eca66d8ef 100644 --- a/tests/test_float_type.py +++ b/tests/test_float_type.py @@ -20,7 +20,7 @@ def setup_method(self, method): "alt": np.arange(0.0, 20.0, 0.1), }, ) - + array = array.to_array() self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) @@ -28,36 +28,42 @@ def setup_method(self, method): def test_slicing_span(self): # TODO: some problems with floating point values and values inside the datacube being slightly off. # This has been fixed by introducing tolerances, but could be better handled using exact arithmetic. - request = Request(Span("lat", 4.1, 4.3), Select("long", [4.1]), Select("alt", [4.1])) + request = Request(Span("lat", 4.1, 4.3), Select("long", [4.1]), Select("alt", [4.1]), + Select("variable", ["param"])) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 3 def test_slicing_point(self): - request = Request(Select("lat", [4.1]), Select("long", [4.1]), Select("alt", [4.1])) + request = Request(Select("lat", [4.1]), Select("long", [4.1]), Select("alt", [4.1]), + Select("variable", ["param"])) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 1 and not result.leaves[0].is_root() @pytest.mark.skip(reason="Points too close, need exact arithmetic") def test_slicing_very_close_point(self): - request = Request(Select("lat", [4.1]), Select("long", [4.0999919, 4.1]), Select("alt", [4.1])) + request = Request(Select("lat", [4.1]), Select("long", [4.0999919, 4.1]), Select("alt", [4.1]), + Select("variable", ["param"])) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 1 def test_slicing_points_higher_precision(self): - request = Request(Select("lat", [4.12]), Select("long", [4.1]), Select("alt", [4.1])) + request = Request(Select("lat", [4.12]), Select("long", [4.1]), Select("alt", [4.1]), + Select("variable", ["param"])) result = self.API.retrieve(request) result.pprint() assert result.leaves[0].is_root() def test_slicing_points_empty_span_higher_precision(self): - request = Request(Span("lat", 4.11, 4.12), Select("long", [4.1]), Select("alt", [4.1])) + request = Request(Span("lat", 4.11, 4.12), Select("long", [4.1]), Select("alt", [4.1]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].is_root() def test_slicing_points_span_higher_precision(self): - request = Request(Span("lat", 4.09, 4.12), Select("long", [4.1]), Select("alt", [4.1])) + request = Request(Span("lat", 4.09, 4.12), Select("long", [4.1]), Select("alt", [4.1]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert not result.leaves[0].is_root() and len(result.leaves) == 1 diff --git a/tests/test_hullslicer_engine.py b/tests/test_hullslicer_engine.py index f113caf65..4bb15fb42 100644 --- a/tests/test_hullslicer_engine.py +++ b/tests/test_hullslicer_engine.py @@ -18,7 +18,6 @@ def setup_method(self, method): "level": np.arange(0, 100, 1), }, ) - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 2e7f77955..e3f55e7cb 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -8,7 +8,7 @@ class TestSlicingEra5Data: def setup_method(self, method): - array = xr.open_dataset("./tests/data/era5-levels-members.grib", engine="cfgrib") + array = xr.open_dataset("./tests/data/era5-levels-members.grib", engine="cfgrib").t self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/tests/test_slicer_xarray.py b/tests/test_slicer_xarray.py index ec67c69f1..564e807f5 100644 --- a/tests/test_slicer_xarray.py +++ b/tests/test_slicer_xarray.py @@ -19,12 +19,13 @@ def setup_method(self, method): "level": range(1, 130), }, ) - + array = array.to_array() self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) def test_2D_box(self): - request = Request(Box(["step", "level"], [3, 10], [6, 11]), Select("date", ["2000-01-01"])) + request = Request(Box(["step", "level"], [3, 10], [6, 11]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) result.pprint() @@ -34,6 +35,7 @@ def test_2D_box_with_date_range(self): # TODO: conversion from numpy to Point class should allow dropping the pd.Timestamp # It should convert to correct type Span("date", lower=pd.Timestamp("2000-01-01"), upper=pd.Timestamp("2000-01-05")), + Select("variable", ["param"]) ) result = self.API.retrieve(request) result.pprint() @@ -41,6 +43,7 @@ def test_2D_box_with_date_range(self): def test_3D_box_with_date(self): request = Request( Box(["step", "level", "date"], [3, 10, pd.Timestamp("2000-01-01")], [6, 11, pd.Timestamp("2000-01-01")]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) result.pprint() diff --git a/tests/test_slicing_unsliceable_axis.py b/tests/test_slicing_unsliceable_axis.py index a16d99378..fdbb1cbd1 100644 --- a/tests/test_slicing_unsliceable_axis.py +++ b/tests/test_slicing_unsliceable_axis.py @@ -18,6 +18,7 @@ def setup_method(self, method): data_vars=dict(param=(["date", "variable", "level"], dims)), coords={"date": pd.date_range("2000-01-01", "2000-01-03", 3), "variable": ["a"], "level": range(1, 130)}, ) + array = array.to_array(dim="parameter") self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) @@ -25,19 +26,22 @@ def setup_method(self, method): # Testing different shapes def test_finding_existing_variable(self): - request = Request(Box(["level"], [10], [11]), Select("date", ["2000-01-01"]), Select("variable", ["a"])) + request = Request(Box(["level"], [10], [11]), Select("date", ["2000-01-01"]), Select("variable", ["a"]), + Select("parameter", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 2 def test_finding_nonexisting_variable(self): - request = Request(Box(["level"], [10], [11]), Select("date", ["2000-01-01"]), Select("variable", ["b"])) + request = Request(Box(["level"], [10], [11]), Select("date", ["2000-01-01"]), Select("variable", ["b"]), + Select("parameter", ["param"])) with pytest.raises(ValueError): result = self.API.retrieve(request) result.pprint() def test_unsliceable_axis_in_a_shape(self): # does it work when we ask a box or disk of an unsliceable axis? - request = Request(Box(["level", "variable"], [10, "a"], [11, "a"]), Select("date", ["2000-01-01"])) + request = Request(Box(["level", "variable"], [10, "a"], [11, "a"]), Select("date", ["2000-01-01"]), + Select("parameter", ["param"])) with pytest.raises(UnsliceableShapeError): result = self.API.retrieve(request) result.pprint() diff --git a/tests/test_slicing_xarray_3D.py b/tests/test_slicing_xarray_3D.py index 6aba39004..aca668c5f 100644 --- a/tests/test_slicing_xarray_3D.py +++ b/tests/test_slicing_xarray_3D.py @@ -33,6 +33,7 @@ def setup_method(self, method): "level": range(1, 130), }, ) + array = array.to_array(dim="variable") self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) @@ -40,68 +41,79 @@ def setup_method(self, method): # Testing different shapes def test_2D_box(self): - request = Request(Box(["step", "level"], [3, 10], [6, 11]), Select("date", ["2000-01-01"])) + request = Request(Box(["step", "level"], [3, 10], [6, 11]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 4 def test_2D_box_union_disjoint_boxes(self): box1 = Box(["step", "level"], [3, 10], [6, 11]) box2 = Box(["step", "level"], [7, 15], [12, 17]) - request = Request(Union(["step", "level"], box1, box2), Select("date", ["2000-01-01"])) + request = Request(Union(["step", "level"], box1, box2), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 4 + 6 def test_2D_box_union_overlapping_boxes(self): box1 = Box(["step", "level"], [3, 9], [6, 11]) box2 = Box(["step", "level"], [6, 10], [12, 17]) - request = Request(Union(["step", "level"], box1, box2), Select("date", ["2000-01-01"])) + request = Request(Union(["step", "level"], box1, box2), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 6 + 24 - 2 def test_point(self): - request = Request(Select("date", ["2000-01-03"]), Select("level", [100]), Select("step", [3])) + request = Request(Select("date", ["2000-01-03"]), Select("level", [100]), Select("step", [3]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 1 def test_segment(self): - request = Request(Span("level", 10, 11), Select("date", ["2000-01-01"]), Select("step", [9])) + request = Request(Span("level", 10, 11), Select("date", ["2000-01-01"]), Select("step", [9]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 2 def test_union_line_point(self): seg1 = Span("step", 4.3, 6.2) pt1 = Select("step", [6.20001]) - request = Request(Union(["step"], seg1, pt1), Select("date", ["2000-01-01"]), Select("level", [100])) + request = Request(Union(["step"], seg1, pt1), Select("date", ["2000-01-01"]), Select("level", [100]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 1 def test_union_boxes_intersect_one_point(self): box1 = Box(["step", "level"], [3, 10], [6, 11]) box2 = Box(["step", "level"], [6, 11], [12, 17]) - request = Request(Union(["step", "level"], box1, box2), Select("date", ["2000-01-01"])) + request = Request(Union(["step", "level"], box1, box2), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 4 + 3 * 7 - 1 def test_mix_existing_nonexisting_data(self): - request = Request(Select("date", ["2000-01-03", "2000-01-04"]), Select("level", [100]), Select("step", [3])) + request = Request(Select("date", ["2000-01-03", "2000-01-04"]), Select("level", [100]), Select("step", [3]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 1 def test_disk(self): - request = Request(Disk(["level", "step"], [6, 6], [3, 3]), Select("date", ["2000-01-01"])) + request = Request(Disk(["level", "step"], [6, 6], [3, 3]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 9 def test_concave_polygon(self): points = [[1, 0], [3, 0], [2, 3], [3, 6], [1, 6]] - request = Request(Polygon(["level", "step"], points), Select("date", ["2000-01-01"])) + request = Request(Polygon(["level", "step"], points), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) self.xarraydatacube.get(result) assert len(result.leaves) == 8 def test_polytope(self): points = [[0, 1], [3, 1], [3, 2], [0, 2]] - request = Request(ConvexPolytope(["step", "level"], points), Select("date", ["2000-01-01"])) + request = Request(ConvexPolytope(["step", "level"], points), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) self.xarraydatacube.get(result) assert len(result.leaves) == 4 @@ -112,49 +124,57 @@ def test_union_empty_lines(self): # Slices non-existing step data seg1 = Span("step", 4, 5) seg2 = Span("step", 10, 11) - request = Request(Union(["step"], seg1, seg2), Select("date", ["2000-01-01"]), Select("level", [100])) + request = Request(Union(["step"], seg1, seg2), Select("date", ["2000-01-01"]), Select("level", [100]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_empty_box_no_level(self): # Slices non-existing level data - request = Request(Box(["step", "level"], [3, 10.5], [7, 10.99]), Select("date", ["2000-01-01"])) + request = Request(Box(["step", "level"], [3, 10.5], [7, 10.99]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_empty_box_no_level_step(self): # Slices non-existing level and step data - request = Request(Box(["step", "level"], [4, 10.5], [5, 10.99]), Select("date", ["2000-01-01"])) + request = Request(Box(["step", "level"], [4, 10.5], [5, 10.99]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_empty_box_no_step(self): # Slices non-existing step and level data - request = Request(Box(["step", "level"], [4, 10], [5, 10.49]), Select("date", ["2000-01-01"])) + request = Request(Box(["step", "level"], [4, 10], [5, 10.49]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_empty_box_floating_steps(self): # Slices through no step data and float type level data - request = Request(Box(["step", "level"], [4.1, 10.3], [5.7, 11.8]), Select("date", ["2000-01-01"])) + request = Request(Box(["step", "level"], [4.1, 10.3], [5.7, 11.8]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_empty_box_no_step_level_float(self): # Slices empty step and level box - request = Request(Box(["step", "level"], [4.1, 10.3], [5.7, 10.8]), Select("date", ["2000-01-01"])) + request = Request(Box(["step", "level"], [4.1, 10.3], [5.7, 10.8]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_empty_no_step_unordered(self): # Slice empty box because no step is available - request = Request(Box(["level", "step"], [10, 4], [10, 5]), Select("date", ["2000-01-01"])) + request = Request(Box(["level", "step"], [10, 4], [10, 5]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_nonexisting_date(self): # Slices non-existing date data - request = Request(Select("date", ["2000-01-04"]), Select("level", [100]), Select("step", [3])) + request = Request(Select("date", ["2000-01-04"]), Select("level", [100]), Select("step", [3]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -162,7 +182,8 @@ def test_two_nonexisting_close_points(self): # Slices two close points neither of which are available in the datacube pt1 = Select("step", [2.99]) pt2 = Select("step", [3.001]) - request = Request(Union(["step"], pt1, pt2), Select("level", [100]), Select("date", ["2000-01-01"])) + request = Request(Union(["step"], pt1, pt2), Select("level", [100]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -171,7 +192,8 @@ def test_union_two_nonexisting_points(self): # However if we round these points, we get points in the datacube pt1 = Select("step", [6.99]) pt2 = Select("step", [3.001]) - request = Request(Union(["step"], pt1, pt2), Select("level", [100]), Select("date", ["2000-01-01"])) + request = Request(Union(["step"], pt1, pt2), Select("level", [100]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -179,19 +201,22 @@ def test_two_close_points_no_level(self): # Slices non-existing step points and non-existing level pt1 = Select("step", [2.99]) pt2 = Select("step", [3.001]) - request = Request(Union(["step"], pt1, pt2), Select("level", [100.1]), Select("date", ["2000-01-01"])) + request = Request(Union(["step"], pt1, pt2), Select("level", [100.1]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_nonexisting_point_float_level(self): # Slices non-existing level data - request = Request(Select("step", [3]), Select("level", [99.1]), Select("date", ["2000-01-02"])) + request = Request(Select("step", [3]), Select("level", [99.1]), Select("date", ["2000-01-02"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_nonexisting_segment(self): # Slices non-existing step data - request = Request(Span("step", 3.2, 3.23), Select("level", [99]), Select("date", ["2000-01-01"])) + request = Request(Span("step", 3.2, 3.23), Select("level", [99]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -199,13 +224,15 @@ def test_nonexisting_segment(self): def test_flat_box(self): # Should slice through a line in the step direction - request = Request(Box(["step", "level"], [4, 10], [7, 10]), Select("date", ["2000-01-01"])) + request = Request(Box(["step", "level"], [4, 10], [7, 10]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 1 def test_box(self): # Should slice a line in the level direction - request = Request(Box(["level", "step"], [3, 3], [6, 3]), Select("date", ["2000-01-01"])) + request = Request(Box(["level", "step"], [3, 3], [6, 3]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 4 @@ -214,7 +241,8 @@ def test_swept_concave_polygon(self): points = [(1, 0), (3, 0), (3, 6), (2, 6), (2, 3), (1, 3)] concave_polygon = Polygon(["level", "step"], points) swept_poly = PathSegment(["level", "step"], concave_polygon, [0, 0], [1, 3]) - request = Request(swept_poly, Select("date", ["2000-01-01"])) + request = Request(swept_poly, Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) self.xarraydatacube.get(result) assert len(result.leaves) == 12 @@ -227,7 +255,8 @@ def test_intersection_point_disk_polygon(self): r1 = math.cos(math.pi / 12) * (8 - 4 * math.sqrt(3)) + sys.float_info.epsilon # note that we need a small perturbation to make up for rounding errors r2 = 3 * math.cos(math.pi / 12) * (math.sqrt(3) - 2) * (8 - 4 * math.sqrt(3)) / (4 * math.sqrt(3) - 7) - request = Request(Disk(["level", "step"], [0, 0], [r1, r2]), Select("date", ["2000-01-01"])) + request = Request(Disk(["level", "step"], [0, 0], [r1, r2]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) paths = [r.flatten().values() for r in result.leaves] - assert (pd.Timestamp("2000-01-01 00:00:00"), 3.0, 1.0) in paths + assert (pd.Timestamp("2000-01-01 00:00:00"), 3.0, 1.0, "param") in paths diff --git a/tests/test_slicing_xarray_4D.py b/tests/test_slicing_xarray_4D.py index 382674822..582b50afc 100644 --- a/tests/test_slicing_xarray_4D.py +++ b/tests/test_slicing_xarray_4D.py @@ -34,7 +34,7 @@ def setup_method(self, method): "lat": np.around(np.arange(0.0, 10.0, 0.1), 15), }, ) - + array = array.to_array() self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) @@ -42,20 +42,23 @@ def setup_method(self, method): # Testing different shapes def test_3D_box(self): - request = Request(Box(["step", "level", "lat"], [3, 10, 5.0], [6, 11, 6.0]), Select("date", ["2000-01-01"])) + request = Request(Box(["step", "level", "lat"], [3, 10, 5.0], [6, 11, 6.0]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 2 * 2 * 11 def test_4D_box(self): request = Request( Box(["step", "level", "lat", "date"], [3, 10, 5.0, "2000-01-01"], [6, 11, 6.0, "2000-01-02"]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) assert len(result.leaves) == 2 * 2 * 11 * 2 def test_circle_int(self): request = Request( - Disk(["step", "level"], [9, 10], [6, 6]), Select("date", ["2000-01-01"]), Select("lat", [5.2]) + Disk(["step", "level"], [9, 10], [6, 6]), Select("date", ["2000-01-01"]), Select("lat", [5.2]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) assert len(result.leaves) == 37 @@ -63,28 +66,32 @@ def test_circle_int(self): def test_circles_barely_touching_int(self): disk1 = Disk(["step", "level"], [6, 10], [5.9, 6]) disk2 = Disk(["step", "level"], [15, 10], [2.9, 3]) - request = Request(Union(["step", "level"], disk1, disk2), Select("date", ["2000-01-01"]), Select("lat", [5.1])) + request = Request(Union(["step", "level"], disk1, disk2), Select("date", ["2000-01-01"]), Select("lat", [5.1]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 45 def test_circles_intersecting_float(self): disk1 = Disk(["step", "lat"], [6, 4.0], [6.99, 0.1]) disk2 = Disk(["step", "lat"], [15, 2.0], [4.99, 0.3]) - request = Request(Union(["step", "lat"], disk1, disk2), Select("date", ["2000-01-01"]), Select("level", [10])) + request = Request(Union(["step", "lat"], disk1, disk2), Select("date", ["2000-01-01"]), Select("level", [10]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 24 def test_circles_touching_float(self): disk1 = Disk(["step", "lat"], [6, 4.0], [3, 1.9]) disk2 = Disk(["step", "lat"], [15, 2.0], [3, 2.1]) - request = Request(Union(["step", "lat"], disk1, disk2), Select("date", ["2000-01-01"]), Select("level", [10])) + request = Request(Union(["step", "lat"], disk1, disk2), Select("date", ["2000-01-01"]), Select("level", [10]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 101 def test_pathsegment_swept_2D_box(self): box1 = Box(["step", "level"], [3, 0], [6, 1]) request = Request( - PathSegment(["step", "level"], box1, [3, 1], [6, 2]), Select("lat", [4.0]), Select("date", ["2000-01-01"]) + PathSegment(["step", "level"], box1, [3, 1], [6, 2]), Select("lat", [4.0]), Select("date", ["2000-01-01"]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() @@ -94,7 +101,8 @@ def test_pathsegment_swept_2D_box_bis(self): # Had a floating point problem because of the latitude box1 = Box(["step", "level"], [3, 3], [6, 5]) request = Request( - PathSegment(["step", "level"], box1, [3, 3], [6, 6]), Select("lat", [4.1]), Select("date", ["2000-01-01"]) + PathSegment(["step", "level"], box1, [3, 3], [6, 6]), Select("lat", [4.1]), Select("date", ["2000-01-01"]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() @@ -103,7 +111,8 @@ def test_pathsegment_swept_2D_box_bis(self): def test_pathsegment_swept_circle(self): circ1 = Disk(["step", "level"], [6, 3], [3, 2]) request = Request( - PathSegment(["step", "level"], circ1, [3, 3], [6, 6]), Select("lat", [5.5]), Select("date", ["2000-01-01"]) + PathSegment(["step", "level"], circ1, [3, 3], [6, 6]), Select("lat", [5.5]), Select("date", ["2000-01-01"]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() @@ -112,7 +121,8 @@ def test_pathsegment_swept_circle(self): def test_path_swept_box_2_points(self): box1 = Box(["step", "level"], [3, 3], [6, 5]) request = Request( - Path(["step", "level"], box1, [3, 3], [6, 6]), Select("lat", [4.3]), Select("date", ["2000-01-01"]) + Path(["step", "level"], box1, [3, 3], [6, 6]), Select("lat", [4.3]), Select("date", ["2000-01-01"]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) # result.pprint() @@ -121,20 +131,22 @@ def test_path_swept_box_2_points(self): def test_path_swept_box_3_points(self): box1 = Box(["step", "level"], [3, 3], [6, 5]) request = Request( - Path(["step", "level"], box1, [3, 3], [6, 6], [9, 9]), Select("lat", [4.3]), Select("date", ["2000-01-01"]) + Path(["step", "level"], box1, [3, 3], [6, 6], [9, 9]), Select("lat", [4.3]), Select("date", ["2000-01-01"]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) assert len(result.leaves) == 18 def test_polygon_other_than_triangle(self): polygon = Polygon(["step", "level"], [[3, 3], [3, 5], [6, 5], [6, 7]]) - request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"])) + request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 6 def test_ellipsoid(self): ellipsoid = Ellipsoid(["step", "level", "lat"], [6, 3, 2.1], [3, 1, 0.1]) - request = Request(ellipsoid, Select("date", ["2000-01-01"])) + request = Request(ellipsoid, Select("date", ["2000-01-01"]), Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 7 @@ -143,7 +155,8 @@ def test_ellipsoid(self): def test_empty_circle(self): # Slices a circle with no data inside request = Request( - Disk(["step", "level"], [5, 3.4], [0.5, 0.2]), Select("date", ["2000-01-01"]), Select("lat", [5.1]) + Disk(["step", "level"], [5, 3.4], [0.5, 0.2]), Select("date", ["2000-01-01"]), Select("lat", [5.1]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -151,7 +164,8 @@ def test_empty_circle(self): def test_float_box(self): # Slices a box with no data inside request = Request( - Box(["step", "lat"], [10.1, 1.01], [10.3, 1.04]), Select("date", ["2000-01-01"]), Select("level", [10]) + Box(["step", "lat"], [10.1, 1.01], [10.3, 1.04]), Select("date", ["2000-01-01"]), Select("level", [10]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -160,7 +174,8 @@ def test_path_empty_box(self): # Slices the path of a box with no data inside, but gives data because the box is swept over a datacube value box1 = Box(["step", "level"], [2.4, 3.1], [2.5, 3.4]) request = Request( - PathSegment(["step", "level"], box1, [3, 3], [6, 6]), Select("lat", [4.0]), Select("date", ["2000-01-01"]) + PathSegment(["step", "level"], box1, [3, 3], [6, 6]), Select("lat", [4.0]), Select("date", ["2000-01-01"]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) assert len(result.leaves) == 1 @@ -172,6 +187,7 @@ def test_path_empty_box_empty(self): PathSegment(["step", "level"], box1, [1.1, 3.3], [2.7, 3.6]), Select("lat", [4.0]), Select("date", ["2000-01-01"]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -179,7 +195,7 @@ def test_path_empty_box_empty(self): def test_ellipsoid_empty(self): # Slices an empty ellipsoid which doesn't have any step value ellipsoid = Ellipsoid(["step", "level", "lat"], [5, 3, 2.1], [0, 0, 0]) - request = Request(ellipsoid, Select("date", ["2000-01-01"])) + request = Request(ellipsoid, Select("date", ["2000-01-01"]), Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -188,7 +204,8 @@ def test_ellipsoid_empty(self): def test_span_bounds(self): # Tests that span also works in reverse order request = Request( - Span("level", 100, 98), Select("step", [3]), Select("lat", [5.5]), Select("date", ["2000-01-01"]) + Span("level", 100, 98), Select("step", [3]), Select("lat", [5.5]), Select("date", ["2000-01-01"]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) assert len(result.leaves) == 3 @@ -198,53 +215,60 @@ def test_span_bounds(self): def test_ellipsoid_one_point(self): # Slices through a point (center of the ellipsoid) ellipsoid = Ellipsoid(["step", "level", "lat"], [6, 3, 2.1], [0, 0, 0]) - request = Request(ellipsoid, Select("date", ["2000-01-01"])) + request = Request(ellipsoid, Select("date", ["2000-01-01"]), Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 1 assert not result.leaves[0].axis == IndexTree.root def test_flat_box_level(self): # Slices a line in the step direction - request = Request(Select("lat", [6]), Box(["level", "step"], [3, 3], [3, 9]), Select("date", ["2000-01-01"])) + request = Request(Select("lat", [6]), Box(["level", "step"], [3, 3], [3, 9]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 3 def test_flat_box_step(self): # Slices a line in the level direction - request = Request(Select("lat", [6]), Box(["level", "step"], [3, 3], [7, 3]), Select("date", ["2000-01-01"])) + request = Request(Select("lat", [6]), Box(["level", "step"], [3, 3], [7, 3]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert len(result.leaves) == 5 def test_flat_disk_nonexisting(self): # Slices an empty disk because there is no step level - request = Request(Disk(["level", "step"], [4, 5], [4, 0]), Select("lat", [6]), Select("date", ["2000-01-01"])) + request = Request(Disk(["level", "step"], [4, 5], [4, 0]), Select("lat", [6]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_flat_disk_line(self): # Slices a line in the level direction - request = Request(Disk(["level", "step"], [4, 6], [4, 0]), Select("lat", [6]), Select("date", ["2000-01-01"])) + request = Request(Disk(["level", "step"], [4, 6], [4, 0]), Select("lat", [6]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 8 def test_flat_disk_line_step(self): # Slices a line in the step direction - request = Request(Disk(["level", "step"], [4, 6], [0, 3]), Select("lat", [6]), Select("date", ["2000-01-01"])) + request = Request(Disk(["level", "step"], [4, 6], [0, 3]), Select("lat", [6]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 3 def test_flat_disk_empty(self): # Slices an empty disk because there is no step - request = Request(Disk(["level", "step"], [4, 5], [0, 0.5]), Select("lat", [6]), Select("date", ["2000-01-01"])) + request = Request(Disk(["level", "step"], [4, 5], [0, 0.5]), Select("lat", [6]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) # result.pprint() assert result.leaves[0].axis == IndexTree.root def test_disk_point(self): # Slices a point because the origin of the disk is a datacube point - request = Request(Disk(["level", "step"], [4, 6], [0, 0]), Select("lat", [6]), Select("date", ["2000-01-01"])) + request = Request(Disk(["level", "step"], [4, 6], [0, 0]), Select("lat", [6]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 1 @@ -252,7 +276,8 @@ def test_disk_point(self): def test_empty_disk(self): # Slices an empty object because the origin of the disk is not a datacube point - request = Request(Disk(["level", "step"], [4, 5], [0, 0]), Select("lat", [6]), Select("date", ["2000-01-01"])) + request = Request(Disk(["level", "step"], [4, 5], [0, 0]), Select("lat", [6]), Select("date", ["2000-01-01"]), + Select("variable", ["param"])) result = self.API.retrieve(request) # result.pprint() assert result.leaves[0].axis == IndexTree.root @@ -260,7 +285,7 @@ def test_empty_disk(self): def test_polygon_line(self): # Slices a line defined through the polygon shape polygon = Polygon(["step", "level"], [[3, 3], [3, 6], [3, 3], [3, 3]]) - request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"])) + request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"]), Select("variable", ["param"])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 4 @@ -269,7 +294,7 @@ def test_polygon_point(self): # Slices a point defined through the polygon object with several initial point entries. # Tests whether duplicate points are removed as they should polygon = Polygon(["step", "level"], [[3, 3], [3, 3], [3, 3], [3, 3]]) - request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"])) + request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"]), Select("variable", ["param"])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 1 @@ -278,7 +303,7 @@ def test_polygon_point(self): def test_polygon_empty(self): # Slices a point which isn't in the datacube (defined through the polygon shape) polygon = Polygon(["step", "level"], [[2, 3.1]]) - request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"])) + request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"]), Select("variable", ["param"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -289,18 +314,19 @@ def test_axis_specified_twice(self): request = Request( Box(["step", "level"], [3, 10], [6, 11]), Box(["step", "lat", "date"], [3, 5.0, "2000-01-01"], [6, 6.0, "2000-01-02"]), + Select("variable", ["param"]) ) result = self.API.retrieve(request) result.pprint() def test_not_all_axes_defined(self): with pytest.raises(AxisUnderdefinedError): - request = Request(Box(["step", "level"], [3, 10], [6, 11])) + request = Request(Box(["step", "level"], [3, 10], [6, 11]), Select("variable", ["param"])) result = self.API.retrieve(request) result.pprint() def test_not_all_axes_exist(self): with pytest.raises(KeyError): - request = Request(Box(["weather", "level"], [3, 10], [6, 11])) + request = Request(Box(["weather", "level"], [3, 10], [6, 11]), Select("variable", ["param"])) result = self.API.retrieve(request) result.pprint() From 4302def2269dfb59b600748dbf9ee87be1c60a63 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 13 Jun 2023 14:46:07 +0200 Subject: [PATCH 002/332] fix todo in xarray get_indices --- polytope/datacube/xarray.py | 5 +++-- tests/test_datacube_xarray.py | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index bc32d7705..5913b3206 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -25,6 +25,7 @@ np.float64: FloatAxis(), np.str_: UnsliceableaAxis(), str: UnsliceableaAxis(), + np.object_ : UnsliceableaAxis() } @@ -116,8 +117,8 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): # Get the indexes of the axis we want to query # XArray does not support branching, so no need to use label, we just take the next axis - # TODO: should assert that the label == next axis + assert axis.name == next(iter(subarray.xindexes)) indexes = next(iter(subarray.xindexes.values())).to_pandas_index() # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube @@ -142,7 +143,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - subarray = self.dataarray.sel(path)[axis.name] + subarray = self.dataarray.sel(path, method='nearest')[axis.name] subarray_vals = subarray.values return index in subarray_vals diff --git a/tests/test_datacube_xarray.py b/tests/test_datacube_xarray.py index b18f91f95..73db69a3c 100644 --- a/tests/test_datacube_xarray.py +++ b/tests/test_datacube_xarray.py @@ -67,12 +67,14 @@ def test_create(self): # Check discretizing along 'date' axis with a range of dates label = PandasTimestampAxis() + label.name = "date" idxs = datacube.get_indices(partial_request, label, pd.Timestamp("2000-01-02"), pd.Timestamp("2000-03-31")) assert (idxs == pd.date_range(pd.Timestamp("2000-01-02"), pd.Timestamp("2000-01-03"), 2)).all() assert type(idxs[0]) == pd.Timestamp # Check discretizing along 'date' axis at a specific date gives one value label = PandasTimestampAxis() + label.name = "date" idxs = datacube.get_indices(partial_request, label, pd.Timestamp("2000-01-02"), pd.Timestamp("2000-01-02")) assert len(idxs) == 1 assert type(idxs[0]) == pd.Timestamp @@ -80,6 +82,7 @@ def test_create(self): # Check discretizing along 'date' axis at a date which does not exist in discrete space gives no values label = PandasTimestampAxis() + label.name = "date" idxs = datacube.get_indices( partial_request, label, pd.Timestamp("2000-01-01-1200"), pd.Timestamp("2000-01-01-1200") ) @@ -94,6 +97,7 @@ def test_create(self): # Check discretizing along 'step' axis with a range of steps label = IntAxis() + label.name = "step" idxs = datacube.get_indices(partial_request, label, 0, 10) assert idxs == [0, 3, 6, 9] assert type(idxs[0]) == int @@ -117,6 +121,7 @@ def test_create(self): # Check discretizing along 'level' axis with a range of levels label = FloatAxis() + label.name = "level" idxs = datacube.get_indices(partial_request, label, -0.3, 10) assert idxs == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] assert type(idxs[0]) == int From cf59427878b9e3ebbe0ab85fb3f5aaed7ea3f1ad Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 13 Jun 2023 14:47:47 +0200 Subject: [PATCH 003/332] black --- polytope/datacube/xarray.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 5913b3206..eef17203e 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -25,7 +25,7 @@ np.float64: FloatAxis(), np.str_: UnsliceableaAxis(), str: UnsliceableaAxis(), - np.object_ : UnsliceableaAxis() + np.object_: UnsliceableaAxis(), } @@ -143,7 +143,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - subarray = self.dataarray.sel(path, method='nearest')[axis.name] + subarray = self.dataarray.sel(path, method="nearest")[axis.name] subarray_vals = subarray.values return index in subarray_vals From a5ff72f604d97e031b5b4648efe2491bf7812528 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 21 Jun 2023 11:01:25 +0200 Subject: [PATCH 004/332] change project name --- .github/workflows/ci.yaml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 46dc23696..9d0b2d82b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -4,7 +4,7 @@ on: # Trigger the workflow on push to master or develop, except tag creation push: branches: - - 'master' + - 'main' - 'develop' # Trigger the workflow on pull request diff --git a/setup.py b/setup.py index b614485a4..3decfd37a 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( - name="polytope", + name="polytope-python", version=__version__, description="Polytope datacube feature extraction library", url="https://github.com/ecmwf/polytope", From 977f1e8b2f734e8ff476b09e5dfba5814291e349 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 22 Jun 2023 10:47:38 +0200 Subject: [PATCH 005/332] remove the method=nearest in has_idx --- polytope/datacube/xarray.py | 7 +- tests/test_cyclic_axis_over_negative_vals.py | 38 +++----- tests/test_cyclic_axis_slicer_not_0.py | 38 +++----- tests/test_cyclic_axis_slicing.py | 75 ++++++--------- tests/test_float_type.py | 27 ++---- tests/test_slicer_xarray.py | 14 +-- tests/test_slicing_unsliceable_axis.py | 16 ++-- tests/test_slicing_xarray_3D.py | 93 +++++++----------- tests/test_slicing_xarray_4D.py | 99 +++++++------------- 9 files changed, 150 insertions(+), 257 deletions(-) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 511faa294..3e079ba62 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -25,7 +25,7 @@ np.float64: FloatAxis(), np.str_: UnsliceableaAxis(), str: UnsliceableaAxis(), - np.object_ : UnsliceableaAxis() + np.object_: UnsliceableaAxis(), } @@ -69,9 +69,6 @@ def get(self, requests: IndexTree): subxarray = self.dataarray.sel(path, method="nearest") value = subxarray.item() key = subxarray.name - # data_variables = subxarray.data_vars - # result_tuples = [(key, value) for key, value in data_variables.items()] - # r.result = dict(result_tuples) r.result = (key, value) else: r.remove_branch() @@ -142,7 +139,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - subarray = self.dataarray.sel(path, method='nearest')[axis.name] + subarray = self.dataarray.sel(path)[axis.name] subarray_vals = subarray.values return index in subarray_vals diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index a0ee1e1c6..57c4a6780 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -11,9 +11,9 @@ class TestSlicing3DXarrayDatacube: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types - dims = np.random.randn(3, 6, 129, 11) - array = xr.Dataset( - data_vars=dict(param=(["date", "step", "level", "long"], dims)), + array = xr.DataArray( + np.random.randn(3, 6, 129, 11), + dims=("date", "step", "level", "long"), coords={ "date": pd.date_range("2000-01-01", "2000-01-03", 3), "step": [0, 3, 6, 9, 12, 15], @@ -21,7 +21,6 @@ def setup_method(self, method): "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1], }, ) - array = array.to_array() options = {"long": {"Cyclic": [-1.1, -0.1]}, "level": {"Cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() @@ -31,13 +30,12 @@ def setup_method(self, method): def test_cyclic_float_axis_across_seam(self): request = Request( - Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 20 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 0.8, 0.9, 1.0, @@ -62,13 +60,12 @@ def test_cyclic_float_axis_across_seam(self): def test_cyclic_float_axis_inside_cyclic_range(self): request = Request( - Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 16 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 0.0, 0.1, 0.2, @@ -89,23 +86,21 @@ def test_cyclic_float_axis_inside_cyclic_range(self): def test_cyclic_float_axis_above_axis_range(self): request = Request( - Box(["step", "long"], [0, 1.3], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 1.3], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.parent.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] + assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] def test_cyclic_float_axis_two_range_loops(self): request = Request( - Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 50 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 0.3, 0.4, 0.5, @@ -160,24 +155,21 @@ def test_cyclic_float_axis_two_range_loops(self): def test_cyclic_float_axis_below_axis_range(self): request = Request( - Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.parent.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, - -0.3] + assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, -0.3] def test_cyclic_float_axis_below_axis_range_crossing_seam(self): request = Request( - Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 22 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ -0.7, -0.6, -0.5, diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index b3aa6f2a2..93147c05b 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -11,9 +11,9 @@ class TestSlicing3DXarrayDatacube: def setup_method(self, method): # create a dataarray with 3 labelled axes using different index types - dims = np.random.randn(3, 6, 129, 11) - array = xr.Dataset( - data_vars=dict(param=(["date", "step", "level", "long"], dims)), + array = xr.DataArray( + np.random.randn(3, 6, 129, 11), + dims=("date", "step", "level", "long"), coords={ "date": pd.date_range("2000-01-01", "2000-01-03", 3), "step": [0, 3, 6, 9, 12, 15], @@ -21,7 +21,6 @@ def setup_method(self, method): "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1], }, ) - array = array.to_array() options = {"long": {"Cyclic": [-1.1, -0.1]}, "level": {"Cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() @@ -31,12 +30,11 @@ def setup_method(self, method): def test_cyclic_float_axis_across_seam(self): request = Request( - Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) assert len(result.leaves) == 20 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 0.8, 0.9, 1.0, @@ -61,13 +59,12 @@ def test_cyclic_float_axis_across_seam(self): def test_cyclic_float_axis_inside_cyclic_range(self): request = Request( - Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 16 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 0.0, 0.1, 0.2, @@ -88,23 +85,21 @@ def test_cyclic_float_axis_inside_cyclic_range(self): def test_cyclic_float_axis_above_axis_range(self): request = Request( - Box(["step", "long"], [0, 1.3], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 1.3], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.parent.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] + assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] def test_cyclic_float_axis_two_range_loops(self): request = Request( - Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 50 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 0.3, 0.4, 0.5, @@ -159,24 +154,21 @@ def test_cyclic_float_axis_two_range_loops(self): def test_cyclic_float_axis_below_axis_range(self): request = Request( - Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.parent.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, - -0.3] + assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, -0.3] def test_cyclic_float_axis_below_axis_range_crossing_seam(self): request = Request( - Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 22 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ -0.7, -0.6, -0.5, diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index 321f9f35f..e5cbfb3af 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -11,9 +11,9 @@ class TestSlicing3DXarrayDatacube: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types - dims = np.random.randn(3, 6, 129, 11) - array = xr.Dataset( - data_vars=dict(param=(["date", "step", "level", "long"], dims)), + array = xr.DataArray( + np.random.randn(3, 6, 129, 11), + dims=("date", "step", "level", "long"), coords={ "date": pd.date_range("2000-01-01", "2000-01-03", 3), "step": [0, 3, 6, 9, 12, 15], @@ -21,7 +21,6 @@ def setup_method(self, method): "long": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], }, ) - array = array.to_array() options = {"long": {"Cyclic": [0, 1.0]}, "level": {"Cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() @@ -31,12 +30,11 @@ def setup_method(self, method): def test_cyclic_float_axis_across_seam(self): request = Request( - Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) assert len(result.leaves) == 20 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 0.8, 0.9, 1.0, @@ -61,13 +59,12 @@ def test_cyclic_float_axis_across_seam(self): def test_cyclic_float_axis_across_seam_repeated(self): request = Request( - Box(["step", "long"], [0, 0.0], [3, 1.0]), Select("date", ["2000-01-01"]), Select("level", [128]), - # Select("variable", ["param"]) + Box(["step", "long"], [0, 0.0], [3, 1.0]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 22 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 0.0, 0.1, 0.2, @@ -94,13 +91,12 @@ def test_cyclic_float_axis_across_seam_repeated(self): def test_cyclic_float_axis_across_seam_repeated_twice(self): request = Request( - Box(["step", "long"], [0, 0.0], [3, 2.0]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 0.0], [3, 2.0]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 22 * 2 - 2 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 0.0, 0.1, 0.2, @@ -147,13 +143,12 @@ def test_cyclic_float_axis_across_seam_repeated_twice(self): def test_cyclic_float_axis_inside_cyclic_range(self): request = Request( - Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 16 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 0.0, 0.1, 0.2, @@ -174,23 +169,23 @@ def test_cyclic_float_axis_inside_cyclic_range(self): def test_cyclic_float_axis_above_axis_range(self): request = Request( - Box(["step", "long"], [0, 1.3], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 1.3], [3, 1.7]), + Select("date", ["2000-01-01"]), + Select("level", [128]), ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.parent.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] + assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] def test_cyclic_float_axis_two_range_loops(self): request = Request( - Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 50 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 0.3, 0.4, 0.5, @@ -245,24 +240,21 @@ def test_cyclic_float_axis_two_range_loops(self): def test_cyclic_float_axis_below_axis_range(self): request = Request( - Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.parent.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, - -0.3] + assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, -0.3] def test_cyclic_float_axis_below_axis_range_crossing_seam(self): request = Request( - Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 22 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ -0.7, -0.6, -0.5, @@ -289,21 +281,19 @@ def test_cyclic_float_axis_below_axis_range_crossing_seam(self): def test_cyclic_float_axis_reversed(self): request = Request( - Box(["step", "long"], [0, 1.7], [3, 1.3]), Select("date", ["2000-01-01"]), Select("level", [128]), - Select("variable", ["param"]) + Box(["step", "long"], [0, 1.7], [3, 1.3]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.parent.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] + assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] def test_two_cyclic_axis_wrong_axis_order(self): - request = Request(Box(["step", "long", "level"], [0, 1.3, 131], [3, 1.7, 132]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["step", "long", "level"], [0, 1.3, 131], [3, 1.7, 132]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 20 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 1.3, 1.4, 1.5, @@ -327,12 +317,11 @@ def test_two_cyclic_axis_wrong_axis_order(self): ] def test_two_cyclic_axis(self): - request = Request(Box(["step", "level", "long"], [0, 131, 1.3], [3, 132, 1.7]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["step", "level", "long"], [0, 131, 1.3], [3, 132, 1.7]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 20 - assert [leaf.parent.value for leaf in result.leaves] == [ + assert [leaf.value for leaf in result.leaves] == [ 1.3, 1.4, 1.5, @@ -356,17 +345,15 @@ def test_two_cyclic_axis(self): ] def test_select_cyclic_float_axis_edge(self): - request = Request(Box(["step", "level"], [0, 3], [3, 5]), Select("date", ["2000-01-01"]), Select("long", [0]), - Select("variable", ["param"])) + request = Request(Box(["step", "level"], [0, 3], [3, 5]), Select("date", ["2000-01-01"]), Select("long", [0])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 6 - assert [leaf.parent.value for leaf in result.leaves] == [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + assert [leaf.value for leaf in result.leaves] == [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] def test_cyclic_int_axis(self): - request = Request(Box(["step", "level"], [0, 3], [3, 5]), Select("date", ["2000-01-01"]), Select("long", [0.1]), - Select("variable", ["param"])) + request = Request(Box(["step", "level"], [0, 3], [3, 5]), Select("date", ["2000-01-01"]), Select("long", [0.1])) result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 6 - assert [leaf.parent.value for leaf in result.leaves] == [0.1, 0.1, 0.1, 0.1, 0.1, 0.1] + assert [leaf.value for leaf in result.leaves] == [0.1, 0.1, 0.1, 0.1, 0.1, 0.1] diff --git a/tests/test_float_type.py b/tests/test_float_type.py index eca66d8ef..0b07c79ae 100644 --- a/tests/test_float_type.py +++ b/tests/test_float_type.py @@ -11,59 +11,50 @@ class TestFloatType: def setup_method(self, method): # Create a dataarray with 3 labelled axes using float type - dims = np.random.randn(100, 101, 200) - array = xr.Dataset( - data_vars=dict(param=(["lat", "long", "alt"], dims)), + array = xr.DataArray( + np.random.randn(100, 101, 200), + dims=("lat", "long", "alt"), coords={ "lat": np.arange(0.0, 10.0, 0.1), "long": np.arange(4.09999, 4.1 + 0.0000001, 0.0000001), "alt": np.arange(0.0, 20.0, 0.1), }, ) - array = array.to_array() self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) def test_slicing_span(self): - # TODO: some problems with floating point values and values inside the datacube being slightly off. - # This has been fixed by introducing tolerances, but could be better handled using exact arithmetic. - request = Request(Span("lat", 4.1, 4.3), Select("long", [4.1]), Select("alt", [4.1]), - Select("variable", ["param"])) + request = Request(Span("lat", 4.1, 4.3), Select("long", [4.1]), Select("alt", [4.1])) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 3 def test_slicing_point(self): - request = Request(Select("lat", [4.1]), Select("long", [4.1]), Select("alt", [4.1]), - Select("variable", ["param"])) + request = Request(Select("lat", [4.1]), Select("long", [4.1]), Select("alt", [4.1])) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 1 and not result.leaves[0].is_root() @pytest.mark.skip(reason="Points too close, need exact arithmetic") def test_slicing_very_close_point(self): - request = Request(Select("lat", [4.1]), Select("long", [4.0999919, 4.1]), Select("alt", [4.1]), - Select("variable", ["param"])) + request = Request(Select("lat", [4.1]), Select("long", [4.0999919, 4.1]), Select("alt", [4.1])) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 1 def test_slicing_points_higher_precision(self): - request = Request(Select("lat", [4.12]), Select("long", [4.1]), Select("alt", [4.1]), - Select("variable", ["param"])) + request = Request(Select("lat", [4.12]), Select("long", [4.1]), Select("alt", [4.1])) result = self.API.retrieve(request) result.pprint() assert result.leaves[0].is_root() def test_slicing_points_empty_span_higher_precision(self): - request = Request(Span("lat", 4.11, 4.12), Select("long", [4.1]), Select("alt", [4.1]), - Select("variable", ["param"])) + request = Request(Span("lat", 4.11, 4.12), Select("long", [4.1]), Select("alt", [4.1])) result = self.API.retrieve(request) assert result.leaves[0].is_root() def test_slicing_points_span_higher_precision(self): - request = Request(Span("lat", 4.09, 4.12), Select("long", [4.1]), Select("alt", [4.1]), - Select("variable", ["param"])) + request = Request(Span("lat", 4.09, 4.12), Select("long", [4.1]), Select("alt", [4.1])) result = self.API.retrieve(request) assert not result.leaves[0].is_root() and len(result.leaves) == 1 diff --git a/tests/test_slicer_xarray.py b/tests/test_slicer_xarray.py index b9b9b927a..a7aecaab6 100644 --- a/tests/test_slicer_xarray.py +++ b/tests/test_slicer_xarray.py @@ -10,22 +10,20 @@ class TestXarraySlicing: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types - dims = np.random.randn(3, 6, 129) - array = xr.Dataset( - data_vars=dict(param=(["date", "step", "level"], dims)), + array = xr.DataArray( + np.random.randn(3, 6, 129), + dims=("date", "step", "level"), coords={ "date": pd.date_range("2000-01-01", "2000-01-03", 3), "step": [0, 3, 6, 9, 12, 15], "level": range(1, 130), }, ) - array = array.to_array() self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) def test_2D_box(self): - request = Request(Box(["step", "level"], [3, 10], [6, 11]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["step", "level"], [3, 10], [6, 11]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) result.pprint() @@ -33,15 +31,13 @@ def test_2D_box_with_date_range(self): request = Request( Box(["step", "level"], [3, 10], [6, 11]), Span("date", lower=pd.Timestamp("2000-01-01"), upper=pd.Timestamp("2000-01-05")), - Select("variable", ["param"]) ) result = self.API.retrieve(request) result.pprint() def test_3D_box_with_date(self): request = Request( - Box(["step", "level", "date"], [3, 10, pd.Timestamp("2000-01-01")], [6, 11, pd.Timestamp("2000-01-01")]), - Select("variable", ["param"]) + Box(["step", "level", "date"], [3, 10, pd.Timestamp("2000-01-01")], [6, 11, pd.Timestamp("2000-01-01")]) ) result = self.API.retrieve(request) result.pprint() diff --git a/tests/test_slicing_unsliceable_axis.py b/tests/test_slicing_unsliceable_axis.py index fdbb1cbd1..45c21cdb5 100644 --- a/tests/test_slicing_unsliceable_axis.py +++ b/tests/test_slicing_unsliceable_axis.py @@ -13,12 +13,11 @@ class TestSlicing3DXarrayDatacube: def setup_method(self, method): # create a dataarray with 3 labelled axes using different index types - dims = np.random.randn(3, 1, 129) - array = xr.Dataset( - data_vars=dict(param=(["date", "variable", "level"], dims)), + array = xr.DataArray( + np.random.randn(3, 1, 129), + dims=("date", "variable", "level"), coords={"date": pd.date_range("2000-01-01", "2000-01-03", 3), "variable": ["a"], "level": range(1, 130)}, ) - array = array.to_array(dim="parameter") self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) @@ -26,22 +25,19 @@ def setup_method(self, method): # Testing different shapes def test_finding_existing_variable(self): - request = Request(Box(["level"], [10], [11]), Select("date", ["2000-01-01"]), Select("variable", ["a"]), - Select("parameter", ["param"])) + request = Request(Box(["level"], [10], [11]), Select("date", ["2000-01-01"]), Select("variable", ["a"])) result = self.API.retrieve(request) assert len(result.leaves) == 2 def test_finding_nonexisting_variable(self): - request = Request(Box(["level"], [10], [11]), Select("date", ["2000-01-01"]), Select("variable", ["b"]), - Select("parameter", ["param"])) + request = Request(Box(["level"], [10], [11]), Select("date", ["2000-01-01"]), Select("variable", ["b"])) with pytest.raises(ValueError): result = self.API.retrieve(request) result.pprint() def test_unsliceable_axis_in_a_shape(self): # does it work when we ask a box or disk of an unsliceable axis? - request = Request(Box(["level", "variable"], [10, "a"], [11, "a"]), Select("date", ["2000-01-01"]), - Select("parameter", ["param"])) + request = Request(Box(["level", "variable"], [10, "a"], [11, "a"]), Select("date", ["2000-01-01"])) with pytest.raises(UnsliceableShapeError): result = self.API.retrieve(request) result.pprint() diff --git a/tests/test_slicing_xarray_3D.py b/tests/test_slicing_xarray_3D.py index aca668c5f..a90943e36 100644 --- a/tests/test_slicing_xarray_3D.py +++ b/tests/test_slicing_xarray_3D.py @@ -24,16 +24,15 @@ class TestSlicing3DXarrayDatacube: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types - dims = np.random.randn(3, 6, 129) - array = xr.Dataset( - data_vars=dict(param=(["date", "step", "level"], dims)), + array = xr.DataArray( + np.random.randn(3, 6, 129), + dims=("date", "step", "level"), coords={ "date": pd.date_range("2000-01-01", "2000-01-03", 3), "step": [0, 3, 6, 9, 12, 15], "level": range(1, 130), }, ) - array = array.to_array(dim="variable") self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) @@ -41,79 +40,68 @@ def setup_method(self, method): # Testing different shapes def test_2D_box(self): - request = Request(Box(["step", "level"], [3, 10], [6, 11]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["step", "level"], [3, 10], [6, 11]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 4 def test_2D_box_union_disjoint_boxes(self): box1 = Box(["step", "level"], [3, 10], [6, 11]) box2 = Box(["step", "level"], [7, 15], [12, 17]) - request = Request(Union(["step", "level"], box1, box2), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Union(["step", "level"], box1, box2), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 4 + 6 def test_2D_box_union_overlapping_boxes(self): box1 = Box(["step", "level"], [3, 9], [6, 11]) box2 = Box(["step", "level"], [6, 10], [12, 17]) - request = Request(Union(["step", "level"], box1, box2), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Union(["step", "level"], box1, box2), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 6 + 24 - 2 def test_point(self): - request = Request(Select("date", ["2000-01-03"]), Select("level", [100]), Select("step", [3]), - Select("variable", ["param"])) + request = Request(Select("date", ["2000-01-03"]), Select("level", [100]), Select("step", [3])) result = self.API.retrieve(request) assert len(result.leaves) == 1 def test_segment(self): - request = Request(Span("level", 10, 11), Select("date", ["2000-01-01"]), Select("step", [9]), - Select("variable", ["param"])) + request = Request(Span("level", 10, 11), Select("date", ["2000-01-01"]), Select("step", [9])) result = self.API.retrieve(request) assert len(result.leaves) == 2 def test_union_line_point(self): seg1 = Span("step", 4.3, 6.2) pt1 = Select("step", [6.20001]) - request = Request(Union(["step"], seg1, pt1), Select("date", ["2000-01-01"]), Select("level", [100]), - Select("variable", ["param"])) + request = Request(Union(["step"], seg1, pt1), Select("date", ["2000-01-01"]), Select("level", [100])) result = self.API.retrieve(request) assert len(result.leaves) == 1 def test_union_boxes_intersect_one_point(self): box1 = Box(["step", "level"], [3, 10], [6, 11]) box2 = Box(["step", "level"], [6, 11], [12, 17]) - request = Request(Union(["step", "level"], box1, box2), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Union(["step", "level"], box1, box2), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 4 + 3 * 7 - 1 def test_mix_existing_nonexisting_data(self): - request = Request(Select("date", ["2000-01-03", "2000-01-04"]), Select("level", [100]), Select("step", [3]), - Select("variable", ["param"])) + request = Request(Select("date", ["2000-01-03", "2000-01-04"]), Select("level", [100]), Select("step", [3])) result = self.API.retrieve(request) assert len(result.leaves) == 1 def test_disk(self): - request = Request(Disk(["level", "step"], [6, 6], [3, 3]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Disk(["level", "step"], [6, 6], [3, 3]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 9 def test_concave_polygon(self): points = [[1, 0], [3, 0], [2, 3], [3, 6], [1, 6]] - request = Request(Polygon(["level", "step"], points), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Polygon(["level", "step"], points), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) self.xarraydatacube.get(result) assert len(result.leaves) == 8 def test_polytope(self): points = [[0, 1], [3, 1], [3, 2], [0, 2]] - request = Request(ConvexPolytope(["step", "level"], points), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(ConvexPolytope(["step", "level"], points), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) self.xarraydatacube.get(result) assert len(result.leaves) == 4 @@ -124,57 +112,49 @@ def test_union_empty_lines(self): # Slices non-existing step data seg1 = Span("step", 4, 5) seg2 = Span("step", 10, 11) - request = Request(Union(["step"], seg1, seg2), Select("date", ["2000-01-01"]), Select("level", [100]), - Select("variable", ["param"])) + request = Request(Union(["step"], seg1, seg2), Select("date", ["2000-01-01"]), Select("level", [100])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_empty_box_no_level(self): # Slices non-existing level data - request = Request(Box(["step", "level"], [3, 10.5], [7, 10.99]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["step", "level"], [3, 10.5], [7, 10.99]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_empty_box_no_level_step(self): # Slices non-existing level and step data - request = Request(Box(["step", "level"], [4, 10.5], [5, 10.99]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["step", "level"], [4, 10.5], [5, 10.99]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_empty_box_no_step(self): # Slices non-existing step and level data - request = Request(Box(["step", "level"], [4, 10], [5, 10.49]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["step", "level"], [4, 10], [5, 10.49]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_empty_box_floating_steps(self): # Slices through no step data and float type level data - request = Request(Box(["step", "level"], [4.1, 10.3], [5.7, 11.8]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["step", "level"], [4.1, 10.3], [5.7, 11.8]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_empty_box_no_step_level_float(self): # Slices empty step and level box - request = Request(Box(["step", "level"], [4.1, 10.3], [5.7, 10.8]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["step", "level"], [4.1, 10.3], [5.7, 10.8]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_empty_no_step_unordered(self): # Slice empty box because no step is available - request = Request(Box(["level", "step"], [10, 4], [10, 5]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["level", "step"], [10, 4], [10, 5]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_nonexisting_date(self): # Slices non-existing date data - request = Request(Select("date", ["2000-01-04"]), Select("level", [100]), Select("step", [3]), - Select("variable", ["param"])) + request = Request(Select("date", ["2000-01-04"]), Select("level", [100]), Select("step", [3])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -182,8 +162,7 @@ def test_two_nonexisting_close_points(self): # Slices two close points neither of which are available in the datacube pt1 = Select("step", [2.99]) pt2 = Select("step", [3.001]) - request = Request(Union(["step"], pt1, pt2), Select("level", [100]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Union(["step"], pt1, pt2), Select("level", [100]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -192,8 +171,7 @@ def test_union_two_nonexisting_points(self): # However if we round these points, we get points in the datacube pt1 = Select("step", [6.99]) pt2 = Select("step", [3.001]) - request = Request(Union(["step"], pt1, pt2), Select("level", [100]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Union(["step"], pt1, pt2), Select("level", [100]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -201,22 +179,19 @@ def test_two_close_points_no_level(self): # Slices non-existing step points and non-existing level pt1 = Select("step", [2.99]) pt2 = Select("step", [3.001]) - request = Request(Union(["step"], pt1, pt2), Select("level", [100.1]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Union(["step"], pt1, pt2), Select("level", [100.1]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_nonexisting_point_float_level(self): # Slices non-existing level data - request = Request(Select("step", [3]), Select("level", [99.1]), Select("date", ["2000-01-02"]), - Select("variable", ["param"])) + request = Request(Select("step", [3]), Select("level", [99.1]), Select("date", ["2000-01-02"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_nonexisting_segment(self): # Slices non-existing step data - request = Request(Span("step", 3.2, 3.23), Select("level", [99]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Span("step", 3.2, 3.23), Select("level", [99]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -224,15 +199,13 @@ def test_nonexisting_segment(self): def test_flat_box(self): # Should slice through a line in the step direction - request = Request(Box(["step", "level"], [4, 10], [7, 10]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["step", "level"], [4, 10], [7, 10]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 1 def test_box(self): # Should slice a line in the level direction - request = Request(Box(["level", "step"], [3, 3], [6, 3]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["level", "step"], [3, 3], [6, 3]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 4 @@ -241,8 +214,7 @@ def test_swept_concave_polygon(self): points = [(1, 0), (3, 0), (3, 6), (2, 6), (2, 3), (1, 3)] concave_polygon = Polygon(["level", "step"], points) swept_poly = PathSegment(["level", "step"], concave_polygon, [0, 0], [1, 3]) - request = Request(swept_poly, Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(swept_poly, Select("date", ["2000-01-01"])) result = self.API.retrieve(request) self.xarraydatacube.get(result) assert len(result.leaves) == 12 @@ -255,8 +227,7 @@ def test_intersection_point_disk_polygon(self): r1 = math.cos(math.pi / 12) * (8 - 4 * math.sqrt(3)) + sys.float_info.epsilon # note that we need a small perturbation to make up for rounding errors r2 = 3 * math.cos(math.pi / 12) * (math.sqrt(3) - 2) * (8 - 4 * math.sqrt(3)) / (4 * math.sqrt(3) - 7) - request = Request(Disk(["level", "step"], [0, 0], [r1, r2]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Disk(["level", "step"], [0, 0], [r1, r2]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) paths = [r.flatten().values() for r in result.leaves] - assert (pd.Timestamp("2000-01-01 00:00:00"), 3.0, 1.0, "param") in paths + assert (pd.Timestamp("2000-01-01 00:00:00"), 3.0, 1.0) in paths diff --git a/tests/test_slicing_xarray_4D.py b/tests/test_slicing_xarray_4D.py index 7d8025d6e..449e5838f 100644 --- a/tests/test_slicing_xarray_4D.py +++ b/tests/test_slicing_xarray_4D.py @@ -24,9 +24,9 @@ class TestSlicing4DXarrayDatacube: def setup_method(self, method): # Create a dataarray with 4 labelled axes using different index types - dims = np.random.randn(3, 7, 129, 100) - array = xr.Dataset( - data_vars=dict(param=(["date", "step", "level", "lat"], dims)), + array = xr.DataArray( + np.random.randn(3, 7, 129, 100), + dims=("date", "step", "level", "lat"), coords={ "date": pd.date_range("2000-01-01", "2000-01-03", 3), "step": [0, 3, 6, 9, 12, 15, 18], @@ -34,7 +34,6 @@ def setup_method(self, method): "lat": np.around(np.arange(0.0, 10.0, 0.1), 15), }, ) - array = array.to_array() self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) @@ -42,23 +41,18 @@ def setup_method(self, method): # Testing different shapes def test_3D_box(self): - request = Request(Box(["step", "level", "lat"], [3, 10, 5.0], [6, 11, 6.0]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Box(["step", "level", "lat"], [3, 10, 5.0], [6, 11, 6.0]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 2 * 2 * 11 def test_4D_box(self): - request = Request( - Box(["step", "level", "lat", "date"], [3, 10, 5.0, "2000-01-01"], [6, 11, 6.0, "2000-01-02"]), - Select("variable", ["param"]) - ) + request = Request(Box(["step", "level", "lat", "date"], [3, 10, 5.0, "2000-01-01"], [6, 11, 6.0, "2000-01-02"])) result = self.API.retrieve(request) assert len(result.leaves) == 2 * 2 * 11 * 2 def test_circle_int(self): request = Request( - Disk(["step", "level"], [9, 10], [6, 6]), Select("date", ["2000-01-01"]), Select("lat", [5.2]), - Select("variable", ["param"]) + Disk(["step", "level"], [9, 10], [6, 6]), Select("date", ["2000-01-01"]), Select("lat", [5.2]) ) result = self.API.retrieve(request) assert len(result.leaves) == 37 @@ -66,32 +60,28 @@ def test_circle_int(self): def test_circles_barely_touching_int(self): disk1 = Disk(["step", "level"], [6, 10], [5.9, 6]) disk2 = Disk(["step", "level"], [15, 10], [2.9, 3]) - request = Request(Union(["step", "level"], disk1, disk2), Select("date", ["2000-01-01"]), Select("lat", [5.1]), - Select("variable", ["param"])) + request = Request(Union(["step", "level"], disk1, disk2), Select("date", ["2000-01-01"]), Select("lat", [5.1])) result = self.API.retrieve(request) assert len(result.leaves) == 45 def test_circles_intersecting_float(self): disk1 = Disk(["step", "lat"], [6, 4.0], [6.99, 0.1]) disk2 = Disk(["step", "lat"], [15, 2.0], [4.99, 0.3]) - request = Request(Union(["step", "lat"], disk1, disk2), Select("date", ["2000-01-01"]), Select("level", [10]), - Select("variable", ["param"])) + request = Request(Union(["step", "lat"], disk1, disk2), Select("date", ["2000-01-01"]), Select("level", [10])) result = self.API.retrieve(request) assert len(result.leaves) == 24 def test_circles_touching_float(self): disk1 = Disk(["step", "lat"], [6, 4.0], [3, 1.9]) disk2 = Disk(["step", "lat"], [15, 2.0], [3, 2.1]) - request = Request(Union(["step", "lat"], disk1, disk2), Select("date", ["2000-01-01"]), Select("level", [10]), - Select("variable", ["param"])) + request = Request(Union(["step", "lat"], disk1, disk2), Select("date", ["2000-01-01"]), Select("level", [10])) result = self.API.retrieve(request) assert len(result.leaves) == 101 def test_pathsegment_swept_2D_box(self): box1 = Box(["step", "level"], [3, 0], [6, 1]) request = Request( - PathSegment(["step", "level"], box1, [3, 1], [6, 2]), Select("lat", [4.0]), Select("date", ["2000-01-01"]), - Select("variable", ["param"]) + PathSegment(["step", "level"], box1, [3, 1], [6, 2]), Select("lat", [4.0]), Select("date", ["2000-01-01"]) ) result = self.API.retrieve(request) # result.pprint() @@ -101,8 +91,7 @@ def test_pathsegment_swept_2D_box_bis(self): # Had a floating point problem because of the latitude box1 = Box(["step", "level"], [3, 3], [6, 5]) request = Request( - PathSegment(["step", "level"], box1, [3, 3], [6, 6]), Select("lat", [4.1]), Select("date", ["2000-01-01"]), - Select("variable", ["param"]) + PathSegment(["step", "level"], box1, [3, 3], [6, 6]), Select("lat", [4.1]), Select("date", ["2000-01-01"]) ) result = self.API.retrieve(request) # result.pprint() @@ -111,8 +100,7 @@ def test_pathsegment_swept_2D_box_bis(self): def test_pathsegment_swept_circle(self): circ1 = Disk(["step", "level"], [6, 3], [3, 2]) request = Request( - PathSegment(["step", "level"], circ1, [3, 3], [6, 6]), Select("lat", [5.5]), Select("date", ["2000-01-01"]), - Select("variable", ["param"]) + PathSegment(["step", "level"], circ1, [3, 3], [6, 6]), Select("lat", [5.5]), Select("date", ["2000-01-01"]) ) result = self.API.retrieve(request) # result.pprint() @@ -121,8 +109,7 @@ def test_pathsegment_swept_circle(self): def test_path_swept_box_2_points(self): box1 = Box(["step", "level"], [3, 3], [6, 5]) request = Request( - Path(["step", "level"], box1, [3, 3], [6, 6]), Select("lat", [4.3]), Select("date", ["2000-01-01"]), - Select("variable", ["param"]) + Path(["step", "level"], box1, [3, 3], [6, 6]), Select("lat", [4.3]), Select("date", ["2000-01-01"]) ) result = self.API.retrieve(request) assert len(result.leaves) == 12 @@ -130,22 +117,20 @@ def test_path_swept_box_2_points(self): def test_path_swept_box_3_points(self): box1 = Box(["step", "level"], [3, 3], [6, 5]) request = Request( - Path(["step", "level"], box1, [3, 3], [6, 6], [9, 9]), Select("lat", [4.3]), Select("date", ["2000-01-01"]), - Select("variable", ["param"]) + Path(["step", "level"], box1, [3, 3], [6, 6], [9, 9]), Select("lat", [4.3]), Select("date", ["2000-01-01"]) ) result = self.API.retrieve(request) assert len(result.leaves) == 18 def test_polygon_other_than_triangle(self): polygon = Polygon(["step", "level"], [[3, 3], [3, 5], [6, 5], [6, 7]]) - request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 6 def test_ellipsoid(self): ellipsoid = Ellipsoid(["step", "level", "lat"], [6, 3, 2.1], [3, 1, 0.1]) - request = Request(ellipsoid, Select("date", ["2000-01-01"]), Select("variable", ["param"])) + request = Request(ellipsoid, Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 7 @@ -154,8 +139,7 @@ def test_ellipsoid(self): def test_empty_circle(self): # Slices a circle with no data inside request = Request( - Disk(["step", "level"], [5, 3.4], [0.5, 0.2]), Select("date", ["2000-01-01"]), Select("lat", [5.1]), - Select("variable", ["param"]) + Disk(["step", "level"], [5, 3.4], [0.5, 0.2]), Select("date", ["2000-01-01"]), Select("lat", [5.1]) ) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -163,8 +147,7 @@ def test_empty_circle(self): def test_float_box(self): # Slices a box with no data inside request = Request( - Box(["step", "lat"], [10.1, 1.01], [10.3, 1.04]), Select("date", ["2000-01-01"]), Select("level", [10]), - Select("variable", ["param"]) + Box(["step", "lat"], [10.1, 1.01], [10.3, 1.04]), Select("date", ["2000-01-01"]), Select("level", [10]) ) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -173,8 +156,7 @@ def test_path_empty_box(self): # Slices the path of a box with no data inside, but gives data because the box is swept over a datacube value box1 = Box(["step", "level"], [2.4, 3.1], [2.5, 3.4]) request = Request( - PathSegment(["step", "level"], box1, [3, 3], [6, 6]), Select("lat", [4.0]), Select("date", ["2000-01-01"]), - Select("variable", ["param"]) + PathSegment(["step", "level"], box1, [3, 3], [6, 6]), Select("lat", [4.0]), Select("date", ["2000-01-01"]) ) result = self.API.retrieve(request) assert len(result.leaves) == 1 @@ -186,7 +168,6 @@ def test_path_empty_box_empty(self): PathSegment(["step", "level"], box1, [1.1, 3.3], [2.7, 3.6]), Select("lat", [4.0]), Select("date", ["2000-01-01"]), - Select("variable", ["param"]) ) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -194,7 +175,7 @@ def test_path_empty_box_empty(self): def test_ellipsoid_empty(self): # Slices an empty ellipsoid which doesn't have any step value ellipsoid = Ellipsoid(["step", "level", "lat"], [5, 3, 2.1], [0, 0, 0]) - request = Request(ellipsoid, Select("date", ["2000-01-01"]), Select("variable", ["param"])) + request = Request(ellipsoid, Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -203,8 +184,7 @@ def test_ellipsoid_empty(self): def test_span_bounds(self): # Tests that span also works in reverse order request = Request( - Span("level", 100, 98), Select("step", [3]), Select("lat", [5.5]), Select("date", ["2000-01-01"]), - Select("variable", ["param"]) + Span("level", 100, 98), Select("step", [3]), Select("lat", [5.5]), Select("date", ["2000-01-01"]) ) result = self.API.retrieve(request) assert len(result.leaves) == 3 @@ -214,72 +194,64 @@ def test_span_bounds(self): def test_ellipsoid_one_point(self): # Slices through a point (center of the ellipsoid) ellipsoid = Ellipsoid(["step", "level", "lat"], [6, 3, 2.1], [0, 0, 0]) - request = Request(ellipsoid, Select("date", ["2000-01-01"]), Select("variable", ["param"])) + request = Request(ellipsoid, Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 1 assert not result.leaves[0].axis == IndexTree.root def test_flat_box_level(self): # Slices a line in the step direction - request = Request(Select("lat", [6]), Box(["level", "step"], [3, 3], [3, 9]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Select("lat", [6]), Box(["level", "step"], [3, 3], [3, 9]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 3 def test_flat_box_step(self): # Slices a line in the level direction - request = Request(Select("lat", [6]), Box(["level", "step"], [3, 3], [7, 3]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Select("lat", [6]), Box(["level", "step"], [3, 3], [7, 3]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 5 def test_flat_disk_nonexisting(self): # Slices an empty disk because there is no step level - request = Request(Disk(["level", "step"], [4, 5], [4, 0]), Select("lat", [6]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Disk(["level", "step"], [4, 5], [4, 0]), Select("lat", [6]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_flat_disk_line(self): # Slices a line in the level direction - request = Request(Disk(["level", "step"], [4, 6], [4, 0]), Select("lat", [6]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Disk(["level", "step"], [4, 6], [4, 0]), Select("lat", [6]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 8 def test_flat_disk_line_step(self): # Slices a line in the step direction - request = Request(Disk(["level", "step"], [4, 6], [0, 3]), Select("lat", [6]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Disk(["level", "step"], [4, 6], [0, 3]), Select("lat", [6]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 3 def test_flat_disk_empty(self): # Slices an empty disk because there is no step - request = Request(Disk(["level", "step"], [4, 5], [0, 0.5]), Select("lat", [6]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Disk(["level", "step"], [4, 5], [0, 0.5]), Select("lat", [6]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_disk_point(self): # Slices a point because the origin of the disk is a datacube point - request = Request(Disk(["level", "step"], [4, 6], [0, 0]), Select("lat", [6]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Disk(["level", "step"], [4, 6], [0, 0]), Select("lat", [6]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 1 assert not result.leaves[0].axis == IndexTree.root def test_empty_disk(self): # Slices an empty object because the origin of the disk is not a datacube point - request = Request(Disk(["level", "step"], [4, 5], [0, 0]), Select("lat", [6]), Select("date", ["2000-01-01"]), - Select("variable", ["param"])) + request = Request(Disk(["level", "step"], [4, 5], [0, 0]), Select("lat", [6]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root def test_polygon_line(self): # Slices a line defined through the polygon shape polygon = Polygon(["step", "level"], [[3, 3], [3, 6], [3, 3], [3, 3]]) - request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"]), Select("variable", ["param"])) + request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 4 @@ -287,7 +259,7 @@ def test_polygon_point(self): # Slices a point defined through the polygon object with several initial point entries. # Tests whether duplicate points are removed as they should polygon = Polygon(["step", "level"], [[3, 3], [3, 3], [3, 3], [3, 3]]) - request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"]), Select("variable", ["param"])) + request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 1 assert not result.leaves[0].axis == IndexTree.root @@ -295,7 +267,7 @@ def test_polygon_point(self): def test_polygon_empty(self): # Slices a point which isn't in the datacube (defined through the polygon shape) polygon = Polygon(["step", "level"], [[2, 3.1]]) - request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"]), Select("variable", ["param"])) + request = Request(polygon, Select("lat", [4.3]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert result.leaves[0].axis == IndexTree.root @@ -306,19 +278,18 @@ def test_axis_specified_twice(self): request = Request( Box(["step", "level"], [3, 10], [6, 11]), Box(["step", "lat", "date"], [3, 5.0, "2000-01-01"], [6, 6.0, "2000-01-02"]), - Select("variable", ["param"]) ) result = self.API.retrieve(request) result.pprint() def test_not_all_axes_defined(self): with pytest.raises(AxisUnderdefinedError): - request = Request(Box(["step", "level"], [3, 10], [6, 11]), Select("variable", ["param"])) + request = Request(Box(["step", "level"], [3, 10], [6, 11])) result = self.API.retrieve(request) result.pprint() def test_not_all_axes_exist(self): with pytest.raises(KeyError): - request = Request(Box(["weather", "level"], [3, 10], [6, 11]), Select("variable", ["param"])) + request = Request(Box(["weather", "level"], [3, 10], [6, 11])) result = self.API.retrieve(request) result.pprint() From 6b32085a0ec4ff1e8069b2a1e14daa3564e78960 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 22 Jun 2023 10:01:53 +0200 Subject: [PATCH 006/332] fix link to paper --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index b2ac63de7..892388bcd 100644 --- a/readme.md +++ b/readme.md @@ -67,7 +67,7 @@ The Polytope algorithm can for example be used to extract: - and many more high-dimensional shapes in arbitrary dimensions... -For more information about the Polytope algorithm, refer to our [paper] (https://arxiv.org/abs/2306.11553). +For more information about the Polytope algorithm, refer to our [paper](https://arxiv.org/abs/2306.11553). If this project is useful for your work, please consider citing this paper. ## Installation From e5d788f7d7879573248033e0dfbd4924193b25d9 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 28 Jun 2023 10:36:02 +0100 Subject: [PATCH 007/332] make octahedral grid work --- polytope/datacube/datacube.py | 2 +- polytope/datacube/octahedral_xarray.py | 251 +++++++++++++++++++++++++ tests/test_octahedral_grid.py | 50 +++++ 3 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 polytope/datacube/octahedral_xarray.py create mode 100644 tests/test_octahedral_grid.py diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index 1fde53de7..2b14c0dc0 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -43,7 +43,7 @@ def validate(self, axes) -> bool: @staticmethod def create(datacube, options): if isinstance(datacube, (xr.core.dataarray.DataArray, xr.core.dataset.Dataset)): - from .xarray import XArrayDatacube + from .octahedral_xarray import XArrayDatacube xadatacube = XArrayDatacube(datacube, options=options) return xadatacube diff --git a/polytope/datacube/octahedral_xarray.py b/polytope/datacube/octahedral_xarray.py new file mode 100644 index 000000000..f14b5fa2f --- /dev/null +++ b/polytope/datacube/octahedral_xarray.py @@ -0,0 +1,251 @@ +import math +import sys +from copy import deepcopy + +import numpy as np +import pandas as pd +import xarray as xr + +from ..utility.combinatorics import unique, validate_axes +from .datacube import Datacube, DatacubePath, IndexTree +from .datacube_axis import ( + FloatAxis, + IntAxis, + PandasTimedeltaAxis, + PandasTimestampAxis, + UnsliceableaAxis, +) + +_mappings = { + pd.Int64Dtype: IntAxis(), + pd.Timestamp: PandasTimestampAxis(), + np.int64: IntAxis(), + np.datetime64: PandasTimestampAxis(), + np.timedelta64: PandasTimedeltaAxis(), + np.float64: FloatAxis(), + np.str_: UnsliceableaAxis(), + str: UnsliceableaAxis(), +} + + +class XArrayDatacube(Datacube): + """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0).""" + + def _set_mapper(self, values, name): + if values.dtype.type not in _mappings: + raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") + if name in self.options.keys(): + # The options argument here is supposed to be a nested dictionary + # like {"latitude":{"Cyclic":range}, ...} + if "Cyclic" in self.options[name].keys(): + value_type = values.dtype.type + axes_type_str = type(_mappings[value_type]).__name__ + axes_type_str += "Cyclic" + cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) + self.mappers[name] = cyclic_axis_type + self.mappers[name].name = name + self.mappers[name].range = self.options[name]["Cyclic"] + else: + self.mappers[name] = deepcopy(_mappings[values.dtype.type]) + self.mappers[name].name = name + + def __init__(self, dataarray: xr.DataArray, options={}): + self.options = options + self.mappers = {} + # print(dataarray.coords) + for name, values in dataarray.coords.variables.items(): + if values.data.size != 1: + dataarray = dataarray.sortby(name) + self._set_mapper(values, name) + else: # drop non-necessary coordinates which we don't slice on + dataarray = dataarray.reset_coords(names=name, drop=True) + + self.dataarray = dataarray + + def get(self, requests: IndexTree): + for r in requests.leaves: + path = r.flatten() + path = self.remap_path(path) + len_coords = 0 + for name, values in self.dataarray.coords.variables.items(): + if values.data.size != 1: + len_coords += 1 + if len(path.items()) == len_coords: + lat_val = path["latitude"] + lon_val = path["longitude"] + path.pop("longitude", None) + path.pop("latitude", None) + subxarray = self.dataarray.sel(path, method="nearest") + # need to remap the lat, lon in path to dataarray index + lat_idx, lon_idx = latlon_val_to_idx(lat_val, lon_val) + octa_idx = latlon_idx_to_octa_idx(lat_idx, lon_idx) + subxarray = subxarray.isel(values=octa_idx) + value = subxarray.item() + key = subxarray.name + r.result = (key, value) + else: + r.remove_branch() + + def get_mapper(self, axis): + return self.mappers[axis] + + def remap_path(self, path: DatacubePath): + for key in path: + value = path[key] + path[key] = self.mappers[key].remap_val_to_axis_range(value) + return path + + def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): + idx_between = [] + for i in range(len(search_ranges)): + r = search_ranges[i] + offset = search_ranges_offset[i] + low = r[0] + up = r[1] + + if axis.name == "latitude" or axis.name == "longitude": + indexes.sort() + indexes_between = [i for i in indexes if low <= i <= up] + else: + # Find the range of indexes between lower and upper + # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html + # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing + start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? + end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? + indexes_between = indexes[start:end].to_list() + + # Now the indexes_between are values on the cyclic range so need to remap them to their original + # values before returning them + for j in range(len(indexes_between)): + if offset is None: + indexes_between[j] = indexes_between[j] + else: + indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) + + idx_between.append(indexes_between[j]) + return idx_between + + def get_indices(self, path: DatacubePath, axis, lower, upper): + path = self.remap_path(path) + print(axis.name) + print(path) + # Open a view on the subset identified by the path + lat_val = path.get("latitude", None) + path.pop("longitude", None) + path.pop("latitude", None) + print(path) + subarray = self.dataarray.sel(path, method="nearest") + + # Get the indexes of the axis we want to query + # XArray does not support branching, so no need to use label, we just take the next axis + # indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + if axis.name == "latitude": + # subxarray_lat = xr.DataArray(self.lat_val_available(), dims=("latitude"), + # coords={ + # "latitude": self.lat_val_available(), + # },) + # indexes = next(iter(subxarray_lat.xindexes.values())).to_pandas_index() + indexes = self.lat_val_available() + # indexes = pd.Index(tuple(indexes)) if indexes else None + elif axis.name == "longitude": + # lat_val = path["latitude"] + # subxarray_lon = xr.DataArray(self.lon_val_available(lat_val), dims=("longitude"), + # coords={ + # "longitude": self.lon_val_available(lat_val), + # },) + # indexes = next(iter(subxarray_lon.xindexes.values())).to_pandas_index() + indexes = self.lon_val_available(lat_val) + # indexes = indexes[0].to_pandas_index() if indexes else None + else: + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + + # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube + search_ranges = axis.remap([lower, upper]) + original_search_ranges = axis.to_intervals([lower, upper]) + + # Find the offsets for each interval in the requested range, which we will need later + search_ranges_offset = [] + for r in original_search_ranges: + offset = axis.offset(r) + search_ranges_offset.append(offset) + + # Look up the values in the datacube for each cyclic interval range + idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis) + + # Remove duplicates even if difference of the order of the axis tolerance + if offset is not None: + # Note that we can only do unique if not dealing with time values + idx_between = unique(idx_between) + + return idx_between + + def has_index(self, path: DatacubePath, axis, index): + # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube + subarray = self.dataarray.sel(path)[axis.name] + subarray_vals = subarray.values + return index in subarray_vals + + @property + def axes(self): + return self.mappers + + def validate(self, axes): + return validate_axes(self.axes, axes) + + def lat_val_available(self): + lat_spacing = 90/1280 + lat_start = 0 + return_lat = [lat_start + i * lat_spacing for i in range(1280)] + return return_lat + + def lon_val_available(self, lat): + lat_spacing = 90/1280 + lat_start = 0 + lat_idx = (lat-lat_start)/lat_spacing + num_points_on_lon = 4 * lat_idx + 16 + lon_spacing = 180/num_points_on_lon + lon_start = 0 + return_lon = [lon_start + i * lon_spacing for i in range(int(num_points_on_lon))] + return return_lon + + +def octa_idx_to_latlon_idx(idx): + lat_j = math.floor(-3.5 + (math.sqrt(81+2*idx)/2)) + lon_j = idx - 2 * lat_j * lat_j - 14 * lat_j + 16 + + # NOTE to get idx of lat and lon, need to substract 1 to start from 0 for lat + + lat_idx = lat_j - 1 + lon_idx = lon_j + + return (lat_idx, lon_idx) + + +def latlon_idx_to_octa_idx(lat_idx, lon_idx): + return int(2*lat_idx*lat_idx + 15*lat_idx - 16 + lon_idx) + + +def latlon_idx_to_val(lat_idx, lon_idx): + # spacing between different lat levels + lat_spacing = 90/1280 + num_points_on_lon = 4 * lat_idx + 16 + lon_spacing = 180/num_points_on_lon + + # TODO: this might be wrong, and the spacing above too for the lat, + # depends on how the grid is laid out onto the sphere + lat_start = 0 + lon_start = 0 + return (lat_idx*lat_spacing + lat_start, lon_idx*lon_spacing + lon_start) + + +def latlon_val_to_idx(lat, lon): + lat_spacing = 90/1280 + lat_start = 0 + lat_idx = (lat-lat_start)/lat_spacing + + num_points_on_lon = 4 * lat_idx + 16 + lon_spacing = 180/num_points_on_lon + lon_start = 0 + lon_idx = (lon-lon_start)/lon_spacing + + return (lat_idx, lon_idx) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py new file mode 100644 index 000000000..6a21c7e03 --- /dev/null +++ b/tests/test_octahedral_grid.py @@ -0,0 +1,50 @@ +import math + +from earthkit import data + +from polytope.datacube.octahedral_xarray import XArrayDatacube +from polytope.datacube.datacube_axis import FloatAxis +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + +ds = data.from_source("file", "./foo.grib") +latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) +latlon_array = latlon_array.t2m + +# print(latlon_array) + + +# # Going from an index in the latlon array to individual lat and lon numbers +# index = 44+28 + +# lat_j = math.floor(-3.5 + (math.sqrt(81+2*index)/2)) +# lon_j = index - 2 * lat_j * lat_j - 14 * lat_j + 16 + +# # NOTE to get idx of lat and lon, need to substract 1 to start from 0 for lat + +# lat_idx = lat_j - 1 +# lon_idx = lon_j + +# print(lat_idx) +# print(lon_idx) + +latlon_xarray_datacube = XArrayDatacube(latlon_array) + +# print(xarraydatacube.mappers) + +# lat_axis = FloatAxis() +# lat_axis.name = "latitude" +# xarraydatacube.get_indices({}, lat_axis, 5, 10) + +# subxarray = latlon_array.isel(values=20) +# value = subxarray.item() +# key = subxarray.name +# print((value, key)) + +slicer = HullSlicer() +API = Polytope(datacube=latlon_array, engine=slicer) + +request = Request(Box(["latitude", "longitude"], [5, 5], [10, 10])) +result = API.retrieve(request) +result.pprint() From 1cd653a8d0585bc7438fcb308eaee5de52a366e2 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 23 Jun 2023 13:37:30 +0200 Subject: [PATCH 008/332] add contributing file and sections in readme --- CONTRIBUTING.rst | 20 ++++++++++++++++++++ readme.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 CONTRIBUTING.rst diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 000000000..d2086657a --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,20 @@ +Contributing to *polytope* +======================================== + +Please report bug_ reports or pull-requests_ on GitHub_ + +.. _bug: https://github.com/ecmwf/polytope/issues + +.. _pull-requests: https://github.com/ecmwf/polytope/pulls + +.. _GitHub: https://github.com/ecmwf/polytope + +We want your feedback, please e-mail: user-services@ecmwf.int + +The package is installed from PyPI with:: + + $ pip install polytope-python + +How you could use polytope: + * plot outputs, examples are provided in the examples folder + * go further than our examples and let us know how it goes \ No newline at end of file diff --git a/readme.md b/readme.md index 892388bcd..6ff8e8008 100644 --- a/readme.md +++ b/readme.md @@ -154,3 +154,50 @@ The Polytope tests and examples require additional Python packages compared to t The additional dependencies are provided in the requirements_test.txt and requirements_examples.txt files, which can respectively be found in the examples and tests folders. Moreover, Polytope's tests and examples also require the installation of eccodes and GDAL. It is possible to install both of these dependencies using either a package manager or manually. + +## Contributing + +The main repository is hosted on GitHub; testing, bug reports and contributions are highly welcomed and appreciated. +Please see the [Contributing](./CONTRIBUTING.rst) document for the best way to help. + +Main contributors: + +- Mathilde Leuridan - [ECMWF](www.ecmwf.int) +- James Hawkes - [ECMWF](www.ecmwf.int) + + +See also the [contributors](https://github.com/ecmwf-projects/thermofeel/contributors) for a more complete list. + +## License + +Copyright 2021 European Centre for Medium-Range Weather Forecasts (ECMWF) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +In applying this licence, ECMWF does not waive the privileges and immunities +granted to it by virtue of its status as an intergovernmental organisation nor +does it submit to any jurisdiction. + +## Citing + +In this software is useful in your work, please consider citing our [paper](https://arxiv.org/abs/2306.11553) as + + Leuridan, M., Hawkes, J., Smart, S., Danovaro, E., and Quintino, T., “Polytope: An Algorithm for Efficient Feature Extraction on Hypercubes”, arXiv e-prints, 2023. doi:10.48550/arXiv.2306.11553. + +## Acknowledgements + +Past and current funding and support for **polytope** is listed in the adjoining [Acknowledgements](./AKNOWLEDGEMENTS.rst) + + From dc2e81976374851a7652bb83d987f5855b33a488 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 23 Jun 2023 13:38:42 +0200 Subject: [PATCH 009/332] add contributing file and sections in readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 6ff8e8008..7ca7876b7 100644 --- a/readme.md +++ b/readme.md @@ -194,7 +194,7 @@ does it submit to any jurisdiction. In this software is useful in your work, please consider citing our [paper](https://arxiv.org/abs/2306.11553) as - Leuridan, M., Hawkes, J., Smart, S., Danovaro, E., and Quintino, T., “Polytope: An Algorithm for Efficient Feature Extraction on Hypercubes”, arXiv e-prints, 2023. doi:10.48550/arXiv.2306.11553. +Leuridan, M., Hawkes, J., Smart, S., Danovaro, E., and Quintino, T., “Polytope: An Algorithm for Efficient Feature Extraction on Hypercubes”, arXiv e-prints, 2023. doi:10.48550/arXiv.2306.11553. ## Acknowledgements From 4c7ce8d3b964cdc0eb7dd9f7736c20629a5ea54e Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 23 Jun 2023 13:44:37 +0200 Subject: [PATCH 010/332] add contributing file and sections in readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 7ca7876b7..f32a5331e 100644 --- a/readme.md +++ b/readme.md @@ -194,7 +194,7 @@ does it submit to any jurisdiction. In this software is useful in your work, please consider citing our [paper](https://arxiv.org/abs/2306.11553) as -Leuridan, M., Hawkes, J., Smart, S., Danovaro, E., and Quintino, T., “Polytope: An Algorithm for Efficient Feature Extraction on Hypercubes”, arXiv e-prints, 2023. doi:10.48550/arXiv.2306.11553. +> Leuridan, M., Hawkes, J., Smart, S., Danovaro, E., and Quintino, T., “Polytope: An Algorithm for Efficient Feature Extraction on Hypercubes”, arXiv e-prints, 2023. doi:10.48550/arXiv.2306.11553. ## Acknowledgements From e81e86170b3eee23c2757c1c666deb007a2f9284 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 23 Jun 2023 13:45:52 +0200 Subject: [PATCH 011/332] add contributing file and sections in readme --- readme.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/readme.md b/readme.md index f32a5331e..c8fdd9431 100644 --- a/readme.md +++ b/readme.md @@ -176,9 +176,7 @@ Copyright 2021 European Centre for Medium-Range Weather Forecasts (ECMWF) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 +You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0). Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, From a3020e2de7f5bef2c7c5acbc4cb87d22984cf4c8 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 23 Jun 2023 13:50:16 +0200 Subject: [PATCH 012/332] add contributing file and sections in readme --- readme.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/readme.md b/readme.md index c8fdd9431..d45e74c4c 100644 --- a/readme.md +++ b/readme.md @@ -85,40 +85,45 @@ or from PyPI with the command Here is a step-by-step example of how to use this software. 1. In this example, we first specify the data which will be in our Xarray datacube. Note that the data here comes from the GRIB file called "winds.grib", which is 3-dimensional with dimensions: step, latitude and longitude. - + ```Python import xarray as xr array = xr.open_dataset("winds.grib", engine="cfgrib") + ``` We then construct the Polytope object, passing in some additional metadata describing properties of the longitude axis. - + ```Python options = {"longitude": {"Cyclic": [0, 360.0]}} from polytope.polytope import Polytope p = Polytope(datacube=array, options=options) + ``` 2. Next, we create a request shape to extract from the datacube. In this example, we want to extract a simple 2D box in latitude and longitude at step 0. We thus create the two relevant shapes we need to build this 3-dimensional object, - + ```Python import numpy as np from polytope.shapes import Box, Select box = Box(["latitude", "longitude"], [0, 0], [1, 1]) step_point = Select("step", [np.timedelta64(0, "s")]) + ``` which we then incorporate into a Polytope request. - + ```Python from polytope.polytope import Request request = Request(box, step_point) + ``` 3. Finally, extract the request from the datacube. - + ```Python result = p.retrieve(request) + ``` The result is stored as an IndexTree containing the retrieved data organised hierarchically with axis indices for each point. - + ```Python result.pprint() @@ -132,6 +137,7 @@ Here is a step-by-step example of how to use this software. ↳latitude=1.0 ↳longitude=0.0 ↳longitude=1.0 + ``` -See also the [contributors](https://github.com/ecmwf-projects/thermofeel/contributors) for a more complete list. +See also the [contributors](https://github.com/ecmwf/polytope/contributors) for a more complete list. ## License From e4ce306242464622c22001b6c21af724be728e67 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 23 Jun 2023 13:56:56 +0200 Subject: [PATCH 016/332] add contributing file and sections in readme --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 420b0bfda..25fdbee10 100644 --- a/readme.md +++ b/readme.md @@ -168,8 +168,8 @@ Please see the [Contributing](./CONTRIBUTING.rst) document for the best way to h Main contributors: -- Mathilde Leuridan - [ECMWF](www.ecmwf.int) -- James Hawkes - [ECMWF](www.ecmwf.int) +- Mathilde Leuridan - [ECMWF](https://www.ecmwf.int) +- James Hawkes - [ECMWF](https://www.ecmwf.int) From 11fdbc7de497399961b5838cdc590826229599d4 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 23 Jun 2023 13:58:26 +0200 Subject: [PATCH 017/332] update readme --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 25fdbee10..1aeecf859 100644 --- a/readme.md +++ b/readme.md @@ -170,9 +170,9 @@ Main contributors: - Mathilde Leuridan - [ECMWF](https://www.ecmwf.int) - James Hawkes - [ECMWF](https://www.ecmwf.int) - +- Tiago Quintino - [ECMWF](www.ecmwf.int) See also the [contributors](https://github.com/ecmwf/polytope/contributors) for a more complete list. From 9d590f6e78e09784ebfc6376fb09cd5d9f69f826 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 23 Jun 2023 14:02:56 +0200 Subject: [PATCH 018/332] update readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 1aeecf859..9e2628a35 100644 --- a/readme.md +++ b/readme.md @@ -202,6 +202,6 @@ In this software is useful in your work, please consider citing our [paper](http ## Acknowledgements -Past and current funding and support for **polytope** is listed in the adjoining [Acknowledgements](./ACKNOWLEDGEMENTS.rst). +Past and current funding and support for *polytope* is listed in the adjoining [Acknowledgements](./ACKNOWLEDGEMENTS.rst). From 7dbaefa0df92139ba87603b18454c564c8531f7a Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 23 Jun 2023 14:03:16 +0200 Subject: [PATCH 019/332] update readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 9e2628a35..cf88e942d 100644 --- a/readme.md +++ b/readme.md @@ -202,6 +202,6 @@ In this software is useful in your work, please consider citing our [paper](http ## Acknowledgements -Past and current funding and support for *polytope* is listed in the adjoining [Acknowledgements](./ACKNOWLEDGEMENTS.rst). +Past and current funding and support for *Polytope* is listed in the adjoining [Acknowledgements](./ACKNOWLEDGEMENTS.rst). From 75d104f51ee15cb19d481019c8b3f395f8b39ac5 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 23 Jun 2023 14:04:13 +0200 Subject: [PATCH 020/332] update readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index cf88e942d..5be58455b 100644 --- a/readme.md +++ b/readme.md @@ -202,6 +202,6 @@ In this software is useful in your work, please consider citing our [paper](http ## Acknowledgements -Past and current funding and support for *Polytope* is listed in the adjoining [Acknowledgements](./ACKNOWLEDGEMENTS.rst). +Past and current funding and support for **Polytope** is listed in the adjoining [Acknowledgements](./ACKNOWLEDGEMENTS.rst). From 3d6bb906b4fd91e1e58ddb99eb419342bba2175f Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 26 Jun 2023 08:59:21 +0100 Subject: [PATCH 021/332] add acknowledgements with grant agreement numbers --- ACKNOWLEDGEMENTS.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ACKNOWLEDGEMENTS.rst b/ACKNOWLEDGEMENTS.rst index c68078a37..af7ce40b5 100644 --- a/ACKNOWLEDGEMENTS.rst +++ b/ACKNOWLEDGEMENTS.rst @@ -2,7 +2,7 @@ Acknowledgements ================ Destination Earth - This work has been partially funded by the European Union's Destination Earth initiative. + This software is developed with co-funding by the European Union under the Destination Earth initiative. LEXIS - This work has been partially funded by the LEXIS project of the European Union’s Horizon 2020 Research and Innovation programme under Grant Agreement no . \ No newline at end of file + This software is part of a project that has received funding from the European Union’s Horizon 2020 research and innovation programme under grant agreement No 825532. \ No newline at end of file From 5c9d652cd7a95d74bee432551229ffb687942914 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 28 Jun 2023 12:16:29 +0100 Subject: [PATCH 022/332] keep all coords in xarray --- examples/3D_shipping_route.py | 14 ++++++-------- examples/4D_flight_path.py | 4 +++- examples/country_slicing.py | 9 +++++++-- examples/cyclic_route_around_earth.py | 9 +++++++-- examples/read_me_example.py | 4 ++++ examples/slicing_all_ecmwf_countries.py | 7 ++++++- examples/timeseries_example.py | 12 +++++++++--- examples/wind_farms.py | 14 +++++--------- polytope/datacube/xarray.py | 7 ++----- tests/test_slicer_era5.py | 2 ++ 10 files changed, 51 insertions(+), 31 deletions(-) diff --git a/examples/3D_shipping_route.py b/examples/3D_shipping_route.py index 49e0bfac5..bfbad8e82 100644 --- a/examples/3D_shipping_route.py +++ b/examples/3D_shipping_route.py @@ -16,6 +16,10 @@ def setup_method(self): ds = data.from_source("file", "./examples/data/winds.grib") array = ds.to_xarray() array = array.isel(time=0).isel(surface=0).isel(number=0).u10 + array = array.reset_coords(names="time", drop=True) + array = array.reset_coords(names="valid_time", drop=True) + array = array.reset_coords(names="number", drop=True) + array = array.reset_coords(names="surface", drop=True) self.array = array self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) @@ -65,8 +69,6 @@ def test_slice_shipping_route(self): lats = [] longs = [] parameter_values = [] - winds_u = [] - winds_v = [] for i in range(len(result.leaves)): cubepath = result.leaves[i].flatten() lat = cubepath["latitude"] @@ -74,13 +76,9 @@ def test_slice_shipping_route(self): lats.append(lat) longs.append(long) - u10_idx = result.leaves[i].result["u10"] + u10_idx = result.leaves[i].result[1] wind_u = u10_idx - v10_idx = result.leaves[i].result["v10"] - wind_v = v10_idx - winds_u.append(wind_u) - winds_v.append(wind_v) - parameter_values.append(math.sqrt(wind_u**2 + wind_v**2)) + parameter_values.append(wind_u) parameter_values = np.array(parameter_values) # Plot this last array according to different colors for the result on a world map diff --git a/examples/4D_flight_path.py b/examples/4D_flight_path.py index da01995fd..c8f6cc14c 100644 --- a/examples/4D_flight_path.py +++ b/examples/4D_flight_path.py @@ -15,6 +15,8 @@ def setup_method(self): ds = data.from_source("file", "./examples/data/temp_model_levels.grib") array = ds.to_xarray() array = array.isel(time=0).t + array = array.reset_coords(names="time", drop=True) + array = array.reset_coords(names="valid_time", drop=True) options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) for dim in array.dims: @@ -99,7 +101,7 @@ def sphere(size, texture): lats.append(lat) longs.append(long) levels.append(level) - t_idx = result.leaves[i].result["t"] + t_idx = result.leaves[i].result[1] parameter_values.append(t_idx) parameter_values = np.array(parameter_values) diff --git a/examples/country_slicing.py b/examples/country_slicing.py index 4248e27a1..1f50c7f70 100644 --- a/examples/country_slicing.py +++ b/examples/country_slicing.py @@ -12,9 +12,14 @@ class Test: def setup_method(self, method): - ds = data.from_source("file", ".examples/data/output8.grib") + ds = data.from_source("file", "./examples/data/output8.grib") array = ds.to_xarray() array = array.isel(surface=0).isel(step=0).isel(number=0).isel(time=0).t2m + array = array.reset_coords(names="time", drop=True) + array = array.reset_coords(names="valid_time", drop=True) + array = array.reset_coords(names="step", drop=True) + array = array.reset_coords(names="surface", drop=True) + array = array.reset_coords(names="number", drop=True) options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() @@ -64,7 +69,7 @@ def test_slice_country(self): latlong_point = [lat, long] lats.append(lat) longs.append(long) - t_idx = result.leaves[i].result["t2m"] + t_idx = result.leaves[i].result[1] temps.append(t_idx) country_points_plotting.append(latlong_point) temps = np.array(temps) diff --git a/examples/cyclic_route_around_earth.py b/examples/cyclic_route_around_earth.py index a7e03eea4..938f014ee 100644 --- a/examples/cyclic_route_around_earth.py +++ b/examples/cyclic_route_around_earth.py @@ -11,9 +11,14 @@ class Test: def setup_method(self, method): - ds = data.from_source("file", ".examples/data/output8.grib") + ds = data.from_source("file", "./examples/data/output8.grib") array = ds.to_xarray() array = array.isel(surface=0).isel(step=0).isel(number=0).isel(time=0).t2m + array = array.reset_coords(names="time", drop=True) + array = array.reset_coords(names="valid_time", drop=True) + array = array.reset_coords(names="step", drop=True) + array = array.reset_coords(names="surface", drop=True) + array = array.reset_coords(names="number", drop=True) options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() @@ -37,7 +42,7 @@ def test_slice_country(self): latlong_point = [lat, long] lats.append(lat) longs.append(long) - t_idx = result.leaves[i].result["t2m"] + t_idx = result.leaves[i].result[1] temps.append(t_idx) country_points_plotting.append(latlong_point) temps = np.array(temps) diff --git a/examples/read_me_example.py b/examples/read_me_example.py index 2800ef270..245de226d 100644 --- a/examples/read_me_example.py +++ b/examples/read_me_example.py @@ -7,6 +7,10 @@ ds = data.from_source("file", "./examples/data/winds.grib") array = ds.to_xarray() array = array.isel(time=0).isel(surface=0).isel(number=0).u10 +array = array.reset_coords(names="time", drop=True) +array = array.reset_coords(names="valid_time", drop=True) +array = array.reset_coords(names="number", drop=True) +array = array.reset_coords(names="surface", drop=True) options = {"longitude": {"Cyclic": [0, 360.0]}} diff --git a/examples/slicing_all_ecmwf_countries.py b/examples/slicing_all_ecmwf_countries.py index 8b9e0b358..012a7b3bd 100644 --- a/examples/slicing_all_ecmwf_countries.py +++ b/examples/slicing_all_ecmwf_countries.py @@ -15,6 +15,11 @@ def setup_method(self, method): ds = data.from_source("file", "./examples/data/output8.grib") array = ds.to_xarray() array = array.isel(surface=0).isel(step=0).isel(number=0).isel(time=0).t2m + array = array.reset_coords(names="time", drop=True) + array = array.reset_coords(names="valid_time", drop=True) + array = array.reset_coords(names="step", drop=True) + array = array.reset_coords(names="surface", drop=True) + array = array.reset_coords(names="number", drop=True) options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() @@ -127,7 +132,7 @@ def test_slice_country(self): latlong_point = [lat, long] countries_lats.append(lat) countries_longs.append(long) - t_idx = result.leaves[i].result["t2m"] + t_idx = result.leaves[i].result[1] t = t_idx countries_temps.append(t) country_points_plotting.append(latlong_point) diff --git a/examples/timeseries_example.py b/examples/timeseries_example.py index 30e26a9d3..f73f2a06a 100644 --- a/examples/timeseries_example.py +++ b/examples/timeseries_example.py @@ -1,7 +1,5 @@ import geopandas as gpd -import matplotlib.pyplot as plt import numpy as np -import pandas as pd from earthkit import data from shapely.geometry import shape @@ -16,6 +14,10 @@ def setup_method(self): ds = data.from_source("file", "./examples/data/timeseries_t2m.grib") array = ds.to_xarray() array = array.isel(step=0).isel(surface=0).isel(number=0).t2m + array = array.reset_coords(names="valid_time", drop=True) + array = array.reset_coords(names="step", drop=True) + array = array.reset_coords(names="surface", drop=True) + array = array.reset_coords(names="number", drop=True) self.xarraydatacube = XArrayDatacube(array) for dim in array.dims: array = array.sortby(dim) @@ -56,9 +58,12 @@ def test_slice_shipping_route(self): result = self.API.retrieve(request) + result.pprint() + # For each date/time, we plot an image # Note that only the temperatures should change so we can store them in different arrays + """ country_points_plotting = [] lats1 = [] lats2 = [] @@ -89,7 +94,7 @@ def test_slice_shipping_route(self): lat = cubepath["latitude"] long = cubepath["longitude"] latlong_point = [lat, long] - t_idx = result.leaves[i].result["t2m"] + t_idx = result.leaves[i].result[1] if cubepath["time"] == pd.Timestamp("2022-05-14T12:00:00"): temps1.append(t_idx) lats1.append(lat) @@ -216,3 +221,4 @@ def test_slice_shipping_route(self): ax[3, 1].set_xticks([]) plt.gca().axes.get_yaxis().set_visible(False) plt.show() + """ diff --git a/examples/wind_farms.py b/examples/wind_farms.py index 596e28bd0..cd9b36899 100644 --- a/examples/wind_farms.py +++ b/examples/wind_farms.py @@ -1,5 +1,3 @@ -import math - import geopandas as gpd import matplotlib as mpl import matplotlib.pyplot as plt @@ -19,6 +17,10 @@ def setup_method(self): ds = data.from_source("file", "./examples/data/winds.grib") array = ds.to_xarray() array = array.isel(time=0).isel(surface=0).isel(number=0).u10 + array = array.reset_coords(names="time", drop=True) + array = array.reset_coords(names="valid_time", drop=True) + array = array.reset_coords(names="number", drop=True) + array = array.reset_coords(names="surface", drop=True) self.array = array options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) @@ -64,22 +66,16 @@ def test_slice_wind_farms(self): longs = [] parameter_values = [] winds_u = [] - winds_v = [] for i in range(len(result.leaves)): cubepath = result.leaves[i].flatten() lat = cubepath["latitude"] long = cubepath["longitude"] lats.append(lat) longs.append(long) - # u10_idx = result.leaves[i].result["u10"] u10_idx = result.leaves[i].result[1] wind_u = u10_idx - # v10_idx = result.leaves[i].result["v10"] - # wind_v = v10_idx - wind_v = 0 winds_u.append(wind_u) - winds_v.append(wind_v) - parameter_values.append(math.sqrt(wind_u**2 + wind_v**2)) + parameter_values.append(wind_u) parameter_values = np.array(parameter_values) # Plot this last array according to different colors for the result on a world map diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 3e079ba62..fae20b02f 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -54,11 +54,8 @@ def __init__(self, dataarray: xr.DataArray, options={}): self.options = options self.mappers = {} for name, values in dataarray.coords.variables.items(): - if name in dataarray.dims: - dataarray = dataarray.sortby(name) - self._set_mapper(values, name) - else: # drop non-necessary coordinates which we don't slice on - dataarray = dataarray.reset_coords(names=name, drop=True) + dataarray = dataarray.sortby(name) + self._set_mapper(values, name) self.dataarray = dataarray def get(self, requests: IndexTree): diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 09e30b1a2..39ad5a94f 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -10,6 +10,8 @@ class TestSlicingEra5Data: def setup_method(self, method): ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t + array = array.reset_coords(names="step", drop=True) + array = array.reset_coords(names="valid_time", drop=True) self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) From 74d6b6830efe329b7c075c1120fa8ce868bd18f1 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 28 Jun 2023 13:08:42 +0100 Subject: [PATCH 023/332] fix lat lon to octahedral index conversion --- polytope/datacube/datacube.py | 5 +- polytope/datacube/octahedral_xarray.py | 46 +++++--------- tests/test_octahedral_grid.py | 84 +++++++++++++++----------- 3 files changed, 66 insertions(+), 69 deletions(-) diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index 2b14c0dc0..4195e40ef 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -43,7 +43,8 @@ def validate(self, axes) -> bool: @staticmethod def create(datacube, options): if isinstance(datacube, (xr.core.dataarray.DataArray, xr.core.dataset.Dataset)): - from .octahedral_xarray import XArrayDatacube + # from .xarray import XArrayDatacube + from .octahedral_xarray import OctahedralXArrayDatacube - xadatacube = XArrayDatacube(datacube, options=options) + xadatacube = OctahedralXArrayDatacube(datacube, options=options) return xadatacube diff --git a/polytope/datacube/octahedral_xarray.py b/polytope/datacube/octahedral_xarray.py index f14b5fa2f..9e76b3e70 100644 --- a/polytope/datacube/octahedral_xarray.py +++ b/polytope/datacube/octahedral_xarray.py @@ -28,7 +28,7 @@ } -class XArrayDatacube(Datacube): +class OctahedralXArrayDatacube(Datacube): """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0).""" def _set_mapper(self, values, name): @@ -52,7 +52,6 @@ def _set_mapper(self, values, name): def __init__(self, dataarray: xr.DataArray, options={}): self.options = options self.mappers = {} - # print(dataarray.coords) for name, values in dataarray.coords.variables.items(): if values.data.size != 1: dataarray = dataarray.sortby(name) @@ -127,35 +126,18 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): def get_indices(self, path: DatacubePath, axis, lower, upper): path = self.remap_path(path) - print(axis.name) - print(path) # Open a view on the subset identified by the path lat_val = path.get("latitude", None) path.pop("longitude", None) path.pop("latitude", None) - print(path) subarray = self.dataarray.sel(path, method="nearest") # Get the indexes of the axis we want to query # XArray does not support branching, so no need to use label, we just take the next axis - # indexes = next(iter(subarray.xindexes.values())).to_pandas_index() if axis.name == "latitude": - # subxarray_lat = xr.DataArray(self.lat_val_available(), dims=("latitude"), - # coords={ - # "latitude": self.lat_val_available(), - # },) - # indexes = next(iter(subxarray_lat.xindexes.values())).to_pandas_index() indexes = self.lat_val_available() - # indexes = pd.Index(tuple(indexes)) if indexes else None elif axis.name == "longitude": - # lat_val = path["latitude"] - # subxarray_lon = xr.DataArray(self.lon_val_available(lat_val), dims=("longitude"), - # coords={ - # "longitude": self.lon_val_available(lat_val), - # },) - # indexes = next(iter(subxarray_lon.xindexes.values())).to_pandas_index() indexes = self.lon_val_available(lat_val) - # indexes = indexes[0].to_pandas_index() if indexes else None else: indexes = next(iter(subarray.xindexes.values())).to_pandas_index() @@ -202,23 +184,23 @@ def lon_val_available(self, lat): lat_spacing = 90/1280 lat_start = 0 lat_idx = (lat-lat_start)/lat_spacing - num_points_on_lon = 4 * lat_idx + 16 - lon_spacing = 180/num_points_on_lon + num_points_on_lon = 4 * (1280-lat_idx) + 16 + lon_spacing = 360/num_points_on_lon lon_start = 0 return_lon = [lon_start + i * lon_spacing for i in range(int(num_points_on_lon))] return return_lon -def octa_idx_to_latlon_idx(idx): - lat_j = math.floor(-3.5 + (math.sqrt(81+2*idx)/2)) - lon_j = idx - 2 * lat_j * lat_j - 14 * lat_j + 16 +# def octa_idx_to_latlon_idx(idx): +# lat_j = math.floor(-3.5 + (math.sqrt(81+2*idx)/2)) +# lon_j = idx - 2 * lat_j * lat_j - 14 * lat_j + 16 - # NOTE to get idx of lat and lon, need to substract 1 to start from 0 for lat +# # NOTE to get idx of lat and lon, need to substract 1 to start from 0 for lat - lat_idx = lat_j - 1 - lon_idx = lon_j +# lat_idx = lat_j - 1 +# lon_idx = lon_j - return (lat_idx, lon_idx) +# return (lat_idx, lon_idx) def latlon_idx_to_octa_idx(lat_idx, lon_idx): @@ -228,8 +210,8 @@ def latlon_idx_to_octa_idx(lat_idx, lon_idx): def latlon_idx_to_val(lat_idx, lon_idx): # spacing between different lat levels lat_spacing = 90/1280 - num_points_on_lon = 4 * lat_idx + 16 - lon_spacing = 180/num_points_on_lon + num_points_on_lon = 4 * (1280-lat_idx) + 16 + lon_spacing = 360/num_points_on_lon # TODO: this might be wrong, and the spacing above too for the lat, # depends on how the grid is laid out onto the sphere @@ -243,8 +225,8 @@ def latlon_val_to_idx(lat, lon): lat_start = 0 lat_idx = (lat-lat_start)/lat_spacing - num_points_on_lon = 4 * lat_idx + 16 - lon_spacing = 180/num_points_on_lon + num_points_on_lon = 4 * (1280-lat_idx) + 16 + lon_spacing = 360/num_points_on_lon lon_start = 0 lon_idx = (lon-lon_start)/lon_spacing diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 6a21c7e03..af0d76a44 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -1,50 +1,64 @@ -import math - +import geopandas as gpd +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np from earthkit import data -from polytope.datacube.octahedral_xarray import XArrayDatacube -from polytope.datacube.datacube_axis import FloatAxis +from polytope.datacube.octahedral_xarray import OctahedralXArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Box, Select +from polytope.shapes import Box ds = data.from_source("file", "./foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m -# print(latlon_array) - - -# # Going from an index in the latlon array to individual lat and lon numbers -# index = 44+28 - -# lat_j = math.floor(-3.5 + (math.sqrt(81+2*index)/2)) -# lon_j = index - 2 * lat_j * lat_j - 14 * lat_j + 16 - -# # NOTE to get idx of lat and lon, need to substract 1 to start from 0 for lat - -# lat_idx = lat_j - 1 -# lon_idx = lon_j - -# print(lat_idx) -# print(lon_idx) - -latlon_xarray_datacube = XArrayDatacube(latlon_array) - -# print(xarraydatacube.mappers) - -# lat_axis = FloatAxis() -# lat_axis.name = "latitude" -# xarraydatacube.get_indices({}, lat_axis, 5, 10) - -# subxarray = latlon_array.isel(values=20) -# value = subxarray.item() -# key = subxarray.name -# print((value, key)) +latlon_xarray_datacube = OctahedralXArrayDatacube(latlon_array) slicer = HullSlicer() API = Polytope(datacube=latlon_array, engine=slicer) -request = Request(Box(["latitude", "longitude"], [5, 5], [10, 10])) +request = Request(Box(["latitude", "longitude"], [3, 3], [5, 5])) result = API.retrieve(request) result.pprint() + +request2 = Request(Box(["latitude", "longitude"], [70, 3], [72, 5])) +result2 = API.retrieve(request2) +result2.pprint() + +lats = [] +longs = [] +parameter_values = [] +for i in range(len(result.leaves)): + cubepath = result.leaves[i].flatten() + lat = cubepath["latitude"] + long = cubepath["longitude"] + lats.append(lat) + longs.append(long) + + t = result.leaves[i].result[1] + parameter_values.append(t) + +for i in range(len(result2.leaves)): + cubepath = result2.leaves[i].flatten() + lat = cubepath["latitude"] + long = cubepath["longitude"] + lats.append(lat) + longs.append(long) + + t = result2.leaves[i].result[1] + parameter_values.append(t) + +parameter_values = np.array(parameter_values) +# Plot this last array according to different colors for the result on a world map +worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) +fig, ax = plt.subplots(figsize=(12, 6)) +worldmap.plot(color="darkgrey", ax=ax) +ax.scatter(longs, lats, s=8, c=parameter_values, cmap="viridis") +norm = mpl.colors.Normalize(vmin=min(parameter_values), vmax=max(parameter_values)) + +sm = plt.cm.ScalarMappable(cmap="viridis", norm=norm) +sm.set_array([]) +plt.colorbar(sm, label="Wind Speed") + +plt.show() From 796e28f619929e15e24557820f7a79d38d8db03f Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 28 Jun 2023 13:10:09 +0100 Subject: [PATCH 024/332] small readme fix --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 5be58455b..48dba68f6 100644 --- a/readme.md +++ b/readme.md @@ -196,7 +196,7 @@ does it submit to any jurisdiction. ## Citing -In this software is useful in your work, please consider citing our [paper](https://arxiv.org/abs/2306.11553) as +If this software is useful in your work, please consider citing our [paper](https://arxiv.org/abs/2306.11553) as > Leuridan, M., Hawkes, J., Smart, S., Danovaro, E., and Quintino, T., “Polytope: An Algorithm for Efficient Feature Extraction on Hypercubes”, arXiv e-prints, 2023. doi:10.48550/arXiv.2306.11553. From d5ae692bb1e7b09036abcfd32ca8c15539e86101 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 29 Jun 2023 13:35:59 +0100 Subject: [PATCH 025/332] add lat_start and compare with eccodes --- polytope/datacube/octahedral_xarray.py | 12 +++-- tests/test_octahedral_grid.py | 69 ++++++++++++++++++++++---- 2 files changed, 67 insertions(+), 14 deletions(-) diff --git a/polytope/datacube/octahedral_xarray.py b/polytope/datacube/octahedral_xarray.py index 9e76b3e70..2ee1b0ae5 100644 --- a/polytope/datacube/octahedral_xarray.py +++ b/polytope/datacube/octahedral_xarray.py @@ -176,13 +176,15 @@ def validate(self, axes): def lat_val_available(self): lat_spacing = 90/1280 - lat_start = 0 + # lat_start = 0.035149384215604956 + lat_start = 0.03515625 return_lat = [lat_start + i * lat_spacing for i in range(1280)] return return_lat def lon_val_available(self, lat): lat_spacing = 90/1280 - lat_start = 0 + # lat_start = 0.035149384215604956 + lat_start = 0.03515625 lat_idx = (lat-lat_start)/lat_spacing num_points_on_lon = 4 * (1280-lat_idx) + 16 lon_spacing = 360/num_points_on_lon @@ -215,14 +217,16 @@ def latlon_idx_to_val(lat_idx, lon_idx): # TODO: this might be wrong, and the spacing above too for the lat, # depends on how the grid is laid out onto the sphere - lat_start = 0 + # lat_start = 0.035149384215604956 + lat_start = 0.03515625 lon_start = 0 return (lat_idx*lat_spacing + lat_start, lon_idx*lon_spacing + lon_start) def latlon_val_to_idx(lat, lon): lat_spacing = 90/1280 - lat_start = 0 + # lat_start = 0.035149384215604956 + lat_start = 0.03515625 lat_idx = (lat-lat_start)/lat_spacing num_points_on_lon = 4 * (1280-lat_idx) + 16 diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index af0d76a44..6a0b35b5c 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -9,16 +9,48 @@ from polytope.polytope import Polytope, Request from polytope.shapes import Box +from eccodes import * + + +def find_nearest_latlon(grib_file, target_lat, target_lon): + # Open the GRIB file + f = open(grib_file) + + # Load the GRIB messages from the file + messages = [] + while True: + message = codes_grib_new_from_file(f) + if message is None: + break + messages.append(message) + + # Find the nearest grid points + nearest_points = [] + for message in messages: + # latitudes = codes_get_array(message, "latitudes") + # longitudes = codes_get_array(message, "longitudes") + # nearest_index = codes_grib_find_nearest(message, target_lat, target_lon, latitudes, longitudes) + nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) + print(nearest_index) + # nearest_points.append((latitudes[nearest_index], longitudes[nearest_index])) + nearest_points.append(nearest_index) + + # Close the GRIB file + f.close() + + return nearest_points + ds = data.from_source("file", "./foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m +# print([val for val in latlon_array["latitude"].values[:] if val>0][-1]) latlon_xarray_datacube = OctahedralXArrayDatacube(latlon_array) slicer = HullSlicer() API = Polytope(datacube=latlon_array, engine=slicer) -request = Request(Box(["latitude", "longitude"], [3, 3], [5, 5])) +request = Request(Box(["latitude", "longitude"], [89, -1], [89.5, 181])) result = API.retrieve(request) result.pprint() @@ -28,6 +60,8 @@ lats = [] longs = [] +eccodes_lats = [] +eccodes_longs = [] parameter_values = [] for i in range(len(result.leaves)): cubepath = result.leaves[i].flatten() @@ -35,26 +69,33 @@ long = cubepath["longitude"] lats.append(lat) longs.append(long) + nearest_points = find_nearest_latlon("./foo.grib", lat, long) + print(nearest_points) + eccodes_lats.append(nearest_points[0][0]["lat"]) + eccodes_longs.append(nearest_points[0][0]["lon"]) t = result.leaves[i].result[1] parameter_values.append(t) -for i in range(len(result2.leaves)): - cubepath = result2.leaves[i].flatten() - lat = cubepath["latitude"] - long = cubepath["longitude"] - lats.append(lat) - longs.append(long) +# for i in range(len(result2.leaves)): +# cubepath = result2.leaves[i].flatten() +# lat = cubepath["latitude"] +# long = cubepath["longitude"] +# lats.append(lat) +# longs.append(long) - t = result2.leaves[i].result[1] - parameter_values.append(t) +# t = result2.leaves[i].result[1] +# parameter_values.append(t) parameter_values = np.array(parameter_values) # Plot this last array according to different colors for the result on a world map worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) fig, ax = plt.subplots(figsize=(12, 6)) worldmap.plot(color="darkgrey", ax=ax) -ax.scatter(longs, lats, s=8, c=parameter_values, cmap="viridis") +from matplotlib import markers +marker = markers.MarkerStyle(marker='s') +ax.scatter(eccodes_longs, eccodes_lats, s=12, c="red", marker=marker, facecolors='none') +ax.scatter(longs, lats, s=4, c=parameter_values, cmap="viridis") norm = mpl.colors.Normalize(vmin=min(parameter_values), vmax=max(parameter_values)) sm = plt.cm.ScalarMappable(cmap="viridis", norm=norm) @@ -62,3 +103,11 @@ plt.colorbar(sm, label="Wind Speed") plt.show() + + +# grib_file = "./foo.grib" +# target_lat = 40.0 +# target_lon = -100.0 + +# nearest_points = find_nearest_latlon(grib_file, target_lat, target_lon) +# print(nearest_points) \ No newline at end of file From 6911f853949412e21ab77a96e881f2e36dbb51dc Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 29 Jun 2023 14:23:20 +0100 Subject: [PATCH 026/332] make octahedral grid work for negative lat --- polytope/datacube/octahedral_xarray.py | 38 +++++++++++++++++--------- tests/test_octahedral_grid.py | 4 +-- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/polytope/datacube/octahedral_xarray.py b/polytope/datacube/octahedral_xarray.py index 2ee1b0ae5..44cf74e62 100644 --- a/polytope/datacube/octahedral_xarray.py +++ b/polytope/datacube/octahedral_xarray.py @@ -176,17 +176,23 @@ def validate(self, axes): def lat_val_available(self): lat_spacing = 90/1280 - # lat_start = 0.035149384215604956 - lat_start = 0.03515625 - return_lat = [lat_start + i * lat_spacing for i in range(1280)] + lat_start = 0.035149384215604956 + # lat_start = 0.026906142167192115 + # lat_start = -(90-89.94618771566562) + return_lat = [lat_start + i * lat_spacing - 90 for i in range(1280*2)] return return_lat def lon_val_available(self, lat): lat_spacing = 90/1280 - # lat_start = 0.035149384215604956 - lat_start = 0.03515625 - lat_idx = (lat-lat_start)/lat_spacing - num_points_on_lon = 4 * (1280-lat_idx) + 16 + lat_start = 0.035149384215604956 + # lat_start = 0.026906142167192115 + # lat_start = -(90-89.94618771566562) + if lat_start <= lat < 90: + lat_idx = 1280 - ((lat-lat_start)/lat_spacing) + else: + lat_idx = (lat+90-lat_start)/lat_spacing + # lat_idx = 1280 - ((lat-lat_start)/lat_spacing) + num_points_on_lon = 4 * lat_idx + 16 lon_spacing = 360/num_points_on_lon lon_start = 0 return_lon = [lon_start + i * lon_spacing for i in range(int(num_points_on_lon))] @@ -217,19 +223,25 @@ def latlon_idx_to_val(lat_idx, lon_idx): # TODO: this might be wrong, and the spacing above too for the lat, # depends on how the grid is laid out onto the sphere - # lat_start = 0.035149384215604956 - lat_start = 0.03515625 + lat_start = 0.035149384215604956 + # lat_start = 0.026906142167192115 + # lat_start = -(90-89.94618771566562) lon_start = 0 return (lat_idx*lat_spacing + lat_start, lon_idx*lon_spacing + lon_start) def latlon_val_to_idx(lat, lon): lat_spacing = 90/1280 - # lat_start = 0.035149384215604956 - lat_start = 0.03515625 + lat_start = 0.035149384215604956 + # lat_start = 0.026906142167192115 + # lat_start = -(90-89.94618771566562) lat_idx = (lat-lat_start)/lat_spacing - - num_points_on_lon = 4 * (1280-lat_idx) + 16 + if lat_start <= lat < 90: + lat_idx = 1280 - ((lat-lat_start)/lat_spacing) + else: + lat_idx = (lat+90-lat_start)/lat_spacing + # lat_idx = 1280 - ((lat-lat_start)/lat_spacing) + num_points_on_lon = 4 * lat_idx + 16 lon_spacing = 360/num_points_on_lon lon_start = 0 lon_idx = (lon-lon_start)/lon_spacing diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 6a0b35b5c..ad4f4a63a 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -43,14 +43,14 @@ def find_nearest_latlon(grib_file, target_lat, target_lon): ds = data.from_source("file", "./foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m -# print([val for val in latlon_array["latitude"].values[:] if val>0][-1]) +print([val for val in latlon_array["latitude"].values[:] if val<0][-1]) latlon_xarray_datacube = OctahedralXArrayDatacube(latlon_array) slicer = HullSlicer() API = Polytope(datacube=latlon_array, engine=slicer) -request = Request(Box(["latitude", "longitude"], [89, -1], [89.5, 181])) +request = Request(Box(["latitude", "longitude"], [-1, 1], [1, 2])) result = API.retrieve(request) result.pprint() From 2fc1a5cc6f266f9fcf778a9f11273f531f2e00cb Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 30 Jun 2023 15:49:00 +0100 Subject: [PATCH 027/332] make it work for negative lats and verify indexes --- examples/octahedral_grid_example.py | 106 +++++++++++++++++++++++++ polytope/datacube/octahedral_xarray.py | 92 +++++++++------------ tests/test_octahedral_grid.py | 44 ++-------- 3 files changed, 152 insertions(+), 90 deletions(-) create mode 100644 examples/octahedral_grid_example.py diff --git a/examples/octahedral_grid_example.py b/examples/octahedral_grid_example.py new file mode 100644 index 000000000..da0fd9a64 --- /dev/null +++ b/examples/octahedral_grid_example.py @@ -0,0 +1,106 @@ +import geopandas as gpd +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +from earthkit import data +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file +from matplotlib import markers +from shapely.geometry import shape + +from polytope.datacube.octahedral_xarray import OctahedralXArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Polygon, Union + + +def find_nearest_latlon(grib_file, target_lat, target_lon): + # Open the GRIB file + f = open(grib_file) + + # Load the GRIB messages from the file + messages = [] + while True: + message = codes_grib_new_from_file(f) + if message is None: + break + messages.append(message) + + # Find the nearest grid points + nearest_points = [] + for message in messages: + nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) + nearest_points.append(nearest_index) + + # Close the GRIB file + f.close() + + return nearest_points + + +ds = data.from_source("file", "./foo.grib") +latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) +latlon_array = latlon_array.t2m + +latlon_xarray_datacube = OctahedralXArrayDatacube(latlon_array) + +slicer = HullSlicer() +API = Polytope(datacube=latlon_array, engine=slicer) + +shapefile = gpd.read_file("./examples/data/World_Countries__Generalized_.shp") +country = shapefile.iloc[13] +multi_polygon = shape(country["geometry"]) +# If country is just a polygon +polygons = [multi_polygon] +polygons_list = [] + +# Now create a list of x,y points for each polygon + +for polygon in polygons: + xx, yy = polygon.exterior.coords.xy + polygon_points = [list(a) for a in zip(xx, yy)] + polygons_list.append(polygon_points) + +# Then do union of the polygon objects and cut using the slicer +poly = [] +for points in polygons_list: + polygon = Polygon(["longitude", "latitude"], points) + poly.append(polygon) +request_obj = poly[0] +for obj in poly: + request_obj = Union(["longitude", "latitude"], request_obj, obj) +request = Request(request_obj) +result = API.retrieve(request) + +lats = [] +longs = [] +eccodes_lats = [] +eccodes_longs = [] +parameter_values = [] +for i in range(len(result.leaves)): + cubepath = result.leaves[i].flatten() + lat = cubepath["latitude"] + long = cubepath["longitude"] + lats.append(lat) + longs.append(long) + nearest_points = find_nearest_latlon("./foo.grib", lat, long) + eccodes_lats.append(nearest_points[0][0]["lat"]) + eccodes_longs.append(nearest_points[0][0]["lon"]) + t = result.leaves[i].result[1] + parameter_values.append(t) + +parameter_values = np.array(parameter_values) +# Plot this last array according to different colors for the result on a world map +worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) +fig, ax = plt.subplots(figsize=(12, 6)) +worldmap.plot(color="darkgrey", ax=ax) + +marker = markers.MarkerStyle(marker="s") +ax.scatter(eccodes_longs, eccodes_lats, s=12, c="red", marker=marker, facecolors="none") +ax.scatter(longs, lats, s=4, c=parameter_values, cmap="viridis") +norm = mpl.colors.Normalize(vmin=min(parameter_values), vmax=max(parameter_values)) + +sm = plt.cm.ScalarMappable(cmap="viridis", norm=norm) +sm.set_array([]) +plt.colorbar(sm, label="Wind Speed") + +plt.show() diff --git a/polytope/datacube/octahedral_xarray.py b/polytope/datacube/octahedral_xarray.py index 44cf74e62..8a98d2b44 100644 --- a/polytope/datacube/octahedral_xarray.py +++ b/polytope/datacube/octahedral_xarray.py @@ -54,7 +54,6 @@ def __init__(self, dataarray: xr.DataArray, options={}): self.mappers = {} for name, values in dataarray.coords.variables.items(): if values.data.size != 1: - dataarray = dataarray.sortby(name) self._set_mapper(values, name) else: # drop non-necessary coordinates which we don't slice on dataarray = dataarray.reset_coords(names=name, drop=True) @@ -103,7 +102,6 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): up = r[1] if axis.name == "latitude" or axis.name == "longitude": - indexes.sort() indexes_between = [i for i in indexes if low <= i <= up] else: # Find the range of indexes between lower and upper @@ -175,75 +173,63 @@ def validate(self, axes): return validate_axes(self.axes, axes) def lat_val_available(self): - lat_spacing = 90/1280 + lat_spacing = 90 / 1280 lat_start = 0.035149384215604956 - # lat_start = 0.026906142167192115 - # lat_start = -(90-89.94618771566562) - return_lat = [lat_start + i * lat_spacing - 90 for i in range(1280*2)] + return_lat = [lat_start + i * lat_spacing - 90 for i in range(1280 * 2)] return return_lat def lon_val_available(self, lat): - lat_spacing = 90/1280 + lat_spacing = 90 / 1280 lat_start = 0.035149384215604956 - # lat_start = 0.026906142167192115 - # lat_start = -(90-89.94618771566562) if lat_start <= lat < 90: - lat_idx = 1280 - ((lat-lat_start)/lat_spacing) + lat_idx = 1280 - ((lat - lat_start) / lat_spacing) else: - lat_idx = (lat+90-lat_start)/lat_spacing - # lat_idx = 1280 - ((lat-lat_start)/lat_spacing) + lat_idx = (lat + 90 - lat_start) / lat_spacing num_points_on_lon = 4 * lat_idx + 16 - lon_spacing = 360/num_points_on_lon + lon_spacing = 360 / num_points_on_lon lon_start = 0 return_lon = [lon_start + i * lon_spacing for i in range(int(num_points_on_lon))] return return_lon -# def octa_idx_to_latlon_idx(idx): -# lat_j = math.floor(-3.5 + (math.sqrt(81+2*idx)/2)) -# lon_j = idx - 2 * lat_j * lat_j - 14 * lat_j + 16 - -# # NOTE to get idx of lat and lon, need to substract 1 to start from 0 for lat - -# lat_idx = lat_j - 1 -# lon_idx = lon_j - -# return (lat_idx, lon_idx) - - def latlon_idx_to_octa_idx(lat_idx, lon_idx): - return int(2*lat_idx*lat_idx + 15*lat_idx - 16 + lon_idx) - - -def latlon_idx_to_val(lat_idx, lon_idx): - # spacing between different lat levels - lat_spacing = 90/1280 - num_points_on_lon = 4 * (1280-lat_idx) + 16 - lon_spacing = 360/num_points_on_lon - - # TODO: this might be wrong, and the spacing above too for the lat, - # depends on how the grid is laid out onto the sphere - lat_start = 0.035149384215604956 - # lat_start = 0.026906142167192115 - # lat_start = -(90-89.94618771566562) - lon_start = 0 - return (lat_idx*lat_spacing + lat_start, lon_idx*lon_spacing + lon_start) + if lat_idx <= 1280: + # we are at lat >0 + # lat_idx 0 is lat=90 + octa_idx = 0 + if lat_idx == 1: + octa_idx = lon_idx + else: + for i in range(lat_idx - 1): + octa_idx += 16 + 4 * (i) + octa_idx = int(octa_idx + lon_idx) + else: + # we are at lat <0 + octa_idx = int(6599680 / 2) + lat_idx = lat_idx - 1280 + if lat_idx == 1: + octa_idx += lon_idx + else: + for i in range(lat_idx - 1): + octa_idx += 16 + 4 * (1280 - i - 1) + octa_idx = octa_idx + lon_idx + if lat_idx == 1280: + lat_idx -= 16 + return octa_idx def latlon_val_to_idx(lat, lon): - lat_spacing = 90/1280 - lat_start = 0.035149384215604956 - # lat_start = 0.026906142167192115 - # lat_start = -(90-89.94618771566562) - lat_idx = (lat-lat_start)/lat_spacing - if lat_start <= lat < 90: - lat_idx = 1280 - ((lat-lat_start)/lat_spacing) + lat_spacing = 90 / 1280 + lat_start = 0.03515625 + if lat_start <= lat <= 90: + # if we are at lat=90, the latitude line index is 0 + lat_idx = 1280 - ((lat - lat_start) / lat_spacing) else: - lat_idx = (lat+90-lat_start)/lat_spacing - # lat_idx = 1280 - ((lat-lat_start)/lat_spacing) + # if we are at lat=-90, the latitude line index is 2560 + lat_idx = 1280 - ((lat - lat_start) / lat_spacing) num_points_on_lon = 4 * lat_idx + 16 - lon_spacing = 360/num_points_on_lon + lon_spacing = 360 / num_points_on_lon lon_start = 0 - lon_idx = (lon-lon_start)/lon_spacing + lon_idx = (lon - lon_start) / lon_spacing - return (lat_idx, lon_idx) + return (int(lat_idx), lon_idx) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index ad4f4a63a..865b2c9fd 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -3,14 +3,14 @@ import matplotlib.pyplot as plt import numpy as np from earthkit import data +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file +from matplotlib import markers from polytope.datacube.octahedral_xarray import OctahedralXArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box -from eccodes import * - def find_nearest_latlon(grib_file, target_lat, target_lon): # Open the GRIB file @@ -27,12 +27,7 @@ def find_nearest_latlon(grib_file, target_lat, target_lon): # Find the nearest grid points nearest_points = [] for message in messages: - # latitudes = codes_get_array(message, "latitudes") - # longitudes = codes_get_array(message, "longitudes") - # nearest_index = codes_grib_find_nearest(message, target_lat, target_lon, latitudes, longitudes) nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) - print(nearest_index) - # nearest_points.append((latitudes[nearest_index], longitudes[nearest_index])) nearest_points.append(nearest_index) # Close the GRIB file @@ -40,24 +35,21 @@ def find_nearest_latlon(grib_file, target_lat, target_lon): return nearest_points + ds = data.from_source("file", "./foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m -print([val for val in latlon_array["latitude"].values[:] if val<0][-1]) +nearest_points = find_nearest_latlon("./foo.grib", 0, 0) latlon_xarray_datacube = OctahedralXArrayDatacube(latlon_array) slicer = HullSlicer() API = Polytope(datacube=latlon_array, engine=slicer) -request = Request(Box(["latitude", "longitude"], [-1, 1], [1, 2])) +request = Request(Box(["latitude", "longitude"], [0, 0], [0.5, 0.5])) result = API.retrieve(request) result.pprint() -request2 = Request(Box(["latitude", "longitude"], [70, 3], [72, 5])) -result2 = API.retrieve(request2) -result2.pprint() - lats = [] longs = [] eccodes_lats = [] @@ -70,44 +62,22 @@ def find_nearest_latlon(grib_file, target_lat, target_lon): lats.append(lat) longs.append(long) nearest_points = find_nearest_latlon("./foo.grib", lat, long) - print(nearest_points) eccodes_lats.append(nearest_points[0][0]["lat"]) eccodes_longs.append(nearest_points[0][0]["lon"]) - t = result.leaves[i].result[1] parameter_values.append(t) -# for i in range(len(result2.leaves)): -# cubepath = result2.leaves[i].flatten() -# lat = cubepath["latitude"] -# long = cubepath["longitude"] -# lats.append(lat) -# longs.append(long) - -# t = result2.leaves[i].result[1] -# parameter_values.append(t) parameter_values = np.array(parameter_values) # Plot this last array according to different colors for the result on a world map worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) fig, ax = plt.subplots(figsize=(12, 6)) worldmap.plot(color="darkgrey", ax=ax) -from matplotlib import markers -marker = markers.MarkerStyle(marker='s') -ax.scatter(eccodes_longs, eccodes_lats, s=12, c="red", marker=marker, facecolors='none') +marker = markers.MarkerStyle(marker="s") +ax.scatter(eccodes_longs, eccodes_lats, s=12, c="red", marker=marker, facecolors="none") ax.scatter(longs, lats, s=4, c=parameter_values, cmap="viridis") norm = mpl.colors.Normalize(vmin=min(parameter_values), vmax=max(parameter_values)) - sm = plt.cm.ScalarMappable(cmap="viridis", norm=norm) sm.set_array([]) plt.colorbar(sm, label="Wind Speed") - plt.show() - - -# grib_file = "./foo.grib" -# target_lat = 40.0 -# target_lon = -100.0 - -# nearest_points = find_nearest_latlon(grib_file, target_lat, target_lon) -# print(nearest_points) \ No newline at end of file From 42bbd6ce7c5d2c9d4f1cede1faacf4d5cc50bb63 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 3 Jul 2023 10:09:21 +0100 Subject: [PATCH 028/332] make better tests --- examples/octahedral_grid_box_example.py | 83 ++++++++++++ ....py => octahedral_grid_country_example.py} | 0 tests/test_octahedral_grid.py | 118 ++++++++---------- 3 files changed, 134 insertions(+), 67 deletions(-) create mode 100644 examples/octahedral_grid_box_example.py rename examples/{octahedral_grid_example.py => octahedral_grid_country_example.py} (100%) diff --git a/examples/octahedral_grid_box_example.py b/examples/octahedral_grid_box_example.py new file mode 100644 index 000000000..865b2c9fd --- /dev/null +++ b/examples/octahedral_grid_box_example.py @@ -0,0 +1,83 @@ +import geopandas as gpd +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +from earthkit import data +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file +from matplotlib import markers + +from polytope.datacube.octahedral_xarray import OctahedralXArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box + + +def find_nearest_latlon(grib_file, target_lat, target_lon): + # Open the GRIB file + f = open(grib_file) + + # Load the GRIB messages from the file + messages = [] + while True: + message = codes_grib_new_from_file(f) + if message is None: + break + messages.append(message) + + # Find the nearest grid points + nearest_points = [] + for message in messages: + nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) + nearest_points.append(nearest_index) + + # Close the GRIB file + f.close() + + return nearest_points + + +ds = data.from_source("file", "./foo.grib") +latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) +latlon_array = latlon_array.t2m +nearest_points = find_nearest_latlon("./foo.grib", 0, 0) + +latlon_xarray_datacube = OctahedralXArrayDatacube(latlon_array) + +slicer = HullSlicer() +API = Polytope(datacube=latlon_array, engine=slicer) + +request = Request(Box(["latitude", "longitude"], [0, 0], [0.5, 0.5])) +result = API.retrieve(request) +result.pprint() + +lats = [] +longs = [] +eccodes_lats = [] +eccodes_longs = [] +parameter_values = [] +for i in range(len(result.leaves)): + cubepath = result.leaves[i].flatten() + lat = cubepath["latitude"] + long = cubepath["longitude"] + lats.append(lat) + longs.append(long) + nearest_points = find_nearest_latlon("./foo.grib", lat, long) + eccodes_lats.append(nearest_points[0][0]["lat"]) + eccodes_longs.append(nearest_points[0][0]["lon"]) + t = result.leaves[i].result[1] + parameter_values.append(t) + + +parameter_values = np.array(parameter_values) +# Plot this last array according to different colors for the result on a world map +worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) +fig, ax = plt.subplots(figsize=(12, 6)) +worldmap.plot(color="darkgrey", ax=ax) +marker = markers.MarkerStyle(marker="s") +ax.scatter(eccodes_longs, eccodes_lats, s=12, c="red", marker=marker, facecolors="none") +ax.scatter(longs, lats, s=4, c=parameter_values, cmap="viridis") +norm = mpl.colors.Normalize(vmin=min(parameter_values), vmax=max(parameter_values)) +sm = plt.cm.ScalarMappable(cmap="viridis", norm=norm) +sm.set_array([]) +plt.colorbar(sm, label="Wind Speed") +plt.show() diff --git a/examples/octahedral_grid_example.py b/examples/octahedral_grid_country_example.py similarity index 100% rename from examples/octahedral_grid_example.py rename to examples/octahedral_grid_country_example.py diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 865b2c9fd..49f880c8b 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -1,10 +1,5 @@ -import geopandas as gpd -import matplotlib as mpl -import matplotlib.pyplot as plt -import numpy as np from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file -from matplotlib import markers from polytope.datacube.octahedral_xarray import OctahedralXArrayDatacube from polytope.engine.hullslicer import HullSlicer @@ -12,72 +7,61 @@ from polytope.shapes import Box -def find_nearest_latlon(grib_file, target_lat, target_lon): - # Open the GRIB file - f = open(grib_file) +class TestOctahedralGrid: + def setup_method(self, method): + ds = data.from_source("file", "./foo.grib") + latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) + latlon_array = latlon_array.t2m + self.xarraydatacube = OctahedralXArrayDatacube(latlon_array) + self.slicer = HullSlicer() + self.API = Polytope(datacube=latlon_array, engine=self.slicer) - # Load the GRIB messages from the file - messages = [] - while True: - message = codes_grib_new_from_file(f) - if message is None: - break - messages.append(message) + def find_nearest_latlon(self, grib_file, target_lat, target_lon): + # Open the GRIB file + f = open(grib_file) - # Find the nearest grid points - nearest_points = [] - for message in messages: - nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) - nearest_points.append(nearest_index) + # Load the GRIB messages from the file + messages = [] + while True: + message = codes_grib_new_from_file(f) + if message is None: + break + messages.append(message) - # Close the GRIB file - f.close() + # Find the nearest grid points + nearest_points = [] + for message in messages: + nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) + nearest_points.append(nearest_index) - return nearest_points + # Close the GRIB file + f.close() + return nearest_points -ds = data.from_source("file", "./foo.grib") -latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) -latlon_array = latlon_array.t2m -nearest_points = find_nearest_latlon("./foo.grib", 0, 0) + def test_octahedral_grid(self): + request = Request(Box(["latitude", "longitude"], [0, 0], [0.5, 0.5])) + result = self.API.retrieve(request) + assert len(result.leaves) == 56 -latlon_xarray_datacube = OctahedralXArrayDatacube(latlon_array) - -slicer = HullSlicer() -API = Polytope(datacube=latlon_array, engine=slicer) - -request = Request(Box(["latitude", "longitude"], [0, 0], [0.5, 0.5])) -result = API.retrieve(request) -result.pprint() - -lats = [] -longs = [] -eccodes_lats = [] -eccodes_longs = [] -parameter_values = [] -for i in range(len(result.leaves)): - cubepath = result.leaves[i].flatten() - lat = cubepath["latitude"] - long = cubepath["longitude"] - lats.append(lat) - longs.append(long) - nearest_points = find_nearest_latlon("./foo.grib", lat, long) - eccodes_lats.append(nearest_points[0][0]["lat"]) - eccodes_longs.append(nearest_points[0][0]["lon"]) - t = result.leaves[i].result[1] - parameter_values.append(t) - - -parameter_values = np.array(parameter_values) -# Plot this last array according to different colors for the result on a world map -worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) -fig, ax = plt.subplots(figsize=(12, 6)) -worldmap.plot(color="darkgrey", ax=ax) -marker = markers.MarkerStyle(marker="s") -ax.scatter(eccodes_longs, eccodes_lats, s=12, c="red", marker=marker, facecolors="none") -ax.scatter(longs, lats, s=4, c=parameter_values, cmap="viridis") -norm = mpl.colors.Normalize(vmin=min(parameter_values), vmax=max(parameter_values)) -sm = plt.cm.ScalarMappable(cmap="viridis", norm=norm) -sm.set_array([]) -plt.colorbar(sm, label="Wind Speed") -plt.show() + lats = [] + lons = [] + eccodes_lats = [] + eccodes_lons = [] + tol = 1e-3 + for i in range(len(result.leaves)): + cubepath = result.leaves[i].flatten() + lat = cubepath["latitude"] + lon = cubepath["longitude"] + lats.append(lat) + lons.append(lon) + nearest_points = self.find_nearest_latlon("./foo.grib", lat, lon) + eccodes_lat = nearest_points[0][0]["lat"] + eccodes_lon = nearest_points[0][0]["lon"] + eccodes_lats.append(eccodes_lat) + eccodes_lons.append(eccodes_lon) + assert eccodes_lat - tol <= lat + assert lat <= eccodes_lat + tol + assert eccodes_lon - tol <= lon + assert lon <= eccodes_lon + tol + assert len(eccodes_lats) == 56 From 968c5fee07760caa2ac22f54607c0961e0a42cd3 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 7 Jul 2023 13:25:48 +0100 Subject: [PATCH 029/332] test --- polytope/datacube/xarray.py | 1 + 1 file changed, 1 insertion(+) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 3e079ba62..5d4e6fdf9 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -60,6 +60,7 @@ def __init__(self, dataarray: xr.DataArray, options={}): else: # drop non-necessary coordinates which we don't slice on dataarray = dataarray.reset_coords(names=name, drop=True) self.dataarray = dataarray + def get(self, requests: IndexTree): for r in requests.leaves: From ce435f0d819383cb93372e6ac28f53f0a9feb5fd Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 11 Jul 2023 15:55:03 +0200 Subject: [PATCH 030/332] make mapper class work --- polytope/datacube/datacube.py | 7 +- polytope/datacube/mappers.py | 175 ++++++++++++++++++++++++++++++++++ polytope/datacube/xarray.py | 99 +++++++++++++++---- polytope/polytope.py | 4 +- tests/test_octahedral_grid.py | 5 +- 5 files changed, 263 insertions(+), 27 deletions(-) create mode 100644 polytope/datacube/mappers.py diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index 4195e40ef..c1d83d8a1 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -41,10 +41,9 @@ def validate(self, axes) -> bool: """returns true if the input axes can be resolved against the datacube axes""" @staticmethod - def create(datacube, options): + def create(datacube, options, grid_options): if isinstance(datacube, (xr.core.dataarray.DataArray, xr.core.dataset.Dataset)): - # from .xarray import XArrayDatacube - from .octahedral_xarray import OctahedralXArrayDatacube + from .xarray import XArrayDatacube - xadatacube = OctahedralXArrayDatacube(datacube, options=options) + xadatacube = XArrayDatacube(datacube, options=options, grid_options=grid_options) return xadatacube diff --git a/polytope/datacube/mappers.py b/polytope/datacube/mappers.py new file mode 100644 index 000000000..a894dc0c1 --- /dev/null +++ b/polytope/datacube/mappers.py @@ -0,0 +1,175 @@ +import math +from abc import ABC, abstractmethod, abstractproperty + + +class GridMappers(ABC): + @abstractproperty + def _mapped_axes(self): + pass + + @abstractproperty + def _base_axis(self): + pass + + @abstractproperty + def _resolution(self): + pass + + @abstractmethod + def map_first_axis(self, lower, upper): + pass + + @abstractmethod + def map_second_axis(self, first_val, lower, upper): + pass + + @abstractmethod + def unmap(self): + pass + + +class OctahedralGridMap(ABC): + def __init__(self, base_axis, mapped_axes, resolution): + self._mapped_axes = mapped_axes + self._base_axis = base_axis + self._resolution = resolution + + def gauss_first_guess(self): + i = 0 + gvals = [2.4048255577E0, + 5.5200781103E0, + 8.6537279129E0, + 11.7915344391E0, + 14.9309177086E0, + 18.0710639679E0, + 21.2116366299E0, + 24.3524715308E0, + 27.4934791320E0, + 30.6346064684E0, + 33.7758202136E0, + 36.9170983537E0, + 40.0584257646E0, + 43.1997917132E0, + 46.3411883717E0, + 49.4826098974E0, + 52.6240518411E0, + 55.7655107550E0, + 58.9069839261E0, + 62.0484691902E0, + 65.1899648002E0, + 68.3314693299E0, + 71.4729816036E0, + 74.6145006437E0, + 77.7560256304E0, + 80.8975558711E0, + 84.0390907769E0, + 87.1806298436E0, + 90.3221726372E0, + 93.4637187819E0, + 96.6052679510E0, + 99.7468198587E0, + 102.8883742542E0, + 106.0299309165E0, + 109.1714896498E0, + 112.3130502805E0, + 115.4546126537E0, + 118.5961766309E0, + 121.7377420880E0, + 124.8793089132E0, + 128.0208770059E0, + 131.1624462752E0, + 134.3040166383E0, + 137.4455880203E0, + 140.5871603528E0, + 143.7287335737E0, + 146.8703076258E0, + 150.0118824570E0, + 153.1534580192E0, + 156.2950342685E0] + + numVals = len(gvals) + vals = [] + for i in range(self._resolution): + if i < numVals: + vals.append(gvals[i]) + else: + vals.append(vals[i-1] + math.pi) + return vals + + def first_axis_vals(self): + precision = 1.0E-14 + nval = self._resolution * 2 + rad2deg = 180/math.pi + convval = (1-((2/math.pi)*(2/math.pi))*0.25) + vals = self.gauss_first_guess() + new_vals = [0]*nval + denom = math.sqrt(((nval+0.5)*(nval+0.5))+convval) + for jval in range(self._resolution): + root = math.cos(vals[jval]/denom) + conv = 1 + while abs(conv) >= precision: + mem2 = 1 + mem1 = root + for legi in range(nval): + legfonc = ((2.0 * (legi + 1) - 1.0) * root * mem1 - legi * mem2) / (legi + 1) + mem2 = mem1 + mem1 = legfonc + conv = legfonc / ((nval * (mem2 - root * legfonc)) / (1.0 - (root * root))) + root = root - conv + # add maybe a max iter here to make sure we converge at some point + new_vals[jval] = math.asin(root) * rad2deg + new_vals[nval-1-jval] = -new_vals[jval] + return new_vals + + def map_first_axis(self, lower, upper): + axis_lines = self.first_axis_vals() + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def second_axis_vals(self, first_val): + first_axis_vals = self.first_axis_vals() + tol = 1E-10 + first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] + first_idx = first_axis_vals.index(first_val) + if first_idx >= self._resolution: + first_idx = (2*self._resolution) - 1 - first_idx + first_idx = first_idx + 1 + npoints = 4 * first_idx + 16 + second_axis_spacing = 360 / npoints + second_axis_start = 0 + second_axis_vals = [second_axis_start + i * second_axis_spacing for i in range(int(npoints))] + return second_axis_vals + + def map_second_axis(self, first_val, lower, upper): + second_axis_vals = self.second_axis_vals(first_val) + return_vals = [val for val in second_axis_vals if lower <= val <= upper] + return return_vals + + def axes_idx_to_octahedral_idx(self, first_idx, second_idx): + octa_idx = 0 + if first_idx == 1: + octa_idx = second_idx + else: + for i in range(first_idx-1): + if i <= self._resolution-1: + octa_idx += 16 + 4 * i + else: + i = i - self._resolution + 1 + if i == 1: + octa_idx += 16 + 4 * self._resolution + else: + i = i - 1 + octa_idx += 16 + 4 * (self._resolution - i + 1) + octa_idx += second_idx + return octa_idx + + def unmap(self, first_val, second_val): + first_axis_vals = self.first_axis_vals() + tol = 1E-10 + first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] + first_idx = first_axis_vals.index(first_val) + 1 + second_axis_vals = self.second_axis_vals(first_val) + second_val = [val for val in second_axis_vals if second_val - tol < val < second_val + tol][0] + second_idx = second_axis_vals.index(second_val) + octahedral_index = self.axes_idx_to_octahedral_idx(first_idx, second_idx) + return octahedral_index diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 5d4e6fdf9..0c9594dfd 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -15,6 +15,7 @@ PandasTimestampAxis, UnsliceableaAxis, ) +from .mappers import OctahedralGridMap _mappings = { pd.Int64Dtype: IntAxis(), @@ -50,27 +51,59 @@ def _set_mapper(self, values, name): self.mappers[name] = deepcopy(_mappings[values.dtype.type]) self.mappers[name].name = name - def __init__(self, dataarray: xr.DataArray, options={}): + def _set_grid_mapper(self, name): + if name in self.grid_options.keys(): + if "grid_map" in self.grid_options[name].keys(): + grid_mapping_options = self.grid_options[name]["grid_map"] + grid_type = grid_mapping_options["type"] + grid_axes = grid_mapping_options["axes"] + if grid_type[0] == "octahedral": + resolution = grid_type[1] + self.grid_mapper = OctahedralGridMap(name, grid_axes, resolution) + + def __init__(self, dataarray: xr.DataArray, options={}, grid_options={}): self.options = options + self.grid_options = grid_options + self.grid_mapper = None + self.axis_counter = 0 + for name in dataarray.dims: + self._set_grid_mapper(name) self.mappers = {} for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: - dataarray = dataarray.sortby(name) self._set_mapper(values, name) - else: # drop non-necessary coordinates which we don't slice on - dataarray = dataarray.reset_coords(names=name, drop=True) + self.axis_counter += 1 + if self.grid_mapper is not None: + if name in self.grid_mapper._mapped_axes: + self._set_mapper(values, name) + self.axis_counter += 1 self.dataarray = dataarray - def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() path = self.remap_path(path) - if len(path.items()) == len(self.dataarray.coords): - subxarray = self.dataarray.sel(path, method="nearest") - value = subxarray.item() - key = subxarray.name - r.result = (key, value) + if len(path.items()) == self.axis_counter: + if self.grid_mapper is not None: + first_axis = self.grid_mapper._mapped_axes[0] + first_val = path[first_axis] + second_axis = self.grid_mapper._mapped_axes[1] + second_val = path[second_axis] + path.pop(first_axis, None) + path.pop(second_axis, None) + subxarray = self.dataarray.sel(path, method="nearest") + # need to remap the lat, lon in path to dataarray index + unmapped_idx = self.grid_mapper.unmap(first_val, second_val) + subxarray = subxarray.isel(values=unmapped_idx) + value = subxarray.item() + key = subxarray.name + r.result = (key, value) + else: + # if we have no grid map, still need to assign values + subxarray = self.dataarray.sel(path, method="nearest") + value = subxarray.item() + key = subxarray.name + r.result = (key, value) else: r.remove_branch() @@ -83,19 +116,32 @@ def remap_path(self, path: DatacubePath): path[key] = self.mappers[key].remap_val_to_axis_range(value) return path - def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): + def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): idx_between = [] for i in range(len(search_ranges)): r = search_ranges[i] offset = search_ranges_offset[i] low = r[0] up = r[1] - # Find the range of indexes between lower and upper - # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html - # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? - end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? - indexes_between = indexes[start:end].to_list() + + if self.grid_mapper is not None: + first_axis = self.grid_mapper._mapped_axes[0] + second_axis = self.grid_mapper._mapped_axes[1] + if axis.name == first_axis: + indexes_between = self.grid_mapper.map_first_axis(low, up) + elif axis.name == second_axis: + indexes_between = self.grid_mapper.map_second_axis(first_val, low, up) + else: + start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? + end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? + indexes_between = indexes[start:end].to_list() + else: + # Find the range of indexes between lower and upper + # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html + # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing + start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? + end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? + indexes_between = indexes[start:end].to_list() # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them @@ -110,13 +156,28 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): def get_indices(self, path: DatacubePath, axis, lower, upper): path = self.remap_path(path) + first_val = None + if self.grid_mapper is not None: + first_axis = self.grid_mapper._mapped_axes[0] + first_val = path.get(first_axis, None) + second_axis = self.grid_mapper._mapped_axes[1] + path.pop(first_axis, None) + path.pop(second_axis, None) # Open a view on the subset identified by the path subarray = self.dataarray.sel(path, method="nearest") # Get the indexes of the axis we want to query # XArray does not support branching, so no need to use label, we just take the next axis - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + if self.grid_mapper is not None: + if axis.name == first_axis: + indexes = [] + elif axis.name == second_axis: + indexes = [] + else: + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + else: + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube search_ranges = axis.remap([lower, upper]) @@ -129,7 +190,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): search_ranges_offset.append(offset) # Look up the values in the datacube for each cyclic interval range - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis) + idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) # Remove duplicates even if difference of the order of the axis tolerance if offset is not None: diff --git a/polytope/polytope.py b/polytope/polytope.py index d84716c02..3eac8aab0 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -31,11 +31,11 @@ def polytopes(self): class Polytope: - def __init__(self, datacube, engine=None, options={}): + def __init__(self, datacube, engine=None, options={}, grid_options={}): from .datacube import Datacube from .engine import Engine - self.datacube = Datacube.create(datacube, options) + self.datacube = Datacube.create(datacube, options, grid_options) self.engine = engine if engine is not None else Engine.default() def slice(self, polytopes: List[ConvexPolytope]): diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 49f880c8b..cd075fadb 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -13,8 +13,9 @@ def setup_method(self, method): latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m self.xarraydatacube = OctahedralXArrayDatacube(latlon_array) + grid_options = {"values" : {"grid_map" : {"type" : ["octahedral", 1280], "axes" : ["latitude", "longitude"]}}} self.slicer = HullSlicer() - self.API = Polytope(datacube=latlon_array, engine=self.slicer) + self.API = Polytope(datacube=latlon_array, engine=self.slicer, grid_options=grid_options) def find_nearest_latlon(self, grib_file, target_lat, target_lon): # Open the GRIB file @@ -48,7 +49,7 @@ def test_octahedral_grid(self): lons = [] eccodes_lats = [] eccodes_lons = [] - tol = 1e-3 + tol = 1e-8 for i in range(len(result.leaves)): cubepath = result.leaves[i].flatten() lat = cubepath["latitude"] From cfd1dea6fdecc12506fa34a579e77ce6adfd9a5a Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 12 Jul 2023 13:55:00 +0200 Subject: [PATCH 031/332] black --- polytope/datacube/mappers.py | 128 +++++++++++++++++----------------- tests/test_octahedral_grid.py | 2 +- 2 files changed, 66 insertions(+), 64 deletions(-) diff --git a/polytope/datacube/mappers.py b/polytope/datacube/mappers.py index a894dc0c1..8727556ab 100644 --- a/polytope/datacube/mappers.py +++ b/polytope/datacube/mappers.py @@ -36,56 +36,58 @@ def __init__(self, base_axis, mapped_axes, resolution): def gauss_first_guess(self): i = 0 - gvals = [2.4048255577E0, - 5.5200781103E0, - 8.6537279129E0, - 11.7915344391E0, - 14.9309177086E0, - 18.0710639679E0, - 21.2116366299E0, - 24.3524715308E0, - 27.4934791320E0, - 30.6346064684E0, - 33.7758202136E0, - 36.9170983537E0, - 40.0584257646E0, - 43.1997917132E0, - 46.3411883717E0, - 49.4826098974E0, - 52.6240518411E0, - 55.7655107550E0, - 58.9069839261E0, - 62.0484691902E0, - 65.1899648002E0, - 68.3314693299E0, - 71.4729816036E0, - 74.6145006437E0, - 77.7560256304E0, - 80.8975558711E0, - 84.0390907769E0, - 87.1806298436E0, - 90.3221726372E0, - 93.4637187819E0, - 96.6052679510E0, - 99.7468198587E0, - 102.8883742542E0, - 106.0299309165E0, - 109.1714896498E0, - 112.3130502805E0, - 115.4546126537E0, - 118.5961766309E0, - 121.7377420880E0, - 124.8793089132E0, - 128.0208770059E0, - 131.1624462752E0, - 134.3040166383E0, - 137.4455880203E0, - 140.5871603528E0, - 143.7287335737E0, - 146.8703076258E0, - 150.0118824570E0, - 153.1534580192E0, - 156.2950342685E0] + gvals = [ + 2.4048255577e0, + 5.5200781103e0, + 8.6537279129e0, + 11.7915344391e0, + 14.9309177086e0, + 18.0710639679e0, + 21.2116366299e0, + 24.3524715308e0, + 27.4934791320e0, + 30.6346064684e0, + 33.7758202136e0, + 36.9170983537e0, + 40.0584257646e0, + 43.1997917132e0, + 46.3411883717e0, + 49.4826098974e0, + 52.6240518411e0, + 55.7655107550e0, + 58.9069839261e0, + 62.0484691902e0, + 65.1899648002e0, + 68.3314693299e0, + 71.4729816036e0, + 74.6145006437e0, + 77.7560256304e0, + 80.8975558711e0, + 84.0390907769e0, + 87.1806298436e0, + 90.3221726372e0, + 93.4637187819e0, + 96.6052679510e0, + 99.7468198587e0, + 102.8883742542e0, + 106.0299309165e0, + 109.1714896498e0, + 112.3130502805e0, + 115.4546126537e0, + 118.5961766309e0, + 121.7377420880e0, + 124.8793089132e0, + 128.0208770059e0, + 131.1624462752e0, + 134.3040166383e0, + 137.4455880203e0, + 140.5871603528e0, + 143.7287335737e0, + 146.8703076258e0, + 150.0118824570e0, + 153.1534580192e0, + 156.2950342685e0, + ] numVals = len(gvals) vals = [] @@ -93,19 +95,19 @@ def gauss_first_guess(self): if i < numVals: vals.append(gvals[i]) else: - vals.append(vals[i-1] + math.pi) + vals.append(vals[i - 1] + math.pi) return vals def first_axis_vals(self): - precision = 1.0E-14 + precision = 1.0e-14 nval = self._resolution * 2 - rad2deg = 180/math.pi - convval = (1-((2/math.pi)*(2/math.pi))*0.25) + rad2deg = 180 / math.pi + convval = 1 - ((2 / math.pi) * (2 / math.pi)) * 0.25 vals = self.gauss_first_guess() - new_vals = [0]*nval - denom = math.sqrt(((nval+0.5)*(nval+0.5))+convval) + new_vals = [0] * nval + denom = math.sqrt(((nval + 0.5) * (nval + 0.5)) + convval) for jval in range(self._resolution): - root = math.cos(vals[jval]/denom) + root = math.cos(vals[jval] / denom) conv = 1 while abs(conv) >= precision: mem2 = 1 @@ -118,7 +120,7 @@ def first_axis_vals(self): root = root - conv # add maybe a max iter here to make sure we converge at some point new_vals[jval] = math.asin(root) * rad2deg - new_vals[nval-1-jval] = -new_vals[jval] + new_vals[nval - 1 - jval] = -new_vals[jval] return new_vals def map_first_axis(self, lower, upper): @@ -128,11 +130,11 @@ def map_first_axis(self, lower, upper): def second_axis_vals(self, first_val): first_axis_vals = self.first_axis_vals() - tol = 1E-10 + tol = 1e-10 first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] first_idx = first_axis_vals.index(first_val) if first_idx >= self._resolution: - first_idx = (2*self._resolution) - 1 - first_idx + first_idx = (2 * self._resolution) - 1 - first_idx first_idx = first_idx + 1 npoints = 4 * first_idx + 16 second_axis_spacing = 360 / npoints @@ -150,8 +152,8 @@ def axes_idx_to_octahedral_idx(self, first_idx, second_idx): if first_idx == 1: octa_idx = second_idx else: - for i in range(first_idx-1): - if i <= self._resolution-1: + for i in range(first_idx - 1): + if i <= self._resolution - 1: octa_idx += 16 + 4 * i else: i = i - self._resolution + 1 @@ -165,7 +167,7 @@ def axes_idx_to_octahedral_idx(self, first_idx, second_idx): def unmap(self, first_val, second_val): first_axis_vals = self.first_axis_vals() - tol = 1E-10 + tol = 1e-10 first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] first_idx = first_axis_vals.index(first_val) + 1 second_axis_vals = self.second_axis_vals(first_val) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index cd075fadb..207a7590c 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -13,7 +13,7 @@ def setup_method(self, method): latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m self.xarraydatacube = OctahedralXArrayDatacube(latlon_array) - grid_options = {"values" : {"grid_map" : {"type" : ["octahedral", 1280], "axes" : ["latitude", "longitude"]}}} + grid_options = {"values": {"grid_map": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, grid_options=grid_options) From cd20268c809a0470429bb71d700dda5b3b51fb4e Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 12 Jul 2023 13:57:13 +0200 Subject: [PATCH 032/332] remove unnecessary file --- polytope/datacube/octahedral_xarray.py | 235 ------------------------- 1 file changed, 235 deletions(-) delete mode 100644 polytope/datacube/octahedral_xarray.py diff --git a/polytope/datacube/octahedral_xarray.py b/polytope/datacube/octahedral_xarray.py deleted file mode 100644 index 8a98d2b44..000000000 --- a/polytope/datacube/octahedral_xarray.py +++ /dev/null @@ -1,235 +0,0 @@ -import math -import sys -from copy import deepcopy - -import numpy as np -import pandas as pd -import xarray as xr - -from ..utility.combinatorics import unique, validate_axes -from .datacube import Datacube, DatacubePath, IndexTree -from .datacube_axis import ( - FloatAxis, - IntAxis, - PandasTimedeltaAxis, - PandasTimestampAxis, - UnsliceableaAxis, -) - -_mappings = { - pd.Int64Dtype: IntAxis(), - pd.Timestamp: PandasTimestampAxis(), - np.int64: IntAxis(), - np.datetime64: PandasTimestampAxis(), - np.timedelta64: PandasTimedeltaAxis(), - np.float64: FloatAxis(), - np.str_: UnsliceableaAxis(), - str: UnsliceableaAxis(), -} - - -class OctahedralXArrayDatacube(Datacube): - """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0).""" - - def _set_mapper(self, values, name): - if values.dtype.type not in _mappings: - raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") - if name in self.options.keys(): - # The options argument here is supposed to be a nested dictionary - # like {"latitude":{"Cyclic":range}, ...} - if "Cyclic" in self.options[name].keys(): - value_type = values.dtype.type - axes_type_str = type(_mappings[value_type]).__name__ - axes_type_str += "Cyclic" - cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) - self.mappers[name] = cyclic_axis_type - self.mappers[name].name = name - self.mappers[name].range = self.options[name]["Cyclic"] - else: - self.mappers[name] = deepcopy(_mappings[values.dtype.type]) - self.mappers[name].name = name - - def __init__(self, dataarray: xr.DataArray, options={}): - self.options = options - self.mappers = {} - for name, values in dataarray.coords.variables.items(): - if values.data.size != 1: - self._set_mapper(values, name) - else: # drop non-necessary coordinates which we don't slice on - dataarray = dataarray.reset_coords(names=name, drop=True) - - self.dataarray = dataarray - - def get(self, requests: IndexTree): - for r in requests.leaves: - path = r.flatten() - path = self.remap_path(path) - len_coords = 0 - for name, values in self.dataarray.coords.variables.items(): - if values.data.size != 1: - len_coords += 1 - if len(path.items()) == len_coords: - lat_val = path["latitude"] - lon_val = path["longitude"] - path.pop("longitude", None) - path.pop("latitude", None) - subxarray = self.dataarray.sel(path, method="nearest") - # need to remap the lat, lon in path to dataarray index - lat_idx, lon_idx = latlon_val_to_idx(lat_val, lon_val) - octa_idx = latlon_idx_to_octa_idx(lat_idx, lon_idx) - subxarray = subxarray.isel(values=octa_idx) - value = subxarray.item() - key = subxarray.name - r.result = (key, value) - else: - r.remove_branch() - - def get_mapper(self, axis): - return self.mappers[axis] - - def remap_path(self, path: DatacubePath): - for key in path: - value = path[key] - path[key] = self.mappers[key].remap_val_to_axis_range(value) - return path - - def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): - idx_between = [] - for i in range(len(search_ranges)): - r = search_ranges[i] - offset = search_ranges_offset[i] - low = r[0] - up = r[1] - - if axis.name == "latitude" or axis.name == "longitude": - indexes_between = [i for i in indexes if low <= i <= up] - else: - # Find the range of indexes between lower and upper - # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html - # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? - end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? - indexes_between = indexes[start:end].to_list() - - # Now the indexes_between are values on the cyclic range so need to remap them to their original - # values before returning them - for j in range(len(indexes_between)): - if offset is None: - indexes_between[j] = indexes_between[j] - else: - indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) - - idx_between.append(indexes_between[j]) - return idx_between - - def get_indices(self, path: DatacubePath, axis, lower, upper): - path = self.remap_path(path) - # Open a view on the subset identified by the path - lat_val = path.get("latitude", None) - path.pop("longitude", None) - path.pop("latitude", None) - subarray = self.dataarray.sel(path, method="nearest") - - # Get the indexes of the axis we want to query - # XArray does not support branching, so no need to use label, we just take the next axis - if axis.name == "latitude": - indexes = self.lat_val_available() - elif axis.name == "longitude": - indexes = self.lon_val_available(lat_val) - else: - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() - - # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube - search_ranges = axis.remap([lower, upper]) - original_search_ranges = axis.to_intervals([lower, upper]) - - # Find the offsets for each interval in the requested range, which we will need later - search_ranges_offset = [] - for r in original_search_ranges: - offset = axis.offset(r) - search_ranges_offset.append(offset) - - # Look up the values in the datacube for each cyclic interval range - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis) - - # Remove duplicates even if difference of the order of the axis tolerance - if offset is not None: - # Note that we can only do unique if not dealing with time values - idx_between = unique(idx_between) - - return idx_between - - def has_index(self, path: DatacubePath, axis, index): - # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - subarray = self.dataarray.sel(path)[axis.name] - subarray_vals = subarray.values - return index in subarray_vals - - @property - def axes(self): - return self.mappers - - def validate(self, axes): - return validate_axes(self.axes, axes) - - def lat_val_available(self): - lat_spacing = 90 / 1280 - lat_start = 0.035149384215604956 - return_lat = [lat_start + i * lat_spacing - 90 for i in range(1280 * 2)] - return return_lat - - def lon_val_available(self, lat): - lat_spacing = 90 / 1280 - lat_start = 0.035149384215604956 - if lat_start <= lat < 90: - lat_idx = 1280 - ((lat - lat_start) / lat_spacing) - else: - lat_idx = (lat + 90 - lat_start) / lat_spacing - num_points_on_lon = 4 * lat_idx + 16 - lon_spacing = 360 / num_points_on_lon - lon_start = 0 - return_lon = [lon_start + i * lon_spacing for i in range(int(num_points_on_lon))] - return return_lon - - -def latlon_idx_to_octa_idx(lat_idx, lon_idx): - if lat_idx <= 1280: - # we are at lat >0 - # lat_idx 0 is lat=90 - octa_idx = 0 - if lat_idx == 1: - octa_idx = lon_idx - else: - for i in range(lat_idx - 1): - octa_idx += 16 + 4 * (i) - octa_idx = int(octa_idx + lon_idx) - else: - # we are at lat <0 - octa_idx = int(6599680 / 2) - lat_idx = lat_idx - 1280 - if lat_idx == 1: - octa_idx += lon_idx - else: - for i in range(lat_idx - 1): - octa_idx += 16 + 4 * (1280 - i - 1) - octa_idx = octa_idx + lon_idx - if lat_idx == 1280: - lat_idx -= 16 - return octa_idx - - -def latlon_val_to_idx(lat, lon): - lat_spacing = 90 / 1280 - lat_start = 0.03515625 - if lat_start <= lat <= 90: - # if we are at lat=90, the latitude line index is 0 - lat_idx = 1280 - ((lat - lat_start) / lat_spacing) - else: - # if we are at lat=-90, the latitude line index is 2560 - lat_idx = 1280 - ((lat - lat_start) / lat_spacing) - num_points_on_lon = 4 * lat_idx + 16 - lon_spacing = 360 / num_points_on_lon - lon_start = 0 - lon_idx = (lon - lon_start) / lon_spacing - - return (int(lat_idx), lon_idx) From c8f80570c01987f1f9600408a162169a4a84d718 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 12 Jul 2023 14:30:37 +0200 Subject: [PATCH 033/332] fix problems with tests --- polytope/datacube/xarray.py | 1 + tests/test_cyclic_axis_slicer_not_0.py | 5 ----- tests/test_datacube_xarray.py | 8 ++++---- tests/test_octahedral_grid.py | 4 ++-- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 0c9594dfd..f9104c140 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -71,6 +71,7 @@ def __init__(self, dataarray: xr.DataArray, options={}, grid_options={}): self.mappers = {} for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: + dataarray = dataarray.sortby(name) self._set_mapper(values, name) self.axis_counter += 1 if self.grid_mapper is not None: diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 93147c05b..6b43d84c4 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -62,7 +62,6 @@ def test_cyclic_float_axis_inside_cyclic_range(self): Box(["step", "long"], [0, 0.0], [3, 0.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) - # result.pprint() assert len(result.leaves) == 16 assert [leaf.value for leaf in result.leaves] == [ 0.0, @@ -88,7 +87,6 @@ def test_cyclic_float_axis_above_axis_range(self): Box(["step", "long"], [0, 1.3], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) - # result.pprint() assert len(result.leaves) == 10 assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] @@ -97,7 +95,6 @@ def test_cyclic_float_axis_two_range_loops(self): Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) - # result.pprint() assert len(result.leaves) == 50 assert [leaf.value for leaf in result.leaves] == [ 0.3, @@ -157,7 +154,6 @@ def test_cyclic_float_axis_below_axis_range(self): Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) - # result.pprint() assert len(result.leaves) == 10 assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, -0.3] @@ -166,7 +162,6 @@ def test_cyclic_float_axis_below_axis_range_crossing_seam(self): Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) - # result.pprint() assert len(result.leaves) == 22 assert [leaf.value for leaf in result.leaves] == [ -0.7, diff --git a/tests/test_datacube_xarray.py b/tests/test_datacube_xarray.py index 52e61172f..81554a26a 100644 --- a/tests/test_datacube_xarray.py +++ b/tests/test_datacube_xarray.py @@ -18,8 +18,8 @@ def test_validate(self): array = xr.Dataset(data_vars=dict(param=(["x", "y", "z"], dims)), coords={"x": [1], "y": [1], "z": [1]}) array = array.to_array() - datacube = Datacube.create(array, options={}) - datacube = Datacube.create(array, options={}) + datacube = Datacube.create(array, options={}, grid_options={}) + datacube = Datacube.create(array, options={}, grid_options={}) datacube.validate(["x", "y", "z", "variable"]) datacube.validate(["x", "z", "y", "variable"]) @@ -48,8 +48,8 @@ def test_create(self): for d, v in array.coords.variables.items(): print(v.dtype) - datacube = Datacube.create(array, options={}) - datacube = Datacube.create(array, options={}) + datacube = Datacube.create(array, options={}, grid_options={}) + datacube = Datacube.create(array, options={}, grid_options={}) # Check the factory created the correct type of datacube assert isinstance(datacube, XArrayDatacube) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 207a7590c..4da295f3c 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -1,7 +1,7 @@ from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file -from polytope.datacube.octahedral_xarray import OctahedralXArrayDatacube +from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box @@ -12,7 +12,7 @@ def setup_method(self, method): ds = data.from_source("file", "./foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m - self.xarraydatacube = OctahedralXArrayDatacube(latlon_array) + self.xarraydatacube = XArrayDatacube(latlon_array) grid_options = {"values": {"grid_map": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, grid_options=grid_options) From 62844bc7ba44793f26076d9e0dc2b3614fdd381f Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 12 Jul 2023 14:33:16 +0200 Subject: [PATCH 034/332] fix examples and remove unnecessary files --- examples/octahedral_grid_box_example.py | 10 +- examples/octahedral_grid_country_example.py | 9 +- polytope/datacube/octahedral_xarray.py | 235 -------------------- 3 files changed, 13 insertions(+), 241 deletions(-) delete mode 100644 polytope/datacube/octahedral_xarray.py diff --git a/examples/octahedral_grid_box_example.py b/examples/octahedral_grid_box_example.py index 865b2c9fd..61af057ed 100644 --- a/examples/octahedral_grid_box_example.py +++ b/examples/octahedral_grid_box_example.py @@ -6,7 +6,7 @@ from eccodes import codes_grib_find_nearest, codes_grib_new_from_file from matplotlib import markers -from polytope.datacube.octahedral_xarray import OctahedralXArrayDatacube +from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box @@ -41,12 +41,16 @@ def find_nearest_latlon(grib_file, target_lat, target_lon): latlon_array = latlon_array.t2m nearest_points = find_nearest_latlon("./foo.grib", 0, 0) -latlon_xarray_datacube = OctahedralXArrayDatacube(latlon_array) +latlon_xarray_datacube = XArrayDatacube(latlon_array) slicer = HullSlicer() -API = Polytope(datacube=latlon_array, engine=slicer) + +grid_options = {"values": {"grid_map": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} + +API = Polytope(datacube=latlon_array, engine=slicer, grid_options=grid_options) request = Request(Box(["latitude", "longitude"], [0, 0], [0.5, 0.5])) + result = API.retrieve(request) result.pprint() diff --git a/examples/octahedral_grid_country_example.py b/examples/octahedral_grid_country_example.py index da0fd9a64..b3ddd10ce 100644 --- a/examples/octahedral_grid_country_example.py +++ b/examples/octahedral_grid_country_example.py @@ -7,7 +7,7 @@ from matplotlib import markers from shapely.geometry import shape -from polytope.datacube.octahedral_xarray import OctahedralXArrayDatacube +from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Polygon, Union @@ -41,10 +41,13 @@ def find_nearest_latlon(grib_file, target_lat, target_lon): latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m -latlon_xarray_datacube = OctahedralXArrayDatacube(latlon_array) +latlon_xarray_datacube = XArrayDatacube(latlon_array) slicer = HullSlicer() -API = Polytope(datacube=latlon_array, engine=slicer) + +grid_options = {"values": {"grid_map": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} + +API = Polytope(datacube=latlon_array, engine=slicer, grid_options=grid_options) shapefile = gpd.read_file("./examples/data/World_Countries__Generalized_.shp") country = shapefile.iloc[13] diff --git a/polytope/datacube/octahedral_xarray.py b/polytope/datacube/octahedral_xarray.py deleted file mode 100644 index 8a98d2b44..000000000 --- a/polytope/datacube/octahedral_xarray.py +++ /dev/null @@ -1,235 +0,0 @@ -import math -import sys -from copy import deepcopy - -import numpy as np -import pandas as pd -import xarray as xr - -from ..utility.combinatorics import unique, validate_axes -from .datacube import Datacube, DatacubePath, IndexTree -from .datacube_axis import ( - FloatAxis, - IntAxis, - PandasTimedeltaAxis, - PandasTimestampAxis, - UnsliceableaAxis, -) - -_mappings = { - pd.Int64Dtype: IntAxis(), - pd.Timestamp: PandasTimestampAxis(), - np.int64: IntAxis(), - np.datetime64: PandasTimestampAxis(), - np.timedelta64: PandasTimedeltaAxis(), - np.float64: FloatAxis(), - np.str_: UnsliceableaAxis(), - str: UnsliceableaAxis(), -} - - -class OctahedralXArrayDatacube(Datacube): - """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0).""" - - def _set_mapper(self, values, name): - if values.dtype.type not in _mappings: - raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") - if name in self.options.keys(): - # The options argument here is supposed to be a nested dictionary - # like {"latitude":{"Cyclic":range}, ...} - if "Cyclic" in self.options[name].keys(): - value_type = values.dtype.type - axes_type_str = type(_mappings[value_type]).__name__ - axes_type_str += "Cyclic" - cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) - self.mappers[name] = cyclic_axis_type - self.mappers[name].name = name - self.mappers[name].range = self.options[name]["Cyclic"] - else: - self.mappers[name] = deepcopy(_mappings[values.dtype.type]) - self.mappers[name].name = name - - def __init__(self, dataarray: xr.DataArray, options={}): - self.options = options - self.mappers = {} - for name, values in dataarray.coords.variables.items(): - if values.data.size != 1: - self._set_mapper(values, name) - else: # drop non-necessary coordinates which we don't slice on - dataarray = dataarray.reset_coords(names=name, drop=True) - - self.dataarray = dataarray - - def get(self, requests: IndexTree): - for r in requests.leaves: - path = r.flatten() - path = self.remap_path(path) - len_coords = 0 - for name, values in self.dataarray.coords.variables.items(): - if values.data.size != 1: - len_coords += 1 - if len(path.items()) == len_coords: - lat_val = path["latitude"] - lon_val = path["longitude"] - path.pop("longitude", None) - path.pop("latitude", None) - subxarray = self.dataarray.sel(path, method="nearest") - # need to remap the lat, lon in path to dataarray index - lat_idx, lon_idx = latlon_val_to_idx(lat_val, lon_val) - octa_idx = latlon_idx_to_octa_idx(lat_idx, lon_idx) - subxarray = subxarray.isel(values=octa_idx) - value = subxarray.item() - key = subxarray.name - r.result = (key, value) - else: - r.remove_branch() - - def get_mapper(self, axis): - return self.mappers[axis] - - def remap_path(self, path: DatacubePath): - for key in path: - value = path[key] - path[key] = self.mappers[key].remap_val_to_axis_range(value) - return path - - def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): - idx_between = [] - for i in range(len(search_ranges)): - r = search_ranges[i] - offset = search_ranges_offset[i] - low = r[0] - up = r[1] - - if axis.name == "latitude" or axis.name == "longitude": - indexes_between = [i for i in indexes if low <= i <= up] - else: - # Find the range of indexes between lower and upper - # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html - # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? - end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? - indexes_between = indexes[start:end].to_list() - - # Now the indexes_between are values on the cyclic range so need to remap them to their original - # values before returning them - for j in range(len(indexes_between)): - if offset is None: - indexes_between[j] = indexes_between[j] - else: - indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) - - idx_between.append(indexes_between[j]) - return idx_between - - def get_indices(self, path: DatacubePath, axis, lower, upper): - path = self.remap_path(path) - # Open a view on the subset identified by the path - lat_val = path.get("latitude", None) - path.pop("longitude", None) - path.pop("latitude", None) - subarray = self.dataarray.sel(path, method="nearest") - - # Get the indexes of the axis we want to query - # XArray does not support branching, so no need to use label, we just take the next axis - if axis.name == "latitude": - indexes = self.lat_val_available() - elif axis.name == "longitude": - indexes = self.lon_val_available(lat_val) - else: - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() - - # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube - search_ranges = axis.remap([lower, upper]) - original_search_ranges = axis.to_intervals([lower, upper]) - - # Find the offsets for each interval in the requested range, which we will need later - search_ranges_offset = [] - for r in original_search_ranges: - offset = axis.offset(r) - search_ranges_offset.append(offset) - - # Look up the values in the datacube for each cyclic interval range - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis) - - # Remove duplicates even if difference of the order of the axis tolerance - if offset is not None: - # Note that we can only do unique if not dealing with time values - idx_between = unique(idx_between) - - return idx_between - - def has_index(self, path: DatacubePath, axis, index): - # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - subarray = self.dataarray.sel(path)[axis.name] - subarray_vals = subarray.values - return index in subarray_vals - - @property - def axes(self): - return self.mappers - - def validate(self, axes): - return validate_axes(self.axes, axes) - - def lat_val_available(self): - lat_spacing = 90 / 1280 - lat_start = 0.035149384215604956 - return_lat = [lat_start + i * lat_spacing - 90 for i in range(1280 * 2)] - return return_lat - - def lon_val_available(self, lat): - lat_spacing = 90 / 1280 - lat_start = 0.035149384215604956 - if lat_start <= lat < 90: - lat_idx = 1280 - ((lat - lat_start) / lat_spacing) - else: - lat_idx = (lat + 90 - lat_start) / lat_spacing - num_points_on_lon = 4 * lat_idx + 16 - lon_spacing = 360 / num_points_on_lon - lon_start = 0 - return_lon = [lon_start + i * lon_spacing for i in range(int(num_points_on_lon))] - return return_lon - - -def latlon_idx_to_octa_idx(lat_idx, lon_idx): - if lat_idx <= 1280: - # we are at lat >0 - # lat_idx 0 is lat=90 - octa_idx = 0 - if lat_idx == 1: - octa_idx = lon_idx - else: - for i in range(lat_idx - 1): - octa_idx += 16 + 4 * (i) - octa_idx = int(octa_idx + lon_idx) - else: - # we are at lat <0 - octa_idx = int(6599680 / 2) - lat_idx = lat_idx - 1280 - if lat_idx == 1: - octa_idx += lon_idx - else: - for i in range(lat_idx - 1): - octa_idx += 16 + 4 * (1280 - i - 1) - octa_idx = octa_idx + lon_idx - if lat_idx == 1280: - lat_idx -= 16 - return octa_idx - - -def latlon_val_to_idx(lat, lon): - lat_spacing = 90 / 1280 - lat_start = 0.03515625 - if lat_start <= lat <= 90: - # if we are at lat=90, the latitude line index is 0 - lat_idx = 1280 - ((lat - lat_start) / lat_spacing) - else: - # if we are at lat=-90, the latitude line index is 2560 - lat_idx = 1280 - ((lat - lat_start) / lat_spacing) - num_points_on_lon = 4 * lat_idx + 16 - lon_spacing = 360 / num_points_on_lon - lon_start = 0 - lon_idx = (lon - lon_start) / lon_spacing - - return (int(lat_idx), lon_idx) From 528d7fb21dba5b4812b2b8cd09dc38f649c543bc Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 12 Jul 2023 16:54:26 +0200 Subject: [PATCH 035/332] start FDB datacube --- polytope/datacube/FDB_datacube.py | 216 ++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 polytope/datacube/FDB_datacube.py diff --git a/polytope/datacube/FDB_datacube.py b/polytope/datacube/FDB_datacube.py new file mode 100644 index 000000000..1183f5f9c --- /dev/null +++ b/polytope/datacube/FDB_datacube.py @@ -0,0 +1,216 @@ +import math +import sys +from copy import deepcopy + +import numpy as np +import pandas as pd +import xarray as xr + +from ..utility.combinatorics import unique, validate_axes +from .datacube import Datacube, DatacubePath, IndexTree +from .datacube_axis import ( + FloatAxis, + IntAxis, + PandasTimedeltaAxis, + PandasTimestampAxis, + UnsliceableaAxis, +) +from .mappers import OctahedralGridMap + +_mappings = { + pd.Int64Dtype: IntAxis(), + pd.Timestamp: PandasTimestampAxis(), + np.int64: IntAxis(), + np.datetime64: PandasTimestampAxis(), + np.timedelta64: PandasTimedeltaAxis(), + np.float64: FloatAxis(), + np.str_: UnsliceableaAxis(), + str: UnsliceableaAxis(), + np.object_: UnsliceableaAxis(), +} + + +class FDBDatacube(Datacube): + + def _set_mapper(self, values, name): + if values.dtype.type not in _mappings: + raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") + if name in self.options.keys(): + # The options argument here is supposed to be a nested dictionary + # like {"latitude":{"Cyclic":range}, ...} + if "Cyclic" in self.options[name].keys(): + value_type = values.dtype.type + axes_type_str = type(_mappings[value_type]).__name__ + axes_type_str += "Cyclic" + cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) + self.mappers[name] = cyclic_axis_type + self.mappers[name].name = name + self.mappers[name].range = self.options[name]["Cyclic"] + else: + self.mappers[name] = deepcopy(_mappings[values.dtype.type]) + self.mappers[name].name = name + + def _set_grid_mapper(self, name): + if name in self.grid_options.keys(): + if "grid_map" in self.grid_options[name].keys(): + grid_mapping_options = self.grid_options[name]["grid_map"] + grid_type = grid_mapping_options["type"] + grid_axes = grid_mapping_options["axes"] + if grid_type[0] == "octahedral": + resolution = grid_type[1] + self.grid_mapper = OctahedralGridMap(name, grid_axes, resolution) + + def __init__(self, config={}, options={}, grid_options={}): + # Need to get the cyclic options and grid options from somewhere + self.options = options + self.grid_options = grid_options + + self.grid_mapper = None + self.axis_counter = 0 + for name in self.grid_options.keys(): + self._set_grid_mapper(name) + self.mappers = {} + + for name, values in dataarray.coords.variables.items(): + if name in dataarray.dims: + dataarray = dataarray.sortby(name) + self._set_mapper(values, name) + self.axis_counter += 1 + if self.grid_mapper is not None: + if name in self.grid_mapper._mapped_axes: + self._set_mapper(values, name) + self.axis_counter += 1 + self.dataarray = dataarray + + def get(self, requests: IndexTree): + for r in requests.leaves: + path = r.flatten() + path = self.remap_path(path) + if len(path.items()) == self.axis_counter: + if self.grid_mapper is not None: + first_axis = self.grid_mapper._mapped_axes[0] + first_val = path[first_axis] + second_axis = self.grid_mapper._mapped_axes[1] + second_val = path[second_axis] + path.pop(first_axis, None) + path.pop(second_axis, None) + subxarray = self.dataarray.sel(path, method="nearest") + # need to remap the lat, lon in path to dataarray index + unmapped_idx = self.grid_mapper.unmap(first_val, second_val) + subxarray = subxarray.isel(values=unmapped_idx) + value = subxarray.item() + key = subxarray.name + r.result = (key, value) + else: + # if we have no grid map, still need to assign values + subxarray = self.dataarray.sel(path, method="nearest") + value = subxarray.item() + key = subxarray.name + r.result = (key, value) + else: + r.remove_branch() + + def get_mapper(self, axis): + return self.mappers[axis] + + def remap_path(self, path: DatacubePath): + for key in path: + value = path[key] + path[key] = self.mappers[key].remap_val_to_axis_range(value) + return path + + def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): + idx_between = [] + for i in range(len(search_ranges)): + r = search_ranges[i] + offset = search_ranges_offset[i] + low = r[0] + up = r[1] + + if self.grid_mapper is not None: + first_axis = self.grid_mapper._mapped_axes[0] + second_axis = self.grid_mapper._mapped_axes[1] + if axis.name == first_axis: + indexes_between = self.grid_mapper.map_first_axis(low, up) + elif axis.name == second_axis: + indexes_between = self.grid_mapper.map_second_axis(first_val, low, up) + else: + start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? + end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? + indexes_between = indexes[start:end].to_list() + else: + # Find the range of indexes between lower and upper + # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html + # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing + start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? + end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? + indexes_between = indexes[start:end].to_list() + + # Now the indexes_between are values on the cyclic range so need to remap them to their original + # values before returning them + for j in range(len(indexes_between)): + if offset is None: + indexes_between[j] = indexes_between[j] + else: + indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) + + idx_between.append(indexes_between[j]) + return idx_between + + def get_indices(self, path: DatacubePath, axis, lower, upper): + path = self.remap_path(path) + first_val = None + if self.grid_mapper is not None: + first_axis = self.grid_mapper._mapped_axes[0] + first_val = path.get(first_axis, None) + second_axis = self.grid_mapper._mapped_axes[1] + path.pop(first_axis, None) + path.pop(second_axis, None) + + # Open a view on the subset identified by the path + subarray = self.dataarray.sel(path, method="nearest") + + # Get the indexes of the axis we want to query + # XArray does not support branching, so no need to use label, we just take the next axis + if self.grid_mapper is not None: + if axis.name == first_axis: + indexes = [] + elif axis.name == second_axis: + indexes = [] + else: + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + else: + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + + # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube + search_ranges = axis.remap([lower, upper]) + original_search_ranges = axis.to_intervals([lower, upper]) + + # Find the offsets for each interval in the requested range, which we will need later + search_ranges_offset = [] + for r in original_search_ranges: + offset = axis.offset(r) + search_ranges_offset.append(offset) + + # Look up the values in the datacube for each cyclic interval range + idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) + + # Remove duplicates even if difference of the order of the axis tolerance + if offset is not None: + # Note that we can only do unique if not dealing with time values + idx_between = unique(idx_between) + + return idx_between + + def has_index(self, path: DatacubePath, axis, index): + # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube + subarray = self.dataarray.sel(path)[axis.name] + subarray_vals = subarray.values + return index in subarray_vals + + @property + def axes(self): + return self.mappers + + def validate(self, axes): + return validate_axes(self.axes, axes) From eb10f57330381a97c236deb2f87de623006ede3d Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 13 Jul 2023 15:09:18 +0200 Subject: [PATCH 036/332] add FDB_datacube --- polytope/datacube/FDB_datacube.py | 69 ++++++++++++++++--------------- polytope/datacube/mappers.py | 5 +++ 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/polytope/datacube/FDB_datacube.py b/polytope/datacube/FDB_datacube.py index 1183f5f9c..9e6d8798b 100644 --- a/polytope/datacube/FDB_datacube.py +++ b/polytope/datacube/FDB_datacube.py @@ -4,7 +4,6 @@ import numpy as np import pandas as pd -import xarray as xr from ..utility.combinatorics import unique, validate_axes from .datacube import Datacube, DatacubePath, IndexTree @@ -30,6 +29,16 @@ } +def get_datacube_indices(partial_request): + datacube_dico = {"step": [0, 3, 6], + "values": [1, 2, 3, 4]} + return datacube_dico + + +def glue(path): + return {"t" : 0} + + class FDBDatacube(Datacube): def _set_mapper(self, values, name): @@ -71,16 +80,24 @@ def __init__(self, config={}, options={}, grid_options={}): self._set_grid_mapper(name) self.mappers = {} - for name, values in dataarray.coords.variables.items(): - if name in dataarray.dims: - dataarray = dataarray.sortby(name) - self._set_mapper(values, name) - self.axis_counter += 1 + partial_request = config + # Find values in the level 3 FDB datacube + # Will be in the form of a dictionary? {axis_name:values_available, ...} + dataarray = get_datacube_indices(partial_request) + + for name, values in dataarray.items(): + values = values.sort() + self._set_mapper(values, name) + self.axis_counter += 1 if self.grid_mapper is not None: - if name in self.grid_mapper._mapped_axes: - self._set_mapper(values, name) - self.axis_counter += 1 - self.dataarray = dataarray + if name == self.grid_mapper._base_axis: + # Need to remove the base axis and replace with the mapped grid axes + del self.mappers[name] + self.axis_counter -= 1 + for axis_name in self.grid_mapper._mapped_axes: + self._set_mapper(self.grid_mapper._value_type, axis_name) + self.axis_counter += 1 + self.dataarray[name] = values def get(self, requests: IndexTree): for r in requests.leaves: @@ -94,16 +111,17 @@ def get(self, requests: IndexTree): second_val = path[second_axis] path.pop(first_axis, None) path.pop(second_axis, None) - subxarray = self.dataarray.sel(path, method="nearest") # need to remap the lat, lon in path to dataarray index unmapped_idx = self.grid_mapper.unmap(first_val, second_val) - subxarray = subxarray.isel(values=unmapped_idx) + path[self.grid_mapper._base_axis] = unmapped_idx + # Ask FDB what values it has on the path + subxarray = glue(path) value = subxarray.item() key = subxarray.name r.result = (key, value) else: # if we have no grid map, still need to assign values - subxarray = self.dataarray.sel(path, method="nearest") + subxarray = glue(path) value = subxarray.item() key = subxarray.name r.result = (key, value) @@ -135,16 +153,9 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, elif axis.name == second_axis: indexes_between = self.grid_mapper.map_second_axis(first_val, low, up) else: - start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? - end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? - indexes_between = indexes[start:end].to_list() + indexes_between = [i for i in indexes if low <= i <= up] else: - # Find the range of indexes between lower and upper - # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html - # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? - end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? - indexes_between = indexes[start:end].to_list() + indexes_between = [i for i in indexes if low <= i <= up] # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them @@ -166,21 +177,14 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): second_axis = self.grid_mapper._mapped_axes[1] path.pop(first_axis, None) path.pop(second_axis, None) - - # Open a view on the subset identified by the path - subarray = self.dataarray.sel(path, method="nearest") - - # Get the indexes of the axis we want to query - # XArray does not support branching, so no need to use label, we just take the next axis - if self.grid_mapper is not None: if axis.name == first_axis: indexes = [] elif axis.name == second_axis: indexes = [] else: - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + indexes = self.dataarray[axis.name] else: - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + indexes = self.dataarray[axis.name] # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube search_ranges = axis.remap([lower, upper]) @@ -204,8 +208,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - subarray = self.dataarray.sel(path)[axis.name] - subarray_vals = subarray.values + subarray_vals = self.dataarray[axis.name] return index in subarray_vals @property diff --git a/polytope/datacube/mappers.py b/polytope/datacube/mappers.py index 8727556ab..fa8841db9 100644 --- a/polytope/datacube/mappers.py +++ b/polytope/datacube/mappers.py @@ -15,6 +15,10 @@ def _base_axis(self): def _resolution(self): pass + @abstractproperty + def _value_type(self): + pass + @abstractmethod def map_first_axis(self, lower, upper): pass @@ -33,6 +37,7 @@ def __init__(self, base_axis, mapped_axes, resolution): self._mapped_axes = mapped_axes self._base_axis = base_axis self._resolution = resolution + self._value_type = 1.2e0 def gauss_first_guess(self): i = 0 From 6a0bf09e8122251c70362f1f1e9a0ff7e7fdcd0b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 13 Jul 2023 15:55:32 +0200 Subject: [PATCH 037/332] make FDB datacube work with tests glue functions --- polytope/datacube/FDB_datacube.py | 20 +++++++++++--------- polytope/datacube/datacube.py | 2 ++ polytope/datacube/mappers.py | 2 +- tests/test_fdb_datacube.py | 22 ++++++++++++++++++++++ 4 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 tests/test_fdb_datacube.py diff --git a/polytope/datacube/FDB_datacube.py b/polytope/datacube/FDB_datacube.py index 9e6d8798b..b727f4b2d 100644 --- a/polytope/datacube/FDB_datacube.py +++ b/polytope/datacube/FDB_datacube.py @@ -26,11 +26,14 @@ np.str_: UnsliceableaAxis(), str: UnsliceableaAxis(), np.object_: UnsliceableaAxis(), + "int" : IntAxis(), + "float" : FloatAxis(), } def get_datacube_indices(partial_request): datacube_dico = {"step": [0, 3, 6], + "level" : [10, 11, 12], "values": [1, 2, 3, 4]} return datacube_dico @@ -42,21 +45,20 @@ def glue(path): class FDBDatacube(Datacube): def _set_mapper(self, values, name): - if values.dtype.type not in _mappings: - raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") + if type(values[0]).__name__ not in _mappings: + raise ValueError(f"Could not create a mapper for index type {type(values[0]).__name__} for axis {name}") if name in self.options.keys(): # The options argument here is supposed to be a nested dictionary # like {"latitude":{"Cyclic":range}, ...} if "Cyclic" in self.options[name].keys(): - value_type = values.dtype.type - axes_type_str = type(_mappings[value_type]).__name__ + axes_type_str = type(values[0]).__name__ axes_type_str += "Cyclic" cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) self.mappers[name] = cyclic_axis_type self.mappers[name].name = name self.mappers[name].range = self.options[name]["Cyclic"] else: - self.mappers[name] = deepcopy(_mappings[values.dtype.type]) + self.mappers[name] = deepcopy(_mappings[type(values[0]).__name__]) self.mappers[name].name = name def _set_grid_mapper(self, name): @@ -86,7 +88,7 @@ def __init__(self, config={}, options={}, grid_options={}): dataarray = get_datacube_indices(partial_request) for name, values in dataarray.items(): - values = values.sort() + values.sort() self._set_mapper(values, name) self.axis_counter += 1 if self.grid_mapper is not None: @@ -97,7 +99,7 @@ def __init__(self, config={}, options={}, grid_options={}): for axis_name in self.grid_mapper._mapped_axes: self._set_mapper(self.grid_mapper._value_type, axis_name) self.axis_counter += 1 - self.dataarray[name] = values + self.dataarray = dataarray def get(self, requests: IndexTree): for r in requests.leaves: @@ -116,8 +118,8 @@ def get(self, requests: IndexTree): path[self.grid_mapper._base_axis] = unmapped_idx # Ask FDB what values it has on the path subxarray = glue(path) - value = subxarray.item() - key = subxarray.name + key = list(subxarray.keys())[0] + value = subxarray[key] r.result = (key, value) else: # if we have no grid map, still need to assign values diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index c1d83d8a1..9b0182c83 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -47,3 +47,5 @@ def create(datacube, options, grid_options): xadatacube = XArrayDatacube(datacube, options=options, grid_options=grid_options) return xadatacube + else: + return datacube diff --git a/polytope/datacube/mappers.py b/polytope/datacube/mappers.py index fa8841db9..5cfb74bcf 100644 --- a/polytope/datacube/mappers.py +++ b/polytope/datacube/mappers.py @@ -37,7 +37,7 @@ def __init__(self, base_axis, mapped_axes, resolution): self._mapped_axes = mapped_axes self._base_axis = base_axis self._resolution = resolution - self._value_type = 1.2e0 + self._value_type = [1.2e0] def gauss_first_guess(self): i = 0 diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py new file mode 100644 index 000000000..065d7e093 --- /dev/null +++ b/tests/test_fdb_datacube.py @@ -0,0 +1,22 @@ +from polytope.datacube.FDB_datacube import FDBDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box + + +class TestSlicing3DXarrayDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + grid_options = {"values": {"grid_map": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} + self.xarraydatacube = FDBDatacube(config={}, options={}, grid_options=grid_options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.xarraydatacube, engine=self.slicer) + + # Testing different shapes + + def test_2D_box(self): + request = Request(Box(["step", "level"], [3, 10], [6, 11]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2])) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 36 From 68a41ad1120878afcb64dffbaadbe200b999ab38 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 17 Jul 2023 15:46:40 +0200 Subject: [PATCH 038/332] make shorter tests and update requirements --- examples/octahedral_grid_box_example.py | 4 ++-- examples/octahedral_grid_country_example.py | 4 ++-- tests/data/foo.grib | 3 +++ tests/requirements_test.txt | 3 ++- tests/test_octahedral_grid.py | 12 +++++------- 5 files changed, 14 insertions(+), 12 deletions(-) create mode 100644 tests/data/foo.grib diff --git a/examples/octahedral_grid_box_example.py b/examples/octahedral_grid_box_example.py index 61af057ed..20b3d0c2b 100644 --- a/examples/octahedral_grid_box_example.py +++ b/examples/octahedral_grid_box_example.py @@ -36,10 +36,10 @@ def find_nearest_latlon(grib_file, target_lat, target_lon): return nearest_points -ds = data.from_source("file", "./foo.grib") +ds = data.from_source("file", "./tests/data/foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m -nearest_points = find_nearest_latlon("./foo.grib", 0, 0) +nearest_points = find_nearest_latlon("./tests/data/foo.grib", 0, 0) latlon_xarray_datacube = XArrayDatacube(latlon_array) diff --git a/examples/octahedral_grid_country_example.py b/examples/octahedral_grid_country_example.py index b3ddd10ce..f64694baa 100644 --- a/examples/octahedral_grid_country_example.py +++ b/examples/octahedral_grid_country_example.py @@ -37,7 +37,7 @@ def find_nearest_latlon(grib_file, target_lat, target_lon): return nearest_points -ds = data.from_source("file", "./foo.grib") +ds = data.from_source("file", "./tests/data/foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m @@ -85,7 +85,7 @@ def find_nearest_latlon(grib_file, target_lat, target_lon): long = cubepath["longitude"] lats.append(lat) longs.append(long) - nearest_points = find_nearest_latlon("./foo.grib", lat, long) + nearest_points = find_nearest_latlon("./tests/data/foo.grib", lat, long) eccodes_lats.append(nearest_points[0][0]["lat"]) eccodes_longs.append(nearest_points[0][0]["lon"]) t = result.leaves[i].result[1] diff --git a/tests/data/foo.grib b/tests/data/foo.grib new file mode 100644 index 000000000..9c5efa68b --- /dev/null +++ b/tests/data/foo.grib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12533bae9b3aa078a3298a82f289c7dbba6dbdb9a21e4e5e24e84b8695a75dee +size 26409360 diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index e09e60e35..23b52cb64 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -5,4 +5,5 @@ eccodes==1.5.2 h5netcdf==1.1.0 h5py==3.8.0 earthkit==0.0.1 -earthkit-data==0.1.3 \ No newline at end of file +earthkit-data==0.1.3 +eccodes==1.5.2 \ No newline at end of file diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 4da295f3c..390e20a4d 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -9,7 +9,7 @@ class TestOctahedralGrid: def setup_method(self, method): - ds = data.from_source("file", "./foo.grib") + ds = data.from_source("file", "./tests/data/foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m self.xarraydatacube = XArrayDatacube(latlon_array) @@ -41,14 +41,13 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points def test_octahedral_grid(self): - request = Request(Box(["latitude", "longitude"], [0, 0], [0.5, 0.5])) + request = Request(Box(["latitude", "longitude"], [0, 0], [0.2, 0.2])) result = self.API.retrieve(request) - assert len(result.leaves) == 56 + assert len(result.leaves) == 9 lats = [] lons = [] eccodes_lats = [] - eccodes_lons = [] tol = 1e-8 for i in range(len(result.leaves)): cubepath = result.leaves[i].flatten() @@ -56,13 +55,12 @@ def test_octahedral_grid(self): lon = cubepath["longitude"] lats.append(lat) lons.append(lon) - nearest_points = self.find_nearest_latlon("./foo.grib", lat, lon) + nearest_points = self.find_nearest_latlon("./tests/data/foo.grib", lat, lon) eccodes_lat = nearest_points[0][0]["lat"] eccodes_lon = nearest_points[0][0]["lon"] eccodes_lats.append(eccodes_lat) - eccodes_lons.append(eccodes_lon) assert eccodes_lat - tol <= lat assert lat <= eccodes_lat + tol assert eccodes_lon - tol <= lon assert lon <= eccodes_lon + tol - assert len(eccodes_lats) == 56 + assert len(eccodes_lats) == 9 From c875567008decd9671f9b618f3de787e4010d32c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 17 Jul 2023 13:34:05 +0200 Subject: [PATCH 039/332] datacube xarray validate function compares strings of axes names --- polytope/datacube/xarray.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 3e079ba62..393c79064 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -59,6 +59,7 @@ def __init__(self, dataarray: xr.DataArray, options={}): self._set_mapper(values, name) else: # drop non-necessary coordinates which we don't slice on dataarray = dataarray.reset_coords(names=name, drop=True) + self._axes = list(self.mappers.keys()) self.dataarray = dataarray def get(self, requests: IndexTree): @@ -148,4 +149,4 @@ def axes(self): return self.mappers def validate(self, axes): - return validate_axes(self.axes, axes) + return validate_axes(self._axes, axes) From 8192dacad285c01230b62b62c27ca87a1866165b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 17 Jul 2023 16:10:14 +0200 Subject: [PATCH 040/332] make more compact --- polytope/datacube/xarray.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 393c79064..a76f17d4c 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -59,7 +59,6 @@ def __init__(self, dataarray: xr.DataArray, options={}): self._set_mapper(values, name) else: # drop non-necessary coordinates which we don't slice on dataarray = dataarray.reset_coords(names=name, drop=True) - self._axes = list(self.mappers.keys()) self.dataarray = dataarray def get(self, requests: IndexTree): @@ -149,4 +148,4 @@ def axes(self): return self.mappers def validate(self, axes): - return validate_axes(self._axes, axes) + return validate_axes(list(self.axes.keys()), axes) From 785bbb7c642d68428e2121c5b516da3afc62c3c2 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 19 Jul 2023 16:19:52 +0200 Subject: [PATCH 041/332] handle coordinates which aren't dimensions --- polytope/datacube/xarray.py | 20 ++++++++++++++------ tests/test_slicer_era5.py | 2 -- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index b13ac3505..2304e747d 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -53,16 +53,26 @@ def _set_mapper(self, values, name): def __init__(self, dataarray: xr.DataArray, options={}): self.options = options self.mappers = {} + self.pre_path = {} + self.axis_counter = 0 for name, values in dataarray.coords.variables.items(): - dataarray = dataarray.sortby(name) - self._set_mapper(values, name) + if name in dataarray.dims: + # dataarray = dataarray.expand_dims(dim={name: 1}) + dataarray = dataarray.sortby(name) + self._set_mapper(values, name) + self.axis_counter += 1 + else: + if dataarray[name].dims == (): + # Here the coordinate is not a dimension, but it also doesn't depend on any other dimension + # So this coordinate just store a "metadata" value that we extract + self.pre_path[name] = dataarray[name].values self.dataarray = dataarray def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() path = self.remap_path(path) - if len(path.items()) == len(self.dataarray.coords): + if len(path.items()) == self.axis_counter: subxarray = self.dataarray.sel(path, method="nearest") value = subxarray.item() key = subxarray.name @@ -112,9 +122,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): # Get the indexes of the axis we want to query # XArray does not support branching, so no need to use label, we just take the next axis - - assert axis.name == next(iter(subarray.xindexes)) - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + indexes = subarray.indexes[axis.name] # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube search_ranges = axis.remap([lower, upper]) diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 39ad5a94f..09e30b1a2 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -10,8 +10,6 @@ class TestSlicingEra5Data: def setup_method(self, method): ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t - array = array.reset_coords(names="step", drop=True) - array = array.reset_coords(names="valid_time", drop=True) self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) From fcfafff57c0631761109abe1c5a24b0ec85cb055 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 19 Jul 2023 17:08:02 +0200 Subject: [PATCH 042/332] try to make it work so that the non-dimension coordinates can still be requested in the request --- polytope/datacube/xarray.py | 29 ++++++++++++++++++----------- tests/test_slicer_era5.py | 5 ++++- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 2304e747d..a9c2d2463 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -56,16 +56,19 @@ def __init__(self, dataarray: xr.DataArray, options={}): self.pre_path = {} self.axis_counter = 0 for name, values in dataarray.coords.variables.items(): - if name in dataarray.dims: - # dataarray = dataarray.expand_dims(dim={name: 1}) + # dataarray = dataarray.expand_dims(dim={name: 1}) + if dataarray[name].dims == (): + self._set_mapper(values, name) + self.axis_counter += 1 + elif name in dataarray.dims: dataarray = dataarray.sortby(name) self._set_mapper(values, name) self.axis_counter += 1 - else: - if dataarray[name].dims == (): - # Here the coordinate is not a dimension, but it also doesn't depend on any other dimension - # So this coordinate just store a "metadata" value that we extract - self.pre_path[name] = dataarray[name].values + # else: + # if dataarray[name].dims == (): + # # Here the coordinate is not a dimension, but it also doesn't depend on any other dimension + # # So this coordinate just stores a "metadata" value that we extract + # self.pre_path[name] = dataarray[name].values self.dataarray = dataarray def get(self, requests: IndexTree): @@ -99,9 +102,10 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? - end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? - indexes_between = indexes[start:end].to_list() + # start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? + # end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? + # indexes_between = indexes[start:end].to_list() + indexes_between = [i for i in indexes if low <= i <= up] # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them @@ -122,7 +126,10 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): # Get the indexes of the axis we want to query # XArray does not support branching, so no need to use label, we just take the next axis - indexes = subarray.indexes[axis.name] + if axis.name in self.dataarray.dims: + indexes = list(subarray.indexes[axis.name]) + else: + indexes = [subarray[axis.name].values] # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube search_ranges = axis.remap([lower, upper]) diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 09e30b1a2..2847df1f2 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -4,12 +4,14 @@ from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select +import numpy as np class TestSlicingEra5Data: def setup_method(self, method): ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t + print(array) self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) @@ -19,9 +21,10 @@ def test_2D_box(self): Box(["number", "isobaricInhPa"], [3, 0.0], [6, 1000.0]), Select("time", ["2017-01-02T12:00:00"]), Box(["latitude", "longitude"], lower_corner=[10.0, 0.0], upper_corner=[0.0, 30.0]), + Select("step", [np.timedelta64(0, "s")]) ) result = self.API.retrieve(request) - result.pprint() + # result.pprint() assert len(result.leaves) == 4 * 1 * 2 * 4 * 11 From bb5f89c8024d81bf56a6bdb566aac942eabf2372 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 20 Jul 2023 13:33:05 +0200 Subject: [PATCH 043/332] correct tests and examples to include all axes --- examples/3D_shipping_route.py | 11 +++++------ examples/4D_flight_path.py | 7 +++---- examples/country_slicing.py | 15 ++++++++------- examples/cyclic_route_around_earth.py | 14 +++++++------- examples/read_me_example.py | 10 +++++----- examples/slicing_all_ecmwf_countries.py | 21 +++++++++++++-------- examples/timeseries_example.py | 10 +++++----- examples/wind_farms.py | 10 +++++----- polytope/datacube/datacube_axis.py | 10 ++++++++-- polytope/datacube/xarray.py | 17 +++++++---------- tests/test_slicer_era5.py | 5 ++--- 11 files changed, 68 insertions(+), 62 deletions(-) diff --git a/examples/3D_shipping_route.py b/examples/3D_shipping_route.py index bfbad8e82..4fd73a649 100644 --- a/examples/3D_shipping_route.py +++ b/examples/3D_shipping_route.py @@ -8,7 +8,7 @@ from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Ellipsoid, Path +from polytope.shapes import Ellipsoid, Path, Select class Test: @@ -16,10 +16,6 @@ def setup_method(self): ds = data.from_source("file", "./examples/data/winds.grib") array = ds.to_xarray() array = array.isel(time=0).isel(surface=0).isel(number=0).u10 - array = array.reset_coords(names="time", drop=True) - array = array.reset_coords(names="valid_time", drop=True) - array = array.reset_coords(names="number", drop=True) - array = array.reset_coords(names="surface", drop=True) self.array = array self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) @@ -62,7 +58,10 @@ def test_slice_shipping_route(self): # Then somehow make this list of points into just a sequence of points ship_route_polytope = Path(["latitude", "longitude", "step"], initial_shape, *new_points) - request = Request(ship_route_polytope) + request = Request(ship_route_polytope, + Select("number", [0]), + Select("surface", [0]), + Select("time", ["2022-09-30T12:00:00"])) result = self.API.retrieve(request) # Associate the results to the lat/long points in an array diff --git a/examples/4D_flight_path.py b/examples/4D_flight_path.py index c8f6cc14c..b727a04c0 100644 --- a/examples/4D_flight_path.py +++ b/examples/4D_flight_path.py @@ -7,7 +7,7 @@ from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Box, Path +from polytope.shapes import Box, Path, Select class Test: @@ -15,8 +15,6 @@ def setup_method(self): ds = data.from_source("file", "./examples/data/temp_model_levels.grib") array = ds.to_xarray() array = array.isel(time=0).t - array = array.reset_coords(names="time", drop=True) - array = array.reset_coords(names="valid_time", drop=True) options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) for dim in array.dims: @@ -86,7 +84,8 @@ def sphere(size, texture): flight_route_polytope = Path(["latitude", "longitude", "step", "hybrid"], initial_shape, *route_point_CDG_LHR) - request = Request(flight_route_polytope) + request = Request(flight_route_polytope, + Select("time", ["2022-12-02T12:00:00"])) result = self.API.retrieve(request) lats = [] diff --git a/examples/country_slicing.py b/examples/country_slicing.py index 1f50c7f70..d0a7f5590 100644 --- a/examples/country_slicing.py +++ b/examples/country_slicing.py @@ -7,7 +7,7 @@ from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Polygon, Union +from polytope.shapes import Polygon, Select, Union class Test: @@ -15,11 +15,7 @@ def setup_method(self, method): ds = data.from_source("file", "./examples/data/output8.grib") array = ds.to_xarray() array = array.isel(surface=0).isel(step=0).isel(number=0).isel(time=0).t2m - array = array.reset_coords(names="time", drop=True) - array = array.reset_coords(names="valid_time", drop=True) - array = array.reset_coords(names="step", drop=True) - array = array.reset_coords(names="surface", drop=True) - array = array.reset_coords(names="number", drop=True) + print(array) options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() @@ -54,7 +50,12 @@ def test_slice_country(self): request_obj = poly[0] for obj in poly: request_obj = Union(["longitude", "latitude"], request_obj, obj) - request = Request(request_obj) + request = Request(request_obj, + Select("number", [0]), + Select("time", ["2022-02-06T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2022-02-06T12:00:00"])) # Extract the values of the long and lat from the tree result = self.API.retrieve(request) diff --git a/examples/cyclic_route_around_earth.py b/examples/cyclic_route_around_earth.py index 938f014ee..38c1aae2f 100644 --- a/examples/cyclic_route_around_earth.py +++ b/examples/cyclic_route_around_earth.py @@ -6,7 +6,7 @@ from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Box, PathSegment +from polytope.shapes import Box, PathSegment, Select class Test: @@ -14,11 +14,6 @@ def setup_method(self, method): ds = data.from_source("file", "./examples/data/output8.grib") array = ds.to_xarray() array = array.isel(surface=0).isel(step=0).isel(number=0).isel(time=0).t2m - array = array.reset_coords(names="time", drop=True) - array = array.reset_coords(names="valid_time", drop=True) - array = array.reset_coords(names="step", drop=True) - array = array.reset_coords(names="surface", drop=True) - array = array.reset_coords(names="number", drop=True) options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() @@ -27,7 +22,12 @@ def setup_method(self, method): def test_slice_country(self): bounding_box = Box(["latitude", "longitude"], [-0.1, -0.1], [0.1, 0.1]) request_obj = PathSegment(["latitude", "longitude"], bounding_box, [-88, -67], [68, 170]) - request = Request(request_obj) + request = Request(request_obj, + Select("number", [0]), + Select("time", ["2022-02-06T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2022-02-06T12:00:00"])) # Extract the values of the long and lat from the tree result = self.API.retrieve(request) diff --git a/examples/read_me_example.py b/examples/read_me_example.py index 245de226d..46cfd073f 100644 --- a/examples/read_me_example.py +++ b/examples/read_me_example.py @@ -7,10 +7,6 @@ ds = data.from_source("file", "./examples/data/winds.grib") array = ds.to_xarray() array = array.isel(time=0).isel(surface=0).isel(number=0).u10 -array = array.reset_coords(names="time", drop=True) -array = array.reset_coords(names="valid_time", drop=True) -array = array.reset_coords(names="number", drop=True) -array = array.reset_coords(names="surface", drop=True) options = {"longitude": {"Cyclic": [0, 360.0]}} @@ -19,7 +15,11 @@ box = Box(["latitude", "longitude"], [0, 0], [1, 1]) step_point = Select("step", [np.timedelta64(0, "s")]) -request = Request(box, step_point) +request = Request(box, + step_point, + Select("number", [0]), + Select("surface", [0]), + Select("time", ["2022-09-30T12:00:00"])) result = p.retrieve(request) diff --git a/examples/slicing_all_ecmwf_countries.py b/examples/slicing_all_ecmwf_countries.py index 012a7b3bd..047c4002c 100644 --- a/examples/slicing_all_ecmwf_countries.py +++ b/examples/slicing_all_ecmwf_countries.py @@ -7,7 +7,7 @@ from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Polygon, Union +from polytope.shapes import Polygon, Select, Union class Test: @@ -15,11 +15,6 @@ def setup_method(self, method): ds = data.from_source("file", "./examples/data/output8.grib") array = ds.to_xarray() array = array.isel(surface=0).isel(step=0).isel(number=0).isel(time=0).t2m - array = array.reset_coords(names="time", drop=True) - array = array.reset_coords(names="valid_time", drop=True) - array = array.reset_coords(names="step", drop=True) - array = array.reset_coords(names="surface", drop=True) - array = array.reset_coords(names="number", drop=True) options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() @@ -73,7 +68,12 @@ def test_slice_country(self): for obj in poly: request_obj = Union(["longitude", "latitude"], request_obj, obj) - request = Request(request_obj) + request = Request(request_obj, + Select("number", [0]), + Select("time", ["2022-02-06T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2022-02-06T12:00:00"])) # Extract the values of the long and lat from the tree result = self.API.retrieve(request) @@ -119,7 +119,12 @@ def test_slice_country(self): for obj in poly: request_obj = Union(["longitude", "latitude"], request_obj, obj) - request = Request(request_obj) + request = Request(request_obj, + Select("number", [0]), + Select("time", ["2022-02-06T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2022-02-06T12:00:00"])) # Extract the values of the long and lat from the tree result = self.API.retrieve(request) diff --git a/examples/timeseries_example.py b/examples/timeseries_example.py index f73f2a06a..40b58d976 100644 --- a/examples/timeseries_example.py +++ b/examples/timeseries_example.py @@ -14,10 +14,6 @@ def setup_method(self): ds = data.from_source("file", "./examples/data/timeseries_t2m.grib") array = ds.to_xarray() array = array.isel(step=0).isel(surface=0).isel(number=0).t2m - array = array.reset_coords(names="valid_time", drop=True) - array = array.reset_coords(names="step", drop=True) - array = array.reset_coords(names="surface", drop=True) - array = array.reset_coords(names="number", drop=True) self.xarraydatacube = XArrayDatacube(array) for dim in array.dims: array = array.sortby(dim) @@ -54,7 +50,11 @@ def test_slice_shipping_route(self): for obj in poly: request_obj = Union(["longitude", "latitude"], request_obj, obj) - request = Request(request_obj, Select("time", [np.datetime64("2022-05-14T12:00:00")])) + request = Request(request_obj, + Select("time", [np.datetime64("2022-05-14T12:00:00")]), + Select("number", [0]), + Select("step", ["00:00:00"]), + Select("surface", [0])) result = self.API.retrieve(request) diff --git a/examples/wind_farms.py b/examples/wind_farms.py index cd9b36899..c988dbee0 100644 --- a/examples/wind_farms.py +++ b/examples/wind_farms.py @@ -17,10 +17,6 @@ def setup_method(self): ds = data.from_source("file", "./examples/data/winds.grib") array = ds.to_xarray() array = array.isel(time=0).isel(surface=0).isel(number=0).u10 - array = array.reset_coords(names="time", drop=True) - array = array.reset_coords(names="valid_time", drop=True) - array = array.reset_coords(names="number", drop=True) - array = array.reset_coords(names="surface", drop=True) self.array = array options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) @@ -58,7 +54,11 @@ def test_slice_wind_farms(self): request_obj = poly[0] for obj in poly: request_obj = Union(["longitude", "latitude"], request_obj, obj) - request = Request(request_obj, Select("step", [np.timedelta64(0, "ns")])) + request = Request(request_obj, + Select("step", [np.timedelta64(0, "ns")]), + Select("number", [0]), + Select("surface", [0]), + Select("time", ["2022-09-30T12:00:00"])) # Extract the values of the long and lat from the tree result = self.API.retrieve(request) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 7a5680a05..bb228b22a 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -378,7 +378,10 @@ def parse(self, value: Any) -> Any: return pd.Timestamp(value) def to_float(self, value: pd.Timestamp): - return float(value.value / 10**9) + if type(value) == np.datetime64: + return float((value - np.datetime64("1970-01-01T00:00:00")).astype("int")) + else: + return float(value.value / 10**9) def from_float(self, value): return pd.Timestamp(int(value), unit="s") @@ -416,7 +419,10 @@ def parse(self, value: Any) -> Any: return pd.Timedelta(value) def to_float(self, value: pd.Timedelta): - return float(value.value / 10**9) + if type(value) == np.timedelta64: + return value.astype('timedelta64[s]').astype(int) + else: + return float(value.value / 10**9) def from_float(self, value): return pd.Timedelta(int(value), unit="s") diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index a9c2d2463..25d6c1fb7 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -56,25 +56,21 @@ def __init__(self, dataarray: xr.DataArray, options={}): self.pre_path = {} self.axis_counter = 0 for name, values in dataarray.coords.variables.items(): - # dataarray = dataarray.expand_dims(dim={name: 1}) if dataarray[name].dims == (): self._set_mapper(values, name) - self.axis_counter += 1 elif name in dataarray.dims: dataarray = dataarray.sortby(name) self._set_mapper(values, name) self.axis_counter += 1 - # else: - # if dataarray[name].dims == (): - # # Here the coordinate is not a dimension, but it also doesn't depend on any other dimension - # # So this coordinate just stores a "metadata" value that we extract - # self.pre_path[name] = dataarray[name].values self.dataarray = dataarray def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() path = self.remap_path(path) + for key in path.keys(): + if self.dataarray[key].dims == (): + path.pop(key) if len(path.items()) == self.axis_counter: subxarray = self.dataarray.sel(path, method="nearest") value = subxarray.item() @@ -102,9 +98,6 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - # start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? - # end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? - # indexes_between = indexes[start:end].to_list() indexes_between = [i for i in indexes if low <= i <= up] # Now the indexes_between are values on the cyclic range so need to remap them to their original @@ -121,6 +114,10 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): def get_indices(self, path: DatacubePath, axis, lower, upper): path = self.remap_path(path) + for key in path.keys(): + if self.dataarray[key].dims == (): + path.pop(key) + # Open a view on the subset identified by the path subarray = self.dataarray.sel(path, method="nearest") diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 2847df1f2..5c98e5eaf 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -1,17 +1,16 @@ +import numpy as np from earthkit import data from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select -import numpy as np class TestSlicingEra5Data: def setup_method(self, method): ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t - print(array) self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) @@ -25,6 +24,6 @@ def test_2D_box(self): ) result = self.API.retrieve(request) - # result.pprint() + result.pprint() assert len(result.leaves) == 4 * 1 * 2 * 4 * 11 From e4cb5d53b55746d18d881497dc0c47ee6a4874d9 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 20 Jul 2023 13:34:50 +0200 Subject: [PATCH 044/332] black --- examples/3D_shipping_route.py | 7 +++---- examples/4D_flight_path.py | 3 +-- examples/country_slicing.py | 14 +++++++------ examples/cyclic_route_around_earth.py | 14 +++++++------ examples/read_me_example.py | 8 +++---- examples/slicing_all_ecmwf_countries.py | 28 ++++++++++++++----------- examples/timeseries_example.py | 12 ++++++----- examples/wind_farms.py | 12 ++++++----- polytope/datacube/datacube_axis.py | 2 +- tests/test_slicer_era5.py | 2 +- 10 files changed, 55 insertions(+), 47 deletions(-) diff --git a/examples/3D_shipping_route.py b/examples/3D_shipping_route.py index 4fd73a649..57322b513 100644 --- a/examples/3D_shipping_route.py +++ b/examples/3D_shipping_route.py @@ -58,10 +58,9 @@ def test_slice_shipping_route(self): # Then somehow make this list of points into just a sequence of points ship_route_polytope = Path(["latitude", "longitude", "step"], initial_shape, *new_points) - request = Request(ship_route_polytope, - Select("number", [0]), - Select("surface", [0]), - Select("time", ["2022-09-30T12:00:00"])) + request = Request( + ship_route_polytope, Select("number", [0]), Select("surface", [0]), Select("time", ["2022-09-30T12:00:00"]) + ) result = self.API.retrieve(request) # Associate the results to the lat/long points in an array diff --git a/examples/4D_flight_path.py b/examples/4D_flight_path.py index b727a04c0..74931d87f 100644 --- a/examples/4D_flight_path.py +++ b/examples/4D_flight_path.py @@ -84,8 +84,7 @@ def sphere(size, texture): flight_route_polytope = Path(["latitude", "longitude", "step", "hybrid"], initial_shape, *route_point_CDG_LHR) - request = Request(flight_route_polytope, - Select("time", ["2022-12-02T12:00:00"])) + request = Request(flight_route_polytope, Select("time", ["2022-12-02T12:00:00"])) result = self.API.retrieve(request) lats = [] diff --git a/examples/country_slicing.py b/examples/country_slicing.py index d0a7f5590..d0661b9bc 100644 --- a/examples/country_slicing.py +++ b/examples/country_slicing.py @@ -50,12 +50,14 @@ def test_slice_country(self): request_obj = poly[0] for obj in poly: request_obj = Union(["longitude", "latitude"], request_obj, obj) - request = Request(request_obj, - Select("number", [0]), - Select("time", ["2022-02-06T12:00:00"]), - Select("step", ["00:00:00"]), - Select("surface", [0]), - Select("valid_time", ["2022-02-06T12:00:00"])) + request = Request( + request_obj, + Select("number", [0]), + Select("time", ["2022-02-06T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2022-02-06T12:00:00"]), + ) # Extract the values of the long and lat from the tree result = self.API.retrieve(request) diff --git a/examples/cyclic_route_around_earth.py b/examples/cyclic_route_around_earth.py index 38c1aae2f..7a262554a 100644 --- a/examples/cyclic_route_around_earth.py +++ b/examples/cyclic_route_around_earth.py @@ -22,12 +22,14 @@ def setup_method(self, method): def test_slice_country(self): bounding_box = Box(["latitude", "longitude"], [-0.1, -0.1], [0.1, 0.1]) request_obj = PathSegment(["latitude", "longitude"], bounding_box, [-88, -67], [68, 170]) - request = Request(request_obj, - Select("number", [0]), - Select("time", ["2022-02-06T12:00:00"]), - Select("step", ["00:00:00"]), - Select("surface", [0]), - Select("valid_time", ["2022-02-06T12:00:00"])) + request = Request( + request_obj, + Select("number", [0]), + Select("time", ["2022-02-06T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2022-02-06T12:00:00"]), + ) # Extract the values of the long and lat from the tree result = self.API.retrieve(request) diff --git a/examples/read_me_example.py b/examples/read_me_example.py index 46cfd073f..f75ab7b0a 100644 --- a/examples/read_me_example.py +++ b/examples/read_me_example.py @@ -15,11 +15,9 @@ box = Box(["latitude", "longitude"], [0, 0], [1, 1]) step_point = Select("step", [np.timedelta64(0, "s")]) -request = Request(box, - step_point, - Select("number", [0]), - Select("surface", [0]), - Select("time", ["2022-09-30T12:00:00"])) +request = Request( + box, step_point, Select("number", [0]), Select("surface", [0]), Select("time", ["2022-09-30T12:00:00"]) +) result = p.retrieve(request) diff --git a/examples/slicing_all_ecmwf_countries.py b/examples/slicing_all_ecmwf_countries.py index 047c4002c..c10902f7d 100644 --- a/examples/slicing_all_ecmwf_countries.py +++ b/examples/slicing_all_ecmwf_countries.py @@ -68,12 +68,14 @@ def test_slice_country(self): for obj in poly: request_obj = Union(["longitude", "latitude"], request_obj, obj) - request = Request(request_obj, - Select("number", [0]), - Select("time", ["2022-02-06T12:00:00"]), - Select("step", ["00:00:00"]), - Select("surface", [0]), - Select("valid_time", ["2022-02-06T12:00:00"])) + request = Request( + request_obj, + Select("number", [0]), + Select("time", ["2022-02-06T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2022-02-06T12:00:00"]), + ) # Extract the values of the long and lat from the tree result = self.API.retrieve(request) @@ -119,12 +121,14 @@ def test_slice_country(self): for obj in poly: request_obj = Union(["longitude", "latitude"], request_obj, obj) - request = Request(request_obj, - Select("number", [0]), - Select("time", ["2022-02-06T12:00:00"]), - Select("step", ["00:00:00"]), - Select("surface", [0]), - Select("valid_time", ["2022-02-06T12:00:00"])) + request = Request( + request_obj, + Select("number", [0]), + Select("time", ["2022-02-06T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2022-02-06T12:00:00"]), + ) # Extract the values of the long and lat from the tree result = self.API.retrieve(request) diff --git a/examples/timeseries_example.py b/examples/timeseries_example.py index 40b58d976..ba153f8a8 100644 --- a/examples/timeseries_example.py +++ b/examples/timeseries_example.py @@ -50,11 +50,13 @@ def test_slice_shipping_route(self): for obj in poly: request_obj = Union(["longitude", "latitude"], request_obj, obj) - request = Request(request_obj, - Select("time", [np.datetime64("2022-05-14T12:00:00")]), - Select("number", [0]), - Select("step", ["00:00:00"]), - Select("surface", [0])) + request = Request( + request_obj, + Select("time", [np.datetime64("2022-05-14T12:00:00")]), + Select("number", [0]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + ) result = self.API.retrieve(request) diff --git a/examples/wind_farms.py b/examples/wind_farms.py index c988dbee0..1b4c34aa5 100644 --- a/examples/wind_farms.py +++ b/examples/wind_farms.py @@ -54,11 +54,13 @@ def test_slice_wind_farms(self): request_obj = poly[0] for obj in poly: request_obj = Union(["longitude", "latitude"], request_obj, obj) - request = Request(request_obj, - Select("step", [np.timedelta64(0, "ns")]), - Select("number", [0]), - Select("surface", [0]), - Select("time", ["2022-09-30T12:00:00"])) + request = Request( + request_obj, + Select("step", [np.timedelta64(0, "ns")]), + Select("number", [0]), + Select("surface", [0]), + Select("time", ["2022-09-30T12:00:00"]), + ) # Extract the values of the long and lat from the tree result = self.API.retrieve(request) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index bb228b22a..999ae5764 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -420,7 +420,7 @@ def parse(self, value: Any) -> Any: def to_float(self, value: pd.Timedelta): if type(value) == np.timedelta64: - return value.astype('timedelta64[s]').astype(int) + return value.astype("timedelta64[s]").astype(int) else: return float(value.value / 10**9) diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 5c98e5eaf..b4550b1d8 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -20,7 +20,7 @@ def test_2D_box(self): Box(["number", "isobaricInhPa"], [3, 0.0], [6, 1000.0]), Select("time", ["2017-01-02T12:00:00"]), Box(["latitude", "longitude"], lower_corner=[10.0, 0.0], upper_corner=[0.0, 30.0]), - Select("step", [np.timedelta64(0, "s")]) + Select("step", [np.timedelta64(0, "s")]), ) result = self.API.retrieve(request) From 155be091e2827d1e49bd3cacff75f066cea5c4d4 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 20 Jul 2023 14:13:32 +0200 Subject: [PATCH 045/332] rename options --- examples/4D_flight_path.py | 4 ++-- examples/country_slicing.py | 4 ++-- examples/cyclic_route_around_earth.py | 4 ++-- examples/read_me_example.py | 4 ++-- examples/slicing_all_ecmwf_countries.py | 4 ++-- examples/wind_farms.py | 4 ++-- polytope/datacube/datacube.py | 4 ++-- polytope/datacube/xarray.py | 14 +++++++------- polytope/polytope.py | 4 ++-- tests/test_cyclic_axis_over_negative_vals.py | 2 +- tests/test_cyclic_axis_slicer_not_0.py | 2 +- tests/test_cyclic_axis_slicing.py | 2 +- tests/test_datacube_xarray.py | 8 ++++---- tests/test_octahedral_grid.py | 2 +- 14 files changed, 31 insertions(+), 31 deletions(-) diff --git a/examples/4D_flight_path.py b/examples/4D_flight_path.py index da01995fd..6a308af4a 100644 --- a/examples/4D_flight_path.py +++ b/examples/4D_flight_path.py @@ -15,13 +15,13 @@ def setup_method(self): ds = data.from_source("file", "./examples/data/temp_model_levels.grib") array = ds.to_xarray() array = array.isel(time=0).t - options = {"longitude": {"Cyclic": [0, 360.0]}} + axis_options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) for dim in array.dims: array = array.sortby(dim) self.array = array self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer, options=options) + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=axis_options) def test_slice_shipping_route(self): colorscale = [ diff --git a/examples/country_slicing.py b/examples/country_slicing.py index 4248e27a1..5218c36ac 100644 --- a/examples/country_slicing.py +++ b/examples/country_slicing.py @@ -15,10 +15,10 @@ def setup_method(self, method): ds = data.from_source("file", ".examples/data/output8.grib") array = ds.to_xarray() array = array.isel(surface=0).isel(step=0).isel(number=0).isel(time=0).t2m - options = {"longitude": {"Cyclic": [0, 360.0]}} + axis_options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer, options=options) + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=axis_options) def test_slice_country(self): # Read a shapefile for a given country and extract the geometry polygons diff --git a/examples/cyclic_route_around_earth.py b/examples/cyclic_route_around_earth.py index a7e03eea4..4b3977365 100644 --- a/examples/cyclic_route_around_earth.py +++ b/examples/cyclic_route_around_earth.py @@ -14,10 +14,10 @@ def setup_method(self, method): ds = data.from_source("file", ".examples/data/output8.grib") array = ds.to_xarray() array = array.isel(surface=0).isel(step=0).isel(number=0).isel(time=0).t2m - options = {"longitude": {"Cyclic": [0, 360.0]}} + axis_options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer, options=options) + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=axis_options) def test_slice_country(self): bounding_box = Box(["latitude", "longitude"], [-0.1, -0.1], [0.1, 0.1]) diff --git a/examples/read_me_example.py b/examples/read_me_example.py index 2800ef270..f0c615187 100644 --- a/examples/read_me_example.py +++ b/examples/read_me_example.py @@ -8,9 +8,9 @@ array = ds.to_xarray() array = array.isel(time=0).isel(surface=0).isel(number=0).u10 -options = {"longitude": {"Cyclic": [0, 360.0]}} +axis_options = {"longitude": {"Cyclic": [0, 360.0]}} -p = Polytope(datacube=array, options=options) +p = Polytope(datacube=array, axis_options=axis_options) box = Box(["latitude", "longitude"], [0, 0], [1, 1]) step_point = Select("step", [np.timedelta64(0, "s")]) diff --git a/examples/slicing_all_ecmwf_countries.py b/examples/slicing_all_ecmwf_countries.py index 8b9e0b358..8abb62ddf 100644 --- a/examples/slicing_all_ecmwf_countries.py +++ b/examples/slicing_all_ecmwf_countries.py @@ -15,10 +15,10 @@ def setup_method(self, method): ds = data.from_source("file", "./examples/data/output8.grib") array = ds.to_xarray() array = array.isel(surface=0).isel(step=0).isel(number=0).isel(time=0).t2m - options = {"longitude": {"Cyclic": [0, 360.0]}} + axis_options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer, options=options) + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=axis_options) def test_slice_country(self): # Read a shapefile for a given country and extract the geometry polygons diff --git a/examples/wind_farms.py b/examples/wind_farms.py index 596e28bd0..9c1e9a5f1 100644 --- a/examples/wind_farms.py +++ b/examples/wind_farms.py @@ -20,10 +20,10 @@ def setup_method(self): array = ds.to_xarray() array = array.isel(time=0).isel(surface=0).isel(number=0).u10 self.array = array - options = {"longitude": {"Cyclic": [0, 360.0]}} + axis_options = {"longitude": {"Cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer, options=options) + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=axis_options) def test_slice_wind_farms(self): gdal.SetConfigOption("SHAPE_RESTORE_SHX", "YES") diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index c1d83d8a1..79278e94e 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -41,9 +41,9 @@ def validate(self, axes) -> bool: """returns true if the input axes can be resolved against the datacube axes""" @staticmethod - def create(datacube, options, grid_options): + def create(datacube, axis_options, grid_options): if isinstance(datacube, (xr.core.dataarray.DataArray, xr.core.dataset.Dataset)): from .xarray import XArrayDatacube - xadatacube = XArrayDatacube(datacube, options=options, grid_options=grid_options) + xadatacube = XArrayDatacube(datacube, axis_options=axis_options, grid_options=grid_options) return xadatacube diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index f9104c140..80bb7984f 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -36,33 +36,33 @@ class XArrayDatacube(Datacube): def _set_mapper(self, values, name): if values.dtype.type not in _mappings: raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") - if name in self.options.keys(): + if name in self.axis_options.keys(): # The options argument here is supposed to be a nested dictionary # like {"latitude":{"Cyclic":range}, ...} - if "Cyclic" in self.options[name].keys(): + if "Cyclic" in self.axis_options[name].keys(): value_type = values.dtype.type axes_type_str = type(_mappings[value_type]).__name__ axes_type_str += "Cyclic" cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) self.mappers[name] = cyclic_axis_type self.mappers[name].name = name - self.mappers[name].range = self.options[name]["Cyclic"] + self.mappers[name].range = self.axis_options[name]["Cyclic"] else: self.mappers[name] = deepcopy(_mappings[values.dtype.type]) self.mappers[name].name = name def _set_grid_mapper(self, name): if name in self.grid_options.keys(): - if "grid_map" in self.grid_options[name].keys(): - grid_mapping_options = self.grid_options[name]["grid_map"] + if "mapper" in self.grid_options[name].keys(): + grid_mapping_options = self.grid_options[name]["mapper"] grid_type = grid_mapping_options["type"] grid_axes = grid_mapping_options["axes"] if grid_type[0] == "octahedral": resolution = grid_type[1] self.grid_mapper = OctahedralGridMap(name, grid_axes, resolution) - def __init__(self, dataarray: xr.DataArray, options={}, grid_options={}): - self.options = options + def __init__(self, dataarray: xr.DataArray, axis_options={}, grid_options={}): + self.axis_options = axis_options self.grid_options = grid_options self.grid_mapper = None self.axis_counter = 0 diff --git a/polytope/polytope.py b/polytope/polytope.py index 3eac8aab0..7b31bd6e5 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -31,11 +31,11 @@ def polytopes(self): class Polytope: - def __init__(self, datacube, engine=None, options={}, grid_options={}): + def __init__(self, datacube, engine=None, axis_options={}, grid_options={}): from .datacube import Datacube from .engine import Engine - self.datacube = Datacube.create(datacube, options, grid_options) + self.datacube = Datacube.create(datacube, axis_options, grid_options) self.engine = engine if engine is not None else Engine.default() def slice(self, polytopes: List[ConvexPolytope]): diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index 57c4a6780..31a74de54 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -24,7 +24,7 @@ def setup_method(self, method): options = {"long": {"Cyclic": [-1.1, -0.1]}, "level": {"Cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer, options=options) + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) # Testing different shapes diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 6b43d84c4..c66fbaacb 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -24,7 +24,7 @@ def setup_method(self, method): options = {"long": {"Cyclic": [-1.1, -0.1]}, "level": {"Cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer, options=options) + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) # Testing different shapes diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index e5cbfb3af..d6af02e88 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -24,7 +24,7 @@ def setup_method(self, method): options = {"long": {"Cyclic": [0, 1.0]}, "level": {"Cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer, options=options) + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) # Testing different shapes diff --git a/tests/test_datacube_xarray.py b/tests/test_datacube_xarray.py index 81554a26a..c3ba445c7 100644 --- a/tests/test_datacube_xarray.py +++ b/tests/test_datacube_xarray.py @@ -18,8 +18,8 @@ def test_validate(self): array = xr.Dataset(data_vars=dict(param=(["x", "y", "z"], dims)), coords={"x": [1], "y": [1], "z": [1]}) array = array.to_array() - datacube = Datacube.create(array, options={}, grid_options={}) - datacube = Datacube.create(array, options={}, grid_options={}) + datacube = Datacube.create(array, axis_options={}, grid_options={}) + datacube = Datacube.create(array, axis_options={}, grid_options={}) datacube.validate(["x", "y", "z", "variable"]) datacube.validate(["x", "z", "y", "variable"]) @@ -48,8 +48,8 @@ def test_create(self): for d, v in array.coords.variables.items(): print(v.dtype) - datacube = Datacube.create(array, options={}, grid_options={}) - datacube = Datacube.create(array, options={}, grid_options={}) + datacube = Datacube.create(array, axis_options={}, grid_options={}) + datacube = Datacube.create(array, axis_options={}, grid_options={}) # Check the factory created the correct type of datacube assert isinstance(datacube, XArrayDatacube) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 390e20a4d..aebd0df6e 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -13,7 +13,7 @@ def setup_method(self, method): latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m self.xarraydatacube = XArrayDatacube(latlon_array) - grid_options = {"values": {"grid_map": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} + grid_options = {"values": {"mapper": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, grid_options=grid_options) From 362e579a9d89448d3035d88462b6d3618d25fb77 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 21 Jul 2023 12:58:35 +0200 Subject: [PATCH 046/332] add axis_options in one single dictionary and refactor into factory functions --- polytope/datacube/xarray.py | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index e7cf732ba..a55128a19 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -80,6 +80,57 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}, grid_options={}): self.axis_counter += 1 self.dataarray = dataarray + def __init__v2(self, dataarray: xr.DataArray, axis_options={}): + self.axis_options = axis_options + self.grid_mapper = None + self.axis_counter = 0 + for name, values in dataarray.coords.variables.items(): + options = axis_options.get(name, {}) + self.create_axis(options, name, values) + + def create_axis(self, options, name, values): + if options == {}: + self.create_standard(name, values) + if "mapper" in options.keys(): + self.create_mapper(options, name) + if "Cyclic" in options.keys(): + self.create_cyclic(options, name, values) + + def create_cyclic(self, options, name, values): + value_type = values.dtype.type + axes_type_str = type(_mappings[value_type]).__name__ + axes_type_str += "Cyclic" + cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) + self.mappers[name] = cyclic_axis_type + self.mappers[name].name = name + self.mappers[name].range = options["Cyclic"] + + def create_mapper(self, options, name): + grid_mapping_options = options["mapper"] + grid_type = grid_mapping_options["type"] + grid_resolution = grid_mapping_options["resolution"] + grid_axes = grid_mapping_options["axes"] + if grid_type == "octahedral": + self.grid_mapper = OctahedralGridMap(name, grid_axes, grid_resolution) + # Once we have created mapper, create axis for the mapped axes + for i in range(len(grid_axes)): + axis_name = grid_axes[i] + new_axis_options = self.axis_options.get(axis_name, {}) + if i == 1: + values = self.grid_mapper.first_axis_vals() + if i == 2: + values = self.grid_mapper.second_axis_vals(values[0]) # the values[0] will be a value on the first axis + self.create_axis(new_axis_options, axis_name, values) + + def create_standard(self, name, values): + self.check_axis_type(name, values) + self.mappers[name] = deepcopy(_mappings[values.dtype.type]) + self.mappers[name].name = name + + def check_axis_type(self, name, values): + if values.dtype.type not in _mappings: + raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") + def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() From dde7de573b46f78228711055a6ea1100fe39b5f3 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 21 Jul 2023 16:41:35 +0200 Subject: [PATCH 047/332] finish putting all options in one dictionary and making factory function to handle axis initialisation in datacube --- examples/octahedral_grid_box_example.py | 15 +++-- examples/octahedral_grid_country_example.py | 15 +++-- polytope/datacube/datacube.py | 4 +- polytope/datacube/datacube_axis.py | 10 +++- polytope/datacube/xarray.py | 65 +++++++++++++++------ polytope/polytope.py | 4 +- tests/test_octahedral_grid.py | 18 ++++-- 7 files changed, 96 insertions(+), 35 deletions(-) diff --git a/examples/octahedral_grid_box_example.py b/examples/octahedral_grid_box_example.py index 20b3d0c2b..860b66d2d 100644 --- a/examples/octahedral_grid_box_example.py +++ b/examples/octahedral_grid_box_example.py @@ -9,7 +9,7 @@ from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Box +from polytope.shapes import Box, Select def find_nearest_latlon(grib_file, target_lat, target_lon): @@ -45,11 +45,18 @@ def find_nearest_latlon(grib_file, target_lat, target_lon): slicer = HullSlicer() -grid_options = {"values": {"grid_map": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} +grid_options = {"values": {"grid_map": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}} -API = Polytope(datacube=latlon_array, engine=slicer, grid_options=grid_options) +API = Polytope(datacube=latlon_array, engine=slicer, axis_options=grid_options) -request = Request(Box(["latitude", "longitude"], [0, 0], [0.5, 0.5])) +request = Request( + Box(["latitude", "longitude"], [0, 0], [0.5, 0.5]), + Select("number", [0]), + Select("time", ["2023-06-25T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2023-06-25T12:00:00"]), +) result = API.retrieve(request) result.pprint() diff --git a/examples/octahedral_grid_country_example.py b/examples/octahedral_grid_country_example.py index f64694baa..397154983 100644 --- a/examples/octahedral_grid_country_example.py +++ b/examples/octahedral_grid_country_example.py @@ -10,7 +10,7 @@ from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Polygon, Union +from polytope.shapes import Polygon, Select, Union def find_nearest_latlon(grib_file, target_lat, target_lon): @@ -45,9 +45,9 @@ def find_nearest_latlon(grib_file, target_lat, target_lon): slicer = HullSlicer() -grid_options = {"values": {"grid_map": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} +grid_options = {"values": {"grid_map": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}} -API = Polytope(datacube=latlon_array, engine=slicer, grid_options=grid_options) +API = Polytope(datacube=latlon_array, engine=slicer, axis_options=grid_options) shapefile = gpd.read_file("./examples/data/World_Countries__Generalized_.shp") country = shapefile.iloc[13] @@ -71,7 +71,14 @@ def find_nearest_latlon(grib_file, target_lat, target_lon): request_obj = poly[0] for obj in poly: request_obj = Union(["longitude", "latitude"], request_obj, obj) -request = Request(request_obj) +request = Request( + request_obj, + Select("number", [0]), + Select("time", ["2023-06-25T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2023-06-25T12:00:00"]), +) result = API.retrieve(request) lats = [] diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index 79278e94e..996e5f19e 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -41,9 +41,9 @@ def validate(self, axes) -> bool: """returns true if the input axes can be resolved against the datacube axes""" @staticmethod - def create(datacube, axis_options, grid_options): + def create(datacube, axis_options): if isinstance(datacube, (xr.core.dataarray.DataArray, xr.core.dataset.Dataset)): from .xarray import XArrayDatacube - xadatacube = XArrayDatacube(datacube, axis_options=axis_options, grid_options=grid_options) + xadatacube = XArrayDatacube(datacube, axis_options=axis_options) return xadatacube diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 7a5680a05..999ae5764 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -378,7 +378,10 @@ def parse(self, value: Any) -> Any: return pd.Timestamp(value) def to_float(self, value: pd.Timestamp): - return float(value.value / 10**9) + if type(value) == np.datetime64: + return float((value - np.datetime64("1970-01-01T00:00:00")).astype("int")) + else: + return float(value.value / 10**9) def from_float(self, value): return pd.Timestamp(int(value), unit="s") @@ -416,7 +419,10 @@ def parse(self, value: Any) -> Any: return pd.Timedelta(value) def to_float(self, value: pd.Timedelta): - return float(value.value / 10**9) + if type(value) == np.timedelta64: + return value.astype("timedelta64[s]").astype(int) + else: + return float(value.value / 10**9) def from_float(self, value): return pd.Timedelta(int(value), unit="s") diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index a55128a19..f4a2a9530 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -61,7 +61,7 @@ def _set_grid_mapper(self, name): resolution = grid_type[1] self.grid_mapper = OctahedralGridMap(name, grid_axes, resolution) - def __init__(self, dataarray: xr.DataArray, axis_options={}, grid_options={}): + def __init__old(self, dataarray: xr.DataArray, axis_options={}, grid_options={}): self.axis_options = axis_options self.grid_options = grid_options self.grid_mapper = None @@ -80,13 +80,23 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}, grid_options={}): self.axis_counter += 1 self.dataarray = dataarray - def __init__v2(self, dataarray: xr.DataArray, axis_options={}): + def __init__(self, dataarray: xr.DataArray, axis_options={}): self.axis_options = axis_options self.grid_mapper = None self.axis_counter = 0 + self.mappers = {} + self.dataarray = dataarray for name, values in dataarray.coords.variables.items(): + if name in dataarray.dims: + options = axis_options.get(name, {}) + self.create_axis(options, name, values) + else: + if self.dataarray[name].dims == (): + options = axis_options.get(name, {}) + self.create_axis(options, name, values) + for name in dataarray.dims: options = axis_options.get(name, {}) - self.create_axis(options, name, values) + self.create_axis(options, name, np.array(0)) def create_axis(self, options, name, values): if options == {}: @@ -104,6 +114,7 @@ def create_cyclic(self, options, name, values): self.mappers[name] = cyclic_axis_type self.mappers[name].name = name self.mappers[name].range = options["Cyclic"] + self.axis_counter += 1 def create_mapper(self, options, name): grid_mapping_options = options["mapper"] @@ -116,16 +127,19 @@ def create_mapper(self, options, name): for i in range(len(grid_axes)): axis_name = grid_axes[i] new_axis_options = self.axis_options.get(axis_name, {}) + if i == 0: + values = np.array(self.grid_mapper.first_axis_vals()) + self.create_axis(new_axis_options, axis_name, values) if i == 1: - values = self.grid_mapper.first_axis_vals() - if i == 2: - values = self.grid_mapper.second_axis_vals(values[0]) # the values[0] will be a value on the first axis - self.create_axis(new_axis_options, axis_name, values) + # the values[0] will be a value on the first axis + values = np.array(self.grid_mapper.second_axis_vals(values[0])) + self.create_axis(new_axis_options, axis_name, values) def create_standard(self, name, values): self.check_axis_type(name, values) self.mappers[name] = deepcopy(_mappings[values.dtype.type]) self.mappers[name].name = name + self.axis_counter += 1 def check_axis_type(self, name, values): if values.dtype.type not in _mappings: @@ -136,6 +150,9 @@ def get(self, requests: IndexTree): path = r.flatten() path = self.remap_path(path) if len(path.items()) == self.axis_counter: + for key in path.keys(): + if self.dataarray[key].dims == (): + path.pop(key) if self.grid_mapper is not None: first_axis = self.grid_mapper._mapped_axes[0] first_val = path[first_axis] @@ -184,16 +201,18 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, elif axis.name == second_axis: indexes_between = self.grid_mapper.map_second_axis(first_val, low, up) else: - start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? - end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? - indexes_between = indexes[start:end].to_list() + indexes_between = [i for i in indexes if low <= i <= up] + # start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? + # end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? + # indexes_between = indexes[start:end].to_list() else: # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? - end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? - indexes_between = indexes[start:end].to_list() + # start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? + # end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? + # indexes_between = indexes[start:end].to_list() + indexes_between = [i for i in indexes if low <= i <= up] # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them @@ -208,6 +227,10 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, def get_indices(self, path: DatacubePath, axis, lower, upper): path = self.remap_path(path) + for key in path.keys(): + if self.dataarray[key].dims == (): + path.pop(key) + first_val = None if self.grid_mapper is not None: first_axis = self.grid_mapper._mapped_axes[0] @@ -228,11 +251,19 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): elif axis.name == second_axis: indexes = [] else: - assert axis.name == next(iter(subarray.xindexes)) - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + # assert axis.name == next(iter(subarray.xindexes)) + # indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + if axis.name in self.dataarray.dims: + indexes = list(subarray.indexes[axis.name]) + else: + indexes = [subarray[axis.name].values] else: - assert axis.name == next(iter(subarray.xindexes)) - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + if axis.name in self.dataarray.dims: + indexes = list(subarray.indexes[axis.name]) + else: + indexes = [subarray[axis.name].values] + # assert axis.name == next(iter(subarray.xindexes)) + # indexes = next(iter(subarray.xindexes.values())).to_pandas_index() # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube search_ranges = axis.remap([lower, upper]) diff --git a/polytope/polytope.py b/polytope/polytope.py index 7b31bd6e5..ed1c708aa 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -31,11 +31,11 @@ def polytopes(self): class Polytope: - def __init__(self, datacube, engine=None, axis_options={}, grid_options={}): + def __init__(self, datacube, engine=None, axis_options={}): from .datacube import Datacube from .engine import Engine - self.datacube = Datacube.create(datacube, axis_options, grid_options) + self.datacube = Datacube.create(datacube, axis_options) self.engine = engine if engine is not None else Engine.default() def slice(self, polytopes: List[ConvexPolytope]): diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index aebd0df6e..c7092fd64 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -4,7 +4,7 @@ from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Box +from polytope.shapes import Box, Select class TestOctahedralGrid: @@ -12,10 +12,13 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m + print(latlon_array) self.xarraydatacube = XArrayDatacube(latlon_array) - grid_options = {"values": {"mapper": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} + grid_options = { + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} + } self.slicer = HullSlicer() - self.API = Polytope(datacube=latlon_array, engine=self.slicer, grid_options=grid_options) + self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=grid_options) def find_nearest_latlon(self, grib_file, target_lat, target_lon): # Open the GRIB file @@ -41,7 +44,14 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points def test_octahedral_grid(self): - request = Request(Box(["latitude", "longitude"], [0, 0], [0.2, 0.2])) + request = Request( + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + Select("number", [0]), + Select("time", ["2023-06-25T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2023-06-25T12:00:00"]), + ) result = self.API.retrieve(request) assert len(result.leaves) == 9 From f4814e909d1dc170981019d1dd27617c43fe27fd Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 24 Jul 2023 09:51:05 +0200 Subject: [PATCH 048/332] only add dimension axis that's not a coordinate in the init --- polytope/datacube/xarray.py | 56 +++++------------------------------ tests/test_datacube_xarray.py | 8 ++--- tests/test_slicer_era5.py | 2 ++ 3 files changed, 13 insertions(+), 53 deletions(-) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index f4a2a9530..45118b2ca 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -33,70 +33,28 @@ class XArrayDatacube(Datacube): """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0).""" - def _set_mapper(self, values, name): - if values.dtype.type not in _mappings: - raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") - if name in self.axis_options.keys(): - # The options argument here is supposed to be a nested dictionary - # like {"latitude":{"Cyclic":range}, ...} - if "Cyclic" in self.axis_options[name].keys(): - value_type = values.dtype.type - axes_type_str = type(_mappings[value_type]).__name__ - axes_type_str += "Cyclic" - cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) - self.mappers[name] = cyclic_axis_type - self.mappers[name].name = name - self.mappers[name].range = self.axis_options[name]["Cyclic"] - else: - self.mappers[name] = deepcopy(_mappings[values.dtype.type]) - self.mappers[name].name = name - - def _set_grid_mapper(self, name): - if name in self.grid_options.keys(): - if "mapper" in self.grid_options[name].keys(): - grid_mapping_options = self.grid_options[name]["mapper"] - grid_type = grid_mapping_options["type"] - grid_axes = grid_mapping_options["axes"] - if grid_type[0] == "octahedral": - resolution = grid_type[1] - self.grid_mapper = OctahedralGridMap(name, grid_axes, resolution) - - def __init__old(self, dataarray: xr.DataArray, axis_options={}, grid_options={}): - self.axis_options = axis_options - self.grid_options = grid_options - self.grid_mapper = None - self.axis_counter = 0 - for name in dataarray.dims: - self._set_grid_mapper(name) - self.mappers = {} - for name, values in dataarray.coords.variables.items(): - if name in dataarray.dims: - dataarray = dataarray.sortby(name) - self._set_mapper(values, name) - self.axis_counter += 1 - if self.grid_mapper is not None: - if name in self.grid_mapper._mapped_axes: - self._set_mapper(values, name) - self.axis_counter += 1 - self.dataarray = dataarray - def __init__(self, dataarray: xr.DataArray, axis_options={}): self.axis_options = axis_options self.grid_mapper = None self.axis_counter = 0 self.mappers = {} self.dataarray = dataarray + treated_axes = [] for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: options = axis_options.get(name, {}) self.create_axis(options, name, values) + treated_axes.append(name) else: if self.dataarray[name].dims == (): options = axis_options.get(name, {}) self.create_axis(options, name, values) + treated_axes.append(name) for name in dataarray.dims: - options = axis_options.get(name, {}) - self.create_axis(options, name, np.array(0)) + if name not in treated_axes: + options = axis_options.get(name, {}) + val = dataarray[name].values[0] + self.create_axis(options, name, val) def create_axis(self, options, name, values): if options == {}: diff --git a/tests/test_datacube_xarray.py b/tests/test_datacube_xarray.py index 13783a7c7..90d5d8010 100644 --- a/tests/test_datacube_xarray.py +++ b/tests/test_datacube_xarray.py @@ -18,8 +18,8 @@ def test_validate(self): array = xr.Dataset(data_vars=dict(param=(["x", "y", "z"], dims)), coords={"x": [1], "y": [1], "z": [1]}) array = array.to_array() - datacube = Datacube.create(array, axis_options={}, grid_options={}) - datacube = Datacube.create(array, axis_options={}, grid_options={}) + datacube = Datacube.create(array, axis_options={}) + datacube = Datacube.create(array, axis_options={}) datacube.validate(["x", "y", "z", "variable"]) datacube.validate(["x", "z", "y", "variable"]) @@ -48,8 +48,8 @@ def test_create(self): for d, v in array.coords.variables.items(): print(v.dtype) - datacube = Datacube.create(array, axis_options={}, grid_options={}) - datacube = Datacube.create(array, axis_options={}, grid_options={}) + datacube = Datacube.create(array, axis_options={}) + datacube = Datacube.create(array, axis_options={}) # Check the factory created the correct type of datacube assert isinstance(datacube, XArrayDatacube) diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 09e30b1a2..b4550b1d8 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -1,3 +1,4 @@ +import numpy as np from earthkit import data from polytope.datacube.xarray import XArrayDatacube @@ -19,6 +20,7 @@ def test_2D_box(self): Box(["number", "isobaricInhPa"], [3, 0.0], [6, 1000.0]), Select("time", ["2017-01-02T12:00:00"]), Box(["latitude", "longitude"], lower_corner=[10.0, 0.0], upper_corner=[0.0, 30.0]), + Select("step", [np.timedelta64(0, "s")]), ) result = self.API.retrieve(request) From 0174836ba3f4861b5c4a5efd6847863c41dcc7c5 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 24 Jul 2023 11:06:59 +0200 Subject: [PATCH 049/332] make factory function for grid_mappers --- polytope/datacube/xarray.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 45118b2ca..564f64ab8 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -1,6 +1,7 @@ import math import sys from copy import deepcopy +from importlib import import_module import numpy as np import pandas as pd @@ -15,7 +16,6 @@ PandasTimestampAxis, UnsliceableaAxis, ) -from .mappers import OctahedralGridMap _mappings = { pd.Int64Dtype: IntAxis(), @@ -29,6 +29,8 @@ np.object_: UnsliceableaAxis(), } +_grid_mappings = {"octahedral": "OctahedralGridMap"} + class XArrayDatacube(Datacube): """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0).""" @@ -79,8 +81,10 @@ def create_mapper(self, options, name): grid_type = grid_mapping_options["type"] grid_resolution = grid_mapping_options["resolution"] grid_axes = grid_mapping_options["axes"] - if grid_type == "octahedral": - self.grid_mapper = OctahedralGridMap(name, grid_axes, grid_resolution) + map_type = _grid_mappings[grid_type] + module = import_module("polytope.datacube.mappers") + constructor = getattr(module, map_type) + self.grid_mapper = constructor(name, grid_axes, grid_resolution) # Once we have created mapper, create axis for the mapped axes for i in range(len(grid_axes)): axis_name = grid_axes[i] From ebbec5f486ec347cdbf56e85ef720df0ff338294 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 24 Jul 2023 11:22:14 +0200 Subject: [PATCH 050/332] add precomputed values for O1280 --- polytope/datacube/mappers.py | 2613 ++++++++++++++++++++++++++++++++- tests/test_octahedral_grid.py | 1 - 2 files changed, 2590 insertions(+), 24 deletions(-) diff --git a/polytope/datacube/mappers.py b/polytope/datacube/mappers.py index 8727556ab..5bd651d83 100644 --- a/polytope/datacube/mappers.py +++ b/polytope/datacube/mappers.py @@ -98,30 +98,2597 @@ def gauss_first_guess(self): vals.append(vals[i - 1] + math.pi) return vals + def get_precomputed_values_N1280(self): + lats = [0] * 2560 + lats[0] = 89.946187715665616 + lats[1] = 89.876478353332288 + lats[2] = 89.806357319542244 + lats[3] = 89.736143271609578 + lats[4] = 89.6658939412157 + lats[5] = 89.595627537554492 + lats[6] = 89.525351592371393 + lats[7] = 89.45506977912261 + lats[8] = 89.3847841013921 + lats[9] = 89.314495744374256 + lats[10] = 89.24420545380525 + lats[11] = 89.173913722284126 + lats[12] = 89.103620888238879 + lats[13] = 89.033327191845927 + lats[14] = 88.96303280826325 + lats[15] = 88.892737868230952 + lats[16] = 88.822442471310097 + lats[17] = 88.752146694650691 + lats[18] = 88.681850598961759 + lats[19] = 88.611554232668382 + lats[20] = 88.541257634868515 + lats[21] = 88.470960837474877 + lats[22] = 88.40066386679355 + lats[23] = 88.330366744702559 + lats[24] = 88.26006948954614 + lats[25] = 88.189772116820762 + lats[26] = 88.119474639706425 + lats[27] = 88.049177069484486 + lats[28] = 87.978879415867283 + lats[29] = 87.908581687261687 + lats[30] = 87.838283890981543 + lats[31] = 87.767986033419561 + lats[32] = 87.697688120188062 + lats[33] = 87.627390156234085 + lats[34] = 87.557092145935584 + lats[35] = 87.486794093180748 + lats[36] = 87.416496001434894 + lats[37] = 87.346197873795816 + lats[38] = 87.275899713041966 + lats[39] = 87.205601521672108 + lats[40] = 87.135303301939786 + lats[41] = 87.065005055882821 + lats[42] = 86.994706785348129 + lats[43] = 86.924408492014166 + lats[44] = 86.854110177408927 + lats[45] = 86.783811842927179 + lats[46] = 86.713513489844246 + lats[47] = 86.643215119328573 + lats[48] = 86.572916732453024 + lats[49] = 86.502618330203831 + lats[50] = 86.432319913489792 + lats[51] = 86.362021483149363 + lats[52] = 86.291723039957418 + lats[53] = 86.221424584631109 + lats[54] = 86.151126117835304 + lats[55] = 86.080827640187209 + lats[56] = 86.010529152260403 + lats[57] = 85.940230654588888 + lats[58] = 85.869932147670127 + lats[59] = 85.799633631968391 + lats[60] = 85.729335107917464 + lats[61] = 85.659036575922883 + lats[62] = 85.588738036364362 + lats[63] = 85.518439489597966 + lats[64] = 85.448140935957483 + lats[65] = 85.377842375756586 + lats[66] = 85.307543809290152 + lats[67] = 85.237245236835548 + lats[68] = 85.16694665865414 + lats[69] = 85.09664807499216 + lats[70] = 85.026349486081983 + lats[71] = 84.95605089214304 + lats[72] = 84.885752293382765 + lats[73] = 84.81545368999717 + lats[74] = 84.745155082171991 + lats[75] = 84.674856470082915 + lats[76] = 84.604557853896708 + lats[77] = 84.534259233771479 + lats[78] = 84.463960609857125 + lats[79] = 84.393661982296322 + lats[80] = 84.323363351224444 + lats[81] = 84.253064716770425 + lats[82] = 84.18276607905679 + lats[83] = 84.112467438200326 + lats[84] = 84.042168794312317 + lats[85] = 83.971870147498763 + lats[86] = 83.901571497860914 + lats[87] = 83.831272845495249 + lats[88] = 83.760974190494011 + lats[89] = 83.690675532945292 + lats[90] = 83.620376872933264 + lats[91] = 83.550078210538487 + lats[92] = 83.479779545838113 + lats[93] = 83.409480878905782 + lats[94] = 83.339182209812321 + lats[95] = 83.268883538625232 + lats[96] = 83.198584865409657 + lats[97] = 83.128286190227698 + lats[98] = 83.057987513139125 + lats[99] = 82.987688834201322 + lats[100] = 82.917390153469313 + lats[101] = 82.84709147099602 + lats[102] = 82.77679278683226 + lats[103] = 82.706494101026948 + lats[104] = 82.63619541362705 + lats[105] = 82.56589672467787 + lats[106] = 82.495598034222837 + lats[107] = 82.425299342304029 + lats[108] = 82.355000648961692 + lats[109] = 82.284701954234833 + lats[110] = 82.214403258160871 + lats[111] = 82.144104560776 + lats[112] = 82.073805862115165 + lats[113] = 82.003507162211946 + lats[114] = 81.933208461098829 + lats[115] = 81.862909758807191 + lats[116] = 81.792611055367345 + lats[117] = 81.722312350808508 + lats[118] = 81.652013645158945 + lats[119] = 81.581714938445955 + lats[120] = 81.511416230696042 + lats[121] = 81.441117521934686 + lats[122] = 81.370818812186627 + lats[123] = 81.300520101475826 + lats[124] = 81.230221389825374 + lats[125] = 81.159922677257711 + lats[126] = 81.089623963794551 + lats[127] = 81.019325249456955 + lats[128] = 80.949026534265244 + lats[129] = 80.878727818239184 + lats[130] = 80.808429101397948 + lats[131] = 80.73813038376008 + lats[132] = 80.667831665343556 + lats[133] = 80.59753294616587 + lats[134] = 80.527234226243991 + lats[135] = 80.456935505594302 + lats[136] = 80.386636784232863 + lats[137] = 80.316338062175078 + lats[138] = 80.246039339436052 + lats[139] = 80.175740616030438 + lats[140] = 80.105441891972376 + lats[141] = 80.035143167275749 + lats[142] = 79.9648444419539 + lats[143] = 79.894545716019948 + lats[144] = 79.824246989486554 + lats[145] = 79.753948262366038 + lats[146] = 79.683649534670437 + lats[147] = 79.61335080641139 + lats[148] = 79.543052077600308 + lats[149] = 79.472753348248219 + lats[150] = 79.402454618365894 + lats[151] = 79.332155887963822 + lats[152] = 79.261857157052191 + lats[153] = 79.191558425640977 + lats[154] = 79.121259693739859 + lats[155] = 79.050960961358285 + lats[156] = 78.980662228505423 + lats[157] = 78.910363495190211 + lats[158] = 78.840064761421445 + lats[159] = 78.769766027207638 + lats[160] = 78.699467292557102 + lats[161] = 78.629168557477882 + lats[162] = 78.558869821977908 + lats[163] = 78.488571086064923 + lats[164] = 78.418272349746417 + lats[165] = 78.347973613029708 + lats[166] = 78.277674875922045 + lats[167] = 78.207376138430348 + lats[168] = 78.137077400561424 + lats[169] = 78.066778662322022 + lats[170] = 77.996479923718596 + lats[171] = 77.926181184757539 + lats[172] = 77.855882445445019 + lats[173] = 77.785583705787161 + lats[174] = 77.71528496578982 + lats[175] = 77.644986225458879 + lats[176] = 77.574687484799924 + lats[177] = 77.504388743818524 + lats[178] = 77.434090002520122 + lats[179] = 77.363791260909963 + lats[180] = 77.293492518993247 + lats[181] = 77.22319377677502 + lats[182] = 77.15289503426024 + lats[183] = 77.082596291453768 + lats[184] = 77.012297548360323 + lats[185] = 76.941998804984564 + lats[186] = 76.871700061330955 + lats[187] = 76.801401317404 + lats[188] = 76.731102573208048 + lats[189] = 76.660803828747362 + lats[190] = 76.59050508402602 + lats[191] = 76.520206339048215 + lats[192] = 76.449907593817869 + lats[193] = 76.379608848338933 + lats[194] = 76.3093101026152 + lats[195] = 76.239011356650423 + lats[196] = 76.16871261044831 + lats[197] = 76.098413864012443 + lats[198] = 76.028115117346374 + lats[199] = 75.957816370453543 + lats[200] = 75.887517623337317 + lats[201] = 75.81721887600105 + lats[202] = 75.746920128447996 + lats[203] = 75.67662138068134 + lats[204] = 75.60632263270422 + lats[205] = 75.536023884519707 + lats[206] = 75.465725136130786 + lats[207] = 75.395426387540439 + lats[208] = 75.325127638751567 + lats[209] = 75.254828889766983 + lats[210] = 75.184530140589501 + lats[211] = 75.114231391221821 + lats[212] = 75.043932641666672 + lats[213] = 74.973633891926625 + lats[214] = 74.903335142004323 + lats[215] = 74.833036391902269 + lats[216] = 74.762737641622991 + lats[217] = 74.692438891168877 + lats[218] = 74.622140140542356 + lats[219] = 74.551841389745761 + lats[220] = 74.481542638781434 + lats[221] = 74.411243887651622 + lats[222] = 74.340945136358584 + lats[223] = 74.270646384904481 + lats[224] = 74.200347633291472 + lats[225] = 74.13004888152166 + lats[226] = 74.059750129597163 + lats[227] = 73.98945137751997 + lats[228] = 73.919152625292114 + lats[229] = 73.848853872915541 + lats[230] = 73.778555120392184 + lats[231] = 73.70825636772399 + lats[232] = 73.637957614912779 + lats[233] = 73.567658861960396 + lats[234] = 73.497360108868662 + lats[235] = 73.427061355639339 + lats[236] = 73.356762602274188 + lats[237] = 73.2864638487749 + lats[238] = 73.216165095143182 + lats[239] = 73.145866341380668 + lats[240] = 73.075567587489019 + lats[241] = 73.005268833469799 + lats[242] = 72.934970079324657 + lats[243] = 72.864671325055056 + lats[244] = 72.794372570662574 + lats[245] = 72.724073816148703 + lats[246] = 72.653775061514935 + lats[247] = 72.583476306762691 + lats[248] = 72.513177551893421 + lats[249] = 72.442878796908545 + lats[250] = 72.3725800418094 + lats[251] = 72.302281286597392 + lats[252] = 72.231982531273843 + lats[253] = 72.161683775840089 + lats[254] = 72.091385020297409 + lats[255] = 72.02108626464711 + lats[256] = 71.950787508890414 + lats[257] = 71.880488753028587 + lats[258] = 71.810189997062835 + lats[259] = 71.739891240994368 + lats[260] = 71.669592484824364 + lats[261] = 71.599293728553988 + lats[262] = 71.528994972184378 + lats[263] = 71.458696215716685 + lats[264] = 71.388397459152031 + lats[265] = 71.318098702491469 + lats[266] = 71.247799945736105 + lats[267] = 71.177501188887007 + lats[268] = 71.107202431945211 + lats[269] = 71.036903674911756 + lats[270] = 70.966604917787635 + lats[271] = 70.896306160573886 + lats[272] = 70.826007403271475 + lats[273] = 70.755708645881384 + lats[274] = 70.685409888404578 + lats[275] = 70.615111130841967 + lats[276] = 70.544812373194532 + lats[277] = 70.474513615463138 + lats[278] = 70.404214857648739 + lats[279] = 70.333916099752187 + lats[280] = 70.263617341774406 + lats[281] = 70.193318583716191 + lats[282] = 70.123019825578467 + lats[283] = 70.052721067362043 + lats[284] = 69.982422309067744 + lats[285] = 69.912123550696421 + lats[286] = 69.841824792248843 + lats[287] = 69.771526033725834 + lats[288] = 69.701227275128161 + lats[289] = 69.630928516456592 + lats[290] = 69.560629757711908 + lats[291] = 69.490330998894862 + lats[292] = 69.420032240006194 + lats[293] = 69.349733481046613 + lats[294] = 69.279434722016902 + lats[295] = 69.209135962917699 + lats[296] = 69.138837203749759 + lats[297] = 69.068538444513763 + lats[298] = 68.998239685210365 + lats[299] = 68.927940925840304 + lats[300] = 68.85764216640419 + lats[301] = 68.787343406902693 + lats[302] = 68.717044647336493 + lats[303] = 68.646745887706189 + lats[304] = 68.576447128012447 + lats[305] = 68.506148368255865 + lats[306] = 68.435849608437067 + lats[307] = 68.365550848556666 + lats[308] = 68.295252088615257 + lats[309] = 68.224953328613438 + lats[310] = 68.154654568551791 + lats[311] = 68.084355808430871 + lats[312] = 68.014057048251274 + lats[313] = 67.943758288013555 + lats[314] = 67.873459527718282 + lats[315] = 67.803160767365966 + lats[316] = 67.732862006957205 + lats[317] = 67.662563246492482 + lats[318] = 67.592264485972336 + lats[319] = 67.521965725397308 + lats[320] = 67.451666964767895 + lats[321] = 67.381368204084609 + lats[322] = 67.311069443347961 + lats[323] = 67.240770682558434 + lats[324] = 67.170471921716526 + lats[325] = 67.100173160822706 + lats[326] = 67.029874399877471 + lats[327] = 66.95957563888129 + lats[328] = 66.889276877834618 + lats[329] = 66.818978116737924 + lats[330] = 66.748679355591662 + lats[331] = 66.678380594396273 + lats[332] = 66.608081833152212 + lats[333] = 66.537783071859891 + lats[334] = 66.467484310519808 + lats[335] = 66.397185549132331 + lats[336] = 66.326886787697887 + lats[337] = 66.256588026216932 + lats[338] = 66.186289264689833 + lats[339] = 66.115990503117033 + lats[340] = 66.045691741498899 + lats[341] = 65.975392979835888 + lats[342] = 65.905094218128355 + lats[343] = 65.834795456376696 + lats[344] = 65.764496694581283 + lats[345] = 65.694197932742526 + lats[346] = 65.623899170860767 + lats[347] = 65.553600408936404 + lats[348] = 65.483301646969792 + lats[349] = 65.413002884961315 + lats[350] = 65.342704122911286 + lats[351] = 65.272405360820116 + lats[352] = 65.202106598688133 + lats[353] = 65.131807836515677 + lats[354] = 65.061509074303089 + lats[355] = 64.991210312050711 + lats[356] = 64.920911549758912 + lats[357] = 64.850612787427963 + lats[358] = 64.780314025058246 + lats[359] = 64.710015262650074 + lats[360] = 64.639716500203733 + lats[361] = 64.569417737719576 + lats[362] = 64.499118975197902 + lats[363] = 64.428820212639039 + lats[364] = 64.358521450043284 + lats[365] = 64.288222687410922 + lats[366] = 64.21792392474228 + lats[367] = 64.147625162037642 + lats[368] = 64.07732639929732 + lats[369] = 64.00702763652157 + lats[370] = 63.93672887371072 + lats[371] = 63.866430110865004 + lats[372] = 63.796131347984762 + lats[373] = 63.725832585070251 + lats[374] = 63.655533822121711 + lats[375] = 63.585235059139464 + lats[376] = 63.514936296123757 + lats[377] = 63.444637533074854 + lats[378] = 63.374338769993031 + lats[379] = 63.304040006878537 + lats[380] = 63.23374124373165 + lats[381] = 63.163442480552604 + lats[382] = 63.093143717341647 + lats[383] = 63.022844954099064 + lats[384] = 62.952546190825068 + lats[385] = 62.882247427519928 + lats[386] = 62.811948664183866 + lats[387] = 62.741649900817137 + lats[388] = 62.67135113741999 + lats[389] = 62.60105237399263 + lats[390] = 62.530753610535321 + lats[391] = 62.460454847048261 + lats[392] = 62.3901560835317 + lats[393] = 62.319857319985871 + lats[394] = 62.249558556410982 + lats[395] = 62.179259792807258 + lats[396] = 62.108961029174914 + lats[397] = 62.038662265514176 + lats[398] = 61.968363501825259 + lats[399] = 61.898064738108381 + lats[400] = 61.827765974363729 + lats[401] = 61.757467210591535 + lats[402] = 61.687168446791986 + lats[403] = 61.616869682965287 + lats[404] = 61.546570919111666 + lats[405] = 61.476272155231321 + lats[406] = 61.405973391324409 + lats[407] = 61.335674627391185 + lats[408] = 61.265375863431785 + lats[409] = 61.195077099446451 + lats[410] = 61.124778335435344 + lats[411] = 61.054479571398652 + lats[412] = 60.984180807336578 + lats[413] = 60.913882043249295 + lats[414] = 60.843583279137007 + lats[415] = 60.773284514999872 + lats[416] = 60.702985750838074 + lats[417] = 60.632686986651805 + lats[418] = 60.562388222441243 + lats[419] = 60.492089458206543 + lats[420] = 60.421790693947884 + lats[421] = 60.35149192966545 + lats[422] = 60.28119316535939 + lats[423] = 60.21089440102989 + lats[424] = 60.140595636677112 + lats[425] = 60.070296872301235 + lats[426] = 59.999998107902378 + lats[427] = 59.929699343480763 + lats[428] = 59.859400579036503 + lats[429] = 59.78910181456979 + lats[430] = 59.718803050080759 + lats[431] = 59.64850428556958 + lats[432] = 59.578205521036402 + lats[433] = 59.507906756481383 + lats[434] = 59.43760799190467 + lats[435] = 59.3673092273064 + lats[436] = 59.29701046268675 + lats[437] = 59.226711698045854 + lats[438] = 59.156412933383855 + lats[439] = 59.086114168700909 + lats[440] = 59.015815403997145 + lats[441] = 58.945516639272725 + lats[442] = 58.875217874527763 + lats[443] = 58.804919109762423 + lats[444] = 58.73462034497684 + lats[445] = 58.664321580171141 + lats[446] = 58.594022815345468 + lats[447] = 58.523724050499972 + lats[448] = 58.453425285634758 + lats[449] = 58.383126520749968 + lats[450] = 58.312827755845746 + lats[451] = 58.242528990922203 + lats[452] = 58.172230225979497 + lats[453] = 58.101931461017728 + lats[454] = 58.031632696037022 + lats[455] = 57.961333931037537 + lats[456] = 57.891035166019364 + lats[457] = 57.820736400982646 + lats[458] = 57.75043763592749 + lats[459] = 57.680138870854037 + lats[460] = 57.60984010576238 + lats[461] = 57.539541340652676 + lats[462] = 57.469242575525016 + lats[463] = 57.398943810379521 + lats[464] = 57.328645045216312 + lats[465] = 57.258346280035504 + lats[466] = 57.188047514837208 + lats[467] = 57.117748749621541 + lats[468] = 57.047449984388614 + lats[469] = 56.977151219138541 + lats[470] = 56.90685245387143 + lats[471] = 56.836553688587379 + lats[472] = 56.766254923286517 + lats[473] = 56.695956157968951 + lats[474] = 56.625657392634771 + lats[475] = 56.555358627284086 + lats[476] = 56.485059861917016 + lats[477] = 56.41476109653366 + lats[478] = 56.34446233113411 + lats[479] = 56.274163565718467 + lats[480] = 56.203864800286865 + lats[481] = 56.133566034839362 + lats[482] = 56.063267269376091 + lats[483] = 55.992968503897131 + lats[484] = 55.922669738402583 + lats[485] = 55.852370972892551 + lats[486] = 55.782072207367136 + lats[487] = 55.711773441826416 + lats[488] = 55.641474676270505 + lats[489] = 55.571175910699488 + lats[490] = 55.500877145113449 + lats[491] = 55.430578379512511 + lats[492] = 55.360279613896743 + lats[493] = 55.289980848266232 + lats[494] = 55.219682082621084 + lats[495] = 55.149383316961377 + lats[496] = 55.07908455128721 + lats[497] = 55.008785785598668 + lats[498] = 54.938487019895831 + lats[499] = 54.868188254178797 + lats[500] = 54.797889488447652 + lats[501] = 54.727590722702473 + lats[502] = 54.657291956943347 + lats[503] = 54.586993191170357 + lats[504] = 54.516694425383605 + lats[505] = 54.446395659583146 + lats[506] = 54.376096893769081 + lats[507] = 54.305798127941479 + lats[508] = 54.235499362100448 + lats[509] = 54.165200596246031 + lats[510] = 54.094901830378333 + lats[511] = 54.024603064497434 + lats[512] = 53.954304298603383 + lats[513] = 53.884005532696307 + lats[514] = 53.813706766776235 + lats[515] = 53.743408000843282 + lats[516] = 53.673109234897495 + lats[517] = 53.602810468938962 + lats[518] = 53.53251170296776 + lats[519] = 53.462212936983953 + lats[520] = 53.391914170987633 + lats[521] = 53.321615404978871 + lats[522] = 53.251316638957725 + lats[523] = 53.181017872924265 + lats[524] = 53.110719106878584 + lats[525] = 53.040420340820731 + lats[526] = 52.970121574750792 + lats[527] = 52.899822808668837 + lats[528] = 52.829524042574917 + lats[529] = 52.759225276469131 + lats[530] = 52.688926510351514 + lats[531] = 52.618627744222159 + lats[532] = 52.548328978081123 + lats[533] = 52.478030211928477 + lats[534] = 52.407731445764284 + lats[535] = 52.337432679588609 + lats[536] = 52.26713391340153 + lats[537] = 52.196835147203096 + lats[538] = 52.126536380993372 + lats[539] = 52.056237614772435 + lats[540] = 51.985938848540336 + lats[541] = 51.915640082297152 + lats[542] = 51.845341316042933 + lats[543] = 51.775042549777737 + lats[544] = 51.704743783501634 + lats[545] = 51.634445017214695 + lats[546] = 51.56414625091697 + lats[547] = 51.493847484608516 + lats[548] = 51.423548718289396 + lats[549] = 51.353249951959683 + lats[550] = 51.282951185619417 + lats[551] = 51.21265241926865 + lats[552] = 51.14235365290746 + lats[553] = 51.072054886535909 + lats[554] = 51.001756120154049 + lats[555] = 50.931457353761914 + lats[556] = 50.86115858735959 + lats[557] = 50.790859820947119 + lats[558] = 50.720561054524559 + lats[559] = 50.650262288091959 + lats[560] = 50.579963521649397 + lats[561] = 50.509664755196901 + lats[562] = 50.439365988734544 + lats[563] = 50.369067222262359 + lats[564] = 50.298768455780426 + lats[565] = 50.228469689288779 + lats[566] = 50.158170922787484 + lats[567] = 50.087872156276575 + lats[568] = 50.017573389756123 + lats[569] = 49.947274623226157 + lats[570] = 49.876975856686762 + lats[571] = 49.80667709013796 + lats[572] = 49.736378323579807 + lats[573] = 49.66607955701236 + lats[574] = 49.595780790435676 + lats[575] = 49.525482023849783 + lats[576] = 49.455183257254745 + lats[577] = 49.384884490650613 + lats[578] = 49.314585724037435 + lats[579] = 49.244286957415234 + lats[580] = 49.173988190784094 + lats[581] = 49.103689424144044 + lats[582] = 49.03339065749514 + lats[583] = 48.963091890837418 + lats[584] = 48.892793124170929 + lats[585] = 48.822494357495721 + lats[586] = 48.752195590811837 + lats[587] = 48.681896824119335 + lats[588] = 48.611598057418242 + lats[589] = 48.541299290708608 + lats[590] = 48.47100052399049 + lats[591] = 48.400701757263917 + lats[592] = 48.330402990528938 + lats[593] = 48.260104223785596 + lats[594] = 48.189805457033941 + lats[595] = 48.119506690274015 + lats[596] = 48.049207923505868 + lats[597] = 47.978909156729507 + lats[598] = 47.908610389945018 + lats[599] = 47.838311623152421 + lats[600] = 47.76801285635176 + lats[601] = 47.697714089543084 + lats[602] = 47.627415322726435 + lats[603] = 47.557116555901828 + lats[604] = 47.486817789069342 + lats[605] = 47.416519022228997 + lats[606] = 47.346220255380835 + lats[607] = 47.275921488524894 + lats[608] = 47.205622721661214 + lats[609] = 47.13532395478984 + lats[610] = 47.065025187910805 + lats[611] = 46.994726421024154 + lats[612] = 46.924427654129929 + lats[613] = 46.85412888722815 + lats[614] = 46.783830120318882 + lats[615] = 46.713531353402139 + lats[616] = 46.643232586477971 + lats[617] = 46.572933819546414 + lats[618] = 46.502635052607502 + lats[619] = 46.432336285661272 + lats[620] = 46.362037518707766 + lats[621] = 46.291738751747012 + lats[622] = 46.221439984779053 + lats[623] = 46.151141217803925 + lats[624] = 46.080842450821663 + lats[625] = 46.01054368383231 + lats[626] = 45.94024491683588 + lats[627] = 45.869946149832437 + lats[628] = 45.799647382821995 + lats[629] = 45.729348615804589 + lats[630] = 45.659049848780263 + lats[631] = 45.588751081749038 + lats[632] = 45.51845231471097 + lats[633] = 45.448153547666081 + lats[634] = 45.377854780614399 + lats[635] = 45.30755601355596 + lats[636] = 45.237257246490813 + lats[637] = 45.166958479418959 + lats[638] = 45.096659712340461 + lats[639] = 45.026360945255341 + lats[640] = 44.956062178163634 + lats[641] = 44.885763411065362 + lats[642] = 44.81546464396056 + lats[643] = 44.745165876849271 + lats[644] = 44.674867109731515 + lats[645] = 44.604568342607337 + lats[646] = 44.534269575476756 + lats[647] = 44.463970808339802 + lats[648] = 44.39367204119651 + lats[649] = 44.323373274046915 + lats[650] = 44.253074506891046 + lats[651] = 44.182775739728925 + lats[652] = 44.112476972560586 + lats[653] = 44.042178205386072 + lats[654] = 43.971879438205391 + lats[655] = 43.9015806710186 + lats[656] = 43.831281903825705 + lats[657] = 43.760983136626741 + lats[658] = 43.690684369421732 + lats[659] = 43.620385602210717 + lats[660] = 43.550086834993728 + lats[661] = 43.479788067770777 + lats[662] = 43.409489300541907 + lats[663] = 43.339190533307139 + lats[664] = 43.26889176606651 + lats[665] = 43.19859299882004 + lats[666] = 43.128294231567757 + lats[667] = 43.057995464309691 + lats[668] = 42.987696697045862 + lats[669] = 42.917397929776307 + lats[670] = 42.847099162501053 + lats[671] = 42.776800395220121 + lats[672] = 42.706501627933541 + lats[673] = 42.63620286064134 + lats[674] = 42.565904093343548 + lats[675] = 42.495605326040177 + lats[676] = 42.425306558731272 + lats[677] = 42.355007791416853 + lats[678] = 42.284709024096927 + lats[679] = 42.214410256771551 + lats[680] = 42.144111489440725 + lats[681] = 42.073812722104492 + lats[682] = 42.003513954762873 + lats[683] = 41.933215187415882 + lats[684] = 41.862916420063563 + lats[685] = 41.792617652705921 + lats[686] = 41.722318885343 + lats[687] = 41.6520201179748 + lats[688] = 41.581721350601363 + lats[689] = 41.511422583222718 + lats[690] = 41.441123815838885 + lats[691] = 41.370825048449873 + lats[692] = 41.300526281055724 + lats[693] = 41.230227513656445 + lats[694] = 41.159928746252085 + lats[695] = 41.089629978842645 + lats[696] = 41.01933121142816 + lats[697] = 40.949032444008644 + lats[698] = 40.878733676584126 + lats[699] = 40.808434909154634 + lats[700] = 40.738136141720176 + lats[701] = 40.667837374280786 + lats[702] = 40.597538606836487 + lats[703] = 40.527239839387299 + lats[704] = 40.456941071933244 + lats[705] = 40.386642304474343 + lats[706] = 40.316343537010617 + lats[707] = 40.246044769542102 + lats[708] = 40.175746002068806 + lats[709] = 40.105447234590748 + lats[710] = 40.035148467107952 + lats[711] = 39.964849699620437 + lats[712] = 39.894550932128247 + lats[713] = 39.824252164631375 + lats[714] = 39.753953397129855 + lats[715] = 39.683654629623703 + lats[716] = 39.613355862112947 + lats[717] = 39.543057094597607 + lats[718] = 39.472758327077692 + lats[719] = 39.402459559553229 + lats[720] = 39.332160792024254 + lats[721] = 39.261862024490775 + lats[722] = 39.191563256952804 + lats[723] = 39.121264489410365 + lats[724] = 39.050965721863491 + lats[725] = 38.980666954312184 + lats[726] = 38.910368186756479 + lats[727] = 38.840069419196389 + lats[728] = 38.769770651631937 + lats[729] = 38.699471884063136 + lats[730] = 38.629173116490001 + lats[731] = 38.558874348912568 + lats[732] = 38.488575581330842 + lats[733] = 38.418276813744846 + lats[734] = 38.347978046154608 + lats[735] = 38.277679278560143 + lats[736] = 38.20738051096145 + lats[737] = 38.137081743358586 + lats[738] = 38.066782975751536 + lats[739] = 37.99648420814033 + lats[740] = 37.926185440524989 + lats[741] = 37.855886672905527 + lats[742] = 37.785587905281965 + lats[743] = 37.715289137654317 + lats[744] = 37.644990370022605 + lats[745] = 37.574691602386856 + lats[746] = 37.504392834747065 + lats[747] = 37.434094067103274 + lats[748] = 37.363795299455489 + lats[749] = 37.293496531803719 + lats[750] = 37.223197764147997 + lats[751] = 37.152898996488332 + lats[752] = 37.082600228824752 + lats[753] = 37.012301461157264 + lats[754] = 36.942002693485883 + lats[755] = 36.871703925810628 + lats[756] = 36.801405158131523 + lats[757] = 36.731106390448581 + lats[758] = 36.660807622761808 + lats[759] = 36.590508855071242 + lats[760] = 36.520210087376888 + lats[761] = 36.449911319678755 + lats[762] = 36.379612551976876 + lats[763] = 36.309313784271254 + lats[764] = 36.239015016561908 + lats[765] = 36.16871624884886 + lats[766] = 36.098417481132117 + lats[767] = 36.028118713411708 + lats[768] = 35.957819945687639 + lats[769] = 35.887521177959933 + lats[770] = 35.817222410228595 + lats[771] = 35.746923642493655 + lats[772] = 35.676624874755113 + lats[773] = 35.606326107012997 + lats[774] = 35.536027339267314 + lats[775] = 35.465728571518085 + lats[776] = 35.395429803765317 + lats[777] = 35.325131036009047 + lats[778] = 35.254832268249267 + lats[779] = 35.184533500486005 + lats[780] = 35.114234732719261 + lats[781] = 35.043935964949064 + lats[782] = 34.973637197175435 + lats[783] = 34.903338429398374 + lats[784] = 34.833039661617903 + lats[785] = 34.762740893834028 + lats[786] = 34.692442126046771 + lats[787] = 34.622143358256153 + lats[788] = 34.551844590462188 + lats[789] = 34.481545822664863 + lats[790] = 34.411247054864234 + lats[791] = 34.340948287060286 + lats[792] = 34.270649519253041 + lats[793] = 34.200350751442521 + lats[794] = 34.130051983628725 + lats[795] = 34.059753215811682 + lats[796] = 33.989454447991392 + lats[797] = 33.919155680167876 + lats[798] = 33.848856912341155 + lats[799] = 33.778558144511237 + lats[800] = 33.708259376678136 + lats[801] = 33.637960608841851 + lats[802] = 33.567661841002426 + lats[803] = 33.497363073159853 + lats[804] = 33.42706430531414 + lats[805] = 33.356765537465314 + lats[806] = 33.286466769613391 + lats[807] = 33.216168001758369 + lats[808] = 33.145869233900278 + lats[809] = 33.075570466039117 + lats[810] = 33.005271698174909 + lats[811] = 32.934972930307666 + lats[812] = 32.864674162437396 + lats[813] = 32.794375394564113 + lats[814] = 32.724076626687825 + lats[815] = 32.653777858808567 + lats[816] = 32.583479090926325 + lats[817] = 32.513180323041112 + lats[818] = 32.442881555152965 + lats[819] = 32.372582787261891 + lats[820] = 32.302284019367875 + lats[821] = 32.231985251470959 + lats[822] = 32.161686483571145 + lats[823] = 32.091387715668439 + lats[824] = 32.021088947762863 + lats[825] = 31.950790179854422 + lats[826] = 31.880491411943137 + lats[827] = 31.810192644029012 + lats[828] = 31.739893876112063 + lats[829] = 31.669595108192297 + lats[830] = 31.599296340269738 + lats[831] = 31.528997572344384 + lats[832] = 31.458698804416255 + lats[833] = 31.388400036485361 + lats[834] = 31.318101268551715 + lats[835] = 31.247802500615318 + lats[836] = 31.177503732676204 + lats[837] = 31.107204964734358 + lats[838] = 31.036906196789811 + lats[839] = 30.966607428842572 + lats[840] = 30.896308660892647 + lats[841] = 30.826009892940046 + lats[842] = 30.755711124984781 + lats[843] = 30.685412357026873 + lats[844] = 30.615113589066322 + lats[845] = 30.544814821103138 + lats[846] = 30.47451605313735 + lats[847] = 30.404217285168947 + lats[848] = 30.333918517197947 + lats[849] = 30.263619749224372 + lats[850] = 30.19332098124822 + lats[851] = 30.123022213269511 + lats[852] = 30.052723445288244 + lats[853] = 29.98242467730444 + lats[854] = 29.91212590931811 + lats[855] = 29.841827141329258 + lats[856] = 29.771528373337894 + lats[857] = 29.701229605344039 + lats[858] = 29.630930837347698 + lats[859] = 29.560632069348884 + lats[860] = 29.490333301347597 + lats[861] = 29.420034533343859 + lats[862] = 29.349735765337677 + lats[863] = 29.279436997329057 + lats[864] = 29.209138229318015 + lats[865] = 29.138839461304556 + lats[866] = 29.068540693288696 + lats[867] = 28.998241925270449 + lats[868] = 28.927943157249814 + lats[869] = 28.857644389226806 + lats[870] = 28.787345621201432 + lats[871] = 28.717046853173709 + lats[872] = 28.646748085143642 + lats[873] = 28.576449317111244 + lats[874] = 28.506150549076519 + lats[875] = 28.435851781039485 + lats[876] = 28.365553013000145 + lats[877] = 28.29525424495851 + lats[878] = 28.224955476914594 + lats[879] = 28.154656708868405 + lats[880] = 28.084357940819952 + lats[881] = 28.014059172769244 + lats[882] = 27.94376040471629 + lats[883] = 27.873461636661098 + lats[884] = 27.803162868603682 + lats[885] = 27.732864100544052 + lats[886] = 27.662565332482213 + lats[887] = 27.592266564418171 + lats[888] = 27.521967796351948 + lats[889] = 27.451669028283543 + lats[890] = 27.381370260212968 + lats[891] = 27.311071492140236 + lats[892] = 27.240772724065348 + lats[893] = 27.170473955988321 + lats[894] = 27.100175187909159 + lats[895] = 27.029876419827872 + lats[896] = 26.959577651744471 + lats[897] = 26.889278883658971 + lats[898] = 26.818980115571364 + lats[899] = 26.748681347481678 + lats[900] = 26.678382579389908 + lats[901] = 26.608083811296069 + lats[902] = 26.53778504320017 + lats[903] = 26.467486275102218 + lats[904] = 26.397187507002222 + lats[905] = 26.326888738900195 + lats[906] = 26.256589970796135 + lats[907] = 26.186291202690064 + lats[908] = 26.115992434581983 + lats[909] = 26.045693666471902 + lats[910] = 25.975394898359827 + lats[911] = 25.90509613024577 + lats[912] = 25.834797362129745 + lats[913] = 25.764498594011751 + lats[914] = 25.694199825891793 + lats[915] = 25.623901057769892 + lats[916] = 25.553602289646051 + lats[917] = 25.483303521520277 + lats[918] = 25.413004753392578 + lats[919] = 25.342705985262967 + lats[920] = 25.272407217131445 + lats[921] = 25.202108448998025 + lats[922] = 25.13180968086272 + lats[923] = 25.061510912725527 + lats[924] = 24.991212144586456 + lats[925] = 24.920913376445526 + lats[926] = 24.850614608302738 + lats[927] = 24.780315840158096 + lats[928] = 24.710017072011613 + lats[929] = 24.639718303863294 + lats[930] = 24.569419535713152 + lats[931] = 24.499120767561195 + lats[932] = 24.428821999407425 + lats[933] = 24.358523231251851 + lats[934] = 24.288224463094483 + lats[935] = 24.217925694935328 + lats[936] = 24.1476269267744 + lats[937] = 24.077328158611696 + lats[938] = 24.007029390447226 + lats[939] = 23.936730622281004 + lats[940] = 23.866431854113038 + lats[941] = 23.796133085943328 + lats[942] = 23.725834317771888 + lats[943] = 23.655535549598721 + lats[944] = 23.585236781423838 + lats[945] = 23.514938013247242 + lats[946] = 23.444639245068949 + lats[947] = 23.374340476888957 + lats[948] = 23.304041708707278 + lats[949] = 23.233742940523921 + lats[950] = 23.163444172338895 + lats[951] = 23.0931454041522 + lats[952] = 23.022846635963852 + lats[953] = 22.952547867773848 + lats[954] = 22.882249099582204 + lats[955] = 22.811950331388925 + lats[956] = 22.741651563194019 + lats[957] = 22.671352794997489 + lats[958] = 22.60105402679935 + lats[959] = 22.530755258599601 + lats[960] = 22.460456490398254 + lats[961] = 22.390157722195315 + lats[962] = 22.319858953990789 + lats[963] = 22.249560185784691 + lats[964] = 22.179261417577013 + lats[965] = 22.108962649367779 + lats[966] = 22.038663881156989 + lats[967] = 21.968365112944642 + lats[968] = 21.898066344730758 + lats[969] = 21.827767576515338 + lats[970] = 21.757468808298391 + lats[971] = 21.687170040079913 + lats[972] = 21.616871271859928 + lats[973] = 21.546572503638437 + lats[974] = 21.47627373541544 + lats[975] = 21.40597496719095 + lats[976] = 21.335676198964972 + lats[977] = 21.265377430737512 + lats[978] = 21.195078662508585 + lats[979] = 21.124779894278181 + lats[980] = 21.054481126046323 + lats[981] = 20.984182357813012 + lats[982] = 20.913883589578251 + lats[983] = 20.843584821342048 + lats[984] = 20.773286053104417 + lats[985] = 20.702987284865355 + lats[986] = 20.632688516624874 + lats[987] = 20.562389748382977 + lats[988] = 20.492090980139672 + lats[989] = 20.421792211894967 + lats[990] = 20.35149344364887 + lats[991] = 20.28119467540138 + lats[992] = 20.210895907152516 + lats[993] = 20.140597138902272 + lats[994] = 20.070298370650661 + lats[995] = 19.999999602397686 + lats[996] = 19.929700834143357 + lats[997] = 19.859402065887682 + lats[998] = 19.789103297630657 + lats[999] = 19.718804529372303 + lats[1000] = 19.648505761112613 + lats[1001] = 19.578206992851602 + lats[1002] = 19.507908224589269 + lats[1003] = 19.437609456325632 + lats[1004] = 19.367310688060684 + lats[1005] = 19.297011919794439 + lats[1006] = 19.226713151526898 + lats[1007] = 19.15641438325807 + lats[1008] = 19.086115614987968 + lats[1009] = 19.015816846716586 + lats[1010] = 18.945518078443939 + lats[1011] = 18.875219310170031 + lats[1012] = 18.804920541894862 + lats[1013] = 18.734621773618446 + lats[1014] = 18.664323005340787 + lats[1015] = 18.594024237061891 + lats[1016] = 18.523725468781763 + lats[1017] = 18.453426700500408 + lats[1018] = 18.383127932217832 + lats[1019] = 18.312829163934047 + lats[1020] = 18.242530395649048 + lats[1021] = 18.172231627362851 + lats[1022] = 18.101932859075458 + lats[1023] = 18.031634090786874 + lats[1024] = 17.96133532249711 + lats[1025] = 17.89103655420616 + lats[1026] = 17.820737785914044 + lats[1027] = 17.75043901762076 + lats[1028] = 17.680140249326314 + lats[1029] = 17.60984148103071 + lats[1030] = 17.539542712733962 + lats[1031] = 17.469243944436066 + lats[1032] = 17.39894517613704 + lats[1033] = 17.328646407836878 + lats[1034] = 17.258347639535586 + lats[1035] = 17.188048871233182 + lats[1036] = 17.117750102929655 + lats[1037] = 17.04745133462502 + lats[1038] = 16.977152566319283 + lats[1039] = 16.906853798012452 + lats[1040] = 16.836555029704527 + lats[1041] = 16.766256261395515 + lats[1042] = 16.69595749308542 + lats[1043] = 16.625658724774254 + lats[1044] = 16.555359956462013 + lats[1045] = 16.485061188148713 + lats[1046] = 16.41476241983435 + lats[1047] = 16.344463651518936 + lats[1048] = 16.274164883202477 + lats[1049] = 16.203866114884974 + lats[1050] = 16.133567346566434 + lats[1051] = 16.063268578246863 + lats[1052] = 15.992969809926265 + lats[1053] = 15.922671041604652 + lats[1054] = 15.852372273282016 + lats[1055] = 15.78207350495838 + lats[1056] = 15.711774736633735 + lats[1057] = 15.641475968308091 + lats[1058] = 15.571177199981456 + lats[1059] = 15.500878431653829 + lats[1060] = 15.430579663325226 + lats[1061] = 15.360280894995643 + lats[1062] = 15.289982126665089 + lats[1063] = 15.219683358333569 + lats[1064] = 15.149384590001089 + lats[1065] = 15.07908582166765 + lats[1066] = 15.008787053333259 + lats[1067] = 14.938488284997929 + lats[1068] = 14.868189516661655 + lats[1069] = 14.797890748324447 + lats[1070] = 14.727591979986309 + lats[1071] = 14.657293211647247 + lats[1072] = 14.586994443307265 + lats[1073] = 14.516695674966371 + lats[1074] = 14.446396906624567 + lats[1075] = 14.376098138281863 + lats[1076] = 14.305799369938256 + lats[1077] = 14.23550060159376 + lats[1078] = 14.165201833248371 + lats[1079] = 14.0949030649021 + lats[1080] = 14.024604296554955 + lats[1081] = 13.954305528206934 + lats[1082] = 13.884006759858046 + lats[1083] = 13.813707991508297 + lats[1084] = 13.743409223157688 + lats[1085] = 13.673110454806226 + lats[1086] = 13.602811686453919 + lats[1087] = 13.532512918100766 + lats[1088] = 13.46221414974678 + lats[1089] = 13.391915381391959 + lats[1090] = 13.32161661303631 + lats[1091] = 13.251317844679837 + lats[1092] = 13.181019076322551 + lats[1093] = 13.110720307964451 + lats[1094] = 13.040421539605545 + lats[1095] = 12.970122771245832 + lats[1096] = 12.899824002885323 + lats[1097] = 12.829525234524022 + lats[1098] = 12.759226466161934 + lats[1099] = 12.688927697799061 + lats[1100] = 12.618628929435411 + lats[1101] = 12.548330161070988 + lats[1102] = 12.478031392705796 + lats[1103] = 12.407732624339841 + lats[1104] = 12.337433855973126 + lats[1105] = 12.267135087605659 + lats[1106] = 12.196836319237443 + lats[1107] = 12.126537550868482 + lats[1108] = 12.056238782498781 + lats[1109] = 11.985940014128348 + lats[1110] = 11.915641245757183 + lats[1111] = 11.845342477385294 + lats[1112] = 11.775043709012685 + lats[1113] = 11.704744940639358 + lats[1114] = 11.634446172265324 + lats[1115] = 11.564147403890583 + lats[1116] = 11.493848635515141 + lats[1117] = 11.423549867139002 + lats[1118] = 11.35325109876217 + lats[1119] = 11.282952330384653 + lats[1120] = 11.212653562006453 + lats[1121] = 11.142354793627575 + lats[1122] = 11.072056025248026 + lats[1123] = 11.001757256867807 + lats[1124] = 10.931458488486923 + lats[1125] = 10.861159720105382 + lats[1126] = 10.790860951723188 + lats[1127] = 10.720562183340341 + lats[1128] = 10.65026341495685 + lats[1129] = 10.579964646572719 + lats[1130] = 10.509665878187954 + lats[1131] = 10.439367109802557 + lats[1132] = 10.369068341416533 + lats[1133] = 10.298769573029887 + lats[1134] = 10.228470804642624 + lats[1135] = 10.158172036254747 + lats[1136] = 10.087873267866264 + lats[1137] = 10.017574499477174 + lats[1138] = 9.9472757310874869 + lats[1139] = 9.8769769626972046 + lats[1140] = 9.8066781943063344 + lats[1141] = 9.7363794259148779 + lats[1142] = 9.6660806575228388 + lats[1143] = 9.5957818891302242 + lats[1144] = 9.5254831207370376 + lats[1145] = 9.4551843523432826 + lats[1146] = 9.3848855839489662 + lats[1147] = 9.3145868155540921 + lats[1148] = 9.2442880471586619 + lats[1149] = 9.1739892787626829 + lats[1150] = 9.1036905103661585 + lats[1151] = 9.0333917419690941 + lats[1152] = 8.963092973571495 + lats[1153] = 8.8927942051733631 + lats[1154] = 8.8224954367747017 + lats[1155] = 8.7521966683755217 + lats[1156] = 8.6818978999758194 + lats[1157] = 8.6115991315756055 + lats[1158] = 8.5413003631748801 + lats[1159] = 8.4710015947736537 + lats[1160] = 8.4007028263719228 + lats[1161] = 8.3304040579696963 + lats[1162] = 8.2601052895669778 + lats[1163] = 8.1898065211637725 + lats[1164] = 8.1195077527600841 + lats[1165] = 8.049208984355916 + lats[1166] = 7.9789102159512737 + lats[1167] = 7.9086114475461606 + lats[1168] = 7.8383126791405831 + lats[1169] = 7.7680139107345463 + lats[1170] = 7.6977151423280494 + lats[1171] = 7.6274163739210996 + lats[1172] = 7.557117605513703 + lats[1173] = 7.4868188371058624 + lats[1174] = 7.4165200686975803 + lats[1175] = 7.3462213002888648 + lats[1176] = 7.2759225318797176 + lats[1177] = 7.2056237634701441 + lats[1178] = 7.1353249950601469 + lats[1179] = 7.0650262266497315 + lats[1180] = 6.994727458238903 + lats[1181] = 6.924428689827665 + lats[1182] = 6.8541299214160212 + lats[1183] = 6.7838311530039768 + lats[1184] = 6.7135323845915353 + lats[1185] = 6.6432336161787013 + lats[1186] = 6.5729348477654792 + lats[1187] = 6.5026360793518734 + lats[1188] = 6.4323373109378874 + lats[1189] = 6.3620385425235257 + lats[1190] = 6.2917397741087928 + lats[1191] = 6.2214410056936931 + lats[1192] = 6.151142237278231 + lats[1193] = 6.0808434688624091 + lats[1194] = 6.0105447004462347 + lats[1195] = 5.9402459320297085 + lats[1196] = 5.869947163612836 + lats[1197] = 5.7996483951956233 + lats[1198] = 5.729349626778073 + lats[1199] = 5.6590508583601888 + lats[1200] = 5.5887520899419751 + lats[1201] = 5.5184533215234373 + lats[1202] = 5.4481545531045787 + lats[1203] = 5.3778557846854023 + lats[1204] = 5.3075570162659149 + lats[1205] = 5.2372582478461194 + lats[1206] = 5.1669594794260192 + lats[1207] = 5.0966607110056197 + lats[1208] = 5.0263619425849244 + lats[1209] = 4.9560631741639369 + lats[1210] = 4.8857644057426626 + lats[1211] = 4.8154656373211049 + lats[1212] = 4.7451668688992683 + lats[1213] = 4.6748681004771564 + lats[1214] = 4.6045693320547736 + lats[1215] = 4.5342705636321252 + lats[1216] = 4.4639717952092139 + lats[1217] = 4.3936730267860451 + lats[1218] = 4.3233742583626205 + lats[1219] = 4.2530754899389471 + lats[1220] = 4.1827767215150269 + lats[1221] = 4.1124779530908659 + lats[1222] = 4.0421791846664661 + lats[1223] = 3.9718804162418326 + lats[1224] = 3.90158164781697 + lats[1225] = 3.8312828793918823 + lats[1226] = 3.7609841109665734 + lats[1227] = 3.6906853425410477 + lats[1228] = 3.6203865741153085 + lats[1229] = 3.5500878056893601 + lats[1230] = 3.4797890372632065 + lats[1231] = 3.4094902688368531 + lats[1232] = 3.339191500410303 + lats[1233] = 3.2688927319835597 + lats[1234] = 3.1985939635566285 + lats[1235] = 3.1282951951295126 + lats[1236] = 3.0579964267022164 + lats[1237] = 2.9876976582747439 + lats[1238] = 2.9173988898470999 + lats[1239] = 2.8471001214192873 + lats[1240] = 2.7768013529913107 + lats[1241] = 2.7065025845631743 + lats[1242] = 2.6362038161348824 + lats[1243] = 2.5659050477064382 + lats[1244] = 2.4956062792778466 + lats[1245] = 2.4253075108491116 + lats[1246] = 2.3550087424202366 + lats[1247] = 2.2847099739912267 + lats[1248] = 2.2144112055620848 + lats[1249] = 2.1441124371328155 + lats[1250] = 2.0738136687034232 + lats[1251] = 2.0035149002739114 + lats[1252] = 1.9332161318442849 + lats[1253] = 1.8629173634145471 + lats[1254] = 1.792618594984702 + lats[1255] = 1.7223198265547539 + lats[1256] = 1.6520210581247066 + lats[1257] = 1.5817222896945646 + lats[1258] = 1.5114235212643317 + lats[1259] = 1.4411247528340119 + lats[1260] = 1.3708259844036093 + lats[1261] = 1.300527215973128 + lats[1262] = 1.2302284475425722 + lats[1263] = 1.1599296791119456 + lats[1264] = 1.0896309106812523 + lats[1265] = 1.0193321422504964 + lats[1266] = 0.949033373819682 + lats[1267] = 0.87873460538881287 + lats[1268] = 0.80843583695789356 + lats[1269] = 0.73813706852692773 + lats[1270] = 0.66783830009591949 + lats[1271] = 0.59753953166487306 + lats[1272] = 0.52724076323379232 + lats[1273] = 0.45694199480268116 + lats[1274] = 0.3866432263715438 + lats[1275] = 0.31634445794038429 + lats[1276] = 0.24604568950920663 + lats[1277] = 0.17574692107801482 + lats[1278] = 0.10544815264681295 + lats[1279] = 0.035149384215604956 + lats[1280] = -0.035149384215604956 + lats[1281] = -0.10544815264681295 + lats[1282] = -0.17574692107801482 + lats[1283] = -0.24604568950920663 + lats[1284] = -0.31634445794038429 + lats[1285] = -0.3866432263715438 + lats[1286] = -0.45694199480268116 + lats[1287] = -0.52724076323379232 + lats[1288] = -0.59753953166487306 + lats[1289] = -0.66783830009591949 + lats[1290] = -0.73813706852692773 + lats[1291] = -0.80843583695789356 + lats[1292] = -0.87873460538881287 + lats[1293] = -0.949033373819682 + lats[1294] = -1.0193321422504964 + lats[1295] = -1.0896309106812523 + lats[1296] = -1.1599296791119456 + lats[1297] = -1.2302284475425722 + lats[1298] = -1.300527215973128 + lats[1299] = -1.3708259844036093 + lats[1300] = -1.4411247528340119 + lats[1301] = -1.5114235212643317 + lats[1302] = -1.5817222896945646 + lats[1303] = -1.6520210581247066 + lats[1304] = -1.7223198265547539 + lats[1305] = -1.792618594984702 + lats[1306] = -1.8629173634145471 + lats[1307] = -1.9332161318442849 + lats[1308] = -2.0035149002739114 + lats[1309] = -2.0738136687034232 + lats[1310] = -2.1441124371328155 + lats[1311] = -2.2144112055620848 + lats[1312] = -2.2847099739912267 + lats[1313] = -2.3550087424202366 + lats[1314] = -2.4253075108491116 + lats[1315] = -2.4956062792778466 + lats[1316] = -2.5659050477064382 + lats[1317] = -2.6362038161348824 + lats[1318] = -2.7065025845631743 + lats[1319] = -2.7768013529913107 + lats[1320] = -2.8471001214192873 + lats[1321] = -2.9173988898470999 + lats[1322] = -2.9876976582747439 + lats[1323] = -3.0579964267022164 + lats[1324] = -3.1282951951295126 + lats[1325] = -3.1985939635566285 + lats[1326] = -3.2688927319835597 + lats[1327] = -3.339191500410303 + lats[1328] = -3.4094902688368531 + lats[1329] = -3.4797890372632065 + lats[1330] = -3.5500878056893601 + lats[1331] = -3.6203865741153085 + lats[1332] = -3.6906853425410477 + lats[1333] = -3.7609841109665734 + lats[1334] = -3.8312828793918823 + lats[1335] = -3.90158164781697 + lats[1336] = -3.9718804162418326 + lats[1337] = -4.0421791846664661 + lats[1338] = -4.1124779530908659 + lats[1339] = -4.1827767215150269 + lats[1340] = -4.2530754899389471 + lats[1341] = -4.3233742583626205 + lats[1342] = -4.3936730267860451 + lats[1343] = -4.4639717952092139 + lats[1344] = -4.5342705636321252 + lats[1345] = -4.6045693320547736 + lats[1346] = -4.6748681004771564 + lats[1347] = -4.7451668688992683 + lats[1348] = -4.8154656373211049 + lats[1349] = -4.8857644057426626 + lats[1350] = -4.9560631741639369 + lats[1351] = -5.0263619425849244 + lats[1352] = -5.0966607110056197 + lats[1353] = -5.1669594794260192 + lats[1354] = -5.2372582478461194 + lats[1355] = -5.3075570162659149 + lats[1356] = -5.3778557846854023 + lats[1357] = -5.4481545531045787 + lats[1358] = -5.5184533215234373 + lats[1359] = -5.5887520899419751 + lats[1360] = -5.6590508583601888 + lats[1361] = -5.729349626778073 + lats[1362] = -5.7996483951956233 + lats[1363] = -5.869947163612836 + lats[1364] = -5.9402459320297085 + lats[1365] = -6.0105447004462347 + lats[1366] = -6.0808434688624091 + lats[1367] = -6.151142237278231 + lats[1368] = -6.2214410056936931 + lats[1369] = -6.2917397741087928 + lats[1370] = -6.3620385425235257 + lats[1371] = -6.4323373109378874 + lats[1372] = -6.5026360793518734 + lats[1373] = -6.5729348477654792 + lats[1374] = -6.6432336161787013 + lats[1375] = -6.7135323845915353 + lats[1376] = -6.7838311530039768 + lats[1377] = -6.8541299214160212 + lats[1378] = -6.924428689827665 + lats[1379] = -6.994727458238903 + lats[1380] = -7.0650262266497315 + lats[1381] = -7.1353249950601469 + lats[1382] = -7.2056237634701441 + lats[1383] = -7.2759225318797176 + lats[1384] = -7.3462213002888648 + lats[1385] = -7.4165200686975803 + lats[1386] = -7.4868188371058624 + lats[1387] = -7.557117605513703 + lats[1388] = -7.6274163739210996 + lats[1389] = -7.6977151423280494 + lats[1390] = -7.7680139107345463 + lats[1391] = -7.8383126791405831 + lats[1392] = -7.9086114475461606 + lats[1393] = -7.9789102159512737 + lats[1394] = -8.049208984355916 + lats[1395] = -8.1195077527600841 + lats[1396] = -8.1898065211637725 + lats[1397] = -8.2601052895669778 + lats[1398] = -8.3304040579696963 + lats[1399] = -8.4007028263719228 + lats[1400] = -8.4710015947736537 + lats[1401] = -8.5413003631748801 + lats[1402] = -8.6115991315756055 + lats[1403] = -8.6818978999758194 + lats[1404] = -8.7521966683755217 + lats[1405] = -8.8224954367747017 + lats[1406] = -8.8927942051733631 + lats[1407] = -8.963092973571495 + lats[1408] = -9.0333917419690941 + lats[1409] = -9.1036905103661585 + lats[1410] = -9.1739892787626829 + lats[1411] = -9.2442880471586619 + lats[1412] = -9.3145868155540921 + lats[1413] = -9.3848855839489662 + lats[1414] = -9.4551843523432826 + lats[1415] = -9.5254831207370376 + lats[1416] = -9.5957818891302242 + lats[1417] = -9.6660806575228388 + lats[1418] = -9.7363794259148779 + lats[1419] = -9.8066781943063344 + lats[1420] = -9.8769769626972046 + lats[1421] = -9.9472757310874869 + lats[1422] = -10.017574499477174 + lats[1423] = -10.087873267866264 + lats[1424] = -10.158172036254747 + lats[1425] = -10.228470804642624 + lats[1426] = -10.298769573029887 + lats[1427] = -10.369068341416533 + lats[1428] = -10.439367109802557 + lats[1429] = -10.509665878187954 + lats[1430] = -10.579964646572719 + lats[1431] = -10.65026341495685 + lats[1432] = -10.720562183340341 + lats[1433] = -10.790860951723188 + lats[1434] = -10.861159720105382 + lats[1435] = -10.931458488486923 + lats[1436] = -11.001757256867807 + lats[1437] = -11.072056025248026 + lats[1438] = -11.142354793627575 + lats[1439] = -11.212653562006453 + lats[1440] = -11.282952330384653 + lats[1441] = -11.35325109876217 + lats[1442] = -11.423549867139002 + lats[1443] = -11.493848635515141 + lats[1444] = -11.564147403890583 + lats[1445] = -11.634446172265324 + lats[1446] = -11.704744940639358 + lats[1447] = -11.775043709012685 + lats[1448] = -11.845342477385294 + lats[1449] = -11.915641245757183 + lats[1450] = -11.985940014128348 + lats[1451] = -12.056238782498781 + lats[1452] = -12.126537550868482 + lats[1453] = -12.196836319237443 + lats[1454] = -12.267135087605659 + lats[1455] = -12.337433855973126 + lats[1456] = -12.407732624339841 + lats[1457] = -12.478031392705796 + lats[1458] = -12.548330161070988 + lats[1459] = -12.618628929435411 + lats[1460] = -12.688927697799061 + lats[1461] = -12.759226466161934 + lats[1462] = -12.829525234524022 + lats[1463] = -12.899824002885323 + lats[1464] = -12.970122771245832 + lats[1465] = -13.040421539605545 + lats[1466] = -13.110720307964451 + lats[1467] = -13.181019076322551 + lats[1468] = -13.251317844679837 + lats[1469] = -13.32161661303631 + lats[1470] = -13.391915381391959 + lats[1471] = -13.46221414974678 + lats[1472] = -13.532512918100766 + lats[1473] = -13.602811686453919 + lats[1474] = -13.673110454806226 + lats[1475] = -13.743409223157688 + lats[1476] = -13.813707991508297 + lats[1477] = -13.884006759858046 + lats[1478] = -13.954305528206934 + lats[1479] = -14.024604296554955 + lats[1480] = -14.0949030649021 + lats[1481] = -14.165201833248371 + lats[1482] = -14.23550060159376 + lats[1483] = -14.305799369938256 + lats[1484] = -14.376098138281863 + lats[1485] = -14.446396906624567 + lats[1486] = -14.516695674966371 + lats[1487] = -14.586994443307265 + lats[1488] = -14.657293211647247 + lats[1489] = -14.727591979986309 + lats[1490] = -14.797890748324447 + lats[1491] = -14.868189516661655 + lats[1492] = -14.938488284997929 + lats[1493] = -15.008787053333259 + lats[1494] = -15.07908582166765 + lats[1495] = -15.149384590001089 + lats[1496] = -15.219683358333569 + lats[1497] = -15.289982126665089 + lats[1498] = -15.360280894995643 + lats[1499] = -15.430579663325226 + lats[1500] = -15.500878431653829 + lats[1501] = -15.571177199981456 + lats[1502] = -15.641475968308091 + lats[1503] = -15.711774736633735 + lats[1504] = -15.78207350495838 + lats[1505] = -15.852372273282016 + lats[1506] = -15.922671041604652 + lats[1507] = -15.992969809926265 + lats[1508] = -16.063268578246863 + lats[1509] = -16.133567346566434 + lats[1510] = -16.203866114884974 + lats[1511] = -16.274164883202477 + lats[1512] = -16.344463651518936 + lats[1513] = -16.41476241983435 + lats[1514] = -16.485061188148713 + lats[1515] = -16.555359956462013 + lats[1516] = -16.625658724774254 + lats[1517] = -16.69595749308542 + lats[1518] = -16.766256261395515 + lats[1519] = -16.836555029704527 + lats[1520] = -16.906853798012452 + lats[1521] = -16.977152566319283 + lats[1522] = -17.04745133462502 + lats[1523] = -17.117750102929655 + lats[1524] = -17.188048871233182 + lats[1525] = -17.258347639535586 + lats[1526] = -17.328646407836878 + lats[1527] = -17.39894517613704 + lats[1528] = -17.469243944436066 + lats[1529] = -17.539542712733962 + lats[1530] = -17.60984148103071 + lats[1531] = -17.680140249326314 + lats[1532] = -17.75043901762076 + lats[1533] = -17.820737785914044 + lats[1534] = -17.89103655420616 + lats[1535] = -17.96133532249711 + lats[1536] = -18.031634090786874 + lats[1537] = -18.101932859075458 + lats[1538] = -18.172231627362851 + lats[1539] = -18.242530395649048 + lats[1540] = -18.312829163934047 + lats[1541] = -18.383127932217832 + lats[1542] = -18.453426700500408 + lats[1543] = -18.523725468781763 + lats[1544] = -18.594024237061891 + lats[1545] = -18.664323005340787 + lats[1546] = -18.734621773618446 + lats[1547] = -18.804920541894862 + lats[1548] = -18.875219310170031 + lats[1549] = -18.945518078443939 + lats[1550] = -19.015816846716586 + lats[1551] = -19.086115614987968 + lats[1552] = -19.15641438325807 + lats[1553] = -19.226713151526898 + lats[1554] = -19.297011919794439 + lats[1555] = -19.367310688060684 + lats[1556] = -19.437609456325632 + lats[1557] = -19.507908224589269 + lats[1558] = -19.578206992851602 + lats[1559] = -19.648505761112613 + lats[1560] = -19.718804529372303 + lats[1561] = -19.789103297630657 + lats[1562] = -19.859402065887682 + lats[1563] = -19.929700834143357 + lats[1564] = -19.999999602397686 + lats[1565] = -20.070298370650661 + lats[1566] = -20.140597138902272 + lats[1567] = -20.210895907152516 + lats[1568] = -20.28119467540138 + lats[1569] = -20.35149344364887 + lats[1570] = -20.421792211894967 + lats[1571] = -20.492090980139672 + lats[1572] = -20.562389748382977 + lats[1573] = -20.632688516624874 + lats[1574] = -20.702987284865355 + lats[1575] = -20.773286053104417 + lats[1576] = -20.843584821342048 + lats[1577] = -20.913883589578251 + lats[1578] = -20.984182357813012 + lats[1579] = -21.054481126046323 + lats[1580] = -21.124779894278181 + lats[1581] = -21.195078662508585 + lats[1582] = -21.265377430737512 + lats[1583] = -21.335676198964972 + lats[1584] = -21.40597496719095 + lats[1585] = -21.47627373541544 + lats[1586] = -21.546572503638437 + lats[1587] = -21.616871271859928 + lats[1588] = -21.687170040079913 + lats[1589] = -21.757468808298391 + lats[1590] = -21.827767576515338 + lats[1591] = -21.898066344730758 + lats[1592] = -21.968365112944642 + lats[1593] = -22.038663881156989 + lats[1594] = -22.108962649367779 + lats[1595] = -22.179261417577013 + lats[1596] = -22.249560185784691 + lats[1597] = -22.319858953990789 + lats[1598] = -22.390157722195315 + lats[1599] = -22.460456490398254 + lats[1600] = -22.530755258599601 + lats[1601] = -22.60105402679935 + lats[1602] = -22.671352794997489 + lats[1603] = -22.741651563194019 + lats[1604] = -22.811950331388925 + lats[1605] = -22.882249099582204 + lats[1606] = -22.952547867773848 + lats[1607] = -23.022846635963852 + lats[1608] = -23.0931454041522 + lats[1609] = -23.163444172338895 + lats[1610] = -23.233742940523921 + lats[1611] = -23.304041708707278 + lats[1612] = -23.374340476888957 + lats[1613] = -23.444639245068949 + lats[1614] = -23.514938013247242 + lats[1615] = -23.585236781423838 + lats[1616] = -23.655535549598721 + lats[1617] = -23.725834317771888 + lats[1618] = -23.796133085943328 + lats[1619] = -23.866431854113038 + lats[1620] = -23.936730622281004 + lats[1621] = -24.007029390447226 + lats[1622] = -24.077328158611696 + lats[1623] = -24.1476269267744 + lats[1624] = -24.217925694935328 + lats[1625] = -24.288224463094483 + lats[1626] = -24.358523231251851 + lats[1627] = -24.428821999407425 + lats[1628] = -24.499120767561195 + lats[1629] = -24.569419535713152 + lats[1630] = -24.639718303863294 + lats[1631] = -24.710017072011613 + lats[1632] = -24.780315840158096 + lats[1633] = -24.850614608302738 + lats[1634] = -24.920913376445526 + lats[1635] = -24.991212144586456 + lats[1636] = -25.061510912725527 + lats[1637] = -25.13180968086272 + lats[1638] = -25.202108448998025 + lats[1639] = -25.272407217131445 + lats[1640] = -25.342705985262967 + lats[1641] = -25.413004753392578 + lats[1642] = -25.483303521520277 + lats[1643] = -25.553602289646051 + lats[1644] = -25.623901057769892 + lats[1645] = -25.694199825891793 + lats[1646] = -25.764498594011751 + lats[1647] = -25.834797362129745 + lats[1648] = -25.90509613024577 + lats[1649] = -25.975394898359827 + lats[1650] = -26.045693666471902 + lats[1651] = -26.115992434581983 + lats[1652] = -26.186291202690064 + lats[1653] = -26.256589970796135 + lats[1654] = -26.326888738900195 + lats[1655] = -26.397187507002222 + lats[1656] = -26.467486275102218 + lats[1657] = -26.53778504320017 + lats[1658] = -26.608083811296069 + lats[1659] = -26.678382579389908 + lats[1660] = -26.748681347481678 + lats[1661] = -26.818980115571364 + lats[1662] = -26.889278883658971 + lats[1663] = -26.959577651744471 + lats[1664] = -27.029876419827872 + lats[1665] = -27.100175187909159 + lats[1666] = -27.170473955988321 + lats[1667] = -27.240772724065348 + lats[1668] = -27.311071492140236 + lats[1669] = -27.381370260212968 + lats[1670] = -27.451669028283543 + lats[1671] = -27.521967796351948 + lats[1672] = -27.592266564418171 + lats[1673] = -27.662565332482213 + lats[1674] = -27.732864100544052 + lats[1675] = -27.803162868603682 + lats[1676] = -27.873461636661098 + lats[1677] = -27.94376040471629 + lats[1678] = -28.014059172769244 + lats[1679] = -28.084357940819952 + lats[1680] = -28.154656708868405 + lats[1681] = -28.224955476914594 + lats[1682] = -28.29525424495851 + lats[1683] = -28.365553013000145 + lats[1684] = -28.435851781039485 + lats[1685] = -28.506150549076519 + lats[1686] = -28.576449317111244 + lats[1687] = -28.646748085143642 + lats[1688] = -28.717046853173709 + lats[1689] = -28.787345621201432 + lats[1690] = -28.857644389226806 + lats[1691] = -28.927943157249814 + lats[1692] = -28.998241925270449 + lats[1693] = -29.068540693288696 + lats[1694] = -29.138839461304556 + lats[1695] = -29.209138229318015 + lats[1696] = -29.279436997329057 + lats[1697] = -29.349735765337677 + lats[1698] = -29.420034533343859 + lats[1699] = -29.490333301347597 + lats[1700] = -29.560632069348884 + lats[1701] = -29.630930837347698 + lats[1702] = -29.701229605344039 + lats[1703] = -29.771528373337894 + lats[1704] = -29.841827141329258 + lats[1705] = -29.91212590931811 + lats[1706] = -29.98242467730444 + lats[1707] = -30.052723445288244 + lats[1708] = -30.123022213269511 + lats[1709] = -30.19332098124822 + lats[1710] = -30.263619749224372 + lats[1711] = -30.333918517197947 + lats[1712] = -30.404217285168947 + lats[1713] = -30.47451605313735 + lats[1714] = -30.544814821103138 + lats[1715] = -30.615113589066322 + lats[1716] = -30.685412357026873 + lats[1717] = -30.755711124984781 + lats[1718] = -30.826009892940046 + lats[1719] = -30.896308660892647 + lats[1720] = -30.966607428842572 + lats[1721] = -31.036906196789811 + lats[1722] = -31.107204964734358 + lats[1723] = -31.177503732676204 + lats[1724] = -31.247802500615318 + lats[1725] = -31.318101268551715 + lats[1726] = -31.388400036485361 + lats[1727] = -31.458698804416255 + lats[1728] = -31.528997572344384 + lats[1729] = -31.599296340269738 + lats[1730] = -31.669595108192297 + lats[1731] = -31.739893876112063 + lats[1732] = -31.810192644029012 + lats[1733] = -31.880491411943137 + lats[1734] = -31.950790179854422 + lats[1735] = -32.021088947762863 + lats[1736] = -32.091387715668439 + lats[1737] = -32.161686483571145 + lats[1738] = -32.231985251470959 + lats[1739] = -32.302284019367875 + lats[1740] = -32.372582787261891 + lats[1741] = -32.442881555152965 + lats[1742] = -32.513180323041112 + lats[1743] = -32.583479090926325 + lats[1744] = -32.653777858808567 + lats[1745] = -32.724076626687825 + lats[1746] = -32.794375394564113 + lats[1747] = -32.864674162437396 + lats[1748] = -32.934972930307666 + lats[1749] = -33.005271698174909 + lats[1750] = -33.075570466039117 + lats[1751] = -33.145869233900278 + lats[1752] = -33.216168001758369 + lats[1753] = -33.286466769613391 + lats[1754] = -33.356765537465314 + lats[1755] = -33.42706430531414 + lats[1756] = -33.497363073159853 + lats[1757] = -33.567661841002426 + lats[1758] = -33.637960608841851 + lats[1759] = -33.708259376678136 + lats[1760] = -33.778558144511237 + lats[1761] = -33.848856912341155 + lats[1762] = -33.919155680167876 + lats[1763] = -33.989454447991392 + lats[1764] = -34.059753215811682 + lats[1765] = -34.130051983628725 + lats[1766] = -34.200350751442521 + lats[1767] = -34.270649519253041 + lats[1768] = -34.340948287060286 + lats[1769] = -34.411247054864234 + lats[1770] = -34.481545822664863 + lats[1771] = -34.551844590462188 + lats[1772] = -34.622143358256153 + lats[1773] = -34.692442126046771 + lats[1774] = -34.762740893834028 + lats[1775] = -34.833039661617903 + lats[1776] = -34.903338429398374 + lats[1777] = -34.973637197175435 + lats[1778] = -35.043935964949064 + lats[1779] = -35.114234732719261 + lats[1780] = -35.184533500486005 + lats[1781] = -35.254832268249267 + lats[1782] = -35.325131036009047 + lats[1783] = -35.395429803765317 + lats[1784] = -35.465728571518085 + lats[1785] = -35.536027339267314 + lats[1786] = -35.606326107012997 + lats[1787] = -35.676624874755113 + lats[1788] = -35.746923642493655 + lats[1789] = -35.817222410228595 + lats[1790] = -35.887521177959933 + lats[1791] = -35.957819945687639 + lats[1792] = -36.028118713411708 + lats[1793] = -36.098417481132117 + lats[1794] = -36.16871624884886 + lats[1795] = -36.239015016561908 + lats[1796] = -36.309313784271254 + lats[1797] = -36.379612551976876 + lats[1798] = -36.449911319678755 + lats[1799] = -36.520210087376888 + lats[1800] = -36.590508855071242 + lats[1801] = -36.660807622761808 + lats[1802] = -36.731106390448581 + lats[1803] = -36.801405158131523 + lats[1804] = -36.871703925810628 + lats[1805] = -36.942002693485883 + lats[1806] = -37.012301461157264 + lats[1807] = -37.082600228824752 + lats[1808] = -37.152898996488332 + lats[1809] = -37.223197764147997 + lats[1810] = -37.293496531803719 + lats[1811] = -37.363795299455489 + lats[1812] = -37.434094067103274 + lats[1813] = -37.504392834747065 + lats[1814] = -37.574691602386856 + lats[1815] = -37.644990370022605 + lats[1816] = -37.715289137654317 + lats[1817] = -37.785587905281965 + lats[1818] = -37.855886672905527 + lats[1819] = -37.926185440524989 + lats[1820] = -37.99648420814033 + lats[1821] = -38.066782975751536 + lats[1822] = -38.137081743358586 + lats[1823] = -38.20738051096145 + lats[1824] = -38.277679278560143 + lats[1825] = -38.347978046154608 + lats[1826] = -38.418276813744846 + lats[1827] = -38.488575581330842 + lats[1828] = -38.558874348912568 + lats[1829] = -38.629173116490001 + lats[1830] = -38.699471884063136 + lats[1831] = -38.769770651631937 + lats[1832] = -38.840069419196389 + lats[1833] = -38.910368186756479 + lats[1834] = -38.980666954312184 + lats[1835] = -39.050965721863491 + lats[1836] = -39.121264489410365 + lats[1837] = -39.191563256952804 + lats[1838] = -39.261862024490775 + lats[1839] = -39.332160792024254 + lats[1840] = -39.402459559553229 + lats[1841] = -39.472758327077692 + lats[1842] = -39.543057094597607 + lats[1843] = -39.613355862112947 + lats[1844] = -39.683654629623703 + lats[1845] = -39.753953397129855 + lats[1846] = -39.824252164631375 + lats[1847] = -39.894550932128247 + lats[1848] = -39.964849699620437 + lats[1849] = -40.035148467107952 + lats[1850] = -40.105447234590748 + lats[1851] = -40.175746002068806 + lats[1852] = -40.246044769542102 + lats[1853] = -40.316343537010617 + lats[1854] = -40.386642304474343 + lats[1855] = -40.456941071933244 + lats[1856] = -40.527239839387299 + lats[1857] = -40.597538606836487 + lats[1858] = -40.667837374280786 + lats[1859] = -40.738136141720176 + lats[1860] = -40.808434909154634 + lats[1861] = -40.878733676584126 + lats[1862] = -40.949032444008644 + lats[1863] = -41.01933121142816 + lats[1864] = -41.089629978842645 + lats[1865] = -41.159928746252085 + lats[1866] = -41.230227513656445 + lats[1867] = -41.300526281055724 + lats[1868] = -41.370825048449873 + lats[1869] = -41.441123815838885 + lats[1870] = -41.511422583222718 + lats[1871] = -41.581721350601363 + lats[1872] = -41.6520201179748 + lats[1873] = -41.722318885343 + lats[1874] = -41.792617652705921 + lats[1875] = -41.862916420063563 + lats[1876] = -41.933215187415882 + lats[1877] = -42.003513954762873 + lats[1878] = -42.073812722104492 + lats[1879] = -42.144111489440725 + lats[1880] = -42.214410256771551 + lats[1881] = -42.284709024096927 + lats[1882] = -42.355007791416853 + lats[1883] = -42.425306558731272 + lats[1884] = -42.495605326040177 + lats[1885] = -42.565904093343548 + lats[1886] = -42.63620286064134 + lats[1887] = -42.706501627933541 + lats[1888] = -42.776800395220121 + lats[1889] = -42.847099162501053 + lats[1890] = -42.917397929776307 + lats[1891] = -42.987696697045862 + lats[1892] = -43.057995464309691 + lats[1893] = -43.128294231567757 + lats[1894] = -43.19859299882004 + lats[1895] = -43.26889176606651 + lats[1896] = -43.339190533307139 + lats[1897] = -43.409489300541907 + lats[1898] = -43.479788067770777 + lats[1899] = -43.550086834993728 + lats[1900] = -43.620385602210717 + lats[1901] = -43.690684369421732 + lats[1902] = -43.760983136626741 + lats[1903] = -43.831281903825705 + lats[1904] = -43.9015806710186 + lats[1905] = -43.971879438205391 + lats[1906] = -44.042178205386072 + lats[1907] = -44.112476972560586 + lats[1908] = -44.182775739728925 + lats[1909] = -44.253074506891046 + lats[1910] = -44.323373274046915 + lats[1911] = -44.39367204119651 + lats[1912] = -44.463970808339802 + lats[1913] = -44.534269575476756 + lats[1914] = -44.604568342607337 + lats[1915] = -44.674867109731515 + lats[1916] = -44.745165876849271 + lats[1917] = -44.81546464396056 + lats[1918] = -44.885763411065362 + lats[1919] = -44.956062178163634 + lats[1920] = -45.026360945255341 + lats[1921] = -45.096659712340461 + lats[1922] = -45.166958479418959 + lats[1923] = -45.237257246490813 + lats[1924] = -45.30755601355596 + lats[1925] = -45.377854780614399 + lats[1926] = -45.448153547666081 + lats[1927] = -45.51845231471097 + lats[1928] = -45.588751081749038 + lats[1929] = -45.659049848780263 + lats[1930] = -45.729348615804589 + lats[1931] = -45.799647382821995 + lats[1932] = -45.869946149832437 + lats[1933] = -45.94024491683588 + lats[1934] = -46.01054368383231 + lats[1935] = -46.080842450821663 + lats[1936] = -46.151141217803925 + lats[1937] = -46.221439984779053 + lats[1938] = -46.291738751747012 + lats[1939] = -46.362037518707766 + lats[1940] = -46.432336285661272 + lats[1941] = -46.502635052607502 + lats[1942] = -46.572933819546414 + lats[1943] = -46.643232586477971 + lats[1944] = -46.713531353402139 + lats[1945] = -46.783830120318882 + lats[1946] = -46.85412888722815 + lats[1947] = -46.924427654129929 + lats[1948] = -46.994726421024154 + lats[1949] = -47.065025187910805 + lats[1950] = -47.13532395478984 + lats[1951] = -47.205622721661214 + lats[1952] = -47.275921488524894 + lats[1953] = -47.346220255380835 + lats[1954] = -47.416519022228997 + lats[1955] = -47.486817789069342 + lats[1956] = -47.557116555901828 + lats[1957] = -47.627415322726435 + lats[1958] = -47.697714089543084 + lats[1959] = -47.76801285635176 + lats[1960] = -47.838311623152421 + lats[1961] = -47.908610389945018 + lats[1962] = -47.978909156729507 + lats[1963] = -48.049207923505868 + lats[1964] = -48.119506690274015 + lats[1965] = -48.189805457033941 + lats[1966] = -48.260104223785596 + lats[1967] = -48.330402990528938 + lats[1968] = -48.400701757263917 + lats[1969] = -48.47100052399049 + lats[1970] = -48.541299290708608 + lats[1971] = -48.611598057418242 + lats[1972] = -48.681896824119335 + lats[1973] = -48.752195590811837 + lats[1974] = -48.822494357495721 + lats[1975] = -48.892793124170929 + lats[1976] = -48.963091890837418 + lats[1977] = -49.03339065749514 + lats[1978] = -49.103689424144044 + lats[1979] = -49.173988190784094 + lats[1980] = -49.244286957415234 + lats[1981] = -49.314585724037435 + lats[1982] = -49.384884490650613 + lats[1983] = -49.455183257254745 + lats[1984] = -49.525482023849783 + lats[1985] = -49.595780790435676 + lats[1986] = -49.66607955701236 + lats[1987] = -49.736378323579807 + lats[1988] = -49.80667709013796 + lats[1989] = -49.876975856686762 + lats[1990] = -49.947274623226157 + lats[1991] = -50.017573389756123 + lats[1992] = -50.087872156276575 + lats[1993] = -50.158170922787484 + lats[1994] = -50.228469689288779 + lats[1995] = -50.298768455780426 + lats[1996] = -50.369067222262359 + lats[1997] = -50.439365988734544 + lats[1998] = -50.509664755196901 + lats[1999] = -50.579963521649397 + lats[2000] = -50.650262288091959 + lats[2001] = -50.720561054524559 + lats[2002] = -50.790859820947119 + lats[2003] = -50.86115858735959 + lats[2004] = -50.931457353761914 + lats[2005] = -51.001756120154049 + lats[2006] = -51.072054886535909 + lats[2007] = -51.14235365290746 + lats[2008] = -51.21265241926865 + lats[2009] = -51.282951185619417 + lats[2010] = -51.353249951959683 + lats[2011] = -51.423548718289396 + lats[2012] = -51.493847484608516 + lats[2013] = -51.56414625091697 + lats[2014] = -51.634445017214695 + lats[2015] = -51.704743783501634 + lats[2016] = -51.775042549777737 + lats[2017] = -51.845341316042933 + lats[2018] = -51.915640082297152 + lats[2019] = -51.985938848540336 + lats[2020] = -52.056237614772435 + lats[2021] = -52.126536380993372 + lats[2022] = -52.196835147203096 + lats[2023] = -52.26713391340153 + lats[2024] = -52.337432679588609 + lats[2025] = -52.407731445764284 + lats[2026] = -52.478030211928477 + lats[2027] = -52.548328978081123 + lats[2028] = -52.618627744222159 + lats[2029] = -52.688926510351514 + lats[2030] = -52.759225276469131 + lats[2031] = -52.829524042574917 + lats[2032] = -52.899822808668837 + lats[2033] = -52.970121574750792 + lats[2034] = -53.040420340820731 + lats[2035] = -53.110719106878584 + lats[2036] = -53.181017872924265 + lats[2037] = -53.251316638957725 + lats[2038] = -53.321615404978871 + lats[2039] = -53.391914170987633 + lats[2040] = -53.462212936983953 + lats[2041] = -53.53251170296776 + lats[2042] = -53.602810468938962 + lats[2043] = -53.673109234897495 + lats[2044] = -53.743408000843282 + lats[2045] = -53.813706766776235 + lats[2046] = -53.884005532696307 + lats[2047] = -53.954304298603383 + lats[2048] = -54.024603064497434 + lats[2049] = -54.094901830378333 + lats[2050] = -54.165200596246031 + lats[2051] = -54.235499362100448 + lats[2052] = -54.305798127941479 + lats[2053] = -54.376096893769081 + lats[2054] = -54.446395659583146 + lats[2055] = -54.516694425383605 + lats[2056] = -54.586993191170357 + lats[2057] = -54.657291956943347 + lats[2058] = -54.727590722702473 + lats[2059] = -54.797889488447652 + lats[2060] = -54.868188254178797 + lats[2061] = -54.938487019895831 + lats[2062] = -55.008785785598668 + lats[2063] = -55.07908455128721 + lats[2064] = -55.149383316961377 + lats[2065] = -55.219682082621084 + lats[2066] = -55.289980848266232 + lats[2067] = -55.360279613896743 + lats[2068] = -55.430578379512511 + lats[2069] = -55.500877145113449 + lats[2070] = -55.571175910699488 + lats[2071] = -55.641474676270505 + lats[2072] = -55.711773441826416 + lats[2073] = -55.782072207367136 + lats[2074] = -55.852370972892551 + lats[2075] = -55.922669738402583 + lats[2076] = -55.992968503897131 + lats[2077] = -56.063267269376091 + lats[2078] = -56.133566034839362 + lats[2079] = -56.203864800286865 + lats[2080] = -56.274163565718467 + lats[2081] = -56.34446233113411 + lats[2082] = -56.41476109653366 + lats[2083] = -56.485059861917016 + lats[2084] = -56.555358627284086 + lats[2085] = -56.625657392634771 + lats[2086] = -56.695956157968951 + lats[2087] = -56.766254923286517 + lats[2088] = -56.836553688587379 + lats[2089] = -56.90685245387143 + lats[2090] = -56.977151219138541 + lats[2091] = -57.047449984388614 + lats[2092] = -57.117748749621541 + lats[2093] = -57.188047514837208 + lats[2094] = -57.258346280035504 + lats[2095] = -57.328645045216312 + lats[2096] = -57.398943810379521 + lats[2097] = -57.469242575525016 + lats[2098] = -57.539541340652676 + lats[2099] = -57.60984010576238 + lats[2100] = -57.680138870854037 + lats[2101] = -57.75043763592749 + lats[2102] = -57.820736400982646 + lats[2103] = -57.891035166019364 + lats[2104] = -57.961333931037537 + lats[2105] = -58.031632696037022 + lats[2106] = -58.101931461017728 + lats[2107] = -58.172230225979497 + lats[2108] = -58.242528990922203 + lats[2109] = -58.312827755845746 + lats[2110] = -58.383126520749968 + lats[2111] = -58.453425285634758 + lats[2112] = -58.523724050499972 + lats[2113] = -58.594022815345468 + lats[2114] = -58.664321580171141 + lats[2115] = -58.73462034497684 + lats[2116] = -58.804919109762423 + lats[2117] = -58.875217874527763 + lats[2118] = -58.945516639272725 + lats[2119] = -59.015815403997145 + lats[2120] = -59.086114168700909 + lats[2121] = -59.156412933383855 + lats[2122] = -59.226711698045854 + lats[2123] = -59.29701046268675 + lats[2124] = -59.3673092273064 + lats[2125] = -59.43760799190467 + lats[2126] = -59.507906756481383 + lats[2127] = -59.578205521036402 + lats[2128] = -59.64850428556958 + lats[2129] = -59.718803050080759 + lats[2130] = -59.78910181456979 + lats[2131] = -59.859400579036503 + lats[2132] = -59.929699343480763 + lats[2133] = -59.999998107902378 + lats[2134] = -60.070296872301235 + lats[2135] = -60.140595636677112 + lats[2136] = -60.21089440102989 + lats[2137] = -60.28119316535939 + lats[2138] = -60.35149192966545 + lats[2139] = -60.421790693947884 + lats[2140] = -60.492089458206543 + lats[2141] = -60.562388222441243 + lats[2142] = -60.632686986651805 + lats[2143] = -60.702985750838074 + lats[2144] = -60.773284514999872 + lats[2145] = -60.843583279137007 + lats[2146] = -60.913882043249295 + lats[2147] = -60.984180807336578 + lats[2148] = -61.054479571398652 + lats[2149] = -61.124778335435344 + lats[2150] = -61.195077099446451 + lats[2151] = -61.265375863431785 + lats[2152] = -61.335674627391185 + lats[2153] = -61.405973391324409 + lats[2154] = -61.476272155231321 + lats[2155] = -61.546570919111666 + lats[2156] = -61.616869682965287 + lats[2157] = -61.687168446791986 + lats[2158] = -61.757467210591535 + lats[2159] = -61.827765974363729 + lats[2160] = -61.898064738108381 + lats[2161] = -61.968363501825259 + lats[2162] = -62.038662265514176 + lats[2163] = -62.108961029174914 + lats[2164] = -62.179259792807258 + lats[2165] = -62.249558556410982 + lats[2166] = -62.319857319985871 + lats[2167] = -62.3901560835317 + lats[2168] = -62.460454847048261 + lats[2169] = -62.530753610535321 + lats[2170] = -62.60105237399263 + lats[2171] = -62.67135113741999 + lats[2172] = -62.741649900817137 + lats[2173] = -62.811948664183866 + lats[2174] = -62.882247427519928 + lats[2175] = -62.952546190825068 + lats[2176] = -63.022844954099064 + lats[2177] = -63.093143717341647 + lats[2178] = -63.163442480552604 + lats[2179] = -63.23374124373165 + lats[2180] = -63.304040006878537 + lats[2181] = -63.374338769993031 + lats[2182] = -63.444637533074854 + lats[2183] = -63.514936296123757 + lats[2184] = -63.585235059139464 + lats[2185] = -63.655533822121711 + lats[2186] = -63.725832585070251 + lats[2187] = -63.796131347984762 + lats[2188] = -63.866430110865004 + lats[2189] = -63.93672887371072 + lats[2190] = -64.00702763652157 + lats[2191] = -64.07732639929732 + lats[2192] = -64.147625162037642 + lats[2193] = -64.21792392474228 + lats[2194] = -64.288222687410922 + lats[2195] = -64.358521450043284 + lats[2196] = -64.428820212639039 + lats[2197] = -64.499118975197902 + lats[2198] = -64.569417737719576 + lats[2199] = -64.639716500203733 + lats[2200] = -64.710015262650074 + lats[2201] = -64.780314025058246 + lats[2202] = -64.850612787427963 + lats[2203] = -64.920911549758912 + lats[2204] = -64.991210312050711 + lats[2205] = -65.061509074303089 + lats[2206] = -65.131807836515677 + lats[2207] = -65.202106598688133 + lats[2208] = -65.272405360820116 + lats[2209] = -65.342704122911286 + lats[2210] = -65.413002884961315 + lats[2211] = -65.483301646969792 + lats[2212] = -65.553600408936404 + lats[2213] = -65.623899170860767 + lats[2214] = -65.694197932742526 + lats[2215] = -65.764496694581283 + lats[2216] = -65.834795456376696 + lats[2217] = -65.905094218128355 + lats[2218] = -65.975392979835888 + lats[2219] = -66.045691741498899 + lats[2220] = -66.115990503117033 + lats[2221] = -66.186289264689833 + lats[2222] = -66.256588026216932 + lats[2223] = -66.326886787697887 + lats[2224] = -66.397185549132331 + lats[2225] = -66.467484310519808 + lats[2226] = -66.537783071859891 + lats[2227] = -66.608081833152212 + lats[2228] = -66.678380594396273 + lats[2229] = -66.748679355591662 + lats[2230] = -66.818978116737924 + lats[2231] = -66.889276877834618 + lats[2232] = -66.95957563888129 + lats[2233] = -67.029874399877471 + lats[2234] = -67.100173160822706 + lats[2235] = -67.170471921716526 + lats[2236] = -67.240770682558434 + lats[2237] = -67.311069443347961 + lats[2238] = -67.381368204084609 + lats[2239] = -67.451666964767895 + lats[2240] = -67.521965725397308 + lats[2241] = -67.592264485972336 + lats[2242] = -67.662563246492482 + lats[2243] = -67.732862006957205 + lats[2244] = -67.803160767365966 + lats[2245] = -67.873459527718282 + lats[2246] = -67.943758288013555 + lats[2247] = -68.014057048251274 + lats[2248] = -68.084355808430871 + lats[2249] = -68.154654568551791 + lats[2250] = -68.224953328613438 + lats[2251] = -68.295252088615257 + lats[2252] = -68.365550848556666 + lats[2253] = -68.435849608437067 + lats[2254] = -68.506148368255865 + lats[2255] = -68.576447128012447 + lats[2256] = -68.646745887706189 + lats[2257] = -68.717044647336493 + lats[2258] = -68.787343406902693 + lats[2259] = -68.85764216640419 + lats[2260] = -68.927940925840304 + lats[2261] = -68.998239685210365 + lats[2262] = -69.068538444513763 + lats[2263] = -69.138837203749759 + lats[2264] = -69.209135962917699 + lats[2265] = -69.279434722016902 + lats[2266] = -69.349733481046613 + lats[2267] = -69.420032240006194 + lats[2268] = -69.490330998894862 + lats[2269] = -69.560629757711908 + lats[2270] = -69.630928516456592 + lats[2271] = -69.701227275128161 + lats[2272] = -69.771526033725834 + lats[2273] = -69.841824792248843 + lats[2274] = -69.912123550696421 + lats[2275] = -69.982422309067744 + lats[2276] = -70.052721067362043 + lats[2277] = -70.123019825578467 + lats[2278] = -70.193318583716191 + lats[2279] = -70.263617341774406 + lats[2280] = -70.333916099752187 + lats[2281] = -70.404214857648739 + lats[2282] = -70.474513615463138 + lats[2283] = -70.544812373194532 + lats[2284] = -70.615111130841967 + lats[2285] = -70.685409888404578 + lats[2286] = -70.755708645881384 + lats[2287] = -70.826007403271475 + lats[2288] = -70.896306160573886 + lats[2289] = -70.966604917787635 + lats[2290] = -71.036903674911756 + lats[2291] = -71.107202431945211 + lats[2292] = -71.177501188887007 + lats[2293] = -71.247799945736105 + lats[2294] = -71.318098702491469 + lats[2295] = -71.388397459152031 + lats[2296] = -71.458696215716685 + lats[2297] = -71.528994972184378 + lats[2298] = -71.599293728553988 + lats[2299] = -71.669592484824364 + lats[2300] = -71.739891240994368 + lats[2301] = -71.810189997062835 + lats[2302] = -71.880488753028587 + lats[2303] = -71.950787508890414 + lats[2304] = -72.02108626464711 + lats[2305] = -72.091385020297409 + lats[2306] = -72.161683775840089 + lats[2307] = -72.231982531273843 + lats[2308] = -72.302281286597392 + lats[2309] = -72.3725800418094 + lats[2310] = -72.442878796908545 + lats[2311] = -72.513177551893421 + lats[2312] = -72.583476306762691 + lats[2313] = -72.653775061514935 + lats[2314] = -72.724073816148703 + lats[2315] = -72.794372570662574 + lats[2316] = -72.864671325055056 + lats[2317] = -72.934970079324657 + lats[2318] = -73.005268833469799 + lats[2319] = -73.075567587489019 + lats[2320] = -73.145866341380668 + lats[2321] = -73.216165095143182 + lats[2322] = -73.2864638487749 + lats[2323] = -73.356762602274188 + lats[2324] = -73.427061355639339 + lats[2325] = -73.497360108868662 + lats[2326] = -73.567658861960396 + lats[2327] = -73.637957614912779 + lats[2328] = -73.70825636772399 + lats[2329] = -73.778555120392184 + lats[2330] = -73.848853872915541 + lats[2331] = -73.919152625292114 + lats[2332] = -73.98945137751997 + lats[2333] = -74.059750129597163 + lats[2334] = -74.13004888152166 + lats[2335] = -74.200347633291472 + lats[2336] = -74.270646384904481 + lats[2337] = -74.340945136358584 + lats[2338] = -74.411243887651622 + lats[2339] = -74.481542638781434 + lats[2340] = -74.551841389745761 + lats[2341] = -74.622140140542356 + lats[2342] = -74.692438891168877 + lats[2343] = -74.762737641622991 + lats[2344] = -74.833036391902269 + lats[2345] = -74.903335142004323 + lats[2346] = -74.973633891926625 + lats[2347] = -75.043932641666672 + lats[2348] = -75.114231391221821 + lats[2349] = -75.184530140589501 + lats[2350] = -75.254828889766983 + lats[2351] = -75.325127638751567 + lats[2352] = -75.395426387540439 + lats[2353] = -75.465725136130786 + lats[2354] = -75.536023884519707 + lats[2355] = -75.60632263270422 + lats[2356] = -75.67662138068134 + lats[2357] = -75.746920128447996 + lats[2358] = -75.81721887600105 + lats[2359] = -75.887517623337317 + lats[2360] = -75.957816370453543 + lats[2361] = -76.028115117346374 + lats[2362] = -76.098413864012443 + lats[2363] = -76.16871261044831 + lats[2364] = -76.239011356650423 + lats[2365] = -76.3093101026152 + lats[2366] = -76.379608848338933 + lats[2367] = -76.449907593817869 + lats[2368] = -76.520206339048215 + lats[2369] = -76.59050508402602 + lats[2370] = -76.660803828747362 + lats[2371] = -76.731102573208048 + lats[2372] = -76.801401317404 + lats[2373] = -76.871700061330955 + lats[2374] = -76.941998804984564 + lats[2375] = -77.012297548360323 + lats[2376] = -77.082596291453768 + lats[2377] = -77.15289503426024 + lats[2378] = -77.22319377677502 + lats[2379] = -77.293492518993247 + lats[2380] = -77.363791260909963 + lats[2381] = -77.434090002520122 + lats[2382] = -77.504388743818524 + lats[2383] = -77.574687484799924 + lats[2384] = -77.644986225458879 + lats[2385] = -77.71528496578982 + lats[2386] = -77.785583705787161 + lats[2387] = -77.855882445445019 + lats[2388] = -77.926181184757539 + lats[2389] = -77.996479923718596 + lats[2390] = -78.066778662322022 + lats[2391] = -78.137077400561424 + lats[2392] = -78.207376138430348 + lats[2393] = -78.277674875922045 + lats[2394] = -78.347973613029708 + lats[2395] = -78.418272349746417 + lats[2396] = -78.488571086064923 + lats[2397] = -78.558869821977908 + lats[2398] = -78.629168557477882 + lats[2399] = -78.699467292557102 + lats[2400] = -78.769766027207638 + lats[2401] = -78.840064761421445 + lats[2402] = -78.910363495190211 + lats[2403] = -78.980662228505423 + lats[2404] = -79.050960961358285 + lats[2405] = -79.121259693739859 + lats[2406] = -79.191558425640977 + lats[2407] = -79.261857157052191 + lats[2408] = -79.332155887963822 + lats[2409] = -79.402454618365894 + lats[2410] = -79.472753348248219 + lats[2411] = -79.543052077600308 + lats[2412] = -79.61335080641139 + lats[2413] = -79.683649534670437 + lats[2414] = -79.753948262366038 + lats[2415] = -79.824246989486554 + lats[2416] = -79.894545716019948 + lats[2417] = -79.9648444419539 + lats[2418] = -80.035143167275749 + lats[2419] = -80.105441891972376 + lats[2420] = -80.175740616030438 + lats[2421] = -80.246039339436052 + lats[2422] = -80.316338062175078 + lats[2423] = -80.386636784232863 + lats[2424] = -80.456935505594302 + lats[2425] = -80.527234226243991 + lats[2426] = -80.59753294616587 + lats[2427] = -80.667831665343556 + lats[2428] = -80.73813038376008 + lats[2429] = -80.808429101397948 + lats[2430] = -80.878727818239184 + lats[2431] = -80.949026534265244 + lats[2432] = -81.019325249456955 + lats[2433] = -81.089623963794551 + lats[2434] = -81.159922677257711 + lats[2435] = -81.230221389825374 + lats[2436] = -81.300520101475826 + lats[2437] = -81.370818812186627 + lats[2438] = -81.441117521934686 + lats[2439] = -81.511416230696042 + lats[2440] = -81.581714938445955 + lats[2441] = -81.652013645158945 + lats[2442] = -81.722312350808508 + lats[2443] = -81.792611055367345 + lats[2444] = -81.862909758807191 + lats[2445] = -81.933208461098829 + lats[2446] = -82.003507162211946 + lats[2447] = -82.073805862115165 + lats[2448] = -82.144104560776 + lats[2449] = -82.214403258160871 + lats[2450] = -82.284701954234833 + lats[2451] = -82.355000648961692 + lats[2452] = -82.425299342304029 + lats[2453] = -82.495598034222837 + lats[2454] = -82.56589672467787 + lats[2455] = -82.63619541362705 + lats[2456] = -82.706494101026948 + lats[2457] = -82.77679278683226 + lats[2458] = -82.84709147099602 + lats[2459] = -82.917390153469313 + lats[2460] = -82.987688834201322 + lats[2461] = -83.057987513139125 + lats[2462] = -83.128286190227698 + lats[2463] = -83.198584865409657 + lats[2464] = -83.268883538625232 + lats[2465] = -83.339182209812321 + lats[2466] = -83.409480878905782 + lats[2467] = -83.479779545838113 + lats[2468] = -83.550078210538487 + lats[2469] = -83.620376872933264 + lats[2470] = -83.690675532945292 + lats[2471] = -83.760974190494011 + lats[2472] = -83.831272845495249 + lats[2473] = -83.901571497860914 + lats[2474] = -83.971870147498763 + lats[2475] = -84.042168794312317 + lats[2476] = -84.112467438200326 + lats[2477] = -84.18276607905679 + lats[2478] = -84.253064716770425 + lats[2479] = -84.323363351224444 + lats[2480] = -84.393661982296322 + lats[2481] = -84.463960609857125 + lats[2482] = -84.534259233771479 + lats[2483] = -84.604557853896708 + lats[2484] = -84.674856470082915 + lats[2485] = -84.745155082171991 + lats[2486] = -84.81545368999717 + lats[2487] = -84.885752293382765 + lats[2488] = -84.95605089214304 + lats[2489] = -85.026349486081983 + lats[2490] = -85.09664807499216 + lats[2491] = -85.16694665865414 + lats[2492] = -85.237245236835548 + lats[2493] = -85.307543809290152 + lats[2494] = -85.377842375756586 + lats[2495] = -85.448140935957483 + lats[2496] = -85.518439489597966 + lats[2497] = -85.588738036364362 + lats[2498] = -85.659036575922883 + lats[2499] = -85.729335107917464 + lats[2500] = -85.799633631968391 + lats[2501] = -85.869932147670127 + lats[2502] = -85.940230654588888 + lats[2503] = -86.010529152260403 + lats[2504] = -86.080827640187209 + lats[2505] = -86.151126117835304 + lats[2506] = -86.221424584631109 + lats[2507] = -86.291723039957418 + lats[2508] = -86.362021483149363 + lats[2509] = -86.432319913489792 + lats[2510] = -86.502618330203831 + lats[2511] = -86.572916732453024 + lats[2512] = -86.643215119328573 + lats[2513] = -86.713513489844246 + lats[2514] = -86.783811842927179 + lats[2515] = -86.854110177408927 + lats[2516] = -86.924408492014166 + lats[2517] = -86.994706785348129 + lats[2518] = -87.065005055882821 + lats[2519] = -87.135303301939786 + lats[2520] = -87.205601521672108 + lats[2521] = -87.275899713041966 + lats[2522] = -87.346197873795816 + lats[2523] = -87.416496001434894 + lats[2524] = -87.486794093180748 + lats[2525] = -87.557092145935584 + lats[2526] = -87.627390156234085 + lats[2527] = -87.697688120188062 + lats[2528] = -87.767986033419561 + lats[2529] = -87.838283890981543 + lats[2530] = -87.908581687261687 + lats[2531] = -87.978879415867283 + lats[2532] = -88.049177069484486 + lats[2533] = -88.119474639706425 + lats[2534] = -88.189772116820762 + lats[2535] = -88.26006948954614 + lats[2536] = -88.330366744702559 + lats[2537] = -88.40066386679355 + lats[2538] = -88.470960837474877 + lats[2539] = -88.541257634868515 + lats[2540] = -88.611554232668382 + lats[2541] = -88.681850598961759 + lats[2542] = -88.752146694650691 + lats[2543] = -88.822442471310097 + lats[2544] = -88.892737868230952 + lats[2545] = -88.96303280826325 + lats[2546] = -89.033327191845927 + lats[2547] = -89.103620888238879 + lats[2548] = -89.173913722284126 + lats[2549] = -89.24420545380525 + lats[2550] = -89.314495744374256 + lats[2551] = -89.3847841013921 + lats[2552] = -89.45506977912261 + lats[2553] = -89.525351592371393 + lats[2554] = -89.595627537554492 + lats[2555] = -89.6658939412157 + lats[2556] = -89.736143271609578 + lats[2557] = -89.806357319542244 + lats[2558] = -89.876478353332288 + lats[2559] = -89.946187715665616 + return lats + def first_axis_vals(self): - precision = 1.0e-14 - nval = self._resolution * 2 - rad2deg = 180 / math.pi - convval = 1 - ((2 / math.pi) * (2 / math.pi)) * 0.25 - vals = self.gauss_first_guess() - new_vals = [0] * nval - denom = math.sqrt(((nval + 0.5) * (nval + 0.5)) + convval) - for jval in range(self._resolution): - root = math.cos(vals[jval] / denom) - conv = 1 - while abs(conv) >= precision: - mem2 = 1 - mem1 = root - for legi in range(nval): - legfonc = ((2.0 * (legi + 1) - 1.0) * root * mem1 - legi * mem2) / (legi + 1) - mem2 = mem1 - mem1 = legfonc - conv = legfonc / ((nval * (mem2 - root * legfonc)) / (1.0 - (root * root))) - root = root - conv - # add maybe a max iter here to make sure we converge at some point - new_vals[jval] = math.asin(root) * rad2deg - new_vals[nval - 1 - jval] = -new_vals[jval] - return new_vals + if self._resolution == 1280: + return self.get_precomputed_values_N1280() + else: + precision = 1.0e-14 + nval = self._resolution * 2 + rad2deg = 180 / math.pi + convval = 1 - ((2 / math.pi) * (2 / math.pi)) * 0.25 + vals = self.gauss_first_guess() + new_vals = [0] * nval + denom = math.sqrt(((nval + 0.5) * (nval + 0.5)) + convval) + for jval in range(self._resolution): + root = math.cos(vals[jval] / denom) + conv = 1 + while abs(conv) >= precision: + mem2 = 1 + mem1 = root + for legi in range(nval): + legfonc = ((2.0 * (legi + 1) - 1.0) * root * mem1 - legi * mem2) / (legi + 1) + mem2 = mem1 + mem1 = legfonc + conv = legfonc / ((nval * (mem2 - root * legfonc)) / (1.0 - (root * root))) + root = root - conv + # add maybe a max iter here to make sure we converge at some point + new_vals[jval] = math.asin(root) * rad2deg + new_vals[nval - 1 - jval] = -new_vals[jval] + return new_vals def map_first_axis(self, lower, upper): axis_lines = self.first_axis_vals() diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index c7092fd64..e4f5a29b5 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -12,7 +12,6 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m - print(latlon_array) self.xarraydatacube = XArrayDatacube(latlon_array) grid_options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} From 4b9495177e4077f5a1f26ddf2b0c482166581361 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 19 Jul 2023 15:02:13 +0200 Subject: [PATCH 051/332] add type hints for abstract classes --- polytope/datacube/datacube.py | 12 ++++++------ polytope/datacube/datacube_axis.py | 20 ++++++++++---------- polytope/engine/engine.py | 3 ++- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index 1fde53de7..3d7bfbf82 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -13,13 +13,13 @@ def get(self, requests: IndexTree) -> Any: """Return data given a set of request trees""" @abstractmethod - def get_mapper(self, axis) -> DatacubeAxis: + def get_mapper(self, axis: str) -> DatacubeAxis: """ Get the type mapper for a subaxis of the datacube given by label """ @abstractmethod - def get_indices(self, path: DatacubePath, axis: str, lower: Any, upper: Any) -> List: + def get_indices(self, path: DatacubePath, axis: DatacubeAxis, lower: Any, upper: Any) -> List[Any]: """ Given a path to a subset of the datacube, return the discrete indexes which exist between two non-discrete values (lower, upper) for a particular axis (given by label) @@ -28,20 +28,20 @@ def get_indices(self, path: DatacubePath, axis: str, lower: Any, upper: Any) -> """ @abstractmethod - def has_index(self, path: DatacubePath, axis, index) -> bool: + def has_index(self, path: DatacubePath, axis: DatacubeAxis, index: Any) -> bool: "Given a path to a subset of the datacube, checks if the index exists on that sub-datacube axis" @property @abstractmethod - def axes(self): + def axes(self) -> dict[str, DatacubeAxis]: pass @abstractmethod - def validate(self, axes) -> bool: + def validate(self, axes: List[str]) -> bool: """returns true if the input axes can be resolved against the datacube axes""" @staticmethod - def create(datacube, options): + def create(datacube, options: dict): if isinstance(datacube, (xr.core.dataarray.DataArray, xr.core.dataset.Dataset)): from .xarray import XArrayDatacube diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 7a5680a05..2c28b4b6c 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -8,15 +8,15 @@ class DatacubeAxis(ABC): @abstractproperty - def name(self): + def name(self) -> str: pass @abstractproperty - def tol(self): + def tol(self) -> Any: pass @abstractproperty - def range(self): + def range(self) -> List[Any]: pass # Convert from user-provided value to CONTINUOUS type (e.g. float, pd.timestamp) @@ -34,25 +34,25 @@ def to_float(self, value: Any) -> float: def from_float(self, value: float) -> Any: pass - def serialize(self, value) -> Any: + def serialize(self, value: Any) -> Any: pass - def remap(self, range: List) -> Any: + def remap(self, range: List[Any]) -> Any: pass - def to_intervals(self, range): + def to_intervals(self, range: List[Any]) -> List[List[Any]]: pass - def remap_val_to_axis_range(self, value): + def remap_val_to_axis_range(self, value: Any) -> Any: pass - def remap_range_to_axis_range(self, range): + def remap_range_to_axis_range(self, range: List[Any]) -> List[Any]: pass - def to_cyclic_value(self, value): + def to_cyclic_value(self, value: Any) -> Any: pass - def offset(self, value): + def offset(self, value: Any) -> int: pass diff --git a/polytope/engine/engine.py b/polytope/engine/engine.py index 85682e46e..f9eeb7047 100644 --- a/polytope/engine/engine.py +++ b/polytope/engine/engine.py @@ -1,6 +1,7 @@ from typing import List from ..datacube.datacube import Datacube +from ..datacube.datacube_request_tree import IndexTree from ..shapes import ConvexPolytope @@ -8,7 +9,7 @@ class Engine: def __init__(self): pass - def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): + def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]) -> IndexTree: pass @staticmethod From 91482992036c8345923e06c89e2a54409315cdc9 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 25 Jul 2023 10:13:47 +0200 Subject: [PATCH 052/332] fix small bug --- polytope/datacube/datacube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index 8c39ed4db..2d10ac728 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -41,7 +41,7 @@ def validate(self, axes: List[str]) -> bool: """returns true if the input axes can be resolved against the datacube axes""" @staticmethod - def create(datacube, options: dict): + def create(datacube, axis_options: dict): if isinstance(datacube, (xr.core.dataarray.DataArray, xr.core.dataset.Dataset)): from .xarray import XArrayDatacube From df94afa2ab5f92e1e92fa58bfa50da123fd23a7b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 25 Jul 2023 11:17:39 +0200 Subject: [PATCH 053/332] add searchsorted back for axis which support it --- polytope/datacube/xarray.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 14c1ad001..905f081ac 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -42,11 +42,13 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.mappers = {} self.dataarray = dataarray treated_axes = [] + self.complete_axes = [] for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: options = axis_options.get(name, {}) self.create_axis(options, name, values) treated_axes.append(name) + self.complete_axes.append(name) else: if self.dataarray[name].dims == (): options = axis_options.get(name, {}) @@ -163,7 +165,12 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, elif axis.name == second_axis: indexes_between = self.grid_mapper.map_second_axis(first_val, low, up) else: - indexes_between = [i for i in indexes if low <= i <= up] + if axis.name in self.complete_axes: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + else: + indexes_between = [i for i in indexes if low <= i <= up] # start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? # end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? # indexes_between = indexes[start:end].to_list() @@ -174,7 +181,12 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, # start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? # end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? # indexes_between = indexes[start:end].to_list() - indexes_between = [i for i in indexes if low <= i <= up] + if axis.name in self.complete_axes: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + else: + indexes_between = [i for i in indexes if low <= i <= up] # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them @@ -219,13 +231,15 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): else: # assert axis.name == next(iter(subarray.xindexes)) # indexes = next(iter(subarray.xindexes.values())).to_pandas_index() - if axis.name in self.dataarray.dims: - indexes = list(subarray.indexes[axis.name]) + if axis.name in self.complete_axes: + # indexes = list(subarray.indexes[axis.name]) + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() else: indexes = [subarray[axis.name].values] else: - if axis.name in self.dataarray.dims: - indexes = list(subarray.indexes[axis.name]) + if axis.name in self.complete_axes: + # indexes = list(subarray.indexes[axis.name]) + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() else: indexes = [subarray[axis.name].values] # assert axis.name == next(iter(subarray.xindexes)) From 4617a8483f99ebf853e27c1849d94d069f5e91c0 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 25 Jul 2023 11:29:25 +0200 Subject: [PATCH 054/332] need axes to be sorted from lowest to biggest value --- polytope/datacube/xarray.py | 1 + 1 file changed, 1 insertion(+) diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 905f081ac..1978913da 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -45,6 +45,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.complete_axes = [] for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: + self.dataarray = self.dataarray.sortby(name) options = axis_options.get(name, {}) self.create_axis(options, name, values) treated_axes.append(name) From dc59e7c69a217ee06f21d8381ee747b93c22700f Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 26 Jul 2023 17:08:48 +0200 Subject: [PATCH 055/332] add mapper tests --- tests/test_mappers.py | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/test_mappers.py diff --git a/tests/test_mappers.py b/tests/test_mappers.py new file mode 100644 index 000000000..4381e2314 --- /dev/null +++ b/tests/test_mappers.py @@ -0,0 +1,45 @@ +from polytope.datacube.mappers import OctahedralGridMap + + +class TestMapper: + def test_octahedral_mapper_init(self): + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 1280 + octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + assert octahedral_mapper._base_axis == base_axis + assert octahedral_mapper._mapped_axes == mapped_axes + assert octahedral_mapper._resolution == resolution + + def test_first_axis_vals_01280_grid(self): + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 1280 + octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + assert octahedral_mapper.first_axis_vals()[:5] == [89.94618771566562, 89.87647835333229, 89.80635731954224, + 89.73614327160958, 89.6658939412157] + assert len(octahedral_mapper.first_axis_vals()) == 1280*2 + + def test_first_axis_vals_other_grid(self): + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 640 + octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + assert octahedral_mapper.first_axis_vals()[:5] == [89.89239644559007, 89.75300494317403, 89.61279025859908, + 89.47238958206113, 89.33191835438183] + assert len(octahedral_mapper.first_axis_vals()) == 640*2 + + def test_map_first_axis(self): + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 1280 + octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + assert octahedral_mapper.map_first_axis(89.7, 89.96) == [89.94618771566562, 89.87647835333229, + 89.80635731954224, 89.73614327160958] + + def test_second_axis_vals(self): + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 1280 + octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + pass From 9b79a2609cbe008f9b47034f44e972bdb32eab85 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 26 Jul 2023 17:49:03 +0200 Subject: [PATCH 056/332] more mapper tests --- tests/test_mappers.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_mappers.py b/tests/test_mappers.py index 4381e2314..c7847559d 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -42,4 +42,18 @@ def test_second_axis_vals(self): base_axis = "base" resolution = 1280 octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + assert octahedral_mapper.second_axis_vals(0.035149384215604956)[0] == 0 + assert octahedral_mapper.second_axis_vals(10.017574499477174)[0] == 0 + assert octahedral_mapper.second_axis_vals(89.94618771566562)[10] == 180 + assert len(octahedral_mapper.second_axis_vals(89.94618771566562)) == 20 + assert len(octahedral_mapper.second_axis_vals(89.87647835333229)) == 24 + assert len(octahedral_mapper.second_axis_vals(0.035149384215604956)) == 5136 + + def test_map_second_axis(self): + pass + + def test_axes_idx_to_octahedral_idx(self): + pass + + def test_unmap(self): pass From 981afb7571a808f4340830c3c61339f2a3107fe6 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 27 Jul 2023 10:03:04 +0200 Subject: [PATCH 057/332] add tests and fix O1280 mapping at latitude 0 --- polytope/datacube/mappers.py | 4 ++-- tests/test_mappers.py | 38 +++++++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/polytope/datacube/mappers.py b/polytope/datacube/mappers.py index 5bd651d83..84cca64ae 100644 --- a/polytope/datacube/mappers.py +++ b/polytope/datacube/mappers.py @@ -2721,14 +2721,14 @@ def axes_idx_to_octahedral_idx(self, first_idx, second_idx): else: for i in range(first_idx - 1): if i <= self._resolution - 1: - octa_idx += 16 + 4 * i + octa_idx += 20 + 4 * i else: i = i - self._resolution + 1 if i == 1: octa_idx += 16 + 4 * self._resolution else: i = i - 1 - octa_idx += 16 + 4 * (self._resolution - i + 1) + octa_idx += 16 + 4 * (self._resolution - i) octa_idx += second_idx return octa_idx diff --git a/tests/test_mappers.py b/tests/test_mappers.py index c7847559d..dd4f6e975 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -50,10 +50,42 @@ def test_second_axis_vals(self): assert len(octahedral_mapper.second_axis_vals(0.035149384215604956)) == 5136 def test_map_second_axis(self): - pass + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 1280 + octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + assert octahedral_mapper.map_second_axis(89.94618771566562, 0, 90) == [0, 18, 36, 54, 72, 90] def test_axes_idx_to_octahedral_idx(self): - pass + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 1280 + octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + assert octahedral_mapper.axes_idx_to_octahedral_idx(1, 0) == 0 + assert octahedral_mapper.axes_idx_to_octahedral_idx(1, 1) == 1 + assert octahedral_mapper.axes_idx_to_octahedral_idx(1, 16) == 16 + assert octahedral_mapper.axes_idx_to_octahedral_idx(1, 17) == 17 + assert octahedral_mapper.axes_idx_to_octahedral_idx(2, 0) == 20 + assert octahedral_mapper.axes_idx_to_octahedral_idx(3, 0) == 44 + assert octahedral_mapper.axes_idx_to_octahedral_idx(1279, 0) == 3299840-5136 - 5136 + 4 + # at lat line 1280, we start the 1280th line, which has 5136 points + assert octahedral_mapper.axes_idx_to_octahedral_idx(1280, 0) == 3299840-5136 + # the 1281th lat line also has 5136 points, and we are exactly at the half of the number of points in the grid + assert octahedral_mapper.axes_idx_to_octahedral_idx(1281, 0) == 3299840 + assert octahedral_mapper.axes_idx_to_octahedral_idx(1281, 12) == 3299852 + # the 1281th lat line has 5136 points, so when we start the 1282nd lat line, we are at the half of the grid + # points + 5136 + assert octahedral_mapper.axes_idx_to_octahedral_idx(1282, 0) == 3299840 + 5136 + assert octahedral_mapper.axes_idx_to_octahedral_idx(1283, 0) == 3299840 + 5136 + 5136 - 4 + assert octahedral_mapper.axes_idx_to_octahedral_idx(1284, 0) == 3299840 + 5136 + 5136 - 4 + 5136 - 8 + # at the last lat line, we only have 20 points left in the grid + assert octahedral_mapper.axes_idx_to_octahedral_idx(2560, 0) == 3299840*2 - 20 def test_unmap(self): - pass + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 1280 + octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + assert octahedral_mapper.unmap(89.94618771566562, 0) == 0 + assert octahedral_mapper.unmap(0.035149384215604956, 0) == 3299840-5136 + assert octahedral_mapper.unmap(-0.035149384215604956, 0) == 3299840 From a6a282f9bcef25b31d68f054fe37a797c8c1a8c3 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 27 Jul 2023 10:04:37 +0200 Subject: [PATCH 058/332] black --- tests/test_mappers.py | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/tests/test_mappers.py b/tests/test_mappers.py index dd4f6e975..8c91bb15a 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -16,26 +16,40 @@ def test_first_axis_vals_01280_grid(self): base_axis = "base" resolution = 1280 octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) - assert octahedral_mapper.first_axis_vals()[:5] == [89.94618771566562, 89.87647835333229, 89.80635731954224, - 89.73614327160958, 89.6658939412157] - assert len(octahedral_mapper.first_axis_vals()) == 1280*2 + assert octahedral_mapper.first_axis_vals()[:5] == [ + 89.94618771566562, + 89.87647835333229, + 89.80635731954224, + 89.73614327160958, + 89.6658939412157, + ] + assert len(octahedral_mapper.first_axis_vals()) == 1280 * 2 def test_first_axis_vals_other_grid(self): mapped_axes = ["lat", "lon"] base_axis = "base" resolution = 640 octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) - assert octahedral_mapper.first_axis_vals()[:5] == [89.89239644559007, 89.75300494317403, 89.61279025859908, - 89.47238958206113, 89.33191835438183] - assert len(octahedral_mapper.first_axis_vals()) == 640*2 + assert octahedral_mapper.first_axis_vals()[:5] == [ + 89.89239644559007, + 89.75300494317403, + 89.61279025859908, + 89.47238958206113, + 89.33191835438183, + ] + assert len(octahedral_mapper.first_axis_vals()) == 640 * 2 def test_map_first_axis(self): mapped_axes = ["lat", "lon"] base_axis = "base" resolution = 1280 octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) - assert octahedral_mapper.map_first_axis(89.7, 89.96) == [89.94618771566562, 89.87647835333229, - 89.80635731954224, 89.73614327160958] + assert octahedral_mapper.map_first_axis(89.7, 89.96) == [ + 89.94618771566562, + 89.87647835333229, + 89.80635731954224, + 89.73614327160958, + ] def test_second_axis_vals(self): mapped_axes = ["lat", "lon"] @@ -67,9 +81,9 @@ def test_axes_idx_to_octahedral_idx(self): assert octahedral_mapper.axes_idx_to_octahedral_idx(1, 17) == 17 assert octahedral_mapper.axes_idx_to_octahedral_idx(2, 0) == 20 assert octahedral_mapper.axes_idx_to_octahedral_idx(3, 0) == 44 - assert octahedral_mapper.axes_idx_to_octahedral_idx(1279, 0) == 3299840-5136 - 5136 + 4 + assert octahedral_mapper.axes_idx_to_octahedral_idx(1279, 0) == 3299840 - 5136 - 5136 + 4 # at lat line 1280, we start the 1280th line, which has 5136 points - assert octahedral_mapper.axes_idx_to_octahedral_idx(1280, 0) == 3299840-5136 + assert octahedral_mapper.axes_idx_to_octahedral_idx(1280, 0) == 3299840 - 5136 # the 1281th lat line also has 5136 points, and we are exactly at the half of the number of points in the grid assert octahedral_mapper.axes_idx_to_octahedral_idx(1281, 0) == 3299840 assert octahedral_mapper.axes_idx_to_octahedral_idx(1281, 12) == 3299852 @@ -79,7 +93,7 @@ def test_axes_idx_to_octahedral_idx(self): assert octahedral_mapper.axes_idx_to_octahedral_idx(1283, 0) == 3299840 + 5136 + 5136 - 4 assert octahedral_mapper.axes_idx_to_octahedral_idx(1284, 0) == 3299840 + 5136 + 5136 - 4 + 5136 - 8 # at the last lat line, we only have 20 points left in the grid - assert octahedral_mapper.axes_idx_to_octahedral_idx(2560, 0) == 3299840*2 - 20 + assert octahedral_mapper.axes_idx_to_octahedral_idx(2560, 0) == 3299840 * 2 - 20 def test_unmap(self): mapped_axes = ["lat", "lon"] @@ -87,5 +101,5 @@ def test_unmap(self): resolution = 1280 octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) assert octahedral_mapper.unmap(89.94618771566562, 0) == 0 - assert octahedral_mapper.unmap(0.035149384215604956, 0) == 3299840-5136 + assert octahedral_mapper.unmap(0.035149384215604956, 0) == 3299840 - 5136 assert octahedral_mapper.unmap(-0.035149384215604956, 0) == 3299840 From 2bae21be9614f4be05d839c49de8a22053906701 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 27 Jul 2023 11:44:54 +0200 Subject: [PATCH 059/332] add the list_axes fdb function --- polytope/datacube/FDB_datacube.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/polytope/datacube/FDB_datacube.py b/polytope/datacube/FDB_datacube.py index b727f4b2d..0bafea977 100644 --- a/polytope/datacube/FDB_datacube.py +++ b/polytope/datacube/FDB_datacube.py @@ -15,6 +15,8 @@ UnsliceableaAxis, ) from .mappers import OctahedralGridMap +import pyfdb + _mappings = { pd.Int64Dtype: IntAxis(), @@ -42,6 +44,26 @@ def glue(path): return {"t" : 0} +def update_fdb_dataarray(fdb_dataarray): + new_dict = {} + for key, values in fdb_dataarray.items(): + if key in ["levelist", "param", "step"]: + new_values = [] + for val in values: + new_values.append(int(val)) + new_dict[key] = new_values + elif key == "date": + time = fdb_dataarray["time"][0] + time = time + "00" + new_val = [fdb_dataarray[key][0] + "T" + time] + new_dict[key] = new_val + elif key == "time": + pass + else: + new_dict[key] = values + new_dict["values"] = [0.0] + + class FDBDatacube(Datacube): def _set_mapper(self, values, name): @@ -85,7 +107,10 @@ def __init__(self, config={}, options={}, grid_options={}): partial_request = config # Find values in the level 3 FDB datacube # Will be in the form of a dictionary? {axis_name:values_available, ...} - dataarray = get_datacube_indices(partial_request) + # dataarray = get_datacube_indices(partial_request) + fdb = pyfdb.FDB() + fdb_dataarray = fdb.axes(partial_request).as_dict() + dataarray = update_fdb_dataarray(fdb_dataarray) for name, values in dataarray.items(): values.sort() From 1a194647b6b1c87097dd71289b11bda4910a7b30 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 28 Jul 2023 13:26:53 +0200 Subject: [PATCH 060/332] add FDB function to FDB datacube --- polytope/datacube/FDB_datacube.py | 3 +-- tests/test_fdb_datacube.py | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/polytope/datacube/FDB_datacube.py b/polytope/datacube/FDB_datacube.py index 0bafea977..8aa089723 100644 --- a/polytope/datacube/FDB_datacube.py +++ b/polytope/datacube/FDB_datacube.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd +import pyfdb from ..utility.combinatorics import unique, validate_axes from .datacube import Datacube, DatacubePath, IndexTree @@ -15,8 +16,6 @@ UnsliceableaAxis, ) from .mappers import OctahedralGridMap -import pyfdb - _mappings = { pd.Int64Dtype: IntAxis(), diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 065d7e093..1ecd5edba 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -8,7 +8,11 @@ class TestSlicing3DXarrayDatacube: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types grid_options = {"values": {"grid_map": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} - self.xarraydatacube = FDBDatacube(config={}, options={}, grid_options=grid_options) + config = {"class" : "od", + "expver" : "0001", + "levtype" : "pl", + "step" : 4} + self.xarraydatacube = FDBDatacube(config, options={}, grid_options=grid_options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.xarraydatacube, engine=self.slicer) From 4df967cb59582beffc3281ee164d835babe7cb35 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 27 Jul 2023 10:16:09 +0200 Subject: [PATCH 061/332] small readme fix --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 5be58455b..48dba68f6 100644 --- a/readme.md +++ b/readme.md @@ -196,7 +196,7 @@ does it submit to any jurisdiction. ## Citing -In this software is useful in your work, please consider citing our [paper](https://arxiv.org/abs/2306.11553) as +If this software is useful in your work, please consider citing our [paper](https://arxiv.org/abs/2306.11553) as > Leuridan, M., Hawkes, J., Smart, S., Danovaro, E., and Quintino, T., “Polytope: An Algorithm for Efficient Feature Extraction on Hypercubes”, arXiv e-prints, 2023. doi:10.48550/arXiv.2306.11553. From 5a793cabc37bc71c78f326224b77f819b147b74b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 31 Jul 2023 16:33:07 +0200 Subject: [PATCH 062/332] fix flake8 --- polytope/datacube/datacube_axis.py | 4 ++-- tests/test_datacube_xarray.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 7d001d0b1..838196692 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -378,7 +378,7 @@ def parse(self, value: Any) -> Any: return pd.Timestamp(value) def to_float(self, value: pd.Timestamp): - if type(value) == np.datetime64: + if isinstance(value, np.datetime64): return float((value - np.datetime64("1970-01-01T00:00:00")).astype("int")) else: return float(value.value / 10**9) @@ -419,7 +419,7 @@ def parse(self, value: Any) -> Any: return pd.Timedelta(value) def to_float(self, value: pd.Timedelta): - if type(value) == np.timedelta64: + if isinstance(value, np.timedelta64): return value.astype("timedelta64[s]").astype(int) else: return float(value.value / 10**9) diff --git a/tests/test_datacube_xarray.py b/tests/test_datacube_xarray.py index 90d5d8010..963090b50 100644 --- a/tests/test_datacube_xarray.py +++ b/tests/test_datacube_xarray.py @@ -71,14 +71,14 @@ def test_create(self): label.name = "date" idxs = datacube.get_indices(partial_request, label, pd.Timestamp("2000-01-02"), pd.Timestamp("2000-03-31")) assert (idxs == pd.date_range(pd.Timestamp("2000-01-02"), pd.Timestamp("2000-01-03"), 2)).all() - assert type(idxs[0]) == pd.Timestamp + assert isinstance(idxs[0], pd.Timestamp) # Check discretizing along 'date' axis at a specific date gives one value label = PandasTimestampAxis() label.name = "date" idxs = datacube.get_indices(partial_request, label, pd.Timestamp("2000-01-02"), pd.Timestamp("2000-01-02")) assert len(idxs) == 1 - assert type(idxs[0]) == pd.Timestamp + assert isinstance(idxs[0], pd.Timestamp) assert idxs[0] == pd.Timestamp(pd.Timestamp("2000-01-02")) # Check discretizing along 'date' axis at a date which does not exist in discrete space gives no values @@ -101,13 +101,13 @@ def test_create(self): label.name = "step" idxs = datacube.get_indices(partial_request, label, 0, 10) assert idxs == [0, 3, 6, 9] - assert type(idxs[0]) == int + assert isinstance(idxs[0], int) # Check discretizing along 'step' axis at a specific step gives one value idxs = datacube.get_indices(partial_request, label, 3, 3) assert len(idxs) == 1 assert idxs[0] == 3 - assert type(idxs[0]) == int + assert isinstance(idxs[0], int) # Check discretizing along 'step' axis at a step which does not exist in discrete space gives no values idxs = datacube.get_indices(partial_request, label, 4, 4) @@ -125,4 +125,4 @@ def test_create(self): label.name = "level" idxs = datacube.get_indices(partial_request, label, -0.3, 10) assert idxs == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - assert type(idxs[0]) == int + assert isinstance(idxs[0], int) From d126ff21219c326f4fa39a921f1d1942ac145b2b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 25 Jul 2023 16:16:24 +0200 Subject: [PATCH 063/332] move methods and rename methods and files --- polytope/datacube/datacube.py | 2 +- polytope/datacube/datacube_axis.py | 87 ++++++++++++-- .../{mappers.py => datacube_mappers.py} | 12 +- ...datacube_request_tree.py => index_tree.py} | 4 +- polytope/datacube/mock.py | 4 +- polytope/datacube/xarray.py | 90 ++------------ polytope/engine/engine.py | 2 +- polytope/engine/hullslicer.py | 6 +- tests/test_axis_mappers.py | 20 ++-- tests/test_datacube_xarray.py | 16 ++- tests/test_hullslicer_engine.py | 2 +- tests/test_request_tree.py | 110 +++++++++--------- tests/test_request_trees_after_slicing.py | 10 +- tests/test_slicer_engine.py | 2 +- tests/test_slicing_xarray_3D.py | 2 +- tests/test_slicing_xarray_4D.py | 2 +- 16 files changed, 186 insertions(+), 185 deletions(-) rename polytope/datacube/{mappers.py => datacube_mappers.py} (99%) rename polytope/datacube/{datacube_request_tree.py => index_tree.py} (98%) diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index 2d10ac728..b7a7a17ac 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -4,7 +4,7 @@ import xarray as xr from .datacube_axis import DatacubeAxis -from .datacube_request_tree import DatacubePath, IndexTree +from .index_tree import DatacubePath, IndexTree class Datacube(ABC): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 838196692..412bda9c0 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -1,10 +1,14 @@ +import sys from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy +from importlib import import_module from typing import Any, List import numpy as np import pandas as pd +from .datacube_mappers import _type_to_datacube_mapper_lookup + class DatacubeAxis(ABC): @abstractproperty @@ -55,8 +59,62 @@ def to_cyclic_value(self, value: Any) -> Any: def offset(self, value: Any) -> int: pass - -class IntAxis(DatacubeAxis): + @staticmethod + def create_axis(options, name, values, datacube): + if options == {}: + DatacubeAxis.create_standard(name, values, datacube) + if "mapper" in options.keys(): + DatacubeAxis.create_mapper(options, name, datacube) + if "Cyclic" in options.keys(): + DatacubeAxis.create_cyclic(options, name, values, datacube) + + @staticmethod + def create_cyclic(options, name, values, datacube): + value_type = values.dtype.type + axes_type_str = type(_type_to_axis_lookup[value_type]).__name__ + axes_type_str += "Cyclic" + cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) + datacube._axes[name] = cyclic_axis_type + datacube._axes[name].name = name + datacube._axes[name].range = options["Cyclic"] + datacube.axis_counter += 1 + + @staticmethod + def create_standard(name, values, datacube): + DatacubeAxis.check_axis_type(name, values) + datacube._axes[name] = deepcopy(_type_to_axis_lookup[values.dtype.type]) + datacube._axes[name].name = name + datacube.axis_counter += 1 + + @staticmethod + def check_axis_type(name, values): + if values.dtype.type not in _type_to_axis_lookup: + raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") + + @staticmethod + def create_mapper(options, name, datacube): + grid_mapping_options = options["mapper"] + grid_type = grid_mapping_options["type"] + grid_resolution = grid_mapping_options["resolution"] + grid_axes = grid_mapping_options["axes"] + map_type = _type_to_datacube_mapper_lookup[grid_type] + module = import_module("polytope.datacube.datacube_mappers") + constructor = getattr(module, map_type) + datacube.grid_mapper = constructor(name, grid_axes, grid_resolution) + # Once we have created mapper, create axis for the mapped axes + for i in range(len(grid_axes)): + axis_name = grid_axes[i] + new_axis_options = datacube.axis_options.get(axis_name, {}) + if i == 0: + values = np.array(datacube.grid_mapper.first_axis_vals()) + DatacubeAxis.create_axis(new_axis_options, axis_name, values, datacube) + if i == 1: + # the values[0] will be a value on the first axis + values = np.array(datacube.grid_mapper.second_axis_vals(values[0])) + DatacubeAxis.create_axis(new_axis_options, axis_name, values, datacube) + + +class IntDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 range = None @@ -92,7 +150,7 @@ def offset(self, value): return 0 -class IntAxisCyclic(DatacubeAxis): +class IntDatacubeAxisCyclic(DatacubeAxis): name = None tol = 1e-12 range = None @@ -210,7 +268,7 @@ def offset(self, range): return offset -class FloatAxis(DatacubeAxis): +class FloatDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 range = None @@ -246,7 +304,7 @@ def offset(self, value): return 0 -class FloatAxisCyclic(DatacubeAxis): +class FloatDatacubeAxisCyclic(DatacubeAxis): # Note that in the cyclic axis here, we only retain the lower part when we remap # so for eg if the datacube has cyclic axis on [0,360] # then if we want 360, we will in reality get back 0 (which is the same) @@ -367,7 +425,7 @@ def offset(self, range): return offset -class PandasTimestampAxis(DatacubeAxis): +class PandasTimestampDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 range = None @@ -408,7 +466,7 @@ def offset(self, value): return None -class PandasTimedeltaAxis(DatacubeAxis): +class PandasTimedeltaDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 range = None @@ -449,7 +507,7 @@ def offset(self, value): return None -class UnsliceableaAxis(DatacubeAxis): +class UnsliceableDatacubeAxis(DatacubeAxis): name = None tol = float("NaN") range = None @@ -468,3 +526,16 @@ def serialize(self, value): def remap_val_to_axis_range(self, value): return value + + +_type_to_axis_lookup = { + pd.Int64Dtype: IntDatacubeAxis(), + pd.Timestamp: PandasTimestampDatacubeAxis(), + np.int64: IntDatacubeAxis(), + np.datetime64: PandasTimestampDatacubeAxis(), + np.timedelta64: PandasTimedeltaDatacubeAxis(), + np.float64: FloatDatacubeAxis(), + np.str_: UnsliceableDatacubeAxis(), + str: UnsliceableDatacubeAxis(), + np.object_: UnsliceableDatacubeAxis(), +} diff --git a/polytope/datacube/mappers.py b/polytope/datacube/datacube_mappers.py similarity index 99% rename from polytope/datacube/mappers.py rename to polytope/datacube/datacube_mappers.py index 5bd651d83..0ece04c98 100644 --- a/polytope/datacube/mappers.py +++ b/polytope/datacube/datacube_mappers.py @@ -1,17 +1,14 @@ import math -from abc import ABC, abstractmethod, abstractproperty +from abc import ABC, abstractmethod -class GridMappers(ABC): - @abstractproperty +class DatacubeMapper(ABC): def _mapped_axes(self): pass - @abstractproperty def _base_axis(self): pass - @abstractproperty def _resolution(self): pass @@ -28,7 +25,7 @@ def unmap(self): pass -class OctahedralGridMap(ABC): +class OctahedralGridMapper(DatacubeMapper): def __init__(self, base_axis, mapped_axes, resolution): self._mapped_axes = mapped_axes self._base_axis = base_axis @@ -2742,3 +2739,6 @@ def unmap(self, first_val, second_val): second_idx = second_axis_vals.index(second_val) octahedral_index = self.axes_idx_to_octahedral_idx(first_idx, second_idx) return octahedral_index + + +_type_to_datacube_mapper_lookup = {"octahedral": "OctahedralGridMapper"} diff --git a/polytope/datacube/datacube_request_tree.py b/polytope/datacube/index_tree.py similarity index 98% rename from polytope/datacube/datacube_request_tree.py rename to polytope/datacube/index_tree.py index 3f659388c..9833fd240 100644 --- a/polytope/datacube/datacube_request_tree.py +++ b/polytope/datacube/index_tree.py @@ -3,7 +3,7 @@ from sortedcontainers import SortedList -from .datacube_axis import IntAxis +from .datacube_axis import IntDatacubeAxis class DatacubePath(OrderedDict): @@ -21,7 +21,7 @@ def pprint(self): class IndexTree(object): - root = IntAxis() + root = IntDatacubeAxis() root.name = "root" def __init__(self, axis=root, value=None): diff --git a/polytope/datacube/mock.py b/polytope/datacube/mock.py index b5fc1b1f6..c31fd9802 100644 --- a/polytope/datacube/mock.py +++ b/polytope/datacube/mock.py @@ -3,7 +3,7 @@ from ..utility.combinatorics import validate_axes from .datacube import Datacube, DatacubePath, IndexTree -from .datacube_axis import IntAxis +from .datacube_axis import IntDatacubeAxis class MockDatacube(Datacube): @@ -14,7 +14,7 @@ def __init__(self, dimensions): self.mappers = {} for name in self.dimensions: - self.mappers[name] = deepcopy(IntAxis()) + self.mappers[name] = deepcopy(IntDatacubeAxis()) self.mappers[name].name = name self.stride = {} diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index 1978913da..bf116c0c4 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -1,35 +1,10 @@ import math -import sys -from copy import deepcopy -from importlib import import_module -import numpy as np -import pandas as pd import xarray as xr from ..utility.combinatorics import unique, validate_axes from .datacube import Datacube, DatacubePath, IndexTree -from .datacube_axis import ( - FloatAxis, - IntAxis, - PandasTimedeltaAxis, - PandasTimestampAxis, - UnsliceableaAxis, -) - -_mappings = { - pd.Int64Dtype: IntAxis(), - pd.Timestamp: PandasTimestampAxis(), - np.int64: IntAxis(), - np.datetime64: PandasTimestampAxis(), - np.timedelta64: PandasTimedeltaAxis(), - np.float64: FloatAxis(), - np.str_: UnsliceableaAxis(), - str: UnsliceableaAxis(), - np.object_: UnsliceableaAxis(), -} - -_grid_mappings = {"octahedral": "OctahedralGridMap"} +from .datacube_axis import DatacubeAxis class XArrayDatacube(Datacube): @@ -39,7 +14,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.axis_options = axis_options self.grid_mapper = None self.axis_counter = 0 - self.mappers = {} + self._axes = {} self.dataarray = dataarray treated_axes = [] self.complete_axes = [] @@ -47,68 +22,19 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): if name in dataarray.dims: self.dataarray = self.dataarray.sortby(name) options = axis_options.get(name, {}) - self.create_axis(options, name, values) + DatacubeAxis.create_axis(options, name, values, self) treated_axes.append(name) self.complete_axes.append(name) else: if self.dataarray[name].dims == (): options = axis_options.get(name, {}) - self.create_axis(options, name, values) + DatacubeAxis.create_axis(options, name, values, self) treated_axes.append(name) for name in dataarray.dims: if name not in treated_axes: options = axis_options.get(name, {}) val = dataarray[name].values[0] - self.create_axis(options, name, val) - - def create_axis(self, options, name, values): - if options == {}: - self.create_standard(name, values) - if "mapper" in options.keys(): - self.create_mapper(options, name) - if "Cyclic" in options.keys(): - self.create_cyclic(options, name, values) - - def create_cyclic(self, options, name, values): - value_type = values.dtype.type - axes_type_str = type(_mappings[value_type]).__name__ - axes_type_str += "Cyclic" - cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) - self.mappers[name] = cyclic_axis_type - self.mappers[name].name = name - self.mappers[name].range = options["Cyclic"] - self.axis_counter += 1 - - def create_mapper(self, options, name): - grid_mapping_options = options["mapper"] - grid_type = grid_mapping_options["type"] - grid_resolution = grid_mapping_options["resolution"] - grid_axes = grid_mapping_options["axes"] - map_type = _grid_mappings[grid_type] - module = import_module("polytope.datacube.mappers") - constructor = getattr(module, map_type) - self.grid_mapper = constructor(name, grid_axes, grid_resolution) - # Once we have created mapper, create axis for the mapped axes - for i in range(len(grid_axes)): - axis_name = grid_axes[i] - new_axis_options = self.axis_options.get(axis_name, {}) - if i == 0: - values = np.array(self.grid_mapper.first_axis_vals()) - self.create_axis(new_axis_options, axis_name, values) - if i == 1: - # the values[0] will be a value on the first axis - values = np.array(self.grid_mapper.second_axis_vals(values[0])) - self.create_axis(new_axis_options, axis_name, values) - - def create_standard(self, name, values): - self.check_axis_type(name, values) - self.mappers[name] = deepcopy(_mappings[values.dtype.type]) - self.mappers[name].name = name - self.axis_counter += 1 - - def check_axis_type(self, name, values): - if values.dtype.type not in _mappings: - raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") + DatacubeAxis.create_axis(options, name, val, self) def get(self, requests: IndexTree): for r in requests.leaves: @@ -142,12 +68,12 @@ def get(self, requests: IndexTree): r.remove_branch() def get_mapper(self, axis): - return self.mappers[axis] + return self._axes[axis] def remap_path(self, path: DatacubePath): for key in path: value = path[key] - path[key] = self.mappers[key].remap_val_to_axis_range(value) + path[key] = self._axes[key].remap_val_to_axis_range(value) return path def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): @@ -274,7 +200,7 @@ def has_index(self, path: DatacubePath, axis, index): @property def axes(self): - return self.mappers + return self._axes def validate(self, axes): return validate_axes(list(self.axes.keys()), axes) diff --git a/polytope/engine/engine.py b/polytope/engine/engine.py index f9eeb7047..e32d130bc 100644 --- a/polytope/engine/engine.py +++ b/polytope/engine/engine.py @@ -1,7 +1,7 @@ from typing import List from ..datacube.datacube import Datacube -from ..datacube.datacube_request_tree import IndexTree +from ..datacube.index_tree import IndexTree from ..shapes import ConvexPolytope diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 161ed5b5f..646d31c74 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -5,7 +5,7 @@ import scipy.spatial from ..datacube.datacube import Datacube, IndexTree -from ..datacube.datacube_axis import UnsliceableaAxis +from ..datacube.datacube_axis import UnsliceableDatacubeAxis from ..shapes import ConvexPolytope from ..utility.combinatorics import argmax, argmin, group, product, unique from ..utility.exceptions import UnsliceableShapeError @@ -20,7 +20,7 @@ def __init__(self): def _unique_continuous_points(self, p: ConvexPolytope, datacube: Datacube): for i, ax in enumerate(p.axes()): mapper = datacube.get_mapper(ax) - if isinstance(mapper, UnsliceableaAxis): + if isinstance(mapper, UnsliceableDatacubeAxis): break for j, val in enumerate(p.points): p.points[j] = list(p.points[j]) @@ -63,7 +63,7 @@ def _build_branch(self, ax, node, datacube, next_nodes): if ax.name in polytope.axes(): lower, upper = polytope.extents(ax.name) # here, first check if the axis is an unsliceable axis and directly build node if it is - if isinstance(ax, UnsliceableaAxis): + if isinstance(ax, UnsliceableDatacubeAxis): self._build_unsliceable_child(polytope, ax, node, datacube, lower, next_nodes) else: self._build_sliceable_child(polytope, ax, node, datacube, lower, upper, next_nodes) diff --git a/tests/test_axis_mappers.py b/tests/test_axis_mappers.py index cf5e94bd2..3f03f06e0 100644 --- a/tests/test_axis_mappers.py +++ b/tests/test_axis_mappers.py @@ -1,17 +1,17 @@ import pandas as pd from polytope.datacube.datacube_axis import ( - FloatAxis, - FloatAxisCyclic, - IntAxis, - PandasTimedeltaAxis, - PandasTimestampAxis, + FloatDatacubeAxis, + FloatDatacubeAxisCyclic, + IntDatacubeAxis, + PandasTimedeltaDatacubeAxis, + PandasTimestampDatacubeAxis, ) class TestAxisMappers: def test_int_axis(self): - axis = IntAxis() + axis = IntDatacubeAxis() assert axis.parse(2) == 2.0 assert axis.to_float(2) == 2.0 assert axis.from_float(2) == 2.0 @@ -19,7 +19,7 @@ def test_int_axis(self): assert axis.remap([2, 3]) == [[2, 3]] def test_float_axis(self): - axis = FloatAxis() + axis = FloatDatacubeAxis() assert axis.parse(2) == 2.0 assert axis.to_float(2) == 2.0 assert axis.from_float(2) == 2.0 @@ -27,7 +27,7 @@ def test_float_axis(self): assert axis.remap([2, 3]) == [[2, 3]] def test_float_axis_cyclic(self): - axis = FloatAxisCyclic() + axis = FloatDatacubeAxisCyclic() assert axis.parse(2) == 2.0 assert axis.to_float(2) == 2.0 assert axis.from_float(2) == 2.0 @@ -76,7 +76,7 @@ def test_float_axis_cyclic(self): assert axis.offset([5.05, 5.1]) == 4 def test_timedelta_axis(self): - axis = PandasTimedeltaAxis() + axis = PandasTimedeltaDatacubeAxis() time1 = pd.Timedelta("1 days") time2 = pd.Timedelta("1 days 2 hours") assert axis.parse(time1) == pd.Timedelta("1 days 00:00:00") @@ -86,7 +86,7 @@ def test_timedelta_axis(self): assert axis.remap([time1, time2]) == [[pd.Timedelta("1 days 00:00:00"), pd.Timedelta("1 days 02:00:00")]] def test_timestamp_axis(self): - axis = PandasTimestampAxis() + axis = PandasTimestampDatacubeAxis() time1 = pd.Timestamp("2017-01-01 11:00:00") time2 = pd.Timestamp("2017-01-01 12:00:00") assert axis.parse(time1) == pd.Timestamp("2017-01-01 11:00:00") diff --git a/tests/test_datacube_xarray.py b/tests/test_datacube_xarray.py index 963090b50..0d6463c9b 100644 --- a/tests/test_datacube_xarray.py +++ b/tests/test_datacube_xarray.py @@ -4,7 +4,11 @@ import xarray as xr from polytope.datacube import Datacube, DatacubePath -from polytope.datacube.datacube_axis import FloatAxis, IntAxis, PandasTimestampAxis +from polytope.datacube.datacube_axis import ( + FloatDatacubeAxis, + IntDatacubeAxis, + PandasTimestampDatacubeAxis, +) from polytope.datacube.xarray import XArrayDatacube from polytope.utility.exceptions import AxisNotFoundError, AxisOverdefinedError @@ -67,14 +71,14 @@ def test_create(self): assert parsed == from_float # Check discretizing along 'date' axis with a range of dates - label = PandasTimestampAxis() + label = PandasTimestampDatacubeAxis() label.name = "date" idxs = datacube.get_indices(partial_request, label, pd.Timestamp("2000-01-02"), pd.Timestamp("2000-03-31")) assert (idxs == pd.date_range(pd.Timestamp("2000-01-02"), pd.Timestamp("2000-01-03"), 2)).all() assert isinstance(idxs[0], pd.Timestamp) # Check discretizing along 'date' axis at a specific date gives one value - label = PandasTimestampAxis() + label = PandasTimestampDatacubeAxis() label.name = "date" idxs = datacube.get_indices(partial_request, label, pd.Timestamp("2000-01-02"), pd.Timestamp("2000-01-02")) assert len(idxs) == 1 @@ -82,7 +86,7 @@ def test_create(self): assert idxs[0] == pd.Timestamp(pd.Timestamp("2000-01-02")) # Check discretizing along 'date' axis at a date which does not exist in discrete space gives no values - label = PandasTimestampAxis() + label = PandasTimestampDatacubeAxis() label.name = "date" idxs = datacube.get_indices( partial_request, label, pd.Timestamp("2000-01-01-1200"), pd.Timestamp("2000-01-01-1200") @@ -97,7 +101,7 @@ def test_create(self): assert type(datacube.get_mapper("step").parse(3.0)) == float # Check discretizing along 'step' axis with a range of steps - label = IntAxis() + label = IntDatacubeAxis() label.name = "step" idxs = datacube.get_indices(partial_request, label, 0, 10) assert idxs == [0, 3, 6, 9] @@ -121,7 +125,7 @@ def test_create(self): assert type(datacube.get_mapper("level").parse(3.0)) == float # Check discretizing along 'level' axis with a range of levels - label = FloatAxis() + label = FloatDatacubeAxis() label.name = "level" idxs = datacube.get_indices(partial_request, label, -0.3, 10) assert idxs == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] diff --git a/tests/test_hullslicer_engine.py b/tests/test_hullslicer_engine.py index e0be7e004..1fe61f6df 100644 --- a/tests/test_hullslicer_engine.py +++ b/tests/test_hullslicer_engine.py @@ -1,7 +1,7 @@ import numpy as np import xarray as xr -from polytope.datacube.datacube_request_tree import IndexTree +from polytope.datacube.index_tree import IndexTree from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope diff --git a/tests/test_request_tree.py b/tests/test_request_tree.py index d2264ff47..030a97ab1 100644 --- a/tests/test_request_tree.py +++ b/tests/test_request_tree.py @@ -1,7 +1,7 @@ from sortedcontainers import SortedList -from polytope.datacube.datacube_axis import IntAxis -from polytope.datacube.datacube_request_tree import IndexTree +from polytope.datacube.datacube_axis import IntDatacubeAxis +from polytope.datacube.index_tree import IndexTree class TestIndexTree: @@ -9,9 +9,9 @@ def setup_method(self, method): pass def test_init(self): - axis1 = IntAxis() - axis2 = IntAxis() - axis3 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() + axis3 = IntDatacubeAxis() axis1.name = "grandchild1" axis2.name = "child1" axis3.name = "child2" @@ -25,8 +25,8 @@ def test_init(self): assert child1.children == SortedList([grandchild1]) def test_add_child(self): - axis1 = IntAxis() - axis2 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() axis1.name = "grandchild" axis2.name = "child" root_node = IndexTree() @@ -41,7 +41,7 @@ def test_add_child(self): assert SortedList([grandchild]) in [c.children for c in root_node.children] def test_get_parent(self): - axis1 = IntAxis() + axis1 = IntDatacubeAxis() axis1.name = "child" child = IndexTree(axis=axis1) root_node = IndexTree() @@ -49,8 +49,8 @@ def test_get_parent(self): assert child.parent == root_node def test_find_child(self): - axis1 = IntAxis() - axis2 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() axis1.name = "child1" axis2.name = "child2" child1 = IndexTree(axis=axis1) @@ -61,9 +61,9 @@ def test_find_child(self): assert child2 not in root_node.children def test_get_root(self): - axis1 = IntAxis() - axis2 = IntAxis() - axis3 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() + axis3 = IntDatacubeAxis() axis1.name = "grandchild1" axis2.name = "child1" axis3.name = "child2" @@ -78,12 +78,12 @@ def test_get_root(self): assert grandparent == root_node def test_merge(self): - axis1 = IntAxis() - axis2 = IntAxis() - axis3 = IntAxis() - axis4 = IntAxis() - axis5 = IntAxis() - axis6 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() + axis3 = IntDatacubeAxis() + axis4 = IntDatacubeAxis() + axis5 = IntDatacubeAxis() + axis6 = IntDatacubeAxis() axis1.name = "grandchild1" axis2.name = "grandchild2" axis3.name = "grandchild3" @@ -106,12 +106,12 @@ def test_merge(self): root_node1.add_child(child2_1) root_node1.pprint() - axis1 = IntAxis() - axis2 = IntAxis() - axis3 = IntAxis() - axis4 = IntAxis() - axis5 = IntAxis() - axis6 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() + axis3 = IntDatacubeAxis() + axis4 = IntDatacubeAxis() + axis5 = IntDatacubeAxis() + axis6 = IntDatacubeAxis() axis1.name = "grandchild1" axis2.name = "grandchild5" axis3.name = "grandchild6" @@ -141,9 +141,9 @@ def test_merge(self): assert set([len(child.children) for child in root_node1.children]) == {2, 2, 3} def test_pprint(self): - axis1 = IntAxis() - axis2 = IntAxis() - axis3 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() + axis3 = IntDatacubeAxis() axis1.name = "grandchild1" axis2.name = "child1" axis3.name = "child2" @@ -157,9 +157,9 @@ def test_pprint(self): root_node.pprint() def test_remove_branch(self): - axis1 = IntAxis() - axis2 = IntAxis() - axis3 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() + axis3 = IntDatacubeAxis() axis1.name = "grandchild1" axis2.name = "child1" axis3.name = "child2" @@ -178,12 +178,12 @@ def test_remove_branch(self): assert root_node.children == SortedList([child2]) def test_intersect(self): - axis1 = IntAxis() - axis2 = IntAxis() - axis3 = IntAxis() - axis4 = IntAxis() - axis5 = IntAxis() - axis6 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() + axis3 = IntDatacubeAxis() + axis4 = IntDatacubeAxis() + axis5 = IntDatacubeAxis() + axis6 = IntDatacubeAxis() axis1.name = "grandchild1" axis2.name = "grandchild2" axis3.name = "grandchild3" @@ -204,12 +204,12 @@ def test_intersect(self): root_node1.add_child(child1_1) root_node1.add_child(child2_1) - axis1 = IntAxis() - axis2 = IntAxis() - axis3 = IntAxis() - axis4 = IntAxis() - axis5 = IntAxis() - axis6 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() + axis3 = IntDatacubeAxis() + axis4 = IntDatacubeAxis() + axis5 = IntDatacubeAxis() + axis6 = IntDatacubeAxis() axis1.name = "grandchild1" axis2.name = "grandchild5" axis3.name = "grandchild6" @@ -241,9 +241,9 @@ def test_intersect(self): assert list(list(root_node1.children)[0].children)[0].axis.name == "grandchild1" def test_flatten(self): - axis1 = IntAxis() - axis2 = IntAxis() - axis3 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() + axis3 = IntDatacubeAxis() axis1.name = "grandchild1" axis2.name = "child1" axis3.name = "child2" @@ -261,9 +261,9 @@ def test_flatten(self): assert path["grandchild1"] is None def test_get_ancestors(self): - axis1 = IntAxis() - axis2 = IntAxis() - axis3 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() + axis3 = IntDatacubeAxis() axis1.name = "greatgrandchild1" axis2.name = "grandchild1" axis3.name = "child1" @@ -277,9 +277,9 @@ def test_get_ancestors(self): assert greatgrandchild1.get_ancestors() == SortedList([greatgrandchild1, grandchild1, child1]) def test_add_or_get_child(self): - axis1 = IntAxis() + axis1 = IntDatacubeAxis() axis1.name = "child1" - axis2 = IntAxis() + axis2 = IntDatacubeAxis() axis2.name = "child2" child1 = IndexTree(axis=axis1) root_node = IndexTree() @@ -288,10 +288,10 @@ def test_add_or_get_child(self): assert root_node.create_child(axis2, None).parent == root_node def test_to_dict(self): - axis1 = IntAxis() - axis2 = IntAxis() - axis3 = IntAxis() - axis4 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() + axis3 = IntDatacubeAxis() + axis4 = IntDatacubeAxis() axis1.name = "greatgrandchild1" axis2.name = "grandchild1" axis3.name = "child1" diff --git a/tests/test_request_trees_after_slicing.py b/tests/test_request_trees_after_slicing.py index b9e90cd1d..061324ff1 100644 --- a/tests/test_request_trees_after_slicing.py +++ b/tests/test_request_trees_after_slicing.py @@ -1,7 +1,7 @@ import numpy as np import xarray as xr -from polytope.datacube.datacube_axis import IntAxis +from polytope.datacube.datacube_axis import IntDatacubeAxis from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope @@ -61,12 +61,12 @@ def test_add_child(self): request1 = request.leaves[0] request2 = request.leaves[0] # Test adding child - axis1 = IntAxis() + axis1 = IntDatacubeAxis() axis1.name = "lat" request2.create_child(axis1, 4.1) assert request2.leaves[0].axis.name == "lat" assert request2.leaves[0].value == 4.1 - axis2 = IntAxis() + axis2 = IntDatacubeAxis() axis2.name = "level" # Test getting child assert request1.create_child(axis2, 3.0).axis.name == "level" @@ -86,8 +86,8 @@ def test_remove_branch(self): request.leaves[0].remove_branch() new_request_size = len(request.leaves) assert prev_request_size == new_request_size + 1 - axis1 = IntAxis() - axis2 = IntAxis() + axis1 = IntDatacubeAxis() + axis2 = IntDatacubeAxis() axis1.name = "step" axis2.name = "level" # Test if remove_branch() also removes longer branches diff --git a/tests/test_slicer_engine.py b/tests/test_slicer_engine.py index e0be7e004..1fe61f6df 100644 --- a/tests/test_slicer_engine.py +++ b/tests/test_slicer_engine.py @@ -1,7 +1,7 @@ import numpy as np import xarray as xr -from polytope.datacube.datacube_request_tree import IndexTree +from polytope.datacube.index_tree import IndexTree from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope diff --git a/tests/test_slicing_xarray_3D.py b/tests/test_slicing_xarray_3D.py index a90943e36..e036321b2 100644 --- a/tests/test_slicing_xarray_3D.py +++ b/tests/test_slicing_xarray_3D.py @@ -5,7 +5,7 @@ import pandas as pd import xarray as xr -from polytope.datacube.datacube_request_tree import IndexTree +from polytope.datacube.index_tree import IndexTree from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request diff --git a/tests/test_slicing_xarray_4D.py b/tests/test_slicing_xarray_4D.py index 449e5838f..971823aeb 100644 --- a/tests/test_slicing_xarray_4D.py +++ b/tests/test_slicing_xarray_4D.py @@ -3,7 +3,7 @@ import pytest import xarray as xr -from polytope.datacube.datacube_request_tree import IndexTree +from polytope.datacube.index_tree import IndexTree from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request From 73d1079230523427204b0d975bbdbf2e827ca4cb Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 26 Jul 2023 17:04:35 +0200 Subject: [PATCH 064/332] add tests for mapper --- tests/test_mappers.py | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/test_mappers.py diff --git a/tests/test_mappers.py b/tests/test_mappers.py new file mode 100644 index 000000000..b1b4f4d6d --- /dev/null +++ b/tests/test_mappers.py @@ -0,0 +1,46 @@ +from polytope.datacube.datacube_mappers import OctahedralGridMapper + + +class TestMapper: + def test_octahedral_mapper_init(self): + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 1280 + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) + assert octahedral_mapper._base_axis == base_axis + assert octahedral_mapper._mapped_axes == mapped_axes + assert octahedral_mapper._resolution == resolution + + def test_first_axis_vals_01280_grid(self): + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 1280 + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) + assert octahedral_mapper.first_axis_vals()[:5] == [89.94618771566562, 89.87647835333229, 89.80635731954224, + 89.73614327160958, 89.6658939412157] + assert len(octahedral_mapper.first_axis_vals()) == 1280*2 + + def test_first_axis_vals_other_grid(self): + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 640 + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) + assert octahedral_mapper.first_axis_vals()[:5] == [89.89239644559007, 89.75300494317403, 89.61279025859908, + 89.47238958206113, 89.33191835438183] + assert len(octahedral_mapper.first_axis_vals()) == 640*2 + + def test_map_first_axis(self): + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 1280 + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) + assert octahedral_mapper.map_first_axis(89.7, 89.96) == [89.94618771566562, 89.87647835333229, + 89.80635731954224, 89.73614327160958] + + def test_second_axis_vals(self): + mapped_axes = ["lat", "lon"] + base_axis = "base" + resolution = 1280 + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) + pass + From ca87650d6e658b821a19d63d8cc0c2b2bec043fb Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 26 Jul 2023 17:09:25 +0200 Subject: [PATCH 065/332] remove tests --- tests/test_mappers.py | 46 ------------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 tests/test_mappers.py diff --git a/tests/test_mappers.py b/tests/test_mappers.py deleted file mode 100644 index b1b4f4d6d..000000000 --- a/tests/test_mappers.py +++ /dev/null @@ -1,46 +0,0 @@ -from polytope.datacube.datacube_mappers import OctahedralGridMapper - - -class TestMapper: - def test_octahedral_mapper_init(self): - mapped_axes = ["lat", "lon"] - base_axis = "base" - resolution = 1280 - octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) - assert octahedral_mapper._base_axis == base_axis - assert octahedral_mapper._mapped_axes == mapped_axes - assert octahedral_mapper._resolution == resolution - - def test_first_axis_vals_01280_grid(self): - mapped_axes = ["lat", "lon"] - base_axis = "base" - resolution = 1280 - octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) - assert octahedral_mapper.first_axis_vals()[:5] == [89.94618771566562, 89.87647835333229, 89.80635731954224, - 89.73614327160958, 89.6658939412157] - assert len(octahedral_mapper.first_axis_vals()) == 1280*2 - - def test_first_axis_vals_other_grid(self): - mapped_axes = ["lat", "lon"] - base_axis = "base" - resolution = 640 - octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) - assert octahedral_mapper.first_axis_vals()[:5] == [89.89239644559007, 89.75300494317403, 89.61279025859908, - 89.47238958206113, 89.33191835438183] - assert len(octahedral_mapper.first_axis_vals()) == 640*2 - - def test_map_first_axis(self): - mapped_axes = ["lat", "lon"] - base_axis = "base" - resolution = 1280 - octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) - assert octahedral_mapper.map_first_axis(89.7, 89.96) == [89.94618771566562, 89.87647835333229, - 89.80635731954224, 89.73614327160958] - - def test_second_axis_vals(self): - mapped_axes = ["lat", "lon"] - base_axis = "base" - resolution = 1280 - octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) - pass - From 6bdd3cd4f5aa88bf14cd00acdffe3dac2dbba532 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 1 Aug 2023 12:55:55 +0200 Subject: [PATCH 066/332] fix cyclic import --- examples/4D_flight_path.py | 2 +- examples/country_slicing.py | 2 +- examples/cyclic_route_around_earth.py | 2 +- examples/slicing_all_ecmwf_countries.py | 2 +- examples/wind_farms.py | 2 +- polytope/datacube/datacube.py | 10 +++++ polytope/datacube/datacube_axis.py | 43 ++++---------------- polytope/datacube/datacube_mappers.py | 27 ++++++++++++ polytope/datacube/xarray.py | 9 ++-- tests/test_cyclic_axis_over_negative_vals.py | 2 +- tests/test_cyclic_axis_slicer_not_0.py | 2 +- tests/test_cyclic_axis_slicing.py | 2 +- 12 files changed, 58 insertions(+), 47 deletions(-) diff --git a/examples/4D_flight_path.py b/examples/4D_flight_path.py index 1315a37fc..b75100fd0 100644 --- a/examples/4D_flight_path.py +++ b/examples/4D_flight_path.py @@ -15,7 +15,7 @@ def setup_method(self): ds = data.from_source("file", "./examples/data/temp_model_levels.grib") array = ds.to_xarray() array = array.isel(time=0).t - axis_options = {"longitude": {"Cyclic": [0, 360.0]}} + axis_options = {"longitude": {"cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) for dim in array.dims: array = array.sortby(dim) diff --git a/examples/country_slicing.py b/examples/country_slicing.py index 861668b59..a2dd197b4 100644 --- a/examples/country_slicing.py +++ b/examples/country_slicing.py @@ -15,7 +15,7 @@ def setup_method(self, method): ds = data.from_source("file", "./examples/data/output8.grib") array = ds.to_xarray() array = array.isel(surface=0).isel(step=0).isel(number=0).isel(time=0).t2m - axis_options = {"longitude": {"Cyclic": [0, 360.0]}} + axis_options = {"longitude": {"cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=axis_options) diff --git a/examples/cyclic_route_around_earth.py b/examples/cyclic_route_around_earth.py index 1a5295c88..da94a43e2 100644 --- a/examples/cyclic_route_around_earth.py +++ b/examples/cyclic_route_around_earth.py @@ -14,7 +14,7 @@ def setup_method(self, method): ds = data.from_source("file", "./examples/data/output8.grib") array = ds.to_xarray() array = array.isel(surface=0).isel(step=0).isel(number=0).isel(time=0).t2m - axis_options = {"longitude": {"Cyclic": [0, 360.0]}} + axis_options = {"longitude": {"cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=axis_options) diff --git a/examples/slicing_all_ecmwf_countries.py b/examples/slicing_all_ecmwf_countries.py index bdba272cb..e02b6deab 100644 --- a/examples/slicing_all_ecmwf_countries.py +++ b/examples/slicing_all_ecmwf_countries.py @@ -15,7 +15,7 @@ def setup_method(self, method): ds = data.from_source("file", "./examples/data/output8.grib") array = ds.to_xarray() array = array.isel(surface=0).isel(step=0).isel(number=0).isel(time=0).t2m - axis_options = {"longitude": {"Cyclic": [0, 360.0]}} + axis_options = {"longitude": {"cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=axis_options) diff --git a/examples/wind_farms.py b/examples/wind_farms.py index bcb0ed4a7..14091c2c0 100644 --- a/examples/wind_farms.py +++ b/examples/wind_farms.py @@ -18,7 +18,7 @@ def setup_method(self): array = ds.to_xarray() array = array.isel(time=0).isel(surface=0).isel(number=0).u10 self.array = array - axis_options = {"longitude": {"Cyclic": [0, 360.0]}} + axis_options = {"longitude": {"cyclic": [0, 360.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=axis_options) diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index b7a7a17ac..938ae5892 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -47,3 +47,13 @@ def create(datacube, axis_options: dict): xadatacube = XArrayDatacube(datacube, axis_options=axis_options) return xadatacube + + +def configure_datacube_axis(options, name, values, datacube): + if options == {}: + DatacubeAxis.create_standard(name, values, datacube) + if "mapper" in options.keys(): + from .datacube_mappers import DatacubeMapper + DatacubeMapper.create_mapper(options, name, datacube) + if "cyclic" in options.keys(): + DatacubeAxis.create_cyclic(options, name, values, datacube) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 412bda9c0..68ecf6599 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -1,14 +1,11 @@ import sys from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy -from importlib import import_module from typing import Any, List import numpy as np import pandas as pd -from .datacube_mappers import _type_to_datacube_mapper_lookup - class DatacubeAxis(ABC): @abstractproperty @@ -59,14 +56,14 @@ def to_cyclic_value(self, value: Any) -> Any: def offset(self, value: Any) -> int: pass - @staticmethod - def create_axis(options, name, values, datacube): - if options == {}: - DatacubeAxis.create_standard(name, values, datacube) - if "mapper" in options.keys(): - DatacubeAxis.create_mapper(options, name, datacube) - if "Cyclic" in options.keys(): - DatacubeAxis.create_cyclic(options, name, values, datacube) + # @staticmethod + # def create_axis(options, name, values, datacube): + # if options == {}: + # DatacubeAxis.create_standard(name, values, datacube) + # if "mapper" in options.keys(): + # DatacubeAxis.create_mapper(options, name, datacube) + # if "Cyclic" in options.keys(): + # DatacubeAxis.create_cyclic(options, name, values, datacube) @staticmethod def create_cyclic(options, name, values, datacube): @@ -76,7 +73,7 @@ def create_cyclic(options, name, values, datacube): cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) datacube._axes[name] = cyclic_axis_type datacube._axes[name].name = name - datacube._axes[name].range = options["Cyclic"] + datacube._axes[name].range = options["cyclic"] datacube.axis_counter += 1 @staticmethod @@ -91,28 +88,6 @@ def check_axis_type(name, values): if values.dtype.type not in _type_to_axis_lookup: raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") - @staticmethod - def create_mapper(options, name, datacube): - grid_mapping_options = options["mapper"] - grid_type = grid_mapping_options["type"] - grid_resolution = grid_mapping_options["resolution"] - grid_axes = grid_mapping_options["axes"] - map_type = _type_to_datacube_mapper_lookup[grid_type] - module = import_module("polytope.datacube.datacube_mappers") - constructor = getattr(module, map_type) - datacube.grid_mapper = constructor(name, grid_axes, grid_resolution) - # Once we have created mapper, create axis for the mapped axes - for i in range(len(grid_axes)): - axis_name = grid_axes[i] - new_axis_options = datacube.axis_options.get(axis_name, {}) - if i == 0: - values = np.array(datacube.grid_mapper.first_axis_vals()) - DatacubeAxis.create_axis(new_axis_options, axis_name, values, datacube) - if i == 1: - # the values[0] will be a value on the first axis - values = np.array(datacube.grid_mapper.second_axis_vals(values[0])) - DatacubeAxis.create_axis(new_axis_options, axis_name, values, datacube) - class IntDatacubeAxis(DatacubeAxis): name = None diff --git a/polytope/datacube/datacube_mappers.py b/polytope/datacube/datacube_mappers.py index 0ece04c98..aa3e54e0b 100644 --- a/polytope/datacube/datacube_mappers.py +++ b/polytope/datacube/datacube_mappers.py @@ -1,5 +1,10 @@ import math from abc import ABC, abstractmethod +from importlib import import_module + +import numpy as np + +from .datacube import configure_datacube_axis class DatacubeMapper(ABC): @@ -24,6 +29,28 @@ def map_second_axis(self, first_val, lower, upper): def unmap(self): pass + @staticmethod + def create_mapper(options, name, datacube): + grid_mapping_options = options["mapper"] + grid_type = grid_mapping_options["type"] + grid_resolution = grid_mapping_options["resolution"] + grid_axes = grid_mapping_options["axes"] + map_type = _type_to_datacube_mapper_lookup[grid_type] + module = import_module("polytope.datacube.datacube_mappers") + constructor = getattr(module, map_type) + datacube.grid_mapper = constructor(name, grid_axes, grid_resolution) + # Once we have created mapper, create axis for the mapped axes + for i in range(len(grid_axes)): + axis_name = grid_axes[i] + new_axis_options = datacube.axis_options.get(axis_name, {}) + if i == 0: + values = np.array(datacube.grid_mapper.first_axis_vals()) + configure_datacube_axis(new_axis_options, axis_name, values, datacube) + if i == 1: + # the values[0] will be a value on the first axis + values = np.array(datacube.grid_mapper.second_axis_vals(values[0])) + configure_datacube_axis(new_axis_options, axis_name, values, datacube) + class OctahedralGridMapper(DatacubeMapper): def __init__(self, base_axis, mapped_axes, resolution): diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index bf116c0c4..dafcdef1b 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -3,8 +3,7 @@ import xarray as xr from ..utility.combinatorics import unique, validate_axes -from .datacube import Datacube, DatacubePath, IndexTree -from .datacube_axis import DatacubeAxis +from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis class XArrayDatacube(Datacube): @@ -22,19 +21,19 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): if name in dataarray.dims: self.dataarray = self.dataarray.sortby(name) options = axis_options.get(name, {}) - DatacubeAxis.create_axis(options, name, values, self) + configure_datacube_axis(options, name, values, self) treated_axes.append(name) self.complete_axes.append(name) else: if self.dataarray[name].dims == (): options = axis_options.get(name, {}) - DatacubeAxis.create_axis(options, name, values, self) + configure_datacube_axis(options, name, values, self) treated_axes.append(name) for name in dataarray.dims: if name not in treated_axes: options = axis_options.get(name, {}) val = dataarray[name].values[0] - DatacubeAxis.create_axis(options, name, val, self) + configure_datacube_axis(options, name, val, self) def get(self, requests: IndexTree): for r in requests.leaves: diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index 31a74de54..ea0c945bf 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -21,7 +21,7 @@ def setup_method(self, method): "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1], }, ) - options = {"long": {"Cyclic": [-1.1, -0.1]}, "level": {"Cyclic": [1, 129]}} + options = {"long": {"cyclic": [-1.1, -0.1]}, "level": {"cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index c66fbaacb..b3a2bd3ab 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -21,7 +21,7 @@ def setup_method(self, method): "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1], }, ) - options = {"long": {"Cyclic": [-1.1, -0.1]}, "level": {"Cyclic": [1, 129]}} + options = {"long": {"cyclic": [-1.1, -0.1]}, "level": {"cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index d6af02e88..f9f4a994d 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -21,7 +21,7 @@ def setup_method(self, method): "long": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], }, ) - options = {"long": {"Cyclic": [0, 1.0]}, "level": {"Cyclic": [1, 129]}} + options = {"long": {"cyclic": [0, 1.0]}, "level": {"cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) From d5be964a7a253d362660d87e85355183e742f8f0 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 1 Aug 2023 12:57:13 +0200 Subject: [PATCH 067/332] black --- polytope/datacube/datacube.py | 1 + 1 file changed, 1 insertion(+) diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index 938ae5892..312f2258b 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -54,6 +54,7 @@ def configure_datacube_axis(options, name, values, datacube): DatacubeAxis.create_standard(name, values, datacube) if "mapper" in options.keys(): from .datacube_mappers import DatacubeMapper + DatacubeMapper.create_mapper(options, name, datacube) if "cyclic" in options.keys(): DatacubeAxis.create_cyclic(options, name, values, datacube) From 3530301b473745468fae692fd4716d6da662a878 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 1 Aug 2023 15:26:50 +0200 Subject: [PATCH 068/332] make fdb datacube work with the pyfdb list_axes method --- polytope/datacube/FDB_datacube.py | 25 ++++++++++--------------- tests/test_fdb_datacube.py | 18 +++++++++++++----- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/polytope/datacube/FDB_datacube.py b/polytope/datacube/FDB_datacube.py index 8aa089723..08dde6838 100644 --- a/polytope/datacube/FDB_datacube.py +++ b/polytope/datacube/FDB_datacube.py @@ -1,21 +1,24 @@ import math +import os import sys from copy import deepcopy import numpy as np import pandas as pd -import pyfdb -from ..utility.combinatorics import unique, validate_axes -from .datacube import Datacube, DatacubePath, IndexTree -from .datacube_axis import ( +os.environ['FDB_HOME'] = '/Users/male/git/fdb-home' +import pyfdb # noqa: E402 + +from ..utility.combinatorics import unique, validate_axes # noqa: E402 +from .datacube import Datacube, DatacubePath, IndexTree # noqa: E402 +from .datacube_axis import ( # noqa: E402 FloatAxis, IntAxis, PandasTimedeltaAxis, PandasTimestampAxis, UnsliceableaAxis, ) -from .mappers import OctahedralGridMap +from .mappers import OctahedralGridMap # noqa: E402 _mappings = { pd.Int64Dtype: IntAxis(), @@ -25,20 +28,13 @@ np.timedelta64: PandasTimedeltaAxis(), np.float64: FloatAxis(), np.str_: UnsliceableaAxis(), - str: UnsliceableaAxis(), + str.__name__: UnsliceableaAxis(), np.object_: UnsliceableaAxis(), "int" : IntAxis(), "float" : FloatAxis(), } -def get_datacube_indices(partial_request): - datacube_dico = {"step": [0, 3, 6], - "level" : [10, 11, 12], - "values": [1, 2, 3, 4]} - return datacube_dico - - def glue(path): return {"t" : 0} @@ -61,6 +57,7 @@ def update_fdb_dataarray(fdb_dataarray): else: new_dict[key] = values new_dict["values"] = [0.0] + return new_dict class FDBDatacube(Datacube): @@ -106,11 +103,9 @@ def __init__(self, config={}, options={}, grid_options={}): partial_request = config # Find values in the level 3 FDB datacube # Will be in the form of a dictionary? {axis_name:values_available, ...} - # dataarray = get_datacube_indices(partial_request) fdb = pyfdb.FDB() fdb_dataarray = fdb.axes(partial_request).as_dict() dataarray = update_fdb_dataarray(fdb_dataarray) - for name, values in dataarray.items(): values.sort() self._set_mapper(values, name) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 1ecd5edba..fdcdf4ebe 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -1,7 +1,7 @@ from polytope.datacube.FDB_datacube import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Box +from polytope.shapes import Box, Select class TestSlicing3DXarrayDatacube: @@ -10,8 +10,8 @@ def setup_method(self, method): grid_options = {"values": {"grid_map": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} config = {"class" : "od", "expver" : "0001", - "levtype" : "pl", - "step" : 4} + "levtype" : "sfc", + "step" : 11} self.xarraydatacube = FDBDatacube(config, options={}, grid_options=grid_options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.xarraydatacube, engine=self.slicer) @@ -19,8 +19,16 @@ def setup_method(self, method): # Testing different shapes def test_2D_box(self): - request = Request(Box(["step", "level"], [3, 10], [6, 11]), + request = Request(Select("step", [11]), + Select("levtype", ["sfc"]), + Select("date", ["20230710T120000"]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", [151130]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2])) result = self.API.retrieve(request) result.pprint() - assert len(result.leaves) == 36 + assert len(result.leaves) == 9 From 6d76e5a2d28f8c0d63c25cb56d5d4fd5ea700c6f Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 1 Aug 2023 15:29:56 +0200 Subject: [PATCH 069/332] black --- polytope/datacube/FDB_datacube.py | 9 ++++----- tests/test_fdb_datacube.py | 27 +++++++++++++-------------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/polytope/datacube/FDB_datacube.py b/polytope/datacube/FDB_datacube.py index 08dde6838..98b5ff715 100644 --- a/polytope/datacube/FDB_datacube.py +++ b/polytope/datacube/FDB_datacube.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd -os.environ['FDB_HOME'] = '/Users/male/git/fdb-home' +os.environ["FDB_HOME"] = "/Users/male/git/fdb-home" import pyfdb # noqa: E402 from ..utility.combinatorics import unique, validate_axes # noqa: E402 @@ -30,13 +30,13 @@ np.str_: UnsliceableaAxis(), str.__name__: UnsliceableaAxis(), np.object_: UnsliceableaAxis(), - "int" : IntAxis(), - "float" : FloatAxis(), + "int": IntAxis(), + "float": FloatAxis(), } def glue(path): - return {"t" : 0} + return {"t": 0} def update_fdb_dataarray(fdb_dataarray): @@ -61,7 +61,6 @@ def update_fdb_dataarray(fdb_dataarray): class FDBDatacube(Datacube): - def _set_mapper(self, values, name): if type(values[0]).__name__ not in _mappings: raise ValueError(f"Could not create a mapper for index type {type(values[0]).__name__} for axis {name}") diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index fdcdf4ebe..e07e2746b 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -8,10 +8,7 @@ class TestSlicing3DXarrayDatacube: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types grid_options = {"values": {"grid_map": {"type": ["octahedral", 1280], "axes": ["latitude", "longitude"]}}} - config = {"class" : "od", - "expver" : "0001", - "levtype" : "sfc", - "step" : 11} + config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} self.xarraydatacube = FDBDatacube(config, options={}, grid_options=grid_options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.xarraydatacube, engine=self.slicer) @@ -19,16 +16,18 @@ def setup_method(self, method): # Testing different shapes def test_2D_box(self): - request = Request(Select("step", [11]), - Select("levtype", ["sfc"]), - Select("date", ["20230710T120000"]), - Select("domain", ["g"]), - Select("expver", ["0001"]), - Select("param", [151130]), - Select("class", ["od"]), - Select("stream", ["oper"]), - Select("type", ["fc"]), - Box(["latitude", "longitude"], [0, 0], [0.2, 0.2])) + request = Request( + Select("step", [11]), + Select("levtype", ["sfc"]), + Select("date", ["20230710T120000"]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", [151130]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 9 From dfdf999a6c4d3a50cde2f4c56e29b215d34d70c9 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 2 Aug 2023 11:55:20 +0200 Subject: [PATCH 070/332] add transformations class to handle merging of axes --- polytope/datacube/FDB_datacube.py | 6 ++- polytope/datacube/datacube.py | 10 ++++ polytope/datacube/datacube_axis.py | 17 +++++++ polytope/datacube/datacube_transformations.py | 51 +++++++++++++++++++ polytope/datacube/xarray.py | 11 ++++ 5 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 polytope/datacube/datacube_transformations.py diff --git a/polytope/datacube/FDB_datacube.py b/polytope/datacube/FDB_datacube.py index 700cda5b7..695b2518e 100644 --- a/polytope/datacube/FDB_datacube.py +++ b/polytope/datacube/FDB_datacube.py @@ -34,7 +34,6 @@ def update_fdb_dataarray(fdb_dataarray): class FDBDatacube(Datacube): - def __init__(self, config={}, axis_options={}): # Need to get the cyclic options and grid options from somewhere self.axis_options = axis_options @@ -174,3 +173,8 @@ def axes(self): def validate(self, axes): return validate_axes(self.axes, axes) + + def ax_vals(self, name): + for _name, values in self.dataarray.items(): + if _name == name: + return values diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index ee4dcbdba..016e7074e 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -4,6 +4,7 @@ import xarray as xr from .datacube_axis import DatacubeAxis +from .datacube_transformations import DatacubeAxisTransformation from .index_tree import DatacubePath, IndexTree @@ -50,12 +51,21 @@ def create(datacube, axis_options: dict): else: return datacube + @abstractmethod + def ax_vals(self, name: str) -> List: + pass + def configure_datacube_axis(options, name, values, datacube): if options == {}: DatacubeAxis.create_standard(name, values, datacube) + if "merge" in options.keys(): + # the merge options will look like "time": {"merge": {"with":"step", "linker": "00T"}} + # Need to make sure we do not loop infinitely over this option + DatacubeAxisTransformation.create_transformation(options, name, values, datacube) if "mapper" in options.keys(): from .datacube_mappers import DatacubeMapper + DatacubeMapper.create_mapper(options, name, datacube) if "cyclic" in options.keys(): DatacubeAxis.create_cyclic(options, name, values, datacube) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 901c85175..7ffd366a9 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -65,6 +65,23 @@ def offset(self, value: Any) -> int: # if "Cyclic" in options.keys(): # DatacubeAxis.create_cyclic(options, name, values, datacube) + # @staticmethod + # def merge(options, name, values, datacube): + # # This function will not actually create an axis, it will compute values of when we merge the axes together + # # the merge options will look like "time": {"merge": {"with":"step", "linker": "00T"}} + # first_ax_vals = values + # second_ax_name = options["merge"]["with"] + # second_ax_vals = datacube.ax_vals(second_ax_name) + # linker = options["merge"]["linker"] + # merged_values = [] + # for first_val in first_ax_vals: + # for second_val in second_ax_vals: + # merged_val = first_val + linker + second_val + # merged_values.append(merged_val) + # merged_values = np.array(merged_values) + + # return merged_values + @staticmethod def create_cyclic(options, name, values, datacube): values = np.array(values) diff --git a/polytope/datacube/datacube_transformations.py b/polytope/datacube/datacube_transformations.py new file mode 100644 index 000000000..dc004781d --- /dev/null +++ b/polytope/datacube/datacube_transformations.py @@ -0,0 +1,51 @@ +from abc import ABC +from importlib import import_module + +import numpy as np + +from .datacube import configure_datacube_axis + + +class DatacubeAxisTransformation(ABC): + @staticmethod + def create_transformation(options, name, values, datacube): + # transformation options look like + # "time":{"transformation": { "type" : {"merge" : {"with":"step", "linker": "00T"}}}} + # But the last dictionary can vary and change according to transformation, which can be handled inside the + # specialised transformations + transformation_options = options["transformation"] + transformation_type = list(transformation_options["type"].keys())[0] + transformation_type = _type_to_datacube_transformation_lookup[transformation_type] + module = import_module("polytope.datacube.datacube_mappers") + constructor = getattr(module, transformation_type) + transformation_type_option = transformation_options["type"][transformation_type] + datacube.transformation = constructor(name, name, transformation_type_option) + # now need to create an axis for the transformed axis + # but need to make sure we don't loop infinitely over the transformation option since we did not change + # the axis name here, unlike in the mappers + merged_values = datacube.transformation.merged_values(values, datacube) + axis_options = datacube.axis_options.get(name) + axis_options.pop("transformation") + configure_datacube_axis(axis_options, name, merged_values, datacube) + + +class DatacubeAxisMerger(DatacubeAxisTransformation): + def __init__(self, name, merge_options): + self._first_axis = name + self._second_axis = merge_options["with"] + self._linker = merge_options["linker"] + + def merged_values(self, values, datacube): + first_ax_vals = values + second_ax_name = self._second_axis + second_ax_vals = datacube.ax_vals(second_ax_name) + linker = self._linker + merged_values = [] + for first_val in first_ax_vals: + for second_val in second_ax_vals: + merged_values.append(first_val + linker + second_val) + merged_values = np.array(merged_values) + return merged_values + + +_type_to_datacube_transformation_lookup = {"merge": "DatacubeAxisMerger"} diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index dafcdef1b..f66b90be2 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -203,3 +203,14 @@ def axes(self): def validate(self, axes): return validate_axes(list(self.axes.keys()), axes) + + def ax_vals(self, name): + treated_axes = [] + for _name, values in self.dataarray.coords.variables.items(): + treated_axes.append(_name) + if _name == name: + return values + for _name in self.dataarray.dims: + if _name not in treated_axes: + if _name == name: + return self.dataarray[name].values[0] From dc524fb54fa20c6a3e5b97aaa48d3bf7d8bfbcd9 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 2 Aug 2023 11:59:21 +0200 Subject: [PATCH 071/332] avoid circular imports --- polytope/datacube/datacube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index 016e7074e..a23e85f32 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -4,7 +4,6 @@ import xarray as xr from .datacube_axis import DatacubeAxis -from .datacube_transformations import DatacubeAxisTransformation from .index_tree import DatacubePath, IndexTree @@ -62,6 +61,7 @@ def configure_datacube_axis(options, name, values, datacube): if "merge" in options.keys(): # the merge options will look like "time": {"merge": {"with":"step", "linker": "00T"}} # Need to make sure we do not loop infinitely over this option + from .datacube_transformations import DatacubeAxisTransformation DatacubeAxisTransformation.create_transformation(options, name, values, datacube) if "mapper" in options.keys(): from .datacube_mappers import DatacubeMapper From 07698a8aa655813fcc19967a0faa2dfebf7254dc Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 2 Aug 2023 12:00:19 +0200 Subject: [PATCH 072/332] black --- polytope/datacube/datacube.py | 1 + 1 file changed, 1 insertion(+) diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index a23e85f32..113fa50f0 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -62,6 +62,7 @@ def configure_datacube_axis(options, name, values, datacube): # the merge options will look like "time": {"merge": {"with":"step", "linker": "00T"}} # Need to make sure we do not loop infinitely over this option from .datacube_transformations import DatacubeAxisTransformation + DatacubeAxisTransformation.create_transformation(options, name, values, datacube) if "mapper" in options.keys(): from .datacube_mappers import DatacubeMapper From ecc3217d31fd52d40cb683dfdecf25b65faff11c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 2 Aug 2023 14:04:06 +0200 Subject: [PATCH 073/332] finish the merge transformation --- polytope/datacube/FDB_datacube.py | 14 ++++------ polytope/datacube/datacube.py | 28 +++++++++---------- polytope/datacube/datacube_transformations.py | 25 +++++++++++------ polytope/datacube/mock.py | 3 ++ polytope/datacube/xarray.py | 1 + tests/test_fdb_datacube.py | 3 +- 6 files changed, 41 insertions(+), 33 deletions(-) diff --git a/polytope/datacube/FDB_datacube.py b/polytope/datacube/FDB_datacube.py index 695b2518e..3cdb98186 100644 --- a/polytope/datacube/FDB_datacube.py +++ b/polytope/datacube/FDB_datacube.py @@ -7,6 +7,8 @@ os.environ["FDB_HOME"] = "/Users/male/git/fdb-home" import pyfdb # noqa: E402 +# TODO: currently, because the date and time are strings, the data will be treated as an unsliceable axis... + def glue(path): return {"t": 0} @@ -20,13 +22,6 @@ def update_fdb_dataarray(fdb_dataarray): for val in values: new_values.append(int(val)) new_dict[key] = new_values - elif key == "date": - time = fdb_dataarray["time"][0] - time = time + "00" - new_val = [fdb_dataarray[key][0] + "T" + time] - new_dict[key] = new_val - elif key == "time": - pass else: new_dict[key] = values new_dict["values"] = [0.0] @@ -40,6 +35,7 @@ def __init__(self, config={}, axis_options={}): self.grid_mapper = None self.axis_counter = 0 self._axes = {} + self.blocked_axes = [] partial_request = config # Find values in the level 3 FDB datacube @@ -47,14 +43,13 @@ def __init__(self, config={}, axis_options={}): fdb = pyfdb.FDB() fdb_dataarray = fdb.axes(partial_request).as_dict() dataarray = update_fdb_dataarray(fdb_dataarray) + self.dataarray = dataarray for name, values in dataarray.items(): values.sort() options = axis_options.get(name, {}) configure_datacube_axis(options, name, values, self) - self.dataarray = dataarray - def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() @@ -175,6 +170,7 @@ def validate(self, axes): return validate_axes(self.axes, axes) def ax_vals(self, name): + print(self.dataarray) for _name, values in self.dataarray.items(): if _name == name: return values diff --git a/polytope/datacube/datacube.py b/polytope/datacube/datacube.py index 113fa50f0..b29276d28 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/datacube.py @@ -56,17 +56,17 @@ def ax_vals(self, name: str) -> List: def configure_datacube_axis(options, name, values, datacube): - if options == {}: - DatacubeAxis.create_standard(name, values, datacube) - if "merge" in options.keys(): - # the merge options will look like "time": {"merge": {"with":"step", "linker": "00T"}} - # Need to make sure we do not loop infinitely over this option - from .datacube_transformations import DatacubeAxisTransformation - - DatacubeAxisTransformation.create_transformation(options, name, values, datacube) - if "mapper" in options.keys(): - from .datacube_mappers import DatacubeMapper - - DatacubeMapper.create_mapper(options, name, datacube) - if "cyclic" in options.keys(): - DatacubeAxis.create_cyclic(options, name, values, datacube) + if name not in datacube.blocked_axes: + if options == {}: + DatacubeAxis.create_standard(name, values, datacube) + if "transformation" in options.keys(): + # the merge options will look like "time": {"merge": {"with":"step", "linker": "00T"}} + # Need to make sure we do not loop infinitely over this option + from .datacube_transformations import DatacubeAxisTransformation + DatacubeAxisTransformation.create_transformation(options, name, values, datacube) + if "mapper" in options.keys(): + from .datacube_mappers import DatacubeMapper + + DatacubeMapper.create_mapper(options, name, datacube) + if "cyclic" in options.keys(): + DatacubeAxis.create_cyclic(options, name, values, datacube) diff --git a/polytope/datacube/datacube_transformations.py b/polytope/datacube/datacube_transformations.py index dc004781d..306076c11 100644 --- a/polytope/datacube/datacube_transformations.py +++ b/polytope/datacube/datacube_transformations.py @@ -10,16 +10,16 @@ class DatacubeAxisTransformation(ABC): @staticmethod def create_transformation(options, name, values, datacube): # transformation options look like - # "time":{"transformation": { "type" : {"merge" : {"with":"step", "linker": "00T"}}}} + # "time":{"transformation": { "type" : {"merge" : {"with":"step", "linkers": ["T", "00"]}}}} # But the last dictionary can vary and change according to transformation, which can be handled inside the # specialised transformations transformation_options = options["transformation"] - transformation_type = list(transformation_options["type"].keys())[0] - transformation_type = _type_to_datacube_transformation_lookup[transformation_type] - module = import_module("polytope.datacube.datacube_mappers") + transformation_type_key = list(transformation_options["type"].keys())[0] + transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] + module = import_module("polytope.datacube.datacube_transformations") constructor = getattr(module, transformation_type) - transformation_type_option = transformation_options["type"][transformation_type] - datacube.transformation = constructor(name, name, transformation_type_option) + transformation_type_option = transformation_options["type"][transformation_type_key] + datacube.transformation = constructor(name, transformation_type_option) # now need to create an axis for the transformed axis # but need to make sure we don't loop infinitely over the transformation option since we did not change # the axis name here, unlike in the mappers @@ -27,25 +27,32 @@ def create_transformation(options, name, values, datacube): axis_options = datacube.axis_options.get(name) axis_options.pop("transformation") configure_datacube_axis(axis_options, name, merged_values, datacube) + datacube.transformation.finish_transformation(datacube, values) class DatacubeAxisMerger(DatacubeAxisTransformation): def __init__(self, name, merge_options): self._first_axis = name self._second_axis = merge_options["with"] - self._linker = merge_options["linker"] + self._linkers = merge_options["linkers"] def merged_values(self, values, datacube): first_ax_vals = values second_ax_name = self._second_axis second_ax_vals = datacube.ax_vals(second_ax_name) - linker = self._linker + linkers = self._linkers merged_values = [] for first_val in first_ax_vals: for second_val in second_ax_vals: - merged_values.append(first_val + linker + second_val) + # TODO: check that the first and second val are strings + merged_values.append(first_val + linkers[0] + second_val + linkers[1]) merged_values = np.array(merged_values) return merged_values + def finish_transformation(self, datacube, values): + datacube.blocked_axes.append(self._second_axis) + # NOTE: we change the axis values here directly + datacube.dataarray[self._first_axis] = self.merged_values(values, datacube) + _type_to_datacube_transformation_lookup = {"merge": "DatacubeAxisMerger"} diff --git a/polytope/datacube/mock.py b/polytope/datacube/mock.py index c31fd9802..1d417a9db 100644 --- a/polytope/datacube/mock.py +++ b/polytope/datacube/mock.py @@ -60,3 +60,6 @@ def axes(self): def validate(self, axes): return validate_axes(self.axes, axes) + + def ax_vals(self, name): + return [] diff --git a/polytope/datacube/xarray.py b/polytope/datacube/xarray.py index f66b90be2..4e95040d3 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/xarray.py @@ -17,6 +17,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.dataarray = dataarray treated_axes = [] self.complete_axes = [] + self.blocked_axes = [] for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: self.dataarray = self.dataarray.sortby(name) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index c7b8e71a1..b4ddb3fe6 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -8,7 +8,8 @@ class TestSlicingFDBDatacube: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types grid_options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date" : {"transformation": {"type" : {"merge" : {"with" : "time", "linkers": ["T", "00"]}}}} } config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} self.xarraydatacube = FDBDatacube(config, axis_options=grid_options) From 7376ea857d88e543538ad05c78ccdc9799e2371b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 2 Aug 2023 16:25:02 +0200 Subject: [PATCH 074/332] trnasformations general structure --- polytope/datacube/FDB_datacube.py | 176 ------------------ .../{ => transformations}/datacube_mappers.py | 12 +- .../transformations/datacube_merger.py | 31 +++ .../datacube_transformations.py | 47 ++--- 4 files changed, 59 insertions(+), 207 deletions(-) delete mode 100644 polytope/datacube/FDB_datacube.py rename polytope/datacube/{ => transformations}/datacube_mappers.py (99%) create mode 100644 polytope/datacube/transformations/datacube_merger.py rename polytope/datacube/{ => transformations}/datacube_transformations.py (60%) diff --git a/polytope/datacube/FDB_datacube.py b/polytope/datacube/FDB_datacube.py deleted file mode 100644 index 3cdb98186..000000000 --- a/polytope/datacube/FDB_datacube.py +++ /dev/null @@ -1,176 +0,0 @@ -import math -import os - -from ..utility.combinatorics import unique, validate_axes -from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis - -os.environ["FDB_HOME"] = "/Users/male/git/fdb-home" -import pyfdb # noqa: E402 - -# TODO: currently, because the date and time are strings, the data will be treated as an unsliceable axis... - - -def glue(path): - return {"t": 0} - - -def update_fdb_dataarray(fdb_dataarray): - new_dict = {} - for key, values in fdb_dataarray.items(): - if key in ["levelist", "param", "step"]: - new_values = [] - for val in values: - new_values.append(int(val)) - new_dict[key] = new_values - else: - new_dict[key] = values - new_dict["values"] = [0.0] - return new_dict - - -class FDBDatacube(Datacube): - def __init__(self, config={}, axis_options={}): - # Need to get the cyclic options and grid options from somewhere - self.axis_options = axis_options - self.grid_mapper = None - self.axis_counter = 0 - self._axes = {} - self.blocked_axes = [] - - partial_request = config - # Find values in the level 3 FDB datacube - # Will be in the form of a dictionary? {axis_name:values_available, ...} - fdb = pyfdb.FDB() - fdb_dataarray = fdb.axes(partial_request).as_dict() - dataarray = update_fdb_dataarray(fdb_dataarray) - self.dataarray = dataarray - - for name, values in dataarray.items(): - values.sort() - options = axis_options.get(name, {}) - configure_datacube_axis(options, name, values, self) - - def get(self, requests: IndexTree): - for r in requests.leaves: - path = r.flatten() - path = self.remap_path(path) - if len(path.items()) == self.axis_counter: - if self.grid_mapper is not None: - first_axis = self.grid_mapper._mapped_axes[0] - first_val = path[first_axis] - second_axis = self.grid_mapper._mapped_axes[1] - second_val = path[second_axis] - path.pop(first_axis, None) - path.pop(second_axis, None) - # need to remap the lat, lon in path to dataarray index - unmapped_idx = self.grid_mapper.unmap(first_val, second_val) - path[self.grid_mapper._base_axis] = unmapped_idx - # Ask FDB what values it has on the path - subxarray = glue(path) - key = list(subxarray.keys())[0] - value = subxarray[key] - r.result = (key, value) - else: - # if we have no grid map, still need to assign values - subxarray = glue(path) - value = subxarray.item() - key = subxarray.name - r.result = (key, value) - else: - r.remove_branch() - - def get_mapper(self, axis): - return self._axes[axis] - - def remap_path(self, path: DatacubePath): - for key in path: - value = path[key] - path[key] = self._axes[key].remap_val_to_axis_range(value) - return path - - def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): - idx_between = [] - for i in range(len(search_ranges)): - r = search_ranges[i] - offset = search_ranges_offset[i] - low = r[0] - up = r[1] - - if self.grid_mapper is not None: - first_axis = self.grid_mapper._mapped_axes[0] - second_axis = self.grid_mapper._mapped_axes[1] - if axis.name == first_axis: - indexes_between = self.grid_mapper.map_first_axis(low, up) - elif axis.name == second_axis: - indexes_between = self.grid_mapper.map_second_axis(first_val, low, up) - else: - indexes_between = [i for i in indexes if low <= i <= up] - else: - indexes_between = [i for i in indexes if low <= i <= up] - - # Now the indexes_between are values on the cyclic range so need to remap them to their original - # values before returning them - for j in range(len(indexes_between)): - if offset is None: - indexes_between[j] = indexes_between[j] - else: - indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) - - idx_between.append(indexes_between[j]) - return idx_between - - def get_indices(self, path: DatacubePath, axis, lower, upper): - path = self.remap_path(path) - first_val = None - if self.grid_mapper is not None: - first_axis = self.grid_mapper._mapped_axes[0] - first_val = path.get(first_axis, None) - second_axis = self.grid_mapper._mapped_axes[1] - path.pop(first_axis, None) - path.pop(second_axis, None) - if axis.name == first_axis: - indexes = [] - elif axis.name == second_axis: - indexes = [] - else: - indexes = self.dataarray[axis.name] - else: - indexes = self.dataarray[axis.name] - - # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube - search_ranges = axis.remap([lower, upper]) - original_search_ranges = axis.to_intervals([lower, upper]) - - # Find the offsets for each interval in the requested range, which we will need later - search_ranges_offset = [] - for r in original_search_ranges: - offset = axis.offset(r) - search_ranges_offset.append(offset) - - # Look up the values in the datacube for each cyclic interval range - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) - - # Remove duplicates even if difference of the order of the axis tolerance - if offset is not None: - # Note that we can only do unique if not dealing with time values - idx_between = unique(idx_between) - - return idx_between - - def has_index(self, path: DatacubePath, axis, index): - # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - subarray_vals = self.dataarray[axis.name] - return index in subarray_vals - - @property - def axes(self): - return self._axes - - def validate(self, axes): - return validate_axes(self.axes, axes) - - def ax_vals(self, name): - print(self.dataarray) - for _name, values in self.dataarray.items(): - if _name == name: - return values diff --git a/polytope/datacube/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py similarity index 99% rename from polytope/datacube/datacube_mappers.py rename to polytope/datacube/transformations/datacube_mappers.py index aa3e54e0b..dc23d3b39 100644 --- a/polytope/datacube/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -1,13 +1,19 @@ import math -from abc import ABC, abstractmethod +from abc import abstractmethod from importlib import import_module import numpy as np -from .datacube import configure_datacube_axis +from ..datacube import configure_datacube_axis +from .datacube_transformations import DatacubeAxisTransformation -class DatacubeMapper(ABC): +class DatacubeMapper(DatacubeAxisTransformation): + + def __init__(self): + # TODO: should create an __init__ which initialises sub-class of itself + pass + def _mapped_axes(self): pass diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py new file mode 100644 index 000000000..0c8ddbec8 --- /dev/null +++ b/polytope/datacube/transformations/datacube_merger.py @@ -0,0 +1,31 @@ +import numpy as np + +from .datacube_transformations import DatacubeAxisTransformation + + +class DatacubeAxisMerger(DatacubeAxisTransformation): + def __init__(self, name, merge_options): + # TODO: not here, but in datacube, create a big flag dictionary where for each axis, + # we add a flag like grid_mapper if there is a mapper or datacube_merger if there is a merger + # with the relevant info for later. Can initalise these flags here/ add them to flag dictionary here + self._first_axis = name + self._second_axis = merge_options["with"] + self._linkers = merge_options["linkers"] + + def merged_values(self, values, datacube): + first_ax_vals = values + second_ax_name = self._second_axis + second_ax_vals = datacube.ax_vals(second_ax_name) + linkers = self._linkers + merged_values = [] + for first_val in first_ax_vals: + for second_val in second_ax_vals: + # TODO: check that the first and second val are strings + merged_values.append(first_val + linkers[0] + second_val + linkers[1]) + merged_values = np.array(merged_values) + return merged_values + + def finish_transformation(self, datacube, values): + datacube.blocked_axes.append(self._second_axis) + # NOTE: we change the axis values here directly + datacube.dataarray[self._first_axis] = self.merged_values(values, datacube) diff --git a/polytope/datacube/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py similarity index 60% rename from polytope/datacube/datacube_transformations.py rename to polytope/datacube/transformations/datacube_transformations.py index 306076c11..fa1fe63ea 100644 --- a/polytope/datacube/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -1,9 +1,7 @@ -from abc import ABC +from abc import ABC, abstractmethod from importlib import import_module -import numpy as np - -from .datacube import configure_datacube_axis +from ..datacube import configure_datacube_axis class DatacubeAxisTransformation(ABC): @@ -14,45 +12,38 @@ def create_transformation(options, name, values, datacube): # But the last dictionary can vary and change according to transformation, which can be handled inside the # specialised transformations transformation_options = options["transformation"] + + # TODO: next line, what happens if there are several transformation options? transformation_type_key = list(transformation_options["type"].keys())[0] transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] + + # TODO: import from each module individually since they are not all in the same file module = import_module("polytope.datacube.datacube_transformations") constructor = getattr(module, transformation_type) transformation_type_option = transformation_options["type"][transformation_type_key] datacube.transformation = constructor(name, transformation_type_option) + # TODO: then in subclasses, create init, and inside init, create sub transformation + # and update datacube.transformation + # now need to create an axis for the transformed axis # but need to make sure we don't loop infinitely over the transformation option since we did not change # the axis name here, unlike in the mappers + + # TODO: the specifics like merged_values should be stored inside the individual transformations... + # Really, this is specific to the merger and creating a merger... merged_values = datacube.transformation.merged_values(values, datacube) axis_options = datacube.axis_options.get(name) axis_options.pop("transformation") configure_datacube_axis(axis_options, name, merged_values, datacube) - datacube.transformation.finish_transformation(datacube, values) + # TODO: does this really belong in the generic transformation class. + # TODO: is this even necessary because we are mapping and not creating axes etc? + datacube.transformation.finish_transformation(datacube, values) -class DatacubeAxisMerger(DatacubeAxisTransformation): - def __init__(self, name, merge_options): - self._first_axis = name - self._second_axis = merge_options["with"] - self._linkers = merge_options["linkers"] - - def merged_values(self, values, datacube): - first_ax_vals = values - second_ax_name = self._second_axis - second_ax_vals = datacube.ax_vals(second_ax_name) - linkers = self._linkers - merged_values = [] - for first_val in first_ax_vals: - for second_val in second_ax_vals: - # TODO: check that the first and second val are strings - merged_values.append(first_val + linkers[0] + second_val + linkers[1]) - merged_values = np.array(merged_values) - return merged_values - + @abstractmethod def finish_transformation(self, datacube, values): - datacube.blocked_axes.append(self._second_axis) - # NOTE: we change the axis values here directly - datacube.dataarray[self._first_axis] = self.merged_values(values, datacube) + pass -_type_to_datacube_transformation_lookup = {"merge": "DatacubeAxisMerger"} +_type_to_datacube_transformation_lookup = {"merge": "DatacubeAxisMerger", + "mapper": "DatacubeMapper"} From 9bee5a6171d65cf704dbb758e76b7cab7c49aa2b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 2 Aug 2023 16:54:30 +0200 Subject: [PATCH 075/332] fix factory for datacube transformation class --- .../transformations/datacube_transformations.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index fa1fe63ea..4a7b16959 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -16,12 +16,16 @@ def create_transformation(options, name, values, datacube): # TODO: next line, what happens if there are several transformation options? transformation_type_key = list(transformation_options["type"].keys())[0] transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] + transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] - # TODO: import from each module individually since they are not all in the same file - module = import_module("polytope.datacube.datacube_transformations") + module = import_module("polytope.datacube.datacube_" + transformation_file_name) constructor = getattr(module, transformation_type) transformation_type_option = transformation_options["type"][transformation_type_key] + # NOTE: the transformation in the datacube takes in now an option dico like + # {"with":"step", "linkers": ["T", "00"]}} + # TODO: here, would be better to have a dico of transformations along with name of axis to keep flags datacube.transformation = constructor(name, transformation_type_option) + # TODO: then in subclasses, create init, and inside init, create sub transformation # and update datacube.transformation @@ -47,3 +51,6 @@ def finish_transformation(self, datacube, values): _type_to_datacube_transformation_lookup = {"merge": "DatacubeAxisMerger", "mapper": "DatacubeMapper"} + +_type_to_transformation_file_lookup = {"merge" : "merger", + "mapper" : "mappers"} From fd79088b552753d83ea779f717d06f1a06de34b8 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 3 Aug 2023 10:37:31 +0200 Subject: [PATCH 076/332] restructure folders --- examples/4D_flight_path.py | 2 +- examples/country_slicing.py | 2 +- examples/cyclic_route_around_earth.py | 2 +- examples/octahedral_grid_box_example.py | 2 +- examples/octahedral_grid_country_example.py | 2 +- examples/slicing_all_ecmwf_countries.py | 2 +- examples/timeseries_example.py | 2 +- examples/wind_farms.py | 2 +- performance/scalability_test.py | 2 +- performance/scalability_test_2.py | 2 +- tests/test_create_axis.py | 55 ++++++++++++++++++++ tests/test_cyclic_axis_over_negative_vals.py | 2 +- tests/test_cyclic_axis_slicer_not_0.py | 2 +- tests/test_cyclic_axis_slicing.py | 2 +- tests/test_datacube_mock.py | 2 +- tests/test_datacube_xarray.py | 2 +- tests/test_engine_slicer.py | 2 +- tests/test_fdb_datacube.py | 36 ------------- tests/test_float_type.py | 2 +- tests/test_hull_slicer.py | 2 +- tests/test_hullslicer_engine.py | 2 +- tests/test_octahedral_grid.py | 2 +- tests/test_profiling_requesttree.py | 2 +- tests/test_request_trees_after_slicing.py | 2 +- tests/test_slicer_engine.py | 2 +- tests/test_slicer_era5.py | 2 +- tests/test_slicing_unsliceable_axis.py | 2 +- tests/test_slicing_xarray_3D.py | 2 +- tests/test_slicing_xarray_4D.py | 2 +- 29 files changed, 82 insertions(+), 63 deletions(-) create mode 100644 tests/test_create_axis.py delete mode 100644 tests/test_fdb_datacube.py diff --git a/examples/4D_flight_path.py b/examples/4D_flight_path.py index b75100fd0..d2d1f1eaa 100644 --- a/examples/4D_flight_path.py +++ b/examples/4D_flight_path.py @@ -4,7 +4,7 @@ from earthkit import data from PIL import Image -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Path, Select diff --git a/examples/country_slicing.py b/examples/country_slicing.py index a2dd197b4..4a5ed30db 100644 --- a/examples/country_slicing.py +++ b/examples/country_slicing.py @@ -4,7 +4,7 @@ from earthkit import data from shapely.geometry import shape -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Polygon, Select, Union diff --git a/examples/cyclic_route_around_earth.py b/examples/cyclic_route_around_earth.py index da94a43e2..4e971d677 100644 --- a/examples/cyclic_route_around_earth.py +++ b/examples/cyclic_route_around_earth.py @@ -3,7 +3,7 @@ import numpy as np from earthkit import data -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, PathSegment, Select diff --git a/examples/octahedral_grid_box_example.py b/examples/octahedral_grid_box_example.py index 860b66d2d..a4cddcacd 100644 --- a/examples/octahedral_grid_box_example.py +++ b/examples/octahedral_grid_box_example.py @@ -6,7 +6,7 @@ from eccodes import codes_grib_find_nearest, codes_grib_new_from_file from matplotlib import markers -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select diff --git a/examples/octahedral_grid_country_example.py b/examples/octahedral_grid_country_example.py index 397154983..280701d2b 100644 --- a/examples/octahedral_grid_country_example.py +++ b/examples/octahedral_grid_country_example.py @@ -7,7 +7,7 @@ from matplotlib import markers from shapely.geometry import shape -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Polygon, Select, Union diff --git a/examples/slicing_all_ecmwf_countries.py b/examples/slicing_all_ecmwf_countries.py index e02b6deab..7178305a9 100644 --- a/examples/slicing_all_ecmwf_countries.py +++ b/examples/slicing_all_ecmwf_countries.py @@ -4,7 +4,7 @@ from earthkit import data from shapely.geometry import shape -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Polygon, Select, Union diff --git a/examples/timeseries_example.py b/examples/timeseries_example.py index ba153f8a8..bd350cc13 100644 --- a/examples/timeseries_example.py +++ b/examples/timeseries_example.py @@ -3,7 +3,7 @@ from earthkit import data from shapely.geometry import shape -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Polygon, Select, Union diff --git a/examples/wind_farms.py b/examples/wind_farms.py index 14091c2c0..23699eb36 100644 --- a/examples/wind_farms.py +++ b/examples/wind_farms.py @@ -6,7 +6,7 @@ from osgeo import gdal from shapely.geometry import shape -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Polygon, Select, Union diff --git a/performance/scalability_test.py b/performance/scalability_test.py index 98eeac9d1..ab3c5b299 100644 --- a/performance/scalability_test.py +++ b/performance/scalability_test.py @@ -3,7 +3,7 @@ import numpy as np import xarray as xr -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Disk, Ellipsoid, Select diff --git a/performance/scalability_test_2.py b/performance/scalability_test_2.py index 9330fd60a..b10e86f4c 100644 --- a/performance/scalability_test_2.py +++ b/performance/scalability_test_2.py @@ -3,7 +3,7 @@ import numpy as np import xarray as xr -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select, Union diff --git a/tests/test_create_axis.py b/tests/test_create_axis.py new file mode 100644 index 000000000..7465b8e10 --- /dev/null +++ b/tests/test_create_axis.py @@ -0,0 +1,55 @@ +import numpy as np +import xarray as xr + +from polytope.datacube.datacube_axis import ( + DatacubeAxis, + FloatDatacubeAxis, + FloatDatacubeAxisCyclic, + IntDatacubeAxis, + IntDatacubeAxisCyclic, +) +from polytope.datacube.backends.xarray import XArrayDatacube + + +class TestCreateAxis: + def test_create_axis(self): + array = xr.DataArray( + np.random.randn(3, 6, 129), + dims=("date", "step", "long"), + coords={ + "date": [0.1, 0.2, 0.3], + "step": [0, 3, 6, 9, 12, 15], + "long": range(1, 130), + }, + ) + options = {"Cyclic": [1, 10]} + datacube = XArrayDatacube(array, options) + DatacubeAxis.create_axis(options, "long", array["long"], datacube) + assert type(datacube._axes["long"]) == IntDatacubeAxisCyclic + options = {} + DatacubeAxis.create_axis(options, "step", array["step"], datacube) + assert type(datacube._axes["step"]) == IntDatacubeAxis + grid_options = {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} + DatacubeAxis.create_axis(grid_options, "date", array["date"], datacube) + assert type(datacube._axes["latitude"]) == FloatDatacubeAxis + assert type(datacube._axes["longitude"]) == FloatDatacubeAxis + + def test_create_cyclic_and_mapper(self): + array = xr.DataArray( + np.random.randn(3, 6, 129), + dims=("date", "step", "long"), + coords={ + "date": [0.1, 0.2, 0.3], + "step": [0, 3, 6, 9, 12, 15], + "long": range(1, 130), + }, + ) + options = { + "date": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "latitude": {"Cyclic": [1, 10]}, + } + datacube = XArrayDatacube(array, options) + grid_options = {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} + DatacubeAxis.create_axis(grid_options, "date", array["date"], datacube) + assert type(datacube._axes["latitude"]) == FloatDatacubeAxisCyclic + assert datacube._axes.get("date", {}) == {} diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index ea0c945bf..fc3aa7dd2 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -2,7 +2,7 @@ import pandas as pd import xarray as xr -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index b3a2bd3ab..1634b0883 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -2,7 +2,7 @@ import pandas as pd import xarray as xr -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index f9f4a994d..9bdea1c11 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -2,7 +2,7 @@ import pandas as pd import xarray as xr -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select diff --git a/tests/test_datacube_mock.py b/tests/test_datacube_mock.py index 149623d17..0b147b95c 100644 --- a/tests/test_datacube_mock.py +++ b/tests/test_datacube_mock.py @@ -1,6 +1,6 @@ import pytest -from polytope.datacube.mock import MockDatacube +from polytope.datacube.backends.mock import MockDatacube from polytope.utility.exceptions import AxisNotFoundError, AxisOverdefinedError diff --git a/tests/test_datacube_xarray.py b/tests/test_datacube_xarray.py index 0d6463c9b..2074c7c8e 100644 --- a/tests/test_datacube_xarray.py +++ b/tests/test_datacube_xarray.py @@ -4,12 +4,12 @@ import xarray as xr from polytope.datacube import Datacube, DatacubePath +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.datacube.datacube_axis import ( FloatDatacubeAxis, IntDatacubeAxis, PandasTimestampDatacubeAxis, ) -from polytope.datacube.xarray import XArrayDatacube from polytope.utility.exceptions import AxisNotFoundError, AxisOverdefinedError diff --git a/tests/test_engine_slicer.py b/tests/test_engine_slicer.py index 94ac15751..a7fda9e12 100644 --- a/tests/test_engine_slicer.py +++ b/tests/test_engine_slicer.py @@ -1,4 +1,4 @@ -from polytope.datacube.mock import MockDatacube +from polytope.datacube.backends.mock import MockDatacube from polytope.engine.hullslicer import HullSlicer from polytope.shapes import Box, Polygon diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py deleted file mode 100644 index b4ddb3fe6..000000000 --- a/tests/test_fdb_datacube.py +++ /dev/null @@ -1,36 +0,0 @@ -from polytope.datacube.FDB_datacube import FDBDatacube -from polytope.engine.hullslicer import HullSlicer -from polytope.polytope import Polytope, Request -from polytope.shapes import Box, Select - - -class TestSlicingFDBDatacube: - def setup_method(self, method): - # Create a dataarray with 3 labelled axes using different index types - grid_options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "date" : {"transformation": {"type" : {"merge" : {"with" : "time", "linkers": ["T", "00"]}}}} - } - config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} - self.xarraydatacube = FDBDatacube(config, axis_options=grid_options) - self.slicer = HullSlicer() - self.API = Polytope(datacube=self.xarraydatacube, engine=self.slicer) - - # Testing different shapes - - def test_2D_box(self): - request = Request( - Select("step", [11]), - Select("levtype", ["sfc"]), - Select("date", ["20230710T120000"]), - Select("domain", ["g"]), - Select("expver", ["0001"]), - Select("param", [151130]), - Select("class", ["od"]), - Select("stream", ["oper"]), - Select("type", ["fc"]), - Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), - ) - result = self.API.retrieve(request) - result.pprint() - assert len(result.leaves) == 9 diff --git a/tests/test_float_type.py b/tests/test_float_type.py index 0b07c79ae..7fb6f94ab 100644 --- a/tests/test_float_type.py +++ b/tests/test_float_type.py @@ -2,7 +2,7 @@ import pytest import xarray as xr -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select, Span diff --git a/tests/test_hull_slicer.py b/tests/test_hull_slicer.py index da7860b31..4e9404e23 100644 --- a/tests/test_hull_slicer.py +++ b/tests/test_hull_slicer.py @@ -4,7 +4,7 @@ import polytope.engine.hullslicer from polytope import ConvexPolytope -from polytope.datacube.mock import MockDatacube +from polytope.datacube.backends.mock import MockDatacube from polytope.utility.profiling import benchmark diff --git a/tests/test_hullslicer_engine.py b/tests/test_hullslicer_engine.py index 1fe61f6df..dd876beb4 100644 --- a/tests/test_hullslicer_engine.py +++ b/tests/test_hullslicer_engine.py @@ -1,8 +1,8 @@ import numpy as np import xarray as xr +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.datacube.index_tree import IndexTree -from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope from polytope.shapes import Box diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index e4f5a29b5..6bb18d11a 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -1,7 +1,7 @@ from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select diff --git a/tests/test_profiling_requesttree.py b/tests/test_profiling_requesttree.py index 1e855ecbe..6bd6afac2 100644 --- a/tests/test_profiling_requesttree.py +++ b/tests/test_profiling_requesttree.py @@ -3,7 +3,7 @@ import pytest import xarray as xr -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select diff --git a/tests/test_request_trees_after_slicing.py b/tests/test_request_trees_after_slicing.py index 061324ff1..e64917aa3 100644 --- a/tests/test_request_trees_after_slicing.py +++ b/tests/test_request_trees_after_slicing.py @@ -1,8 +1,8 @@ import numpy as np import xarray as xr +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.datacube.datacube_axis import IntDatacubeAxis -from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope from polytope.shapes import Box diff --git a/tests/test_slicer_engine.py b/tests/test_slicer_engine.py index 1fe61f6df..dd876beb4 100644 --- a/tests/test_slicer_engine.py +++ b/tests/test_slicer_engine.py @@ -1,8 +1,8 @@ import numpy as np import xarray as xr +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.datacube.index_tree import IndexTree -from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope from polytope.shapes import Box diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index b4550b1d8..06d34b63a 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -1,7 +1,7 @@ import numpy as np from earthkit import data -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select diff --git a/tests/test_slicing_unsliceable_axis.py b/tests/test_slicing_unsliceable_axis.py index 45c21cdb5..3f3bb1c63 100644 --- a/tests/test_slicing_unsliceable_axis.py +++ b/tests/test_slicing_unsliceable_axis.py @@ -3,7 +3,7 @@ import pytest import xarray as xr -from polytope.datacube.xarray import XArrayDatacube +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select diff --git a/tests/test_slicing_xarray_3D.py b/tests/test_slicing_xarray_3D.py index e036321b2..9742f909e 100644 --- a/tests/test_slicing_xarray_3D.py +++ b/tests/test_slicing_xarray_3D.py @@ -5,8 +5,8 @@ import pandas as pd import xarray as xr +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.datacube.index_tree import IndexTree -from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import ( diff --git a/tests/test_slicing_xarray_4D.py b/tests/test_slicing_xarray_4D.py index 971823aeb..301bd874c 100644 --- a/tests/test_slicing_xarray_4D.py +++ b/tests/test_slicing_xarray_4D.py @@ -3,8 +3,8 @@ import pytest import xarray as xr +from polytope.datacube.backends.xarray import XArrayDatacube from polytope.datacube.index_tree import IndexTree -from polytope.datacube.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import ( From fcb71ad36aaf2cf7c0d4a7cc6619de8187197730 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 3 Aug 2023 10:38:56 +0200 Subject: [PATCH 077/332] restructure datacube folder and finish generic transformation class and merger transformation --- polytope/datacube/FDB_datacube.py | 176 ++++++++++++++++++ polytope/datacube/__init__.py | 2 +- polytope/datacube/{ => backends}/datacube.py | 14 +- polytope/datacube/{ => backends}/mock.py | 4 +- polytope/datacube/{ => backends}/xarray.py | 2 +- .../transformations/datacube_mappers.py | 2 +- .../transformations/datacube_merger.py | 18 +- .../datacube_transformations.py | 62 +++--- polytope/engine/engine.py | 2 +- polytope/engine/hullslicer.py | 2 +- 10 files changed, 237 insertions(+), 47 deletions(-) create mode 100644 polytope/datacube/FDB_datacube.py rename polytope/datacube/{ => backends}/datacube.py (83%) rename polytope/datacube/{ => backends}/mock.py (95%) rename polytope/datacube/{ => backends}/xarray.py (99%) diff --git a/polytope/datacube/FDB_datacube.py b/polytope/datacube/FDB_datacube.py new file mode 100644 index 000000000..f441b18dd --- /dev/null +++ b/polytope/datacube/FDB_datacube.py @@ -0,0 +1,176 @@ +import math +import os + +from ..utility.combinatorics import unique, validate_axes +from .backends.datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis + +os.environ["FDB_HOME"] = "/Users/male/git/fdb-home" +import pyfdb # noqa: E402 + +# TODO: currently, because the date and time are strings, the data will be treated as an unsliceable axis... + + +def glue(path): + return {"t": 0} + + +def update_fdb_dataarray(fdb_dataarray): + new_dict = {} + for key, values in fdb_dataarray.items(): + if key in ["levelist", "param", "step"]: + new_values = [] + for val in values: + new_values.append(int(val)) + new_dict[key] = new_values + else: + new_dict[key] = values + new_dict["values"] = [0.0] + return new_dict + + +class FDBDatacube(Datacube): + def __init__(self, config={}, axis_options={}): + # Need to get the cyclic options and grid options from somewhere + self.axis_options = axis_options + self.grid_mapper = None + self.axis_counter = 0 + self._axes = {} + self.blocked_axes = [] + + partial_request = config + # Find values in the level 3 FDB datacube + # Will be in the form of a dictionary? {axis_name:values_available, ...} + fdb = pyfdb.FDB() + fdb_dataarray = fdb.axes(partial_request).as_dict() + dataarray = update_fdb_dataarray(fdb_dataarray) + self.dataarray = dataarray + + for name, values in dataarray.items(): + values.sort() + options = axis_options.get(name, {}) + configure_datacube_axis(options, name, values, self) + + def get(self, requests: IndexTree): + for r in requests.leaves: + path = r.flatten() + path = self.remap_path(path) + if len(path.items()) == self.axis_counter: + if self.grid_mapper is not None: + first_axis = self.grid_mapper._mapped_axes[0] + first_val = path[first_axis] + second_axis = self.grid_mapper._mapped_axes[1] + second_val = path[second_axis] + path.pop(first_axis, None) + path.pop(second_axis, None) + # need to remap the lat, lon in path to dataarray index + unmapped_idx = self.grid_mapper.unmap(first_val, second_val) + path[self.grid_mapper._base_axis] = unmapped_idx + # Ask FDB what values it has on the path + subxarray = glue(path) + key = list(subxarray.keys())[0] + value = subxarray[key] + r.result = (key, value) + else: + # if we have no grid map, still need to assign values + subxarray = glue(path) + value = subxarray.item() + key = subxarray.name + r.result = (key, value) + else: + r.remove_branch() + + def get_mapper(self, axis): + return self._axes[axis] + + def remap_path(self, path: DatacubePath): + for key in path: + value = path[key] + path[key] = self._axes[key].remap_val_to_axis_range(value) + return path + + def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): + idx_between = [] + for i in range(len(search_ranges)): + r = search_ranges[i] + offset = search_ranges_offset[i] + low = r[0] + up = r[1] + + if self.grid_mapper is not None: + first_axis = self.grid_mapper._mapped_axes[0] + second_axis = self.grid_mapper._mapped_axes[1] + if axis.name == first_axis: + indexes_between = self.grid_mapper.map_first_axis(low, up) + elif axis.name == second_axis: + indexes_between = self.grid_mapper.map_second_axis(first_val, low, up) + else: + indexes_between = [i for i in indexes if low <= i <= up] + else: + indexes_between = [i for i in indexes if low <= i <= up] + + # Now the indexes_between are values on the cyclic range so need to remap them to their original + # values before returning them + for j in range(len(indexes_between)): + if offset is None: + indexes_between[j] = indexes_between[j] + else: + indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) + + idx_between.append(indexes_between[j]) + return idx_between + + def get_indices(self, path: DatacubePath, axis, lower, upper): + path = self.remap_path(path) + first_val = None + if self.grid_mapper is not None: + first_axis = self.grid_mapper._mapped_axes[0] + first_val = path.get(first_axis, None) + second_axis = self.grid_mapper._mapped_axes[1] + path.pop(first_axis, None) + path.pop(second_axis, None) + if axis.name == first_axis: + indexes = [] + elif axis.name == second_axis: + indexes = [] + else: + indexes = self.dataarray[axis.name] + else: + indexes = self.dataarray[axis.name] + + # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube + search_ranges = axis.remap([lower, upper]) + original_search_ranges = axis.to_intervals([lower, upper]) + + # Find the offsets for each interval in the requested range, which we will need later + search_ranges_offset = [] + for r in original_search_ranges: + offset = axis.offset(r) + search_ranges_offset.append(offset) + + # Look up the values in the datacube for each cyclic interval range + idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) + + # Remove duplicates even if difference of the order of the axis tolerance + if offset is not None: + # Note that we can only do unique if not dealing with time values + idx_between = unique(idx_between) + + return idx_between + + def has_index(self, path: DatacubePath, axis, index): + # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube + subarray_vals = self.dataarray[axis.name] + return index in subarray_vals + + @property + def axes(self): + return self._axes + + def validate(self, axes): + return validate_axes(self.axes, axes) + + def ax_vals(self, name): + print(self.dataarray) + for _name, values in self.dataarray.items(): + if _name == name: + return values diff --git a/polytope/datacube/__init__.py b/polytope/datacube/__init__.py index d261e197f..798718589 100644 --- a/polytope/datacube/__init__.py +++ b/polytope/datacube/__init__.py @@ -1 +1 @@ -from .datacube import * +from .backends.datacube import * diff --git a/polytope/datacube/datacube.py b/polytope/datacube/backends/datacube.py similarity index 83% rename from polytope/datacube/datacube.py rename to polytope/datacube/backends/datacube.py index b29276d28..bcc0988ec 100644 --- a/polytope/datacube/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -3,8 +3,8 @@ import xarray as xr -from .datacube_axis import DatacubeAxis -from .index_tree import DatacubePath, IndexTree +from ..datacube_axis import DatacubeAxis +from ..index_tree import DatacubePath, IndexTree class Datacube(ABC): @@ -54,18 +54,24 @@ def create(datacube, axis_options: dict): def ax_vals(self, name: str) -> List: pass + # TODO: need to add transformation properties like the datacube.transformations dico + def configure_datacube_axis(options, name, values, datacube): + # TODO: this will not work with the generic transformation class anymore + # TODO: need to see where the axes are created now if name not in datacube.blocked_axes: if options == {}: DatacubeAxis.create_standard(name, values, datacube) if "transformation" in options.keys(): # the merge options will look like "time": {"merge": {"with":"step", "linker": "00T"}} # Need to make sure we do not loop infinitely over this option - from .datacube_transformations import DatacubeAxisTransformation + from ..transformations.datacube_transformations import ( + DatacubeAxisTransformation, + ) DatacubeAxisTransformation.create_transformation(options, name, values, datacube) if "mapper" in options.keys(): - from .datacube_mappers import DatacubeMapper + from ..transformations.datacube_mappers import DatacubeMapper DatacubeMapper.create_mapper(options, name, datacube) if "cyclic" in options.keys(): diff --git a/polytope/datacube/mock.py b/polytope/datacube/backends/mock.py similarity index 95% rename from polytope/datacube/mock.py rename to polytope/datacube/backends/mock.py index 1d417a9db..3370b4028 100644 --- a/polytope/datacube/mock.py +++ b/polytope/datacube/backends/mock.py @@ -1,9 +1,9 @@ import math from copy import deepcopy -from ..utility.combinatorics import validate_axes +from ...utility.combinatorics import validate_axes +from ..datacube_axis import IntDatacubeAxis from .datacube import Datacube, DatacubePath, IndexTree -from .datacube_axis import IntDatacubeAxis class MockDatacube(Datacube): diff --git a/polytope/datacube/xarray.py b/polytope/datacube/backends/xarray.py similarity index 99% rename from polytope/datacube/xarray.py rename to polytope/datacube/backends/xarray.py index 4e95040d3..5ebf29047 100644 --- a/polytope/datacube/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -2,7 +2,7 @@ import xarray as xr -from ..utility.combinatorics import unique, validate_axes +from ...utility.combinatorics import unique, validate_axes from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index dc23d3b39..bf5cafa0b 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -4,7 +4,7 @@ import numpy as np -from ..datacube import configure_datacube_axis +from ..backends.datacube import configure_datacube_axis from .datacube_transformations import DatacubeAxisTransformation diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 0c8ddbec8..4588a7c92 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -1,13 +1,13 @@ import numpy as np +from ..backends.datacube import configure_datacube_axis from .datacube_transformations import DatacubeAxisTransformation class DatacubeAxisMerger(DatacubeAxisTransformation): + def __init__(self, name, merge_options): - # TODO: not here, but in datacube, create a big flag dictionary where for each axis, - # we add a flag like grid_mapper if there is a mapper or datacube_merger if there is a merger - # with the relevant info for later. Can initalise these flags here/ add them to flag dictionary here + self.name = name self._first_axis = name self._second_axis = merge_options["with"] self._linkers = merge_options["linkers"] @@ -25,7 +25,15 @@ def merged_values(self, values, datacube): merged_values = np.array(merged_values) return merged_values + def apply_transformation(self, name, datacube, values): + merged_values = self.merged_values(values, datacube) + axis_options = datacube.axis_options.get(name) + axis_options.pop("merge") + configure_datacube_axis(axis_options, name, merged_values, datacube) + self.finish_transformation(datacube, merged_values) + def finish_transformation(self, datacube, values): + # Need to "delete" the second axis we do not use anymore datacube.blocked_axes.append(self._second_axis) - # NOTE: we change the axis values here directly - datacube.dataarray[self._first_axis] = self.merged_values(values, datacube) + # NOTE: we change the axis values here directly, which will not work with xarray + datacube.dataarray[self._first_axis] = values diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 4a7b16959..148bf7744 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -1,51 +1,51 @@ -from abc import ABC, abstractmethod +from abc import ABC, abstractmethod, abstractproperty +from copy import deepcopy from importlib import import_module -from ..datacube import configure_datacube_axis - class DatacubeAxisTransformation(ABC): + @staticmethod def create_transformation(options, name, values, datacube): + # transformation options look like # "time":{"transformation": { "type" : {"merge" : {"with":"step", "linkers": ["T", "00"]}}}} # But the last dictionary can vary and change according to transformation, which can be handled inside the # specialised transformations transformation_options = options["transformation"] - # TODO: next line, what happens if there are several transformation options? - transformation_type_key = list(transformation_options["type"].keys())[0] - transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] - transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] - - module = import_module("polytope.datacube.datacube_" + transformation_file_name) - constructor = getattr(module, transformation_type) - transformation_type_option = transformation_options["type"][transformation_type_key] - # NOTE: the transformation in the datacube takes in now an option dico like - # {"with":"step", "linkers": ["T", "00"]}} - # TODO: here, would be better to have a dico of transformations along with name of axis to keep flags - datacube.transformation = constructor(name, transformation_type_option) + # NOTE: we do the following for each transformation of each axis + for transformation_type_key in transformation_options["type"].keys(): + transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] + transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] + + module = import_module("polytope.datacube.datacube_" + transformation_file_name) + constructor = getattr(module, transformation_type) + transformation_type_option = transformation_options["type"][transformation_type_key] + # NOTE: the transformation in the datacube takes in now an option dico like + # {"with":"step", "linkers": ["T", "00"]}} + + # Here, we keep track of all the transformation objects along with the associated axis within the datacube + # We generate a transformation dictionary that looks like + # {"lat": [merger, cyclic], "lon": [mapper, cyclic], etc...} + new_transformation = deepcopy(constructor(name, transformation_type_option)) + # TODO: instead of adding directly the transformation, could be we have an add_transformation method + # where each transformation can choose the name that it is assigned to, ie the axis name it is assigned to + # and then for eg for grid mapper transformation, can have the first axis name in there to make things + # easier to handle in the datacube + new_transformation.name = name + datacube.transformation[name].append(new_transformation) + new_transformation.apply_transformation(name, datacube, values) # TODO: then in subclasses, create init, and inside init, create sub transformation - # and update datacube.transformation - # now need to create an axis for the transformed axis - # but need to make sure we don't loop infinitely over the transformation option since we did not change - # the axis name here, unlike in the mappers - - # TODO: the specifics like merged_values should be stored inside the individual transformations... - # Really, this is specific to the merger and creating a merger... - merged_values = datacube.transformation.merged_values(values, datacube) - axis_options = datacube.axis_options.get(name) - axis_options.pop("transformation") - configure_datacube_axis(axis_options, name, merged_values, datacube) - - # TODO: does this really belong in the generic transformation class. - # TODO: is this even necessary because we are mapping and not creating axes etc? - datacube.transformation.finish_transformation(datacube, values) + @abstractproperty + def name(self): + pass + # TODO: do we need this? to apply transformation to datacube yes... @abstractmethod - def finish_transformation(self, datacube, values): + def apply_transformation(self, datacube, values): pass diff --git a/polytope/engine/engine.py b/polytope/engine/engine.py index e32d130bc..259aaa6de 100644 --- a/polytope/engine/engine.py +++ b/polytope/engine/engine.py @@ -1,6 +1,6 @@ from typing import List -from ..datacube.datacube import Datacube +from ..datacube.backends.datacube import Datacube from ..datacube.index_tree import IndexTree from ..shapes import ConvexPolytope diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 646d31c74..809fa641c 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -4,7 +4,7 @@ import scipy.spatial -from ..datacube.datacube import Datacube, IndexTree +from ..datacube.backends.datacube import Datacube, IndexTree from ..datacube.datacube_axis import UnsliceableDatacubeAxis from ..shapes import ConvexPolytope from ..utility.combinatorics import argmax, argmin, group, product, unique From e10c45c74b65392566c7a581ef3a2421ce694c32 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 3 Aug 2023 16:56:24 +0200 Subject: [PATCH 078/332] try to make generic transformations class work --- polytope/datacube/backends/datacube.py | 37 ++++++-- polytope/datacube/backends/xarray.py | 4 + .../transformations/datacube_mappers.py | 92 +++++++++++++------ .../transformations/datacube_merger.py | 14 ++- .../datacube_transformations.py | 24 +++-- tests/test_create_axis.py | 55 ----------- tests/test_octahedral_grid.py | 4 +- 7 files changed, 125 insertions(+), 105 deletions(-) delete mode 100644 tests/test_create_axis.py diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index bcc0988ec..aecbb4d31 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -60,19 +60,36 @@ def ax_vals(self, name: str) -> List: def configure_datacube_axis(options, name, values, datacube): # TODO: this will not work with the generic transformation class anymore # TODO: need to see where the axes are created now + + # NOTE: Maybe here, with new generic transformation class, + # instead of handling the options, we should just create an axis and look + # up the associated transformation to + + # First need to check we are not initialising an axis which should not be initialised + print("first name") + print(name) if name not in datacube.blocked_axes: + # Now look at the options passed in if options == {}: DatacubeAxis.create_standard(name, values, datacube) if "transformation" in options.keys(): - # the merge options will look like "time": {"merge": {"with":"step", "linker": "00T"}} - # Need to make sure we do not loop infinitely over this option - from ..transformations.datacube_transformations import ( - DatacubeAxisTransformation, - ) - DatacubeAxisTransformation.create_transformation(options, name, values, datacube) - if "mapper" in options.keys(): - from ..transformations.datacube_mappers import DatacubeMapper - - DatacubeMapper.create_mapper(options, name, datacube) + # options = options["transformation"] + if options == {}: + DatacubeAxis.create_standard(name, values, datacube) + else: + # the merge options will look like "time": {"merge": {"with":"step", "linker": "00T"}} + # Need to make sure we do not loop infinitely over this option + from ..transformations.datacube_transformations import ( + DatacubeAxisTransformation, + ) + print(name) + print(options) + DatacubeAxisTransformation.create_transformation(options, name, values, datacube) + # if "mapper" in options.keys(): + # from ..transformations.datacube_mappers import DatacubeMapper + + # DatacubeMapper.create_mapper(options, name, datacube) + + # TODO: This will need to come as a transformation later if "cyclic" in options.keys(): DatacubeAxis.create_cyclic(options, name, values, datacube) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 5ebf29047..723936126 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -18,8 +18,10 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): treated_axes = [] self.complete_axes = [] self.blocked_axes = [] + self.transformation = {} for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: + self.transformation[name] = [] self.dataarray = self.dataarray.sortby(name) options = axis_options.get(name, {}) configure_datacube_axis(options, name, values, self) @@ -27,11 +29,13 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.complete_axes.append(name) else: if self.dataarray[name].dims == (): + self.transformation[name] = [] options = axis_options.get(name, {}) configure_datacube_axis(options, name, values, self) treated_axes.append(name) for name in dataarray.dims: if name not in treated_axes: + self.transformation[name] = [] options = axis_options.get(name, {}) val = dataarray[name].values[0] configure_datacube_axis(options, name, val, self) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index bf5cafa0b..546e17e70 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -1,5 +1,5 @@ import math -from abc import abstractmethod +from copy import deepcopy from importlib import import_module import numpy as np @@ -10,53 +10,87 @@ class DatacubeMapper(DatacubeAxisTransformation): - def __init__(self): - # TODO: should create an __init__ which initialises sub-class of itself - pass + # # TODO: check if we still need this and it's right + # @staticmethod + # def create_mapper(options, name, datacube): + # grid_mapping_options = options["mapper"] + # grid_type = grid_mapping_options["type"] + # grid_resolution = grid_mapping_options["resolution"] + # grid_axes = grid_mapping_options["axes"] + # map_type = _type_to_datacube_mapper_lookup[grid_type] + # module = import_module("polytope.datacube.datacube_mappers") + # constructor = getattr(module, map_type) + # datacube.grid_mapper = constructor(name, grid_axes, grid_resolution) + # # Once we have created mapper, create axis for the mapped axes + # for i in range(len(grid_axes)): + # axis_name = grid_axes[i] + # new_axis_options = datacube.axis_options.get(axis_name, {}) + # if i == 0: + # values = np.array(datacube.grid_mapper.first_axis_vals()) + # configure_datacube_axis(new_axis_options, axis_name, values, datacube) + # if i == 1: + # # the values[0] will be a value on the first axis + # values = np.array(datacube.grid_mapper.second_axis_vals(values[0])) + # configure_datacube_axis(new_axis_options, axis_name, values, datacube) + + # Needs to implements DatacubeAxisTransformation methods + + def __init__(self, name , mapper_options): + self.transformation_options = mapper_options + self.grid_type = mapper_options["type"] + self.grid_resolution = mapper_options["resolution"] + self.grid_axes = mapper_options["axes"] + self.old_axis = name + + def generate_final_transformation(self): + map_type = _type_to_datacube_mapper_lookup[self.grid_type] + module = import_module("polytope.datacube.transformations.datacube_mappers") + constructor = getattr(module, map_type) + transformation = deepcopy(constructor(self.old_axis, self.grid_axes, self.grid_resolution)) + print("here") + print(self.grid_axes) + return transformation + + def apply_transformation(self, name, datacube, values): + # TODO: how is this going to work once the cyclicity is a transformation? + # Create mapped axes here + for i in range(len(self._mapped_axes)): + # axis_name = self.grid_axes[i] + axis_name = name + new_axis_options = datacube.axis_options.get(axis_name, {}) + if i == 0: + values = np.array(self.first_axis_vals()) + configure_datacube_axis(new_axis_options, axis_name, values, datacube) + if i == 1: + # the values[0] will be a value on the first axis + values = np.array(self.second_axis_vals(values[0])) + configure_datacube_axis(new_axis_options, axis_name, values, datacube) + # Needs to also implement its own methods + # @abstractproperty def _mapped_axes(self): pass + # @abstractproperty def _base_axis(self): pass + # @abstractproperty def _resolution(self): pass - @abstractmethod + # @abstractmethod def map_first_axis(self, lower, upper): pass - @abstractmethod + # @abstractmethod def map_second_axis(self, first_val, lower, upper): pass - @abstractmethod + # @abstractmethod def unmap(self): pass - @staticmethod - def create_mapper(options, name, datacube): - grid_mapping_options = options["mapper"] - grid_type = grid_mapping_options["type"] - grid_resolution = grid_mapping_options["resolution"] - grid_axes = grid_mapping_options["axes"] - map_type = _type_to_datacube_mapper_lookup[grid_type] - module = import_module("polytope.datacube.datacube_mappers") - constructor = getattr(module, map_type) - datacube.grid_mapper = constructor(name, grid_axes, grid_resolution) - # Once we have created mapper, create axis for the mapped axes - for i in range(len(grid_axes)): - axis_name = grid_axes[i] - new_axis_options = datacube.axis_options.get(axis_name, {}) - if i == 0: - values = np.array(datacube.grid_mapper.first_axis_vals()) - configure_datacube_axis(new_axis_options, axis_name, values, datacube) - if i == 1: - # the values[0] will be a value on the first axis - values = np.array(datacube.grid_mapper.second_axis_vals(values[0])) - configure_datacube_axis(new_axis_options, axis_name, values, datacube) - class OctahedralGridMapper(DatacubeMapper): def __init__(self, base_axis, mapped_axes, resolution): diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 4588a7c92..ff6d73951 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -7,6 +7,7 @@ class DatacubeAxisMerger(DatacubeAxisTransformation): def __init__(self, name, merge_options): + self.transformation_options = merge_options self.name = name self._first_axis = name self._second_axis = merge_options["with"] @@ -27,9 +28,15 @@ def merged_values(self, values, datacube): def apply_transformation(self, name, datacube, values): merged_values = self.merged_values(values, datacube) - axis_options = datacube.axis_options.get(name) + # Remove the merge option from the axis options since we have already handled it + # so do not want to handle it again + axis_options = datacube.axis_options[name]["transformation"] axis_options.pop("merge") - configure_datacube_axis(axis_options, name, merged_values, datacube) + # Update the nested dictionary with the modified axis option for our axis + new_datacube_axis_options = datacube.axis_options + new_datacube_axis_options[name]["transformation"] = axis_options + # Reconfigure the axis with the rest of its configurations + configure_datacube_axis(new_datacube_axis_options, name, merged_values, datacube) self.finish_transformation(datacube, merged_values) def finish_transformation(self, datacube, values): @@ -37,3 +44,6 @@ def finish_transformation(self, datacube, values): datacube.blocked_axes.append(self._second_axis) # NOTE: we change the axis values here directly, which will not work with xarray datacube.dataarray[self._first_axis] = values + + def generate_final_transformation(self): + return self diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 148bf7744..ac35f310f 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -1,4 +1,4 @@ -from abc import ABC, abstractmethod, abstractproperty +from abc import ABC, abstractmethod from copy import deepcopy from importlib import import_module @@ -15,13 +15,13 @@ def create_transformation(options, name, values, datacube): transformation_options = options["transformation"] # NOTE: we do the following for each transformation of each axis - for transformation_type_key in transformation_options["type"].keys(): + for transformation_type_key in transformation_options.keys(): transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] - module = import_module("polytope.datacube.datacube_" + transformation_file_name) + module = import_module("polytope.datacube.transformations.datacube_" + transformation_file_name) constructor = getattr(module, transformation_type) - transformation_type_option = transformation_options["type"][transformation_type_key] + transformation_type_option = transformation_options[transformation_type_key] # NOTE: the transformation in the datacube takes in now an option dico like # {"with":"step", "linkers": ["T", "00"]}} @@ -33,19 +33,27 @@ def create_transformation(options, name, values, datacube): # where each transformation can choose the name that it is assigned to, ie the axis name it is assigned to # and then for eg for grid mapper transformation, can have the first axis name in there to make things # easier to handle in the datacube + + # In case there are nested derived classes, want to get the final transformation in our + # transformation dico + new_transformation = new_transformation.generate_final_transformation() new_transformation.name = name datacube.transformation[name].append(new_transformation) new_transformation.apply_transformation(name, datacube, values) - # TODO: then in subclasses, create init, and inside init, create sub transformation - - @abstractproperty def name(self): pass + def transformation_options(self): + pass + + @abstractmethod + def generate_final_transformation(self): + pass + # TODO: do we need this? to apply transformation to datacube yes... @abstractmethod - def apply_transformation(self, datacube, values): + def apply_transformation(self, name, datacube, values): pass diff --git a/tests/test_create_axis.py b/tests/test_create_axis.py deleted file mode 100644 index 7465b8e10..000000000 --- a/tests/test_create_axis.py +++ /dev/null @@ -1,55 +0,0 @@ -import numpy as np -import xarray as xr - -from polytope.datacube.datacube_axis import ( - DatacubeAxis, - FloatDatacubeAxis, - FloatDatacubeAxisCyclic, - IntDatacubeAxis, - IntDatacubeAxisCyclic, -) -from polytope.datacube.backends.xarray import XArrayDatacube - - -class TestCreateAxis: - def test_create_axis(self): - array = xr.DataArray( - np.random.randn(3, 6, 129), - dims=("date", "step", "long"), - coords={ - "date": [0.1, 0.2, 0.3], - "step": [0, 3, 6, 9, 12, 15], - "long": range(1, 130), - }, - ) - options = {"Cyclic": [1, 10]} - datacube = XArrayDatacube(array, options) - DatacubeAxis.create_axis(options, "long", array["long"], datacube) - assert type(datacube._axes["long"]) == IntDatacubeAxisCyclic - options = {} - DatacubeAxis.create_axis(options, "step", array["step"], datacube) - assert type(datacube._axes["step"]) == IntDatacubeAxis - grid_options = {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} - DatacubeAxis.create_axis(grid_options, "date", array["date"], datacube) - assert type(datacube._axes["latitude"]) == FloatDatacubeAxis - assert type(datacube._axes["longitude"]) == FloatDatacubeAxis - - def test_create_cyclic_and_mapper(self): - array = xr.DataArray( - np.random.randn(3, 6, 129), - dims=("date", "step", "long"), - coords={ - "date": [0.1, 0.2, 0.3], - "step": [0, 3, 6, 9, 12, 15], - "long": range(1, 130), - }, - ) - options = { - "date": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - "latitude": {"Cyclic": [1, 10]}, - } - datacube = XArrayDatacube(array, options) - grid_options = {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} - DatacubeAxis.create_axis(grid_options, "date", array["date"], datacube) - assert type(datacube._axes["latitude"]) == FloatDatacubeAxisCyclic - assert datacube._axes.get("date", {}) == {} diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 6bb18d11a..84894e4d8 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -14,7 +14,9 @@ def setup_method(self, method): latlon_array = latlon_array.t2m self.xarraydatacube = XArrayDatacube(latlon_array) grid_options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} + "values": {"transformation" : {"mapper": {"type": "octahedral", + "resolution": 1280, + "axes": ["latitude", "longitude"]}}} } self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=grid_options) From 7d241bcaf2fb0c0157c7922eaf3d40d875d8305a Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 4 Aug 2023 10:43:27 +0200 Subject: [PATCH 079/332] finish creating a generic transformation class, but only works for datacube mappers now --- polytope/datacube/backends/datacube.py | 31 ++---- polytope/datacube/backends/xarray.py | 103 +++++++++--------- .../transformations/datacube_mappers.py | 64 ++++------- .../transformations/datacube_merger.py | 3 + .../datacube_transformations.py | 18 ++- 5 files changed, 103 insertions(+), 116 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index aecbb4d31..a1affa5ff 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -66,30 +66,17 @@ def configure_datacube_axis(options, name, values, datacube): # up the associated transformation to # First need to check we are not initialising an axis which should not be initialised - print("first name") - print(name) if name not in datacube.blocked_axes: # Now look at the options passed in if options == {}: DatacubeAxis.create_standard(name, values, datacube) - if "transformation" in options.keys(): - # options = options["transformation"] - if options == {}: - DatacubeAxis.create_standard(name, values, datacube) - else: - # the merge options will look like "time": {"merge": {"with":"step", "linker": "00T"}} - # Need to make sure we do not loop infinitely over this option - from ..transformations.datacube_transformations import ( - DatacubeAxisTransformation, - ) - print(name) - print(options) - DatacubeAxisTransformation.create_transformation(options, name, values, datacube) - # if "mapper" in options.keys(): - # from ..transformations.datacube_mappers import DatacubeMapper - - # DatacubeMapper.create_mapper(options, name, datacube) - - # TODO: This will need to come as a transformation later - if "cyclic" in options.keys(): + + # TODO: this will need to be a transformation + elif "cyclic" in options.keys(): DatacubeAxis.create_cyclic(options, name, values, datacube) + + else: + from ..transformations.datacube_transformations import ( + DatacubeAxisTransformation, + ) + DatacubeAxisTransformation.create_transformation(options, name, values, datacube) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 723936126..dfeda23b6 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -3,6 +3,7 @@ import xarray as xr from ...utility.combinatorics import unique, validate_axes +from ..transformations.datacube_mappers import DatacubeMapper from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis @@ -21,7 +22,6 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.transformation = {} for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: - self.transformation[name] = [] self.dataarray = self.dataarray.sortby(name) options = axis_options.get(name, {}) configure_datacube_axis(options, name, values, self) @@ -29,13 +29,11 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.complete_axes.append(name) else: if self.dataarray[name].dims == (): - self.transformation[name] = [] options = axis_options.get(name, {}) configure_datacube_axis(options, name, values, self) treated_axes.append(name) for name in dataarray.dims: if name not in treated_axes: - self.transformation[name] = [] options = axis_options.get(name, {}) val = dataarray[name].values[0] configure_datacube_axis(options, name, val, self) @@ -45,19 +43,27 @@ def get(self, requests: IndexTree): path = r.flatten() path = self.remap_path(path) if len(path.items()) == self.axis_counter: + # first, find the grid mapper transform + grid_map_transform = None for key in path.keys(): if self.dataarray[key].dims == (): path.pop(key) - if self.grid_mapper is not None: - first_axis = self.grid_mapper._mapped_axes[0] + if key in self.transformation.keys(): + axis_transforms = self.transformation[key] + for transform in axis_transforms: + if isinstance(transform, DatacubeMapper): + grid_map_transform = transform + if grid_map_transform is not None: + # if we have a grid_mapper transform, find the new axis indices + first_axis = grid_map_transform._mapped_axes()[0] first_val = path[first_axis] - second_axis = self.grid_mapper._mapped_axes[1] + second_axis = grid_map_transform._mapped_axes()[1] second_val = path[second_axis] path.pop(first_axis, None) path.pop(second_axis, None) subxarray = self.dataarray.sel(path, method="nearest") # need to remap the lat, lon in path to dataarray index - unmapped_idx = self.grid_mapper.unmap(first_val, second_val) + unmapped_idx = grid_map_transform.unmap(first_val, second_val) subxarray = subxarray.isel(values=unmapped_idx) value = subxarray.item() key = subxarray.name @@ -88,30 +94,29 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, low = r[0] up = r[1] - if self.grid_mapper is not None: - first_axis = self.grid_mapper._mapped_axes[0] - second_axis = self.grid_mapper._mapped_axes[1] - if axis.name == first_axis: - indexes_between = self.grid_mapper.map_first_axis(low, up) - elif axis.name == second_axis: - indexes_between = self.grid_mapper.map_second_axis(first_val, low, up) - else: - if axis.name in self.complete_axes: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - indexes_between = indexes[start:end].to_list() - else: - indexes_between = [i for i in indexes if low <= i <= up] - # start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? - # end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? - # indexes_between = indexes[start:end].to_list() + if axis.name in self.transformation.keys(): + axis_transforms = self.transformation[axis.name] + for transform in axis_transforms: + if isinstance(transform, DatacubeMapper): + # The if and for above loop through the transforms for that axis and + # determine if there is a grid mapper transform + first_axis = transform._mapped_axes()[0] + second_axis = transform._mapped_axes()[1] + if axis.name == first_axis: + indexes_between = transform.map_first_axis(low, up) + elif axis.name == second_axis: + indexes_between = transform.map_second_axis(first_val, low, up) + else: + if axis.name in self.complete_axes: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + else: + indexes_between = [i for i in indexes if low <= i <= up] else: # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - # start = indexes.searchsorted(low, "left") # TODO: catch start=0 (not found)? - # end = indexes.searchsorted(up, "right") # TODO: catch end=length (not found)? - # indexes_between = indexes[start:end].to_list() if axis.name in self.complete_axes: start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") @@ -137,12 +142,15 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): path.pop(key) first_val = None - if self.grid_mapper is not None: - first_axis = self.grid_mapper._mapped_axes[0] - first_val = path.get(first_axis, None) - second_axis = self.grid_mapper._mapped_axes[1] - path.pop(first_axis, None) - path.pop(second_axis, None) + if axis.name in self.transformation.keys(): + axis_transforms = self.transformation[axis.name] + for transform in axis_transforms: + if isinstance(transform, DatacubeMapper): + first_axis = transform._mapped_axes()[0] + first_val = path.get(first_axis, None) + second_axis = transform._mapped_axes()[1] + path.pop(first_axis, None) + path.pop(second_axis, None) for key in path.keys(): if self.dataarray[key].dims == (): @@ -154,27 +162,24 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): # Get the indexes of the axis we want to query # XArray does not support branching, so no need to use label, we just take the next axis - if self.grid_mapper is not None: - if axis.name == first_axis: - indexes = [] - elif axis.name == second_axis: - indexes = [] - else: - # assert axis.name == next(iter(subarray.xindexes)) - # indexes = next(iter(subarray.xindexes.values())).to_pandas_index() - if axis.name in self.complete_axes: - # indexes = list(subarray.indexes[axis.name]) - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() - else: - indexes = [subarray[axis.name].values] + if axis.name in self.transformation.keys(): + axis_transforms = self.transformation[axis.name] + for transform in axis_transforms: + if isinstance(transform, DatacubeMapper): + if axis.name == first_axis: + indexes = [] + elif axis.name == second_axis: + indexes = [] + else: + if axis.name in self.complete_axes: + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + else: + indexes = [subarray[axis.name].values] else: if axis.name in self.complete_axes: - # indexes = list(subarray.indexes[axis.name]) indexes = next(iter(subarray.xindexes.values())).to_pandas_index() else: indexes = [subarray[axis.name].values] - # assert axis.name == next(iter(subarray.xindexes)) - # indexes = next(iter(subarray.xindexes.values())).to_pandas_index() # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube search_ranges = axis.remap([lower, upper]) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 546e17e70..355aa615b 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -10,29 +10,6 @@ class DatacubeMapper(DatacubeAxisTransformation): - # # TODO: check if we still need this and it's right - # @staticmethod - # def create_mapper(options, name, datacube): - # grid_mapping_options = options["mapper"] - # grid_type = grid_mapping_options["type"] - # grid_resolution = grid_mapping_options["resolution"] - # grid_axes = grid_mapping_options["axes"] - # map_type = _type_to_datacube_mapper_lookup[grid_type] - # module = import_module("polytope.datacube.datacube_mappers") - # constructor = getattr(module, map_type) - # datacube.grid_mapper = constructor(name, grid_axes, grid_resolution) - # # Once we have created mapper, create axis for the mapped axes - # for i in range(len(grid_axes)): - # axis_name = grid_axes[i] - # new_axis_options = datacube.axis_options.get(axis_name, {}) - # if i == 0: - # values = np.array(datacube.grid_mapper.first_axis_vals()) - # configure_datacube_axis(new_axis_options, axis_name, values, datacube) - # if i == 1: - # # the values[0] will be a value on the first axis - # values = np.array(datacube.grid_mapper.second_axis_vals(values[0])) - # configure_datacube_axis(new_axis_options, axis_name, values, datacube) - # Needs to implements DatacubeAxisTransformation methods def __init__(self, name , mapper_options): @@ -47,49 +24,54 @@ def generate_final_transformation(self): module = import_module("polytope.datacube.transformations.datacube_mappers") constructor = getattr(module, map_type) transformation = deepcopy(constructor(self.old_axis, self.grid_axes, self.grid_resolution)) - print("here") - print(self.grid_axes) return transformation def apply_transformation(self, name, datacube, values): # TODO: how is this going to work once the cyclicity is a transformation? # Create mapped axes here - for i in range(len(self._mapped_axes)): - # axis_name = self.grid_axes[i] - axis_name = name + transformation = self.generate_final_transformation() + for i in range(len(transformation._mapped_axes)): + axis_name = transformation._mapped_axes[i] + # axis_name = name new_axis_options = datacube.axis_options.get(axis_name, {}) if i == 0: - values = np.array(self.first_axis_vals()) + values = np.array(transformation.first_axis_vals()) configure_datacube_axis(new_axis_options, axis_name, values, datacube) if i == 1: # the values[0] will be a value on the first axis - values = np.array(self.second_axis_vals(values[0])) + values = np.array(transformation.second_axis_vals(values[0])) configure_datacube_axis(new_axis_options, axis_name, values, datacube) + def transformation_axes_final(self): + final_transformation = self.generate_final_transformation() + final_axes = final_transformation._mapped_axes + return final_axes + # Needs to also implement its own methods - # @abstractproperty + def _mapped_axes(self): - pass + # NOTE: Each of the mapper method needs to call it's sub mapper method + final_transformation = self.generate_final_transformation() + final_axes = final_transformation._mapped_axes + return final_axes - # @abstractproperty def _base_axis(self): pass - # @abstractproperty def _resolution(self): pass - # @abstractmethod def map_first_axis(self, lower, upper): - pass + final_transformation = self.generate_final_transformation() + return final_transformation.map_first_axis(lower, upper) - # @abstractmethod def map_second_axis(self, first_val, lower, upper): - pass + final_transformation = self.generate_final_transformation() + return final_transformation.map_second_axis(first_val, lower, upper) - # @abstractmethod - def unmap(self): - pass + def unmap(self, first_val, second_val): + final_transformation = self.generate_final_transformation() + return final_transformation.unmap(first_val, second_val) class OctahedralGridMapper(DatacubeMapper): diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index ff6d73951..ac31e8941 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -39,6 +39,9 @@ def apply_transformation(self, name, datacube, values): configure_datacube_axis(new_datacube_axis_options, name, merged_values, datacube) self.finish_transformation(datacube, merged_values) + def transformation_axes_final(self): + return [self._first_axis] + def finish_transformation(self, datacube, values): # Need to "delete" the second axis we do not use anymore datacube.blocked_axes.append(self._second_axis) diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index ac35f310f..9d88873ee 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -34,11 +34,17 @@ def create_transformation(options, name, values, datacube): # and then for eg for grid mapper transformation, can have the first axis name in there to make things # easier to handle in the datacube - # In case there are nested derived classes, want to get the final transformation in our - # transformation dico - new_transformation = new_transformation.generate_final_transformation() new_transformation.name = name - datacube.transformation[name].append(new_transformation) + transformation_axis_names = new_transformation.transformation_axes_final() + for axis_name in transformation_axis_names: + # if there are no transformations for that axis yet, create an empty list of transforms. + # else, take the old list and append new transformation we are working on + key_val = datacube.transformation.get(axis_name, []) + datacube.transformation[axis_name] = key_val + # the transformation dico keeps track of the type of transformation, not the exact transformations + # For grid mappers, it keeps track that we have a grid_mapper, but won't know the exact grid map we + # implement + datacube.transformation[axis_name].append(new_transformation) new_transformation.apply_transformation(name, datacube, values) def name(self): @@ -51,6 +57,10 @@ def transformation_options(self): def generate_final_transformation(self): pass + @abstractmethod + def transformation_axes_final(self): + pass + # TODO: do we need this? to apply transformation to datacube yes... @abstractmethod def apply_transformation(self, name, datacube, values): From e9d7b9893e666554f681aea6ea942d8cd8ac61cd Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 4 Aug 2023 10:53:43 +0200 Subject: [PATCH 080/332] remove merger transformation --- polytope/datacube/FDB_datacube.py | 176 ------------------ .../transformations/datacube_merger.py | 52 ------ .../datacube_transformations.py | 6 +- 3 files changed, 2 insertions(+), 232 deletions(-) delete mode 100644 polytope/datacube/FDB_datacube.py delete mode 100644 polytope/datacube/transformations/datacube_merger.py diff --git a/polytope/datacube/FDB_datacube.py b/polytope/datacube/FDB_datacube.py deleted file mode 100644 index f441b18dd..000000000 --- a/polytope/datacube/FDB_datacube.py +++ /dev/null @@ -1,176 +0,0 @@ -import math -import os - -from ..utility.combinatorics import unique, validate_axes -from .backends.datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis - -os.environ["FDB_HOME"] = "/Users/male/git/fdb-home" -import pyfdb # noqa: E402 - -# TODO: currently, because the date and time are strings, the data will be treated as an unsliceable axis... - - -def glue(path): - return {"t": 0} - - -def update_fdb_dataarray(fdb_dataarray): - new_dict = {} - for key, values in fdb_dataarray.items(): - if key in ["levelist", "param", "step"]: - new_values = [] - for val in values: - new_values.append(int(val)) - new_dict[key] = new_values - else: - new_dict[key] = values - new_dict["values"] = [0.0] - return new_dict - - -class FDBDatacube(Datacube): - def __init__(self, config={}, axis_options={}): - # Need to get the cyclic options and grid options from somewhere - self.axis_options = axis_options - self.grid_mapper = None - self.axis_counter = 0 - self._axes = {} - self.blocked_axes = [] - - partial_request = config - # Find values in the level 3 FDB datacube - # Will be in the form of a dictionary? {axis_name:values_available, ...} - fdb = pyfdb.FDB() - fdb_dataarray = fdb.axes(partial_request).as_dict() - dataarray = update_fdb_dataarray(fdb_dataarray) - self.dataarray = dataarray - - for name, values in dataarray.items(): - values.sort() - options = axis_options.get(name, {}) - configure_datacube_axis(options, name, values, self) - - def get(self, requests: IndexTree): - for r in requests.leaves: - path = r.flatten() - path = self.remap_path(path) - if len(path.items()) == self.axis_counter: - if self.grid_mapper is not None: - first_axis = self.grid_mapper._mapped_axes[0] - first_val = path[first_axis] - second_axis = self.grid_mapper._mapped_axes[1] - second_val = path[second_axis] - path.pop(first_axis, None) - path.pop(second_axis, None) - # need to remap the lat, lon in path to dataarray index - unmapped_idx = self.grid_mapper.unmap(first_val, second_val) - path[self.grid_mapper._base_axis] = unmapped_idx - # Ask FDB what values it has on the path - subxarray = glue(path) - key = list(subxarray.keys())[0] - value = subxarray[key] - r.result = (key, value) - else: - # if we have no grid map, still need to assign values - subxarray = glue(path) - value = subxarray.item() - key = subxarray.name - r.result = (key, value) - else: - r.remove_branch() - - def get_mapper(self, axis): - return self._axes[axis] - - def remap_path(self, path: DatacubePath): - for key in path: - value = path[key] - path[key] = self._axes[key].remap_val_to_axis_range(value) - return path - - def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): - idx_between = [] - for i in range(len(search_ranges)): - r = search_ranges[i] - offset = search_ranges_offset[i] - low = r[0] - up = r[1] - - if self.grid_mapper is not None: - first_axis = self.grid_mapper._mapped_axes[0] - second_axis = self.grid_mapper._mapped_axes[1] - if axis.name == first_axis: - indexes_between = self.grid_mapper.map_first_axis(low, up) - elif axis.name == second_axis: - indexes_between = self.grid_mapper.map_second_axis(first_val, low, up) - else: - indexes_between = [i for i in indexes if low <= i <= up] - else: - indexes_between = [i for i in indexes if low <= i <= up] - - # Now the indexes_between are values on the cyclic range so need to remap them to their original - # values before returning them - for j in range(len(indexes_between)): - if offset is None: - indexes_between[j] = indexes_between[j] - else: - indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) - - idx_between.append(indexes_between[j]) - return idx_between - - def get_indices(self, path: DatacubePath, axis, lower, upper): - path = self.remap_path(path) - first_val = None - if self.grid_mapper is not None: - first_axis = self.grid_mapper._mapped_axes[0] - first_val = path.get(first_axis, None) - second_axis = self.grid_mapper._mapped_axes[1] - path.pop(first_axis, None) - path.pop(second_axis, None) - if axis.name == first_axis: - indexes = [] - elif axis.name == second_axis: - indexes = [] - else: - indexes = self.dataarray[axis.name] - else: - indexes = self.dataarray[axis.name] - - # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube - search_ranges = axis.remap([lower, upper]) - original_search_ranges = axis.to_intervals([lower, upper]) - - # Find the offsets for each interval in the requested range, which we will need later - search_ranges_offset = [] - for r in original_search_ranges: - offset = axis.offset(r) - search_ranges_offset.append(offset) - - # Look up the values in the datacube for each cyclic interval range - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) - - # Remove duplicates even if difference of the order of the axis tolerance - if offset is not None: - # Note that we can only do unique if not dealing with time values - idx_between = unique(idx_between) - - return idx_between - - def has_index(self, path: DatacubePath, axis, index): - # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - subarray_vals = self.dataarray[axis.name] - return index in subarray_vals - - @property - def axes(self): - return self._axes - - def validate(self, axes): - return validate_axes(self.axes, axes) - - def ax_vals(self, name): - print(self.dataarray) - for _name, values in self.dataarray.items(): - if _name == name: - return values diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py deleted file mode 100644 index ac31e8941..000000000 --- a/polytope/datacube/transformations/datacube_merger.py +++ /dev/null @@ -1,52 +0,0 @@ -import numpy as np - -from ..backends.datacube import configure_datacube_axis -from .datacube_transformations import DatacubeAxisTransformation - - -class DatacubeAxisMerger(DatacubeAxisTransformation): - - def __init__(self, name, merge_options): - self.transformation_options = merge_options - self.name = name - self._first_axis = name - self._second_axis = merge_options["with"] - self._linkers = merge_options["linkers"] - - def merged_values(self, values, datacube): - first_ax_vals = values - second_ax_name = self._second_axis - second_ax_vals = datacube.ax_vals(second_ax_name) - linkers = self._linkers - merged_values = [] - for first_val in first_ax_vals: - for second_val in second_ax_vals: - # TODO: check that the first and second val are strings - merged_values.append(first_val + linkers[0] + second_val + linkers[1]) - merged_values = np.array(merged_values) - return merged_values - - def apply_transformation(self, name, datacube, values): - merged_values = self.merged_values(values, datacube) - # Remove the merge option from the axis options since we have already handled it - # so do not want to handle it again - axis_options = datacube.axis_options[name]["transformation"] - axis_options.pop("merge") - # Update the nested dictionary with the modified axis option for our axis - new_datacube_axis_options = datacube.axis_options - new_datacube_axis_options[name]["transformation"] = axis_options - # Reconfigure the axis with the rest of its configurations - configure_datacube_axis(new_datacube_axis_options, name, merged_values, datacube) - self.finish_transformation(datacube, merged_values) - - def transformation_axes_final(self): - return [self._first_axis] - - def finish_transformation(self, datacube, values): - # Need to "delete" the second axis we do not use anymore - datacube.blocked_axes.append(self._second_axis) - # NOTE: we change the axis values here directly, which will not work with xarray - datacube.dataarray[self._first_axis] = values - - def generate_final_transformation(self): - return self diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 9d88873ee..40c241c0b 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -67,8 +67,6 @@ def apply_transformation(self, name, datacube, values): pass -_type_to_datacube_transformation_lookup = {"merge": "DatacubeAxisMerger", - "mapper": "DatacubeMapper"} +_type_to_datacube_transformation_lookup = {"mapper": "DatacubeMapper"} -_type_to_transformation_file_lookup = {"merge" : "merger", - "mapper" : "mappers"} +_type_to_transformation_file_lookup = {"mapper" : "mappers"} From 3665e951016bacd6c491d010a28ffac3dc85fbfa Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 4 Aug 2023 11:00:25 +0200 Subject: [PATCH 081/332] black --- polytope/datacube/backends/datacube.py | 1 + polytope/datacube/transformations/datacube_mappers.py | 3 +-- .../datacube/transformations/datacube_transformations.py | 4 +--- tests/test_octahedral_grid.py | 8 +++++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index a1affa5ff..12003ba9f 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -79,4 +79,5 @@ def configure_datacube_axis(options, name, values, datacube): from ..transformations.datacube_transformations import ( DatacubeAxisTransformation, ) + DatacubeAxisTransformation.create_transformation(options, name, values, datacube) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 355aa615b..b752924ef 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -9,10 +9,9 @@ class DatacubeMapper(DatacubeAxisTransformation): - # Needs to implements DatacubeAxisTransformation methods - def __init__(self, name , mapper_options): + def __init__(self, name, mapper_options): self.transformation_options = mapper_options self.grid_type = mapper_options["type"] self.grid_resolution = mapper_options["resolution"] diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 40c241c0b..4e504995a 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -4,10 +4,8 @@ class DatacubeAxisTransformation(ABC): - @staticmethod def create_transformation(options, name, values, datacube): - # transformation options look like # "time":{"transformation": { "type" : {"merge" : {"with":"step", "linkers": ["T", "00"]}}}} # But the last dictionary can vary and change according to transformation, which can be handled inside the @@ -69,4 +67,4 @@ def apply_transformation(self, name, datacube, values): _type_to_datacube_transformation_lookup = {"mapper": "DatacubeMapper"} -_type_to_transformation_file_lookup = {"mapper" : "mappers"} +_type_to_transformation_file_lookup = {"mapper": "mappers"} diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 84894e4d8..4746f3110 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -14,9 +14,11 @@ def setup_method(self, method): latlon_array = latlon_array.t2m self.xarraydatacube = XArrayDatacube(latlon_array) grid_options = { - "values": {"transformation" : {"mapper": {"type": "octahedral", - "resolution": 1280, - "axes": ["latitude", "longitude"]}}} + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + } + } } self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=grid_options) From 3de9482ad994eb576b555c13fa4008d1972c02b8 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 4 Aug 2023 13:10:41 +0200 Subject: [PATCH 082/332] add framework for cyclic transformation --- .../transformations/datacube_cyclic.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 polytope/datacube/transformations/datacube_cyclic.py diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py new file mode 100644 index 000000000..e81e436d6 --- /dev/null +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -0,0 +1,130 @@ +from copy import deepcopy + +from datacube_transformations import DatacubeAxisTransformation + + +class DatacubeAxisCyclic(DatacubeAxisTransformation): + # The transformation here will be to point the old axes to the new cyclic axes + + def __init__(self, name, cyclic_options): + self.name = name + self.transformation_options = cyclic_options + + def generate_final_transformation(self): + return self + + def transformation_axes_final(self): + return [self.name] + + def apply_transformation(self, name, datacube, values): + # NOTE: we will handle all the cyclicity mapping here instead of in the DatacubeAxis + # then we can generate just create_standard in the configure_axis at the end + # Also, in the datacube implementations, we then have to deal with the transformation dico + # and see if there is a cyclic axis, where we then need to generate the relevant offsets etc + # from the transformation object + + # OR, we generate a transformation, which we call in a new function create_axis. + # In create_axis, if we have a cyclic transformation, we generate a cyclic axis, else, a standard one + pass + + # Cyclic functions + + def to_intervals(self, range): + axis_lower = self.range[0] + axis_upper = self.range[1] + axis_range = axis_upper - axis_lower + lower = range[0] + upper = range[1] + intervals = [] + if lower < axis_upper: + # In this case, we want to go from lower to the first remapped cyclic axis upper + # or the asked upper range value. + # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, + # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper + # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], + # where -180 is the asked upper range value. + loops = int((axis_upper - lower) / axis_range) + remapped_up = axis_upper - (loops) * axis_range + new_upper = min(upper, remapped_up) + else: + # In this case, since lower >= axis_upper, we need to either go to the asked upper range + # or we need to go to the first remapped cyclic axis upper which is higher than lower + new_upper = min(axis_upper + axis_range, upper) + while new_upper < lower: + new_upper = min(new_upper + axis_range, upper) + intervals.append([lower, new_upper]) + # Now that we have established what the first interval should be, we should just jump from cyclic range + # to cyclic range until we hit the asked upper range value. + new_up = deepcopy(new_upper) + while new_up < upper: + new_upper = new_up + new_up = min(upper, new_upper + axis_range) + intervals.append([new_upper, new_up]) + # Once we have added all the in-between ranges, we need to add the last interval + intervals.append([new_up, upper]) + return intervals + + def remap_range_to_axis_range(self, range): + axis_lower = self.range[0] + axis_upper = self.range[1] + axis_range = axis_upper - axis_lower + lower = range[0] + upper = range[1] + if lower < axis_lower: + # In this case we need to calculate the number of loops between the axis lower + # and the lower to recenter the lower + loops = int((axis_lower - lower - self.tol) / axis_range) + return_lower = lower + (loops + 1) * axis_range + return_upper = upper + (loops + 1) * axis_range + elif lower >= axis_upper: + # In this case we need to calculate the number of loops between the axis upper + # and the lower to recenter the lower + loops = int((lower - axis_upper) / axis_range) + return_lower = lower - (loops + 1) * axis_range + return_upper = upper - (loops + 1) * axis_range + else: + # In this case, the lower value is already in the right range + return_lower = lower + return_upper = upper + return [return_lower, return_upper] + + def remap_val_to_axis_range(self, value): + return_range = self.remap_range_to_axis_range([value, value]) + return return_range[0] + + def remap(self, range): + if self.range[0] - self.tol <= range[0] <= self.range[1] + self.tol: + if self.range[0] - self.tol <= range[1] <= self.range[1] + self.tol: + # If we are already in the cyclic range, return it + return [range] + elif abs(range[0] - range[1]) <= 2 * self.tol: + # If we have a range that is just one point, then it should still be counted + # and so we should take a small interval around it to find values inbetween + range = [ + self.remap_val_to_axis_range(range[0]) - self.tol, + self.remap_val_to_axis_range(range[0]) + self.tol, + ] + return [range] + range_intervals = self.to_intervals(range) + ranges = [] + for interval in range_intervals: + if abs(interval[0] - interval[1]) > 0: + # If the interval is not just a single point, we remap it to the axis range + range = self.remap_range_to_axis_range([interval[0], interval[1]]) + up = range[1] + low = range[0] + if up < low: + # Make sure we remap in the right order + ranges.append([up - self.tol, low + self.tol]) + else: + ranges.append([low - self.tol, up + self.tol]) + return ranges + + def offset(self, range): + # We first unpad the range by the axis tolerance to make sure that + # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. + # Also, it's safer that we find the offset of a value inside the range instead of on the border + unpadded_range = [range[0] + 1.5 * self.tol, range[1] - 1.5 * self.tol] + cyclic_range = self.remap_range_to_axis_range(unpadded_range) + offset = unpadded_range[0] - cyclic_range[0] + return offset From 2c161658aaf7f2b60cda7bade17ed2a51e8679e0 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 4 Aug 2023 16:52:39 +0200 Subject: [PATCH 083/332] make merge transformation work --- polytope/datacube/backends/datacube.py | 7 +- polytope/datacube/backends/xarray.py | 14 +++ polytope/datacube/datacube_axis.py | 28 +++-- .../transformations/datacube_cyclic.py | 114 ++---------------- .../datacube_transformations.py | 5 +- tests/test_cyclic_axis_over_negative_vals.py | 5 +- tests/test_cyclic_axis_slicer_not_0.py | 5 +- tests/test_cyclic_axis_slicing.py | 2 +- 8 files changed, 55 insertions(+), 125 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 12003ba9f..db5a05786 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -69,12 +69,7 @@ def configure_datacube_axis(options, name, values, datacube): if name not in datacube.blocked_axes: # Now look at the options passed in if options == {}: - DatacubeAxis.create_standard(name, values, datacube) - - # TODO: this will need to be a transformation - elif "cyclic" in options.keys(): - DatacubeAxis.create_cyclic(options, name, values, datacube) - + DatacubeAxis.create_axis(name, values, datacube) else: from ..transformations.datacube_transformations import ( DatacubeAxisTransformation, diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index dfeda23b6..9b0c8644a 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -3,6 +3,7 @@ import xarray as xr from ...utility.combinatorics import unique, validate_axes +from ..transformations.datacube_cyclic import DatacubeAxisCyclic from ..transformations.datacube_mappers import DatacubeMapper from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis @@ -113,6 +114,14 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, indexes_between = indexes[start:end].to_list() else: indexes_between = [i for i in indexes if low <= i <= up] + if isinstance(transform, DatacubeAxisCyclic): + if axis.name in self.complete_axes: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + else: + indexes_between = [i for i in indexes if low <= i <= up] + else: # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html @@ -175,6 +184,11 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): indexes = next(iter(subarray.xindexes.values())).to_pandas_index() else: indexes = [subarray[axis.name].values] + if isinstance(transform, DatacubeAxisCyclic): + if axis.name in self.complete_axes: + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + else: + indexes = [subarray[axis.name].values] else: if axis.name in self.complete_axes: indexes = next(iter(subarray.xindexes.values())).to_pandas_index() diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 7ffd366a9..5eab3744b 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -56,15 +56,6 @@ def to_cyclic_value(self, value: Any) -> Any: def offset(self, value: Any) -> int: pass - # @staticmethod - # def create_axis(options, name, values, datacube): - # if options == {}: - # DatacubeAxis.create_standard(name, values, datacube) - # if "mapper" in options.keys(): - # DatacubeAxis.create_mapper(options, name, datacube) - # if "Cyclic" in options.keys(): - # DatacubeAxis.create_cyclic(options, name, values, datacube) - # @staticmethod # def merge(options, name, values, datacube): # # This function will not actually create an axis, it will compute values of when we merge the axes together @@ -83,7 +74,22 @@ def offset(self, value: Any) -> int: # return merged_values @staticmethod - def create_cyclic(options, name, values, datacube): + def create_axis(name, values, datacube): + if name in datacube.transformation.keys(): + axis_transforms = datacube.transformation[name] + for transform in axis_transforms: + from .transformations.datacube_cyclic import DatacubeAxisCyclic + + if isinstance(transform, DatacubeAxisCyclic): + DatacubeAxis.create_cyclic(transform, name, values, datacube) + else: + # if we don't have a cyclic transform, then we create a standard axis + DatacubeAxis.create_standard(name, values, datacube) + else: + DatacubeAxis.create_standard(name, values, datacube) + + @staticmethod + def create_cyclic(cyclic_transform, name, values, datacube): values = np.array(values) value_type = values.dtype.type axes_type_str = type(_type_to_axis_lookup[value_type]).__name__ @@ -91,7 +97,7 @@ def create_cyclic(options, name, values, datacube): cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) datacube._axes[name] = cyclic_axis_type datacube._axes[name].name = name - datacube._axes[name].range = options["cyclic"] + datacube._axes[name].range = cyclic_transform.range datacube.axis_counter += 1 @staticmethod diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index e81e436d6..7d37e7753 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -1,6 +1,7 @@ from copy import deepcopy -from datacube_transformations import DatacubeAxisTransformation +from ..backends.datacube import configure_datacube_axis +from .datacube_transformations import DatacubeAxisTransformation class DatacubeAxisCyclic(DatacubeAxisTransformation): @@ -9,6 +10,7 @@ class DatacubeAxisCyclic(DatacubeAxisTransformation): def __init__(self, name, cyclic_options): self.name = name self.transformation_options = cyclic_options + self.range = cyclic_options def generate_final_transformation(self): return self @@ -25,106 +27,14 @@ def apply_transformation(self, name, datacube, values): # OR, we generate a transformation, which we call in a new function create_axis. # In create_axis, if we have a cyclic transformation, we generate a cyclic axis, else, a standard one - pass - # Cyclic functions - - def to_intervals(self, range): - axis_lower = self.range[0] - axis_upper = self.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - intervals = [] - if lower < axis_upper: - # In this case, we want to go from lower to the first remapped cyclic axis upper - # or the asked upper range value. - # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, - # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper - # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], - # where -180 is the asked upper range value. - loops = int((axis_upper - lower) / axis_range) - remapped_up = axis_upper - (loops) * axis_range - new_upper = min(upper, remapped_up) - else: - # In this case, since lower >= axis_upper, we need to either go to the asked upper range - # or we need to go to the first remapped cyclic axis upper which is higher than lower - new_upper = min(axis_upper + axis_range, upper) - while new_upper < lower: - new_upper = min(new_upper + axis_range, upper) - intervals.append([lower, new_upper]) - # Now that we have established what the first interval should be, we should just jump from cyclic range - # to cyclic range until we hit the asked upper range value. - new_up = deepcopy(new_upper) - while new_up < upper: - new_upper = new_up - new_up = min(upper, new_upper + axis_range) - intervals.append([new_upper, new_up]) - # Once we have added all the in-between ranges, we need to add the last interval - intervals.append([new_up, upper]) - return intervals - - def remap_range_to_axis_range(self, range): - axis_lower = self.range[0] - axis_upper = self.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - if lower < axis_lower: - # In this case we need to calculate the number of loops between the axis lower - # and the lower to recenter the lower - loops = int((axis_lower - lower - self.tol) / axis_range) - return_lower = lower + (loops + 1) * axis_range - return_upper = upper + (loops + 1) * axis_range - elif lower >= axis_upper: - # In this case we need to calculate the number of loops between the axis upper - # and the lower to recenter the lower - loops = int((lower - axis_upper) / axis_range) - return_lower = lower - (loops + 1) * axis_range - return_upper = upper - (loops + 1) * axis_range + axis_options = deepcopy(datacube.axis_options[name]["transformation"]) + axis_options.pop("cyclic") + # Update the nested dictionary with the modified axis option for our axis + new_datacube_axis_options = deepcopy(datacube.axis_options) + # if we have no transformations left, then empty the transformation dico + if axis_options == {}: + new_datacube_axis_options[name] = {} else: - # In this case, the lower value is already in the right range - return_lower = lower - return_upper = upper - return [return_lower, return_upper] - - def remap_val_to_axis_range(self, value): - return_range = self.remap_range_to_axis_range([value, value]) - return return_range[0] - - def remap(self, range): - if self.range[0] - self.tol <= range[0] <= self.range[1] + self.tol: - if self.range[0] - self.tol <= range[1] <= self.range[1] + self.tol: - # If we are already in the cyclic range, return it - return [range] - elif abs(range[0] - range[1]) <= 2 * self.tol: - # If we have a range that is just one point, then it should still be counted - # and so we should take a small interval around it to find values inbetween - range = [ - self.remap_val_to_axis_range(range[0]) - self.tol, - self.remap_val_to_axis_range(range[0]) + self.tol, - ] - return [range] - range_intervals = self.to_intervals(range) - ranges = [] - for interval in range_intervals: - if abs(interval[0] - interval[1]) > 0: - # If the interval is not just a single point, we remap it to the axis range - range = self.remap_range_to_axis_range([interval[0], interval[1]]) - up = range[1] - low = range[0] - if up < low: - # Make sure we remap in the right order - ranges.append([up - self.tol, low + self.tol]) - else: - ranges.append([low - self.tol, up + self.tol]) - return ranges - - def offset(self, range): - # We first unpad the range by the axis tolerance to make sure that - # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. - # Also, it's safer that we find the offset of a value inside the range instead of on the border - unpadded_range = [range[0] + 1.5 * self.tol, range[1] - 1.5 * self.tol] - cyclic_range = self.remap_range_to_axis_range(unpadded_range) - offset = unpadded_range[0] - cyclic_range[0] - return offset + new_datacube_axis_options[name]["transformation"] = axis_options + configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 4e504995a..94ef8235e 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -11,7 +11,6 @@ def create_transformation(options, name, values, datacube): # But the last dictionary can vary and change according to transformation, which can be handled inside the # specialised transformations transformation_options = options["transformation"] - # NOTE: we do the following for each transformation of each axis for transformation_type_key in transformation_options.keys(): transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] @@ -65,6 +64,6 @@ def apply_transformation(self, name, datacube, values): pass -_type_to_datacube_transformation_lookup = {"mapper": "DatacubeMapper"} +_type_to_datacube_transformation_lookup = {"mapper": "DatacubeMapper", "cyclic": "DatacubeAxisCyclic"} -_type_to_transformation_file_lookup = {"mapper": "mappers"} +_type_to_transformation_file_lookup = {"mapper": "mappers", "cyclic": "cyclic"} diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index fc3aa7dd2..7559baded 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -21,7 +21,10 @@ def setup_method(self, method): "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1], }, ) - options = {"long": {"cyclic": [-1.1, -0.1]}, "level": {"cyclic": [1, 129]}} + options = { + "long": {"transformation": {"cyclic": [-1.1, -0.1]}}, + "level": {"transformation": {"cyclic": [1, 129]}}, + } self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 1634b0883..2a0b4143b 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -21,7 +21,10 @@ def setup_method(self, method): "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1], }, ) - options = {"long": {"cyclic": [-1.1, -0.1]}, "level": {"cyclic": [1, 129]}} + options = { + "long": {"transformation": {"cyclic": [-1.1, -0.1]}}, + "level": {"transformation": {"cyclic": [1, 129]}}, + } self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index 9bdea1c11..fcf213480 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -21,7 +21,7 @@ def setup_method(self, method): "long": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], }, ) - options = {"long": {"cyclic": [0, 1.0]}, "level": {"cyclic": [1, 129]}} + options = {"long": {"transformation": {"cyclic": [0, 1.0]}}, "level": {"transformation": {"cyclic": [1, 129]}}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) From 6c6a16bcaba998585782f472e7cd5e0359cba4aa Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 4 Aug 2023 17:11:47 +0200 Subject: [PATCH 084/332] add merge transformation without tests --- .../transformations/datacube_merger.py | 51 +++++++++++++++++++ .../datacube_transformations.py | 8 ++- 2 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 polytope/datacube/transformations/datacube_merger.py diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py new file mode 100644 index 000000000..01fe20704 --- /dev/null +++ b/polytope/datacube/transformations/datacube_merger.py @@ -0,0 +1,51 @@ +import numpy as np + +from ..backends.datacube import configure_datacube_axis +from .datacube_transformations import DatacubeAxisTransformation + + +class DatacubeAxisMerger(DatacubeAxisTransformation): + def __init__(self, name, merge_options): + self.transformation_options = merge_options + self.name = name + self._first_axis = name + self._second_axis = merge_options["with"] + self._linkers = merge_options["linkers"] + + def merged_values(self, values, datacube): + first_ax_vals = values + second_ax_name = self._second_axis + second_ax_vals = datacube.ax_vals(second_ax_name) + linkers = self._linkers + merged_values = [] + for first_val in first_ax_vals: + for second_val in second_ax_vals: + # TODO: check that the first and second val are strings + merged_values.append(first_val + linkers[0] + second_val + linkers[1]) + merged_values = np.array(merged_values) + return merged_values + + def apply_transformation(self, name, datacube, values): + merged_values = self.merged_values(values, datacube) + # Remove the merge option from the axis options since we have already handled it + # so do not want to handle it again + axis_options = datacube.axis_options[name]["transformation"] + axis_options.pop("merge") + # Update the nested dictionary with the modified axis option for our axis + new_datacube_axis_options = datacube.axis_options + new_datacube_axis_options[name]["transformation"] = axis_options + # Reconfigure the axis with the rest of its configurations + configure_datacube_axis(new_datacube_axis_options, name, merged_values, datacube) + self.finish_transformation(datacube, merged_values) + + def transformation_axes_final(self): + return [self._first_axis] + + def finish_transformation(self, datacube, values): + # Need to "delete" the second axis we do not use anymore + datacube.blocked_axes.append(self._second_axis) + # NOTE: we change the axis values here directly, which will not work with xarray + datacube.dataarray[self._first_axis] = values + + def generate_final_transformation(self): + return self diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 94ef8235e..9171b52b3 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -64,6 +64,10 @@ def apply_transformation(self, name, datacube, values): pass -_type_to_datacube_transformation_lookup = {"mapper": "DatacubeMapper", "cyclic": "DatacubeAxisCyclic"} +_type_to_datacube_transformation_lookup = { + "mapper": "DatacubeMapper", + "cyclic": "DatacubeAxisCyclic", + "merge": "DatacubeAxisMerger", +} -_type_to_transformation_file_lookup = {"mapper": "mappers", "cyclic": "cyclic"} +_type_to_transformation_file_lookup = {"mapper": "mappers", "cyclic": "cyclic", "merge": "merger"} From 0e5833adeba48cfc7d1f5fedfe1de4553d5dc698 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 4 Aug 2023 17:24:00 +0200 Subject: [PATCH 085/332] add test --- tests/test_merge_transformation.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/test_merge_transformation.py diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py new file mode 100644 index 000000000..a9488c2a5 --- /dev/null +++ b/tests/test_merge_transformation.py @@ -0,0 +1,29 @@ +import numpy as np +import xarray as xr + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Select + + +class TestSlicing4DXarrayDatacube: + def setup_method(self, method): + # Create a dataarray with 4 labelled axes using different index types + array = xr.DataArray( + np.random.randn(1, 1), + dims=("date", "time"), + coords={ + "date": ["2000-01-01"], + "time": ["06:00"], + }, + ) + options = {"date": {"transformation": {"merge": {"with": "time", "linkers": ["T", ":00"]}}}} + self.xarraydatacube = XArrayDatacube(array) + self.slicer = HullSlicer() + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + + def test_merge_axis(self): + request = Request(Select("date", ["2000-01-01T06:00:00"])) + result = self.API.retrieve(request) + assert result.flatten()["date"] == "2000-01-01T06:00:00" From 19d4382061e2c52c75acec1146680b70c2c038ba Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 4 Aug 2023 22:07:58 +0200 Subject: [PATCH 086/332] make merge transformation work --- polytope/datacube/backends/xarray.py | 3 ++- .../datacube/transformations/datacube_merger.py | 17 ++++++++++++----- tests/test_merge_transformation.py | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 9b0c8644a..f8bf60a17 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -233,7 +233,8 @@ def ax_vals(self, name): for _name, values in self.dataarray.coords.variables.items(): treated_axes.append(_name) if _name == name: - return values + # print(values.values) + return values.values for _name in self.dataarray.dims: if _name not in treated_axes: if _name == name: diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 01fe20704..faef6de26 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -1,3 +1,5 @@ +from copy import deepcopy + import numpy as np from ..backends.datacube import configure_datacube_axis @@ -13,7 +15,8 @@ def __init__(self, name, merge_options): self._linkers = merge_options["linkers"] def merged_values(self, values, datacube): - first_ax_vals = values + # first_ax_vals = values + first_ax_vals = datacube.ax_vals(self.name) second_ax_name = self._second_axis second_ax_vals = datacube.ax_vals(second_ax_name) linkers = self._linkers @@ -29,13 +32,17 @@ def apply_transformation(self, name, datacube, values): merged_values = self.merged_values(values, datacube) # Remove the merge option from the axis options since we have already handled it # so do not want to handle it again - axis_options = datacube.axis_options[name]["transformation"] + axis_options = deepcopy(datacube.axis_options[name]["transformation"]) axis_options.pop("merge") # Update the nested dictionary with the modified axis option for our axis - new_datacube_axis_options = datacube.axis_options - new_datacube_axis_options[name]["transformation"] = axis_options + new_datacube_axis_options = deepcopy(datacube.axis_options) + # new_datacube_axis_options[name]["transformation"] = axis_options + if axis_options == {}: + new_datacube_axis_options[name] = {} + else: + new_datacube_axis_options[name]["transformation"] = axis_options # Reconfigure the axis with the rest of its configurations - configure_datacube_axis(new_datacube_axis_options, name, merged_values, datacube) + configure_datacube_axis(new_datacube_axis_options[name], name, merged_values, datacube) self.finish_transformation(datacube, merged_values) def transformation_axes_final(self): diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index a9488c2a5..33e05e00f 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -26,4 +26,4 @@ def setup_method(self, method): def test_merge_axis(self): request = Request(Select("date", ["2000-01-01T06:00:00"])) result = self.API.retrieve(request) - assert result.flatten()["date"] == "2000-01-01T06:00:00" + assert result.leaves[0].flatten()["date"] == "2000-01-01T06:00:00" From 6b5efb3e7c4e276cad7b8c9b3dc0a994faa5e83a Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 7 Aug 2023 12:52:03 +0200 Subject: [PATCH 087/332] add reverse transformation and tests --- polytope/datacube/backends/xarray.py | 17 ++++++- polytope/datacube/datacube_axis.py | 12 +++-- .../transformations/datacube_merger.py | 1 + .../transformations/datacube_reverse.py | 50 +++++++++++++++++++ .../datacube_transformations.py | 6 ++- tests/test_cyclic_axis_over_negative_vals.py | 2 +- tests/test_cyclic_axis_slicer_not_0.py | 2 +- tests/test_reverse_transformation.py | 29 +++++++++++ tests/test_slicer_era5.py | 7 +-- 9 files changed, 114 insertions(+), 12 deletions(-) create mode 100644 polytope/datacube/transformations/datacube_reverse.py create mode 100644 tests/test_reverse_transformation.py diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index f8bf60a17..32ee0f1f8 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -5,6 +5,7 @@ from ...utility.combinatorics import unique, validate_axes from ..transformations.datacube_cyclic import DatacubeAxisCyclic from ..transformations.datacube_mappers import DatacubeMapper +from ..transformations.datacube_reverse import DatacubeAxisReverse from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis @@ -23,7 +24,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.transformation = {} for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: - self.dataarray = self.dataarray.sortby(name) + # self.dataarray = self.dataarray.sortby(name) options = axis_options.get(name, {}) configure_datacube_axis(options, name, values, self) treated_axes.append(name) @@ -121,6 +122,14 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, indexes_between = indexes[start:end].to_list() else: indexes_between = [i for i in indexes if low <= i <= up] + if isinstance(transform, DatacubeAxisReverse): + if axis.name in self.complete_axes: + sorted_indexes = indexes.sort_values() + start = sorted_indexes.searchsorted(low, "left") + end = sorted_indexes.searchsorted(up, "right") + indexes_between = sorted_indexes[start:end].to_list() + else: + indexes_between = [i for i in indexes if low <= i <= up] else: # Find the range of indexes between lower and upper @@ -189,6 +198,12 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): indexes = next(iter(subarray.xindexes.values())).to_pandas_index() else: indexes = [subarray[axis.name].values] + if isinstance(transform, DatacubeAxisReverse): + if axis.name in self.complete_axes: + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + else: + indexes = [subarray[axis.name].values] + else: if axis.name in self.complete_axes: indexes = next(iter(subarray.xindexes.values())).to_pandas_index() diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 5eab3744b..d1996ed6c 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -75,16 +75,18 @@ def offset(self, value: Any) -> int: @staticmethod def create_axis(name, values, datacube): + cyclic_transform = None + # First check if axis has any cyclicity transformation if name in datacube.transformation.keys(): axis_transforms = datacube.transformation[name] for transform in axis_transforms: from .transformations.datacube_cyclic import DatacubeAxisCyclic - if isinstance(transform, DatacubeAxisCyclic): - DatacubeAxis.create_cyclic(transform, name, values, datacube) - else: - # if we don't have a cyclic transform, then we create a standard axis - DatacubeAxis.create_standard(name, values, datacube) + cyclic_transform = transform + + if cyclic_transform is not None: + # the axis has a cyclic transformation + DatacubeAxis.create_cyclic(transform, name, values, datacube) else: DatacubeAxis.create_standard(name, values, datacube) diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index faef6de26..96e524912 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -52,6 +52,7 @@ def finish_transformation(self, datacube, values): # Need to "delete" the second axis we do not use anymore datacube.blocked_axes.append(self._second_axis) # NOTE: we change the axis values here directly, which will not work with xarray + # TODO: fix this? datacube.dataarray[self._first_axis] = values def generate_final_transformation(self): diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py new file mode 100644 index 000000000..733b97f4b --- /dev/null +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -0,0 +1,50 @@ +from copy import deepcopy + +from ..backends.datacube import configure_datacube_axis +from .datacube_transformations import DatacubeAxisTransformation + + +class DatacubeAxisReverse(DatacubeAxisTransformation): + + def __init__(self, name, mapper_options): + self.name = name + self.transformation_options = mapper_options + + def generate_final_transformation(self): + return self + + def apply_transformation(self, name, datacube, values): + # axis_options = deepcopy(datacube.axis_options[name]["transformation"]) + # axis_options.pop("reverse") + # # Update the nested dictionary with the modified axis option for our axis + # new_datacube_axis_options = deepcopy(datacube.axis_options) + # # if we have no transformations left, then empty the transformation dico + # if axis_options == {}: + # new_datacube_axis_options[name] = {} + # else: + # new_datacube_axis_options[name]["transformation"] = axis_options + # configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) + + reversed_values = self.reversed_values(datacube) + # Remove the merge option from the axis options since we have already handled it + # so do not want to handle it again + axis_options = deepcopy(datacube.axis_options[name]["transformation"]) + axis_options.pop("reverse") + # Update the nested dictionary with the modified axis option for our axis + new_datacube_axis_options = deepcopy(datacube.axis_options) + # new_datacube_axis_options[name]["transformation"] = axis_options + if axis_options == {}: + new_datacube_axis_options[name] = {} + else: + new_datacube_axis_options[name]["transformation"] = axis_options + # Reconfigure the axis with the rest of its configurations + configure_datacube_axis(new_datacube_axis_options[name], name, reversed_values, datacube) + + def transformation_axes_final(self): + return [self.name] + + def reversed_values(self, datacube): + # TODO: do we really need this? or can just use the normal values to configure axes? + vals = datacube.ax_vals(self.name) + reversed_vals = vals[::-1] + return reversed_vals diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 9171b52b3..595696abb 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -68,6 +68,10 @@ def apply_transformation(self, name, datacube, values): "mapper": "DatacubeMapper", "cyclic": "DatacubeAxisCyclic", "merge": "DatacubeAxisMerger", + "reverse": "DatacubeAxisReverse" } -_type_to_transformation_file_lookup = {"mapper": "mappers", "cyclic": "cyclic", "merge": "merger"} +_type_to_transformation_file_lookup = {"mapper": "mappers", + "cyclic": "cyclic", + "merge": "merger", + "reverse": "reverse"} diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index 7559baded..151065ad0 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -18,7 +18,7 @@ def setup_method(self, method): "date": pd.date_range("2000-01-01", "2000-01-03", 3), "step": [0, 3, 6, 9, 12, 15], "level": range(1, 130), - "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1], + "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1][::-1], }, ) options = { diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 2a0b4143b..3173ee77a 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -18,7 +18,7 @@ def setup_method(self, method): "date": pd.date_range("2000-01-01", "2000-01-03", 3), "step": [0, 3, 6, 9, 12, 15], "level": range(1, 130), - "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1], + "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1][::-1], }, ) options = { diff --git a/tests/test_reverse_transformation.py b/tests/test_reverse_transformation.py new file mode 100644 index 000000000..7b9151004 --- /dev/null +++ b/tests/test_reverse_transformation.py @@ -0,0 +1,29 @@ +import numpy as np +import xarray as xr + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Select + + +class TestSlicing4DXarrayDatacube: + def setup_method(self, method): + # Create a dataarray with 4 labelled axes using different index types + array = xr.DataArray( + np.random.randn(4), + dims=("lat"), + coords={ + "lat": [4, 3, 2, 1], + }, + ) + options = {"lat": {"transformation": {"reverse": {True}}}} + self.xarraydatacube = XArrayDatacube(array) + self.slicer = HullSlicer() + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + + def test_reverse_transformation(self): + request = Request(Select("lat", [1, 2, 3])) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 3 diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 06d34b63a..27346227e 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -13,17 +13,18 @@ def setup_method(self, method): array = ds.to_xarray().isel(step=0).t self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer) + options = {"lat": {"transformation": {"reverse": {True}}}} + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) def test_2D_box(self): request = Request( Box(["number", "isobaricInhPa"], [3, 0.0], [6, 1000.0]), Select("time", ["2017-01-02T12:00:00"]), - Box(["latitude", "longitude"], lower_corner=[10.0, 0.0], upper_corner=[0.0, 30.0]), + Box(["latitude", "longitude"], lower_corner=[0.0, 0.0], upper_corner=[10.0, 30.0]), Select("step", [np.timedelta64(0, "s")]), ) result = self.API.retrieve(request) - result.pprint() + # result.pprint() assert len(result.leaves) == 4 * 1 * 2 * 4 * 11 From 664353b38a8aad731b54fcb1b6d0cf263db53343 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 7 Aug 2023 13:55:34 +0200 Subject: [PATCH 088/332] make merge work for xarray without modifying the actual datacube --- polytope/datacube/backends/xarray.py | 93 ++++++++++++++++++- .../transformations/datacube_merger.py | 16 +++- 2 files changed, 101 insertions(+), 8 deletions(-) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 32ee0f1f8..a36c9fa6b 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -5,6 +5,7 @@ from ...utility.combinatorics import unique, validate_axes from ..transformations.datacube_cyclic import DatacubeAxisCyclic from ..transformations.datacube_mappers import DatacubeMapper +from ..transformations.datacube_merger import DatacubeAxisMerger from ..transformations.datacube_reverse import DatacubeAxisReverse from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis @@ -47,6 +48,7 @@ def get(self, requests: IndexTree): if len(path.items()) == self.axis_counter: # first, find the grid mapper transform grid_map_transform = None + merger_transform = None for key in path.keys(): if self.dataarray[key].dims == (): path.pop(key) @@ -55,6 +57,8 @@ def get(self, requests: IndexTree): for transform in axis_transforms: if isinstance(transform, DatacubeMapper): grid_map_transform = transform + if isinstance(transform, DatacubeAxisMerger): + merger_transform = transform if grid_map_transform is not None: # if we have a grid_mapper transform, find the new axis indices first_axis = grid_map_transform._mapped_axes()[0] @@ -70,8 +74,28 @@ def get(self, requests: IndexTree): value = subxarray.item() key = subxarray.name r.result = (key, value) + elif merger_transform is not None: + merged_ax = merger_transform._first_axis + merged_val = path[merged_ax] + removed_ax = merger_transform._second_axis + path.pop(merged_ax, None) + path.pop(removed_ax, None) + unmapped_first_val = merger_transform.unmerge(merged_val)[0] + unmapped_second_val = merger_transform.unmerge(merged_val)[1] + first_unmap_path = {merged_ax : unmapped_first_val} + second_unmap_path = {removed_ax : unmapped_second_val} + # Here, need to unmap the merged val into the two original merged axes + # and select these values + subxarray = self.dataarray.sel(path, method="nearest") + # subxarray = subxarray.isel(merged_ax=val) + subxarray = subxarray.sel(first_unmap_path) + subxarray = subxarray.sel(second_unmap_path) + value = subxarray.item() + key = subxarray.name + r.result = (key, value) else: # if we have no grid map, still need to assign values + print(path) subxarray = self.dataarray.sel(path, method="nearest") value = subxarray.item() key = subxarray.name @@ -130,6 +154,14 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, indexes_between = sorted_indexes[start:end].to_list() else: indexes_between = [i for i in indexes if low <= i <= up] + if isinstance(transform, DatacubeAxisMerger): + # TODO: does this work? + if axis.name in self.complete_axes: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + else: + indexes_between = [i for i in indexes if low <= i <= up] else: # Find the range of indexes between lower and upper @@ -169,6 +201,11 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): second_axis = transform._mapped_axes()[1] path.pop(first_axis, None) path.pop(second_axis, None) + if isinstance(transform, DatacubeAxisMerger): + # Need to remove the path key that it won't find because we created new + # merged values + merged_ax = transform._first_axis + path.pop(merged_ax, None) for key in path.keys(): if self.dataarray[key].dims == (): @@ -203,6 +240,8 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): indexes = next(iter(subarray.xindexes.values())).to_pandas_index() else: indexes = [subarray[axis.name].values] + if isinstance(transform, DatacubeAxisMerger): + indexes = [transform.merged_values(self)] else: if axis.name in self.complete_axes: @@ -232,9 +271,57 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - subarray = self.dataarray.sel(path)[axis.name] - subarray_vals = subarray.values - return index in subarray_vals + # subarray = self.dataarray.sel(path)[axis.name] + # subarray_vals = subarray.values + path = self.remap_path(path) + for key in path.keys(): + if self.dataarray[key].dims == (): + path.pop(key) + + if axis.name in self.transformation.keys(): + axis_transforms = self.transformation[axis.name] + for transform in axis_transforms: + if isinstance(transform, DatacubeMapper): + first_axis = transform._mapped_axes()[0] + second_axis = transform._mapped_axes()[1] + path.pop(first_axis, None) + path.pop(second_axis, None) + + for key in path.keys(): + if self.dataarray[key].dims == (): + path.pop(key) + + # Open a view on the subset identified by the path + subarray = self.dataarray.sel(path, method="nearest") + if axis.name in self.transformation.keys(): + axis_transforms = self.transformation[axis.name] + for transform in axis_transforms: + if isinstance(transform, DatacubeMapper): + first_axis = transform._mapped_axes()[0] + second_axis = transform._mapped_axes()[1] + if axis.name == first_axis: + indexes = [] + elif axis.name == second_axis: + indexes = [] + else: + if axis.name in self.complete_axes: + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + else: + indexes = [subarray[axis.name].values] + if isinstance(transform, DatacubeAxisCyclic): + if axis.name in self.complete_axes: + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + else: + indexes = [subarray[axis.name].values] + if isinstance(transform, DatacubeAxisReverse): + if axis.name in self.complete_axes: + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + else: + indexes = [subarray[axis.name].values] + if isinstance(transform, DatacubeAxisMerger): + indexes = [transform.merged_values(self)] + # return index in subarray_vals + return index in indexes @property def axes(self): diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 96e524912..77e58468e 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -14,7 +14,7 @@ def __init__(self, name, merge_options): self._second_axis = merge_options["with"] self._linkers = merge_options["linkers"] - def merged_values(self, values, datacube): + def merged_values(self, datacube): # first_ax_vals = values first_ax_vals = datacube.ax_vals(self.name) second_ax_name = self._second_axis @@ -29,7 +29,7 @@ def merged_values(self, values, datacube): return merged_values def apply_transformation(self, name, datacube, values): - merged_values = self.merged_values(values, datacube) + merged_values = self.merged_values(datacube) # Remove the merge option from the axis options since we have already handled it # so do not want to handle it again axis_options = deepcopy(datacube.axis_options[name]["transformation"]) @@ -51,9 +51,15 @@ def transformation_axes_final(self): def finish_transformation(self, datacube, values): # Need to "delete" the second axis we do not use anymore datacube.blocked_axes.append(self._second_axis) - # NOTE: we change the axis values here directly, which will not work with xarray - # TODO: fix this? - datacube.dataarray[self._first_axis] = values def generate_final_transformation(self): return self + + def unmerge(self, merged_val): + first_idx = merged_val.find(self._linkers[0]) + # second_idx = merged_val.find(self._linkers[1]) + first_val = merged_val[:first_idx] + first_linker_size = len(self._linkers[0]) + second_linked_size = len(self._linkers[1]) + second_val = merged_val[first_idx + first_linker_size:-second_linked_size] + return (first_val, second_val) From 9504777d8b5b500d0c9c9c5b3204a3f4c9fc1866 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 7 Aug 2023 14:10:24 +0200 Subject: [PATCH 089/332] create new branch for reverse transformation --- polytope/datacube/transformations/datacube_reverse.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index 733b97f4b..51de7581b 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -14,17 +14,6 @@ def generate_final_transformation(self): return self def apply_transformation(self, name, datacube, values): - # axis_options = deepcopy(datacube.axis_options[name]["transformation"]) - # axis_options.pop("reverse") - # # Update the nested dictionary with the modified axis option for our axis - # new_datacube_axis_options = deepcopy(datacube.axis_options) - # # if we have no transformations left, then empty the transformation dico - # if axis_options == {}: - # new_datacube_axis_options[name] = {} - # else: - # new_datacube_axis_options[name]["transformation"] = axis_options - # configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) - reversed_values = self.reversed_values(datacube) # Remove the merge option from the axis options since we have already handled it # so do not want to handle it again From e28ac0204fb3066553f41f759233cd0d0ac7ccd9 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 7 Aug 2023 14:19:19 +0200 Subject: [PATCH 090/332] black --- polytope/datacube/backends/xarray.py | 6 ++++-- polytope/datacube/datacube_axis.py | 1 + polytope/datacube/transformations/datacube_merger.py | 2 +- polytope/datacube/transformations/datacube_reverse.py | 1 - .../datacube/transformations/datacube_transformations.py | 7 ++----- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index a36c9fa6b..569682972 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -82,8 +82,8 @@ def get(self, requests: IndexTree): path.pop(removed_ax, None) unmapped_first_val = merger_transform.unmerge(merged_val)[0] unmapped_second_val = merger_transform.unmerge(merged_val)[1] - first_unmap_path = {merged_ax : unmapped_first_val} - second_unmap_path = {removed_ax : unmapped_second_val} + first_unmap_path = {merged_ax: unmapped_first_val} + second_unmap_path = {removed_ax: unmapped_second_val} # Here, need to unmap the merged val into the two original merged axes # and select these values subxarray = self.dataarray.sel(path, method="nearest") @@ -321,6 +321,8 @@ def has_index(self, path: DatacubePath, axis, index): if isinstance(transform, DatacubeAxisMerger): indexes = [transform.merged_values(self)] # return index in subarray_vals + else: + indexes = subarray[axis.name].values return index in indexes @property diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index d1996ed6c..ef9ea3c96 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -81,6 +81,7 @@ def create_axis(name, values, datacube): axis_transforms = datacube.transformation[name] for transform in axis_transforms: from .transformations.datacube_cyclic import DatacubeAxisCyclic + if isinstance(transform, DatacubeAxisCyclic): cyclic_transform = transform diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 77e58468e..03ecdfa13 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -61,5 +61,5 @@ def unmerge(self, merged_val): first_val = merged_val[:first_idx] first_linker_size = len(self._linkers[0]) second_linked_size = len(self._linkers[1]) - second_val = merged_val[first_idx + first_linker_size:-second_linked_size] + second_val = merged_val[first_idx + first_linker_size : -second_linked_size] return (first_val, second_val) diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index 51de7581b..c57872aba 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -5,7 +5,6 @@ class DatacubeAxisReverse(DatacubeAxisTransformation): - def __init__(self, name, mapper_options): self.name = name self.transformation_options = mapper_options diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 595696abb..61a9afb51 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -68,10 +68,7 @@ def apply_transformation(self, name, datacube, values): "mapper": "DatacubeMapper", "cyclic": "DatacubeAxisCyclic", "merge": "DatacubeAxisMerger", - "reverse": "DatacubeAxisReverse" + "reverse": "DatacubeAxisReverse", } -_type_to_transformation_file_lookup = {"mapper": "mappers", - "cyclic": "cyclic", - "merge": "merger", - "reverse": "reverse"} +_type_to_transformation_file_lookup = {"mapper": "mappers", "cyclic": "cyclic", "merge": "merger", "reverse": "reverse"} From e3329d5a44c5b64799fc12f0983f48621218d8ba Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 7 Aug 2023 16:28:46 +0200 Subject: [PATCH 091/332] refactor the xarray backend a bit --- polytope/datacube/backends/xarray.py | 46 +++++++++------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 569682972..ac3ebcae5 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -112,6 +112,15 @@ def remap_path(self, path: DatacubePath): path[key] = self._axes[key].remap_val_to_axis_range(value) return path + def _find_indexes_between(self, axis, indexes, low, up): + if axis.name in self.complete_axes: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + else: + indexes_between = [i for i in indexes if low <= i <= up] + return indexes_between + def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): idx_between = [] for i in range(len(search_ranges)): @@ -133,46 +142,21 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, elif axis.name == second_axis: indexes_between = transform.map_second_axis(first_val, low, up) else: - if axis.name in self.complete_axes: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - indexes_between = indexes[start:end].to_list() - else: - indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = self._find_indexes_between(axis, indexes, low, up) if isinstance(transform, DatacubeAxisCyclic): - if axis.name in self.complete_axes: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - indexes_between = indexes[start:end].to_list() - else: - indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = self._find_indexes_between(axis, indexes, low, up) if isinstance(transform, DatacubeAxisReverse): - if axis.name in self.complete_axes: - sorted_indexes = indexes.sort_values() - start = sorted_indexes.searchsorted(low, "left") - end = sorted_indexes.searchsorted(up, "right") - indexes_between = sorted_indexes[start:end].to_list() - else: - indexes_between = [i for i in indexes if low <= i <= up] + sorted_indexes = indexes.sort_values() + indexes_between = self._find_indexes_between(axis, sorted_indexes, low, up) if isinstance(transform, DatacubeAxisMerger): # TODO: does this work? - if axis.name in self.complete_axes: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - indexes_between = indexes[start:end].to_list() - else: - indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = self._find_indexes_between(axis, indexes, low, up) else: # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - if axis.name in self.complete_axes: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - indexes_between = indexes[start:end].to_list() - else: - indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = self._find_indexes_between(axis, indexes, low, up) # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them From 2f9fbb00d4c8974a78d82aa7dfee730339eca38f Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 7 Aug 2023 17:09:21 +0200 Subject: [PATCH 092/332] add TODO --- polytope/datacube/backends/datacube.py | 9 -------- polytope/datacube/backends/xarray.py | 4 +++- polytope/datacube/datacube_axis.py | 22 +++++-------------- .../transformations/datacube_mappers.py | 1 - .../transformations/datacube_merger.py | 2 -- .../transformations/datacube_reverse.py | 10 +-------- .../datacube_transformations.py | 1 - 7 files changed, 9 insertions(+), 40 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index db5a05786..e5f11742c 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -44,7 +44,6 @@ def validate(self, axes: List[str]) -> bool: def create(datacube, axis_options: dict): if isinstance(datacube, (xr.core.dataarray.DataArray, xr.core.dataset.Dataset)): from .xarray import XArrayDatacube - xadatacube = XArrayDatacube(datacube, axis_options=axis_options) return xadatacube else: @@ -58,13 +57,6 @@ def ax_vals(self, name: str) -> List: def configure_datacube_axis(options, name, values, datacube): - # TODO: this will not work with the generic transformation class anymore - # TODO: need to see where the axes are created now - - # NOTE: Maybe here, with new generic transformation class, - # instead of handling the options, we should just create an axis and look - # up the associated transformation to - # First need to check we are not initialising an axis which should not be initialised if name not in datacube.blocked_axes: # Now look at the options passed in @@ -74,5 +66,4 @@ def configure_datacube_axis(options, name, values, datacube): from ..transformations.datacube_transformations import ( DatacubeAxisTransformation, ) - DatacubeAxisTransformation.create_transformation(options, name, values, datacube) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index ac3ebcae5..9cc0f0672 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -106,6 +106,7 @@ def get(self, requests: IndexTree): def get_mapper(self, axis): return self._axes[axis] + # TODO: should this be in DatacubePath? def remap_path(self, path: DatacubePath): for key in path: value = path[key] @@ -133,6 +134,7 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, axis_transforms = self.transformation[axis.name] for transform in axis_transforms: if isinstance(transform, DatacubeMapper): + # TODO: add this in the transformations as a find_transformation_idx_between method # The if and for above loop through the transforms for that axis and # determine if there is a grid mapper transform first_axis = transform._mapped_axes()[0] @@ -204,6 +206,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): if axis.name in self.transformation.keys(): axis_transforms = self.transformation[axis.name] for transform in axis_transforms: + # TODO: put this within the transform as transformed_indices method if isinstance(transform, DatacubeMapper): if axis.name == first_axis: indexes = [] @@ -321,7 +324,6 @@ def ax_vals(self, name): for _name, values in self.dataarray.coords.variables.items(): treated_axes.append(_name) if _name == name: - # print(values.values) return values.values for _name in self.dataarray.dims: if _name not in treated_axes: diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index ef9ea3c96..55961e210 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -6,6 +6,10 @@ import numpy as np import pandas as pd +# TODO: for the cyclic axes, implement them as a decorator that can be turned on or off according to +# a special is_cyclic property of the DatacubeAxis +# TODO: maybe create dico of which axes can be cyclic too + class DatacubeAxis(ABC): @abstractproperty @@ -56,23 +60,6 @@ def to_cyclic_value(self, value: Any) -> Any: def offset(self, value: Any) -> int: pass - # @staticmethod - # def merge(options, name, values, datacube): - # # This function will not actually create an axis, it will compute values of when we merge the axes together - # # the merge options will look like "time": {"merge": {"with":"step", "linker": "00T"}} - # first_ax_vals = values - # second_ax_name = options["merge"]["with"] - # second_ax_vals = datacube.ax_vals(second_ax_name) - # linker = options["merge"]["linker"] - # merged_values = [] - # for first_val in first_ax_vals: - # for second_val in second_ax_vals: - # merged_val = first_val + linker + second_val - # merged_values.append(merged_val) - # merged_values = np.array(merged_values) - - # return merged_values - @staticmethod def create_axis(name, values, datacube): cyclic_transform = None @@ -94,6 +81,7 @@ def create_axis(name, values, datacube): @staticmethod def create_cyclic(cyclic_transform, name, values, datacube): values = np.array(values) + DatacubeAxis.check_axis_type(name, values) value_type = values.dtype.type axes_type_str = type(_type_to_axis_lookup[value_type]).__name__ axes_type_str += "Cyclic" diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index b752924ef..72cc73dba 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -26,7 +26,6 @@ def generate_final_transformation(self): return transformation def apply_transformation(self, name, datacube, values): - # TODO: how is this going to work once the cyclicity is a transformation? # Create mapped axes here transformation = self.generate_final_transformation() for i in range(len(transformation._mapped_axes)): diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 03ecdfa13..2a55c2890 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -15,7 +15,6 @@ def __init__(self, name, merge_options): self._linkers = merge_options["linkers"] def merged_values(self, datacube): - # first_ax_vals = values first_ax_vals = datacube.ax_vals(self.name) second_ax_name = self._second_axis second_ax_vals = datacube.ax_vals(second_ax_name) @@ -36,7 +35,6 @@ def apply_transformation(self, name, datacube, values): axis_options.pop("merge") # Update the nested dictionary with the modified axis option for our axis new_datacube_axis_options = deepcopy(datacube.axis_options) - # new_datacube_axis_options[name]["transformation"] = axis_options if axis_options == {}: new_datacube_axis_options[name] = {} else: diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index c57872aba..f25678e39 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -13,26 +13,18 @@ def generate_final_transformation(self): return self def apply_transformation(self, name, datacube, values): - reversed_values = self.reversed_values(datacube) # Remove the merge option from the axis options since we have already handled it # so do not want to handle it again axis_options = deepcopy(datacube.axis_options[name]["transformation"]) axis_options.pop("reverse") # Update the nested dictionary with the modified axis option for our axis new_datacube_axis_options = deepcopy(datacube.axis_options) - # new_datacube_axis_options[name]["transformation"] = axis_options if axis_options == {}: new_datacube_axis_options[name] = {} else: new_datacube_axis_options[name]["transformation"] = axis_options # Reconfigure the axis with the rest of its configurations - configure_datacube_axis(new_datacube_axis_options[name], name, reversed_values, datacube) + configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) def transformation_axes_final(self): return [self.name] - - def reversed_values(self, datacube): - # TODO: do we really need this? or can just use the normal values to configure axes? - vals = datacube.ax_vals(self.name) - reversed_vals = vals[::-1] - return reversed_vals diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 61a9afb51..b7fe64f4c 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -58,7 +58,6 @@ def generate_final_transformation(self): def transformation_axes_final(self): pass - # TODO: do we need this? to apply transformation to datacube yes... @abstractmethod def apply_transformation(self, name, datacube, values): pass From 2d5b7a27f2c74a596c282eb1ef3e1983c5a51ee0 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 8 Aug 2023 10:11:53 +0200 Subject: [PATCH 093/332] make cyclic axes a decorator --- polytope/datacube/backends/datacube.py | 2 + polytope/datacube/datacube_axis.py | 464 +++++++------------------ tests/test_axis_mappers.py | 5 +- 3 files changed, 140 insertions(+), 331 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index e5f11742c..22b65d272 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -44,6 +44,7 @@ def validate(self, axes: List[str]) -> bool: def create(datacube, axis_options: dict): if isinstance(datacube, (xr.core.dataarray.DataArray, xr.core.dataset.Dataset)): from .xarray import XArrayDatacube + xadatacube = XArrayDatacube(datacube, axis_options=axis_options) return xadatacube else: @@ -66,4 +67,5 @@ def configure_datacube_axis(options, name, values, datacube): from ..transformations.datacube_transformations import ( DatacubeAxisTransformation, ) + DatacubeAxisTransformation.create_transformation(options, name, values, datacube) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 55961e210..bc04bade7 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -1,4 +1,4 @@ -import sys +# import sys from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy from typing import Any, List @@ -6,12 +6,129 @@ import numpy as np import pandas as pd -# TODO: for the cyclic axes, implement them as a decorator that can be turned on or off according to -# a special is_cyclic property of the DatacubeAxis # TODO: maybe create dico of which axes can be cyclic too +def cyclic(cls): + if cls.is_cyclic: + + def to_intervals(range): + axis_lower = cls.range[0] + axis_upper = cls.range[1] + axis_range = axis_upper - axis_lower + lower = range[0] + upper = range[1] + intervals = [] + if lower < axis_upper: + # In this case, we want to go from lower to the first remapped cyclic axis upper + # or the asked upper range value. + # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, + # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper + # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], + # where -180 is the asked upper range value. + loops = int((axis_upper - lower) / axis_range) + remapped_up = axis_upper - (loops) * axis_range + new_upper = min(upper, remapped_up) + else: + # In this case, since lower >= axis_upper, we need to either go to the asked upper range + # or we need to go to the first remapped cyclic axis upper which is higher than lower + new_upper = min(axis_upper + axis_range, upper) + while new_upper < lower: + new_upper = min(new_upper + axis_range, upper) + intervals.append([lower, new_upper]) + # Now that we have established what the first interval should be, we should just jump from cyclic range + # to cyclic range until we hit the asked upper range value. + new_up = deepcopy(new_upper) + while new_up < upper: + new_upper = new_up + new_up = min(upper, new_upper + axis_range) + intervals.append([new_upper, new_up]) + # Once we have added all the in-between ranges, we need to add the last interval + intervals.append([new_up, upper]) + return intervals + + def remap_range_to_axis_range(range): + axis_lower = cls.range[0] + axis_upper = cls.range[1] + axis_range = axis_upper - axis_lower + lower = range[0] + upper = range[1] + if lower < axis_lower: + # In this case we need to calculate the number of loops between the axis lower + # and the lower to recenter the lower + loops = int((axis_lower - lower - cls.tol) / axis_range) + return_lower = lower + (loops + 1) * axis_range + return_upper = upper + (loops + 1) * axis_range + elif lower >= axis_upper: + # In this case we need to calculate the number of loops between the axis upper + # and the lower to recenter the lower + loops = int((lower - axis_upper) / axis_range) + return_lower = lower - (loops + 1) * axis_range + return_upper = upper - (loops + 1) * axis_range + else: + # In this case, the lower value is already in the right range + return_lower = lower + return_upper = upper + return [return_lower, return_upper] + + def remap_val_to_axis_range(value): + return_range = cls.remap_range_to_axis_range([value, value]) + return return_range[0] + + def remap(range: List): + if cls.range[0] - cls.tol <= range[0] <= cls.range[1] + cls.tol: + if cls.range[0] - cls.tol <= range[1] <= cls.range[1] + cls.tol: + # If we are already in the cyclic range, return it + return [range] + elif abs(range[0] - range[1]) <= 2 * cls.tol: + # If we have a range that is just one point, then it should still be counted + # and so we should take a small interval around it to find values inbetween + range = [ + cls.remap_val_to_axis_range(range[0]) - cls.tol, + cls.remap_val_to_axis_range(range[0]) + cls.tol, + ] + return [range] + range_intervals = cls.to_intervals(range) + ranges = [] + for interval in range_intervals: + if abs(interval[0] - interval[1]) > 0: + # If the interval is not just a single point, we remap it to the axis range + range = cls.remap_range_to_axis_range([interval[0], interval[1]]) + up = range[1] + low = range[0] + if up < low: + # Make sure we remap in the right order + ranges.append([up - cls.tol, low + cls.tol]) + else: + ranges.append([low - cls.tol, up + cls.tol]) + return ranges + + def offset(range): + # We first unpad the range by the axis tolerance to make sure that + # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. + # Also, it's safer that we find the offset of a value inside the range instead of on the border + unpadded_range = [range[0] + 1.5 * cls.tol, range[1] - 1.5 * cls.tol] + cyclic_range = cls.remap_range_to_axis_range(unpadded_range) + offset = unpadded_range[0] - cyclic_range[0] + return offset + + cls.to_intervals = to_intervals + cls.remap_range_to_axis_range = remap_range_to_axis_range + cls.remap_val_to_axis_range = remap_val_to_axis_range + cls.remap = remap + cls.offset = offset + + return cls + + class DatacubeAxis(ABC): + is_cyclic = False + + def update_axis(self): + if self.is_cyclic: + self = cyclic(self) + return self + @abstractproperty def name(self) -> str: pass @@ -42,23 +159,20 @@ def from_float(self, value: float) -> Any: def serialize(self, value: Any) -> Any: pass - def remap(self, range: List[Any]) -> Any: - pass - - def to_intervals(self, range: List[Any]) -> List[List[Any]]: - pass + def to_intervals(self, range): + return [range] - def remap_val_to_axis_range(self, value: Any) -> Any: - pass + def remap_val_to_axis_range(self, value): + return value - def remap_range_to_axis_range(self, range: List[Any]) -> List[Any]: - pass + def remap_range_to_axis_range(self, range): + return range - def to_cyclic_value(self, value: Any) -> Any: - pass + def remap(self, range: List) -> Any: + return [range] - def offset(self, value: Any) -> int: - pass + def offset(self, value): + return 0 @staticmethod def create_axis(name, values, datacube): @@ -82,11 +196,9 @@ def create_axis(name, values, datacube): def create_cyclic(cyclic_transform, name, values, datacube): values = np.array(values) DatacubeAxis.check_axis_type(name, values) - value_type = values.dtype.type - axes_type_str = type(_type_to_axis_lookup[value_type]).__name__ - axes_type_str += "Cyclic" - cyclic_axis_type = deepcopy(getattr(sys.modules["polytope.datacube.datacube_axis"], axes_type_str)()) - datacube._axes[name] = cyclic_axis_type + datacube._axes[name] = deepcopy(_type_to_axis_lookup[values.dtype.type]) + datacube._axes[name].is_cyclic = True + datacube._axes[name].update_axis() datacube._axes[name].name = name datacube._axes[name].range = cyclic_transform.range datacube.axis_counter += 1 @@ -106,6 +218,7 @@ def check_axis_type(name, values): raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") +@cyclic class IntDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 @@ -123,143 +236,8 @@ def from_float(self, value): def serialize(self, value): return value - def to_intervals(self, range): - return [range] - - def remap_val_to_axis_range(self, value): - return value - - def remap_range_to_axis_range(self, range): - return range - - def remap(self, range: List) -> Any: - return [range] - - def to_cyclic_value(self, value): - return value - - def offset(self, value): - return 0 - - -class IntDatacubeAxisCyclic(DatacubeAxis): - name = None - tol = 1e-12 - range = None - - def parse(self, value: Any) -> Any: - return float(value) - - def to_float(self, value): - return float(value) - - def from_float(self, value): - return float(value) - - def serialize(self, value): - return value - - def to_intervals(self, range): - axis_lower = self.range[0] - axis_upper = self.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - intervals = [] - if lower < axis_upper: - # In this case, we want to go from lower to the first remapped cyclic axis upper - # or the asked upper range value. - # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, - # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper - # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], - # where -180 is the asked upper range value. - loops = int((axis_upper - lower) / axis_range) - remapped_up = axis_upper - (loops) * axis_range - new_upper = min(upper, remapped_up) - else: - # In this case, since lower >= axis_upper, we need to either go to the asked upper range - # or we need to go to the first remapped cyclic axis upper which is higher than lower - new_upper = min(axis_upper + axis_range, upper) - while new_upper < lower: - new_upper = min(new_upper + axis_range, upper) - intervals.append([lower, new_upper]) - # Now that we have established what the first interval should be, we should just jump from cyclic range - # to cyclic range until we hit the asked upper range value. - new_up = deepcopy(new_upper) - while new_up < upper: - new_upper = new_up - new_up = min(upper, new_upper + axis_range) - intervals.append([new_upper, new_up]) - # Once we have added all the in-between ranges, we need to add the last interval - intervals.append([new_up, upper]) - return intervals - - def remap_range_to_axis_range(self, range): - axis_lower = self.range[0] - axis_upper = self.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - if lower < axis_lower: - # In this case we need to calculate the number of loops between the axis lower - # and the lower to recenter the lower - loops = int((axis_lower - lower - self.tol) / axis_range) - return_lower = lower + (loops + 1) * axis_range - return_upper = upper + (loops + 1) * axis_range - elif lower >= axis_upper: - # In this case we need to calculate the number of loops between the axis upper - # and the lower to recenter the lower - loops = int((lower - axis_upper) / axis_range) - return_lower = lower - (loops + 1) * axis_range - return_upper = upper - (loops + 1) * axis_range - else: - # In this case, the lower value is already in the right range - return_lower = lower - return_upper = upper - return [return_lower, return_upper] - - def remap_val_to_axis_range(self, value): - return_range = self.remap_range_to_axis_range([value, value]) - return return_range[0] - - def remap(self, range: List): - if abs(range[0] - range[1]) <= 2 * self.tol: - # If we have a range that is just one point, then it should still be counted - # and so we should take a small interval around it to find values inbetween - range = [ - self.remap_val_to_axis_range(range[0]) - self.tol, - self.remap_val_to_axis_range(range[0]) + self.tol, - ] - return [range] - if self.range[0] - self.tol <= range[0] <= self.range[1] + self.tol: - if self.range[0] - self.tol <= range[1] <= self.range[1] + self.tol: - # If we are in cyclic range, return it - return [range] - range_intervals = self.to_intervals(range) - ranges = [] - for interval in range_intervals: - if abs(interval[0] - interval[1]) > 0: - # If the interval is not just a single point, we remap it to the axis range - range = self.remap_range_to_axis_range([interval[0], interval[1]]) - up = range[1] - low = range[0] - if up < low: - # Make sure we remap in the right order - ranges.append([up - self.tol, low + self.tol]) - else: - ranges.append([low - self.tol, up + self.tol]) - return ranges - - def offset(self, range): - # We first unpad the range by the axis tolerance to make sure that - # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. - # Also, it's safer that we find the offset of a value inside the range instead of on the border. - unpadded_range = [range[0] + 1.5 * self.tol, range[1] - 1.5 * self.tol] - cyclic_range = self.remap_range_to_axis_range(unpadded_range) - offset = unpadded_range[0] - cyclic_range[0] - return offset - +@cyclic class FloatDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 @@ -277,145 +255,6 @@ def from_float(self, value): def serialize(self, value): return value - def to_intervals(self, range): - return [range] - - def remap_val_to_axis_range(self, value): - return value - - def remap_range_to_axis_range(self, range): - return range - - def remap(self, range: List) -> Any: - return [range] - - def to_cyclic_value(self, value): - return value - - def offset(self, value): - return 0 - - -class FloatDatacubeAxisCyclic(DatacubeAxis): - # Note that in the cyclic axis here, we only retain the lower part when we remap - # so for eg if the datacube has cyclic axis on [0,360] - # then if we want 360, we will in reality get back 0 (which is the same) - name = None - tol = 1e-12 - range = None - - def parse(self, value: Any) -> Any: - return float(value) - - def to_float(self, value): - return float(value) - - def from_float(self, value): - return float(value) - - def serialize(self, value): - return value - - def to_intervals(self, range): - axis_lower = self.range[0] - axis_upper = self.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - intervals = [] - if lower < axis_upper: - # In this case, we want to go from lower to the first remapped cyclic axis upper - # or the asked upper range value. - # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, - # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper - # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], - # where -180 is the asked upper range value. - loops = int((axis_upper - lower) / axis_range) - remapped_up = axis_upper - (loops) * axis_range - new_upper = min(upper, remapped_up) - else: - # In this case, since lower >= axis_upper, we need to either go to the asked upper range - # or we need to go to the first remapped cyclic axis upper which is higher than lower - new_upper = min(axis_upper + axis_range, upper) - while new_upper < lower: - new_upper = min(new_upper + axis_range, upper) - intervals.append([lower, new_upper]) - # Now that we have established what the first interval should be, we should just jump from cyclic range - # to cyclic range until we hit the asked upper range value. - new_up = deepcopy(new_upper) - while new_up < upper: - new_upper = new_up - new_up = min(upper, new_upper + axis_range) - intervals.append([new_upper, new_up]) - # Once we have added all the in-between ranges, we need to add the last interval - intervals.append([new_up, upper]) - return intervals - - def remap_range_to_axis_range(self, range): - axis_lower = self.range[0] - axis_upper = self.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - if lower < axis_lower: - # In this case we need to calculate the number of loops between the axis lower - # and the lower to recenter the lower - loops = int((axis_lower - lower - self.tol) / axis_range) - return_lower = lower + (loops + 1) * axis_range - return_upper = upper + (loops + 1) * axis_range - elif lower >= axis_upper: - # In this case we need to calculate the number of loops between the axis upper - # and the lower to recenter the lower - loops = int((lower - axis_upper) / axis_range) - return_lower = lower - (loops + 1) * axis_range - return_upper = upper - (loops + 1) * axis_range - else: - # In this case, the lower value is already in the right range - return_lower = lower - return_upper = upper - return [return_lower, return_upper] - - def remap_val_to_axis_range(self, value): - return_range = self.remap_range_to_axis_range([value, value]) - return return_range[0] - - def remap(self, range: List): - if self.range[0] - self.tol <= range[0] <= self.range[1] + self.tol: - if self.range[0] - self.tol <= range[1] <= self.range[1] + self.tol: - # If we are already in the cyclic range, return it - return [range] - elif abs(range[0] - range[1]) <= 2 * self.tol: - # If we have a range that is just one point, then it should still be counted - # and so we should take a small interval around it to find values inbetween - range = [ - self.remap_val_to_axis_range(range[0]) - self.tol, - self.remap_val_to_axis_range(range[0]) + self.tol, - ] - return [range] - range_intervals = self.to_intervals(range) - ranges = [] - for interval in range_intervals: - if abs(interval[0] - interval[1]) > 0: - # If the interval is not just a single point, we remap it to the axis range - range = self.remap_range_to_axis_range([interval[0], interval[1]]) - up = range[1] - low = range[0] - if up < low: - # Make sure we remap in the right order - ranges.append([up - self.tol, low + self.tol]) - else: - ranges.append([low - self.tol, up + self.tol]) - return ranges - - def offset(self, range): - # We first unpad the range by the axis tolerance to make sure that - # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. - # Also, it's safer that we find the offset of a value inside the range instead of on the border - unpadded_range = [range[0] + 1.5 * self.tol, range[1] - 1.5 * self.tol] - cyclic_range = self.remap_range_to_axis_range(unpadded_range) - offset = unpadded_range[0] - cyclic_range[0] - return offset - class PandasTimestampDatacubeAxis(DatacubeAxis): name = None @@ -439,21 +278,6 @@ def from_float(self, value): def serialize(self, value): return str(value) - def to_intervals(self, range): - return [range] - - def remap_val_to_axis_range(self, value): - return value - - def remap_range_to_axis_range(self, range): - return range - - def remap(self, range: List) -> Any: - return [range] - - def to_cyclic_value(self, value): - return value - def offset(self, value): return None @@ -480,21 +304,6 @@ def from_float(self, value): def serialize(self, value): return str(value) - def to_intervals(self, range): - return [range] - - def remap_val_to_axis_range(self, value): - return value - - def remap_range_to_axis_range(self, range): - return range - - def remap(self, range: List) -> Any: - return [range] - - def to_cyclic_value(self, value): - return value - def offset(self, value): return None @@ -516,9 +325,6 @@ def from_float(self, value): def serialize(self, value): raise TypeError("Tried to slice unsliceable axis") - def remap_val_to_axis_range(self, value): - return value - _type_to_axis_lookup = { pd.Int64Dtype: IntDatacubeAxis(), diff --git a/tests/test_axis_mappers.py b/tests/test_axis_mappers.py index 3f03f06e0..301769ec4 100644 --- a/tests/test_axis_mappers.py +++ b/tests/test_axis_mappers.py @@ -2,7 +2,6 @@ from polytope.datacube.datacube_axis import ( FloatDatacubeAxis, - FloatDatacubeAxisCyclic, IntDatacubeAxis, PandasTimedeltaDatacubeAxis, PandasTimestampDatacubeAxis, @@ -27,7 +26,9 @@ def test_float_axis(self): assert axis.remap([2, 3]) == [[2, 3]] def test_float_axis_cyclic(self): - axis = FloatDatacubeAxisCyclic() + axis = FloatDatacubeAxis() + axis.is_cyclic = True + axis = axis.update_axis() assert axis.parse(2) == 2.0 assert axis.to_float(2) == 2.0 assert axis.from_float(2) == 2.0 From 29ea2cafb435bf2401bf2213bae9c3434b304872 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 8 Aug 2023 16:40:22 +0200 Subject: [PATCH 094/332] add specific transformation functions within transformation classes --- polytope/datacube/backends/datacube.py | 4 + polytope/datacube/backends/mock.py | 3 + polytope/datacube/backends/xarray.py | 109 +++++------------- .../transformations/datacube_cyclic.py | 4 + .../transformations/datacube_mappers.py | 11 ++ .../transformations/datacube_merger.py | 4 + .../transformations/datacube_reverse.py | 5 + .../datacube_transformations.py | 4 + 8 files changed, 63 insertions(+), 81 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 22b65d272..2f23b46ab 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -54,6 +54,10 @@ def create(datacube, axis_options: dict): def ax_vals(self, name: str) -> List: pass + @abstractmethod + def _find_indexes_between(self, axis, indexes, low, up): + pass + # TODO: need to add transformation properties like the datacube.transformations dico diff --git a/polytope/datacube/backends/mock.py b/polytope/datacube/backends/mock.py index 3370b4028..351c0208e 100644 --- a/polytope/datacube/backends/mock.py +++ b/polytope/datacube/backends/mock.py @@ -63,3 +63,6 @@ def validate(self, axes): def ax_vals(self, name): return [] + + def _find_indexes_between(self, axis, indexes, low, up): + pass diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 9cc0f0672..22cb010e2 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -25,7 +25,6 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.transformation = {} for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: - # self.dataarray = self.dataarray.sortby(name) options = axis_options.get(name, {}) configure_datacube_axis(options, name, values, self) treated_axes.append(name) @@ -95,7 +94,6 @@ def get(self, requests: IndexTree): r.result = (key, value) else: # if we have no grid map, still need to assign values - print(path) subxarray = self.dataarray.sel(path, method="nearest") value = subxarray.item() key = subxarray.name @@ -115,6 +113,9 @@ def remap_path(self, path: DatacubePath): def _find_indexes_between(self, axis, indexes, low, up): if axis.name in self.complete_axes: + # Find the range of indexes between lower and upper + # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html + # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") indexes_between = indexes[start:end].to_list() @@ -133,31 +134,9 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, if axis.name in self.transformation.keys(): axis_transforms = self.transformation[axis.name] for transform in axis_transforms: - if isinstance(transform, DatacubeMapper): - # TODO: add this in the transformations as a find_transformation_idx_between method - # The if and for above loop through the transforms for that axis and - # determine if there is a grid mapper transform - first_axis = transform._mapped_axes()[0] - second_axis = transform._mapped_axes()[1] - if axis.name == first_axis: - indexes_between = transform.map_first_axis(low, up) - elif axis.name == second_axis: - indexes_between = transform.map_second_axis(first_val, low, up) - else: - indexes_between = self._find_indexes_between(axis, indexes, low, up) - if isinstance(transform, DatacubeAxisCyclic): - indexes_between = self._find_indexes_between(axis, indexes, low, up) - if isinstance(transform, DatacubeAxisReverse): - sorted_indexes = indexes.sort_values() - indexes_between = self._find_indexes_between(axis, sorted_indexes, low, up) - if isinstance(transform, DatacubeAxisMerger): - # TODO: does this work? - indexes_between = self._find_indexes_between(axis, indexes, low, up) - + indexes_between = transform._find_transformed_indices_between(axis, self, indexes, low, up, + first_val) else: - # Find the range of indexes between lower and upper - # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html - # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing indexes_between = self._find_indexes_between(axis, indexes, low, up) # Now the indexes_between are values on the cyclic range so need to remap them to their original @@ -167,16 +146,22 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, indexes_between[j] = indexes_between[j] else: indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) - idx_between.append(indexes_between[j]) return idx_between - def get_indices(self, path: DatacubePath, axis, lower, upper): + def datacube_natural_indexes(self, axis, subarray): + if axis.name in self.complete_axes: + indexes = next(iter(subarray.xindexes.values())).to_pandas_index() + else: + indexes = [subarray[axis.name].values] + return indexes + + def fit_path_to_datacube(self, axis, path): + # TODO: how to make this also work for the get method? path = self.remap_path(path) for key in path.keys(): if self.dataarray[key].dims == (): path.pop(key) - first_val = None if axis.name in self.transformation.keys(): axis_transforms = self.transformation[axis.name] @@ -192,10 +177,10 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): # merged values merged_ax = transform._first_axis path.pop(merged_ax, None) + return (path, first_val) - for key in path.keys(): - if self.dataarray[key].dims == (): - path.pop(key) + def get_indices(self, path: DatacubePath, axis, lower, upper): + (path, first_val) = self.fit_path_to_datacube(axis, path) # Open a view on the subset identified by the path subarray = self.dataarray.sel(path, method="nearest") @@ -208,33 +193,22 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): for transform in axis_transforms: # TODO: put this within the transform as transformed_indices method if isinstance(transform, DatacubeMapper): + first_axis = transform._mapped_axes()[0] + second_axis = transform._mapped_axes()[1] if axis.name == first_axis: indexes = [] elif axis.name == second_axis: indexes = [] else: - if axis.name in self.complete_axes: - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() - else: - indexes = [subarray[axis.name].values] + indexes = self.datacube_natural_indexes(axis, subarray) if isinstance(transform, DatacubeAxisCyclic): - if axis.name in self.complete_axes: - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() - else: - indexes = [subarray[axis.name].values] + indexes = self.datacube_natural_indexes(axis, subarray) if isinstance(transform, DatacubeAxisReverse): - if axis.name in self.complete_axes: - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() - else: - indexes = [subarray[axis.name].values] + indexes = self.datacube_natural_indexes(axis, subarray) if isinstance(transform, DatacubeAxisMerger): indexes = [transform.merged_values(self)] - else: - if axis.name in self.complete_axes: - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() - else: - indexes = [subarray[axis.name].values] + indexes = self.datacube_natural_indexes(axis, subarray) # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube search_ranges = axis.remap([lower, upper]) @@ -258,25 +232,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - # subarray = self.dataarray.sel(path)[axis.name] - # subarray_vals = subarray.values - path = self.remap_path(path) - for key in path.keys(): - if self.dataarray[key].dims == (): - path.pop(key) - - if axis.name in self.transformation.keys(): - axis_transforms = self.transformation[axis.name] - for transform in axis_transforms: - if isinstance(transform, DatacubeMapper): - first_axis = transform._mapped_axes()[0] - second_axis = transform._mapped_axes()[1] - path.pop(first_axis, None) - path.pop(second_axis, None) - - for key in path.keys(): - if self.dataarray[key].dims == (): - path.pop(key) + path = self.fit_path_to_datacube(axis, path)[0] # Open a view on the subset identified by the path subarray = self.dataarray.sel(path, method="nearest") @@ -291,25 +247,16 @@ def has_index(self, path: DatacubePath, axis, index): elif axis.name == second_axis: indexes = [] else: - if axis.name in self.complete_axes: - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() - else: - indexes = [subarray[axis.name].values] + indexes = self.datacube_natural_indexes(axis, subarray) if isinstance(transform, DatacubeAxisCyclic): - if axis.name in self.complete_axes: - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() - else: - indexes = [subarray[axis.name].values] + indexes = self.datacube_natural_indexes(axis, subarray) if isinstance(transform, DatacubeAxisReverse): - if axis.name in self.complete_axes: - indexes = next(iter(subarray.xindexes.values())).to_pandas_index() - else: - indexes = [subarray[axis.name].values] + indexes = self.datacube_natural_indexes(axis, subarray) if isinstance(transform, DatacubeAxisMerger): indexes = [transform.merged_values(self)] # return index in subarray_vals else: - indexes = subarray[axis.name].values + indexes = self.datacube_natural_indexes(axis, subarray) return index in indexes @property diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index 7d37e7753..7c30f2f16 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -38,3 +38,7 @@ def apply_transformation(self, name, datacube, values): else: new_datacube_axis_options[name]["transformation"] = axis_options configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) + + def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): + indexes_between = datacube._find_indexes_between(axis, indexes, low, up) + return indexes_between diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 72cc73dba..c75ff5fc1 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -71,6 +71,17 @@ def unmap(self, first_val, second_val): final_transformation = self.generate_final_transformation() return final_transformation.unmap(first_val, second_val) + def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): + first_axis = self._mapped_axes()[0] + second_axis = self._mapped_axes()[1] + if axis.name == first_axis: + indexes_between = self.map_first_axis(low, up) + elif axis.name == second_axis: + indexes_between = self.map_second_axis(first_val, low, up) + else: + indexes_between = datacube._find_indexes_between(axis, indexes, low, up) + return indexes_between + class OctahedralGridMapper(DatacubeMapper): def __init__(self, base_axis, mapped_axes, resolution): diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 2a55c2890..23619f774 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -61,3 +61,7 @@ def unmerge(self, merged_val): second_linked_size = len(self._linkers[1]) second_val = merged_val[first_idx + first_linker_size : -second_linked_size] return (first_val, second_val) + + def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): + indexes_between = datacube._find_indexes_between(axis, indexes, low, up) + return indexes_between diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index f25678e39..51116c22c 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -28,3 +28,8 @@ def apply_transformation(self, name, datacube, values): def transformation_axes_final(self): return [self.name] + + def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): + sorted_indexes = indexes.sort_values() + indexes_between = datacube._find_indexes_between(axis, sorted_indexes, low, up) + return indexes_between diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index b7fe64f4c..b7fb2243f 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -62,6 +62,10 @@ def transformation_axes_final(self): def apply_transformation(self, name, datacube, values): pass + @abstractmethod + def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): + pass + _type_to_datacube_transformation_lookup = { "mapper": "DatacubeMapper", From 7eeb5e03f7b6cad33fae102f17151fca0ae45046 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 8 Aug 2023 17:06:44 +0200 Subject: [PATCH 095/332] separate datacube backend and transformation more --- polytope/datacube/backends/xarray.py | 56 +------------------ .../transformations/datacube_cyclic.py | 7 +++ .../transformations/datacube_mappers.py | 19 +++++++ .../transformations/datacube_merger.py | 9 +++ .../transformations/datacube_reverse.py | 7 +++ .../datacube_transformations.py | 9 +++ 6 files changed, 54 insertions(+), 53 deletions(-) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 22cb010e2..ec3aa929a 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -3,10 +3,8 @@ import xarray as xr from ...utility.combinatorics import unique, validate_axes -from ..transformations.datacube_cyclic import DatacubeAxisCyclic from ..transformations.datacube_mappers import DatacubeMapper from ..transformations.datacube_merger import DatacubeAxisMerger -from ..transformations.datacube_reverse import DatacubeAxisReverse from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis @@ -130,7 +128,6 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, offset = search_ranges_offset[i] low = r[0] up = r[1] - if axis.name in self.transformation.keys(): axis_transforms = self.transformation[axis.name] for transform in axis_transforms: @@ -138,7 +135,6 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val) else: indexes_between = self._find_indexes_between(axis, indexes, low, up) - # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them for j in range(len(indexes_between)): @@ -166,68 +162,36 @@ def fit_path_to_datacube(self, axis, path): if axis.name in self.transformation.keys(): axis_transforms = self.transformation[axis.name] for transform in axis_transforms: - if isinstance(transform, DatacubeMapper): - first_axis = transform._mapped_axes()[0] - first_val = path.get(first_axis, None) - second_axis = transform._mapped_axes()[1] - path.pop(first_axis, None) - path.pop(second_axis, None) - if isinstance(transform, DatacubeAxisMerger): - # Need to remove the path key that it won't find because we created new - # merged values - merged_ax = transform._first_axis - path.pop(merged_ax, None) + (path, first_val) = transform._adjust_path(path) return (path, first_val) def get_indices(self, path: DatacubePath, axis, lower, upper): (path, first_val) = self.fit_path_to_datacube(axis, path) - # Open a view on the subset identified by the path subarray = self.dataarray.sel(path, method="nearest") - # Get the indexes of the axis we want to query # XArray does not support branching, so no need to use label, we just take the next axis - if axis.name in self.transformation.keys(): axis_transforms = self.transformation[axis.name] for transform in axis_transforms: # TODO: put this within the transform as transformed_indices method - if isinstance(transform, DatacubeMapper): - first_axis = transform._mapped_axes()[0] - second_axis = transform._mapped_axes()[1] - if axis.name == first_axis: - indexes = [] - elif axis.name == second_axis: - indexes = [] - else: - indexes = self.datacube_natural_indexes(axis, subarray) - if isinstance(transform, DatacubeAxisCyclic): - indexes = self.datacube_natural_indexes(axis, subarray) - if isinstance(transform, DatacubeAxisReverse): - indexes = self.datacube_natural_indexes(axis, subarray) - if isinstance(transform, DatacubeAxisMerger): - indexes = [transform.merged_values(self)] + indexes = transform._find_transformed_axis_indices(self, axis, subarray) else: indexes = self.datacube_natural_indexes(axis, subarray) - # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube search_ranges = axis.remap([lower, upper]) original_search_ranges = axis.to_intervals([lower, upper]) - # Find the offsets for each interval in the requested range, which we will need later search_ranges_offset = [] for r in original_search_ranges: offset = axis.offset(r) search_ranges_offset.append(offset) - # Look up the values in the datacube for each cyclic interval range idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) - # Remove duplicates even if difference of the order of the axis tolerance if offset is not None: # Note that we can only do unique if not dealing with time values idx_between = unique(idx_between) - return idx_between def has_index(self, path: DatacubePath, axis, index): @@ -239,21 +203,7 @@ def has_index(self, path: DatacubePath, axis, index): if axis.name in self.transformation.keys(): axis_transforms = self.transformation[axis.name] for transform in axis_transforms: - if isinstance(transform, DatacubeMapper): - first_axis = transform._mapped_axes()[0] - second_axis = transform._mapped_axes()[1] - if axis.name == first_axis: - indexes = [] - elif axis.name == second_axis: - indexes = [] - else: - indexes = self.datacube_natural_indexes(axis, subarray) - if isinstance(transform, DatacubeAxisCyclic): - indexes = self.datacube_natural_indexes(axis, subarray) - if isinstance(transform, DatacubeAxisReverse): - indexes = self.datacube_natural_indexes(axis, subarray) - if isinstance(transform, DatacubeAxisMerger): - indexes = [transform.merged_values(self)] + indexes = transform._find_transformed_axis_indices(self, axis, subarray) # return index in subarray_vals else: indexes = self.datacube_natural_indexes(axis, subarray) diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index 7c30f2f16..490d9aa4f 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -42,3 +42,10 @@ def apply_transformation(self, name, datacube, values): def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): indexes_between = datacube._find_indexes_between(axis, indexes, low, up) return indexes_between + + def _adjust_path(self, path): + return (path, None) + + def _find_transformed_axis_indices(self, datacube, axis, subarray): + indexes = datacube.datacube_natural_indexes(axis, subarray) + return indexes diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index c75ff5fc1..18c8f8828 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -82,6 +82,25 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi indexes_between = datacube._find_indexes_between(axis, indexes, low, up) return indexes_between + def _adjust_path(self, path): + first_axis = self._mapped_axes()[0] + first_val = path.get(first_axis, None) + second_axis = self._mapped_axes()[1] + path.pop(first_axis, None) + path.pop(second_axis, None) + return (path, first_val) + + def _find_transformed_axis_indices(self, datacube, axis, subarray): + first_axis = self._mapped_axes()[0] + second_axis = self._mapped_axes()[1] + if axis.name == first_axis: + indexes = [] + elif axis.name == second_axis: + indexes = [] + else: + indexes = datacube.datacube_natural_indexes(axis, subarray) + return indexes + class OctahedralGridMapper(DatacubeMapper): def __init__(self, base_axis, mapped_axes, resolution): diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 23619f774..332a8b8ef 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -65,3 +65,12 @@ def unmerge(self, merged_val): def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): indexes_between = datacube._find_indexes_between(axis, indexes, low, up) return indexes_between + + def _adjust_path(self, path): + merged_ax = self._first_axis + path.pop(merged_ax, None) + return (path, None) + + def _find_transformed_axis_indices(self, datacube, axis, subarray): + indexes = [self.merged_values(datacube)] + return indexes diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index 51116c22c..d78e36213 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -33,3 +33,10 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi sorted_indexes = indexes.sort_values() indexes_between = datacube._find_indexes_between(axis, sorted_indexes, low, up) return indexes_between + + def _adjust_path(self, path): + return (path, None) + + def _find_transformed_axis_indices(self, datacube, axis, subarray): + indexes = datacube.datacube_natural_indexes(axis, subarray) + return indexes diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index b7fb2243f..4de3f96f1 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -62,10 +62,19 @@ def transformation_axes_final(self): def apply_transformation(self, name, datacube, values): pass + # Methods to deal with transformation in datacube backends @abstractmethod def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): pass + @abstractmethod + def _adjust_path(self, path): + pass + + @abstractmethod + def _find_transformed_axis_indices(self, datacube, axis, subarray): + pass + _type_to_datacube_transformation_lookup = { "mapper": "DatacubeMapper", From 9976933246492d5ebb3162a1bf193ddc50a3ad31 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 9 Aug 2023 12:21:26 +0200 Subject: [PATCH 096/332] separate transformations and datacube backends --- polytope/datacube/backends/xarray.py | 84 +++++-------------- .../transformations/datacube_cyclic.py | 4 +- .../transformations/datacube_mappers.py | 11 ++- .../transformations/datacube_merger.py | 12 ++- .../transformations/datacube_reverse.py | 4 +- .../datacube_transformations.py | 2 +- 6 files changed, 46 insertions(+), 71 deletions(-) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index ec3aa929a..a616b90c4 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -3,8 +3,6 @@ import xarray as xr from ...utility.combinatorics import unique, validate_axes -from ..transformations.datacube_mappers import DatacubeMapper -from ..transformations.datacube_merger import DatacubeAxisMerger from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis @@ -41,61 +39,20 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() - path = self.remap_path(path) if len(path.items()) == self.axis_counter: # first, find the grid mapper transform - grid_map_transform = None - merger_transform = None + unmap_path = {} + considered_axes = [] for key in path.keys(): - if self.dataarray[key].dims == (): - path.pop(key) - if key in self.transformation.keys(): - axis_transforms = self.transformation[key] - for transform in axis_transforms: - if isinstance(transform, DatacubeMapper): - grid_map_transform = transform - if isinstance(transform, DatacubeAxisMerger): - merger_transform = transform - if grid_map_transform is not None: - # if we have a grid_mapper transform, find the new axis indices - first_axis = grid_map_transform._mapped_axes()[0] - first_val = path[first_axis] - second_axis = grid_map_transform._mapped_axes()[1] - second_val = path[second_axis] - path.pop(first_axis, None) - path.pop(second_axis, None) - subxarray = self.dataarray.sel(path, method="nearest") - # need to remap the lat, lon in path to dataarray index - unmapped_idx = grid_map_transform.unmap(first_val, second_val) - subxarray = subxarray.isel(values=unmapped_idx) - value = subxarray.item() - key = subxarray.name - r.result = (key, value) - elif merger_transform is not None: - merged_ax = merger_transform._first_axis - merged_val = path[merged_ax] - removed_ax = merger_transform._second_axis - path.pop(merged_ax, None) - path.pop(removed_ax, None) - unmapped_first_val = merger_transform.unmerge(merged_val)[0] - unmapped_second_val = merger_transform.unmerge(merged_val)[1] - first_unmap_path = {merged_ax: unmapped_first_val} - second_unmap_path = {removed_ax: unmapped_second_val} - # Here, need to unmap the merged val into the two original merged axes - # and select these values - subxarray = self.dataarray.sel(path, method="nearest") - # subxarray = subxarray.isel(merged_ax=val) - subxarray = subxarray.sel(first_unmap_path) - subxarray = subxarray.sel(second_unmap_path) - value = subxarray.item() - key = subxarray.name - r.result = (key, value) - else: - # if we have no grid map, still need to assign values - subxarray = self.dataarray.sel(path, method="nearest") - value = subxarray.item() - key = subxarray.name - r.result = (key, value) + (path, first_val, considered_axes, unmap_path) = self.fit_path_to_datacube( + key, path, considered_axes, unmap_path + ) + considered_axes.append(key) + subxarray = self.dataarray.sel(path, method="nearest") + subxarray = subxarray.sel(unmap_path) + value = subxarray.item() + key = subxarray.name + r.result = (key, value) else: r.remove_branch() @@ -131,8 +88,9 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, if axis.name in self.transformation.keys(): axis_transforms = self.transformation[axis.name] for transform in axis_transforms: - indexes_between = transform._find_transformed_indices_between(axis, self, indexes, low, up, - first_val) + indexes_between = transform._find_transformed_indices_between( + axis, self, indexes, low, up, first_val + ) else: indexes_between = self._find_indexes_between(axis, indexes, low, up) # Now the indexes_between are values on the cyclic range so need to remap them to their original @@ -152,21 +110,23 @@ def datacube_natural_indexes(self, axis, subarray): indexes = [subarray[axis.name].values] return indexes - def fit_path_to_datacube(self, axis, path): + def fit_path_to_datacube(self, axis_name, path, considered_axes=[], unmap_path={}): # TODO: how to make this also work for the get method? path = self.remap_path(path) for key in path.keys(): if self.dataarray[key].dims == (): path.pop(key) first_val = None - if axis.name in self.transformation.keys(): - axis_transforms = self.transformation[axis.name] + if axis_name in self.transformation.keys(): + axis_transforms = self.transformation[axis_name] for transform in axis_transforms: - (path, first_val) = transform._adjust_path(path) - return (path, first_val) + (path, first_val, considered_axes, unmap_path) = transform._adjust_path( + path, considered_axes, unmap_path + ) + return (path, first_val, considered_axes, unmap_path) def get_indices(self, path: DatacubePath, axis, lower, upper): - (path, first_val) = self.fit_path_to_datacube(axis, path) + (path, first_val, considered_axes, unmap_path) = self.fit_path_to_datacube(axis.name, path) # Open a view on the subset identified by the path subarray = self.dataarray.sel(path, method="nearest") # Get the indexes of the axis we want to query diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index 490d9aa4f..e936d3943 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -43,8 +43,8 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi indexes_between = datacube._find_indexes_between(axis, indexes, low, up) return indexes_between - def _adjust_path(self, path): - return (path, None) + def _adjust_path(self, path, considered_axes=[], unmap_path={}): + return (path, None, [], None) def _find_transformed_axis_indices(self, datacube, axis, subarray): indexes = datacube.datacube_natural_indexes(axis, subarray) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 18c8f8828..7df23b2d9 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -82,13 +82,20 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi indexes_between = datacube._find_indexes_between(axis, indexes, low, up) return indexes_between - def _adjust_path(self, path): + def _adjust_path(self, path, considered_axes=[], unmap_path={}): first_axis = self._mapped_axes()[0] first_val = path.get(first_axis, None) second_axis = self._mapped_axes()[1] + second_val = path.get(second_axis, None) path.pop(first_axis, None) path.pop(second_axis, None) - return (path, first_val) + considered_axes.append(first_axis) + considered_axes.append(second_axis) + # need to remap the lat, lon in path to dataarray index + if first_val is not None and second_val is not None: + unmapped_idx = self.unmap(first_val, second_val) + unmap_path[self.old_axis] = unmapped_idx + return (path, first_val, considered_axes, unmap_path) def _find_transformed_axis_indices(self, datacube, axis, subarray): first_axis = self._mapped_axes()[0] diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 332a8b8ef..a0c331177 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -66,10 +66,18 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi indexes_between = datacube._find_indexes_between(axis, indexes, low, up) return indexes_between - def _adjust_path(self, path): + def _adjust_path(self, path, considered_axes=[], unmap_path={}): merged_ax = self._first_axis + merged_val = path.get(merged_ax, None) + removed_ax = self._second_axis + path.pop(removed_ax, None) path.pop(merged_ax, None) - return (path, None) + if merged_val is not None: + unmapped_first_val = self.unmerge(merged_val)[0] + unmapped_second_val = self.unmerge(merged_val)[1] + unmap_path[merged_ax] = unmapped_first_val + unmap_path[removed_ax] = unmapped_second_val + return (path, None, considered_axes, unmap_path) def _find_transformed_axis_indices(self, datacube, axis, subarray): indexes = [self.merged_values(datacube)] diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index d78e36213..1b924a74a 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -34,8 +34,8 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi indexes_between = datacube._find_indexes_between(axis, sorted_indexes, low, up) return indexes_between - def _adjust_path(self, path): - return (path, None) + def _adjust_path(self, path, considered_axes=[], unmap_path={}): + return (path, None, [], None) def _find_transformed_axis_indices(self, datacube, axis, subarray): indexes = datacube.datacube_natural_indexes(axis, subarray) diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 4de3f96f1..8c3ca8c97 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -68,7 +68,7 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi pass @abstractmethod - def _adjust_path(self, path): + def _adjust_path(self, path, considered_axes=[], unmap_path={}): pass @abstractmethod From 548bf5fa017b9f577c722bf29a3f239b195e1e38 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 14 Aug 2023 09:55:13 +0200 Subject: [PATCH 097/332] add explanation --- polytope/datacube/transformations/datacube_transformations.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 8c3ca8c97..005859194 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -69,6 +69,9 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi @abstractmethod def _adjust_path(self, path, considered_axes=[], unmap_path={}): + # Some of the axes in the datacube should appear or disappear due to transformations + # When we ask the datacube for a path, we should thus remove these axes from the path + # But we want to keep track of the removed axes to be able to request data on the datacube still in unmap_path pass @abstractmethod From b26a4691b4683b0b1837a425f80bad93cb842b3d Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 14 Aug 2023 10:01:08 +0200 Subject: [PATCH 098/332] add explanation --- .../datacube/transformations/datacube_transformations.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 005859194..3967ccd02 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -65,6 +65,9 @@ def apply_transformation(self, name, datacube, values): # Methods to deal with transformation in datacube backends @abstractmethod def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): + # Some of the axes in the datacube appear or disappear due to transformations + # When we look up the datacube, for those axes, we should take particular care to find the right + # values between low and up pass @abstractmethod @@ -76,6 +79,9 @@ def _adjust_path(self, path, considered_axes=[], unmap_path={}): @abstractmethod def _find_transformed_axis_indices(self, datacube, axis, subarray): + # Some of the axes in the datacube appear or disappear due to transformations + # When we look up the datacube, for those axes, we should take particular care to find the right + # values which exist on those axes pass From f83eb2e2ff0b3c0fe2d292f89f339f5db941eb4f Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 14 Aug 2023 14:48:11 +0200 Subject: [PATCH 099/332] start supporting lookup of indices for multiple transformations on same axis --- polytope/datacube/backends/xarray.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index a616b90c4..b866ae955 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -1,6 +1,7 @@ import math import xarray as xr +from copy import deepcopy from ...utility.combinatorics import unique, validate_axes from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis @@ -87,10 +88,15 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, up = r[1] if axis.name in self.transformation.keys(): axis_transforms = self.transformation[axis.name] + temp_indexes = deepcopy(indexes) for transform in axis_transforms: - indexes_between = transform._find_transformed_indices_between( - axis, self, indexes, low, up, first_val + # indexes_between = transform._find_transformed_indices_between( + # axis, self, indexes, low, up, first_val + # ) + temp_indexes = transform._find_transformed_indices_between( + axis, self, temp_indexes, low, up, first_val ) + indexes_between = temp_indexes else: indexes_between = self._find_indexes_between(axis, indexes, low, up) # Now the indexes_between are values on the cyclic range so need to remap them to their original From d9a32fd7437e36e943c68a1061cd68dc09d570a1 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 15 Aug 2023 10:51:42 +0200 Subject: [PATCH 100/332] make multiple transformations on one axis work, but merge only works with unsliceable strings for now --- polytope/datacube/backends/xarray.py | 46 ++++++++++++++++--- .../transformations/datacube_cyclic.py | 15 +++--- .../transformations/datacube_mappers.py | 6 +-- .../transformations/datacube_merger.py | 6 +-- .../transformations/datacube_reverse.py | 15 +++--- .../datacube_transformations.py | 4 +- tests/test_merge_cyclic_octahedral.py | 42 +++++++++++++++++ tests/test_merge_octahedral_one_axis.py | 36 +++++++++++++++ tests/test_merge_transformation.py | 2 +- 9 files changed, 144 insertions(+), 28 deletions(-) create mode 100644 tests/test_merge_cyclic_octahedral.py create mode 100644 tests/test_merge_octahedral_one_axis.py diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index b866ae955..132cef2a5 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -1,7 +1,7 @@ import math +from copy import deepcopy import xarray as xr -from copy import deepcopy from ...utility.combinatorics import unique, validate_axes from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis @@ -51,6 +51,7 @@ def get(self, requests: IndexTree): considered_axes.append(key) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmap_path) + # print(subxarray) value = subxarray.item() key = subxarray.name r.result = (key, value) @@ -81,6 +82,8 @@ def _find_indexes_between(self, axis, indexes, low, up): def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): idx_between = [] + # print("inside look up datacube") + # print(search_ranges) for i in range(len(search_ranges)): r = search_ranges[i] offset = search_ranges_offset[i] @@ -93,12 +96,16 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, # indexes_between = transform._find_transformed_indices_between( # axis, self, indexes, low, up, first_val # ) - temp_indexes = transform._find_transformed_indices_between( - axis, self, temp_indexes, low, up, first_val + # print("inside look up") + # print(temp_indexes) + (offset, temp_indexes) = transform._find_transformed_indices_between( + axis, self, temp_indexes, low, up, first_val, offset ) indexes_between = temp_indexes else: indexes_between = self._find_indexes_between(axis, indexes, low, up) + print("INSIDE LOOK UP ") + print(indexes_between) # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them for j in range(len(indexes_between)): @@ -107,6 +114,8 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, else: indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) idx_between.append(indexes_between[j]) + print("IDX BETWEEN") + print(idx_between) return idx_between def datacube_natural_indexes(self, axis, subarray): @@ -119,29 +128,45 @@ def datacube_natural_indexes(self, axis, subarray): def fit_path_to_datacube(self, axis_name, path, considered_axes=[], unmap_path={}): # TODO: how to make this also work for the get method? path = self.remap_path(path) + # print(unmap_path) for key in path.keys(): + # print(key) if self.dataarray[key].dims == (): path.pop(key) first_val = None if axis_name in self.transformation.keys(): axis_transforms = self.transformation[axis_name] + first_val = None for transform in axis_transforms: - (path, first_val, considered_axes, unmap_path) = transform._adjust_path( + (path, temp_first_val, considered_axes, unmap_path) = transform._adjust_path( path, considered_axes, unmap_path ) + if temp_first_val: + first_val = temp_first_val return (path, first_val, considered_axes, unmap_path) def get_indices(self, path: DatacubePath, axis, lower, upper): + print("inside get_indices") (path, first_val, considered_axes, unmap_path) = self.fit_path_to_datacube(axis.name, path) + print(axis.name) + print(path) # Open a view on the subset identified by the path + # NOTE: THE UNMAP INFO DOES NOT COMMUNICATE BETWEEN UNSLICEABLE AND SLICEABLE CHILDREN NODES + # IS THIS OK BECAUSE UNSLICEABLE CHILDREN SHOULD NOT HAVE TRANSFORMATIONS? subarray = self.dataarray.sel(path, method="nearest") # Get the indexes of the axis we want to query # XArray does not support branching, so no need to use label, we just take the next axis if axis.name in self.transformation.keys(): axis_transforms = self.transformation[axis.name] + # This bool will help us decide for which axes we need to calculate the indexes again or not + # in case there are multiple relevant transformations for an axis + already_has_indexes = False for transform in axis_transforms: # TODO: put this within the transform as transformed_indices method - indexes = transform._find_transformed_axis_indices(self, axis, subarray) + indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) + # print("inside get_indices") + # print(indexes) + already_has_indexes = True else: indexes = self.datacube_natural_indexes(axis, subarray) # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube @@ -158,21 +183,28 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): if offset is not None: # Note that we can only do unique if not dealing with time values idx_between = unique(idx_between) + print(axis.name) + print(idx_between) return idx_between def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - path = self.fit_path_to_datacube(axis, path)[0] + path = self.fit_path_to_datacube(axis.name, path)[0] + print("inside has_index") + print(path) # Open a view on the subset identified by the path subarray = self.dataarray.sel(path, method="nearest") if axis.name in self.transformation.keys(): axis_transforms = self.transformation[axis.name] + already_has_indexes = False for transform in axis_transforms: - indexes = transform._find_transformed_axis_indices(self, axis, subarray) + indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) + already_has_indexes = True # return index in subarray_vals else: indexes = self.datacube_natural_indexes(axis, subarray) + print(index in indexes) return index in indexes @property diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index e936d3943..44f4f8f2a 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -39,13 +39,16 @@ def apply_transformation(self, name, datacube, values): new_datacube_axis_options[name]["transformation"] = axis_options configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) - def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): + def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): indexes_between = datacube._find_indexes_between(axis, indexes, low, up) - return indexes_between + return (offset, indexes_between) def _adjust_path(self, path, considered_axes=[], unmap_path={}): - return (path, None, [], None) + return (path, None, considered_axes, unmap_path) - def _find_transformed_axis_indices(self, datacube, axis, subarray): - indexes = datacube.datacube_natural_indexes(axis, subarray) - return indexes + def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): + if not already_has_indexes: + indexes = datacube.datacube_natural_indexes(axis, subarray) + return indexes + else: + pass diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 7df23b2d9..333bd88f9 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -71,7 +71,7 @@ def unmap(self, first_val, second_val): final_transformation = self.generate_final_transformation() return final_transformation.unmap(first_val, second_val) - def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): + def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): first_axis = self._mapped_axes()[0] second_axis = self._mapped_axes()[1] if axis.name == first_axis: @@ -80,7 +80,7 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi indexes_between = self.map_second_axis(first_val, low, up) else: indexes_between = datacube._find_indexes_between(axis, indexes, low, up) - return indexes_between + return (None, indexes_between) def _adjust_path(self, path, considered_axes=[], unmap_path={}): first_axis = self._mapped_axes()[0] @@ -97,7 +97,7 @@ def _adjust_path(self, path, considered_axes=[], unmap_path={}): unmap_path[self.old_axis] = unmapped_idx return (path, first_val, considered_axes, unmap_path) - def _find_transformed_axis_indices(self, datacube, axis, subarray): + def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): first_axis = self._mapped_axes()[0] second_axis = self._mapped_axes()[1] if axis.name == first_axis: diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index a0c331177..9faf13449 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -62,9 +62,9 @@ def unmerge(self, merged_val): second_val = merged_val[first_idx + first_linker_size : -second_linked_size] return (first_val, second_val) - def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): + def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): indexes_between = datacube._find_indexes_between(axis, indexes, low, up) - return indexes_between + return (offset, indexes_between) def _adjust_path(self, path, considered_axes=[], unmap_path={}): merged_ax = self._first_axis @@ -79,6 +79,6 @@ def _adjust_path(self, path, considered_axes=[], unmap_path={}): unmap_path[removed_ax] = unmapped_second_val return (path, None, considered_axes, unmap_path) - def _find_transformed_axis_indices(self, datacube, axis, subarray): + def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): indexes = [self.merged_values(datacube)] return indexes diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index 1b924a74a..c6e84990a 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -29,14 +29,17 @@ def apply_transformation(self, name, datacube, values): def transformation_axes_final(self): return [self.name] - def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): + def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): sorted_indexes = indexes.sort_values() indexes_between = datacube._find_indexes_between(axis, sorted_indexes, low, up) - return indexes_between + return (offset, indexes_between) def _adjust_path(self, path, considered_axes=[], unmap_path={}): - return (path, None, [], None) + return (path, None, considered_axes, unmap_path) - def _find_transformed_axis_indices(self, datacube, axis, subarray): - indexes = datacube.datacube_natural_indexes(axis, subarray) - return indexes + def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): + if not already_has_indexes: + indexes = datacube.datacube_natural_indexes(axis, subarray) + return indexes + else: + pass diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 3967ccd02..551d286d0 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -64,7 +64,7 @@ def apply_transformation(self, name, datacube, values): # Methods to deal with transformation in datacube backends @abstractmethod - def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val): + def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): # Some of the axes in the datacube appear or disappear due to transformations # When we look up the datacube, for those axes, we should take particular care to find the right # values between low and up @@ -78,7 +78,7 @@ def _adjust_path(self, path, considered_axes=[], unmap_path={}): pass @abstractmethod - def _find_transformed_axis_indices(self, datacube, axis, subarray): + def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): # Some of the axes in the datacube appear or disappear due to transformations # When we look up the datacube, for those axes, we should take particular care to find the right # values which exist on those axes diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py new file mode 100644 index 000000000..5c4bec030 --- /dev/null +++ b/tests/test_merge_cyclic_octahedral.py @@ -0,0 +1,42 @@ +import numpy as np +import pytest +import xarray as xr + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select, Span + + +class TestSlicing4DXarrayDatacube: + def setup_method(self, method): + # Create a dataarray with 4 labelled axes using different index types + array = xr.DataArray( + np.random.randn(1, 1, 2, 3), + dims=("date", "time", "values", "step"), + coords={ + "date": ["2000-01-01"], + "time": ["06:00"], + "values": [0, 1], + "step": [0, 1, 2], + }, + ) + options = {"date": {"transformation": {"merge": {"with": "time", "linkers": ["T", ":00"]}}}, + "values": {"transformation": {"mapper": {"type": "octahedral", + "resolution": 1280, + "axes": ["latitude", "longitude"]}}}, + "step": {"transformation": {"cyclic": [0, 1]}}, + } + self.xarraydatacube = XArrayDatacube(array) + self.slicer = HullSlicer() + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + + @pytest.mark.skip(reason="Need date time to not be strings") + def test_merge_axis(self): + # NOTE: does not work because the date is a string in the merge option... + request = Request(Select("date", ["2000-01-01T06:00:00"]), + Span("step", 0, 3), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2])) + result = self.API.retrieve(request) + result.pprint() + assert result.leaves[0].flatten()["date"] == "2000-01-01T06:00:00" diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py new file mode 100644 index 000000000..88b4c0dca --- /dev/null +++ b/tests/test_merge_octahedral_one_axis.py @@ -0,0 +1,36 @@ +from earthkit import data + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestSlicing4DXarrayDatacube: + def setup_method(self, method): + ds = data.from_source("file", "./tests/data/foo.grib") + latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) + latlon_array = latlon_array.t2m + self.xarraydatacube = XArrayDatacube(latlon_array) + grid_options = { + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + } + }, + "longitude": {"transformation": {"cyclic": [0, 360.0]}} + } + self.slicer = HullSlicer() + self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=grid_options) + + def test_merge_axis(self): + request = Request( + Select("number", [0]), + Select("time", ["2023-06-25T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2023-06-25T12:00:00"]), + Box(["latitude", "longitude"], [0, 359.8], [0.2, 361.2])) + result = self.API.retrieve(request) + # result.pprint() + assert result.leaves[0].flatten()["longitude"] == 0.0 diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index 33e05e00f..4057929df 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -7,7 +7,7 @@ from polytope.shapes import Select -class TestSlicing4DXarrayDatacube: +class TestMergeTransformation: def setup_method(self, method): # Create a dataarray with 4 labelled axes using different index types array = xr.DataArray( From 6acda47b569109157252819ebe46b80210eddd57 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 16 Aug 2023 09:27:51 +0200 Subject: [PATCH 101/332] make multiple transformations work --- polytope/datacube/backends/xarray.py | 96 +++++++++++++------ .../transformations/datacube_cyclic.py | 1 + .../transformations/datacube_mappers.py | 1 + .../transformations/datacube_merger.py | 10 +- tests/test_merge_cyclic_octahedral.py | 15 +-- tests/test_merge_transformation.py | 2 +- 6 files changed, 86 insertions(+), 39 deletions(-) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 132cef2a5..031f6c0b1 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -20,6 +20,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.complete_axes = [] self.blocked_axes = [] self.transformation = {} + self.fake_axes = [] for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: options = axis_options.get(name, {}) @@ -40,18 +41,23 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() + print(path) if len(path.items()) == self.axis_counter: # first, find the grid mapper transform unmap_path = {} considered_axes = [] - for key in path.keys(): - (path, first_val, considered_axes, unmap_path) = self.fit_path_to_datacube( - key, path, considered_axes, unmap_path + # for key in path.keys(): + if True: + # (path, first_val, considered_axes, unmap_path) = self.fit_path_to_datacube( + # key, path, considered_axes, unmap_path + # ) + (path, first_val, considered_axes, unmap_path) = self.fit_path_to_original_datacube( + path ) - considered_axes.append(key) + # considered_axes.append(key) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmap_path) - # print(subxarray) + print(subxarray) value = subxarray.item() key = subxarray.name r.result = (key, value) @@ -82,8 +88,6 @@ def _find_indexes_between(self, axis, indexes, low, up): def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): idx_between = [] - # print("inside look up datacube") - # print(search_ranges) for i in range(len(search_ranges)): r = search_ranges[i] offset = search_ranges_offset[i] @@ -93,19 +97,14 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, axis_transforms = self.transformation[axis.name] temp_indexes = deepcopy(indexes) for transform in axis_transforms: - # indexes_between = transform._find_transformed_indices_between( - # axis, self, indexes, low, up, first_val - # ) - # print("inside look up") - # print(temp_indexes) (offset, temp_indexes) = transform._find_transformed_indices_between( axis, self, temp_indexes, low, up, first_val, offset ) indexes_between = temp_indexes + # print("idx between") + # print(indexes_between) else: indexes_between = self._find_indexes_between(axis, indexes, low, up) - print("INSIDE LOOK UP ") - print(indexes_between) # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them for j in range(len(indexes_between)): @@ -114,8 +113,6 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, else: indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) idx_between.append(indexes_between[j]) - print("IDX BETWEEN") - print(idx_between) return idx_between def datacube_natural_indexes(self, axis, subarray): @@ -128,9 +125,7 @@ def datacube_natural_indexes(self, axis, subarray): def fit_path_to_datacube(self, axis_name, path, considered_axes=[], unmap_path={}): # TODO: how to make this also work for the get method? path = self.remap_path(path) - # print(unmap_path) for key in path.keys(): - # print(key) if self.dataarray[key].dims == (): path.pop(key) first_val = None @@ -145,15 +140,64 @@ def fit_path_to_datacube(self, axis_name, path, considered_axes=[], unmap_path={ first_val = temp_first_val return (path, first_val, considered_axes, unmap_path) + def fit_path_to_original_datacube(self, path): + path = self.remap_path(path) + # for key in path.keys(): + # # if key in self.dataarray.keys(): + # print(key) + # print(self.dataarray["step"]) + # if key in self.dataarray: + # if self.dataarray[key].dims == (): + # path.pop(key) + first_val = None + unmap_path = {} + considered_axes = [] + for axis_name in self.transformation.keys(): + axis_transforms = self.transformation[axis_name] + for transform in axis_transforms: + (path, temp_first_val, considered_axes, unmap_path) = transform._adjust_path( + path, considered_axes, unmap_path + ) + if temp_first_val: + first_val = temp_first_val + for key in path.keys(): + if self.dataarray[key].dims == (): + path.pop(key) + return (path, first_val, considered_axes, unmap_path) + def get_indices(self, path: DatacubePath, axis, lower, upper): - print("inside get_indices") - (path, first_val, considered_axes, unmap_path) = self.fit_path_to_datacube(axis.name, path) - print(axis.name) - print(path) + # print("inside get indices") + # print(path) + print(self.dataarray) + (path, first_val, considered_axes, unmap_path) = self.fit_path_to_original_datacube(path) + # print("HERE NOW") + # print(path) + # print(unmap_path) # Open a view on the subset identified by the path # NOTE: THE UNMAP INFO DOES NOT COMMUNICATE BETWEEN UNSLICEABLE AND SLICEABLE CHILDREN NODES # IS THIS OK BECAUSE UNSLICEABLE CHILDREN SHOULD NOT HAVE TRANSFORMATIONS? + # print("inside get_indices") + # print(path) + # print(self.dataarray["date"]) + # first_val = None + # unmap_path = {} + # considered_axes = [] + # for key in path.keys(): + # if key in self.fake_axes: + # path.pop(key) + # for key in path.keys(): + # print(key) + # (path, first_val, considered_axes, unmap_path) = self.fit_path_to_datacube( + # key, path, considered_axes, unmap_path + # ) + # print("here") + # print(path) + # TODO: here, now go through the axes transformations and call something like transformation.remove_axes() + # to remove the axes that do not actually exist on the datacube + # considered_axes.append(key) + subarray = self.dataarray.sel(path, method="nearest") + subarray = subarray.sel(unmap_path) # Get the indexes of the axis we want to query # XArray does not support branching, so no need to use label, we just take the next axis if axis.name in self.transformation.keys(): @@ -162,10 +206,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): # in case there are multiple relevant transformations for an axis already_has_indexes = False for transform in axis_transforms: - # TODO: put this within the transform as transformed_indices method indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) - # print("inside get_indices") - # print(indexes) already_has_indexes = True else: indexes = self.datacube_natural_indexes(axis, subarray) @@ -183,15 +224,11 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): if offset is not None: # Note that we can only do unique if not dealing with time values idx_between = unique(idx_between) - print(axis.name) - print(idx_between) return idx_between def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube path = self.fit_path_to_datacube(axis.name, path)[0] - print("inside has_index") - print(path) # Open a view on the subset identified by the path subarray = self.dataarray.sel(path, method="nearest") @@ -204,7 +241,6 @@ def has_index(self, path: DatacubePath, axis, index): # return index in subarray_vals else: indexes = self.datacube_natural_indexes(axis, subarray) - print(index in indexes) return index in indexes @property diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index 44f4f8f2a..bb842f14a 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -19,6 +19,7 @@ def transformation_axes_final(self): return [self.name] def apply_transformation(self, name, datacube, values): + print(name) # NOTE: we will handle all the cyclicity mapping here instead of in the DatacubeAxis # then we can generate just create_standard in the configure_axis at the end # Also, in the datacube implementations, we then have to deal with the transformation dico diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 333bd88f9..1ed66c595 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -39,6 +39,7 @@ def apply_transformation(self, name, datacube, values): # the values[0] will be a value on the first axis values = np.array(transformation.second_axis_vals(values[0])) configure_datacube_axis(new_axis_options, axis_name, values, datacube) + datacube.fake_axes.append(axis_name) def transformation_axes_final(self): final_transformation = self.generate_final_transformation() diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 9faf13449..bf746b2a0 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -23,7 +23,7 @@ def merged_values(self, datacube): for first_val in first_ax_vals: for second_val in second_ax_vals: # TODO: check that the first and second val are strings - merged_values.append(first_val + linkers[0] + second_val + linkers[1]) + merged_values.append(np.datetime64(first_val + linkers[0] + second_val + linkers[1])) merged_values = np.array(merged_values) return merged_values @@ -54,6 +54,7 @@ def generate_final_transformation(self): return self def unmerge(self, merged_val): + merged_val = str(merged_val) first_idx = merged_val.find(self._linkers[0]) # second_idx = merged_val.find(self._linkers[1]) first_val = merged_val[:first_idx] @@ -67,8 +68,11 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi return (offset, indexes_between) def _adjust_path(self, path, considered_axes=[], unmap_path={}): + print("inside adjust path") merged_ax = self._first_axis + print(merged_ax) merged_val = path.get(merged_ax, None) + print(merged_val) removed_ax = self._second_axis path.pop(removed_ax, None) path.pop(merged_ax, None) @@ -80,5 +84,7 @@ def _adjust_path(self, path, considered_axes=[], unmap_path={}): return (path, None, considered_axes, unmap_path) def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): - indexes = [self.merged_values(datacube)] + datacube.complete_axes.remove(axis.name) + print(self.merged_values(datacube)) + indexes = self.merged_values(datacube) return indexes diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index 5c4bec030..35a5489d0 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -12,12 +12,12 @@ class TestSlicing4DXarrayDatacube: def setup_method(self, method): # Create a dataarray with 4 labelled axes using different index types array = xr.DataArray( - np.random.randn(1, 1, 2, 3), + np.random.randn(1, 1, 4289589, 3), dims=("date", "time", "values", "step"), coords={ "date": ["2000-01-01"], "time": ["06:00"], - "values": [0, 1], + "values": list(range(4289589)), "step": [0, 1, 2], }, ) @@ -25,18 +25,21 @@ def setup_method(self, method): "values": {"transformation": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}}, - "step": {"transformation": {"cyclic": [0, 1]}}, + "step": {"transformation": {"cyclic": [0, 2]}}, } self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) - @pytest.mark.skip(reason="Need date time to not be strings") + # @pytest.mark.skip(reason="Need date time to not be strings") def test_merge_axis(self): # NOTE: does not work because the date is a string in the merge option... - request = Request(Select("date", ["2000-01-01T06:00:00"]), + date = np.datetime64("2000-01-01T06:00:00") + request = Request(Select("date", [date]), Span("step", 0, 3), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2])) result = self.API.retrieve(request) result.pprint() - assert result.leaves[0].flatten()["date"] == "2000-01-01T06:00:00" + assert result.leaves[0].flatten()["date"] == np.datetime64("2000-01-01T06:00:00") + for leaf in result.leaves: + assert leaf.flatten()["step"] in [0, 1, 2] diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index 4057929df..d7dcf95b2 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -26,4 +26,4 @@ def setup_method(self, method): def test_merge_axis(self): request = Request(Select("date", ["2000-01-01T06:00:00"])) result = self.API.retrieve(request) - assert result.leaves[0].flatten()["date"] == "2000-01-01T06:00:00" + assert result.leaves[0].flatten()["date"] == np.datetime64("2000-01-01T06:00:00") From e3a5d87c97c3f3bf4892ea45e23b95b58a1b929b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 16 Aug 2023 09:56:41 +0200 Subject: [PATCH 102/332] clean up and test --- polytope/datacube/backends/xarray.py | 44 ------------------- .../transformations/datacube_cyclic.py | 1 - .../transformations/datacube_merger.py | 4 -- tests/test_merge_cyclic_octahedral.py | 3 +- 4 files changed, 1 insertion(+), 51 deletions(-) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 031f6c0b1..fc16dd183 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -41,23 +41,16 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() - print(path) if len(path.items()) == self.axis_counter: # first, find the grid mapper transform unmap_path = {} considered_axes = [] - # for key in path.keys(): if True: - # (path, first_val, considered_axes, unmap_path) = self.fit_path_to_datacube( - # key, path, considered_axes, unmap_path - # ) (path, first_val, considered_axes, unmap_path) = self.fit_path_to_original_datacube( path ) - # considered_axes.append(key) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmap_path) - print(subxarray) value = subxarray.item() key = subxarray.name r.result = (key, value) @@ -101,8 +94,6 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, axis, self, temp_indexes, low, up, first_val, offset ) indexes_between = temp_indexes - # print("idx between") - # print(indexes_between) else: indexes_between = self._find_indexes_between(axis, indexes, low, up) # Now the indexes_between are values on the cyclic range so need to remap them to their original @@ -142,13 +133,6 @@ def fit_path_to_datacube(self, axis_name, path, considered_axes=[], unmap_path={ def fit_path_to_original_datacube(self, path): path = self.remap_path(path) - # for key in path.keys(): - # # if key in self.dataarray.keys(): - # print(key) - # print(self.dataarray["step"]) - # if key in self.dataarray: - # if self.dataarray[key].dims == (): - # path.pop(key) first_val = None unmap_path = {} considered_axes = [] @@ -166,35 +150,7 @@ def fit_path_to_original_datacube(self, path): return (path, first_val, considered_axes, unmap_path) def get_indices(self, path: DatacubePath, axis, lower, upper): - # print("inside get indices") - # print(path) - print(self.dataarray) (path, first_val, considered_axes, unmap_path) = self.fit_path_to_original_datacube(path) - # print("HERE NOW") - # print(path) - # print(unmap_path) - # Open a view on the subset identified by the path - # NOTE: THE UNMAP INFO DOES NOT COMMUNICATE BETWEEN UNSLICEABLE AND SLICEABLE CHILDREN NODES - # IS THIS OK BECAUSE UNSLICEABLE CHILDREN SHOULD NOT HAVE TRANSFORMATIONS? - # print("inside get_indices") - # print(path) - # print(self.dataarray["date"]) - # first_val = None - # unmap_path = {} - # considered_axes = [] - # for key in path.keys(): - # if key in self.fake_axes: - # path.pop(key) - # for key in path.keys(): - # print(key) - # (path, first_val, considered_axes, unmap_path) = self.fit_path_to_datacube( - # key, path, considered_axes, unmap_path - # ) - # print("here") - # print(path) - # TODO: here, now go through the axes transformations and call something like transformation.remove_axes() - # to remove the axes that do not actually exist on the datacube - # considered_axes.append(key) subarray = self.dataarray.sel(path, method="nearest") subarray = subarray.sel(unmap_path) diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index bb842f14a..44f4f8f2a 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -19,7 +19,6 @@ def transformation_axes_final(self): return [self.name] def apply_transformation(self, name, datacube, values): - print(name) # NOTE: we will handle all the cyclicity mapping here instead of in the DatacubeAxis # then we can generate just create_standard in the configure_axis at the end # Also, in the datacube implementations, we then have to deal with the transformation dico diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index bf746b2a0..60c5a7c3f 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -68,11 +68,8 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi return (offset, indexes_between) def _adjust_path(self, path, considered_axes=[], unmap_path={}): - print("inside adjust path") merged_ax = self._first_axis - print(merged_ax) merged_val = path.get(merged_ax, None) - print(merged_val) removed_ax = self._second_axis path.pop(removed_ax, None) path.pop(merged_ax, None) @@ -85,6 +82,5 @@ def _adjust_path(self, path, considered_axes=[], unmap_path={}): def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): datacube.complete_axes.remove(axis.name) - print(self.merged_values(datacube)) indexes = self.merged_values(datacube) return indexes diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index 35a5489d0..c8a71efa7 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -1,5 +1,4 @@ import numpy as np -import pytest import xarray as xr from polytope.datacube.backends.xarray import XArrayDatacube @@ -42,4 +41,4 @@ def test_merge_axis(self): result.pprint() assert result.leaves[0].flatten()["date"] == np.datetime64("2000-01-01T06:00:00") for leaf in result.leaves: - assert leaf.flatten()["step"] in [0, 1, 2] + assert leaf.flatten()["step"] in [0, 1, 2, 3] From d565335acc4d6000563bd27784bbb30a43475299 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 16 Aug 2023 13:34:50 +0200 Subject: [PATCH 103/332] add transformation which transforms str type axis indices to other types --- polytope/datacube/backends/xarray.py | 2 + .../transformations/datacube_type_change.py | 78 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 polytope/datacube/transformations/datacube_type_change.py diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index fc16dd183..c0ce61db1 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -162,6 +162,8 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): # in case there are multiple relevant transformations for an axis already_has_indexes = False for transform in axis_transforms: + # TODO: here, instead of creating the indices, would be better to create the standard datacube axes and + # then succesively map them to what they should be indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) already_has_indexes = True else: diff --git a/polytope/datacube/transformations/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change.py new file mode 100644 index 000000000..6d1883aed --- /dev/null +++ b/polytope/datacube/transformations/datacube_type_change.py @@ -0,0 +1,78 @@ +from copy import deepcopy +from importlib import import_module + +from ..backends.datacube import configure_datacube_axis +from .datacube_transformations import DatacubeAxisTransformation + + +class DatacubeAxisTypeChange(DatacubeAxisTransformation): + # The transformation here will be to point the old axes to the new cyclic axes + + def __init__(self, name, type_options): + self.name = name + self.transformation_options = type_options + self.new_type = type_options + + def generate_final_transformation(self): + map_type = _type_to_datacube_type_change_lookup[self.new_type] + module = import_module("polytope.datacube.transformations.datacube_mappers") + constructor = getattr(module, map_type) + transformation = deepcopy(constructor(self.name, self.new_type)) + return transformation + + def transformation_axes_final(self): + final_transformation = self.generate_final_transformation() + return [final_transformation.axis_name] + + def apply_transformation(self, name, datacube, values): + transformation = self.generate_final_transformation() + axis_options = deepcopy(datacube.axis_options[name]["transformation"]) + axis_options.pop("type_change") + # Update the nested dictionary with the modified axis option for our axis + new_datacube_axis_options = deepcopy(datacube.axis_options) + # if we have no transformations left, then empty the transformation dico + if axis_options == {}: + new_datacube_axis_options[name] = {} + else: + new_datacube_axis_options[name]["transformation"] = axis_options + values = [transformation.transform_type(values[0])] # only need 1 value to determine type in datacube config + configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) + + def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): + # NOTE: needs to be in new type + indexes_between = datacube._find_indexes_between(axis, indexes, low, up) + return (offset, indexes_between) + + def _adjust_path(self, path, considered_axes=[], unmap_path={}): + # NOTE: used to access values, so path for the axis needs to be replaced to original type index + axis_new_val = path.get(self.name, None) + axis_val_str = str(axis_new_val) + path.pop(self.name) + path[self.name] = axis_val_str + return (path, None, considered_axes, unmap_path) + + def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): + # NOTE: needs to be in new type + transformation = self.generate_final_transformation() + if not already_has_indexes: + indexes = datacube.datacube_natural_indexes(axis, subarray) + indexes = [transformation.transform_type(i) for i in indexes] + return indexes + else: + pass + + +class TypeChangeStrToInt(DatacubeAxisTypeChange): + + def __init__(self, axis_name, new_type): + self.axis_name = axis_name + self._new_type = new_type + + def transform_type(self, value): + return int(value) + + def make_str(self, value): + return str(value) + + +_type_to_datacube_type_change_lookup = {"int" : "TypeChangeStrToInt"} From fedb29fc87262850f29a00c75fceb2771356cf73 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 16 Aug 2023 14:49:08 +0200 Subject: [PATCH 104/332] fix returned path in get to select string indices --- polytope/datacube/backends/xarray.py | 23 +++++++++----- .../transformations/datacube_cyclic.py | 4 +-- .../transformations/datacube_mappers.py | 4 +-- .../transformations/datacube_merger.py | 4 +-- .../transformations/datacube_reverse.py | 4 +-- .../datacube_transformations.py | 10 +++++-- .../transformations/datacube_type_change.py | 23 +++++++++----- tests/test_type_change_transformation.py | 30 +++++++++++++++++++ 8 files changed, 76 insertions(+), 26 deletions(-) create mode 100644 tests/test_type_change_transformation.py diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index c0ce61db1..062bc84e6 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -46,11 +46,15 @@ def get(self, requests: IndexTree): unmap_path = {} considered_axes = [] if True: - (path, first_val, considered_axes, unmap_path) = self.fit_path_to_original_datacube( + (path, first_val, considered_axes, unmap_path, + changed_type_path) = self.fit_path_to_original_datacube( path ) + print(path) + print(self.dataarray) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmap_path) + subxarray = subxarray.sel(changed_type_path) value = subxarray.item() key = subxarray.name r.result = (key, value) @@ -120,40 +124,43 @@ def fit_path_to_datacube(self, axis_name, path, considered_axes=[], unmap_path={ if self.dataarray[key].dims == (): path.pop(key) first_val = None + changed_type_path = {} if axis_name in self.transformation.keys(): axis_transforms = self.transformation[axis_name] first_val = None for transform in axis_transforms: - (path, temp_first_val, considered_axes, unmap_path) = transform._adjust_path( - path, considered_axes, unmap_path + (path, temp_first_val, considered_axes, unmap_path, changed_type_path) = transform._adjust_path( + path, considered_axes, unmap_path, changed_type_path ) if temp_first_val: first_val = temp_first_val - return (path, first_val, considered_axes, unmap_path) + return (path, first_val, considered_axes, unmap_path, changed_type_path) def fit_path_to_original_datacube(self, path): path = self.remap_path(path) first_val = None unmap_path = {} considered_axes = [] + changed_type_path = {} for axis_name in self.transformation.keys(): axis_transforms = self.transformation[axis_name] for transform in axis_transforms: - (path, temp_first_val, considered_axes, unmap_path) = transform._adjust_path( - path, considered_axes, unmap_path + (path, temp_first_val, considered_axes, unmap_path, changed_type_path) = transform._adjust_path( + path, considered_axes, unmap_path, changed_type_path ) if temp_first_val: first_val = temp_first_val for key in path.keys(): if self.dataarray[key].dims == (): path.pop(key) - return (path, first_val, considered_axes, unmap_path) + return (path, first_val, considered_axes, unmap_path, changed_type_path) def get_indices(self, path: DatacubePath, axis, lower, upper): - (path, first_val, considered_axes, unmap_path) = self.fit_path_to_original_datacube(path) + (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube(path) subarray = self.dataarray.sel(path, method="nearest") subarray = subarray.sel(unmap_path) + subarray = subarray.sel(changed_type_path) # Get the indexes of the axis we want to query # XArray does not support branching, so no need to use label, we just take the next axis if axis.name in self.transformation.keys(): diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index 44f4f8f2a..434f02a0c 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -43,8 +43,8 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi indexes_between = datacube._find_indexes_between(axis, indexes, low, up) return (offset, indexes_between) - def _adjust_path(self, path, considered_axes=[], unmap_path={}): - return (path, None, considered_axes, unmap_path) + def _adjust_path(self, path, considered_axes=[], unmap_path={}, changed_type_path={}): + return (path, None, considered_axes, unmap_path, changed_type_path) def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): if not already_has_indexes: diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 1ed66c595..629298160 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -83,7 +83,7 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi indexes_between = datacube._find_indexes_between(axis, indexes, low, up) return (None, indexes_between) - def _adjust_path(self, path, considered_axes=[], unmap_path={}): + def _adjust_path(self, path, considered_axes=[], unmap_path={}, changed_type_path={}): first_axis = self._mapped_axes()[0] first_val = path.get(first_axis, None) second_axis = self._mapped_axes()[1] @@ -96,7 +96,7 @@ def _adjust_path(self, path, considered_axes=[], unmap_path={}): if first_val is not None and second_val is not None: unmapped_idx = self.unmap(first_val, second_val) unmap_path[self.old_axis] = unmapped_idx - return (path, first_val, considered_axes, unmap_path) + return (path, first_val, considered_axes, unmap_path, changed_type_path) def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): first_axis = self._mapped_axes()[0] diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 60c5a7c3f..474fa45d1 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -67,7 +67,7 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi indexes_between = datacube._find_indexes_between(axis, indexes, low, up) return (offset, indexes_between) - def _adjust_path(self, path, considered_axes=[], unmap_path={}): + def _adjust_path(self, path, considered_axes=[], unmap_path={}, changed_type_path={}): merged_ax = self._first_axis merged_val = path.get(merged_ax, None) removed_ax = self._second_axis @@ -78,7 +78,7 @@ def _adjust_path(self, path, considered_axes=[], unmap_path={}): unmapped_second_val = self.unmerge(merged_val)[1] unmap_path[merged_ax] = unmapped_first_val unmap_path[removed_ax] = unmapped_second_val - return (path, None, considered_axes, unmap_path) + return (path, None, considered_axes, unmap_path, changed_type_path) def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): datacube.complete_axes.remove(axis.name) diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index c6e84990a..742306d9a 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -34,8 +34,8 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi indexes_between = datacube._find_indexes_between(axis, sorted_indexes, low, up) return (offset, indexes_between) - def _adjust_path(self, path, considered_axes=[], unmap_path={}): - return (path, None, considered_axes, unmap_path) + def _adjust_path(self, path, considered_axes=[], unmap_path={}, changed_type_path={}): + return (path, None, considered_axes, unmap_path, changed_type_path) def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): if not already_has_indexes: diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 551d286d0..ed420aabd 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -71,7 +71,7 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi pass @abstractmethod - def _adjust_path(self, path, considered_axes=[], unmap_path={}): + def _adjust_path(self, path, considered_axes=[], unmap_path={}, changed_type_path={}): # Some of the axes in the datacube should appear or disappear due to transformations # When we ask the datacube for a path, we should thus remove these axes from the path # But we want to keep track of the removed axes to be able to request data on the datacube still in unmap_path @@ -90,6 +90,12 @@ def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_i "cyclic": "DatacubeAxisCyclic", "merge": "DatacubeAxisMerger", "reverse": "DatacubeAxisReverse", + "type_change" : "DatacubeAxisTypeChange" } -_type_to_transformation_file_lookup = {"mapper": "mappers", "cyclic": "cyclic", "merge": "merger", "reverse": "reverse"} +_type_to_transformation_file_lookup = {"mapper": "mappers", + "cyclic": "cyclic", + "merge": "merger", + "reverse": "reverse", + "type_change" : "type_change" + } diff --git a/polytope/datacube/transformations/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change.py index 6d1883aed..5f3dd077a 100644 --- a/polytope/datacube/transformations/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change.py @@ -15,7 +15,7 @@ def __init__(self, name, type_options): def generate_final_transformation(self): map_type = _type_to_datacube_type_change_lookup[self.new_type] - module = import_module("polytope.datacube.transformations.datacube_mappers") + module = import_module("polytope.datacube.transformations.datacube_type_change") constructor = getattr(module, map_type) transformation = deepcopy(constructor(self.name, self.new_type)) return transformation @@ -40,23 +40,30 @@ def apply_transformation(self, name, datacube, values): def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): # NOTE: needs to be in new type - indexes_between = datacube._find_indexes_between(axis, indexes, low, up) + if axis.name == self.name: + transformation = self.generate_final_transformation() + indexes_between = [transformation.transform_type(i) for i in indexes + if low <= transformation.transform_type(i) <= up] + # indexes_between = [] + else: + indexes_between = datacube._find_indexes_between(axis, indexes, low, up) return (offset, indexes_between) - def _adjust_path(self, path, considered_axes=[], unmap_path={}): + def _adjust_path(self, path, considered_axes=[], unmap_path={}, changed_type_path={}): # NOTE: used to access values, so path for the axis needs to be replaced to original type index axis_new_val = path.get(self.name, None) axis_val_str = str(axis_new_val) - path.pop(self.name) - path[self.name] = axis_val_str - return (path, None, considered_axes, unmap_path) + if axis_new_val is not None: + path.pop(self.name) + changed_type_path[self.name] = axis_val_str + return (path, None, considered_axes, unmap_path, changed_type_path) def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): # NOTE: needs to be in new type - transformation = self.generate_final_transformation() + # transformation = self.generate_final_transformation() if not already_has_indexes: indexes = datacube.datacube_natural_indexes(axis, subarray) - indexes = [transformation.transform_type(i) for i in indexes] + # indexes = [transformation.transform_type(i) for i in indexes] return indexes else: pass diff --git a/tests/test_type_change_transformation.py b/tests/test_type_change_transformation.py new file mode 100644 index 000000000..fc7152452 --- /dev/null +++ b/tests/test_type_change_transformation.py @@ -0,0 +1,30 @@ +import numpy as np +import xarray as xr + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Select + + +class TestMergeTransformation: + def setup_method(self, method): + # Create a dataarray with 4 labelled axes using different index types + array = xr.DataArray( + np.random.randn(2), + dims=("step"), + coords={ + "step": ["0", "1"], + }, + ) + self.array = array + options = {"step": {"transformation": {"type_change": "int"}}} + self.xarraydatacube = XArrayDatacube(array) + self.slicer = HullSlicer() + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + + def test_merge_axis(self): + request = Request(Select("step", [0])) + result = self.API.retrieve(request) + result.pprint() + assert result.leaves[0].flatten()["step"] == 0 From 4870c29d1243bf0c6986a978e16f5805604bd95e Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 21 Aug 2023 11:03:45 +0200 Subject: [PATCH 105/332] make FDB datacube backend work with transformations --- polytope/datacube/backends/FDB_datacube.py | 285 ++++++++++++++++++ polytope/datacube/backends/xarray.py | 16 +- .../transformations/datacube_merger.py | 8 +- tests/test_fdb_datacube.py | 37 +++ 4 files changed, 343 insertions(+), 3 deletions(-) create mode 100644 polytope/datacube/backends/FDB_datacube.py create mode 100644 tests/test_fdb_datacube.py diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py new file mode 100644 index 000000000..348b012f4 --- /dev/null +++ b/polytope/datacube/backends/FDB_datacube.py @@ -0,0 +1,285 @@ +import math +import os +from copy import deepcopy + +from ...utility.combinatorics import unique, validate_axes +from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis + +os.environ["DYLD_LIBRARY_PATH"] = "/Users/male/build/fdb-bundle/lib" +os.environ["FDB_HOME"] = "/Users/male/git/fdb-home" +import pyfdb # noqa: E402 + +# TODO: currently, because the date and time are strings, the data will be treated as an unsliceable axis... + + +def glue(path): + return {"t": 0} + + +def update_fdb_dataarray(fdb_dataarray): + new_dict = {} + for key, values in fdb_dataarray.items(): + if key in ["levelist", "param", "step"]: + new_values = [] + for val in values: + new_values.append(int(val)) + new_dict[key] = new_values + else: + new_dict[key] = values + new_dict["values"] = [0.0] + return new_dict + + +class FDBDatacube(Datacube): + def __init__(self, config={}, axis_options={}): + # Need to get the cyclic options and grid options from somewhere + self.axis_options = axis_options + self.grid_mapper = None + self.axis_counter = 0 + self._axes = {} + self.blocked_axes = [] + self.transformation = {} + self.fake_axes = [] + self.complete_axes = [] + partial_request = config + # Find values in the level 3 FDB datacube + # Will be in the form of a dictionary? {axis_name:values_available, ...} + fdb = pyfdb.FDB() + fdb_dataarray = fdb.axes(partial_request).as_dict() + dataarray = update_fdb_dataarray(fdb_dataarray) + self.dataarray = dataarray + + for name, values in dataarray.items(): + values.sort() + options = axis_options.get(name, {}) + configure_datacube_axis(options, name, values, self) + self.complete_axes.append(name) + + def get(self, requests: IndexTree): + for r in requests.leaves: + path = r.flatten() + path = self.remap_path(path) + if len(path.items()) == self.axis_counter: + if self.grid_mapper is not None: + first_axis = self.grid_mapper._mapped_axes[0] + first_val = path[first_axis] + second_axis = self.grid_mapper._mapped_axes[1] + second_val = path[second_axis] + path.pop(first_axis, None) + path.pop(second_axis, None) + # need to remap the lat, lon in path to dataarray index + unmapped_idx = self.grid_mapper.unmap(first_val, second_val) + path[self.grid_mapper._base_axis] = unmapped_idx + # Ask FDB what values it has on the path + subxarray = glue(path) + key = list(subxarray.keys())[0] + value = subxarray[key] + r.result = (key, value) + else: + # if we have no grid map, still need to assign values + subxarray = glue(path) + # value = subxarray.item() + # key = subxarray.name + key = list(subxarray.keys())[0] + value = subxarray[key] + r.result = (key, value) + else: + r.remove_branch() + + def get_mapper(self, axis): + return self._axes[axis] + + def remap_path(self, path: DatacubePath): + for key in path: + value = path[key] + path[key] = self._axes[key].remap_val_to_axis_range(value) + return path + + def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): + idx_between = [] + for i in range(len(search_ranges)): + print(search_ranges[i]) + r = search_ranges[i] + offset = search_ranges_offset[i] + low = r[0] + up = r[1] + if axis.name in self.transformation.keys(): + axis_transforms = self.transformation[axis.name] + temp_indexes = deepcopy(indexes) + for transform in axis_transforms: + print(low) + print(up) + print(first_val) + print(axis) + print(offset) + print(temp_indexes) + (offset, temp_indexes) = transform._find_transformed_indices_between( + axis, self, temp_indexes, low, up, first_val, offset + ) + print(offset) + indexes_between = temp_indexes + else: + indexes_between = self._find_indexes_between(axis, indexes, low, up) + # Now the indexes_between are values on the cyclic range so need to remap them to their original + # values before returning them + for j in range(len(indexes_between)): + if offset is None: + indexes_between[j] = indexes_between[j] + else: + indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) + idx_between.append(indexes_between[j]) + return idx_between + + # def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): + # idx_between = [] + # for i in range(len(search_ranges)): + # r = search_ranges[i] + # offset = search_ranges_offset[i] + # low = r[0] + # up = r[1] + + # if self.grid_mapper is not None: + # first_axis = self.grid_mapper._mapped_axes[0] + # second_axis = self.grid_mapper._mapped_axes[1] + # if axis.name == first_axis: + # indexes_between = self.grid_mapper.map_first_axis(low, up) + # elif axis.name == second_axis: + # indexes_between = self.grid_mapper.map_second_axis(first_val, low, up) + # else: + # indexes_between = [i for i in indexes if low <= i <= up] + # else: + # indexes_between = [i for i in indexes if low <= i <= up] + + # # Now the indexes_between are values on the cyclic range so need to remap them to their original + # # values before returning them + # for j in range(len(indexes_between)): + # if offset is None: + # indexes_between[j] = indexes_between[j] + # else: + # indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) + + # idx_between.append(indexes_between[j]) + # return idx_between + + def fit_path_to_original_datacube(self, path): + path = self.remap_path(path) + first_val = None + unmap_path = {} + considered_axes = [] + changed_type_path = {} + for axis_name in self.transformation.keys(): + axis_transforms = self.transformation[axis_name] + for transform in axis_transforms: + (path, temp_first_val, considered_axes, unmap_path, changed_type_path) = transform._adjust_path( + path, considered_axes, unmap_path, changed_type_path + ) + if temp_first_val: + first_val = temp_first_val + # for key in path.keys(): + # if self.dataarray[key].dims == (): + # path.pop(key) + return (path, first_val, considered_axes, unmap_path, changed_type_path) + + # def get_indices(self, path: DatacubePath, axis, lower, upper): + # path = self.remap_path(path) + # first_val = None + # if self.grid_mapper is not None: + # first_axis = self.grid_mapper._mapped_axes[0] + # first_val = path.get(first_axis, None) + # second_axis = self.grid_mapper._mapped_axes[1] + # path.pop(first_axis, None) + # path.pop(second_axis, None) + # if axis.name == first_axis: + # indexes = [] + # elif axis.name == second_axis: + # indexes = [] + # else: + # indexes = self.dataarray[axis.name] + # else: + # indexes = self.dataarray[axis.name] + + # # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube + # search_ranges = axis.remap([lower, upper]) + # original_search_ranges = axis.to_intervals([lower, upper]) + + # # Find the offsets for each interval in the requested range, which we will need later + # search_ranges_offset = [] + # for r in original_search_ranges: + # offset = axis.offset(r) + # search_ranges_offset.append(offset) + + # # Look up the values in the datacube for each cyclic interval range + # idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) + + # # Remove duplicates even if difference of the order of the axis tolerance + # if offset is not None: + # # Note that we can only do unique if not dealing with time values + # idx_between = unique(idx_between) + + # return idx_between + + def get_indices(self, path: DatacubePath, axis, lower, upper): + # NEW VERSION OF THIS METHOD + (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube(path) + + subarray = self.dataarray + # subarray = self.dataarray.sel(path, method="nearest") + # subarray = subarray.sel(unmap_path) + # subarray = subarray.sel(changed_type_path) + # Get the indexes of the axis we want to query + # XArray does not support branching, so no need to use label, we just take the next axis + if axis.name in self.transformation.keys(): + axis_transforms = self.transformation[axis.name] + # This bool will help us decide for which axes we need to calculate the indexes again or not + # in case there are multiple relevant transformations for an axis + already_has_indexes = False + for transform in axis_transforms: + # TODO: here, instead of creating the indices, would be better to create the standard datacube axes and + # then succesively map them to what they should be + indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) + already_has_indexes = True + else: + indexes = self.datacube_natural_indexes(axis, subarray) + # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube + search_ranges = axis.remap([lower, upper]) + original_search_ranges = axis.to_intervals([lower, upper]) + # Find the offsets for each interval in the requested range, which we will need later + search_ranges_offset = [] + for r in original_search_ranges: + offset = axis.offset(r) + search_ranges_offset.append(offset) + # Look up the values in the datacube for each cyclic interval range + idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) + # Remove duplicates even if difference of the order of the axis tolerance + if offset is not None: + # Note that we can only do unique if not dealing with time values + idx_between = unique(idx_between) + return idx_between + + def datacube_natural_indexes(self, axis, subarray): + indexes = subarray[axis.name] + return indexes + + def has_index(self, path: DatacubePath, axis, index): + # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube + subarray_vals = self.dataarray[axis.name] + return index in subarray_vals + + @property + def axes(self): + return self._axes + + def validate(self, axes): + return validate_axes(self.axes, axes) + + def ax_vals(self, name): + for _name, values in self.dataarray.items(): + if _name == name: + return values + + def _find_indexes_between(self, axis, indexes, low, up): + print("INSIDE FIND INDEXES") + print(indexes) + print(low) + print(up) + return [i for i in indexes if low <= i <= up] diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 062bc84e6..52b9db06d 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -50,8 +50,8 @@ def get(self, requests: IndexTree): changed_type_path) = self.fit_path_to_original_datacube( path ) - print(path) - print(self.dataarray) + # print(path) + # print(self.dataarray) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmap_path) subxarray = subxarray.sel(changed_type_path) @@ -72,6 +72,8 @@ def remap_path(self, path: DatacubePath): return path def _find_indexes_between(self, axis, indexes, low, up): + print("INSIDE FIND INDEXES") + print(indexes) if axis.name in self.complete_axes: # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html @@ -80,12 +82,15 @@ def _find_indexes_between(self, axis, indexes, low, up): end = indexes.searchsorted(up, "right") indexes_between = indexes[start:end].to_list() else: + print(indexes) + print(low) indexes_between = [i for i in indexes if low <= i <= up] return indexes_between def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): idx_between = [] for i in range(len(search_ranges)): + print(search_ranges[i]) r = search_ranges[i] offset = search_ranges_offset[i] low = r[0] @@ -94,9 +99,16 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, axis_transforms = self.transformation[axis.name] temp_indexes = deepcopy(indexes) for transform in axis_transforms: + print(low) + print(up) + print(first_val) + print(axis) + print(offset) + print(temp_indexes) (offset, temp_indexes) = transform._find_transformed_indices_between( axis, self, temp_indexes, low, up, first_val, offset ) + print(offset) indexes_between = temp_indexes else: indexes_between = self._find_indexes_between(axis, indexes, low, up) diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 474fa45d1..f5c9025b6 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -1,6 +1,7 @@ from copy import deepcopy import numpy as np +import pandas as pd from ..backends.datacube import configure_datacube_axis from .datacube_transformations import DatacubeAxisTransformation @@ -23,7 +24,12 @@ def merged_values(self, datacube): for first_val in first_ax_vals: for second_val in second_ax_vals: # TODO: check that the first and second val are strings - merged_values.append(np.datetime64(first_val + linkers[0] + second_val + linkers[1])) + # merged_values.append(np.datetime64(first_val + linkers[0] + second_val + linkers[1])) + val_to_add = pd.to_datetime(first_val + linkers[0] + second_val + linkers[1]) + val_to_add = val_to_add.to_numpy() + val_to_add = val_to_add.astype('datetime64[s]') + # merged_values.append(pd.to_datetime(first_val + linkers[0] + second_val + linkers[1])) + merged_values.append(val_to_add) merged_values = np.array(merged_values) return merged_values diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py new file mode 100644 index 000000000..c94dea7b5 --- /dev/null +++ b/tests/test_fdb_datacube.py @@ -0,0 +1,37 @@ +from polytope.datacube.backends.FDB_datacube import FDBDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + grid_options = { + "values": {"transformation": {"mapper": {"type": "octahedral", "resolution": 1280, + "axes": ["latitude", "longitude"]}}}, + "date" : {"transformation": {"merge" : {"with" : "time", "linkers": ["T", "00"]}}} + } + config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} + self.xarraydatacube = FDBDatacube(config, axis_options=grid_options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.xarraydatacube, engine=self.slicer) + + # Testing different shapes + + def test_2D_box(self): + request = Request( + Select("step", [11]), + Select("levtype", ["sfc"]), + Select("date", ["20230710T120000"]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", [151130]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 9 From 9a3a83483de561e3f2398963bbac098ed38b8d9a Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 21 Aug 2023 11:15:58 +0200 Subject: [PATCH 106/332] clean up and add type change option to FDB datacube test --- polytope/datacube/backends/FDB_datacube.py | 104 +-------------------- polytope/datacube/backends/xarray.py | 14 --- tests/test_fdb_datacube.py | 5 +- 3 files changed, 6 insertions(+), 117 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 348b012f4..cafbbf9ca 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -17,17 +17,8 @@ def glue(path): def update_fdb_dataarray(fdb_dataarray): - new_dict = {} - for key, values in fdb_dataarray.items(): - if key in ["levelist", "param", "step"]: - new_values = [] - for val in values: - new_values.append(int(val)) - new_dict[key] = new_values - else: - new_dict[key] = values - new_dict["values"] = [0.0] - return new_dict + fdb_dataarray["values"] = [0.0] + return fdb_dataarray class FDBDatacube(Datacube): @@ -78,8 +69,6 @@ def get(self, requests: IndexTree): else: # if we have no grid map, still need to assign values subxarray = glue(path) - # value = subxarray.item() - # key = subxarray.name key = list(subxarray.keys())[0] value = subxarray[key] r.result = (key, value) @@ -98,7 +87,6 @@ def remap_path(self, path: DatacubePath): def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): idx_between = [] for i in range(len(search_ranges)): - print(search_ranges[i]) r = search_ranges[i] offset = search_ranges_offset[i] low = r[0] @@ -107,16 +95,9 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, axis_transforms = self.transformation[axis.name] temp_indexes = deepcopy(indexes) for transform in axis_transforms: - print(low) - print(up) - print(first_val) - print(axis) - print(offset) - print(temp_indexes) (offset, temp_indexes) = transform._find_transformed_indices_between( axis, self, temp_indexes, low, up, first_val, offset ) - print(offset) indexes_between = temp_indexes else: indexes_between = self._find_indexes_between(axis, indexes, low, up) @@ -130,37 +111,6 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, idx_between.append(indexes_between[j]) return idx_between - # def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): - # idx_between = [] - # for i in range(len(search_ranges)): - # r = search_ranges[i] - # offset = search_ranges_offset[i] - # low = r[0] - # up = r[1] - - # if self.grid_mapper is not None: - # first_axis = self.grid_mapper._mapped_axes[0] - # second_axis = self.grid_mapper._mapped_axes[1] - # if axis.name == first_axis: - # indexes_between = self.grid_mapper.map_first_axis(low, up) - # elif axis.name == second_axis: - # indexes_between = self.grid_mapper.map_second_axis(first_val, low, up) - # else: - # indexes_between = [i for i in indexes if low <= i <= up] - # else: - # indexes_between = [i for i in indexes if low <= i <= up] - - # # Now the indexes_between are values on the cyclic range so need to remap them to their original - # # values before returning them - # for j in range(len(indexes_between)): - # if offset is None: - # indexes_between[j] = indexes_between[j] - # else: - # indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) - - # idx_between.append(indexes_between[j]) - # return idx_between - def fit_path_to_original_datacube(self, path): path = self.remap_path(path) first_val = None @@ -175,57 +125,13 @@ def fit_path_to_original_datacube(self, path): ) if temp_first_val: first_val = temp_first_val - # for key in path.keys(): - # if self.dataarray[key].dims == (): - # path.pop(key) return (path, first_val, considered_axes, unmap_path, changed_type_path) - # def get_indices(self, path: DatacubePath, axis, lower, upper): - # path = self.remap_path(path) - # first_val = None - # if self.grid_mapper is not None: - # first_axis = self.grid_mapper._mapped_axes[0] - # first_val = path.get(first_axis, None) - # second_axis = self.grid_mapper._mapped_axes[1] - # path.pop(first_axis, None) - # path.pop(second_axis, None) - # if axis.name == first_axis: - # indexes = [] - # elif axis.name == second_axis: - # indexes = [] - # else: - # indexes = self.dataarray[axis.name] - # else: - # indexes = self.dataarray[axis.name] - - # # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube - # search_ranges = axis.remap([lower, upper]) - # original_search_ranges = axis.to_intervals([lower, upper]) - - # # Find the offsets for each interval in the requested range, which we will need later - # search_ranges_offset = [] - # for r in original_search_ranges: - # offset = axis.offset(r) - # search_ranges_offset.append(offset) - - # # Look up the values in the datacube for each cyclic interval range - # idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) - - # # Remove duplicates even if difference of the order of the axis tolerance - # if offset is not None: - # # Note that we can only do unique if not dealing with time values - # idx_between = unique(idx_between) - - # return idx_between - def get_indices(self, path: DatacubePath, axis, lower, upper): # NEW VERSION OF THIS METHOD (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube(path) subarray = self.dataarray - # subarray = self.dataarray.sel(path, method="nearest") - # subarray = subarray.sel(unmap_path) - # subarray = subarray.sel(changed_type_path) # Get the indexes of the axis we want to query # XArray does not support branching, so no need to use label, we just take the next axis if axis.name in self.transformation.keys(): @@ -255,7 +161,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): # Note that we can only do unique if not dealing with time values idx_between = unique(idx_between) return idx_between - + def datacube_natural_indexes(self, axis, subarray): indexes = subarray[axis.name] return indexes @@ -278,8 +184,4 @@ def ax_vals(self, name): return values def _find_indexes_between(self, axis, indexes, low, up): - print("INSIDE FIND INDEXES") - print(indexes) - print(low) - print(up) return [i for i in indexes if low <= i <= up] diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 52b9db06d..870ad1b95 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -50,8 +50,6 @@ def get(self, requests: IndexTree): changed_type_path) = self.fit_path_to_original_datacube( path ) - # print(path) - # print(self.dataarray) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmap_path) subxarray = subxarray.sel(changed_type_path) @@ -72,8 +70,6 @@ def remap_path(self, path: DatacubePath): return path def _find_indexes_between(self, axis, indexes, low, up): - print("INSIDE FIND INDEXES") - print(indexes) if axis.name in self.complete_axes: # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html @@ -82,15 +78,12 @@ def _find_indexes_between(self, axis, indexes, low, up): end = indexes.searchsorted(up, "right") indexes_between = indexes[start:end].to_list() else: - print(indexes) - print(low) indexes_between = [i for i in indexes if low <= i <= up] return indexes_between def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): idx_between = [] for i in range(len(search_ranges)): - print(search_ranges[i]) r = search_ranges[i] offset = search_ranges_offset[i] low = r[0] @@ -99,16 +92,9 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, axis_transforms = self.transformation[axis.name] temp_indexes = deepcopy(indexes) for transform in axis_transforms: - print(low) - print(up) - print(first_val) - print(axis) - print(offset) - print(temp_indexes) (offset, temp_indexes) = transform._find_transformed_indices_between( axis, self, temp_indexes, low, up, first_val, offset ) - print(offset) indexes_between = temp_indexes else: indexes_between = self._find_indexes_between(axis, indexes, low, up) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index c94dea7b5..df58f4dac 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -10,7 +10,8 @@ def setup_method(self, method): grid_options = { "values": {"transformation": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}}, - "date" : {"transformation": {"merge" : {"with" : "time", "linkers": ["T", "00"]}}} + "date" : {"transformation": {"merge" : {"with" : "time", "linkers": ["T", "00"]}}}, + "step": {"transformation": {"type_change": "int"}} } config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} self.xarraydatacube = FDBDatacube(config, axis_options=grid_options) @@ -26,7 +27,7 @@ def test_2D_box(self): Select("date", ["20230710T120000"]), Select("domain", ["g"]), Select("expver", ["0001"]), - Select("param", [151130]), + Select("param", ["151130"]), Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["fc"]), From dde36b94f68012ff2d090cd6efe6f3cab7219828 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 21 Aug 2023 11:53:50 +0200 Subject: [PATCH 107/332] change get to deal with transformations for FDB backend --- polytope/datacube/backends/FDB_datacube.py | 35 ++++++++-------------- polytope/datacube/backends/xarray.py | 9 ++---- 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index cafbbf9ca..3b308a0ea 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -12,7 +12,7 @@ # TODO: currently, because the date and time are strings, the data will be treated as an unsliceable axis... -def glue(path): +def glue(path, unmap_path): return {"t": 0} @@ -49,29 +49,18 @@ def __init__(self, config={}, axis_options={}): def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() - path = self.remap_path(path) if len(path.items()) == self.axis_counter: - if self.grid_mapper is not None: - first_axis = self.grid_mapper._mapped_axes[0] - first_val = path[first_axis] - second_axis = self.grid_mapper._mapped_axes[1] - second_val = path[second_axis] - path.pop(first_axis, None) - path.pop(second_axis, None) - # need to remap the lat, lon in path to dataarray index - unmapped_idx = self.grid_mapper.unmap(first_val, second_val) - path[self.grid_mapper._base_axis] = unmapped_idx - # Ask FDB what values it has on the path - subxarray = glue(path) - key = list(subxarray.keys())[0] - value = subxarray[key] - r.result = (key, value) - else: - # if we have no grid map, still need to assign values - subxarray = glue(path) - key = list(subxarray.keys())[0] - value = subxarray[key] - r.result = (key, value) + # first, find the grid mapper transform + unmap_path = {} + considered_axes = [] + (path, first_val, considered_axes, unmap_path, + changed_type_path) = self.fit_path_to_original_datacube(path) + unmap_path.update(changed_type_path) + # Here, need to give the FDB the path and the unmap_path to select data + subxarray = glue(path, unmap_path) + key = list(subxarray.keys())[0] + value = subxarray[key] + r.result = (key, value) else: r.remove_branch() diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 870ad1b95..2baeb2583 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -45,14 +45,11 @@ def get(self, requests: IndexTree): # first, find the grid mapper transform unmap_path = {} considered_axes = [] - if True: - (path, first_val, considered_axes, unmap_path, - changed_type_path) = self.fit_path_to_original_datacube( - path - ) + (path, first_val, considered_axes, unmap_path, + changed_type_path) = self.fit_path_to_original_datacube(path) + unmap_path.update(changed_type_path) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmap_path) - subxarray = subxarray.sel(changed_type_path) value = subxarray.item() key = subxarray.name r.result = (key, value) From 4a8328605afe6653654ca54ef6fbaa050c0f7776 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 21 Aug 2023 11:55:53 +0200 Subject: [PATCH 108/332] black --- polytope/datacube/backends/FDB_datacube.py | 5 +++-- polytope/datacube/backends/xarray.py | 5 +++-- .../transformations/datacube_merger.py | 2 +- .../datacube_transformations.py | 15 ++++++------- .../transformations/datacube_type_change.py | 8 +++---- tests/test_fdb_datacube.py | 11 ++++++---- tests/test_merge_cyclic_octahedral.py | 21 +++++++++++-------- tests/test_merge_octahedral_one_axis.py | 15 ++++++------- 8 files changed, 46 insertions(+), 36 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 3b308a0ea..7e177b1e2 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -53,8 +53,9 @@ def get(self, requests: IndexTree): # first, find the grid mapper transform unmap_path = {} considered_axes = [] - (path, first_val, considered_axes, unmap_path, - changed_type_path) = self.fit_path_to_original_datacube(path) + (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube( + path + ) unmap_path.update(changed_type_path) # Here, need to give the FDB the path and the unmap_path to select data subxarray = glue(path, unmap_path) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 2baeb2583..658a097a8 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -45,8 +45,9 @@ def get(self, requests: IndexTree): # first, find the grid mapper transform unmap_path = {} considered_axes = [] - (path, first_val, considered_axes, unmap_path, - changed_type_path) = self.fit_path_to_original_datacube(path) + (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube( + path + ) unmap_path.update(changed_type_path) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmap_path) diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index f5c9025b6..96e8a9190 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -27,7 +27,7 @@ def merged_values(self, datacube): # merged_values.append(np.datetime64(first_val + linkers[0] + second_val + linkers[1])) val_to_add = pd.to_datetime(first_val + linkers[0] + second_val + linkers[1]) val_to_add = val_to_add.to_numpy() - val_to_add = val_to_add.astype('datetime64[s]') + val_to_add = val_to_add.astype("datetime64[s]") # merged_values.append(pd.to_datetime(first_val + linkers[0] + second_val + linkers[1])) merged_values.append(val_to_add) merged_values = np.array(merged_values) diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index ed420aabd..b954325ba 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -90,12 +90,13 @@ def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_i "cyclic": "DatacubeAxisCyclic", "merge": "DatacubeAxisMerger", "reverse": "DatacubeAxisReverse", - "type_change" : "DatacubeAxisTypeChange" + "type_change": "DatacubeAxisTypeChange", } -_type_to_transformation_file_lookup = {"mapper": "mappers", - "cyclic": "cyclic", - "merge": "merger", - "reverse": "reverse", - "type_change" : "type_change" - } +_type_to_transformation_file_lookup = { + "mapper": "mappers", + "cyclic": "cyclic", + "merge": "merger", + "reverse": "reverse", + "type_change": "type_change", +} diff --git a/polytope/datacube/transformations/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change.py index 5f3dd077a..5ad66b163 100644 --- a/polytope/datacube/transformations/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change.py @@ -42,8 +42,9 @@ def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, fi # NOTE: needs to be in new type if axis.name == self.name: transformation = self.generate_final_transformation() - indexes_between = [transformation.transform_type(i) for i in indexes - if low <= transformation.transform_type(i) <= up] + indexes_between = [ + transformation.transform_type(i) for i in indexes if low <= transformation.transform_type(i) <= up + ] # indexes_between = [] else: indexes_between = datacube._find_indexes_between(axis, indexes, low, up) @@ -70,7 +71,6 @@ def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_i class TypeChangeStrToInt(DatacubeAxisTypeChange): - def __init__(self, axis_name, new_type): self.axis_name = axis_name self._new_type = new_type @@ -82,4 +82,4 @@ def make_str(self, value): return str(value) -_type_to_datacube_type_change_lookup = {"int" : "TypeChangeStrToInt"} +_type_to_datacube_type_change_lookup = {"int": "TypeChangeStrToInt"} diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index df58f4dac..568eb97cf 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -8,10 +8,13 @@ class TestSlicingFDBDatacube: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types grid_options = { - "values": {"transformation": {"mapper": {"type": "octahedral", "resolution": 1280, - "axes": ["latitude", "longitude"]}}}, - "date" : {"transformation": {"merge" : {"with" : "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}} + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + } + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, } config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} self.xarraydatacube = FDBDatacube(config, axis_options=grid_options) diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index c8a71efa7..85c3a5891 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -20,12 +20,15 @@ def setup_method(self, method): "step": [0, 1, 2], }, ) - options = {"date": {"transformation": {"merge": {"with": "time", "linkers": ["T", ":00"]}}}, - "values": {"transformation": {"mapper": {"type": "octahedral", - "resolution": 1280, - "axes": ["latitude", "longitude"]}}}, - "step": {"transformation": {"cyclic": [0, 2]}}, - } + options = { + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", ":00"]}}}, + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + } + }, + "step": {"transformation": {"cyclic": [0, 2]}}, + } self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) @@ -34,9 +37,9 @@ def setup_method(self, method): def test_merge_axis(self): # NOTE: does not work because the date is a string in the merge option... date = np.datetime64("2000-01-01T06:00:00") - request = Request(Select("date", [date]), - Span("step", 0, 3), - Box(["latitude", "longitude"], [0, 0], [0.2, 0.2])) + request = Request( + Select("date", [date]), Span("step", 0, 3), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]) + ) result = self.API.retrieve(request) result.pprint() assert result.leaves[0].flatten()["date"] == np.datetime64("2000-01-01T06:00:00") diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index 88b4c0dca..f2de6b3d0 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -18,19 +18,20 @@ def setup_method(self, method): "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} } }, - "longitude": {"transformation": {"cyclic": [0, 360.0]}} + "longitude": {"transformation": {"cyclic": [0, 360.0]}}, } self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=grid_options) def test_merge_axis(self): request = Request( - Select("number", [0]), - Select("time", ["2023-06-25T12:00:00"]), - Select("step", ["00:00:00"]), - Select("surface", [0]), - Select("valid_time", ["2023-06-25T12:00:00"]), - Box(["latitude", "longitude"], [0, 359.8], [0.2, 361.2])) + Select("number", [0]), + Select("time", ["2023-06-25T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2023-06-25T12:00:00"]), + Box(["latitude", "longitude"], [0, 359.8], [0.2, 361.2]), + ) result = self.API.retrieve(request) # result.pprint() assert result.leaves[0].flatten()["longitude"] == 0.0 From 0bbe82994b9653c97f5ef2b17ff9a46711196955 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 22 Aug 2023 10:28:15 +0200 Subject: [PATCH 109/332] add TODO --- polytope/datacube/backends/FDB_datacube.py | 1 + 1 file changed, 1 insertion(+) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 7e177b1e2..4170baf40 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -5,6 +5,7 @@ from ...utility.combinatorics import unique, validate_axes from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis +# TODO: probably need to do this more general... os.environ["DYLD_LIBRARY_PATH"] = "/Users/male/build/fdb-bundle/lib" os.environ["FDB_HOME"] = "/Users/male/git/fdb-home" import pyfdb # noqa: E402 From 0dd95f0ac4c37d57c0b79aba51f9baa65e97e622 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 23 Aug 2023 09:05:44 +0200 Subject: [PATCH 110/332] refactor some small cyclic axis functions --- polytope/datacube/datacube_axis.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index bc04bade7..68c2ca104 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -72,7 +72,7 @@ def remap_range_to_axis_range(range): return [return_lower, return_upper] def remap_val_to_axis_range(value): - return_range = cls.remap_range_to_axis_range([value, value]) + return_range = remap_range_to_axis_range([value, value]) return return_range[0] def remap(range: List): @@ -93,7 +93,7 @@ def remap(range: List): for interval in range_intervals: if abs(interval[0] - interval[1]) > 0: # If the interval is not just a single point, we remap it to the axis range - range = cls.remap_range_to_axis_range([interval[0], interval[1]]) + range = remap_range_to_axis_range([interval[0], interval[1]]) up = range[1] low = range[0] if up < low: @@ -108,12 +108,11 @@ def offset(range): # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. # Also, it's safer that we find the offset of a value inside the range instead of on the border unpadded_range = [range[0] + 1.5 * cls.tol, range[1] - 1.5 * cls.tol] - cyclic_range = cls.remap_range_to_axis_range(unpadded_range) + cyclic_range = remap_range_to_axis_range(unpadded_range) offset = unpadded_range[0] - cyclic_range[0] return offset cls.to_intervals = to_intervals - cls.remap_range_to_axis_range = remap_range_to_axis_range cls.remap_val_to_axis_range = remap_val_to_axis_range cls.remap = remap cls.offset = offset @@ -165,9 +164,6 @@ def to_intervals(self, range): def remap_val_to_axis_range(self, value): return value - def remap_range_to_axis_range(self, range): - return range - def remap(self, range: List) -> Any: return [range] From 1ca1853f7d6f4c4fa88bbd8894778eea8d854f6a Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 23 Aug 2023 12:26:10 +0200 Subject: [PATCH 111/332] start refactoring by creating all axes the same way and adding tags to differentiate transformations --- polytope/datacube/backends/FDB_datacube.py | 2 +- polytope/datacube/backends/xarray.py | 23 +++++- polytope/datacube/datacube_axis.py | 26 +++--- .../datacube_transformations.py | 20 +++++ .../transformation_pseudocode.py | 81 +++++++++++++++++++ 5 files changed, 139 insertions(+), 13 deletions(-) create mode 100644 polytope/datacube/transformations/transformation_pseudocode.py diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 4170baf40..db2fd2522 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -72,7 +72,7 @@ def get_mapper(self, axis): def remap_path(self, path: DatacubePath): for key in path: value = path[key] - path[key] = self._axes[key].remap_val_to_axis_range(value) + path[key] = self._axes[key].remap([value, value])[0][0] return path def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 658a097a8..994d48915 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -4,12 +4,33 @@ import xarray as xr from ...utility.combinatorics import unique, validate_axes +from ..datacube_axis import DatacubeAxis +from ..transformations.datacube_transformations import ( + DatacubeAxisTransformation, + has_transform, +) from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis class XArrayDatacube(Datacube): """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0).""" + def create_axes(self, name, values, transformation_type_key, transformation_options): + # NOTE: THIS IS ONE OF THE REFACTORED FUNCTIONS + # NOTE: need to do this for all transformations on an axis + + # first check what the final axes are for this axis name given transformations + final_axis_names = DatacubeAxisTransformation.get_final_axes(name, transformation_type_key, + transformation_options) + for axis_name in final_axis_names: + # if axis does not yet exist, create it + if axis_name not in self._axes.keys(): + DatacubeAxis.create_standard(axis_name, values, self) + # add transformation tag to axis, as well as transformation options for later + setattr(self, has_transform[transformation_type_key], True) # where has_transform is a factory inside + # datacube_transformations to set the has_transform, is_cyclic etc axis properties + # TODO: here now, still add the transformation to the axis object here + def __init__(self, dataarray: xr.DataArray, axis_options={}): self.axis_options = axis_options self.grid_mapper = None @@ -64,7 +85,7 @@ def get_mapper(self, axis): def remap_path(self, path: DatacubePath): for key in path: value = path[key] - path[key] = self._axes[key].remap_val_to_axis_range(value) + path[key] = self._axes[key].remap([value, value])[0][0] return path def _find_indexes_between(self, axis, indexes, low, up): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 68c2ca104..683e65f13 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -47,7 +47,7 @@ def to_intervals(range): intervals.append([new_up, upper]) return intervals - def remap_range_to_axis_range(range): + def _remap_range_to_axis_range(range): axis_lower = cls.range[0] axis_upper = cls.range[1] axis_range = axis_upper - axis_lower @@ -71,8 +71,8 @@ def remap_range_to_axis_range(range): return_upper = upper return [return_lower, return_upper] - def remap_val_to_axis_range(value): - return_range = remap_range_to_axis_range([value, value]) + def _remap_val_to_axis_range(value): + return_range = _remap_range_to_axis_range([value, value]) return return_range[0] def remap(range: List): @@ -84,8 +84,8 @@ def remap(range: List): # If we have a range that is just one point, then it should still be counted # and so we should take a small interval around it to find values inbetween range = [ - cls.remap_val_to_axis_range(range[0]) - cls.tol, - cls.remap_val_to_axis_range(range[0]) + cls.tol, + _remap_val_to_axis_range(range[0]) - cls.tol, + _remap_val_to_axis_range(range[0]) + cls.tol, ] return [range] range_intervals = cls.to_intervals(range) @@ -93,7 +93,7 @@ def remap(range: List): for interval in range_intervals: if abs(interval[0] - interval[1]) > 0: # If the interval is not just a single point, we remap it to the axis range - range = remap_range_to_axis_range([interval[0], interval[1]]) + range = _remap_range_to_axis_range([interval[0], interval[1]]) up = range[1] low = range[0] if up < low: @@ -108,18 +108,25 @@ def offset(range): # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. # Also, it's safer that we find the offset of a value inside the range instead of on the border unpadded_range = [range[0] + 1.5 * cls.tol, range[1] - 1.5 * cls.tol] - cyclic_range = remap_range_to_axis_range(unpadded_range) + cyclic_range = _remap_range_to_axis_range(unpadded_range) offset = unpadded_range[0] - cyclic_range[0] return offset cls.to_intervals = to_intervals - cls.remap_val_to_axis_range = remap_val_to_axis_range cls.remap = remap cls.offset = offset return cls +def mapper(cls): + # NOTE: THIS IS ONE OF THE REFACTORED FUNCTIONS + if cls.has_mapper: + if cls.name in cls.transformation.keys(): + # the name of the class is in the transformation dictionary + pass + + class DatacubeAxis(ABC): is_cyclic = False @@ -161,9 +168,6 @@ def serialize(self, value: Any) -> Any: def to_intervals(self, range): return [range] - def remap_val_to_axis_range(self, value): - return value - def remap(self, range: List) -> Any: return [range] diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index b954325ba..19ce0e846 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -4,6 +4,21 @@ class DatacubeAxisTransformation(ABC): + + @staticmethod + def get_final_axes(name, transformation_type_key, transformation_options): + # NOTE: THIS IS ONE OF THE REFACTORED FUNCTIONS + # TODO: refactor this because now it's creating whole transformations which we might not need yet? + transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] + transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] + + module = import_module("polytope.datacube.transformations.datacube_" + transformation_file_name) + constructor = getattr(module, transformation_type) + transformation_type_option = transformation_options[transformation_type_key] + new_transformation = deepcopy(constructor(name, transformation_type_option)) + transformation_axis_names = new_transformation.transformation_axes_final() + return transformation_axis_names + @staticmethod def create_transformation(options, name, values, datacube): # transformation options look like @@ -100,3 +115,8 @@ def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_i "reverse": "reverse", "type_change": "type_change", } + +has_transform = { + "mapper": "has_mapper", + "cyclic" : "is_cyclic", +} diff --git a/polytope/datacube/transformations/transformation_pseudocode.py b/polytope/datacube/transformations/transformation_pseudocode.py new file mode 100644 index 000000000..c9405123e --- /dev/null +++ b/polytope/datacube/transformations/transformation_pseudocode.py @@ -0,0 +1,81 @@ +from copy import deepcopy + + +def create_axis_transformations(options, name, values, datacube): + transformation_options = options["transformation"] + for transformation_type_key in transformation_options(): + # create transformations + new_transformation = create_axis_transformation(name, transformation_type_key, transformation_options) + transformation_axis_names = new_transformation.transformation_axes_final() + + for axis_name in transformation_axis_names: + # if axis exists, just add a tag of the transformation, else create axis first and add tag after + + + + + + # if there are no transformations for that axis yet, create an empty list of transforms. + # else, take the old list and append new transformation we are working on + key_val = datacube.transformation.get(axis_name, []) + datacube.transformation[axis_name] = key_val + # the transformation dico keeps track of the type of transformation, not the exact transformations + # For grid mappers, it keeps track that we have a grid_mapper, but won't know the exact grid map we + # implement + datacube.transformation[axis_name].append(new_transformation) + new_transformation.apply_transformation(name, datacube, values) + + +def create_axis_transformation(name, transformation_type_key, transformation_options): + transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] + transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] + module = import_module("polytope.datacube.transformations.datacube_" + transformation_file_name) + constructor = getattr(module, transformation_type) + transformation_type_option = transformation_options[transformation_type_key] + new_transformation = deepcopy(constructor(name, transformation_type_option)) + new_transformation.name = name + return new_transformation + + + + + + + + # transformation options look like + # "time":{"transformation": { "type" : {"merge" : {"with":"step", "linkers": ["T", "00"]}}}} + # But the last dictionary can vary and change according to transformation, which can be handled inside the + # specialised transformations + transformation_options = options["transformation"] + # NOTE: we do the following for each transformation of each axis + for transformation_type_key in transformation_options.keys(): + transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] + transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] + + module = import_module("polytope.datacube.transformations.datacube_" + transformation_file_name) + constructor = getattr(module, transformation_type) + transformation_type_option = transformation_options[transformation_type_key] + # NOTE: the transformation in the datacube takes in now an option dico like + # {"with":"step", "linkers": ["T", "00"]}} + + # Here, we keep track of all the transformation objects along with the associated axis within the datacube + # We generate a transformation dictionary that looks like + # {"lat": [merger, cyclic], "lon": [mapper, cyclic], etc...} + new_transformation = deepcopy(constructor(name, transformation_type_option)) + # TODO: instead of adding directly the transformation, could be we have an add_transformation method + # where each transformation can choose the name that it is assigned to, ie the axis name it is assigned to + # and then for eg for grid mapper transformation, can have the first axis name in there to make things + # easier to handle in the datacube + + new_transformation.name = name + transformation_axis_names = new_transformation.transformation_axes_final() + for axis_name in transformation_axis_names: + # if there are no transformations for that axis yet, create an empty list of transforms. + # else, take the old list and append new transformation we are working on + key_val = datacube.transformation.get(axis_name, []) + datacube.transformation[axis_name] = key_val + # the transformation dico keeps track of the type of transformation, not the exact transformations + # For grid mappers, it keeps track that we have a grid_mapper, but won't know the exact grid map we + # implement + datacube.transformation[axis_name].append(new_transformation) + new_transformation.apply_transformation(name, datacube, values) \ No newline at end of file From 2f2ba4e7454ed5202d8c1a1c1c111ed4182b00ad Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 25 Aug 2023 11:25:54 +0200 Subject: [PATCH 112/332] make mapper transformation a decorator --- polytope/datacube/backends/xarray.py | 171 +++++++++++++----- polytope/datacube/datacube_axis.py | 94 +++++++++- .../transformations/datacube_cyclic.py | 3 + .../transformations/datacube_mappers.py | 16 +- .../transformations/datacube_merger.py | 3 + .../transformations/datacube_reverse.py | 3 + .../datacube_transformations.py | 18 +- .../transformations/datacube_type_change.py | 4 + .../transformation_pseudocode.py | 4 +- tests/test_datacube_axes_init.py | 48 +++++ 10 files changed, 313 insertions(+), 51 deletions(-) create mode 100644 tests/test_datacube_axes_init.py diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 994d48915..b0d47199e 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -1,3 +1,4 @@ +import importlib import math from copy import deepcopy @@ -9,7 +10,7 @@ DatacubeAxisTransformation, has_transform, ) -from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis +from .datacube import Datacube, DatacubePath, IndexTree class XArrayDatacube(Datacube): @@ -22,14 +23,37 @@ def create_axes(self, name, values, transformation_type_key, transformation_opti # first check what the final axes are for this axis name given transformations final_axis_names = DatacubeAxisTransformation.get_final_axes(name, transformation_type_key, transformation_options) + transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key, + transformation_options) for axis_name in final_axis_names: # if axis does not yet exist, create it + + # first need to change the values so that we have right type + values = transformation.change_val_type(axis_name, values) if axis_name not in self._axes.keys(): DatacubeAxis.create_standard(axis_name, values, self) # add transformation tag to axis, as well as transformation options for later - setattr(self, has_transform[transformation_type_key], True) # where has_transform is a factory inside - # datacube_transformations to set the has_transform, is_cyclic etc axis properties - # TODO: here now, still add the transformation to the axis object here + setattr(self._axes[axis_name], has_transform[transformation_type_key], True) # where has_transform is a + # factory inside datacube_transformations to set the has_transform, is_cyclic etc axis properties + # add the specific transformation handled here to the relevant axes + # Modify the axis to update with the tag + decorator_module = importlib.import_module("polytope.datacube.datacube_axis") + decorator = getattr(decorator_module, transformation_type_key) + decorator(self._axes[axis_name]) + if transformation not in self._axes[axis_name].transformations: # Avoids duplicates being stored + # NOTE: TODO: here, the same transformation gets added twice to each axis? not sure why + self._axes[axis_name].transformations.append(transformation) + + def add_all_transformation_axes(self, options, name, values): + transformation_options = options["transformation"] + for transformation_type_key in transformation_options.keys(): + self.create_axes(name, values, transformation_type_key, transformation_options) + + def check_and_add_axes(self, options, name, values): + if "transformation" in options: + self.add_all_transformation_axes(options, name, values) + else: + DatacubeAxis.create_standard(name, values, self) def __init__(self, dataarray: xr.DataArray, axis_options={}): self.axis_options = axis_options @@ -45,19 +69,84 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: options = axis_options.get(name, {}) - configure_datacube_axis(options, name, values, self) + self.check_and_add_axes(options, name, values) treated_axes.append(name) self.complete_axes.append(name) else: if self.dataarray[name].dims == (): options = axis_options.get(name, {}) - configure_datacube_axis(options, name, values, self) + self.check_and_add_axes(options, name, values) treated_axes.append(name) for name in dataarray.dims: if name not in treated_axes: options = axis_options.get(name, {}) val = dataarray[name].values[0] - configure_datacube_axis(options, name, val, self) + self.check_and_add_axes(options, name, val) + + def get_indices(self, path: DatacubePath, axis, lower, upper): + # TODO: here now, use the axis decorator to handle all transformations + (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube(path) + + subarray = self.dataarray.sel(path, method="nearest") + subarray = subarray.sel(unmap_path) + subarray = subarray.sel(changed_type_path) + # Get the indexes of the axis we want to query + # XArray does not support branching, so no need to use label, we just take the next axis + if axis.name in self.transformation.keys(): + axis_transforms = self.transformation[axis.name] + # This bool will help us decide for which axes we need to calculate the indexes again or not + # in case there are multiple relevant transformations for an axis + already_has_indexes = False + for transform in axis_transforms: + # TODO: here, instead of creating the indices, would be better to create the standard datacube axes and + # then succesively map them to what they should be + indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) + already_has_indexes = True + else: + indexes = self.datacube_natural_indexes(axis, subarray) + # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube + search_ranges = axis.remap([lower, upper]) + original_search_ranges = axis.to_intervals([lower, upper]) + # Find the offsets for each interval in the requested range, which we will need later + search_ranges_offset = [] + for r in original_search_ranges: + offset = axis.offset(r) + search_ranges_offset.append(offset) + # Look up the values in the datacube for each cyclic interval range + idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) + # Remove duplicates even if difference of the order of the axis tolerance + if offset is not None: + # Note that we can only do unique if not dealing with time values + idx_between = unique(idx_between) + return idx_between + + # def __init__(self, dataarray: xr.DataArray, axis_options={}): + # self.axis_options = axis_options + # self.grid_mapper = None + # self.axis_counter = 0 + # self._axes = {} + # self.dataarray = dataarray + # treated_axes = [] + # self.complete_axes = [] + # self.blocked_axes = [] + # self.transformation = {} + # self.fake_axes = [] + # for name, values in dataarray.coords.variables.items(): + # if name in dataarray.dims: + # options = axis_options.get(name, {}) + # configure_datacube_axis(options, name, values, self) + # treated_axes.append(name) + # self.complete_axes.append(name) + # else: + # if self.dataarray[name].dims == (): + # options = axis_options.get(name, {}) + # configure_datacube_axis(options, name, values, self) + # treated_axes.append(name) + # for name in dataarray.dims: + # if name not in treated_axes: + # options = axis_options.get(name, {}) + # val = dataarray[name].values[0] + # configure_datacube_axis(options, name, val, self) def get(self, requests: IndexTree): for r in requests.leaves: @@ -172,41 +261,41 @@ def fit_path_to_original_datacube(self, path): path.pop(key) return (path, first_val, considered_axes, unmap_path, changed_type_path) - def get_indices(self, path: DatacubePath, axis, lower, upper): - (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube(path) + # def get_indices(self, path: DatacubePath, axis, lower, upper): + # (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube(path) - subarray = self.dataarray.sel(path, method="nearest") - subarray = subarray.sel(unmap_path) - subarray = subarray.sel(changed_type_path) - # Get the indexes of the axis we want to query - # XArray does not support branching, so no need to use label, we just take the next axis - if axis.name in self.transformation.keys(): - axis_transforms = self.transformation[axis.name] - # This bool will help us decide for which axes we need to calculate the indexes again or not - # in case there are multiple relevant transformations for an axis - already_has_indexes = False - for transform in axis_transforms: - # TODO: here, instead of creating the indices, would be better to create the standard datacube axes and - # then succesively map them to what they should be - indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) - already_has_indexes = True - else: - indexes = self.datacube_natural_indexes(axis, subarray) - # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube - search_ranges = axis.remap([lower, upper]) - original_search_ranges = axis.to_intervals([lower, upper]) - # Find the offsets for each interval in the requested range, which we will need later - search_ranges_offset = [] - for r in original_search_ranges: - offset = axis.offset(r) - search_ranges_offset.append(offset) - # Look up the values in the datacube for each cyclic interval range - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) - # Remove duplicates even if difference of the order of the axis tolerance - if offset is not None: - # Note that we can only do unique if not dealing with time values - idx_between = unique(idx_between) - return idx_between + # subarray = self.dataarray.sel(path, method="nearest") + # subarray = subarray.sel(unmap_path) + # subarray = subarray.sel(changed_type_path) + # # Get the indexes of the axis we want to query + # # XArray does not support branching, so no need to use label, we just take the next axis + # if axis.name in self.transformation.keys(): + # axis_transforms = self.transformation[axis.name] + # # This bool will help us decide for which axes we need to calculate the indexes again or not + # # in case there are multiple relevant transformations for an axis + # already_has_indexes = False + # for transform in axis_transforms: + # # TODO: here, instead of creating the indices, would be better to create the standard datacube axes and + # # then succesively map them to what they should be + # indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) + # already_has_indexes = True + # else: + # indexes = self.datacube_natural_indexes(axis, subarray) + # # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube + # search_ranges = axis.remap([lower, upper]) + # original_search_ranges = axis.to_intervals([lower, upper]) + # # Find the offsets for each interval in the requested range, which we will need later + # search_ranges_offset = [] + # for r in original_search_ranges: + # offset = axis.offset(r) + # search_ranges_offset.append(offset) + # # Look up the values in the datacube for each cyclic interval range + # idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) + # # Remove duplicates even if difference of the order of the axis tolerance + # if offset is not None: + # # Note that we can only do unique if not dealing with time values + # idx_between = unique(idx_between) + # return idx_between def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 683e65f13..79f3f077d 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -75,7 +75,7 @@ def _remap_val_to_axis_range(value): return_range = _remap_range_to_axis_range([value, value]) return return_range[0] - def remap(range: List): + def remap(range: List, path): if cls.range[0] - cls.tol <= range[0] <= cls.range[1] + cls.tol: if cls.range[0] - cls.tol <= range[1] <= cls.range[1] + cls.tol: # If we are already in the cyclic range, return it @@ -120,15 +120,71 @@ def offset(range): def mapper(cls): + from .transformations.datacube_mappers import DatacubeMapper + # NOTE: THIS IS ONE OF THE REFACTORED FUNCTIONS if cls.has_mapper: - if cls.name in cls.transformation.keys(): - # the name of the class is in the transformation dictionary - pass + # if cls.name in cls.transformation.keys(): + # # the name of the class is in the transformation dictionary + def remap(range, path): + # first, find the relevant transformation object that is a mapping in the cls.transformation dico + for transform in cls.transformations: + if isinstance(transform, DatacubeMapper): + transformation = transform + if cls.name == transformation._mapped_axes()[0]: + return [transformation.first_axis_vals()] + if cls.name == transformation._mapped_axes()[1]: + first_val = path[transformation._mapped_axes()[0]] + return [transformation.second_axis_vals(first_val)] + + def unmap_to_datacube(path, unmapped_path): + for transform in cls.transformations: + if isinstance(transform, DatacubeMapper): + transformation = transform + if cls.name == transformation._mapped_axes()[0]: + # if we are on the first axis, then need to add the first val to unmapped_path + first_val = path.get(cls.name, None) + path.pop(cls.name, None) + if cls.name not in unmapped_path: + # if for some reason, the unmapped_path already has the first axis val, then don't update + unmapped_path[cls.name] = first_val + if cls.name == transformation._mapped_axes()[1]: + # if we are on the second axis, then the val of the first axis is stored + # inside unmapped_path so can get it from there + second_val = path.get(cls.name, None) + path.pop(cls.name, None) + first_val = unmapped_path.get(transformation._mapped_axes()[0], None) + unmapped_path.pop(transformation._mapped_axes()[0], None) + if first_val is not None and second_val is not None: + unmapped_idx = transformation.unmap(first_val, second_val) + unmapped_path[transformation.old_axis] = unmapped_idx + return (path, unmapped_path) + + def remap_to_requested(path, unmapped_path): + return (path, unmapped_path) + + def find_indices_between(cls, index_ranges, low, up, datacube): + indexes_between_ranges = [] + for transform in cls.transformations: + if isinstance(transform, DatacubeMapper): + transformation = transform + if cls.name in transformation._mapped_axes(): + for indexes in index_ranges: + indexes_between = [i for i in indexes if low <= i <= up] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + cls.remap = remap + cls.unmap_to_datacube = unmap_to_datacube + cls.remap_to_requested = remap_to_requested + cls.find_indices_between = find_indices_between + + return cls class DatacubeAxis(ABC): is_cyclic = False + has_mapper = False def update_axis(self): if self.is_cyclic: @@ -174,6 +230,30 @@ def remap(self, range: List) -> Any: def offset(self, value): return 0 + def unmap_to_datacube(path, unmapped_path): + return (path, unmapped_path) + + def remap_to_requeest(path, unmapped_path): + return (path, unmapped_path) + + def find_indices_between(self, index_ranges, low, up, datacube): + # TODO: do we need this ? + # TODO: how does this work for xarray where we get back indices as pandas.Index? + indexes_between_ranges = [] + for indexes in index_ranges: + if self.name in datacube.complete_axes: + # Find the range of indexes between lower and upper + # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html + # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.append(indexes_between) + else: + indexes_between = [i for i in indexes if low <= i <= up] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + @staticmethod def create_axis(name, values, datacube): cyclic_transform = None @@ -223,6 +303,7 @@ class IntDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 range = None + transformations = [] def parse(self, value: Any) -> Any: return float(value) @@ -237,11 +318,13 @@ def serialize(self, value): return value +@mapper @cyclic class FloatDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 range = None + transformations = [] def parse(self, value: Any) -> Any: return float(value) @@ -260,6 +343,7 @@ class PandasTimestampDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 range = None + transformations = [] def parse(self, value: Any) -> Any: if isinstance(value, np.str_): @@ -286,6 +370,7 @@ class PandasTimedeltaDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 range = None + transformations = [] def parse(self, value: Any) -> Any: if isinstance(value, np.str_): @@ -312,6 +397,7 @@ class UnsliceableDatacubeAxis(DatacubeAxis): name = None tol = float("NaN") range = None + transformations = [] def parse(self, value: Any) -> Any: return value diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index 434f02a0c..fa37ae4d1 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -18,6 +18,9 @@ def generate_final_transformation(self): def transformation_axes_final(self): return [self.name] + def change_val_type(self, axis_name, values): + return values + def apply_transformation(self, name, datacube, values): # NOTE: we will handle all the cyclicity mapping here instead of in the DatacubeAxis # then we can generate just create_standard in the configure_axis at the end diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 629298160..57ff83aec 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -4,7 +4,7 @@ import numpy as np -from ..backends.datacube import configure_datacube_axis +# from ..backends.datacube import configure_datacube_axis from .datacube_transformations import DatacubeAxisTransformation @@ -33,10 +33,12 @@ def apply_transformation(self, name, datacube, values): # axis_name = name new_axis_options = datacube.axis_options.get(axis_name, {}) if i == 0: + from ..backends.datacube import configure_datacube_axis values = np.array(transformation.first_axis_vals()) configure_datacube_axis(new_axis_options, axis_name, values, datacube) if i == 1: # the values[0] will be a value on the first axis + from ..backends.datacube import configure_datacube_axis values = np.array(transformation.second_axis_vals(values[0])) configure_datacube_axis(new_axis_options, axis_name, values, datacube) datacube.fake_axes.append(axis_name) @@ -48,6 +50,10 @@ def transformation_axes_final(self): # Needs to also implement its own methods + def change_val_type(self, axis_name, values): + # the new axis_vals created will be floats + return [0.0] + def _mapped_axes(self): # NOTE: Each of the mapper method needs to call it's sub mapper method final_transformation = self.generate_final_transformation() @@ -60,6 +66,14 @@ def _base_axis(self): def _resolution(self): pass + def first_axis_vals(self): + final_transformation = self.generate_final_transformation() + return final_transformation.first_axis_vals() + + def second_axis_vals(self, first_val): + final_transformation = self.generate_final_transformation() + return final_transformation.second_axis_vals(first_val) + def map_first_axis(self, lower, upper): final_transformation = self.generate_final_transformation() return final_transformation.map_first_axis(lower, upper) diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 96e8a9190..b670e72ce 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -69,6 +69,9 @@ def unmerge(self, merged_val): second_val = merged_val[first_idx + first_linker_size : -second_linked_size] return (first_val, second_val) + def change_val_type(self, axis_name, values): + return values + def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): indexes_between = datacube._find_indexes_between(axis, indexes, low, up) return (offset, indexes_between) diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index 742306d9a..da2fa4308 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -43,3 +43,6 @@ def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_i return indexes else: pass + + def change_val_type(self, axis_name, values): + return values diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 19ce0e846..0dd505bf1 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -6,9 +6,7 @@ class DatacubeAxisTransformation(ABC): @staticmethod - def get_final_axes(name, transformation_type_key, transformation_options): - # NOTE: THIS IS ONE OF THE REFACTORED FUNCTIONS - # TODO: refactor this because now it's creating whole transformations which we might not need yet? + def create_transform(name, transformation_type_key, transformation_options): transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] @@ -16,6 +14,16 @@ def get_final_axes(name, transformation_type_key, transformation_options): constructor = getattr(module, transformation_type) transformation_type_option = transformation_options[transformation_type_key] new_transformation = deepcopy(constructor(name, transformation_type_option)) + + new_transformation.name = name + return new_transformation + + @staticmethod + def get_final_axes(name, transformation_type_key, transformation_options): + # NOTE: THIS IS ONE OF THE REFACTORED FUNCTIONS + # TODO: refactor this because now it's creating whole transformations which we might not need yet? + new_transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key, + transformation_options) transformation_axis_names = new_transformation.transformation_axes_final() return transformation_axis_names @@ -99,6 +107,10 @@ def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_i # values which exist on those axes pass + @abstractmethod + def change_val_type(self, axis_name, values): + pass + _type_to_datacube_transformation_lookup = { "mapper": "DatacubeMapper", diff --git a/polytope/datacube/transformations/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change.py index 5ad66b163..7c341f418 100644 --- a/polytope/datacube/transformations/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change.py @@ -69,6 +69,10 @@ def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_i else: pass + def change_val_type(self, axis_name, values): + transformation = self.generate_final_transformation() + return [transformation.transform_type(val) for val in values] + class TypeChangeStrToInt(DatacubeAxisTypeChange): def __init__(self, axis_name, new_type): diff --git a/polytope/datacube/transformations/transformation_pseudocode.py b/polytope/datacube/transformations/transformation_pseudocode.py index c9405123e..07fc0b2c6 100644 --- a/polytope/datacube/transformations/transformation_pseudocode.py +++ b/polytope/datacube/transformations/transformation_pseudocode.py @@ -1,4 +1,4 @@ -from copy import deepcopy +"""from copy import deepcopy def create_axis_transformations(options, name, values, datacube): @@ -78,4 +78,4 @@ def create_axis_transformation(name, transformation_type_key, transformation_opt # For grid mappers, it keeps track that we have a grid_mapper, but won't know the exact grid map we # implement datacube.transformation[axis_name].append(new_transformation) - new_transformation.apply_transformation(name, datacube, values) \ No newline at end of file + new_transformation.apply_transformation(name, datacube, values)""" diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py new file mode 100644 index 000000000..12871c77c --- /dev/null +++ b/tests/test_datacube_axes_init.py @@ -0,0 +1,48 @@ +from earthkit import data + +from polytope.datacube.backends.datacube import Datacube +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.datacube.datacube_axis import FloatDatacubeAxis +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope + + +class TestOctahedralGrid: + def setup_method(self, method): + ds = data.from_source("file", "./tests/data/foo.grib") + latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) + latlon_array = latlon_array.t2m + self.xarraydatacube = XArrayDatacube(latlon_array) + grid_options = { + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + } + } + } + self.slicer = HullSlicer() + self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=grid_options) + self.datacube = self.API.datacube + + def test_created_axes(self): + assert self.datacube._axes["latitude"].has_mapper + assert self.datacube._axes["longitude"].has_mapper + assert type(self.datacube._axes["longitude"]) == FloatDatacubeAxis + assert not ("values" in self.datacube._axes.keys()) + assert self.datacube._axes["latitude"].remap([0], {})[0][:5] == [89.94618771566562, 89.87647835333229, + 89.80635731954224, 89.73614327160958, + 89.6658939412157] + assert self.datacube._axes["longitude"].remap([0], {"latitude": 89.94618771566562})[0][:8] == [0.0, 18.0, 36.0, + 54.0, 72.0, 90.0, + 108.0, 126.0] + assert len(self.datacube._axes["longitude"].remap([0], {"latitude": 89.94618771566562})[0]) == 20 + lon_ax = self.datacube._axes["longitude"] + lat_ax = self.datacube._axes["latitude"] + (path, unmapped_path) = lat_ax.unmap_to_datacube({"latitude": 89.94618771566562}, {}) + assert path == {} + assert unmapped_path == {'latitude': 89.94618771566562} + (path, unmapped_path) = lon_ax.unmap_to_datacube({"longitude": 0.0}, {"latitude": 89.94618771566562}) + assert path == {} + assert unmapped_path == {"values": 0} + assert lat_ax.find_indices_between(lat_ax, [[89.94618771566562, 89.87647835333229]], + 89.87, 90, self.datacube) == [[89.94618771566562, 89.87647835333229]] From 3c5c17d592402c5598fa8347f7132092fc3d68b9 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 28 Aug 2023 16:11:59 +0200 Subject: [PATCH 113/332] make transformations as decorators, but only works for one transformations at a time for now --- polytope/datacube/backends/datacube.py | 8 - polytope/datacube/backends/xarray.py | 267 +++++------------- polytope/datacube/datacube_axis.py | 209 +++++++++++++- .../transformations/datacube_cyclic.py | 3 + .../transformations/datacube_mappers.py | 37 +-- .../transformations/datacube_merger.py | 36 +-- .../transformations/datacube_reverse.py | 31 +- .../datacube_transformations.py | 9 +- .../transformations/datacube_type_change.py | 35 ++- tests/test_datacube_axes_init.py | 36 ++- tests/test_merge_octahedral_one_axis.py | 2 +- tests/test_merge_transformation.py | 3 +- 12 files changed, 390 insertions(+), 286 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 2f23b46ab..2de302a0e 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -50,14 +50,6 @@ def create(datacube, axis_options: dict): else: return datacube - @abstractmethod - def ax_vals(self, name: str) -> List: - pass - - @abstractmethod - def _find_indexes_between(self, axis, indexes, low, up): - pass - # TODO: need to add transformation properties like the datacube.transformations dico diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index b0d47199e..d6a4d1aa4 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -16,15 +16,14 @@ class XArrayDatacube(Datacube): """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0).""" - def create_axes(self, name, values, transformation_type_key, transformation_options): - # NOTE: THIS IS ONE OF THE REFACTORED FUNCTIONS - # NOTE: need to do this for all transformations on an axis - + def _create_axes(self, name, values, transformation_type_key, transformation_options): # first check what the final axes are for this axis name given transformations final_axis_names = DatacubeAxisTransformation.get_final_axes(name, transformation_type_key, transformation_options) transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key, transformation_options) + for blocked_axis in transformation.blocked_axes(): + self.blocked_axes.append(blocked_axis) for axis_name in final_axis_names: # if axis does not yet exist, create it @@ -41,19 +40,19 @@ def create_axes(self, name, values, transformation_type_key, transformation_opti decorator = getattr(decorator_module, transformation_type_key) decorator(self._axes[axis_name]) if transformation not in self._axes[axis_name].transformations: # Avoids duplicates being stored - # NOTE: TODO: here, the same transformation gets added twice to each axis? not sure why self._axes[axis_name].transformations.append(transformation) - def add_all_transformation_axes(self, options, name, values): + def _add_all_transformation_axes(self, options, name, values): transformation_options = options["transformation"] for transformation_type_key in transformation_options.keys(): - self.create_axes(name, values, transformation_type_key, transformation_options) + self._create_axes(name, values, transformation_type_key, transformation_options) - def check_and_add_axes(self, options, name, values): + def _check_and_add_axes(self, options, name, values): if "transformation" in options: - self.add_all_transformation_axes(options, name, values) + self._add_all_transformation_axes(options, name, values) else: - DatacubeAxis.create_standard(name, values, self) + if name not in self.blocked_axes: + DatacubeAxis.create_standard(name, values, self) def __init__(self, dataarray: xr.DataArray, axis_options={}): self.axis_options = axis_options @@ -62,6 +61,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self._axes = {} self.dataarray = dataarray treated_axes = [] + self.non_complete_axes = [] self.complete_axes = [] self.blocked_axes = [] self.transformation = {} @@ -69,42 +69,59 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: options = axis_options.get(name, {}) - self.check_and_add_axes(options, name, values) + self._check_and_add_axes(options, name, values) treated_axes.append(name) self.complete_axes.append(name) else: if self.dataarray[name].dims == (): options = axis_options.get(name, {}) - self.check_and_add_axes(options, name, values) + self._check_and_add_axes(options, name, values) treated_axes.append(name) + self.non_complete_axes.append(name) for name in dataarray.dims: if name not in treated_axes: options = axis_options.get(name, {}) val = dataarray[name].values[0] - self.check_and_add_axes(options, name, val) + self._check_and_add_axes(options, name, val) + treated_axes.append(name) + + # add other options to axis which were just created above like "lat" for the mapper transformations for eg + # for name in self._axes: + # if name not in treated_axes: + # options = axis_options.get(name, {}) + # val = dataarray[name].values[0] + # self._check_and_add_axes(options, name, val) + # if "longitude" in self._axes: + # print(self._axes["longitude"].transformations) + + def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): + idx_between = [] + for i in range(len(search_ranges)): + r = search_ranges[i] + offset = search_ranges_offset[i] + low = r[0] + up = r[1] + print(up) + print(low) + print(indexes) + indexes_between = axis.find_indices_between([indexes], low, up, self) + # Now the indexes_between are values on the cyclic range so need to remap them to their original + # values before returning them + for j in range(len(indexes_between)): + for k in range(len(indexes_between[j])): + if offset is None: + indexes_between[j][k] = indexes_between[j][k] + else: + indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) + idx_between.append(indexes_between[j][k]) + return idx_between def get_indices(self, path: DatacubePath, axis, lower, upper): - # TODO: here now, use the axis decorator to handle all transformations - (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube(path) + path = self.fit_path(path) + indexes = axis.find_indexes(path, self) + print("inside get indices") + print(indexes) - subarray = self.dataarray.sel(path, method="nearest") - subarray = subarray.sel(unmap_path) - subarray = subarray.sel(changed_type_path) - # Get the indexes of the axis we want to query - # XArray does not support branching, so no need to use label, we just take the next axis - if axis.name in self.transformation.keys(): - axis_transforms = self.transformation[axis.name] - # This bool will help us decide for which axes we need to calculate the indexes again or not - # in case there are multiple relevant transformations for an axis - already_has_indexes = False - for transform in axis_transforms: - # TODO: here, instead of creating the indices, would be better to create the standard datacube axes and - # then succesively map them to what they should be - indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) - already_has_indexes = True - else: - indexes = self.datacube_natural_indexes(axis, subarray) - # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube search_ranges = axis.remap([lower, upper]) original_search_ranges = axis.to_intervals([lower, upper]) # Find the offsets for each interval in the requested range, which we will need later @@ -112,55 +129,27 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): for r in original_search_ranges: offset = axis.offset(r) search_ranges_offset.append(offset) - # Look up the values in the datacube for each cyclic interval range - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) + + idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis) # Remove duplicates even if difference of the order of the axis tolerance if offset is not None: # Note that we can only do unique if not dealing with time values idx_between = unique(idx_between) return idx_between - # def __init__(self, dataarray: xr.DataArray, axis_options={}): - # self.axis_options = axis_options - # self.grid_mapper = None - # self.axis_counter = 0 - # self._axes = {} - # self.dataarray = dataarray - # treated_axes = [] - # self.complete_axes = [] - # self.blocked_axes = [] - # self.transformation = {} - # self.fake_axes = [] - # for name, values in dataarray.coords.variables.items(): - # if name in dataarray.dims: - # options = axis_options.get(name, {}) - # configure_datacube_axis(options, name, values, self) - # treated_axes.append(name) - # self.complete_axes.append(name) - # else: - # if self.dataarray[name].dims == (): - # options = axis_options.get(name, {}) - # configure_datacube_axis(options, name, values, self) - # treated_axes.append(name) - # for name in dataarray.dims: - # if name not in treated_axes: - # options = axis_options.get(name, {}) - # val = dataarray[name].values[0] - # configure_datacube_axis(options, name, val, self) - def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() if len(path.items()) == self.axis_counter: # first, find the grid mapper transform - unmap_path = {} - considered_axes = [] - (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube( - path - ) - unmap_path.update(changed_type_path) + unmapped_path = {} + path_copy = deepcopy(path) + for key in path_copy: + axis = self._axes[key] + (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) + path = self.fit_path(path) subxarray = self.dataarray.sel(path, method="nearest") - subxarray = subxarray.sel(unmap_path) + subxarray = subxarray.sel(unmapped_path) value = subxarray.item() key = subxarray.name r.result = (key, value) @@ -177,45 +166,6 @@ def remap_path(self, path: DatacubePath): path[key] = self._axes[key].remap([value, value])[0][0] return path - def _find_indexes_between(self, axis, indexes, low, up): - if axis.name in self.complete_axes: - # Find the range of indexes between lower and upper - # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html - # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - indexes_between = indexes[start:end].to_list() - else: - indexes_between = [i for i in indexes if low <= i <= up] - return indexes_between - - def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): - idx_between = [] - for i in range(len(search_ranges)): - r = search_ranges[i] - offset = search_ranges_offset[i] - low = r[0] - up = r[1] - if axis.name in self.transformation.keys(): - axis_transforms = self.transformation[axis.name] - temp_indexes = deepcopy(indexes) - for transform in axis_transforms: - (offset, temp_indexes) = transform._find_transformed_indices_between( - axis, self, temp_indexes, low, up, first_val, offset - ) - indexes_between = temp_indexes - else: - indexes_between = self._find_indexes_between(axis, indexes, low, up) - # Now the indexes_between are values on the cyclic range so need to remap them to their original - # values before returning them - for j in range(len(indexes_between)): - if offset is None: - indexes_between[j] = indexes_between[j] - else: - indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) - idx_between.append(indexes_between[j]) - return idx_between - def datacube_natural_indexes(self, axis, subarray): if axis.name in self.complete_axes: indexes = next(iter(subarray.xindexes.values())).to_pandas_index() @@ -223,95 +173,30 @@ def datacube_natural_indexes(self, axis, subarray): indexes = [subarray[axis.name].values] return indexes - def fit_path_to_datacube(self, axis_name, path, considered_axes=[], unmap_path={}): - # TODO: how to make this also work for the get method? - path = self.remap_path(path) - for key in path.keys(): - if self.dataarray[key].dims == (): - path.pop(key) - first_val = None - changed_type_path = {} - if axis_name in self.transformation.keys(): - axis_transforms = self.transformation[axis_name] - first_val = None - for transform in axis_transforms: - (path, temp_first_val, considered_axes, unmap_path, changed_type_path) = transform._adjust_path( - path, considered_axes, unmap_path, changed_type_path - ) - if temp_first_val: - first_val = temp_first_val - return (path, first_val, considered_axes, unmap_path, changed_type_path) - - def fit_path_to_original_datacube(self, path): - path = self.remap_path(path) - first_val = None - unmap_path = {} - considered_axes = [] - changed_type_path = {} - for axis_name in self.transformation.keys(): - axis_transforms = self.transformation[axis_name] - for transform in axis_transforms: - (path, temp_first_val, considered_axes, unmap_path, changed_type_path) = transform._adjust_path( - path, considered_axes, unmap_path, changed_type_path - ) - if temp_first_val: - first_val = temp_first_val + def fit_path(self, path): + # path = self.remap_path(path) for key in path.keys(): - if self.dataarray[key].dims == (): + if key in self.non_complete_axes: path.pop(key) - return (path, first_val, considered_axes, unmap_path, changed_type_path) - - # def get_indices(self, path: DatacubePath, axis, lower, upper): - # (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube(path) - - # subarray = self.dataarray.sel(path, method="nearest") - # subarray = subarray.sel(unmap_path) - # subarray = subarray.sel(changed_type_path) - # # Get the indexes of the axis we want to query - # # XArray does not support branching, so no need to use label, we just take the next axis - # if axis.name in self.transformation.keys(): - # axis_transforms = self.transformation[axis.name] - # # This bool will help us decide for which axes we need to calculate the indexes again or not - # # in case there are multiple relevant transformations for an axis - # already_has_indexes = False - # for transform in axis_transforms: - # # TODO: here, instead of creating the indices, would be better to create the standard datacube axes and - # # then succesively map them to what they should be - # indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) - # already_has_indexes = True - # else: - # indexes = self.datacube_natural_indexes(axis, subarray) - # # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube - # search_ranges = axis.remap([lower, upper]) - # original_search_ranges = axis.to_intervals([lower, upper]) - # # Find the offsets for each interval in the requested range, which we will need later - # search_ranges_offset = [] - # for r in original_search_ranges: - # offset = axis.offset(r) - # search_ranges_offset.append(offset) - # # Look up the values in the datacube for each cyclic interval range - # idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) - # # Remove duplicates even if difference of the order of the axis tolerance - # if offset is not None: - # # Note that we can only do unique if not dealing with time values - # idx_between = unique(idx_between) - # return idx_between + return path def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - path = self.fit_path_to_datacube(axis.name, path)[0] + # path = self.fit_path_to_datacube(axis.name, path)[0] + path = self.fit_path(path) + indexes = axis.find_indexes(path, self) # Open a view on the subset identified by the path - subarray = self.dataarray.sel(path, method="nearest") - if axis.name in self.transformation.keys(): - axis_transforms = self.transformation[axis.name] - already_has_indexes = False - for transform in axis_transforms: - indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) - already_has_indexes = True - # return index in subarray_vals - else: - indexes = self.datacube_natural_indexes(axis, subarray) + # subarray = self.dataarray.sel(path, method="nearest") + # if axis.name in self.transformation.keys(): + # axis_transforms = self.transformation[axis.name] + # already_has_indexes = False + # for transform in axis_transforms: + # indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) + # already_has_indexes = True + # # return index in subarray_vals + # else: + # indexes = self.datacube_natural_indexes(axis, subarray) return index in indexes @property diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 79f3f077d..ac8a4049e 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -12,7 +12,15 @@ def cyclic(cls): if cls.is_cyclic: + def update_range(): + from .transformations.datacube_cyclic import DatacubeAxisCyclic + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisCyclic): + transformation = transform + cls.range = transformation.range + def to_intervals(range): + update_range() axis_lower = cls.range[0] axis_upper = cls.range[1] axis_range = axis_upper - axis_lower @@ -48,6 +56,7 @@ def to_intervals(range): return intervals def _remap_range_to_axis_range(range): + update_range() axis_lower = cls.range[0] axis_upper = cls.range[1] axis_range = axis_upper - axis_lower @@ -75,7 +84,8 @@ def _remap_val_to_axis_range(value): return_range = _remap_range_to_axis_range([value, value]) return return_range[0] - def remap(range: List, path): + def remap(range: List): + update_range() if cls.range[0] - cls.tol <= range[0] <= cls.range[1] + cls.tol: if cls.range[0] - cls.tol <= range[1] <= cls.range[1] + cls.tol: # If we are already in the cyclic range, return it @@ -103,6 +113,13 @@ def remap(range: List, path): ranges.append([low - cls.tol, up + cls.tol]) return ranges + def find_indexes(path, datacube): + unmapped_path = {} + (new_path, unmapped_path) = cls.unmap_to_datacube(path, unmapped_path) + subarray = datacube.dataarray.sel(new_path, method="nearest") + subarray = subarray.sel(unmapped_path) + return datacube.datacube_natural_indexes(cls, subarray) + def offset(range): # We first unpad the range by the axis tolerance to make sure that # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. @@ -115,6 +132,7 @@ def offset(range): cls.to_intervals = to_intervals cls.remap = remap cls.offset = offset + cls.find_indexes = find_indexes return cls @@ -122,20 +140,19 @@ def offset(range): def mapper(cls): from .transformations.datacube_mappers import DatacubeMapper - # NOTE: THIS IS ONE OF THE REFACTORED FUNCTIONS if cls.has_mapper: - # if cls.name in cls.transformation.keys(): - # # the name of the class is in the transformation dictionary - def remap(range, path): + def find_indexes(path, datacube): # first, find the relevant transformation object that is a mapping in the cls.transformation dico for transform in cls.transformations: if isinstance(transform, DatacubeMapper): transformation = transform if cls.name == transformation._mapped_axes()[0]: - return [transformation.first_axis_vals()] + # print("INSIDE THE MAPPING") + # print(transformation.first_axis_vals()) + return transformation.first_axis_vals() if cls.name == transformation._mapped_axes()[1]: first_val = path[transformation._mapped_axes()[0]] - return [transformation.second_axis_vals(first_val)] + return transformation.second_axis_vals(first_val) def unmap_to_datacube(path, unmapped_path): for transform in cls.transformations: @@ -163,7 +180,7 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(cls, index_ranges, low, up, datacube): + def find_indices_between(index_ranges, low, up, datacube): indexes_between_ranges = [] for transform in cls.transformations: if isinstance(transform, DatacubeMapper): @@ -174,7 +191,159 @@ def find_indices_between(cls, index_ranges, low, up, datacube): indexes_between_ranges.append(indexes_between) return indexes_between_ranges + def remap(range): + return [range] + + cls.remap = remap + cls.find_indexes = find_indexes + cls.unmap_to_datacube = unmap_to_datacube + cls.remap_to_requested = remap_to_requested + cls.find_indices_between = find_indices_between + + return cls + + +def merge(cls): + from .transformations.datacube_merger import DatacubeAxisMerger + + if cls.has_merger: + def find_indexes(path, datacube): + # first, find the relevant transformation object that is a mapping in the cls.transformation dico + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisMerger): + transformation = transform + if cls.name == transformation._first_axis: + return transformation.merged_values(datacube) + + def unmap_to_datacube(path, unmapped_path): + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisMerger): + transformation = transform + if cls.name == transformation._first_axis: + old_val = path.get(cls.name, None) + (first_val, second_val) = transformation.unmerge(old_val) + path.pop(cls.name, None) + path[transformation._first_axis] = first_val + path[transformation._second_axis] = second_val + return (path, unmapped_path) + + def remap_to_requested(path, unmapped_path): + return (path, unmapped_path) + + def find_indices_between(index_ranges, low, up, datacube): + indexes_between_ranges = [] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisMerger): + transformation = transform + if cls.name in transformation._mapped_axes(): + for indexes in index_ranges: + indexes_between = [i for i in indexes if low <= i <= up] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + def remap(range): + return [range] + cls.remap = remap + cls.find_indexes = find_indexes + cls.unmap_to_datacube = unmap_to_datacube + cls.remap_to_requested = remap_to_requested + cls.find_indices_between = find_indices_between + + return cls + + +def reverse(cls): + from .transformations.datacube_reverse import DatacubeAxisReverse + + if cls.reorder: + def find_indexes(path, datacube): + # first, find the relevant transformation object that is a mapping in the cls.transformation dico + subarray = datacube.dataarray.sel(path, method="nearest") + unordered_indices = datacube.datacube_natural_indexes(cls, subarray) + if cls.name in datacube.complete_axes: + ordered_indices = unordered_indices.sort_values() + else: + ordered_indices = unordered_indices + return ordered_indices + + def unmap_to_datacube(path, unmapped_path): + return (path, unmapped_path) + + def remap_to_requested(path, unmapped_path): + return (path, unmapped_path) + + def find_indices_between(index_ranges, low, up, datacube): + indexes_between_ranges = [] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisReverse): + transformation = transform + if cls.name == transformation.name: + for indexes in index_ranges: + indexes_between = [i for i in indexes if low <= i <= up] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + def remap(range): + return [range] + + cls.remap = remap + cls.find_indexes = find_indexes + cls.unmap_to_datacube = unmap_to_datacube + cls.remap_to_requested = remap_to_requested + cls.find_indices_between = find_indices_between + + return cls + + +def type_change(cls): + from .transformations.datacube_type_change import DatacubeAxisTypeChange + + if cls.type_change: + + def find_indexes(path, datacube): + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisTypeChange): + transformation = transform + if cls.name == transformation.name: + unmapped_path = {} + (new_path, unmapped_path) = cls.unmap_to_datacube(path, unmapped_path) + subarray = datacube.dataarray.sel(new_path, method="nearest") + subarray = subarray.sel(unmapped_path) + original_vals = datacube.datacube_natural_indexes(cls, subarray) + return transformation.change_val_type(cls.name, original_vals) + + def unmap_to_datacube(path, unmapped_path): + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisTypeChange): + transformation = transform + if cls.name == transformation.name: + changed_val = path.get(cls.name, None) + unchanged_val = transformation.make_str(changed_val) + if cls.name in path: + path.pop(cls.name, None) + unmapped_path[cls.name] = unchanged_val + return (path, unmapped_path) + + def remap_to_requested(path, unmapped_path): + return (path, unmapped_path) + + def find_indices_between(index_ranges, low, up, datacube): + indexes_between_ranges = [] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisTypeChange): + transformation = transform + if cls.name == transformation.name: + for indexes in index_ranges: + indexes_between = [i for i in indexes if low <= i <= up] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + def remap(range): + return [range] + + cls.remap = remap + cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.remap_to_requested = remap_to_requested cls.find_indices_between = find_indices_between @@ -185,6 +354,9 @@ def find_indices_between(cls, index_ranges, low, up, datacube): class DatacubeAxis(ABC): is_cyclic = False has_mapper = False + has_merger = False + reorder = False + type_change = False def update_axis(self): if self.is_cyclic: @@ -227,10 +399,18 @@ def to_intervals(self, range): def remap(self, range: List) -> Any: return [range] + def find_indexes(self, path, datacube): + # TODO: does this do what it should? + unmapped_path = {} + (new_path, unmapped_path) = self.unmap_to_datacube(path, unmapped_path) + subarray = datacube.dataarray.sel(new_path, method="nearest") + subarray = subarray.sel(unmapped_path) + return datacube.datacube_natural_indexes(self, subarray) + def offset(self, value): return 0 - def unmap_to_datacube(path, unmapped_path): + def unmap_to_datacube(self, path, unmapped_path): return (path, unmapped_path) def remap_to_requeest(path, unmapped_path): @@ -238,7 +418,6 @@ def remap_to_requeest(path, unmapped_path): def find_indices_between(self, index_ranges, low, up, datacube): # TODO: do we need this ? - # TODO: how does this work for xarray where we get back indices as pandas.Index? indexes_between_ranges = [] for indexes in index_ranges: if self.name in datacube.complete_axes: @@ -298,7 +477,10 @@ def check_axis_type(name, values): raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") +@reverse @cyclic +@mapper +@type_change class IntDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 @@ -318,8 +500,10 @@ def serialize(self, value): return value -@mapper +@reverse @cyclic +@mapper +@type_change class FloatDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 @@ -339,6 +523,7 @@ def serialize(self, value): return value +@merge class PandasTimestampDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 @@ -366,6 +551,7 @@ def offset(self, value): return None +@merge class PandasTimedeltaDatacubeAxis(DatacubeAxis): name = None tol = 1e-12 @@ -393,6 +579,7 @@ def offset(self, value): return None +@type_change class UnsliceableDatacubeAxis(DatacubeAxis): name = None tol = float("NaN") diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index fa37ae4d1..5859c0f1f 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -21,6 +21,9 @@ def transformation_axes_final(self): def change_val_type(self, axis_name, values): return values + def blocked_axes(self): + return [] + def apply_transformation(self, name, datacube, values): # NOTE: we will handle all the cyclicity mapping here instead of in the DatacubeAxis # then we can generate just create_standard in the configure_axis at the end diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 57ff83aec..40b9eb68c 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -25,23 +25,26 @@ def generate_final_transformation(self): transformation = deepcopy(constructor(self.old_axis, self.grid_axes, self.grid_resolution)) return transformation - def apply_transformation(self, name, datacube, values): - # Create mapped axes here - transformation = self.generate_final_transformation() - for i in range(len(transformation._mapped_axes)): - axis_name = transformation._mapped_axes[i] - # axis_name = name - new_axis_options = datacube.axis_options.get(axis_name, {}) - if i == 0: - from ..backends.datacube import configure_datacube_axis - values = np.array(transformation.first_axis_vals()) - configure_datacube_axis(new_axis_options, axis_name, values, datacube) - if i == 1: - # the values[0] will be a value on the first axis - from ..backends.datacube import configure_datacube_axis - values = np.array(transformation.second_axis_vals(values[0])) - configure_datacube_axis(new_axis_options, axis_name, values, datacube) - datacube.fake_axes.append(axis_name) + def blocked_axes(self): + return [] + + # def apply_transformation(self, name, datacube, values): + # # Create mapped axes here + # transformation = self.generate_final_transformation() + # for i in range(len(transformation._mapped_axes)): + # axis_name = transformation._mapped_axes[i] + # # axis_name = name + # new_axis_options = datacube.axis_options.get(axis_name, {}) + # if i == 0: + # from ..backends.datacube import configure_datacube_axis + # values = np.array(transformation.first_axis_vals()) + # configure_datacube_axis(new_axis_options, axis_name, values, datacube) + # if i == 1: + # # the values[0] will be a value on the first axis + # from ..backends.datacube import configure_datacube_axis + # values = np.array(transformation.second_axis_vals(values[0])) + # configure_datacube_axis(new_axis_options, axis_name, values, datacube) + # datacube.fake_axes.append(axis_name) def transformation_axes_final(self): final_transformation = self.generate_final_transformation() diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index b670e72ce..1e2a469de 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -3,7 +3,7 @@ import numpy as np import pandas as pd -from ..backends.datacube import configure_datacube_axis +# from ..backends.datacube import configure_datacube_axis from .datacube_transformations import DatacubeAxisTransformation @@ -15,6 +15,9 @@ def __init__(self, name, merge_options): self._second_axis = merge_options["with"] self._linkers = merge_options["linkers"] + def blocked_axes(self): + return [self._second_axis] + def merged_values(self, datacube): first_ax_vals = datacube.ax_vals(self.name) second_ax_name = self._second_axis @@ -29,25 +32,26 @@ def merged_values(self, datacube): val_to_add = val_to_add.to_numpy() val_to_add = val_to_add.astype("datetime64[s]") # merged_values.append(pd.to_datetime(first_val + linkers[0] + second_val + linkers[1])) + val_to_add = str(val_to_add) merged_values.append(val_to_add) merged_values = np.array(merged_values) return merged_values - def apply_transformation(self, name, datacube, values): - merged_values = self.merged_values(datacube) - # Remove the merge option from the axis options since we have already handled it - # so do not want to handle it again - axis_options = deepcopy(datacube.axis_options[name]["transformation"]) - axis_options.pop("merge") - # Update the nested dictionary with the modified axis option for our axis - new_datacube_axis_options = deepcopy(datacube.axis_options) - if axis_options == {}: - new_datacube_axis_options[name] = {} - else: - new_datacube_axis_options[name]["transformation"] = axis_options - # Reconfigure the axis with the rest of its configurations - configure_datacube_axis(new_datacube_axis_options[name], name, merged_values, datacube) - self.finish_transformation(datacube, merged_values) + # def apply_transformation(self, name, datacube, values): + # merged_values = self.merged_values(datacube) + # # Remove the merge option from the axis options since we have already handled it + # # so do not want to handle it again + # axis_options = deepcopy(datacube.axis_options[name]["transformation"]) + # axis_options.pop("merge") + # # Update the nested dictionary with the modified axis option for our axis + # new_datacube_axis_options = deepcopy(datacube.axis_options) + # if axis_options == {}: + # new_datacube_axis_options[name] = {} + # else: + # new_datacube_axis_options[name]["transformation"] = axis_options + # # Reconfigure the axis with the rest of its configurations + # configure_datacube_axis(new_datacube_axis_options[name], name, merged_values, datacube) + # self.finish_transformation(datacube, merged_values) def transformation_axes_final(self): return [self._first_axis] diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index da2fa4308..d933e8f47 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -1,6 +1,6 @@ from copy import deepcopy -from ..backends.datacube import configure_datacube_axis +# from ..backends.datacube import configure_datacube_axis from .datacube_transformations import DatacubeAxisTransformation @@ -12,19 +12,19 @@ def __init__(self, name, mapper_options): def generate_final_transformation(self): return self - def apply_transformation(self, name, datacube, values): - # Remove the merge option from the axis options since we have already handled it - # so do not want to handle it again - axis_options = deepcopy(datacube.axis_options[name]["transformation"]) - axis_options.pop("reverse") - # Update the nested dictionary with the modified axis option for our axis - new_datacube_axis_options = deepcopy(datacube.axis_options) - if axis_options == {}: - new_datacube_axis_options[name] = {} - else: - new_datacube_axis_options[name]["transformation"] = axis_options - # Reconfigure the axis with the rest of its configurations - configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) + # def apply_transformation(self, name, datacube, values): + # # Remove the merge option from the axis options since we have already handled it + # # so do not want to handle it again + # axis_options = deepcopy(datacube.axis_options[name]["transformation"]) + # axis_options.pop("reverse") + # # Update the nested dictionary with the modified axis option for our axis + # new_datacube_axis_options = deepcopy(datacube.axis_options) + # if axis_options == {}: + # new_datacube_axis_options[name] = {} + # else: + # new_datacube_axis_options[name]["transformation"] = axis_options + # # Reconfigure the axis with the rest of its configurations + # configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) def transformation_axes_final(self): return [self.name] @@ -46,3 +46,6 @@ def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_i def change_val_type(self, axis_name, values): return values + + def blocked_axes(self): + return [] diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 0dd505bf1..e9509b34e 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -81,9 +81,9 @@ def generate_final_transformation(self): def transformation_axes_final(self): pass - @abstractmethod - def apply_transformation(self, name, datacube, values): - pass + # @abstractmethod + # def apply_transformation(self, name, datacube, values): + # pass # Methods to deal with transformation in datacube backends @abstractmethod @@ -131,4 +131,7 @@ def change_val_type(self, axis_name, values): has_transform = { "mapper": "has_mapper", "cyclic" : "is_cyclic", + "merge" : "has_merger", + "reverse": "reorder", + "type_change": "type_change" } diff --git a/polytope/datacube/transformations/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change.py index 7c341f418..fe3bf7783 100644 --- a/polytope/datacube/transformations/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change.py @@ -1,7 +1,7 @@ from copy import deepcopy from importlib import import_module -from ..backends.datacube import configure_datacube_axis +# from ..backends.datacube import configure_datacube_axis from .datacube_transformations import DatacubeAxisTransformation @@ -24,19 +24,19 @@ def transformation_axes_final(self): final_transformation = self.generate_final_transformation() return [final_transformation.axis_name] - def apply_transformation(self, name, datacube, values): - transformation = self.generate_final_transformation() - axis_options = deepcopy(datacube.axis_options[name]["transformation"]) - axis_options.pop("type_change") - # Update the nested dictionary with the modified axis option for our axis - new_datacube_axis_options = deepcopy(datacube.axis_options) - # if we have no transformations left, then empty the transformation dico - if axis_options == {}: - new_datacube_axis_options[name] = {} - else: - new_datacube_axis_options[name]["transformation"] = axis_options - values = [transformation.transform_type(values[0])] # only need 1 value to determine type in datacube config - configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) + # def apply_transformation(self, name, datacube, values): + # transformation = self.generate_final_transformation() + # axis_options = deepcopy(datacube.axis_options[name]["transformation"]) + # axis_options.pop("type_change") + # # Update the nested dictionary with the modified axis option for our axis + # new_datacube_axis_options = deepcopy(datacube.axis_options) + # # if we have no transformations left, then empty the transformation dico + # if axis_options == {}: + # new_datacube_axis_options[name] = {} + # else: + # new_datacube_axis_options[name]["transformation"] = axis_options + # values = [transformation.transform_type(values[0])] # only need 1 value to determine type in datacube config + # configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): # NOTE: needs to be in new type @@ -73,6 +73,13 @@ def change_val_type(self, axis_name, values): transformation = self.generate_final_transformation() return [transformation.transform_type(val) for val in values] + def make_str(self, value): + transformation = self.generate_final_transformation() + return transformation.make_str(value) + + def blocked_axes(self): + return [] + class TypeChangeStrToInt(DatacubeAxisTypeChange): def __init__(self, axis_name, new_type): diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 12871c77c..83461e926 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -1,10 +1,10 @@ from earthkit import data -from polytope.datacube.backends.datacube import Datacube from polytope.datacube.backends.xarray import XArrayDatacube from polytope.datacube.datacube_axis import FloatDatacubeAxis from polytope.engine.hullslicer import HullSlicer -from polytope.polytope import Polytope +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select class TestOctahedralGrid: @@ -29,13 +29,17 @@ def test_created_axes(self): assert self.datacube._axes["longitude"].has_mapper assert type(self.datacube._axes["longitude"]) == FloatDatacubeAxis assert not ("values" in self.datacube._axes.keys()) - assert self.datacube._axes["latitude"].remap([0], {})[0][:5] == [89.94618771566562, 89.87647835333229, - 89.80635731954224, 89.73614327160958, - 89.6658939412157] - assert self.datacube._axes["longitude"].remap([0], {"latitude": 89.94618771566562})[0][:8] == [0.0, 18.0, 36.0, - 54.0, 72.0, 90.0, - 108.0, 126.0] - assert len(self.datacube._axes["longitude"].remap([0], {"latitude": 89.94618771566562})[0]) == 20 + assert self.datacube._axes["latitude"].find_indexes({}, + self.datacube)[:5] == [89.94618771566562, + 89.87647835333229, + 89.80635731954224, + 89.73614327160958, + 89.6658939412157] + assert self.datacube._axes["longitude"].find_indexes({"latitude": 89.94618771566562}, + self.datacube)[:8] == [0.0, 18.0, 36.0, 54.0, + 72.0, 90.0, 108.0, 126.0] + assert len(self.datacube._axes["longitude"].find_indexes({"latitude": 89.94618771566562}, + self.datacube)) == 20 lon_ax = self.datacube._axes["longitude"] lat_ax = self.datacube._axes["latitude"] (path, unmapped_path) = lat_ax.unmap_to_datacube({"latitude": 89.94618771566562}, {}) @@ -44,5 +48,17 @@ def test_created_axes(self): (path, unmapped_path) = lon_ax.unmap_to_datacube({"longitude": 0.0}, {"latitude": 89.94618771566562}) assert path == {} assert unmapped_path == {"values": 0} - assert lat_ax.find_indices_between(lat_ax, [[89.94618771566562, 89.87647835333229]], + assert lat_ax.find_indices_between([[89.94618771566562, 89.87647835333229]], 89.87, 90, self.datacube) == [[89.94618771566562, 89.87647835333229]] + + def test_mapper_transformation_request(self): + request = Request( + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + Select("number", [0]), + Select("time", ["2023-06-25T12:00:00"]), + Select("step", ["00:00:00"]), + Select("surface", [0]), + Select("valid_time", ["2023-06-25T12:00:00"]), + ) + result = self.API.retrieve(request) + assert len(result.leaves) == 9 diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index f2de6b3d0..6d0f96aa8 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -33,5 +33,5 @@ def test_merge_axis(self): Box(["latitude", "longitude"], [0, 359.8], [0.2, 361.2]), ) result = self.API.retrieve(request) - # result.pprint() + result.pprint() assert result.leaves[0].flatten()["longitude"] == 0.0 diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index d7dcf95b2..6fc33c553 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -26,4 +26,5 @@ def setup_method(self, method): def test_merge_axis(self): request = Request(Select("date", ["2000-01-01T06:00:00"])) result = self.API.retrieve(request) - assert result.leaves[0].flatten()["date"] == np.datetime64("2000-01-01T06:00:00") + # assert result.leaves[0].flatten()["date"] == np.datetime64("2000-01-01T06:00:00") + assert result.leaves[0].flatten()["date"] == "2000-01-01T06:00:00" From 84c2ea50d02086190bacffde88fec2ad649d2bb7 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 28 Aug 2023 17:12:14 +0200 Subject: [PATCH 114/332] start supporting multiple transformations --- polytope/datacube/backends/xarray.py | 29 ++++++++++++++-------------- polytope/datacube/datacube_axis.py | 8 ++++++-- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index d6a4d1aa4..1f66a996d 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -52,7 +52,8 @@ def _check_and_add_axes(self, options, name, values): self._add_all_transformation_axes(options, name, values) else: if name not in self.blocked_axes: - DatacubeAxis.create_standard(name, values, self) + if name not in self._axes.keys(): + DatacubeAxis.create_standard(name, values, self) def __init__(self, dataarray: xr.DataArray, axis_options={}): self.axis_options = axis_options @@ -86,13 +87,15 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): treated_axes.append(name) # add other options to axis which were just created above like "lat" for the mapper transformations for eg - # for name in self._axes: - # if name not in treated_axes: - # options = axis_options.get(name, {}) - # val = dataarray[name].values[0] - # self._check_and_add_axes(options, name, val) - # if "longitude" in self._axes: - # print(self._axes["longitude"].transformations) + for name in self._axes: + if name not in treated_axes: + options = axis_options.get(name, {}) + val = dataarray[name].values[0] + self._check_and_add_axes(options, name, val) + # if "latitude" in self._axes: + # print(self._axes["latitude"].transformations) + # print(self._axes["longitude"].has_mapper) + # print(self._axes["longitude"].is_cyclic) def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): idx_between = [] @@ -101,9 +104,6 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): offset = search_ranges_offset[i] low = r[0] up = r[1] - print(up) - print(low) - print(indexes) indexes_between = axis.find_indices_between([indexes], low, up, self) # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them @@ -119,8 +119,6 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): def get_indices(self, path: DatacubePath, axis, lower, upper): path = self.fit_path(path) indexes = axis.find_indexes(path, self) - print("inside get indices") - print(indexes) search_ranges = axis.remap([lower, upper]) original_search_ranges = axis.to_intervals([lower, upper]) @@ -170,7 +168,10 @@ def datacube_natural_indexes(self, axis, subarray): if axis.name in self.complete_axes: indexes = next(iter(subarray.xindexes.values())).to_pandas_index() else: - indexes = [subarray[axis.name].values] + if subarray[axis.name].values.ndim == 0: + indexes = [subarray[axis.name].values] + else: + indexes = subarray[axis.name].values return indexes def fit_path(self, path): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index ac8a4049e..97501862d 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -118,6 +118,8 @@ def find_indexes(path, datacube): (new_path, unmapped_path) = cls.unmap_to_datacube(path, unmapped_path) subarray = datacube.dataarray.sel(new_path, method="nearest") subarray = subarray.sel(unmapped_path) + # print(cls.name) + # print(datacube.datacube_natural_indexes(cls, subarray)) return datacube.datacube_natural_indexes(cls, subarray) def offset(range): @@ -147,8 +149,6 @@ def find_indexes(path, datacube): if isinstance(transform, DatacubeMapper): transformation = transform if cls.name == transformation._mapped_axes()[0]: - # print("INSIDE THE MAPPING") - # print(transformation.first_axis_vals()) return transformation.first_axis_vals() if cls.name == transformation._mapped_axes()[1]: first_val = path[transformation._mapped_axes()[0]] @@ -172,6 +172,10 @@ def unmap_to_datacube(path, unmapped_path): path.pop(cls.name, None) first_val = unmapped_path.get(transformation._mapped_axes()[0], None) unmapped_path.pop(transformation._mapped_axes()[0], None) + # if the first_val was not in the unmapped_path, then it's still in path + if first_val is None: + first_val = path.get(transformation._mapped_axes()[0], None) + path.pop(transformation._mapped_axes()[0], None) if first_val is not None and second_val is not None: unmapped_idx = transformation.unmap(first_val, second_val) unmapped_path[transformation.old_axis] = unmapped_idx From 23151aab9681506410e42ae6cb86d5cb62c301f1 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 29 Aug 2023 10:00:07 +0200 Subject: [PATCH 115/332] try to make multiple transformations work --- polytope/datacube/backends/xarray.py | 8 ++++++-- polytope/datacube/datacube_axis.py | 10 ++++++++-- polytope/datacube/transformations/datacube_mappers.py | 2 ++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 1f66a996d..c19ffa040 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -104,6 +104,8 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): offset = search_ranges_offset[i] low = r[0] up = r[1] + print("IN LOOK UP DATACUBE") + print([indexes]) indexes_between = axis.find_indices_between([indexes], low, up, self) # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them @@ -117,9 +119,10 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): return idx_between def get_indices(self, path: DatacubePath, axis, lower, upper): + print(path) path = self.fit_path(path) indexes = axis.find_indexes(path, self) - + print(indexes[0:2]) search_ranges = axis.remap([lower, upper]) original_search_ranges = axis.to_intervals([lower, upper]) # Find the offsets for each interval in the requested range, which we will need later @@ -127,12 +130,13 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): for r in original_search_ranges: offset = axis.offset(r) search_ranges_offset.append(offset) - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis) # Remove duplicates even if difference of the order of the axis tolerance if offset is not None: # Note that we can only do unique if not dealing with time values idx_between = unique(idx_between) + # if axis.name == "latitude": + # print(idx_between) return idx_between def get(self, requests: IndexTree): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 97501862d..51ad0f9ba 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -190,8 +190,14 @@ def find_indices_between(index_ranges, low, up, datacube): if isinstance(transform, DatacubeMapper): transformation = transform if cls.name in transformation._mapped_axes(): - for indexes in index_ranges: - indexes_between = [i for i in indexes if low <= i <= up] + for idxs in index_ranges: + print(idxs) + indexes_between = [i for i in idxs if low <= i <= up] + print(low) + print(up) + print(indexes_between[0] in idxs) + print("HERE") + print(indexes_between[0]) indexes_between_ranges.append(indexes_between) return indexes_between_ranges diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 40b9eb68c..fdcd6d06c 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -2837,6 +2837,8 @@ def unmap(self, first_val, second_val): first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] first_idx = first_axis_vals.index(first_val) + 1 second_axis_vals = self.second_axis_vals(first_val) + print(second_val) + print([val for val in second_axis_vals if second_val - tol < val < second_val + tol]) second_val = [val for val in second_axis_vals if second_val - tol < val < second_val + tol][0] second_idx = second_axis_vals.index(second_val) octahedral_index = self.axes_idx_to_octahedral_idx(first_idx, second_idx) From 651d2d1525bad100282e20e9b5a7f72914ddb62a Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 29 Aug 2023 17:10:21 +0200 Subject: [PATCH 116/332] make transformation decorators work for several transformations for xarray datacubes --- polytope/datacube/backends/FDB_datacube.py | 76 ++++++++- polytope/datacube/backends/xarray.py | 15 +- polytope/datacube/datacube_axis.py | 145 +++++++++++++++--- .../transformations/datacube_mappers.py | 2 - .../transformations/datacube_merger.py | 2 +- tests/test_fdb_datacube.py | 3 +- tests/test_merge_octahedral_one_axis.py | 4 +- tests/test_merge_transformation.py | 7 +- 8 files changed, 210 insertions(+), 44 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index db2fd2522..25d27b2f1 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -4,6 +4,12 @@ from ...utility.combinatorics import unique, validate_axes from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis +import importlib +from ..datacube_axis import DatacubeAxis +from ..transformations.datacube_transformations import ( + DatacubeAxisTransformation, + has_transform, +) # TODO: probably need to do this more general... os.environ["DYLD_LIBRARY_PATH"] = "/Users/male/build/fdb-bundle/lib" @@ -23,16 +29,58 @@ def update_fdb_dataarray(fdb_dataarray): class FDBDatacube(Datacube): + + def _create_axes(self, name, values, transformation_type_key, transformation_options): + # first check what the final axes are for this axis name given transformations + final_axis_names = DatacubeAxisTransformation.get_final_axes(name, transformation_type_key, + transformation_options) + transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key, + transformation_options) + for blocked_axis in transformation.blocked_axes(): + self.blocked_axes.append(blocked_axis) + for axis_name in final_axis_names: + # if axis does not yet exist, create it + + # first need to change the values so that we have right type + values = transformation.change_val_type(axis_name, values) + if axis_name not in self._axes.keys(): + DatacubeAxis.create_standard(axis_name, values, self) + # add transformation tag to axis, as well as transformation options for later + setattr(self._axes[axis_name], has_transform[transformation_type_key], True) # where has_transform is a + # factory inside datacube_transformations to set the has_transform, is_cyclic etc axis properties + # add the specific transformation handled here to the relevant axes + # Modify the axis to update with the tag + decorator_module = importlib.import_module("polytope.datacube.datacube_axis") + decorator = getattr(decorator_module, transformation_type_key) + decorator(self._axes[axis_name]) + if transformation not in self._axes[axis_name].transformations: # Avoids duplicates being stored + self._axes[axis_name].transformations.append(transformation) + + def _add_all_transformation_axes(self, options, name, values): + transformation_options = options["transformation"] + for transformation_type_key in transformation_options.keys(): + self._create_axes(name, values, transformation_type_key, transformation_options) + + def _check_and_add_axes(self, options, name, values): + if "transformation" in options: + self._add_all_transformation_axes(options, name, values) + else: + if name not in self.blocked_axes: + if name not in self._axes.keys(): + DatacubeAxis.create_standard(name, values, self) + def __init__(self, config={}, axis_options={}): - # Need to get the cyclic options and grid options from somewhere self.axis_options = axis_options self.grid_mapper = None self.axis_counter = 0 self._axes = {} + treated_axes = [] + self.non_complete_axes = [] + self.complete_axes = [] self.blocked_axes = [] self.transformation = {} self.fake_axes = [] - self.complete_axes = [] + partial_request = config # Find values in the level 3 FDB datacube # Will be in the form of a dictionary? {axis_name:values_available, ...} @@ -44,9 +92,17 @@ def __init__(self, config={}, axis_options={}): for name, values in dataarray.items(): values.sort() options = axis_options.get(name, {}) - configure_datacube_axis(options, name, values, self) + self._check_and_add_axes(options, name, values) + treated_axes.append(name) self.complete_axes.append(name) + # add other options to axis which were just created above like "lat" for the mapper transformations for eg + for name in self._axes: + if name not in treated_axes: + options = axis_options.get(name, {}) + val = self._axes[name].type + self._check_and_add_axes(options, name, val) + def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() @@ -159,13 +215,23 @@ def datacube_natural_indexes(self, axis, subarray): def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - subarray_vals = self.dataarray[axis.name] - return index in subarray_vals + # subarray_vals = self.dataarray[axis.name] + path = self.fit_path(path) + indexes = axis.find_indexes(path, self) + # return index in subarray_vals + return index in indexes @property def axes(self): return self._axes + def fit_path(self, path): + # path = self.remap_path(path) + for key in path.keys(): + if key not in self.complete_axes: + path.pop(key) + return path + def validate(self, axes): return validate_axes(self.axes, axes) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index c19ffa040..ca4318ef4 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -90,12 +90,8 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): for name in self._axes: if name not in treated_axes: options = axis_options.get(name, {}) - val = dataarray[name].values[0] + val = self._axes[name].type self._check_and_add_axes(options, name, val) - # if "latitude" in self._axes: - # print(self._axes["latitude"].transformations) - # print(self._axes["longitude"].has_mapper) - # print(self._axes["longitude"].is_cyclic) def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): idx_between = [] @@ -104,8 +100,6 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): offset = search_ranges_offset[i] low = r[0] up = r[1] - print("IN LOOK UP DATACUBE") - print([indexes]) indexes_between = axis.find_indices_between([indexes], low, up, self) # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them @@ -119,10 +113,8 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): return idx_between def get_indices(self, path: DatacubePath, axis, lower, upper): - print(path) path = self.fit_path(path) indexes = axis.find_indexes(path, self) - print(indexes[0:2]) search_ranges = axis.remap([lower, upper]) original_search_ranges = axis.to_intervals([lower, upper]) # Find the offsets for each interval in the requested range, which we will need later @@ -135,20 +127,19 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): if offset is not None: # Note that we can only do unique if not dealing with time values idx_between = unique(idx_between) - # if axis.name == "latitude": - # print(idx_between) return idx_between def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() + path = self.remap_path(path) if len(path.items()) == self.axis_counter: # first, find the grid mapper transform unmapped_path = {} path_copy = deepcopy(path) for key in path_copy: axis = self._axes[key] - (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) + (path, unmapped_path) = axis.unmap_total_path_to_datacube(path, unmapped_path) path = self.fit_path(path) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmapped_path) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 51ad0f9ba..adcf9e467 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -11,9 +11,9 @@ def cyclic(cls): if cls.is_cyclic: + from .transformations.datacube_cyclic import DatacubeAxisCyclic def update_range(): - from .transformations.datacube_cyclic import DatacubeAxisCyclic for transform in cls.transformations: if isinstance(transform, DatacubeAxisCyclic): transformation = transform @@ -84,7 +84,10 @@ def _remap_val_to_axis_range(value): return_range = _remap_range_to_axis_range([value, value]) return return_range[0] + old_remap = cls.remap + def remap(range: List): + # range = old_remap(range) update_range() if cls.range[0] - cls.tol <= range[0] <= cls.range[1] + cls.tol: if cls.range[0] - cls.tol <= range[1] <= cls.range[1] + cls.tol: @@ -113,14 +116,34 @@ def remap(range: List): ranges.append([low - cls.tol, up + cls.tol]) return ranges + old_find_indexes = cls.find_indexes + def find_indexes(path, datacube): - unmapped_path = {} - (new_path, unmapped_path) = cls.unmap_to_datacube(path, unmapped_path) - subarray = datacube.dataarray.sel(new_path, method="nearest") - subarray = subarray.sel(unmapped_path) - # print(cls.name) - # print(datacube.datacube_natural_indexes(cls, subarray)) - return datacube.datacube_natural_indexes(cls, subarray) + return old_find_indexes(path, datacube) + + old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube + + def unmap_total_path_to_datacube(path, unmapped_path): + # (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) + print(path) + print(unmapped_path) + print(cls.name) + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisCyclic): + transformation = transform + if cls.name == transformation.name: + old_val = path.get(cls.name, None) + path.pop(cls.name, None) + new_val = _remap_val_to_axis_range(old_val) + path[cls.name] = new_val + (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) + return (path, unmapped_path) + + old_unmap_to_datacube = cls.unmap_to_datacube + + def unmap_to_datacube(path, unmapped_path): + (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) + return (path, unmapped_path) def offset(range): # We first unpad the range by the axis tolerance to make sure that @@ -135,6 +158,8 @@ def offset(range): cls.remap = remap cls.offset = offset cls.find_indexes = find_indexes + cls.unmap_to_datacube = unmap_to_datacube + cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube return cls @@ -154,7 +179,40 @@ def find_indexes(path, datacube): first_val = path[transformation._mapped_axes()[0]] return transformation.second_axis_vals(first_val) + old_unmap_to_datacube = cls.unmap_to_datacube + def unmap_to_datacube(path, unmapped_path): + (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) + for transform in cls.transformations: + if isinstance(transform, DatacubeMapper): + transformation = transform + if cls.name == transformation._mapped_axes()[0]: + # if we are on the first axis, then need to add the first val to unmapped_path + first_val = path.get(cls.name, None) + path.pop(cls.name, None) + if cls.name not in unmapped_path: + # if for some reason, the unmapped_path already has the first axis val, then don't update + unmapped_path[cls.name] = first_val + if cls.name == transformation._mapped_axes()[1]: + # if we are on the second axis, then the val of the first axis is stored + # inside unmapped_path so can get it from there + second_val = path.get(cls.name, None) + path.pop(cls.name, None) + first_val = unmapped_path.get(transformation._mapped_axes()[0], None) + unmapped_path.pop(transformation._mapped_axes()[0], None) + # if the first_val was not in the unmapped_path, then it's still in path + if first_val is None: + first_val = path.get(transformation._mapped_axes()[0], None) + path.pop(transformation._mapped_axes()[0], None) + if first_val is not None and second_val is not None: + unmapped_idx = transformation.unmap(first_val, second_val) + unmapped_path[transformation.old_axis] = unmapped_idx + return (path, unmapped_path) + + old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube + + def unmap_total_path_to_datacube(path, unmapped_path): + (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeMapper): transformation = transform @@ -191,24 +249,21 @@ def find_indices_between(index_ranges, low, up, datacube): transformation = transform if cls.name in transformation._mapped_axes(): for idxs in index_ranges: - print(idxs) indexes_between = [i for i in idxs if low <= i <= up] - print(low) - print(up) - print(indexes_between[0] in idxs) - print("HERE") - print(indexes_between[0]) indexes_between_ranges.append(indexes_between) return indexes_between_ranges + old_remap = cls.remap + def remap(range): - return [range] + return old_remap(range) cls.remap = remap cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.remap_to_requested = remap_to_requested cls.find_indices_between = find_indices_between + cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube return cls @@ -224,8 +279,24 @@ def find_indexes(path, datacube): transformation = transform if cls.name == transformation._first_axis: return transformation.merged_values(datacube) + + def unmap_total_path_to_datacube(path, unmapped_path): + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisMerger): + transformation = transform + if cls.name == transformation._first_axis: + old_val = path.get(cls.name, None) + (first_val, second_val) = transformation.unmerge(old_val) + path.pop(cls.name, None) + path[transformation._first_axis] = first_val + path[transformation._second_axis] = second_val + return (path, unmapped_path) + + old_unmap_to_datacube = cls.unmap_to_datacube def unmap_to_datacube(path, unmapped_path): + print("UNMAP TO DATACUBE INSIDE MERGE") + (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): transformation = transform @@ -259,6 +330,7 @@ def remap(range): cls.unmap_to_datacube = unmap_to_datacube cls.remap_to_requested = remap_to_requested cls.find_indices_between = find_indices_between + cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube return cls @@ -322,6 +394,18 @@ def find_indexes(path, datacube): subarray = subarray.sel(unmapped_path) original_vals = datacube.datacube_natural_indexes(cls, subarray) return transformation.change_val_type(cls.name, original_vals) + + def unmap_total_path_to_datacube(path, unmapped_path): + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisTypeChange): + transformation = transform + if cls.name == transformation.name: + changed_val = path.get(cls.name, None) + unchanged_val = transformation.make_str(changed_val) + if cls.name in path: + path.pop(cls.name, None) + unmapped_path[cls.name] = unchanged_val + return (path, unmapped_path) def unmap_to_datacube(path, unmapped_path): for transform in cls.transformations: @@ -357,6 +441,7 @@ def remap(range): cls.unmap_to_datacube = unmap_to_datacube cls.remap_to_requested = remap_to_requested cls.find_indices_between = find_indices_between + cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube return cls @@ -409,18 +494,38 @@ def to_intervals(self, range): def remap(self, range: List) -> Any: return [range] + def unmap_to_datacube(self, path, unmapped_path): + return (path, unmapped_path) + def find_indexes(self, path, datacube): # TODO: does this do what it should? unmapped_path = {} - (new_path, unmapped_path) = self.unmap_to_datacube(path, unmapped_path) - subarray = datacube.dataarray.sel(new_path, method="nearest") + # print("HERE") + # if self.name == "date": + # print(self.transformations) + path_copy = deepcopy(path) + for key in path_copy: + axis = datacube._axes[key] + (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) + # (new_path, unmapped_path) = self.unmap_to_datacube(path, unmapped_path) + # print(path) + # new_path_copy = deepcopy(new_path) + # for name in new_path_copy: + # if name not in datacube.complete_axes: + # print(datacube.complete_axes) + # new_path.pop(name, None) + # subarray = datacube.dataarray.sel(new_path, method="nearest") + # subarray = datacube.dataarray.sel(path, method="nearest") + # if isinstance(datacube, XArrayDatacube): + subarray = datacube.dataarray.sel(path, method="nearest") subarray = subarray.sel(unmapped_path) + # subarray = datacube.dataarray return datacube.datacube_natural_indexes(self, subarray) def offset(self, value): return 0 - def unmap_to_datacube(self, path, unmapped_path): + def unmap_total_path_to_datacube(self, path, unmapped_path): return (path, unmapped_path) def remap_to_requeest(path, unmapped_path): @@ -496,6 +601,7 @@ class IntDatacubeAxis(DatacubeAxis): tol = 1e-12 range = None transformations = [] + type = 0 def parse(self, value: Any) -> Any: return float(value) @@ -519,6 +625,7 @@ class FloatDatacubeAxis(DatacubeAxis): tol = 1e-12 range = None transformations = [] + type = 0. def parse(self, value: Any) -> Any: return float(value) @@ -539,6 +646,7 @@ class PandasTimestampDatacubeAxis(DatacubeAxis): tol = 1e-12 range = None transformations = [] + type = pd.Timestamp("2000-01-01T00:00:00") def parse(self, value: Any) -> Any: if isinstance(value, np.str_): @@ -567,6 +675,7 @@ class PandasTimedeltaDatacubeAxis(DatacubeAxis): tol = 1e-12 range = None transformations = [] + type = np.timedelta64(0, "s") def parse(self, value: Any) -> Any: if isinstance(value, np.str_): diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index fdcd6d06c..40b9eb68c 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -2837,8 +2837,6 @@ def unmap(self, first_val, second_val): first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] first_idx = first_axis_vals.index(first_val) + 1 second_axis_vals = self.second_axis_vals(first_val) - print(second_val) - print([val for val in second_axis_vals if second_val - tol < val < second_val + tol]) second_val = [val for val in second_axis_vals if second_val - tol < val < second_val + tol][0] second_idx = second_axis_vals.index(second_val) octahedral_index = self.axes_idx_to_octahedral_idx(first_idx, second_idx) diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 1e2a469de..9dcdf4b9d 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -32,7 +32,7 @@ def merged_values(self, datacube): val_to_add = val_to_add.to_numpy() val_to_add = val_to_add.astype("datetime64[s]") # merged_values.append(pd.to_datetime(first_val + linkers[0] + second_val + linkers[1])) - val_to_add = str(val_to_add) + # val_to_add = str(val_to_add) merged_values.append(val_to_add) merged_values = np.array(merged_values) return merged_values diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 568eb97cf..b57d11092 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -2,6 +2,7 @@ from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select +import pandas as pd class TestSlicingFDBDatacube: @@ -27,7 +28,7 @@ def test_2D_box(self): request = Request( Select("step", [11]), Select("levtype", ["sfc"]), - Select("date", ["20230710T120000"]), + Select("date", [pd.Timestamp("20230710T120000")]), Select("domain", ["g"]), Select("expver", ["0001"]), Select("param", ["151130"]), diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index 6d0f96aa8..9739a6c79 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -33,5 +33,5 @@ def test_merge_axis(self): Box(["latitude", "longitude"], [0, 359.8], [0.2, 361.2]), ) result = self.API.retrieve(request) - result.pprint() - assert result.leaves[0].flatten()["longitude"] == 0.0 + # result.pprint() + assert result.leaves[2].flatten()["longitude"] == 360.0 diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index 6fc33c553..2e33ba6c8 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -1,5 +1,6 @@ import numpy as np import xarray as xr +import pandas as pd from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer @@ -18,13 +19,13 @@ def setup_method(self, method): "time": ["06:00"], }, ) - options = {"date": {"transformation": {"merge": {"with": "time", "linkers": ["T", ":00"]}}}} + options = {"date": {"transformation": {"merge": {"with": "time", "linkers": [" ", ":00"]}}}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) def test_merge_axis(self): - request = Request(Select("date", ["2000-01-01T06:00:00"])) + request = Request(Select("date", [pd.Timestamp("2000-01-01T06:00:00")])) result = self.API.retrieve(request) # assert result.leaves[0].flatten()["date"] == np.datetime64("2000-01-01T06:00:00") - assert result.leaves[0].flatten()["date"] == "2000-01-01T06:00:00" + assert result.leaves[0].flatten()["date"] == pd.Timestamp("2000-01-01T06:00:00") From a30a1f619aed8c9c7ccf9bd95ad37537667d69ee Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 30 Aug 2023 09:25:00 +0200 Subject: [PATCH 117/332] clean up --- polytope/datacube/datacube_axis.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index adcf9e467..3ed386ff9 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -84,8 +84,6 @@ def _remap_val_to_axis_range(value): return_range = _remap_range_to_axis_range([value, value]) return return_range[0] - old_remap = cls.remap - def remap(range: List): # range = old_remap(range) update_range() @@ -124,10 +122,6 @@ def find_indexes(path, datacube): old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube def unmap_total_path_to_datacube(path, unmapped_path): - # (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) - print(path) - print(unmapped_path) - print(cls.name) for transform in cls.transformations: if isinstance(transform, DatacubeAxisCyclic): transformation = transform @@ -208,7 +202,7 @@ def unmap_to_datacube(path, unmapped_path): unmapped_idx = transformation.unmap(first_val, second_val) unmapped_path[transformation.old_axis] = unmapped_idx return (path, unmapped_path) - + old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube def unmap_total_path_to_datacube(path, unmapped_path): @@ -295,7 +289,6 @@ def unmap_total_path_to_datacube(path, unmapped_path): old_unmap_to_datacube = cls.unmap_to_datacube def unmap_to_datacube(path, unmapped_path): - print("UNMAP TO DATACUBE INSIDE MERGE") (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): @@ -500,26 +493,12 @@ def unmap_to_datacube(self, path, unmapped_path): def find_indexes(self, path, datacube): # TODO: does this do what it should? unmapped_path = {} - # print("HERE") - # if self.name == "date": - # print(self.transformations) path_copy = deepcopy(path) for key in path_copy: axis = datacube._axes[key] (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) - # (new_path, unmapped_path) = self.unmap_to_datacube(path, unmapped_path) - # print(path) - # new_path_copy = deepcopy(new_path) - # for name in new_path_copy: - # if name not in datacube.complete_axes: - # print(datacube.complete_axes) - # new_path.pop(name, None) - # subarray = datacube.dataarray.sel(new_path, method="nearest") - # subarray = datacube.dataarray.sel(path, method="nearest") - # if isinstance(datacube, XArrayDatacube): subarray = datacube.dataarray.sel(path, method="nearest") subarray = subarray.sel(unmapped_path) - # subarray = datacube.dataarray return datacube.datacube_natural_indexes(self, subarray) def offset(self, value): From fd5b1a9a8c68bf068e5eb6326950e6b4c8157ee2 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 30 Aug 2023 09:52:19 +0200 Subject: [PATCH 118/332] make new decorator transformations work with fdb backend --- polytope/datacube/backends/FDB_datacube.py | 60 ++++++---------------- polytope/datacube/backends/xarray.py | 22 +++----- polytope/datacube/datacube_axis.py | 28 ++++++---- 3 files changed, 43 insertions(+), 67 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 25d27b2f1..123ee0e0e 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -39,6 +39,7 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) for axis_name in final_axis_names: + self.complete_axes.append(axis_name) # if axis does not yet exist, create it # first need to change the values so that we have right type @@ -130,32 +131,24 @@ def remap_path(self, path: DatacubePath): value = path[key] path[key] = self._axes[key].remap([value, value])[0][0] return path - - def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, first_val): + + def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): idx_between = [] for i in range(len(search_ranges)): r = search_ranges[i] offset = search_ranges_offset[i] low = r[0] up = r[1] - if axis.name in self.transformation.keys(): - axis_transforms = self.transformation[axis.name] - temp_indexes = deepcopy(indexes) - for transform in axis_transforms: - (offset, temp_indexes) = transform._find_transformed_indices_between( - axis, self, temp_indexes, low, up, first_val, offset - ) - indexes_between = temp_indexes - else: - indexes_between = self._find_indexes_between(axis, indexes, low, up) + indexes_between = axis.find_indices_between([indexes], low, up, self) # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them for j in range(len(indexes_between)): - if offset is None: - indexes_between[j] = indexes_between[j] - else: - indexes_between[j] = round(indexes_between[j] + offset, int(-math.log10(axis.tol))) - idx_between.append(indexes_between[j]) + for k in range(len(indexes_between[j])): + if offset is None: + indexes_between[j][k] = indexes_between[j][k] + else: + indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) + idx_between.append(indexes_between[j][k]) return idx_between def fit_path_to_original_datacube(self, path): @@ -173,27 +166,10 @@ def fit_path_to_original_datacube(self, path): if temp_first_val: first_val = temp_first_val return (path, first_val, considered_axes, unmap_path, changed_type_path) - + def get_indices(self, path: DatacubePath, axis, lower, upper): - # NEW VERSION OF THIS METHOD - (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube(path) - - subarray = self.dataarray - # Get the indexes of the axis we want to query - # XArray does not support branching, so no need to use label, we just take the next axis - if axis.name in self.transformation.keys(): - axis_transforms = self.transformation[axis.name] - # This bool will help us decide for which axes we need to calculate the indexes again or not - # in case there are multiple relevant transformations for an axis - already_has_indexes = False - for transform in axis_transforms: - # TODO: here, instead of creating the indices, would be better to create the standard datacube axes and - # then succesively map them to what they should be - indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) - already_has_indexes = True - else: - indexes = self.datacube_natural_indexes(axis, subarray) - # Here, we do a cyclic remapping so we look up on the right existing values in the cyclic range on the datacube + path = self.fit_path(path) + indexes = axis.find_indexes(path, self) search_ranges = axis.remap([lower, upper]) original_search_ranges = axis.to_intervals([lower, upper]) # Find the offsets for each interval in the requested range, which we will need later @@ -201,8 +177,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): for r in original_search_ranges: offset = axis.offset(r) search_ranges_offset.append(offset) - # Look up the values in the datacube for each cyclic interval range - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, first_val) + idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis) # Remove duplicates even if difference of the order of the axis tolerance if offset is not None: # Note that we can only do unique if not dealing with time values @@ -213,6 +188,9 @@ def datacube_natural_indexes(self, axis, subarray): indexes = subarray[axis.name] return indexes + def select(self, path, unmapped_path): + return self.dataarray + def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube # subarray_vals = self.dataarray[axis.name] @@ -226,7 +204,6 @@ def axes(self): return self._axes def fit_path(self, path): - # path = self.remap_path(path) for key in path.keys(): if key not in self.complete_axes: path.pop(key) @@ -239,6 +216,3 @@ def ax_vals(self, name): for _name, values in self.dataarray.items(): if _name == name: return values - - def _find_indexes_between(self, axis, indexes, low, up): - return [i for i in indexes if low <= i <= up] diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index ca4318ef4..6a4dfb9fc 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -25,6 +25,7 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) for axis_name in final_axis_names: + self.complete_axes.append(axis_name) # if axis does not yet exist, create it # first need to change the values so that we have right type @@ -170,29 +171,20 @@ def datacube_natural_indexes(self, axis, subarray): return indexes def fit_path(self, path): - # path = self.remap_path(path) for key in path.keys(): - if key in self.non_complete_axes: + if key not in self.complete_axes: path.pop(key) return path + def select(self, path, unmapped_path): + subarray = self.dataarray.sel(path, method="nearest") + subarray = subarray.sel(unmapped_path) + return subarray + def has_index(self, path: DatacubePath, axis, index): # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - # path = self.fit_path_to_datacube(axis.name, path)[0] path = self.fit_path(path) indexes = axis.find_indexes(path, self) - - # Open a view on the subset identified by the path - # subarray = self.dataarray.sel(path, method="nearest") - # if axis.name in self.transformation.keys(): - # axis_transforms = self.transformation[axis.name] - # already_has_indexes = False - # for transform in axis_transforms: - # indexes = transform._find_transformed_axis_indices(self, axis, subarray, already_has_indexes) - # already_has_indexes = True - # # return index in subarray_vals - # else: - # indexes = self.datacube_natural_indexes(axis, subarray) return index in indexes @property diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 3ed386ff9..e0a24ad96 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -1,4 +1,3 @@ -# import sys from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy from typing import Any, List @@ -273,8 +272,11 @@ def find_indexes(path, datacube): transformation = transform if cls.name == transformation._first_axis: return transformation.merged_values(datacube) - + + old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube + def unmap_total_path_to_datacube(path, unmapped_path): + (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): transformation = transform @@ -376,19 +378,26 @@ def type_change(cls): if cls.type_change: + old_find_indexes = cls.find_indexes + def find_indexes(path, datacube): for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): transformation = transform if cls.name == transformation.name: - unmapped_path = {} - (new_path, unmapped_path) = cls.unmap_to_datacube(path, unmapped_path) - subarray = datacube.dataarray.sel(new_path, method="nearest") - subarray = subarray.sel(unmapped_path) - original_vals = datacube.datacube_natural_indexes(cls, subarray) + + # unmapped_path = {} + # (new_path, unmapped_path) = cls.unmap_to_datacube(path, unmapped_path) + # subarray = datacube.dataarray.sel(new_path, method="nearest") + # subarray = subarray.sel(unmapped_path) + # original_vals = datacube.datacube_natural_indexes(cls, subarray) + original_vals = old_find_indexes(path, datacube) return transformation.change_val_type(cls.name, original_vals) + old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube + def unmap_total_path_to_datacube(path, unmapped_path): + (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): transformation = transform @@ -497,8 +506,9 @@ def find_indexes(self, path, datacube): for key in path_copy: axis = datacube._axes[key] (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) - subarray = datacube.dataarray.sel(path, method="nearest") - subarray = subarray.sel(unmapped_path) + subarray = datacube.select(path, unmapped_path) + # subarray = datacube.dataarray.sel(path, method="nearest") + # subarray = subarray.sel(unmapped_path) return datacube.datacube_natural_indexes(self, subarray) def offset(self, value): From 96d9a8ddb849176f12f4a84ffdf117c9f0effc95 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 30 Aug 2023 11:50:12 +0200 Subject: [PATCH 119/332] clean up --- polytope/datacube/backends/FDB_datacube.py | 150 ++---------------- polytope/datacube/backends/datacube.py | 145 +++++++++++++---- polytope/datacube/backends/xarray.py | 116 +------------- polytope/datacube/datacube_axis.py | 50 +----- .../transformations/datacube_cyclic.py | 38 ----- .../transformations/datacube_mappers.py | 58 ------- .../transformations/datacube_merger.py | 49 ------ .../transformations/datacube_reverse.py | 32 ---- .../datacube_transformations.py | 28 ---- .../transformations/datacube_type_change.py | 46 ------ .../transformation_pseudocode.py | 81 ---------- tests/test_fdb_datacube.py | 3 +- tests/test_merge_transformation.py | 2 +- 13 files changed, 128 insertions(+), 670 deletions(-) delete mode 100644 polytope/datacube/transformations/transformation_pseudocode.py diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 123ee0e0e..46e2c673c 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -1,23 +1,13 @@ -import math import os from copy import deepcopy -from ...utility.combinatorics import unique, validate_axes -from .datacube import Datacube, DatacubePath, IndexTree, configure_datacube_axis -import importlib -from ..datacube_axis import DatacubeAxis -from ..transformations.datacube_transformations import ( - DatacubeAxisTransformation, - has_transform, -) +from .datacube import Datacube, IndexTree # TODO: probably need to do this more general... os.environ["DYLD_LIBRARY_PATH"] = "/Users/male/build/fdb-bundle/lib" os.environ["FDB_HOME"] = "/Users/male/git/fdb-home" import pyfdb # noqa: E402 -# TODO: currently, because the date and time are strings, the data will be treated as an unsliceable axis... - def glue(path, unmap_path): return {"t": 0} @@ -30,46 +20,6 @@ def update_fdb_dataarray(fdb_dataarray): class FDBDatacube(Datacube): - def _create_axes(self, name, values, transformation_type_key, transformation_options): - # first check what the final axes are for this axis name given transformations - final_axis_names = DatacubeAxisTransformation.get_final_axes(name, transformation_type_key, - transformation_options) - transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key, - transformation_options) - for blocked_axis in transformation.blocked_axes(): - self.blocked_axes.append(blocked_axis) - for axis_name in final_axis_names: - self.complete_axes.append(axis_name) - # if axis does not yet exist, create it - - # first need to change the values so that we have right type - values = transformation.change_val_type(axis_name, values) - if axis_name not in self._axes.keys(): - DatacubeAxis.create_standard(axis_name, values, self) - # add transformation tag to axis, as well as transformation options for later - setattr(self._axes[axis_name], has_transform[transformation_type_key], True) # where has_transform is a - # factory inside datacube_transformations to set the has_transform, is_cyclic etc axis properties - # add the specific transformation handled here to the relevant axes - # Modify the axis to update with the tag - decorator_module = importlib.import_module("polytope.datacube.datacube_axis") - decorator = getattr(decorator_module, transformation_type_key) - decorator(self._axes[axis_name]) - if transformation not in self._axes[axis_name].transformations: # Avoids duplicates being stored - self._axes[axis_name].transformations.append(transformation) - - def _add_all_transformation_axes(self, options, name, values): - transformation_options = options["transformation"] - for transformation_type_key in transformation_options.keys(): - self._create_axes(name, values, transformation_type_key, transformation_options) - - def _check_and_add_axes(self, options, name, values): - if "transformation" in options: - self._add_all_transformation_axes(options, name, values) - else: - if name not in self.blocked_axes: - if name not in self._axes.keys(): - DatacubeAxis.create_standard(name, values, self) - def __init__(self, config={}, axis_options={}): self.axis_options = axis_options self.grid_mapper = None @@ -107,83 +57,22 @@ def __init__(self, config={}, axis_options={}): def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() + path = self.remap_path(path) if len(path.items()) == self.axis_counter: # first, find the grid mapper transform - unmap_path = {} - considered_axes = [] - (path, first_val, considered_axes, unmap_path, changed_type_path) = self.fit_path_to_original_datacube( - path - ) - unmap_path.update(changed_type_path) - # Here, need to give the FDB the path and the unmap_path to select data - subxarray = glue(path, unmap_path) + unmapped_path = {} + path_copy = deepcopy(path) + for key in path_copy: + axis = self._axes[key] + (path, unmapped_path) = axis.unmap_total_path_to_datacube(path, unmapped_path) + path = self.fit_path(path) + subxarray = glue(path, unmapped_path) key = list(subxarray.keys())[0] value = subxarray[key] r.result = (key, value) else: r.remove_branch() - def get_mapper(self, axis): - return self._axes[axis] - - def remap_path(self, path: DatacubePath): - for key in path: - value = path[key] - path[key] = self._axes[key].remap([value, value])[0][0] - return path - - def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): - idx_between = [] - for i in range(len(search_ranges)): - r = search_ranges[i] - offset = search_ranges_offset[i] - low = r[0] - up = r[1] - indexes_between = axis.find_indices_between([indexes], low, up, self) - # Now the indexes_between are values on the cyclic range so need to remap them to their original - # values before returning them - for j in range(len(indexes_between)): - for k in range(len(indexes_between[j])): - if offset is None: - indexes_between[j][k] = indexes_between[j][k] - else: - indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) - idx_between.append(indexes_between[j][k]) - return idx_between - - def fit_path_to_original_datacube(self, path): - path = self.remap_path(path) - first_val = None - unmap_path = {} - considered_axes = [] - changed_type_path = {} - for axis_name in self.transformation.keys(): - axis_transforms = self.transformation[axis_name] - for transform in axis_transforms: - (path, temp_first_val, considered_axes, unmap_path, changed_type_path) = transform._adjust_path( - path, considered_axes, unmap_path, changed_type_path - ) - if temp_first_val: - first_val = temp_first_val - return (path, first_val, considered_axes, unmap_path, changed_type_path) - - def get_indices(self, path: DatacubePath, axis, lower, upper): - path = self.fit_path(path) - indexes = axis.find_indexes(path, self) - search_ranges = axis.remap([lower, upper]) - original_search_ranges = axis.to_intervals([lower, upper]) - # Find the offsets for each interval in the requested range, which we will need later - search_ranges_offset = [] - for r in original_search_ranges: - offset = axis.offset(r) - search_ranges_offset.append(offset) - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis) - # Remove duplicates even if difference of the order of the axis tolerance - if offset is not None: - # Note that we can only do unique if not dealing with time values - idx_between = unique(idx_between) - return idx_between - def datacube_natural_indexes(self, axis, subarray): indexes = subarray[axis.name] return indexes @@ -191,27 +80,6 @@ def datacube_natural_indexes(self, axis, subarray): def select(self, path, unmapped_path): return self.dataarray - def has_index(self, path: DatacubePath, axis, index): - # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - # subarray_vals = self.dataarray[axis.name] - path = self.fit_path(path) - indexes = axis.find_indexes(path, self) - # return index in subarray_vals - return index in indexes - - @property - def axes(self): - return self._axes - - def fit_path(self, path): - for key in path.keys(): - if key not in self.complete_axes: - path.pop(key) - return path - - def validate(self, axes): - return validate_axes(self.axes, axes) - def ax_vals(self, name): for _name, values in self.dataarray.items(): if _name == name: diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 2de302a0e..d26f3f952 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -1,10 +1,17 @@ +import importlib +import math from abc import ABC, abstractmethod -from typing import Any, List +from typing import Any import xarray as xr +from ...utility.combinatorics import unique, validate_axes from ..datacube_axis import DatacubeAxis from ..index_tree import DatacubePath, IndexTree +from ..transformations.datacube_transformations import ( + DatacubeAxisTransformation, + has_transform, +) class Datacube(ABC): @@ -12,33 +19,119 @@ class Datacube(ABC): def get(self, requests: IndexTree) -> Any: """Return data given a set of request trees""" - @abstractmethod - def get_mapper(self, axis: str) -> DatacubeAxis: - """ - Get the type mapper for a subaxis of the datacube given by label - """ + @property + def axes(self): + return self._axes - @abstractmethod - def get_indices(self, path: DatacubePath, axis: DatacubeAxis, lower: Any, upper: Any) -> List[Any]: + def validate(self, axes): + """returns true if the input axes can be resolved against the datacube axes""" + return validate_axes(list(self.axes.keys()), axes) + + def _create_axes(self, name, values, transformation_type_key, transformation_options): + # first check what the final axes are for this axis name given transformations + final_axis_names = DatacubeAxisTransformation.get_final_axes(name, transformation_type_key, + transformation_options) + transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key, + transformation_options) + for blocked_axis in transformation.blocked_axes(): + self.blocked_axes.append(blocked_axis) + for axis_name in final_axis_names: + self.complete_axes.append(axis_name) + # if axis does not yet exist, create it + + # first need to change the values so that we have right type + values = transformation.change_val_type(axis_name, values) + if axis_name not in self._axes.keys(): + DatacubeAxis.create_standard(axis_name, values, self) + # add transformation tag to axis, as well as transformation options for later + setattr(self._axes[axis_name], has_transform[transformation_type_key], True) # where has_transform is a + # factory inside datacube_transformations to set the has_transform, is_cyclic etc axis properties + # add the specific transformation handled here to the relevant axes + # Modify the axis to update with the tag + decorator_module = importlib.import_module("polytope.datacube.datacube_axis") + decorator = getattr(decorator_module, transformation_type_key) + decorator(self._axes[axis_name]) + if transformation not in self._axes[axis_name].transformations: # Avoids duplicates being stored + self._axes[axis_name].transformations.append(transformation) + + def _add_all_transformation_axes(self, options, name, values): + transformation_options = options["transformation"] + for transformation_type_key in transformation_options.keys(): + self._create_axes(name, values, transformation_type_key, transformation_options) + + def _check_and_add_axes(self, options, name, values): + if "transformation" in options: + self._add_all_transformation_axes(options, name, values) + else: + if name not in self.blocked_axes: + if name not in self._axes.keys(): + DatacubeAxis.create_standard(name, values, self) + + def has_index(self, path: DatacubePath, axis, index): + "Given a path to a subset of the datacube, checks if the index exists on that sub-datacube axis" + path = self.fit_path(path) + indexes = axis.find_indexes(path, self) + return index in indexes + + def fit_path(self, path): + for key in path.keys(): + if key not in self.complete_axes: + path.pop(key) + return path + + def get_indices(self, path: DatacubePath, axis, lower, upper): """ Given a path to a subset of the datacube, return the discrete indexes which exist between two non-discrete values (lower, upper) for a particular axis (given by label) If lower and upper are equal, returns the index which exactly matches that value (if it exists) e.g. returns integer discrete points between two floats """ + path = self.fit_path(path) + indexes = axis.find_indexes(path, self) + search_ranges = axis.remap([lower, upper]) + original_search_ranges = axis.to_intervals([lower, upper]) + # Find the offsets for each interval in the requested range, which we will need later + search_ranges_offset = [] + for r in original_search_ranges: + offset = axis.offset(r) + search_ranges_offset.append(offset) + idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis) + # Remove duplicates even if difference of the order of the axis tolerance + if offset is not None: + # Note that we can only do unique if not dealing with time values + idx_between = unique(idx_between) + return idx_between - @abstractmethod - def has_index(self, path: DatacubePath, axis: DatacubeAxis, index: Any) -> bool: - "Given a path to a subset of the datacube, checks if the index exists on that sub-datacube axis" + def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): + idx_between = [] + for i in range(len(search_ranges)): + r = search_ranges[i] + offset = search_ranges_offset[i] + low = r[0] + up = r[1] + indexes_between = axis.find_indices_between([indexes], low, up, self) + # Now the indexes_between are values on the cyclic range so need to remap them to their original + # values before returning them + for j in range(len(indexes_between)): + for k in range(len(indexes_between[j])): + if offset is None: + indexes_between[j][k] = indexes_between[j][k] + else: + indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) + idx_between.append(indexes_between[j][k]) + return idx_between - @property - @abstractmethod - def axes(self) -> dict[str, DatacubeAxis]: - pass + def get_mapper(self, axis): + """ + Get the type mapper for a subaxis of the datacube given by label + """ + return self._axes[axis] - @abstractmethod - def validate(self, axes: List[str]) -> bool: - """returns true if the input axes can be resolved against the datacube axes""" + def remap_path(self, path: DatacubePath): + for key in path: + value = path[key] + path[key] = self._axes[key].remap([value, value])[0][0] + return path @staticmethod def create(datacube, axis_options: dict): @@ -49,19 +142,3 @@ def create(datacube, axis_options: dict): return xadatacube else: return datacube - - # TODO: need to add transformation properties like the datacube.transformations dico - - -def configure_datacube_axis(options, name, values, datacube): - # First need to check we are not initialising an axis which should not be initialised - if name not in datacube.blocked_axes: - # Now look at the options passed in - if options == {}: - DatacubeAxis.create_axis(name, values, datacube) - else: - from ..transformations.datacube_transformations import ( - DatacubeAxisTransformation, - ) - - DatacubeAxisTransformation.create_transformation(options, name, values, datacube) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 6a4dfb9fc..b7c92a8b7 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -1,61 +1,13 @@ -import importlib -import math from copy import deepcopy import xarray as xr -from ...utility.combinatorics import unique, validate_axes -from ..datacube_axis import DatacubeAxis -from ..transformations.datacube_transformations import ( - DatacubeAxisTransformation, - has_transform, -) -from .datacube import Datacube, DatacubePath, IndexTree +from .datacube import Datacube, IndexTree class XArrayDatacube(Datacube): """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0).""" - def _create_axes(self, name, values, transformation_type_key, transformation_options): - # first check what the final axes are for this axis name given transformations - final_axis_names = DatacubeAxisTransformation.get_final_axes(name, transformation_type_key, - transformation_options) - transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key, - transformation_options) - for blocked_axis in transformation.blocked_axes(): - self.blocked_axes.append(blocked_axis) - for axis_name in final_axis_names: - self.complete_axes.append(axis_name) - # if axis does not yet exist, create it - - # first need to change the values so that we have right type - values = transformation.change_val_type(axis_name, values) - if axis_name not in self._axes.keys(): - DatacubeAxis.create_standard(axis_name, values, self) - # add transformation tag to axis, as well as transformation options for later - setattr(self._axes[axis_name], has_transform[transformation_type_key], True) # where has_transform is a - # factory inside datacube_transformations to set the has_transform, is_cyclic etc axis properties - # add the specific transformation handled here to the relevant axes - # Modify the axis to update with the tag - decorator_module = importlib.import_module("polytope.datacube.datacube_axis") - decorator = getattr(decorator_module, transformation_type_key) - decorator(self._axes[axis_name]) - if transformation not in self._axes[axis_name].transformations: # Avoids duplicates being stored - self._axes[axis_name].transformations.append(transformation) - - def _add_all_transformation_axes(self, options, name, values): - transformation_options = options["transformation"] - for transformation_type_key in transformation_options.keys(): - self._create_axes(name, values, transformation_type_key, transformation_options) - - def _check_and_add_axes(self, options, name, values): - if "transformation" in options: - self._add_all_transformation_axes(options, name, values) - else: - if name not in self.blocked_axes: - if name not in self._axes.keys(): - DatacubeAxis.create_standard(name, values, self) - def __init__(self, dataarray: xr.DataArray, axis_options={}): self.axis_options = axis_options self.grid_mapper = None @@ -86,7 +38,6 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): val = dataarray[name].values[0] self._check_and_add_axes(options, name, val) treated_axes.append(name) - # add other options to axis which were just created above like "lat" for the mapper transformations for eg for name in self._axes: if name not in treated_axes: @@ -94,42 +45,6 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): val = self._axes[name].type self._check_and_add_axes(options, name, val) - def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): - idx_between = [] - for i in range(len(search_ranges)): - r = search_ranges[i] - offset = search_ranges_offset[i] - low = r[0] - up = r[1] - indexes_between = axis.find_indices_between([indexes], low, up, self) - # Now the indexes_between are values on the cyclic range so need to remap them to their original - # values before returning them - for j in range(len(indexes_between)): - for k in range(len(indexes_between[j])): - if offset is None: - indexes_between[j][k] = indexes_between[j][k] - else: - indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) - idx_between.append(indexes_between[j][k]) - return idx_between - - def get_indices(self, path: DatacubePath, axis, lower, upper): - path = self.fit_path(path) - indexes = axis.find_indexes(path, self) - search_ranges = axis.remap([lower, upper]) - original_search_ranges = axis.to_intervals([lower, upper]) - # Find the offsets for each interval in the requested range, which we will need later - search_ranges_offset = [] - for r in original_search_ranges: - offset = axis.offset(r) - search_ranges_offset.append(offset) - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis) - # Remove duplicates even if difference of the order of the axis tolerance - if offset is not None: - # Note that we can only do unique if not dealing with time values - idx_between = unique(idx_between) - return idx_between - def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() @@ -150,16 +65,6 @@ def get(self, requests: IndexTree): else: r.remove_branch() - def get_mapper(self, axis): - return self._axes[axis] - - # TODO: should this be in DatacubePath? - def remap_path(self, path: DatacubePath): - for key in path: - value = path[key] - path[key] = self._axes[key].remap([value, value])[0][0] - return path - def datacube_natural_indexes(self, axis, subarray): if axis.name in self.complete_axes: indexes = next(iter(subarray.xindexes.values())).to_pandas_index() @@ -170,30 +75,11 @@ def datacube_natural_indexes(self, axis, subarray): indexes = subarray[axis.name].values return indexes - def fit_path(self, path): - for key in path.keys(): - if key not in self.complete_axes: - path.pop(key) - return path - def select(self, path, unmapped_path): subarray = self.dataarray.sel(path, method="nearest") subarray = subarray.sel(unmapped_path) return subarray - def has_index(self, path: DatacubePath, axis, index): - # when we want to obtain the value of an unsliceable axis, need to check the values does exist in the datacube - path = self.fit_path(path) - indexes = axis.find_indexes(path, self) - return index in indexes - - @property - def axes(self): - return self._axes - - def validate(self, axes): - return validate_axes(list(self.axes.keys()), axes) - def ax_vals(self, name): treated_axes = [] for _name, values in self.dataarray.coords.variables.items(): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index e0a24ad96..80fec89ae 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -5,8 +5,6 @@ import numpy as np import pandas as pd -# TODO: maybe create dico of which axes can be cyclic too - def cyclic(cls): if cls.is_cyclic: @@ -84,7 +82,6 @@ def _remap_val_to_axis_range(value): return return_range[0] def remap(range: List): - # range = old_remap(range) update_range() if cls.range[0] - cls.tol <= range[0] <= cls.range[1] + cls.tol: if cls.range[0] - cls.tol <= range[1] <= cls.range[1] + cls.tol: @@ -131,7 +128,7 @@ def unmap_total_path_to_datacube(path, unmapped_path): path[cls.name] = new_val (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) return (path, unmapped_path) - + old_unmap_to_datacube = cls.unmap_to_datacube def unmap_to_datacube(path, unmapped_path): @@ -272,7 +269,7 @@ def find_indexes(path, datacube): transformation = transform if cls.name == transformation._first_axis: return transformation.merged_values(datacube) - + old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube def unmap_total_path_to_datacube(path, unmapped_path): @@ -378,22 +375,16 @@ def type_change(cls): if cls.type_change: - old_find_indexes = cls.find_indexes + old_find_indexes = cls.find_indexes def find_indexes(path, datacube): for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): transformation = transform if cls.name == transformation.name: - - # unmapped_path = {} - # (new_path, unmapped_path) = cls.unmap_to_datacube(path, unmapped_path) - # subarray = datacube.dataarray.sel(new_path, method="nearest") - # subarray = subarray.sel(unmapped_path) - # original_vals = datacube.datacube_natural_indexes(cls, subarray) original_vals = old_find_indexes(path, datacube) return transformation.change_val_type(cls.name, original_vals) - + old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube def unmap_total_path_to_datacube(path, unmapped_path): @@ -500,15 +491,12 @@ def unmap_to_datacube(self, path, unmapped_path): return (path, unmapped_path) def find_indexes(self, path, datacube): - # TODO: does this do what it should? unmapped_path = {} path_copy = deepcopy(path) for key in path_copy: axis = datacube._axes[key] (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) subarray = datacube.select(path, unmapped_path) - # subarray = datacube.dataarray.sel(path, method="nearest") - # subarray = subarray.sel(unmapped_path) return datacube.datacube_natural_indexes(self, subarray) def offset(self, value): @@ -521,7 +509,6 @@ def remap_to_requeest(path, unmapped_path): return (path, unmapped_path) def find_indices_between(self, index_ranges, low, up, datacube): - # TODO: do we need this ? indexes_between_ranges = [] for indexes in index_ranges: if self.name in datacube.complete_axes: @@ -537,35 +524,6 @@ def find_indices_between(self, index_ranges, low, up, datacube): indexes_between_ranges.append(indexes_between) return indexes_between_ranges - @staticmethod - def create_axis(name, values, datacube): - cyclic_transform = None - # First check if axis has any cyclicity transformation - if name in datacube.transformation.keys(): - axis_transforms = datacube.transformation[name] - for transform in axis_transforms: - from .transformations.datacube_cyclic import DatacubeAxisCyclic - - if isinstance(transform, DatacubeAxisCyclic): - cyclic_transform = transform - - if cyclic_transform is not None: - # the axis has a cyclic transformation - DatacubeAxis.create_cyclic(transform, name, values, datacube) - else: - DatacubeAxis.create_standard(name, values, datacube) - - @staticmethod - def create_cyclic(cyclic_transform, name, values, datacube): - values = np.array(values) - DatacubeAxis.check_axis_type(name, values) - datacube._axes[name] = deepcopy(_type_to_axis_lookup[values.dtype.type]) - datacube._axes[name].is_cyclic = True - datacube._axes[name].update_axis() - datacube._axes[name].name = name - datacube._axes[name].range = cyclic_transform.range - datacube.axis_counter += 1 - @staticmethod def create_standard(name, values, datacube): values = np.array(values) diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index 5859c0f1f..a48dedf3d 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -1,6 +1,3 @@ -from copy import deepcopy - -from ..backends.datacube import configure_datacube_axis from .datacube_transformations import DatacubeAxisTransformation @@ -23,38 +20,3 @@ def change_val_type(self, axis_name, values): def blocked_axes(self): return [] - - def apply_transformation(self, name, datacube, values): - # NOTE: we will handle all the cyclicity mapping here instead of in the DatacubeAxis - # then we can generate just create_standard in the configure_axis at the end - # Also, in the datacube implementations, we then have to deal with the transformation dico - # and see if there is a cyclic axis, where we then need to generate the relevant offsets etc - # from the transformation object - - # OR, we generate a transformation, which we call in a new function create_axis. - # In create_axis, if we have a cyclic transformation, we generate a cyclic axis, else, a standard one - - axis_options = deepcopy(datacube.axis_options[name]["transformation"]) - axis_options.pop("cyclic") - # Update the nested dictionary with the modified axis option for our axis - new_datacube_axis_options = deepcopy(datacube.axis_options) - # if we have no transformations left, then empty the transformation dico - if axis_options == {}: - new_datacube_axis_options[name] = {} - else: - new_datacube_axis_options[name]["transformation"] = axis_options - configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) - - def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): - indexes_between = datacube._find_indexes_between(axis, indexes, low, up) - return (offset, indexes_between) - - def _adjust_path(self, path, considered_axes=[], unmap_path={}, changed_type_path={}): - return (path, None, considered_axes, unmap_path, changed_type_path) - - def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): - if not already_has_indexes: - indexes = datacube.datacube_natural_indexes(axis, subarray) - return indexes - else: - pass diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 40b9eb68c..60ea5cdb6 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -2,9 +2,6 @@ from copy import deepcopy from importlib import import_module -import numpy as np - -# from ..backends.datacube import configure_datacube_axis from .datacube_transformations import DatacubeAxisTransformation @@ -28,24 +25,6 @@ def generate_final_transformation(self): def blocked_axes(self): return [] - # def apply_transformation(self, name, datacube, values): - # # Create mapped axes here - # transformation = self.generate_final_transformation() - # for i in range(len(transformation._mapped_axes)): - # axis_name = transformation._mapped_axes[i] - # # axis_name = name - # new_axis_options = datacube.axis_options.get(axis_name, {}) - # if i == 0: - # from ..backends.datacube import configure_datacube_axis - # values = np.array(transformation.first_axis_vals()) - # configure_datacube_axis(new_axis_options, axis_name, values, datacube) - # if i == 1: - # # the values[0] will be a value on the first axis - # from ..backends.datacube import configure_datacube_axis - # values = np.array(transformation.second_axis_vals(values[0])) - # configure_datacube_axis(new_axis_options, axis_name, values, datacube) - # datacube.fake_axes.append(axis_name) - def transformation_axes_final(self): final_transformation = self.generate_final_transformation() final_axes = final_transformation._mapped_axes @@ -89,43 +68,6 @@ def unmap(self, first_val, second_val): final_transformation = self.generate_final_transformation() return final_transformation.unmap(first_val, second_val) - def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): - first_axis = self._mapped_axes()[0] - second_axis = self._mapped_axes()[1] - if axis.name == first_axis: - indexes_between = self.map_first_axis(low, up) - elif axis.name == second_axis: - indexes_between = self.map_second_axis(first_val, low, up) - else: - indexes_between = datacube._find_indexes_between(axis, indexes, low, up) - return (None, indexes_between) - - def _adjust_path(self, path, considered_axes=[], unmap_path={}, changed_type_path={}): - first_axis = self._mapped_axes()[0] - first_val = path.get(first_axis, None) - second_axis = self._mapped_axes()[1] - second_val = path.get(second_axis, None) - path.pop(first_axis, None) - path.pop(second_axis, None) - considered_axes.append(first_axis) - considered_axes.append(second_axis) - # need to remap the lat, lon in path to dataarray index - if first_val is not None and second_val is not None: - unmapped_idx = self.unmap(first_val, second_val) - unmap_path[self.old_axis] = unmapped_idx - return (path, first_val, considered_axes, unmap_path, changed_type_path) - - def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): - first_axis = self._mapped_axes()[0] - second_axis = self._mapped_axes()[1] - if axis.name == first_axis: - indexes = [] - elif axis.name == second_axis: - indexes = [] - else: - indexes = datacube.datacube_natural_indexes(axis, subarray) - return indexes - class OctahedralGridMapper(DatacubeMapper): def __init__(self, base_axis, mapped_axes, resolution): diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 9dcdf4b9d..eb404498d 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -1,9 +1,6 @@ -from copy import deepcopy - import numpy as np import pandas as pd -# from ..backends.datacube import configure_datacube_axis from .datacube_transformations import DatacubeAxisTransformation @@ -27,46 +24,22 @@ def merged_values(self, datacube): for first_val in first_ax_vals: for second_val in second_ax_vals: # TODO: check that the first and second val are strings - # merged_values.append(np.datetime64(first_val + linkers[0] + second_val + linkers[1])) val_to_add = pd.to_datetime(first_val + linkers[0] + second_val + linkers[1]) val_to_add = val_to_add.to_numpy() val_to_add = val_to_add.astype("datetime64[s]") - # merged_values.append(pd.to_datetime(first_val + linkers[0] + second_val + linkers[1])) - # val_to_add = str(val_to_add) merged_values.append(val_to_add) merged_values = np.array(merged_values) return merged_values - # def apply_transformation(self, name, datacube, values): - # merged_values = self.merged_values(datacube) - # # Remove the merge option from the axis options since we have already handled it - # # so do not want to handle it again - # axis_options = deepcopy(datacube.axis_options[name]["transformation"]) - # axis_options.pop("merge") - # # Update the nested dictionary with the modified axis option for our axis - # new_datacube_axis_options = deepcopy(datacube.axis_options) - # if axis_options == {}: - # new_datacube_axis_options[name] = {} - # else: - # new_datacube_axis_options[name]["transformation"] = axis_options - # # Reconfigure the axis with the rest of its configurations - # configure_datacube_axis(new_datacube_axis_options[name], name, merged_values, datacube) - # self.finish_transformation(datacube, merged_values) - def transformation_axes_final(self): return [self._first_axis] - def finish_transformation(self, datacube, values): - # Need to "delete" the second axis we do not use anymore - datacube.blocked_axes.append(self._second_axis) - def generate_final_transformation(self): return self def unmerge(self, merged_val): merged_val = str(merged_val) first_idx = merged_val.find(self._linkers[0]) - # second_idx = merged_val.find(self._linkers[1]) first_val = merged_val[:first_idx] first_linker_size = len(self._linkers[0]) second_linked_size = len(self._linkers[1]) @@ -75,25 +48,3 @@ def unmerge(self, merged_val): def change_val_type(self, axis_name, values): return values - - def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): - indexes_between = datacube._find_indexes_between(axis, indexes, low, up) - return (offset, indexes_between) - - def _adjust_path(self, path, considered_axes=[], unmap_path={}, changed_type_path={}): - merged_ax = self._first_axis - merged_val = path.get(merged_ax, None) - removed_ax = self._second_axis - path.pop(removed_ax, None) - path.pop(merged_ax, None) - if merged_val is not None: - unmapped_first_val = self.unmerge(merged_val)[0] - unmapped_second_val = self.unmerge(merged_val)[1] - unmap_path[merged_ax] = unmapped_first_val - unmap_path[removed_ax] = unmapped_second_val - return (path, None, considered_axes, unmap_path, changed_type_path) - - def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): - datacube.complete_axes.remove(axis.name) - indexes = self.merged_values(datacube) - return indexes diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index d933e8f47..bda03c4a0 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -1,6 +1,3 @@ -from copy import deepcopy - -# from ..backends.datacube import configure_datacube_axis from .datacube_transformations import DatacubeAxisTransformation @@ -12,38 +9,9 @@ def __init__(self, name, mapper_options): def generate_final_transformation(self): return self - # def apply_transformation(self, name, datacube, values): - # # Remove the merge option from the axis options since we have already handled it - # # so do not want to handle it again - # axis_options = deepcopy(datacube.axis_options[name]["transformation"]) - # axis_options.pop("reverse") - # # Update the nested dictionary with the modified axis option for our axis - # new_datacube_axis_options = deepcopy(datacube.axis_options) - # if axis_options == {}: - # new_datacube_axis_options[name] = {} - # else: - # new_datacube_axis_options[name]["transformation"] = axis_options - # # Reconfigure the axis with the rest of its configurations - # configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) - def transformation_axes_final(self): return [self.name] - def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): - sorted_indexes = indexes.sort_values() - indexes_between = datacube._find_indexes_between(axis, sorted_indexes, low, up) - return (offset, indexes_between) - - def _adjust_path(self, path, considered_axes=[], unmap_path={}, changed_type_path={}): - return (path, None, considered_axes, unmap_path, changed_type_path) - - def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): - if not already_has_indexes: - indexes = datacube.datacube_natural_indexes(axis, subarray) - return indexes - else: - pass - def change_val_type(self, axis_name, values): return values diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index e9509b34e..d0af6dcec 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -20,8 +20,6 @@ def create_transform(name, transformation_type_key, transformation_options): @staticmethod def get_final_axes(name, transformation_type_key, transformation_options): - # NOTE: THIS IS ONE OF THE REFACTORED FUNCTIONS - # TODO: refactor this because now it's creating whole transformations which we might not need yet? new_transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key, transformation_options) transformation_axis_names = new_transformation.transformation_axes_final() @@ -81,32 +79,6 @@ def generate_final_transformation(self): def transformation_axes_final(self): pass - # @abstractmethod - # def apply_transformation(self, name, datacube, values): - # pass - - # Methods to deal with transformation in datacube backends - @abstractmethod - def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): - # Some of the axes in the datacube appear or disappear due to transformations - # When we look up the datacube, for those axes, we should take particular care to find the right - # values between low and up - pass - - @abstractmethod - def _adjust_path(self, path, considered_axes=[], unmap_path={}, changed_type_path={}): - # Some of the axes in the datacube should appear or disappear due to transformations - # When we ask the datacube for a path, we should thus remove these axes from the path - # But we want to keep track of the removed axes to be able to request data on the datacube still in unmap_path - pass - - @abstractmethod - def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): - # Some of the axes in the datacube appear or disappear due to transformations - # When we look up the datacube, for those axes, we should take particular care to find the right - # values which exist on those axes - pass - @abstractmethod def change_val_type(self, axis_name, values): pass diff --git a/polytope/datacube/transformations/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change.py index fe3bf7783..e3e8a682c 100644 --- a/polytope/datacube/transformations/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change.py @@ -1,7 +1,6 @@ from copy import deepcopy from importlib import import_module -# from ..backends.datacube import configure_datacube_axis from .datacube_transformations import DatacubeAxisTransformation @@ -24,51 +23,6 @@ def transformation_axes_final(self): final_transformation = self.generate_final_transformation() return [final_transformation.axis_name] - # def apply_transformation(self, name, datacube, values): - # transformation = self.generate_final_transformation() - # axis_options = deepcopy(datacube.axis_options[name]["transformation"]) - # axis_options.pop("type_change") - # # Update the nested dictionary with the modified axis option for our axis - # new_datacube_axis_options = deepcopy(datacube.axis_options) - # # if we have no transformations left, then empty the transformation dico - # if axis_options == {}: - # new_datacube_axis_options[name] = {} - # else: - # new_datacube_axis_options[name]["transformation"] = axis_options - # values = [transformation.transform_type(values[0])] # only need 1 value to determine type in datacube config - # configure_datacube_axis(new_datacube_axis_options[name], name, values, datacube) - - def _find_transformed_indices_between(self, axis, datacube, indexes, low, up, first_val, offset): - # NOTE: needs to be in new type - if axis.name == self.name: - transformation = self.generate_final_transformation() - indexes_between = [ - transformation.transform_type(i) for i in indexes if low <= transformation.transform_type(i) <= up - ] - # indexes_between = [] - else: - indexes_between = datacube._find_indexes_between(axis, indexes, low, up) - return (offset, indexes_between) - - def _adjust_path(self, path, considered_axes=[], unmap_path={}, changed_type_path={}): - # NOTE: used to access values, so path for the axis needs to be replaced to original type index - axis_new_val = path.get(self.name, None) - axis_val_str = str(axis_new_val) - if axis_new_val is not None: - path.pop(self.name) - changed_type_path[self.name] = axis_val_str - return (path, None, considered_axes, unmap_path, changed_type_path) - - def _find_transformed_axis_indices(self, datacube, axis, subarray, already_has_indexes): - # NOTE: needs to be in new type - # transformation = self.generate_final_transformation() - if not already_has_indexes: - indexes = datacube.datacube_natural_indexes(axis, subarray) - # indexes = [transformation.transform_type(i) for i in indexes] - return indexes - else: - pass - def change_val_type(self, axis_name, values): transformation = self.generate_final_transformation() return [transformation.transform_type(val) for val in values] diff --git a/polytope/datacube/transformations/transformation_pseudocode.py b/polytope/datacube/transformations/transformation_pseudocode.py deleted file mode 100644 index 07fc0b2c6..000000000 --- a/polytope/datacube/transformations/transformation_pseudocode.py +++ /dev/null @@ -1,81 +0,0 @@ -"""from copy import deepcopy - - -def create_axis_transformations(options, name, values, datacube): - transformation_options = options["transformation"] - for transformation_type_key in transformation_options(): - # create transformations - new_transformation = create_axis_transformation(name, transformation_type_key, transformation_options) - transformation_axis_names = new_transformation.transformation_axes_final() - - for axis_name in transformation_axis_names: - # if axis exists, just add a tag of the transformation, else create axis first and add tag after - - - - - - # if there are no transformations for that axis yet, create an empty list of transforms. - # else, take the old list and append new transformation we are working on - key_val = datacube.transformation.get(axis_name, []) - datacube.transformation[axis_name] = key_val - # the transformation dico keeps track of the type of transformation, not the exact transformations - # For grid mappers, it keeps track that we have a grid_mapper, but won't know the exact grid map we - # implement - datacube.transformation[axis_name].append(new_transformation) - new_transformation.apply_transformation(name, datacube, values) - - -def create_axis_transformation(name, transformation_type_key, transformation_options): - transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] - transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] - module = import_module("polytope.datacube.transformations.datacube_" + transformation_file_name) - constructor = getattr(module, transformation_type) - transformation_type_option = transformation_options[transformation_type_key] - new_transformation = deepcopy(constructor(name, transformation_type_option)) - new_transformation.name = name - return new_transformation - - - - - - - - # transformation options look like - # "time":{"transformation": { "type" : {"merge" : {"with":"step", "linkers": ["T", "00"]}}}} - # But the last dictionary can vary and change according to transformation, which can be handled inside the - # specialised transformations - transformation_options = options["transformation"] - # NOTE: we do the following for each transformation of each axis - for transformation_type_key in transformation_options.keys(): - transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] - transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] - - module = import_module("polytope.datacube.transformations.datacube_" + transformation_file_name) - constructor = getattr(module, transformation_type) - transformation_type_option = transformation_options[transformation_type_key] - # NOTE: the transformation in the datacube takes in now an option dico like - # {"with":"step", "linkers": ["T", "00"]}} - - # Here, we keep track of all the transformation objects along with the associated axis within the datacube - # We generate a transformation dictionary that looks like - # {"lat": [merger, cyclic], "lon": [mapper, cyclic], etc...} - new_transformation = deepcopy(constructor(name, transformation_type_option)) - # TODO: instead of adding directly the transformation, could be we have an add_transformation method - # where each transformation can choose the name that it is assigned to, ie the axis name it is assigned to - # and then for eg for grid mapper transformation, can have the first axis name in there to make things - # easier to handle in the datacube - - new_transformation.name = name - transformation_axis_names = new_transformation.transformation_axes_final() - for axis_name in transformation_axis_names: - # if there are no transformations for that axis yet, create an empty list of transforms. - # else, take the old list and append new transformation we are working on - key_val = datacube.transformation.get(axis_name, []) - datacube.transformation[axis_name] = key_val - # the transformation dico keeps track of the type of transformation, not the exact transformations - # For grid mappers, it keeps track that we have a grid_mapper, but won't know the exact grid map we - # implement - datacube.transformation[axis_name].append(new_transformation) - new_transformation.apply_transformation(name, datacube, values)""" diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index b57d11092..bc7f5959a 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -1,8 +1,9 @@ +import pandas as pd + from polytope.datacube.backends.FDB_datacube import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select -import pandas as pd class TestSlicingFDBDatacube: diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index 2e33ba6c8..a85d780eb 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -1,6 +1,6 @@ import numpy as np -import xarray as xr import pandas as pd +import xarray as xr from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer From f29d9e8e20d9e666b42e6d7668a81f9f2f48ac01 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 30 Aug 2023 11:59:24 +0200 Subject: [PATCH 120/332] merge with develop --- tests/test_mappers.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_mappers.py b/tests/test_mappers.py index 8c91bb15a..1250a02db 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -1,4 +1,4 @@ -from polytope.datacube.mappers import OctahedralGridMap +from polytope.datacube.datacube_mappers import OctahedralGridMapper class TestMapper: @@ -6,7 +6,7 @@ def test_octahedral_mapper_init(self): mapped_axes = ["lat", "lon"] base_axis = "base" resolution = 1280 - octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) assert octahedral_mapper._base_axis == base_axis assert octahedral_mapper._mapped_axes == mapped_axes assert octahedral_mapper._resolution == resolution @@ -15,7 +15,7 @@ def test_first_axis_vals_01280_grid(self): mapped_axes = ["lat", "lon"] base_axis = "base" resolution = 1280 - octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) assert octahedral_mapper.first_axis_vals()[:5] == [ 89.94618771566562, 89.87647835333229, @@ -29,7 +29,7 @@ def test_first_axis_vals_other_grid(self): mapped_axes = ["lat", "lon"] base_axis = "base" resolution = 640 - octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) assert octahedral_mapper.first_axis_vals()[:5] == [ 89.89239644559007, 89.75300494317403, @@ -43,7 +43,7 @@ def test_map_first_axis(self): mapped_axes = ["lat", "lon"] base_axis = "base" resolution = 1280 - octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) assert octahedral_mapper.map_first_axis(89.7, 89.96) == [ 89.94618771566562, 89.87647835333229, @@ -55,7 +55,7 @@ def test_second_axis_vals(self): mapped_axes = ["lat", "lon"] base_axis = "base" resolution = 1280 - octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) assert octahedral_mapper.second_axis_vals(0.035149384215604956)[0] == 0 assert octahedral_mapper.second_axis_vals(10.017574499477174)[0] == 0 assert octahedral_mapper.second_axis_vals(89.94618771566562)[10] == 180 @@ -67,14 +67,14 @@ def test_map_second_axis(self): mapped_axes = ["lat", "lon"] base_axis = "base" resolution = 1280 - octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) assert octahedral_mapper.map_second_axis(89.94618771566562, 0, 90) == [0, 18, 36, 54, 72, 90] def test_axes_idx_to_octahedral_idx(self): mapped_axes = ["lat", "lon"] base_axis = "base" resolution = 1280 - octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) assert octahedral_mapper.axes_idx_to_octahedral_idx(1, 0) == 0 assert octahedral_mapper.axes_idx_to_octahedral_idx(1, 1) == 1 assert octahedral_mapper.axes_idx_to_octahedral_idx(1, 16) == 16 @@ -99,7 +99,7 @@ def test_unmap(self): mapped_axes = ["lat", "lon"] base_axis = "base" resolution = 1280 - octahedral_mapper = OctahedralGridMap(base_axis, mapped_axes, resolution) + octahedral_mapper = OctahedralGridMapper(base_axis, mapped_axes, resolution) assert octahedral_mapper.unmap(89.94618771566562, 0) == 0 assert octahedral_mapper.unmap(0.035149384215604956, 0) == 3299840 - 5136 assert octahedral_mapper.unmap(-0.035149384215604956, 0) == 3299840 From 5f39743dca8351b94c9e075dc521da4a8b08f6ad Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 30 Aug 2023 16:44:28 +0200 Subject: [PATCH 121/332] start doing surrounding points option --- polytope/datacube/backends/datacube.py | 8 +-- polytope/datacube/datacube_axis.py | 44 ++++++++++---- polytope/engine/hullslicer.py | 3 +- polytope/polytope.py | 2 +- polytope/shapes.py | 9 ++- tests/test_snapping.py | 79 ++++++++++++++++++++++++++ 6 files changed, 125 insertions(+), 20 deletions(-) create mode 100644 tests/test_snapping.py diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index d26f3f952..6b81e0b4d 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -79,7 +79,7 @@ def fit_path(self, path): path.pop(key) return path - def get_indices(self, path: DatacubePath, axis, lower, upper): + def get_indices(self, path: DatacubePath, axis, lower, upper, method): """ Given a path to a subset of the datacube, return the discrete indexes which exist between two non-discrete values (lower, upper) for a particular axis (given by label) @@ -95,21 +95,21 @@ def get_indices(self, path: DatacubePath, axis, lower, upper): for r in original_search_ranges: offset = axis.offset(r) search_ranges_offset.append(offset) - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis) + idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, method) # Remove duplicates even if difference of the order of the axis tolerance if offset is not None: # Note that we can only do unique if not dealing with time values idx_between = unique(idx_between) return idx_between - def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis): + def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, method): idx_between = [] for i in range(len(search_ranges)): r = search_ranges[i] offset = search_ranges_offset[i] low = r[0] up = r[1] - indexes_between = axis.find_indices_between([indexes], low, up, self) + indexes_between = axis.find_indices_between([indexes], low, up, self, method) # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them for j in range(len(indexes_between)): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 80fec89ae..508495633 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -232,7 +232,8 @@ def unmap_total_path_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube): + def find_indices_between(index_ranges, low, up, datacube, method): + # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: if isinstance(transform, DatacubeMapper): @@ -303,7 +304,8 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube): + def find_indices_between(index_ranges, low, up, datacube, method): + # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): @@ -347,7 +349,8 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube): + def find_indices_between(index_ranges, low, up, datacube, method): + # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: if isinstance(transform, DatacubeAxisReverse): @@ -415,7 +418,8 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube): + def find_indices_between(index_ranges, low, up, datacube, method): + # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): @@ -508,20 +512,38 @@ def unmap_total_path_to_datacube(self, path, unmapped_path): def remap_to_requeest(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(self, index_ranges, low, up, datacube): + def find_indices_between(self, index_ranges, low, up, datacube, method): + # TODO: add method for snappping indexes_between_ranges = [] for indexes in index_ranges: if self.name in datacube.complete_axes: # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - indexes_between = indexes[start:end].to_list() - indexes_between_ranges.append(indexes_between) + if method == "snap": + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + start = max(start-1, 0) + end = min(end+1, len(indexes)) + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.append(indexes_between) + else: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.append(indexes_between) else: - indexes_between = [i for i in indexes if low <= i <= up] - indexes_between_ranges.append(indexes_between) + if method == "snap": + start = indexes.index(low) + end = indexes.index(up) + start = max(start-1, 0) + end = min(end+1, len(indexes)) + # indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + indexes_between = [i for i in indexes if low <= i <= up] + indexes_between_ranges.append(indexes_between) return indexes_between_ranges @staticmethod diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 809fa641c..1015f193e 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -46,7 +46,8 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex lower = ax.from_float(lower - tol) upper = ax.from_float(upper + tol) flattened = node.flatten() - for value in datacube.get_indices(flattened, ax, lower, upper): + method = polytope.method + for value in datacube.get_indices(flattened, ax, lower, upper, method): # convert to float for slicing fvalue = ax.to_float(value) new_polytope = slice(polytope, ax.name, fvalue) diff --git a/polytope/polytope.py b/polytope/polytope.py index ed1c708aa..5e8fc7133 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -42,7 +42,7 @@ def slice(self, polytopes: List[ConvexPolytope]): """Low-level API which takes a polytope geometry object and uses it to slice the datacube""" return self.engine.extract(self.datacube, polytopes) - def retrieve(self, request: Request): + def retrieve(self, request: Request, method="standard"): """Higher-level API which takes a request and uses it to slice the datacube""" request_tree = self.engine.extract(self.datacube, request.polytopes()) self.datacube.get(request_tree) diff --git a/polytope/shapes.py b/polytope/shapes.py index acc456731..e0d0b0bd4 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -23,9 +23,11 @@ def axes(self) -> List[str]: class ConvexPolytope(Shape): - def __init__(self, axes, points): + + def __init__(self, axes, points, method=None): self._axes = list(axes) self.points = points + self.method = method def extents(self, axis): slice_axis_idx = self.axes().index(axis) @@ -48,15 +50,16 @@ def polytope(self): class Select(Shape): """Matches several discrete value""" - def __init__(self, axis, values): + def __init__(self, axis, values, method=None): self.axis = axis self.values = values + self.method = method def axes(self): return [self.axis] def polytope(self): - return [ConvexPolytope([self.axis], [[v]]) for v in self.values] + return [ConvexPolytope([self.axis], [[v]], self.method) for v in self.values] class Span(Shape): diff --git a/tests/test_snapping.py b/tests/test_snapping.py new file mode 100644 index 000000000..672e678ed --- /dev/null +++ b/tests/test_snapping.py @@ -0,0 +1,79 @@ +import numpy as np +import xarray as xr + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Select + + +class TestSlicing3DXarrayDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + array = xr.DataArray( + np.random.randn(3, 3), + dims=("level", "step"), + coords={ + "level": [1, 3, 5], + "step": [1, 3, 5], + }, + ) + self.xarraydatacube = XArrayDatacube(array) + self.slicer = HullSlicer() + self.API = Polytope(datacube=array, engine=self.slicer) + + # Testing different shapes + + def test_2D_point(self): + request = Request(Select("level", [2], method="snap"), Select("step", [4], method="snap")) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 4 + for leaf in result.leaves: + path = leaf.flatten() + assert path["level"] in [1, 3] + assert path["step"] in [3, 5] + + def test_2D_point_outside_datacube_left(self): + request = Request(Select("level", [2], method="snap"), Select("step", [0], method="snap")) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 2 + for leaf in result.leaves: + path = leaf.flatten() + assert path["level"] in [1, 3] + assert path["step"] == 1 + + def test_2D_point_outside_datacube_right(self): + request = Request(Select("level", [2], method="snap"), Select("step", [6], method="snap")) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 2 + for leaf in result.leaves: + path = leaf.flatten() + assert path["level"] in [1, 3] + assert path["step"] == 5 + + def test_1D_point_outside_datacube_right(self): + request = Request(Select("level", [1]), Select("step", [6], method="snap")) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + for leaf in result.leaves: + path = leaf.flatten() + assert path["level"] == 1 + assert path["step"] == 5 + + def test_1D_nonexisting_point(self): + request = Request(Select("level", [2]), Select("step", [6], method="snap")) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + assert result.is_root() + + def test_1D_nonexisting_point_v2(self): + request = Request(Select("level", [2], method="snap"), Select("step", [6])) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + assert result.is_root() From 63857e3e14790075c3f25ce82ad79aa9d8bccac3 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 31 Aug 2023 10:26:06 +0200 Subject: [PATCH 122/332] make surrounding method work for all transformations --- polytope/datacube/backends/datacube.py | 26 ++++-- polytope/datacube/backends/mock.py | 2 +- polytope/datacube/datacube_axis.py | 110 +++++++++++++++++++++---- tests/test_cyclic_snapping.py | 36 ++++++++ tests/test_snapping.py | 12 +-- 5 files changed, 157 insertions(+), 29 deletions(-) create mode 100644 tests/test_cyclic_snapping.py diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 6b81e0b4d..1c02ed423 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -79,7 +79,7 @@ def fit_path(self, path): path.pop(key) return path - def get_indices(self, path: DatacubePath, axis, lower, upper, method): + def get_indices(self, path: DatacubePath, axis, lower, upper, method=None): """ Given a path to a subset of the datacube, return the discrete indexes which exist between two non-discrete values (lower, upper) for a particular axis (given by label) @@ -113,12 +113,24 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them for j in range(len(indexes_between)): - for k in range(len(indexes_between[j])): - if offset is None: - indexes_between[j][k] = indexes_between[j][k] - else: - indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) - idx_between.append(indexes_between[j][k]) + # if we have a special indexes between range that needs additional offset, treat it here + if indexes_between[j][0] == "need_offset": + new_offset = indexes_between[j][1] + for k in range(2, len(indexes_between[j])): + if offset is None: + indexes_between[j][k] = indexes_between[j][k] + else: + offset = offset + new_offset + indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) + idx_between.append(indexes_between[j][k]) + else: + # do normal offset if no new offset + for k in range(len(indexes_between[j])): + if offset is None: + indexes_between[j][k] = indexes_between[j][k] + else: + indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) + idx_between.append(indexes_between[j][k]) return idx_between def get_mapper(self, axis): diff --git a/polytope/datacube/backends/mock.py b/polytope/datacube/backends/mock.py index 351c0208e..b6bc32d3e 100644 --- a/polytope/datacube/backends/mock.py +++ b/polytope/datacube/backends/mock.py @@ -41,7 +41,7 @@ def get(self, requests: IndexTree): def get_mapper(self, axis): return self.mappers[axis] - def get_indices(self, path: DatacubePath, axis, lower, upper): + def get_indices(self, path: DatacubePath, axis, lower, upper, method=None): if lower == upper == math.ceil(lower): if lower >= 0: return [int(lower)] diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 508495633..01e8eda85 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -135,6 +135,41 @@ def unmap_to_datacube(path, unmapped_path): (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) return (path, unmapped_path) + old_find_indices_between = cls.find_indices_between + + def find_indices_between(index_ranges, low, up, datacube, method=None): + update_range() + range_length = cls.range[1] - cls.range[0] + indexes_between_ranges = [] + if method == "surrounding": + for indexes in index_ranges: + if cls.name in datacube.complete_axes: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + else: + start = indexes.index(low) + end = indexes.index(up) + if start-1 < 0: + start_offset_indicator = "need_offset" + new_offset = - range_length # TODO: not sure about this offset + index_val_found = indexes[-1:][0] + indexes_between_before = [start_offset_indicator, new_offset, index_val_found].to_list() + indexes_between_ranges.append(indexes_between_before) + if end+1 > len(indexes): + start_offset_indicator = "need_offset" + new_offset = range_length + index_val_found = indexes[:1][0] + indexes_between_after = [start_offset_indicator, new_offset, index_val_found].to_list() + indexes_between_ranges.append(indexes_between_after) + start = max(start-1, 0) + end = min(end+1, len(indexes)) + # indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + else: + return old_find_indices_between(index_ranges, low, up, datacube, method) + def offset(range): # We first unpad the range by the axis tolerance to make sure that # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. @@ -150,6 +185,7 @@ def offset(range): cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube + cls.find_indices_between = find_indices_between return cls @@ -232,7 +268,7 @@ def unmap_total_path_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method): + def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: @@ -240,8 +276,19 @@ def find_indices_between(index_ranges, low, up, datacube, method): transformation = transform if cls.name in transformation._mapped_axes(): for idxs in index_ranges: - indexes_between = [i for i in idxs if low <= i <= up] - indexes_between_ranges.append(indexes_between) + if method == "surrounding": + start = idxs.index(low) + end = idxs.index(up) + start = max(start-1, 0) + end = min(end+1, len(idxs)) + # indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = idxs[start:end] + indexes_between_ranges.append(indexes_between) + else: + indexes_between = [i for i in idxs if low <= i <= up] + indexes_between_ranges.append(indexes_between) + # indexes_between = [i for i in idxs if low <= i <= up] + # indexes_between_ranges.append(indexes_between) return indexes_between_ranges old_remap = cls.remap @@ -304,7 +351,7 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method): + def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: @@ -312,8 +359,19 @@ def find_indices_between(index_ranges, low, up, datacube, method): transformation = transform if cls.name in transformation._mapped_axes(): for indexes in index_ranges: - indexes_between = [i for i in indexes if low <= i <= up] - indexes_between_ranges.append(indexes_between) + if method == "surrounding": + start = indexes.index(low) + end = indexes.index(up) + start = max(start-1, 0) + end = min(end+1, len(indexes)) + # indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + indexes_between = [i for i in indexes if low <= i <= up] + indexes_between_ranges.append(indexes_between) + # indexes_between = [i for i in indexes if low <= i <= up] + # indexes_between_ranges.append(indexes_between) return indexes_between_ranges def remap(range): @@ -349,7 +407,7 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method): + def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: @@ -357,8 +415,19 @@ def find_indices_between(index_ranges, low, up, datacube, method): transformation = transform if cls.name == transformation.name: for indexes in index_ranges: - indexes_between = [i for i in indexes if low <= i <= up] - indexes_between_ranges.append(indexes_between) + if method == "surrounding": + start = indexes.index(low) + end = indexes.index(up) + start = max(start-1, 0) + end = min(end+1, len(indexes)) + # indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + indexes_between = [i for i in indexes if low <= i <= up] + indexes_between_ranges.append(indexes_between) + # indexes_between = [i for i in indexes if low <= i <= up] + # indexes_between_ranges.append(indexes_between) return indexes_between_ranges def remap(range): @@ -418,7 +487,7 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method): + def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: @@ -426,8 +495,19 @@ def find_indices_between(index_ranges, low, up, datacube, method): transformation = transform if cls.name == transformation.name: for indexes in index_ranges: - indexes_between = [i for i in indexes if low <= i <= up] - indexes_between_ranges.append(indexes_between) + if method == "surrounding": + start = indexes.index(low) + end = indexes.index(up) + start = max(start-1, 0) + end = min(end+1, len(indexes)) + # indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + indexes_between = [i for i in indexes if low <= i <= up] + indexes_between_ranges.append(indexes_between) + # indexes_between = [i for i in indexes if low <= i <= up] + # indexes_between_ranges.append(indexes_between) return indexes_between_ranges def remap(range): @@ -512,7 +592,7 @@ def unmap_total_path_to_datacube(self, path, unmapped_path): def remap_to_requeest(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(self, index_ranges, low, up, datacube, method): + def find_indices_between(self, index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] for indexes in index_ranges: @@ -520,7 +600,7 @@ def find_indices_between(self, index_ranges, low, up, datacube, method): # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - if method == "snap": + if method == "surrounding": start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") start = max(start-1, 0) @@ -533,7 +613,7 @@ def find_indices_between(self, index_ranges, low, up, datacube, method): indexes_between = indexes[start:end].to_list() indexes_between_ranges.append(indexes_between) else: - if method == "snap": + if method == "surrounding": start = indexes.index(low) end = indexes.index(up) start = max(start-1, 0) diff --git a/tests/test_cyclic_snapping.py b/tests/test_cyclic_snapping.py new file mode 100644 index 000000000..9f7b045f2 --- /dev/null +++ b/tests/test_cyclic_snapping.py @@ -0,0 +1,36 @@ +import xarray as xr + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Select + + +class TestSlicing3DXarrayDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + array = xr.DataArray( + [0, 1, 2], + dims=("long"), + coords={ + "long": [0, 0.5, 1.0], + }, + ) + options = {"long": {"transformation": {"cyclic": [0, 1.0]}}} + self.xarraydatacube = XArrayDatacube(array) + self.slicer = HullSlicer() + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + + # Testing different shapes + + def test_cyclic_float_axis_across_seam(self): + request = Request( + Select("long", [-0.2], method="surrounding") + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 2 + assert result.leaves[0].flatten()["long"] == -0.5 + assert result.leaves[1].flatten()["long"] == 0. + assert result.leaves[0].result == (None, 1) + assert result.leaves[1].result == (None, 0) diff --git a/tests/test_snapping.py b/tests/test_snapping.py index 672e678ed..7d127a877 100644 --- a/tests/test_snapping.py +++ b/tests/test_snapping.py @@ -25,7 +25,7 @@ def setup_method(self, method): # Testing different shapes def test_2D_point(self): - request = Request(Select("level", [2], method="snap"), Select("step", [4], method="snap")) + request = Request(Select("level", [2], method="surrounding"), Select("step", [4], method="surrounding")) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 4 @@ -35,7 +35,7 @@ def test_2D_point(self): assert path["step"] in [3, 5] def test_2D_point_outside_datacube_left(self): - request = Request(Select("level", [2], method="snap"), Select("step", [0], method="snap")) + request = Request(Select("level", [2], method="surrounding"), Select("step", [0], method="surrounding")) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 2 @@ -45,7 +45,7 @@ def test_2D_point_outside_datacube_left(self): assert path["step"] == 1 def test_2D_point_outside_datacube_right(self): - request = Request(Select("level", [2], method="snap"), Select("step", [6], method="snap")) + request = Request(Select("level", [2], method="surrounding"), Select("step", [6], method="surrounding")) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 2 @@ -55,7 +55,7 @@ def test_2D_point_outside_datacube_right(self): assert path["step"] == 5 def test_1D_point_outside_datacube_right(self): - request = Request(Select("level", [1]), Select("step", [6], method="snap")) + request = Request(Select("level", [1]), Select("step", [6], method="surrounding")) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 1 @@ -65,14 +65,14 @@ def test_1D_point_outside_datacube_right(self): assert path["step"] == 5 def test_1D_nonexisting_point(self): - request = Request(Select("level", [2]), Select("step", [6], method="snap")) + request = Request(Select("level", [2]), Select("step", [6], method="surrounding")) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 1 assert result.is_root() def test_1D_nonexisting_point_v2(self): - request = Request(Select("level", [2], method="snap"), Select("step", [6])) + request = Request(Select("level", [2], method="surrounding"), Select("step", [6])) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 1 From 7917b68a0e69135bcc9bb348f2a25b58e7a7675b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 31 Aug 2023 16:56:12 +0200 Subject: [PATCH 123/332] try to make cyclic surrounding method work --- polytope/datacube/backends/datacube.py | 41 ++-- polytope/datacube/datacube_axis.py | 328 +++++++++++++++++++++---- tests/test_snapping_real_data.py | 59 +++++ 3 files changed, 367 insertions(+), 61 deletions(-) create mode 100644 tests/test_snapping_real_data.py diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 1c02ed423..ae1ba527b 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -109,28 +109,35 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, offset = search_ranges_offset[i] low = r[0] up = r[1] - indexes_between = axis.find_indices_between([indexes], low, up, self, method) + indexes_between = axis.find_indices_between([indexes], low, up, self, offset, method) # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them for j in range(len(indexes_between)): # if we have a special indexes between range that needs additional offset, treat it here - if indexes_between[j][0] == "need_offset": - new_offset = indexes_between[j][1] - for k in range(2, len(indexes_between[j])): - if offset is None: - indexes_between[j][k] = indexes_between[j][k] - else: - offset = offset + new_offset - indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) - idx_between.append(indexes_between[j][k]) + if len(indexes_between[j]) == 0: + idx_between = idx_between else: - # do normal offset if no new offset - for k in range(len(indexes_between[j])): - if offset is None: - indexes_between[j][k] = indexes_between[j][k] - else: - indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) - idx_between.append(indexes_between[j][k]) + if indexes_between[j][0] == "need_offset": + print(indexes_between[j]) + new_offset = indexes_between[j][1] + for k in range(2, len(indexes_between[j])): + if offset is None: + indexes_between[j][k] = indexes_between[j][k] + else: + # offset = offset + new_offset + offset = offset + new_offset + indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) + idx_between.append(indexes_between[j][k]) + else: + # do normal offset if no new offset + for k in range(len(indexes_between[j])): + print("HERE") + print(offset) + if offset is None: + indexes_between[j][k] = indexes_between[j][k] + else: + indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) + idx_between.append(indexes_between[j][k]) return idx_between def get_mapper(self, axis): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 01e8eda85..ed716d8f7 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -137,38 +137,250 @@ def unmap_to_datacube(path, unmapped_path): old_find_indices_between = cls.find_indices_between - def find_indices_between(index_ranges, low, up, datacube, method=None): + def find_indices_between(index_ranges, low, up, datacube, offset, method=None): update_range() range_length = cls.range[1] - cls.range[0] indexes_between_ranges = [] - if method == "surrounding": - for indexes in index_ranges: - if cls.name in datacube.complete_axes: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - else: - start = indexes.index(low) - end = indexes.index(up) - if start-1 < 0: - start_offset_indicator = "need_offset" - new_offset = - range_length # TODO: not sure about this offset - index_val_found = indexes[-1:][0] - indexes_between_before = [start_offset_indicator, new_offset, index_val_found].to_list() - indexes_between_ranges.append(indexes_between_before) - if end+1 > len(indexes): - start_offset_indicator = "need_offset" - new_offset = range_length - index_val_found = indexes[:1][0] - indexes_between_after = [start_offset_indicator, new_offset, index_val_found].to_list() - indexes_between_ranges.append(indexes_between_after) - start = max(start-1, 0) - end = min(end+1, len(indexes)) - # indexes_between = [i for i in indexes if low <= i <= up] - indexes_between = indexes[start:end].to_list() - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - else: - return old_find_indices_between(index_ranges, low, up, datacube, method) + + if offset < 0: + new_offset = 0 + while low > cls.range[0]: + low = low - range_length + new_offset -= range_length + up = up - range_length + if method == "surrounding": + for indexes in index_ranges: + if cls.name in datacube.complete_axes: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + # TODO: need to include the offset here to understand when start-1< 0 with the offset accounted for + if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) + start_offset_indicator = "need_offset" + # new_offset = new_offset - range_length + # new_offset = offset - range_length + index_val_found = indexes[-1:][0] + indexes_between_before = [start_offset_indicator, new_offset, index_val_found] + indexes_between_ranges.append(indexes_between_before) + if end+1 > len(indexes): + start_offset_indicator = "need_offset" + index_val_found = indexes[:1][0] + indexes_between_after = [start_offset_indicator, new_offset, index_val_found] + indexes_between_ranges.append(indexes_between_after) + start = max(start-1, 0) + end = min(end+1, len(indexes)) + # indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = indexes[start:end].to_list() + indexes_between = [i - new_offset for i in indexes_between] + indexes_between_ranges.append(indexes_between) + print("IDXS") + print(indexes_between_ranges) + else: + start = indexes.index(low) + end = indexes.index(up) + if start-1 < 0: + start_offset_indicator = "need_offset" + index_val_found = indexes[-1:][0] + indexes_between_before = [start_offset_indicator, new_offset, index_val_found] + indexes_between_ranges.append(indexes_between_before) + if end+1 > len(indexes): + start_offset_indicator = "need_offset" + index_val_found = indexes[:1][0] + indexes_between_after = [start_offset_indicator, new_offset, index_val_found] + indexes_between_ranges.append(indexes_between_after) + start = max(start-1, 0) + end = min(end+1, len(indexes)) + # indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + else: + return old_find_indices_between(index_ranges, low, up, datacube, method, offset) + + + + # if offset > 0: + # low = low - offset + # up = up - offset + # else: + # low = low + range_length + offset + # up = up + range_length + offset + # if method == "surrounding": + # for indexes in index_ranges: + # if cls.name in datacube.complete_axes: + # start = indexes.searchsorted(low, "left") + # end = indexes.searchsorted(up, "right") + # # TODO: need to include the offset here to understand when start-1< 0 with the offset accounted for + # if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) + # start_offset_indicator = "need_offset" + # # new_offset = - range_length # TODO: not sure about this offset + # # new_offset = 0 + # if offset == 0: + # new_offset = 0 + # elif offset < 0: + # print(range_length) + # new_offset = range_length + # else: + # new_offset = - range_length + # # new_offset = - range_length + # index_val_found = indexes[-1:][0] + # indexes_between_before = [start_offset_indicator, new_offset, index_val_found] + # indexes_between_ranges.append(indexes_between_before) + # if end+1 > len(indexes): + # start_offset_indicator = "need_offset" + # # new_offset = range_length + # # new_offset = 0 + # if offset == 0: + # new_offset = 0 + # elif offset < 0: + # new_offset = - range_length + # else: + # new_offset = range_length + # index_val_found = indexes[:1][0] + # indexes_between_after = [start_offset_indicator, new_offset, index_val_found] + # indexes_between_ranges.append(indexes_between_after) + # start = max(start-1, 0) + # end = min(end+1, len(indexes)) + # # indexes_between = [i for i in indexes if low <= i <= up] + # indexes_between = indexes[start:end].to_list() + # indexes_between_ranges.append(indexes_between) + # else: + # start = indexes.index(low) + # end = indexes.index(up) + # if start-1 < 0: + # start_offset_indicator = "need_offset" + # # new_offset = - range_length # TODO: not sure about this offset + # # new_offset = 0 + # if offset == 0: + # new_offset = 0 + # elif offset < 0: + # new_offset = range_length + # else: + # new_offset = - range_length + # index_val_found = indexes[-1:][0] + # indexes_between_before = [start_offset_indicator, new_offset, index_val_found] + # indexes_between_ranges.append(indexes_between_before) + # if end+1 > len(indexes): + # start_offset_indicator = "need_offset" + # # new_offset = range_length + # # new_offset = 0 + # if offset == 0: + # new_offset = 0 + # elif offset < 0: + # new_offset = - range_length + # else: + # new_offset = range_length + # index_val_found = indexes[:1][0] + # indexes_between_after = [start_offset_indicator, new_offset, index_val_found] + # indexes_between_ranges.append(indexes_between_after) + # start = max(start-1, 0) + # end = min(end+1, len(indexes)) + # # indexes_between = [i for i in indexes if low <= i <= up] + # indexes_between = indexes[start:end] + # indexes_between_ranges.append(indexes_between) + # return indexes_between_ranges + # else: + # return old_find_indices_between(index_ranges, low, up, datacube, method, offset) + + # def find_indices_between(index_ranges, low, up, datacube, offset, method=None): + # update_range() + # range_length = cls.range[1] - cls.range[0] + # indexes_between_ranges = [] + # if offset > 0: + # low = low - offset + # up = up - offset + # else: + # print("HERE") + # print(low) + # # low = low + offset + # low = low + range_length + offset + # print(low) + # # up = up + offset + # up = up + range_length + offset + # if method == "surrounding": + # for indexes in index_ranges: + # # if offset > 0: + # # low = low - offset + # # up = up - offset + # # else: + # # print("HERE") + # # print(low) + # # low = low + offset + # # print(low) + # # up = up + offset + # if cls.name in datacube.complete_axes: + # start = indexes.searchsorted(low, "left") + # end = indexes.searchsorted(up, "right") + # # TODO: need to include the offset here to understand when start-1< 0 with the offset accounted for + # if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) + # start_offset_indicator = "need_offset" + # # new_offset = - range_length # TODO: not sure about this offset + # # new_offset = 0 + # if offset == 0: + # new_offset = 0 + # elif offset < 0: + # print(range_length) + # new_offset = range_length + # else: + # new_offset = - range_length + # # new_offset = - range_length + # index_val_found = indexes[-1:][0] + # indexes_between_before = [start_offset_indicator, new_offset, index_val_found] + # indexes_between_ranges.append(indexes_between_before) + # if end+1 > len(indexes): + # start_offset_indicator = "need_offset" + # # new_offset = range_length + # # new_offset = 0 + # if offset == 0: + # new_offset = 0 + # elif offset < 0: + # new_offset = - range_length + # else: + # new_offset = range_length + # index_val_found = indexes[:1][0] + # indexes_between_after = [start_offset_indicator, new_offset, index_val_found] + # indexes_between_ranges.append(indexes_between_after) + # start = max(start-1, 0) + # end = min(end+1, len(indexes)) + # # indexes_between = [i for i in indexes if low <= i <= up] + # indexes_between = indexes[start:end].to_list() + # indexes_between_ranges.append(indexes_between) + # else: + # start = indexes.index(low) + # end = indexes.index(up) + # if start-1 < 0: + # start_offset_indicator = "need_offset" + # # new_offset = - range_length # TODO: not sure about this offset + # # new_offset = 0 + # if offset == 0: + # new_offset = 0 + # elif offset < 0: + # new_offset = range_length + # else: + # new_offset = - range_length + # index_val_found = indexes[-1:][0] + # indexes_between_before = [start_offset_indicator, new_offset, index_val_found] + # indexes_between_ranges.append(indexes_between_before) + # if end+1 > len(indexes): + # start_offset_indicator = "need_offset" + # # new_offset = range_length + # # new_offset = 0 + # if offset == 0: + # new_offset = 0 + # elif offset < 0: + # new_offset = - range_length + # else: + # new_offset = range_length + # index_val_found = indexes[:1][0] + # indexes_between_after = [start_offset_indicator, new_offset, index_val_found] + # indexes_between_ranges.append(indexes_between_after) + # start = max(start-1, 0) + # end = min(end+1, len(indexes)) + # # indexes_between = [i for i in indexes if low <= i <= up] + # indexes_between = indexes[start:end] + # indexes_between_ranges.append(indexes_between) + # return indexes_between_ranges + # else: + # return old_find_indices_between(index_ranges, low, up, datacube, method, offset) def offset(range): # We first unpad the range by the axis tolerance to make sure that @@ -268,7 +480,7 @@ def unmap_total_path_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): + def find_indices_between(index_ranges, low, up, datacube, offset, method=None): # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: @@ -351,7 +563,7 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): + def find_indices_between(index_ranges, low, up, datacube, offset, method=None): # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: @@ -407,7 +619,7 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): + def find_indices_between(index_ranges, low, up, datacube, offset, method=None): # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: @@ -415,17 +627,45 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): transformation = transform if cls.name == transformation.name: for indexes in index_ranges: - if method == "surrounding": - start = indexes.index(low) - end = indexes.index(up) - start = max(start-1, 0) - end = min(end+1, len(indexes)) - # indexes_between = [i for i in indexes if low <= i <= up] - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) + if cls.name in datacube.complete_axes: + # Find the range of indexes between lower and upper + # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html + # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing + if method == "surrounding": + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + start = max(start-1, 0) + end = min(end+1, len(indexes)) + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.append(indexes_between) + else: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.append(indexes_between) else: - indexes_between = [i for i in indexes if low <= i <= up] - indexes_between_ranges.append(indexes_between) + if method == "surrounding": + start = indexes.index(low) + end = indexes.index(up) + start = max(start-1, 0) + end = min(end+1, len(indexes)) + # indexes_between = [i for i in indexes if low <= i <= up] + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + indexes_between = [i for i in indexes if low <= i <= up] + indexes_between_ranges.append(indexes_between) + # if method == "surrounding": + # start = indexes.index(low) + # end = indexes.index(up) + # start = max(start-1, 0) + # end = min(end+1, len(indexes)) + # # indexes_between = [i for i in indexes if low <= i <= up] + # indexes_between = indexes[start:end] + # indexes_between_ranges.append(indexes_between) + # else: + # indexes_between = [i for i in indexes if low <= i <= up] + # indexes_between_ranges.append(indexes_between) # indexes_between = [i for i in indexes if low <= i <= up] # indexes_between_ranges.append(indexes_between) return indexes_between_ranges @@ -487,7 +727,7 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): + def find_indices_between(index_ranges, low, up, datacube, offset, method=None): # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: @@ -592,7 +832,7 @@ def unmap_total_path_to_datacube(self, path, unmapped_path): def remap_to_requeest(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(self, index_ranges, low, up, datacube, method=None): + def find_indices_between(self, index_ranges, low, up, datacube, offset, method=None): # TODO: add method for snappping indexes_between_ranges = [] for indexes in index_ranges: diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py new file mode 100644 index 000000000..9513daa5f --- /dev/null +++ b/tests/test_snapping_real_data.py @@ -0,0 +1,59 @@ +import geopandas as gpd +import matplotlib.pyplot as plt +import numpy as np +from earthkit import data + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestSlicingEra5Data: + def setup_method(self, method): + ds = data.from_source("file", "./tests/data/era5-levels-members.grib") + array = ds.to_xarray().isel(step=0).t + self.xarraydatacube = XArrayDatacube(array) + self.slicer = HullSlicer() + options = {"latitude": {"transformation": {"reverse": {True}}}, + "longitude": {"transformation": {"cyclic": [0, 360.0]}}} + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + + def test_surrounding_on_grid_point(self): + requested_lat = 0 + requested_lon = -720 + request = Request( + Box(["number", "isobaricInhPa"], [6, 500.0], [6, 850.0]), + Select("time", ["2017-01-02T12:00:00"]), + # Box(["latitude", "longitude"], lower_corner=[0.0, 0.0], upper_corner=[10.0, 30.0]), + Select("latitude", [requested_lat], method="surrounding"), + Select("longitude", [requested_lon], method="surrounding"), + Select("step", [np.timedelta64(0, "s")]), + ) + result = self.API.retrieve(request) + result.pprint() + country_points_plotting = [] + lats = [] + longs = [] + temps = [] + for i in range(len(result.leaves)): + cubepath = result.leaves[i].flatten() + lat = cubepath["latitude"] + long = cubepath["longitude"] + latlong_point = [lat, long] + lats.append(lat) + longs.append(long) + t_idx = result.leaves[i].result[1] + temps.append(t_idx) + country_points_plotting.append(latlong_point) + temps = np.array(temps) + + # Plot all the points on a world map + worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + fig, ax = plt.subplots(figsize=(12, 6)) + worldmap.plot(color="darkgrey", ax=ax) + + plt.scatter(longs, lats, s=16, c=temps, cmap="YlOrRd") + plt.scatter([requested_lon], [requested_lat], s=16, c="blue") + plt.colorbar(label="Temperature") + plt.show() From 3bf4f2db5be6c3ebff3cb36cba74d34cd03221dd Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 1 Sep 2023 11:14:08 +0200 Subject: [PATCH 124/332] make surrounding method work with cyclic axes --- polytope/datacube/backends/datacube.py | 10 +- polytope/datacube/datacube_axis.py | 245 +++++-------------------- tests/test_snapping_real_data.py | 2 +- 3 files changed, 55 insertions(+), 202 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index ae1ba527b..45a320d9d 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -118,25 +118,23 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, idx_between = idx_between else: if indexes_between[j][0] == "need_offset": - print(indexes_between[j]) new_offset = indexes_between[j][1] for k in range(2, len(indexes_between[j])): if offset is None: indexes_between[j][k] = indexes_between[j][k] else: - # offset = offset + new_offset offset = offset + new_offset - indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) + indexes_between[j][k] = round(indexes_between[j][k] + offset, + int(-math.log10(axis.tol))) idx_between.append(indexes_between[j][k]) else: # do normal offset if no new offset for k in range(len(indexes_between[j])): - print("HERE") - print(offset) if offset is None: indexes_between[j][k] = indexes_between[j][k] else: - indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) + indexes_between[j][k] = round(indexes_between[j][k] + offset, + int(-math.log10(axis.tol))) idx_between.append(indexes_between[j][k]) return idx_between diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index ed716d8f7..1c77b1912 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -142,9 +142,11 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): range_length = cls.range[1] - cls.range[0] indexes_between_ranges = [] - if offset < 0: + if offset != 0: + # NOTE that if the offset is not 0, then we need to recenter the low and up + # values to be within the datacube range new_offset = 0 - while low > cls.range[0]: + while low >= cls.range[0] - cls.tol: low = low - range_length new_offset -= range_length up = up - range_length @@ -153,11 +155,8 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): if cls.name in datacube.complete_axes: start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") - # TODO: need to include the offset here to understand when start-1< 0 with the offset accounted for if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) start_offset_indicator = "need_offset" - # new_offset = new_offset - range_length - # new_offset = offset - range_length index_val_found = indexes[-1:][0] indexes_between_before = [start_offset_indicator, new_offset, index_val_found] indexes_between_ranges.append(indexes_between_before) @@ -168,12 +167,9 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): indexes_between_ranges.append(indexes_between_after) start = max(start-1, 0) end = min(end+1, len(indexes)) - # indexes_between = [i for i in indexes if low <= i <= up] indexes_between = indexes[start:end].to_list() indexes_between = [i - new_offset for i in indexes_between] indexes_between_ranges.append(indexes_between) - print("IDXS") - print(indexes_between_ranges) else: start = indexes.index(low) end = indexes.index(up) @@ -189,198 +185,56 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): indexes_between_ranges.append(indexes_between_after) start = max(start-1, 0) end = min(end+1, len(indexes)) - # indexes_between = [i for i in indexes if low <= i <= up] indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) return indexes_between_ranges else: return old_find_indices_between(index_ranges, low, up, datacube, method, offset) - - - # if offset > 0: - # low = low - offset - # up = up - offset - # else: - # low = low + range_length + offset - # up = up + range_length + offset - # if method == "surrounding": - # for indexes in index_ranges: - # if cls.name in datacube.complete_axes: - # start = indexes.searchsorted(low, "left") - # end = indexes.searchsorted(up, "right") - # # TODO: need to include the offset here to understand when start-1< 0 with the offset accounted for - # if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) - # start_offset_indicator = "need_offset" - # # new_offset = - range_length # TODO: not sure about this offset - # # new_offset = 0 - # if offset == 0: - # new_offset = 0 - # elif offset < 0: - # print(range_length) - # new_offset = range_length - # else: - # new_offset = - range_length - # # new_offset = - range_length - # index_val_found = indexes[-1:][0] - # indexes_between_before = [start_offset_indicator, new_offset, index_val_found] - # indexes_between_ranges.append(indexes_between_before) - # if end+1 > len(indexes): - # start_offset_indicator = "need_offset" - # # new_offset = range_length - # # new_offset = 0 - # if offset == 0: - # new_offset = 0 - # elif offset < 0: - # new_offset = - range_length - # else: - # new_offset = range_length - # index_val_found = indexes[:1][0] - # indexes_between_after = [start_offset_indicator, new_offset, index_val_found] - # indexes_between_ranges.append(indexes_between_after) - # start = max(start-1, 0) - # end = min(end+1, len(indexes)) - # # indexes_between = [i for i in indexes if low <= i <= up] - # indexes_between = indexes[start:end].to_list() - # indexes_between_ranges.append(indexes_between) - # else: - # start = indexes.index(low) - # end = indexes.index(up) - # if start-1 < 0: - # start_offset_indicator = "need_offset" - # # new_offset = - range_length # TODO: not sure about this offset - # # new_offset = 0 - # if offset == 0: - # new_offset = 0 - # elif offset < 0: - # new_offset = range_length - # else: - # new_offset = - range_length - # index_val_found = indexes[-1:][0] - # indexes_between_before = [start_offset_indicator, new_offset, index_val_found] - # indexes_between_ranges.append(indexes_between_before) - # if end+1 > len(indexes): - # start_offset_indicator = "need_offset" - # # new_offset = range_length - # # new_offset = 0 - # if offset == 0: - # new_offset = 0 - # elif offset < 0: - # new_offset = - range_length - # else: - # new_offset = range_length - # index_val_found = indexes[:1][0] - # indexes_between_after = [start_offset_indicator, new_offset, index_val_found] - # indexes_between_ranges.append(indexes_between_after) - # start = max(start-1, 0) - # end = min(end+1, len(indexes)) - # # indexes_between = [i for i in indexes if low <= i <= up] - # indexes_between = indexes[start:end] - # indexes_between_ranges.append(indexes_between) - # return indexes_between_ranges - # else: - # return old_find_indices_between(index_ranges, low, up, datacube, method, offset) - - # def find_indices_between(index_ranges, low, up, datacube, offset, method=None): - # update_range() - # range_length = cls.range[1] - cls.range[0] - # indexes_between_ranges = [] - # if offset > 0: - # low = low - offset - # up = up - offset - # else: - # print("HERE") - # print(low) - # # low = low + offset - # low = low + range_length + offset - # print(low) - # # up = up + offset - # up = up + range_length + offset - # if method == "surrounding": - # for indexes in index_ranges: - # # if offset > 0: - # # low = low - offset - # # up = up - offset - # # else: - # # print("HERE") - # # print(low) - # # low = low + offset - # # print(low) - # # up = up + offset - # if cls.name in datacube.complete_axes: - # start = indexes.searchsorted(low, "left") - # end = indexes.searchsorted(up, "right") - # # TODO: need to include the offset here to understand when start-1< 0 with the offset accounted for - # if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) - # start_offset_indicator = "need_offset" - # # new_offset = - range_length # TODO: not sure about this offset - # # new_offset = 0 - # if offset == 0: - # new_offset = 0 - # elif offset < 0: - # print(range_length) - # new_offset = range_length - # else: - # new_offset = - range_length - # # new_offset = - range_length - # index_val_found = indexes[-1:][0] - # indexes_between_before = [start_offset_indicator, new_offset, index_val_found] - # indexes_between_ranges.append(indexes_between_before) - # if end+1 > len(indexes): - # start_offset_indicator = "need_offset" - # # new_offset = range_length - # # new_offset = 0 - # if offset == 0: - # new_offset = 0 - # elif offset < 0: - # new_offset = - range_length - # else: - # new_offset = range_length - # index_val_found = indexes[:1][0] - # indexes_between_after = [start_offset_indicator, new_offset, index_val_found] - # indexes_between_ranges.append(indexes_between_after) - # start = max(start-1, 0) - # end = min(end+1, len(indexes)) - # # indexes_between = [i for i in indexes if low <= i <= up] - # indexes_between = indexes[start:end].to_list() - # indexes_between_ranges.append(indexes_between) - # else: - # start = indexes.index(low) - # end = indexes.index(up) - # if start-1 < 0: - # start_offset_indicator = "need_offset" - # # new_offset = - range_length # TODO: not sure about this offset - # # new_offset = 0 - # if offset == 0: - # new_offset = 0 - # elif offset < 0: - # new_offset = range_length - # else: - # new_offset = - range_length - # index_val_found = indexes[-1:][0] - # indexes_between_before = [start_offset_indicator, new_offset, index_val_found] - # indexes_between_ranges.append(indexes_between_before) - # if end+1 > len(indexes): - # start_offset_indicator = "need_offset" - # # new_offset = range_length - # # new_offset = 0 - # if offset == 0: - # new_offset = 0 - # elif offset < 0: - # new_offset = - range_length - # else: - # new_offset = range_length - # index_val_found = indexes[:1][0] - # indexes_between_after = [start_offset_indicator, new_offset, index_val_found] - # indexes_between_ranges.append(indexes_between_after) - # start = max(start-1, 0) - # end = min(end+1, len(indexes)) - # # indexes_between = [i for i in indexes if low <= i <= up] - # indexes_between = indexes[start:end] - # indexes_between_ranges.append(indexes_between) - # return indexes_between_ranges - # else: - # return old_find_indices_between(index_ranges, low, up, datacube, method, offset) + else: + # If the offset is 0, then the first value found on the left has an offset of range_length + new_offset = 0 + if method == "surrounding": + for indexes in index_ranges: + if cls.name in datacube.complete_axes: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) + start_offset_indicator = "need_offset" + index_val_found = indexes[-1:][0] + new_offset = -range_length + indexes_between_before = [start_offset_indicator, new_offset, index_val_found] + indexes_between_ranges.append(indexes_between_before) + if end+1 > len(indexes): + start_offset_indicator = "need_offset" + index_val_found = indexes[:1][0] + indexes_between_after = [start_offset_indicator, new_offset, index_val_found] + indexes_between_ranges.append(indexes_between_after) + start = max(start-1, 0) + end = min(end+1, len(indexes)) + indexes_between = indexes[start:end].to_list() + indexes_between = [i - new_offset for i in indexes_between] + indexes_between_ranges.append(indexes_between) + else: + start = indexes.index(low) + end = indexes.index(up) + if start-1 < 0: + start_offset_indicator = "need_offset" + index_val_found = indexes[-1:][0] + indexes_between_before = [start_offset_indicator, new_offset, index_val_found] + indexes_between_ranges.append(indexes_between_before) + if end+1 > len(indexes): + start_offset_indicator = "need_offset" + index_val_found = indexes[:1][0] + indexes_between_after = [start_offset_indicator, new_offset, index_val_found] + indexes_between_ranges.append(indexes_between_after) + start = max(start-1, 0) + end = min(end+1, len(indexes)) + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + else: + return old_find_indices_between(index_ranges, low, up, datacube, method, offset) def offset(range): # We first unpad the range by the axis tolerance to make sure that @@ -630,7 +484,8 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): if cls.name in datacube.complete_axes: # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html - # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing + # Assumes the indexes are already sorted (could sort to be sure) and monotonically + # increasing if method == "surrounding": start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index 9513daa5f..feb9e01f9 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -21,7 +21,7 @@ def setup_method(self, method): def test_surrounding_on_grid_point(self): requested_lat = 0 - requested_lon = -720 + requested_lon = 720 request = Request( Box(["number", "isobaricInhPa"], [6, 500.0], [6, 850.0]), Select("time", ["2017-01-02T12:00:00"]), From 2b7cd43fc54784d106a2323a25b797531929beba Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 5 Sep 2023 09:49:31 +0200 Subject: [PATCH 125/332] make all tests work --- polytope/datacube/datacube_axis.py | 12 ++++++---- tests/test_cyclic_simple.py | 37 ++++++++++++++++++++++++++++++ tests/test_cyclic_snapping.py | 2 +- tests/test_datacube_axes_init.py | 2 +- tests/test_snapping_real_data.py | 24 ++++++++++--------- 5 files changed, 60 insertions(+), 17 deletions(-) create mode 100644 tests/test_cyclic_simple.py diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 1c77b1912..db364b4b1 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -142,14 +142,18 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): range_length = cls.range[1] - cls.range[0] indexes_between_ranges = [] + if method != "surrounding": + return old_find_indices_between(index_ranges, low, up, datacube, method, offset) + if offset != 0: # NOTE that if the offset is not 0, then we need to recenter the low and up # values to be within the datacube range new_offset = 0 - while low >= cls.range[0] - cls.tol: - low = low - range_length - new_offset -= range_length - up = up - range_length + if low <= cls.range[0] + cls.tol: + while low >= cls.range[0] - cls.tol: + low = low - range_length + new_offset -= range_length + up = up - range_length if method == "surrounding": for indexes in index_ranges: if cls.name in datacube.complete_axes: diff --git a/tests/test_cyclic_simple.py b/tests/test_cyclic_simple.py new file mode 100644 index 000000000..be48cb8b9 --- /dev/null +++ b/tests/test_cyclic_simple.py @@ -0,0 +1,37 @@ +import numpy as np +import pandas as pd +import xarray as xr + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestSlicing3DXarrayDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + array = xr.DataArray( + np.random.randn(3, 6, 129, 11), + dims=("date", "step", "level", "long"), + coords={ + "date": pd.date_range("2000-01-01", "2000-01-03", 3), + "step": [0, 3, 6, 9, 12, 15], + "level": range(1, 130), + "long": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], + }, + ) + options = {"long": {"transformation": {"cyclic": [0, 1.0]}}, "level": {"transformation": {"cyclic": [1, 129]}}} + self.xarraydatacube = XArrayDatacube(array) + self.slicer = HullSlicer() + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + + # Testing different shapes + + def test_cyclic_float_axis_across_seam(self): + request = Request( + Box(["step", "long"], [0, 0.9], [0, 1.2]), Select("date", ["2000-01-01"]), Select("level", [128]) + ) + result = self.API.retrieve(request) + assert len(result.leaves) == 4 + assert [leaf.value for leaf in result.leaves] == [0.9, 1.0, 1.1, 1.2] diff --git a/tests/test_cyclic_snapping.py b/tests/test_cyclic_snapping.py index 9f7b045f2..ac1b11c7f 100644 --- a/tests/test_cyclic_snapping.py +++ b/tests/test_cyclic_snapping.py @@ -31,6 +31,6 @@ def test_cyclic_float_axis_across_seam(self): result.pprint() assert len(result.leaves) == 2 assert result.leaves[0].flatten()["long"] == -0.5 - assert result.leaves[1].flatten()["long"] == 0. + assert result.leaves[1].flatten()["long"] == 0.0 assert result.leaves[0].result == (None, 1) assert result.leaves[1].result == (None, 0) diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 83461e926..bea08086b 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -49,7 +49,7 @@ def test_created_axes(self): assert path == {} assert unmapped_path == {"values": 0} assert lat_ax.find_indices_between([[89.94618771566562, 89.87647835333229]], - 89.87, 90, self.datacube) == [[89.94618771566562, 89.87647835333229]] + 89.87, 90, self.datacube, 0) == [[89.94618771566562, 89.87647835333229]] def test_mapper_transformation_request(self): request = Request( diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index feb9e01f9..247a276b8 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -1,5 +1,5 @@ -import geopandas as gpd -import matplotlib.pyplot as plt +# import geopandas as gpd +# import matplotlib.pyplot as plt import numpy as np from earthkit import data @@ -21,7 +21,7 @@ def setup_method(self, method): def test_surrounding_on_grid_point(self): requested_lat = 0 - requested_lon = 720 + requested_lon = 0 request = Request( Box(["number", "isobaricInhPa"], [6, 500.0], [6, 850.0]), Select("time", ["2017-01-02T12:00:00"]), @@ -48,12 +48,14 @@ def test_surrounding_on_grid_point(self): country_points_plotting.append(latlong_point) temps = np.array(temps) - # Plot all the points on a world map - worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) - fig, ax = plt.subplots(figsize=(12, 6)) - worldmap.plot(color="darkgrey", ax=ax) + # # Plot all the points on a world map + # worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + # fig, ax = plt.subplots(figsize=(12, 6)) + # worldmap.plot(color="darkgrey", ax=ax) - plt.scatter(longs, lats, s=16, c=temps, cmap="YlOrRd") - plt.scatter([requested_lon], [requested_lat], s=16, c="blue") - plt.colorbar(label="Temperature") - plt.show() + # plt.scatter(longs, lats, s=16, c=temps, cmap="YlOrRd") + # plt.scatter([requested_lon], [requested_lat], s=16, c="blue") + # plt.colorbar(label="Temperature") + # plt.show() + for lon in longs: + assert lon in [-3, 0, 3] From ce0661f9cb39e2d3c72c9299fabc56bc05995b74 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 5 Sep 2023 13:08:29 +0200 Subject: [PATCH 126/332] try to make cyclic without offsets --- polytope/datacube/backends/datacube.py | 15 +- polytope/datacube/datacube_axis.py | 28 ++- polytope/datacube/index_tree.py | 17 +- polytope/engine/hullslicer.py | 10 +- tests/test_cyclic_axis_over_negative_vals.py | 167 ++++++-------- tests/test_cyclic_axis_slicer_not_0.py | 173 +++++++-------- tests/test_cyclic_axis_slicing.py | 215 +++++++------------ tests/test_cyclic_simple.py | 31 ++- tests/test_cyclic_snapping.py | 8 +- 9 files changed, 315 insertions(+), 349 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 45a320d9d..0bc710c93 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -1,4 +1,5 @@ import importlib +import itertools import math from abc import ABC, abstractmethod from typing import Any @@ -100,10 +101,15 @@ def get_indices(self, path: DatacubePath, axis, lower, upper, method=None): if offset is not None: # Note that we can only do unique if not dealing with time values idx_between = unique(idx_between) + print("inside get indices") + print(idx_between) return idx_between def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, method): + print("inside look up datacube") idx_between = [] + print(axis.name) + print(search_ranges) for i in range(len(search_ranges)): r = search_ranges[i] offset = search_ranges_offset[i] @@ -118,12 +124,11 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, idx_between = idx_between else: if indexes_between[j][0] == "need_offset": - new_offset = indexes_between[j][1] for k in range(2, len(indexes_between[j])): if offset is None: indexes_between[j][k] = indexes_between[j][k] else: - offset = offset + new_offset + offset = offset indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) idx_between.append(indexes_between[j][k]) @@ -133,9 +138,12 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, if offset is None: indexes_between[j][k] = indexes_between[j][k] else: + offset = offset indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) idx_between.append(indexes_between[j][k]) + print("indexes between") + print(idx_between) return idx_between def get_mapper(self, axis): @@ -149,6 +157,9 @@ def remap_path(self, path: DatacubePath): value = path[key] path[key] = self._axes[key].remap([value, value])[0][0] return path + + # def remap_tree(self, tree: IndexTree): + # for @staticmethod def create(datacube, axis_options: dict): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index db364b4b1..4c7a66d50 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -149,24 +149,28 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): # NOTE that if the offset is not 0, then we need to recenter the low and up # values to be within the datacube range new_offset = 0 - if low <= cls.range[0] + cls.tol: - while low >= cls.range[0] - cls.tol: - low = low - range_length - new_offset -= range_length - up = up - range_length + # if low <= cls.range[0] + cls.tol: + # while low >= cls.range[0] - cls.tol: + # low = low - range_length + # new_offset -= range_length + # up = up - range_length + print("INSIDE THE DATACUBE AXIS DECORATOR") + print((low, up)) if method == "surrounding": for indexes in index_ranges: if cls.name in datacube.complete_axes: start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") + print(start, end) if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) start_offset_indicator = "need_offset" - index_val_found = indexes[-1:][0] + index_val_found = indexes[-2:][0] indexes_between_before = [start_offset_indicator, new_offset, index_val_found] indexes_between_ranges.append(indexes_between_before) if end+1 > len(indexes): + print("end exceeded indices length") start_offset_indicator = "need_offset" - index_val_found = indexes[:1][0] + index_val_found = indexes[:2][1] indexes_between_after = [start_offset_indicator, new_offset, index_val_found] indexes_between_ranges.append(indexes_between_after) start = max(start-1, 0) @@ -199,25 +203,29 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): # If the offset is 0, then the first value found on the left has an offset of range_length new_offset = 0 if method == "surrounding": + print("no offset") for indexes in index_ranges: if cls.name in datacube.complete_axes: start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") + print((start, end)) if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) + print("low start") start_offset_indicator = "need_offset" - index_val_found = indexes[-1:][0] + index_val_found = indexes[-2:][0] + print(index_val_found) new_offset = -range_length indexes_between_before = [start_offset_indicator, new_offset, index_val_found] indexes_between_ranges.append(indexes_between_before) if end+1 > len(indexes): start_offset_indicator = "need_offset" - index_val_found = indexes[:1][0] + index_val_found = indexes[:2][1] indexes_between_after = [start_offset_indicator, new_offset, index_val_found] indexes_between_ranges.append(indexes_between_after) start = max(start-1, 0) end = min(end+1, len(indexes)) indexes_between = indexes[start:end].to_list() - indexes_between = [i - new_offset for i in indexes_between] + indexes_between = [i for i in indexes_between] indexes_between_ranges.append(indexes_between) else: start = indexes.index(low) diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index 9833fd240..2d285327e 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -58,7 +58,18 @@ def __hash__(self): def __eq__(self, other): if not isinstance(other, IndexTree): return False - return (self.axis.name, self.value) == (other.axis.name, other.value) + if self.axis.name != other.axis.name: + return False + else: + if other.value == self.value: + return True + if other.value - 2*other.axis.tol <= self.value <= other.value + 2*other.axis.tol: + return True + elif self.value - 2*self.axis.tol <= other.value <= self.value + 2*self.axis.tol: + return True + else: + return False + # return (self.axis.name, self.value) == (other.axis.name, other.value) def __lt__(self, other): return (self.axis.name, self.value) < (other.axis.name, other.value) @@ -73,12 +84,12 @@ def add_child(self, node): self.children.add(node) node._parent = self - def create_child(self, axis, value): + def create_child_not_safe(self, axis, value): node = IndexTree(axis, value) self.add_child(node) return node - def create_child_safe(self, axis, value): + def create_child(self, axis, value): node = IndexTree(axis, value) existing = self.find_child(node) if not existing: diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 1015f193e..5553f4e5a 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -1,6 +1,7 @@ from copy import copy from itertools import chain from typing import List +import math import scipy.spatial @@ -52,7 +53,14 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex fvalue = ax.to_float(value) new_polytope = slice(polytope, ax.name, fvalue) # store the native type - child = node.create_child(ax, value) + # remapped_val = (ax.remap([value, value])[0][0] + ax.remap([value, value])[0][1])/2 + remapped_val = value + if ax.is_cyclic: + remapped_val = (ax.remap([value, value])[0][0] + ax.remap([value, value])[0][1])/2 + remapped_val = round(remapped_val, int(-math.log10(ax.tol))) + print(remapped_val) + # child = node.create_child(ax, value) + child = node.create_child(ax, remapped_val) child["unsliced_polytopes"] = copy(node["unsliced_polytopes"]) child["unsliced_polytopes"].remove(polytope) if new_polytope is not None: diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index 151065ad0..a67333cce 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -36,29 +36,29 @@ def test_cyclic_float_axis_across_seam(self): Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) - # result.pprint() + result.pprint() assert len(result.leaves) == 20 assert [leaf.value for leaf in result.leaves] == [ - 0.8, - 0.9, - 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 0.8, - 0.9, - 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, + -1.1, + -1.0, + -0.9, + -0.8, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, + -0.2, + -1.1, + -1.0, + -0.9, + -0.8, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, + -0.2 ] def test_cyclic_float_axis_inside_cyclic_range(self): @@ -69,22 +69,22 @@ def test_cyclic_float_axis_inside_cyclic_range(self): # result.pprint() assert len(result.leaves) == 16 assert [leaf.value for leaf in result.leaves] == [ - 0.0, - 0.1, - 0.2, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.0, - 0.1, - 0.2, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, + -1.0, + -0.9, + -0.8, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, + -1.0, + -0.9, + -0.8, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, ] def test_cyclic_float_axis_above_axis_range(self): @@ -94,66 +94,37 @@ def test_cyclic_float_axis_above_axis_range(self): result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] + assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, + -0.7, -0.6, -0.5, -0.4, -0.3] def test_cyclic_float_axis_two_range_loops(self): request = Request( Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) - # result.pprint() - assert len(result.leaves) == 50 + result.pprint() + assert len(result.leaves) == 20 assert [leaf.value for leaf in result.leaves] == [ - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.8, - 0.9, - 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.8, - 1.9, - 2.0, - 2.1, - 2.2, - 2.3, - 2.4, - 2.5, - 2.6, - 2.7, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.8, - 0.9, - 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.8, - 1.9, - 2.0, - 2.1, - 2.2, - 2.3, - 2.4, - 2.5, - 2.6, - 2.7, + -1.1, + -1.0, + -0.9, + -0.8, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, + -0.2, + -1.1, + -1.0, + -0.9, + -0.8, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, + -0.2 ] def test_cyclic_float_axis_below_axis_range(self): @@ -171,8 +142,11 @@ def test_cyclic_float_axis_below_axis_range_crossing_seam(self): ) result = self.API.retrieve(request) # result.pprint() - assert len(result.leaves) == 22 + assert len(result.leaves) == 20 assert [leaf.value for leaf in result.leaves] == [ + -1.0, + -0.9, + -0.8, -0.7, -0.6, -0.5, @@ -180,10 +154,9 @@ def test_cyclic_float_axis_below_axis_range_crossing_seam(self): -0.3, -0.2, -0.1, - 0.0, - 0.1, - 0.2, - 0.3, + -1.0, + -0.9, + -0.8, -0.7, -0.6, -0.5, @@ -191,8 +164,4 @@ def test_cyclic_float_axis_below_axis_range_crossing_seam(self): -0.3, -0.2, -0.1, - 0.0, - 0.1, - 0.2, - 0.3, ] diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 3173ee77a..33ef062e6 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -36,28 +36,29 @@ def test_cyclic_float_axis_across_seam(self): Box(["step", "long"], [0, 0.8], [3, 1.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) + result.pprint() assert len(result.leaves) == 20 assert [leaf.value for leaf in result.leaves] == [ - 0.8, - 0.9, - 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 0.8, - 0.9, - 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, + -1.1, + -1.0, + -0.9, + -0.8, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, + -0.2, + -1.1, + -1.0, + -0.9, + -0.8, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, + -0.2, ] def test_cyclic_float_axis_inside_cyclic_range(self): @@ -67,22 +68,22 @@ def test_cyclic_float_axis_inside_cyclic_range(self): result = self.API.retrieve(request) assert len(result.leaves) == 16 assert [leaf.value for leaf in result.leaves] == [ - 0.0, - 0.1, - 0.2, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.0, - 0.1, - 0.2, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, + -1.0, + -0.9, + -0.8, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, + -1.0, + -0.9, + -0.8, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, ] def test_cyclic_float_axis_above_axis_range(self): @@ -91,65 +92,45 @@ def test_cyclic_float_axis_above_axis_range(self): ) result = self.API.retrieve(request) assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] + assert [leaf.value for leaf in result.leaves] == [ + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3] def test_cyclic_float_axis_two_range_loops(self): request = Request( Box(["step", "long"], [0, 0.3], [3, 2.7]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) - assert len(result.leaves) == 50 + assert len(result.leaves) == 20 assert [leaf.value for leaf in result.leaves] == [ - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.8, - 0.9, - 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.8, - 1.9, - 2.0, - 2.1, - 2.2, - 2.3, - 2.4, - 2.5, - 2.6, - 2.7, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - 0.8, - 0.9, - 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.8, - 1.9, - 2.0, - 2.1, - 2.2, - 2.3, - 2.4, - 2.5, - 2.6, - 2.7, + -1.1, + -1.0, + -0.9, + -0.8, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, + -0.2, + -1.1, + -1.0, + -0.9, + -0.8, + -0.7, + -0.6, + -0.5, + -0.4, + -0.3, + -0.2, ] def test_cyclic_float_axis_below_axis_range(self): @@ -165,8 +146,11 @@ def test_cyclic_float_axis_below_axis_range_crossing_seam(self): Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) - assert len(result.leaves) == 22 + assert len(result.leaves) == 20 assert [leaf.value for leaf in result.leaves] == [ + -1.0, + -0.9, + -0.8, -0.7, -0.6, -0.5, @@ -174,10 +158,9 @@ def test_cyclic_float_axis_below_axis_range_crossing_seam(self): -0.3, -0.2, -0.1, - 0.0, - 0.1, - 0.2, - 0.3, + -1.0, + -0.9, + -0.8, -0.7, -0.6, -0.5, @@ -185,8 +168,4 @@ def test_cyclic_float_axis_below_axis_range_crossing_seam(self): -0.3, -0.2, -0.1, - 0.0, - 0.1, - 0.2, - 0.3, ] diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index fcf213480..3d3c68465 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -34,27 +34,28 @@ def test_cyclic_float_axis_across_seam(self): ) result = self.API.retrieve(request) assert len(result.leaves) == 20 + result.pprint() assert [leaf.value for leaf in result.leaves] == [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, 0.8, 0.9, - 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, + 1., + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, 0.8, 0.9, - 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, + 1., ] def test_cyclic_float_axis_across_seam_repeated(self): @@ -94,8 +95,8 @@ def test_cyclic_float_axis_across_seam_repeated_twice(self): Box(["step", "long"], [0, 0.0], [3, 2.0]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) - # result.pprint() - assert len(result.leaves) == 22 * 2 - 2 + result.pprint() + assert len(result.leaves) == 22 assert [leaf.value for leaf in result.leaves] == [ 0.0, 0.1, @@ -108,16 +109,6 @@ def test_cyclic_float_axis_across_seam_repeated_twice(self): 0.8, 0.9, 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.8, - 1.9, - 2.0, 0.0, 0.1, 0.2, @@ -129,16 +120,6 @@ def test_cyclic_float_axis_across_seam_repeated_twice(self): 0.8, 0.9, 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.8, - 1.9, - 2.0, ] def test_cyclic_float_axis_inside_cyclic_range(self): @@ -176,7 +157,7 @@ def test_cyclic_float_axis_above_axis_range(self): result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] + assert [leaf.value for leaf in result.leaves] == [0.3, 0.4, 0.5, 0.6, 0.7, 0.3, 0.4, 0.5, 0.6, 0.7] def test_cyclic_float_axis_two_range_loops(self): request = Request( @@ -184,8 +165,11 @@ def test_cyclic_float_axis_two_range_loops(self): ) result = self.API.retrieve(request) # result.pprint() - assert len(result.leaves) == 50 + assert len(result.leaves) == 22 assert [leaf.value for leaf in result.leaves] == [ + 0.0, + 0.1, + 0.2, 0.3, 0.4, 0.5, @@ -194,23 +178,9 @@ def test_cyclic_float_axis_two_range_loops(self): 0.8, 0.9, 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.8, - 1.9, - 2.0, - 2.1, - 2.2, - 2.3, - 2.4, - 2.5, - 2.6, - 2.7, + 0.0, + 0.1, + 0.2, 0.3, 0.4, 0.5, @@ -219,23 +189,6 @@ def test_cyclic_float_axis_two_range_loops(self): 0.8, 0.9, 1.0, - 1.1, - 1.2, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.8, - 1.9, - 2.0, - 2.1, - 2.2, - 2.3, - 2.4, - 2.5, - 2.6, - 2.7, ] def test_cyclic_float_axis_below_axis_range(self): @@ -245,7 +198,7 @@ def test_cyclic_float_axis_below_axis_range(self): result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, -0.3] + assert [leaf.value for leaf in result.leaves] == [0.3, 0.4, 0.5, 0.6, 0.7, 0.3, 0.4, 0.5, 0.6, 0.7] def test_cyclic_float_axis_below_axis_range_crossing_seam(self): request = Request( @@ -253,30 +206,28 @@ def test_cyclic_float_axis_below_axis_range_crossing_seam(self): ) result = self.API.retrieve(request) # result.pprint() - assert len(result.leaves) == 22 + assert len(result.leaves) == 20 assert [leaf.value for leaf in result.leaves] == [ - -0.7, - -0.6, - -0.5, - -0.4, - -0.3, - -0.2, - -0.1, 0.0, 0.1, 0.2, 0.3, - -0.7, - -0.6, - -0.5, - -0.4, - -0.3, - -0.2, - -0.1, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, 0.0, 0.1, 0.2, 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, ] def test_cyclic_float_axis_reversed(self): @@ -286,34 +237,34 @@ def test_cyclic_float_axis_reversed(self): result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [1.3, 1.4, 1.5, 1.6, 1.7, 1.3, 1.4, 1.5, 1.6, 1.7] + assert [leaf.value for leaf in result.leaves] == [0.3, 0.4, 0.5, 0.6, 0.7, 0.3, 0.4, 0.5, 0.6, 0.7] def test_two_cyclic_axis_wrong_axis_order(self): request = Request(Box(["step", "long", "level"], [0, 1.3, 131], [3, 1.7, 132]), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) - # result.pprint() + result.pprint() assert len(result.leaves) == 20 assert [leaf.value for leaf in result.leaves] == [ - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, ] def test_two_cyclic_axis(self): @@ -322,26 +273,26 @@ def test_two_cyclic_axis(self): # result.pprint() assert len(result.leaves) == 20 assert [leaf.value for leaf in result.leaves] == [ - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, - 1.3, - 1.4, - 1.5, - 1.6, - 1.7, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, ] def test_select_cyclic_float_axis_edge(self): diff --git a/tests/test_cyclic_simple.py b/tests/test_cyclic_simple.py index be48cb8b9..ec213e2c8 100644 --- a/tests/test_cyclic_simple.py +++ b/tests/test_cyclic_simple.py @@ -33,5 +33,34 @@ def test_cyclic_float_axis_across_seam(self): Box(["step", "long"], [0, 0.9], [0, 1.2]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) + result.pprint() assert len(result.leaves) == 4 - assert [leaf.value for leaf in result.leaves] == [0.9, 1.0, 1.1, 1.2] + assert [leaf.value for leaf in result.leaves] == [0.1, 0.2, 0.9, 1.0] + + def test_cyclic_float_surrounding(self): + request = Request( + Select("step", [0]), + Select("long", [1.], method="surrounding"), + Select("date", ["2000-01-01"]), + Select("level", [128]) + ) + result = self.API.retrieve(request) + result.pprint() + for leaf in result.leaves: + path = leaf.flatten() + lon_val = path["long"] + assert lon_val in [0., 0.1, 0.9, 1.] + + def test_cyclic_float_surrounding_below_seam(self): + request = Request( + Select("step", [0]), + Select("long", [0.], method="surrounding"), + Select("date", ["2000-01-01"]), + Select("level", [128]) + ) + result = self.API.retrieve(request) + result.pprint() + for leaf in result.leaves: + path = leaf.flatten() + lon_val = path["long"] + assert lon_val in [0., 0.1, 0.9] diff --git a/tests/test_cyclic_snapping.py b/tests/test_cyclic_snapping.py index ac1b11c7f..b0a6580ba 100644 --- a/tests/test_cyclic_snapping.py +++ b/tests/test_cyclic_snapping.py @@ -30,7 +30,7 @@ def test_cyclic_float_axis_across_seam(self): result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 2 - assert result.leaves[0].flatten()["long"] == -0.5 - assert result.leaves[1].flatten()["long"] == 0.0 - assert result.leaves[0].result == (None, 1) - assert result.leaves[1].result == (None, 0) + assert result.leaves[0].flatten()["long"] == 0.0 + assert result.leaves[1].flatten()["long"] == 0.5 + assert result.leaves[0].result == (None, 0) + assert result.leaves[1].result == (None, 1) From 1449f4a87086f40c8984a3958aee37c3b80e8d26 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 6 Sep 2023 11:08:25 +0200 Subject: [PATCH 127/332] make all tests work together --- polytope/datacube/backends/FDB_datacube.py | 4 +- polytope/datacube/backends/datacube.py | 8 +- polytope/datacube/backends/xarray.py | 4 +- polytope/datacube/datacube_axis.py | 81 ++++++++++--------- .../datacube_transformations.py | 40 --------- tests/test_axis_mappers.py | 3 + tests/test_cyclic_axis_over_negative_vals.py | 4 +- tests/test_cyclic_axis_slicer_not_0.py | 6 +- tests/test_cyclic_axis_slicing.py | 7 +- tests/test_datacube_axes_init.py | 6 +- tests/test_fdb_datacube.py | 9 +-- tests/test_hullslicer_engine.py | 6 +- tests/test_merge_cyclic_octahedral.py | 12 +-- tests/test_merge_octahedral_one_axis.py | 12 +-- tests/test_merge_transformation.py | 8 +- tests/test_octahedral_grid.py | 10 +-- tests/test_reverse_transformation.py | 2 +- tests/test_slicing_unsliceable_axis.py | 2 +- tests/test_type_change_transformation.py | 2 +- 19 files changed, 97 insertions(+), 129 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 46e2c673c..a478811e8 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -24,12 +24,12 @@ def __init__(self, config={}, axis_options={}): self.axis_options = axis_options self.grid_mapper = None self.axis_counter = 0 - self._axes = {} + self._axes = None treated_axes = [] self.non_complete_axes = [] self.complete_axes = [] self.blocked_axes = [] - self.transformation = {} + self.transformation = None self.fake_axes = [] partial_request = config diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index d26f3f952..fe5b044b3 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -41,7 +41,9 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt # first need to change the values so that we have right type values = transformation.change_val_type(axis_name, values) - if axis_name not in self._axes.keys(): + if self._axes is None: + DatacubeAxis.create_standard(axis_name, values, self) + elif axis_name not in self._axes.keys(): DatacubeAxis.create_standard(axis_name, values, self) # add transformation tag to axis, as well as transformation options for later setattr(self._axes[axis_name], has_transform[transformation_type_key], True) # where has_transform is a @@ -64,7 +66,9 @@ def _check_and_add_axes(self, options, name, values): self._add_all_transformation_axes(options, name, values) else: if name not in self.blocked_axes: - if name not in self._axes.keys(): + if self._axes is None: + DatacubeAxis.create_standard(name, values, self) + elif name not in self._axes.keys(): DatacubeAxis.create_standard(name, values, self) def has_index(self, path: DatacubePath, axis, index): diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index b7c92a8b7..321f0f644 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -12,13 +12,13 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.axis_options = axis_options self.grid_mapper = None self.axis_counter = 0 - self._axes = {} + self._axes = None self.dataarray = dataarray treated_axes = [] self.non_complete_axes = [] self.complete_axes = [] self.blocked_axes = [] - self.transformation = {} + self.transformation = None self.fake_axes = [] for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 80fec89ae..245c2501a 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -1,4 +1,4 @@ -from abc import ABC, abstractmethod, abstractproperty +from abc import ABC, abstractmethod from copy import deepcopy from typing import Any, List @@ -210,7 +210,9 @@ def unmap_total_path_to_datacube(path, unmapped_path): # if we are on the first axis, then need to add the first val to unmapped_path first_val = path.get(cls.name, None) path.pop(cls.name, None) - if cls.name not in unmapped_path: + if unmapped_path is None: + unmapped_path[cls.name] = first_val + elif cls.name not in unmapped_path: # if for some reason, the unmapped_path already has the first axis val, then don't update unmapped_path[cls.name] = first_val if cls.name == transformation._mapped_axes()[1]: @@ -451,18 +453,6 @@ def update_axis(self): self = cyclic(self) return self - @abstractproperty - def name(self) -> str: - pass - - @abstractproperty - def tol(self) -> Any: - pass - - @abstractproperty - def range(self) -> List[Any]: - pass - # Convert from user-provided value to CONTINUOUS type (e.g. float, pd.timestamp) @abstractmethod def parse(self, value: Any) -> Any: @@ -528,7 +518,10 @@ def find_indices_between(self, index_ranges, low, up, datacube): def create_standard(name, values, datacube): values = np.array(values) DatacubeAxis.check_axis_type(name, values) - datacube._axes[name] = deepcopy(_type_to_axis_lookup[values.dtype.type]) + if datacube._axes is None: + datacube._axes = {name: deepcopy(_type_to_axis_lookup[values.dtype.type])} + else: + datacube._axes[name] = deepcopy(_type_to_axis_lookup[values.dtype.type]) datacube._axes[name].name = name datacube.axis_counter += 1 @@ -544,11 +537,13 @@ def check_axis_type(name, values): @mapper @type_change class IntDatacubeAxis(DatacubeAxis): - name = None - tol = 1e-12 - range = None - transformations = [] - type = 0 + + def __init__(self): + self.name = None + self.tol = 1e-12 + self.range = None + self.transformations = [] + self.type = 0 def parse(self, value: Any) -> Any: return float(value) @@ -568,11 +563,13 @@ def serialize(self, value): @mapper @type_change class FloatDatacubeAxis(DatacubeAxis): - name = None - tol = 1e-12 - range = None - transformations = [] - type = 0. + + def __init__(self): + self.name = None + self.tol = 1e-12 + self.range = None + self.transformations = [] + self.type = 0. def parse(self, value: Any) -> Any: return float(value) @@ -589,11 +586,13 @@ def serialize(self, value): @merge class PandasTimestampDatacubeAxis(DatacubeAxis): - name = None - tol = 1e-12 - range = None - transformations = [] - type = pd.Timestamp("2000-01-01T00:00:00") + + def __init__(self): + self.name = None + self.tol = 1e-12 + self.range = None + self.transformations = [] + self.type = pd.Timestamp("2000-01-01T00:00:00") def parse(self, value: Any) -> Any: if isinstance(value, np.str_): @@ -618,11 +617,13 @@ def offset(self, value): @merge class PandasTimedeltaDatacubeAxis(DatacubeAxis): - name = None - tol = 1e-12 - range = None - transformations = [] - type = np.timedelta64(0, "s") + + def __init__(self): + self.name = None + self.tol = 1e-12 + self.range = None + self.transformations = [] + self.type = np.timedelta64(0, "s") def parse(self, value: Any) -> Any: if isinstance(value, np.str_): @@ -647,10 +648,12 @@ def offset(self, value): @type_change class UnsliceableDatacubeAxis(DatacubeAxis): - name = None - tol = float("NaN") - range = None - transformations = [] + + def __init__(self): + self.name = None + self.tol = float("NaN") + self.range = None + self.transformations = [] def parse(self, value: Any) -> Any: return value diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index d0af6dcec..e266c5ce7 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -25,46 +25,6 @@ def get_final_axes(name, transformation_type_key, transformation_options): transformation_axis_names = new_transformation.transformation_axes_final() return transformation_axis_names - @staticmethod - def create_transformation(options, name, values, datacube): - # transformation options look like - # "time":{"transformation": { "type" : {"merge" : {"with":"step", "linkers": ["T", "00"]}}}} - # But the last dictionary can vary and change according to transformation, which can be handled inside the - # specialised transformations - transformation_options = options["transformation"] - # NOTE: we do the following for each transformation of each axis - for transformation_type_key in transformation_options.keys(): - transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] - transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] - - module = import_module("polytope.datacube.transformations.datacube_" + transformation_file_name) - constructor = getattr(module, transformation_type) - transformation_type_option = transformation_options[transformation_type_key] - # NOTE: the transformation in the datacube takes in now an option dico like - # {"with":"step", "linkers": ["T", "00"]}} - - # Here, we keep track of all the transformation objects along with the associated axis within the datacube - # We generate a transformation dictionary that looks like - # {"lat": [merger, cyclic], "lon": [mapper, cyclic], etc...} - new_transformation = deepcopy(constructor(name, transformation_type_option)) - # TODO: instead of adding directly the transformation, could be we have an add_transformation method - # where each transformation can choose the name that it is assigned to, ie the axis name it is assigned to - # and then for eg for grid mapper transformation, can have the first axis name in there to make things - # easier to handle in the datacube - - new_transformation.name = name - transformation_axis_names = new_transformation.transformation_axes_final() - for axis_name in transformation_axis_names: - # if there are no transformations for that axis yet, create an empty list of transforms. - # else, take the old list and append new transformation we are working on - key_val = datacube.transformation.get(axis_name, []) - datacube.transformation[axis_name] = key_val - # the transformation dico keeps track of the type of transformation, not the exact transformations - # For grid mappers, it keeps track that we have a grid_mapper, but won't know the exact grid map we - # implement - datacube.transformation[axis_name].append(new_transformation) - new_transformation.apply_transformation(name, datacube, values) - def name(self): pass diff --git a/tests/test_axis_mappers.py b/tests/test_axis_mappers.py index 301769ec4..991dbb1f0 100644 --- a/tests/test_axis_mappers.py +++ b/tests/test_axis_mappers.py @@ -9,6 +9,9 @@ class TestAxisMappers: + def setup_method(self, method): + pass + def test_int_axis(self): axis = IntDatacubeAxis() assert axis.parse(2) == 2.0 diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index 151065ad0..fec097c9c 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -8,7 +8,7 @@ from polytope.shapes import Box, Select -class TestSlicing3DXarrayDatacube: +class TestSlicingCyclicAxisNegVals: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types array = xr.DataArray( @@ -161,7 +161,6 @@ def test_cyclic_float_axis_below_axis_range(self): Box(["step", "long"], [0, -0.7], [3, -0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) - # result.pprint() assert len(result.leaves) == 10 assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, -0.3] @@ -170,7 +169,6 @@ def test_cyclic_float_axis_below_axis_range_crossing_seam(self): Box(["step", "long"], [0, -0.7], [3, 0.3]), Select("date", ["2000-01-01"]), Select("level", [128]) ) result = self.API.retrieve(request) - # result.pprint() assert len(result.leaves) == 22 assert [leaf.value for leaf in result.leaves] == [ -0.7, diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 3173ee77a..074c8b1cf 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -8,7 +8,7 @@ from polytope.shapes import Box, Select -class TestSlicing3DXarrayDatacube: +class TestSlicingCyclicAxisNotOverZero: def setup_method(self, method): # create a dataarray with 3 labelled axes using different index types array = xr.DataArray( @@ -21,13 +21,13 @@ def setup_method(self, method): "long": [-0.1, -0.2, -0.3, -0.4, -0.5, -0.6, -0.7, -0.8, -0.9, -1.0, -1.1][::-1], }, ) - options = { + self.options = { "long": {"transformation": {"cyclic": [-1.1, -0.1]}}, "level": {"transformation": {"cyclic": [1, 129]}}, } self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) # Testing different shapes diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index fcf213480..ceaa40d13 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -8,7 +8,7 @@ from polytope.shapes import Box, Select -class TestSlicing3DXarrayDatacube: +class TestSlicingCyclic: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types array = xr.DataArray( @@ -21,10 +21,11 @@ def setup_method(self, method): "long": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], }, ) - options = {"long": {"transformation": {"cyclic": [0, 1.0]}}, "level": {"transformation": {"cyclic": [1, 129]}}} + self.options = {"long": {"transformation": {"cyclic": [0, 1.0]}}, + "level": {"transformation": {"cyclic": [1, 129]}}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) # Testing different shapes diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 83461e926..db9b30544 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -7,13 +7,13 @@ from polytope.shapes import Box, Select -class TestOctahedralGrid: +class TestInitDatacubeAxes: def setup_method(self, method): ds = data.from_source("file", "./tests/data/foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m self.xarraydatacube = XArrayDatacube(latlon_array) - grid_options = { + self.options = { "values": { "transformation": { "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} @@ -21,7 +21,7 @@ def setup_method(self, method): } } self.slicer = HullSlicer() - self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=grid_options) + self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=self.options) self.datacube = self.API.datacube def test_created_axes(self): diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index bc7f5959a..8614ad3d5 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -9,7 +9,7 @@ class TestSlicingFDBDatacube: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types - grid_options = { + self.options = { "values": { "transformation": { "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} @@ -18,13 +18,12 @@ def setup_method(self, method): "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, } - config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} - self.xarraydatacube = FDBDatacube(config, axis_options=grid_options) + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() - self.API = Polytope(datacube=self.xarraydatacube, engine=self.slicer) + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - def test_2D_box(self): request = Request( Select("step", [11]), diff --git a/tests/test_hullslicer_engine.py b/tests/test_hullslicer_engine.py index dd876beb4..ded90cb84 100644 --- a/tests/test_hullslicer_engine.py +++ b/tests/test_hullslicer_engine.py @@ -10,7 +10,7 @@ class TestSlicerComponents: def setup_method(self, method): - array = xr.DataArray( + self.array = xr.DataArray( np.random.randn(4, 100), dims=("step", "level"), coords={ @@ -18,9 +18,9 @@ def setup_method(self, method): "level": np.arange(0, 100, 1), }, ) - self.xarraydatacube = XArrayDatacube(array) + self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer) + self.API = Polytope(datacube=self.array, engine=self.slicer) def test_extract(self): box = Box(["step", "level"], [3.0, 1.0], [6.0, 3.0]) diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index 85c3a5891..ac5d5abc5 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -7,10 +7,11 @@ from polytope.shapes import Box, Select, Span -class TestSlicing4DXarrayDatacube: +class TestMultipleTransformations: + @classmethod def setup_method(self, method): # Create a dataarray with 4 labelled axes using different index types - array = xr.DataArray( + self.array = xr.DataArray( np.random.randn(1, 1, 4289589, 3), dims=("date", "time", "values", "step"), coords={ @@ -20,7 +21,7 @@ def setup_method(self, method): "step": [0, 1, 2], }, ) - options = { + self.options = { "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", ":00"]}}}, "values": { "transformation": { @@ -29,9 +30,9 @@ def setup_method(self, method): }, "step": {"transformation": {"cyclic": [0, 2]}}, } - self.xarraydatacube = XArrayDatacube(array) + self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) # @pytest.mark.skip(reason="Need date time to not be strings") def test_merge_axis(self): @@ -41,7 +42,6 @@ def test_merge_axis(self): Select("date", [date]), Span("step", 0, 3), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]) ) result = self.API.retrieve(request) - result.pprint() assert result.leaves[0].flatten()["date"] == np.datetime64("2000-01-01T06:00:00") for leaf in result.leaves: assert leaf.flatten()["step"] in [0, 1, 2, 3] diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index 9739a6c79..0a304e300 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -6,13 +6,13 @@ from polytope.shapes import Box, Select -class TestSlicing4DXarrayDatacube: +class TestSlicingMultipleTransformationsOneAxis: def setup_method(self, method): ds = data.from_source("file", "./tests/data/foo.grib") - latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) - latlon_array = latlon_array.t2m - self.xarraydatacube = XArrayDatacube(latlon_array) - grid_options = { + self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) + self.latlon_array = self.latlon_array.t2m + self.xarraydatacube = XArrayDatacube(self.latlon_array) + self.options = { "values": { "transformation": { "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} @@ -21,7 +21,7 @@ def setup_method(self, method): "longitude": {"transformation": {"cyclic": [0, 360.0]}}, } self.slicer = HullSlicer() - self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=grid_options) + self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) def test_merge_axis(self): request = Request( diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index a85d780eb..af3b0d79c 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -11,7 +11,7 @@ class TestMergeTransformation: def setup_method(self, method): # Create a dataarray with 4 labelled axes using different index types - array = xr.DataArray( + self.array = xr.DataArray( np.random.randn(1, 1), dims=("date", "time"), coords={ @@ -19,10 +19,10 @@ def setup_method(self, method): "time": ["06:00"], }, ) - options = {"date": {"transformation": {"merge": {"with": "time", "linkers": [" ", ":00"]}}}} - self.xarraydatacube = XArrayDatacube(array) + self.options = {"date": {"transformation": {"merge": {"with": "time", "linkers": [" ", ":00"]}}}} + self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() - self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) def test_merge_axis(self): request = Request(Select("date", [pd.Timestamp("2000-01-01T06:00:00")])) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 4746f3110..15177f4e7 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -10,10 +10,10 @@ class TestOctahedralGrid: def setup_method(self, method): ds = data.from_source("file", "./tests/data/foo.grib") - latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) - latlon_array = latlon_array.t2m - self.xarraydatacube = XArrayDatacube(latlon_array) - grid_options = { + self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) + self.latlon_array = self.latlon_array.t2m + self.xarraydatacube = XArrayDatacube(self.latlon_array) + self.options = { "values": { "transformation": { "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} @@ -21,7 +21,7 @@ def setup_method(self, method): } } self.slicer = HullSlicer() - self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=grid_options) + self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) def find_nearest_latlon(self, grib_file, target_lat, target_lon): # Open the GRIB file diff --git a/tests/test_reverse_transformation.py b/tests/test_reverse_transformation.py index 7b9151004..858f2f29f 100644 --- a/tests/test_reverse_transformation.py +++ b/tests/test_reverse_transformation.py @@ -7,7 +7,7 @@ from polytope.shapes import Select -class TestSlicing4DXarrayDatacube: +class TestSlicingReverseTransformation: def setup_method(self, method): # Create a dataarray with 4 labelled axes using different index types array = xr.DataArray( diff --git a/tests/test_slicing_unsliceable_axis.py b/tests/test_slicing_unsliceable_axis.py index 3f3bb1c63..9d02abf76 100644 --- a/tests/test_slicing_unsliceable_axis.py +++ b/tests/test_slicing_unsliceable_axis.py @@ -10,7 +10,7 @@ from polytope.utility.exceptions import UnsliceableShapeError -class TestSlicing3DXarrayDatacube: +class TestSlicingUnsliceableAxis: def setup_method(self, method): # create a dataarray with 3 labelled axes using different index types array = xr.DataArray( diff --git a/tests/test_type_change_transformation.py b/tests/test_type_change_transformation.py index fc7152452..80f5755aa 100644 --- a/tests/test_type_change_transformation.py +++ b/tests/test_type_change_transformation.py @@ -7,7 +7,7 @@ from polytope.shapes import Select -class TestMergeTransformation: +class TestTypeChangeTransformation: def setup_method(self, method): # Create a dataarray with 4 labelled axes using different index types array = xr.DataArray( From c5167afd9ebf5af173fe16c2df172067ab78ebcd Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 6 Sep 2023 11:09:24 +0200 Subject: [PATCH 128/332] black --- polytope/datacube/backends/FDB_datacube.py | 1 - polytope/datacube/backends/datacube.py | 10 +++--- polytope/datacube/datacube_axis.py | 11 +++--- .../datacube_transformations.py | 12 +++---- tests/test_cyclic_axis_slicing.py | 6 ++-- tests/test_datacube_axes_init.py | 36 +++++++++++-------- 6 files changed, 42 insertions(+), 34 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index a478811e8..564344a41 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -19,7 +19,6 @@ def update_fdb_dataarray(fdb_dataarray): class FDBDatacube(Datacube): - def __init__(self, config={}, axis_options={}): self.axis_options = axis_options self.grid_mapper = None diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index fe5b044b3..596a8038a 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -29,10 +29,12 @@ def validate(self, axes): def _create_axes(self, name, values, transformation_type_key, transformation_options): # first check what the final axes are for this axis name given transformations - final_axis_names = DatacubeAxisTransformation.get_final_axes(name, transformation_type_key, - transformation_options) - transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key, - transformation_options) + final_axis_names = DatacubeAxisTransformation.get_final_axes( + name, transformation_type_key, transformation_options + ) + transformation = DatacubeAxisTransformation.create_transform( + name, transformation_type_key, transformation_options + ) for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) for axis_name in final_axis_names: diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 245c2501a..5b9899670 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -158,6 +158,7 @@ def mapper(cls): from .transformations.datacube_mappers import DatacubeMapper if cls.has_mapper: + def find_indexes(path, datacube): # first, find the relevant transformation object that is a mapping in the cls.transformation dico for transform in cls.transformations: @@ -264,6 +265,7 @@ def merge(cls): from .transformations.datacube_merger import DatacubeAxisMerger if cls.has_merger: + def find_indexes(path, datacube): # first, find the relevant transformation object that is a mapping in the cls.transformation dico for transform in cls.transformations: @@ -333,6 +335,7 @@ def reverse(cls): from .transformations.datacube_reverse import DatacubeAxisReverse if cls.reorder: + def find_indexes(path, datacube): # first, find the relevant transformation object that is a mapping in the cls.transformation dico subarray = datacube.dataarray.sel(path, method="nearest") @@ -376,7 +379,6 @@ def type_change(cls): from .transformations.datacube_type_change import DatacubeAxisTypeChange if cls.type_change: - old_find_indexes = cls.find_indexes def find_indexes(path, datacube): @@ -537,7 +539,6 @@ def check_axis_type(name, values): @mapper @type_change class IntDatacubeAxis(DatacubeAxis): - def __init__(self): self.name = None self.tol = 1e-12 @@ -563,13 +564,12 @@ def serialize(self, value): @mapper @type_change class FloatDatacubeAxis(DatacubeAxis): - def __init__(self): self.name = None self.tol = 1e-12 self.range = None self.transformations = [] - self.type = 0. + self.type = 0.0 def parse(self, value: Any) -> Any: return float(value) @@ -586,7 +586,6 @@ def serialize(self, value): @merge class PandasTimestampDatacubeAxis(DatacubeAxis): - def __init__(self): self.name = None self.tol = 1e-12 @@ -617,7 +616,6 @@ def offset(self, value): @merge class PandasTimedeltaDatacubeAxis(DatacubeAxis): - def __init__(self): self.name = None self.tol = 1e-12 @@ -648,7 +646,6 @@ def offset(self, value): @type_change class UnsliceableDatacubeAxis(DatacubeAxis): - def __init__(self): self.name = None self.tol = float("NaN") diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index e266c5ce7..f4ce357a9 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -4,7 +4,6 @@ class DatacubeAxisTransformation(ABC): - @staticmethod def create_transform(name, transformation_type_key, transformation_options): transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] @@ -20,8 +19,9 @@ def create_transform(name, transformation_type_key, transformation_options): @staticmethod def get_final_axes(name, transformation_type_key, transformation_options): - new_transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key, - transformation_options) + new_transformation = DatacubeAxisTransformation.create_transform( + name, transformation_type_key, transformation_options + ) transformation_axis_names = new_transformation.transformation_axes_final() return transformation_axis_names @@ -62,8 +62,8 @@ def change_val_type(self, axis_name, values): has_transform = { "mapper": "has_mapper", - "cyclic" : "is_cyclic", - "merge" : "has_merger", + "cyclic": "is_cyclic", + "merge": "has_merger", "reverse": "reorder", - "type_change": "type_change" + "type_change": "type_change", } diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index ceaa40d13..7fbaf4f53 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -21,8 +21,10 @@ def setup_method(self, method): "long": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], }, ) - self.options = {"long": {"transformation": {"cyclic": [0, 1.0]}}, - "level": {"transformation": {"cyclic": [1, 129]}}} + self.options = { + "long": {"transformation": {"cyclic": [0, 1.0]}}, + "level": {"transformation": {"cyclic": [1, 129]}}, + } self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index db9b30544..1a108a4ba 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -29,27 +29,35 @@ def test_created_axes(self): assert self.datacube._axes["longitude"].has_mapper assert type(self.datacube._axes["longitude"]) == FloatDatacubeAxis assert not ("values" in self.datacube._axes.keys()) - assert self.datacube._axes["latitude"].find_indexes({}, - self.datacube)[:5] == [89.94618771566562, - 89.87647835333229, - 89.80635731954224, - 89.73614327160958, - 89.6658939412157] - assert self.datacube._axes["longitude"].find_indexes({"latitude": 89.94618771566562}, - self.datacube)[:8] == [0.0, 18.0, 36.0, 54.0, - 72.0, 90.0, 108.0, 126.0] - assert len(self.datacube._axes["longitude"].find_indexes({"latitude": 89.94618771566562}, - self.datacube)) == 20 + assert self.datacube._axes["latitude"].find_indexes({}, self.datacube)[:5] == [ + 89.94618771566562, + 89.87647835333229, + 89.80635731954224, + 89.73614327160958, + 89.6658939412157, + ] + assert self.datacube._axes["longitude"].find_indexes({"latitude": 89.94618771566562}, self.datacube)[:8] == [ + 0.0, + 18.0, + 36.0, + 54.0, + 72.0, + 90.0, + 108.0, + 126.0, + ] + assert len(self.datacube._axes["longitude"].find_indexes({"latitude": 89.94618771566562}, self.datacube)) == 20 lon_ax = self.datacube._axes["longitude"] lat_ax = self.datacube._axes["latitude"] (path, unmapped_path) = lat_ax.unmap_to_datacube({"latitude": 89.94618771566562}, {}) assert path == {} - assert unmapped_path == {'latitude': 89.94618771566562} + assert unmapped_path == {"latitude": 89.94618771566562} (path, unmapped_path) = lon_ax.unmap_to_datacube({"longitude": 0.0}, {"latitude": 89.94618771566562}) assert path == {} assert unmapped_path == {"values": 0} - assert lat_ax.find_indices_between([[89.94618771566562, 89.87647835333229]], - 89.87, 90, self.datacube) == [[89.94618771566562, 89.87647835333229]] + assert lat_ax.find_indices_between([[89.94618771566562, 89.87647835333229]], 89.87, 90, self.datacube) == [ + [89.94618771566562, 89.87647835333229] + ] def test_mapper_transformation_request(self): request = Request( From bb3ce15e129a19bab49739a0d5ebccda978b2dcd Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 6 Sep 2023 11:41:48 +0200 Subject: [PATCH 129/332] add pyfdb to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 50594c4fc..7d14ef2be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ sortedcontainers==2.4.0 tripy==1.0.0 typing==3.7.4.3 xarray==2022.12.0 +pyfdb==0.0.3 From bb830f38d8ee5bab3c7a4e3e89461ad389983979 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 6 Sep 2023 11:42:43 +0200 Subject: [PATCH 130/332] add pyfdb to requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7d14ef2be..8ff6ac4b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ decorator==5.1.1 numpy==1.23.5 pandas==1.5.2 +pyfdb==0.0.3 pypi==2.1 requests==2.28.1 scipy==1.9.3 @@ -8,4 +9,3 @@ sortedcontainers==2.4.0 tripy==1.0.0 typing==3.7.4.3 xarray==2022.12.0 -pyfdb==0.0.3 From 6a980e99110f92f129eccddd8e8cfdff4a5c4c41 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 6 Sep 2023 12:15:45 +0200 Subject: [PATCH 131/332] remove specific FDB location --- polytope/datacube/backends/FDB_datacube.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 564344a41..846ac840e 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -1,12 +1,8 @@ -import os from copy import deepcopy -from .datacube import Datacube, IndexTree +import pyfdb -# TODO: probably need to do this more general... -os.environ["DYLD_LIBRARY_PATH"] = "/Users/male/build/fdb-bundle/lib" -os.environ["FDB_HOME"] = "/Users/male/git/fdb-home" -import pyfdb # noqa: E402 +from .datacube import Datacube, IndexTree def glue(path, unmap_path): From 88f972bea46679d2f39eae951254cdff25fe4d3d Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 6 Sep 2023 12:22:01 +0200 Subject: [PATCH 132/332] add github CI --- .github/workflows/ci.yaml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4b2f16b72..43b2d82b7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,25 +1,19 @@ name: ci - on: # Trigger the workflow on push to master or develop, except tag creation push: branches: - 'main' - 'develop' - # Trigger the workflow on pull request pull_request: ~ - # Trigger the workflow manually workflow_dispatch: ~ - # Trigger after public PR approved for CI pull_request_target: types: [labeled] - release: types: [created] - jobs: qa: name: qa @@ -48,7 +42,6 @@ jobs: - name: Check flake8 run: flake8 . - setup: name: setup runs-on: ubuntu-20.04 @@ -95,7 +88,6 @@ jobs: run: | echo inputs=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '.' --output-format json --indent 0 -) >> $GITHUB_OUTPUT echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-20.04","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT - test: name: test needs: @@ -150,16 +142,11 @@ jobs: uses: codecov/codecov-action@v3 with: files: coverage.xml - deploy: needs: test - if: ${{ github.event_name == 'release' }} - name: Upload to Pypi - runs-on: ubuntu-latest - steps: - uses: actions/checkout@v3 - name: Set up Python @@ -176,4 +163,4 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python setup.py sdist - twine upload dist/* + twine upload dist/* \ No newline at end of file From 7199fc1688d3075705b696bf3901d1ac62a2a444 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 6 Sep 2023 15:35:46 +0200 Subject: [PATCH 133/332] remove pyfdb --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8ff6ac4b7..50594c4fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ decorator==5.1.1 numpy==1.23.5 pandas==1.5.2 -pyfdb==0.0.3 pypi==2.1 requests==2.28.1 scipy==1.9.3 From 7b36653058b18f5fc12d00985f3e07ddd84ed2f8 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 6 Sep 2023 15:38:00 +0200 Subject: [PATCH 134/332] skip fdb test --- tests/test_fdb_datacube.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 8614ad3d5..7d364864e 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -1,4 +1,5 @@ import pandas as pd +import pytest from polytope.datacube.backends.FDB_datacube import FDBDatacube from polytope.engine.hullslicer import HullSlicer @@ -24,7 +25,8 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - def test_2D_box(self): + @pytest.mark.skip(reason="can't install fdb branch on CI") + def test_fdb_datacube(self): request = Request( Select("step", [11]), Select("levtype", ["sfc"]), From 38c40a7d5926197a2c390653ad2962f349e4b53b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 6 Sep 2023 15:39:22 +0200 Subject: [PATCH 135/332] flake8 --- tests/test_datacube_axes_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 1a108a4ba..77f3bf88a 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -27,7 +27,7 @@ def setup_method(self, method): def test_created_axes(self): assert self.datacube._axes["latitude"].has_mapper assert self.datacube._axes["longitude"].has_mapper - assert type(self.datacube._axes["longitude"]) == FloatDatacubeAxis + assert isinstance(self.datacube._axes["longitude"], FloatDatacubeAxis) assert not ("values" in self.datacube._axes.keys()) assert self.datacube._axes["latitude"].find_indexes({}, self.datacube)[:5] == [ 89.94618771566562, From 1d9c035bcfd89914a5257ad393cdbc8cbf6e0c61 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 6 Sep 2023 15:43:51 +0200 Subject: [PATCH 136/332] make tests work --- tests/test_mappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_mappers.py b/tests/test_mappers.py index 1250a02db..b9a39477f 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -1,4 +1,4 @@ -from polytope.datacube.datacube_mappers import OctahedralGridMapper +from polytope.datacube.transformations.datacube_mappers import OctahedralGridMapper class TestMapper: From abc896b4e4925ec63306d7f587048ce749da177c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Sep 2023 10:31:08 +0200 Subject: [PATCH 137/332] make surrounding method work with the cyclic axis --- polytope/datacube/backends/datacube.py | 11 ----------- polytope/datacube/datacube_axis.py | 14 ++++---------- polytope/engine/hullslicer.py | 3 +-- tests/test_cyclic_simple.py | 2 +- tests/test_merge_octahedral_one_axis.py | 3 ++- tests/test_snapping.py | 10 ++++++++++ tests/test_snapping_real_data.py | 4 ++-- 7 files changed, 20 insertions(+), 27 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 0bc710c93..8510a68f9 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -1,5 +1,4 @@ import importlib -import itertools import math from abc import ABC, abstractmethod from typing import Any @@ -101,15 +100,10 @@ def get_indices(self, path: DatacubePath, axis, lower, upper, method=None): if offset is not None: # Note that we can only do unique if not dealing with time values idx_between = unique(idx_between) - print("inside get indices") - print(idx_between) return idx_between def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, method): - print("inside look up datacube") idx_between = [] - print(axis.name) - print(search_ranges) for i in range(len(search_ranges)): r = search_ranges[i] offset = search_ranges_offset[i] @@ -142,8 +136,6 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) idx_between.append(indexes_between[j][k]) - print("indexes between") - print(idx_between) return idx_between def get_mapper(self, axis): @@ -157,9 +149,6 @@ def remap_path(self, path: DatacubePath): value = path[key] path[key] = self._axes[key].remap([value, value])[0][0] return path - - # def remap_tree(self, tree: IndexTree): - # for @staticmethod def create(datacube, axis_options: dict): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 4c7a66d50..3efa48573 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -146,6 +146,7 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): return old_find_indices_between(index_ranges, low, up, datacube, method, offset) if offset != 0: + # print("HERE OFFSET NOT 0") # NOTE that if the offset is not 0, then we need to recenter the low and up # values to be within the datacube range new_offset = 0 @@ -154,21 +155,17 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): # low = low - range_length # new_offset -= range_length # up = up - range_length - print("INSIDE THE DATACUBE AXIS DECORATOR") - print((low, up)) if method == "surrounding": for indexes in index_ranges: if cls.name in datacube.complete_axes: start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") - print(start, end) if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) start_offset_indicator = "need_offset" - index_val_found = indexes[-2:][0] + index_val_found = indexes[-1:][0] indexes_between_before = [start_offset_indicator, new_offset, index_val_found] indexes_between_ranges.append(indexes_between_before) if end+1 > len(indexes): - print("end exceeded indices length") start_offset_indicator = "need_offset" index_val_found = indexes[:2][1] indexes_between_after = [start_offset_indicator, new_offset, index_val_found] @@ -200,20 +197,17 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): return old_find_indices_between(index_ranges, low, up, datacube, method, offset) else: + # print("HERE OFFSET IS 0") # If the offset is 0, then the first value found on the left has an offset of range_length new_offset = 0 if method == "surrounding": - print("no offset") for indexes in index_ranges: if cls.name in datacube.complete_axes: start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") - print((start, end)) if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) - print("low start") start_offset_indicator = "need_offset" - index_val_found = indexes[-2:][0] - print(index_val_found) + index_val_found = indexes[-1:][0] new_offset = -range_length indexes_between_before = [start_offset_indicator, new_offset, index_val_found] indexes_between_ranges.append(indexes_between_before) diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 5553f4e5a..39a88d350 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -1,7 +1,7 @@ +import math from copy import copy from itertools import chain from typing import List -import math import scipy.spatial @@ -58,7 +58,6 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex if ax.is_cyclic: remapped_val = (ax.remap([value, value])[0][0] + ax.remap([value, value])[0][1])/2 remapped_val = round(remapped_val, int(-math.log10(ax.tol))) - print(remapped_val) # child = node.create_child(ax, value) child = node.create_child(ax, remapped_val) child["unsliced_polytopes"] = copy(node["unsliced_polytopes"]) diff --git a/tests/test_cyclic_simple.py b/tests/test_cyclic_simple.py index ec213e2c8..f7dda6dd8 100644 --- a/tests/test_cyclic_simple.py +++ b/tests/test_cyclic_simple.py @@ -63,4 +63,4 @@ def test_cyclic_float_surrounding_below_seam(self): for leaf in result.leaves: path = leaf.flatten() lon_val = path["long"] - assert lon_val in [0., 0.1, 0.9] + assert lon_val in [0., 0.1, 0.9, 1.] diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index 9739a6c79..aec9ee7c6 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -34,4 +34,5 @@ def test_merge_axis(self): ) result = self.API.retrieve(request) # result.pprint() - assert result.leaves[2].flatten()["longitude"] == 360.0 + assert result.leaves[-1].flatten()["longitude"] == 360.0 + assert result.leaves[0].flatten()["longitude"] == 0.070093457944 diff --git a/tests/test_snapping.py b/tests/test_snapping.py index 7d127a877..83d472203 100644 --- a/tests/test_snapping.py +++ b/tests/test_snapping.py @@ -77,3 +77,13 @@ def test_1D_nonexisting_point_v2(self): result.pprint() assert len(result.leaves) == 1 assert result.is_root() + + def test_1D_nonexisting_point_surrounding(self): + request = Request(Select("level", [0], method="surrounding"), Select("step", [6], method="surrounding")) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + for leaf in result.leaves: + path = leaf.flatten() + assert path["level"] == 1 + assert path["step"] == 5 diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index 247a276b8..bfe32b21d 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -21,7 +21,7 @@ def setup_method(self, method): def test_surrounding_on_grid_point(self): requested_lat = 0 - requested_lon = 0 + requested_lon = -720 request = Request( Box(["number", "isobaricInhPa"], [6, 500.0], [6, 850.0]), Select("time", ["2017-01-02T12:00:00"]), @@ -58,4 +58,4 @@ def test_surrounding_on_grid_point(self): # plt.colorbar(label="Temperature") # plt.show() for lon in longs: - assert lon in [-3, 0, 3] + assert lon in [357, 0, 3] From c03b7a5686dee321d3bf8c7932bd6fad8a900b17 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Sep 2023 10:37:23 +0200 Subject: [PATCH 138/332] clean up --- polytope/datacube/datacube_axis.py | 59 +----------------------------- 1 file changed, 2 insertions(+), 57 deletions(-) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 3efa48573..5467ec60a 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -139,22 +139,15 @@ def unmap_to_datacube(path, unmapped_path): def find_indices_between(index_ranges, low, up, datacube, offset, method=None): update_range() - range_length = cls.range[1] - cls.range[0] indexes_between_ranges = [] if method != "surrounding": return old_find_indices_between(index_ranges, low, up, datacube, method, offset) - if offset != 0: - # print("HERE OFFSET NOT 0") + if True: # NOTE that if the offset is not 0, then we need to recenter the low and up # values to be within the datacube range new_offset = 0 - # if low <= cls.range[0] + cls.tol: - # while low >= cls.range[0] - cls.tol: - # low = low - range_length - # new_offset -= range_length - # up = up - range_length if method == "surrounding": for indexes in index_ranges: if cls.name in datacube.complete_axes: @@ -185,7 +178,7 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): indexes_between_ranges.append(indexes_between_before) if end+1 > len(indexes): start_offset_indicator = "need_offset" - index_val_found = indexes[:1][0] + index_val_found = indexes[:2][0] indexes_between_after = [start_offset_indicator, new_offset, index_val_found] indexes_between_ranges.append(indexes_between_after) start = max(start-1, 0) @@ -193,54 +186,6 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) return indexes_between_ranges - else: - return old_find_indices_between(index_ranges, low, up, datacube, method, offset) - - else: - # print("HERE OFFSET IS 0") - # If the offset is 0, then the first value found on the left has an offset of range_length - new_offset = 0 - if method == "surrounding": - for indexes in index_ranges: - if cls.name in datacube.complete_axes: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) - start_offset_indicator = "need_offset" - index_val_found = indexes[-1:][0] - new_offset = -range_length - indexes_between_before = [start_offset_indicator, new_offset, index_val_found] - indexes_between_ranges.append(indexes_between_before) - if end+1 > len(indexes): - start_offset_indicator = "need_offset" - index_val_found = indexes[:2][1] - indexes_between_after = [start_offset_indicator, new_offset, index_val_found] - indexes_between_ranges.append(indexes_between_after) - start = max(start-1, 0) - end = min(end+1, len(indexes)) - indexes_between = indexes[start:end].to_list() - indexes_between = [i for i in indexes_between] - indexes_between_ranges.append(indexes_between) - else: - start = indexes.index(low) - end = indexes.index(up) - if start-1 < 0: - start_offset_indicator = "need_offset" - index_val_found = indexes[-1:][0] - indexes_between_before = [start_offset_indicator, new_offset, index_val_found] - indexes_between_ranges.append(indexes_between_before) - if end+1 > len(indexes): - start_offset_indicator = "need_offset" - index_val_found = indexes[:1][0] - indexes_between_after = [start_offset_indicator, new_offset, index_val_found] - indexes_between_ranges.append(indexes_between_after) - start = max(start-1, 0) - end = min(end+1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - else: - return old_find_indices_between(index_ranges, low, up, datacube, method, offset) def offset(range): # We first unpad the range by the axis tolerance to make sure that From 78306d00020bd260a4a2ada7532a270c946d50c5 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Sep 2023 10:50:26 +0200 Subject: [PATCH 139/332] refactor find_indices_between for cyclic axis --- polytope/datacube/datacube_axis.py | 68 ++++++++++++------------------ 1 file changed, 26 insertions(+), 42 deletions(-) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 5467ec60a..80d6920e1 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -143,49 +143,33 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): if method != "surrounding": return old_find_indices_between(index_ranges, low, up, datacube, method, offset) - - if True: - # NOTE that if the offset is not 0, then we need to recenter the low and up - # values to be within the datacube range + else: new_offset = 0 - if method == "surrounding": - for indexes in index_ranges: - if cls.name in datacube.complete_axes: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - if start-1 < 0: # NOTE TODO: here the boundaries will not necessarily be 0 or len(indexes) - start_offset_indicator = "need_offset" - index_val_found = indexes[-1:][0] - indexes_between_before = [start_offset_indicator, new_offset, index_val_found] - indexes_between_ranges.append(indexes_between_before) - if end+1 > len(indexes): - start_offset_indicator = "need_offset" - index_val_found = indexes[:2][1] - indexes_between_after = [start_offset_indicator, new_offset, index_val_found] - indexes_between_ranges.append(indexes_between_after) - start = max(start-1, 0) - end = min(end+1, len(indexes)) - indexes_between = indexes[start:end].to_list() - indexes_between = [i - new_offset for i in indexes_between] - indexes_between_ranges.append(indexes_between) - else: - start = indexes.index(low) - end = indexes.index(up) - if start-1 < 0: - start_offset_indicator = "need_offset" - index_val_found = indexes[-1:][0] - indexes_between_before = [start_offset_indicator, new_offset, index_val_found] - indexes_between_ranges.append(indexes_between_before) - if end+1 > len(indexes): - start_offset_indicator = "need_offset" - index_val_found = indexes[:2][0] - indexes_between_after = [start_offset_indicator, new_offset, index_val_found] - indexes_between_ranges.append(indexes_between_after) - start = max(start-1, 0) - end = min(end+1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges + for indexes in index_ranges: + if cls.name in datacube.complete_axes: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + else: + start = indexes.index(low) + end = indexes.index(up) + if start-1 < 0: + start_offset_indicator = "need_offset" + index_val_found = indexes[-1:][0] + indexes_between_before = [start_offset_indicator, new_offset, index_val_found] + indexes_between_ranges.append(indexes_between_before) + if end+1 > len(indexes): + start_offset_indicator = "need_offset" + index_val_found = indexes[:2][0] + indexes_between_after = [start_offset_indicator, new_offset, index_val_found] + indexes_between_ranges.append(indexes_between_after) + start = max(start-1, 0) + end = min(end+1, len(indexes)) + if cls.name in datacube.complete_axes: + indexes_between = indexes[start:end].to_list() + else: + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges def offset(range): # We first unpad the range by the axis tolerance to make sure that From f04ded57f93c06e01c963ca17a3221b663af3030 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Sep 2023 10:58:22 +0200 Subject: [PATCH 140/332] refactor the surrounding method code in datacube.py _look_up_datacube and find_indices_between in cyclic datacube axis decorator --- polytope/datacube/backends/datacube.py | 27 +++++++------------------- polytope/datacube/datacube_axis.py | 23 +++++++++------------- 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 8510a68f9..96554550f 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -109,7 +109,7 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, offset = search_ranges_offset[i] low = r[0] up = r[1] - indexes_between = axis.find_indices_between([indexes], low, up, self, offset, method) + indexes_between = axis.find_indices_between([indexes], low, up, self, method) # Now the indexes_between are values on the cyclic range so need to remap them to their original # values before returning them for j in range(len(indexes_between)): @@ -117,25 +117,12 @@ def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, if len(indexes_between[j]) == 0: idx_between = idx_between else: - if indexes_between[j][0] == "need_offset": - for k in range(2, len(indexes_between[j])): - if offset is None: - indexes_between[j][k] = indexes_between[j][k] - else: - offset = offset - indexes_between[j][k] = round(indexes_between[j][k] + offset, - int(-math.log10(axis.tol))) - idx_between.append(indexes_between[j][k]) - else: - # do normal offset if no new offset - for k in range(len(indexes_between[j])): - if offset is None: - indexes_between[j][k] = indexes_between[j][k] - else: - offset = offset - indexes_between[j][k] = round(indexes_between[j][k] + offset, - int(-math.log10(axis.tol))) - idx_between.append(indexes_between[j][k]) + for k in range(len(indexes_between[j])): + if offset is None: + indexes_between[j][k] = indexes_between[j][k] + else: + indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) + idx_between.append(indexes_between[j][k]) return idx_between def get_mapper(self, axis): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 80d6920e1..736349d22 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -137,14 +137,13 @@ def unmap_to_datacube(path, unmapped_path): old_find_indices_between = cls.find_indices_between - def find_indices_between(index_ranges, low, up, datacube, offset, method=None): + def find_indices_between(index_ranges, low, up, datacube, method=None): update_range() indexes_between_ranges = [] if method != "surrounding": - return old_find_indices_between(index_ranges, low, up, datacube, method, offset) + return old_find_indices_between(index_ranges, low, up, datacube, method) else: - new_offset = 0 for indexes in index_ranges: if cls.name in datacube.complete_axes: start = indexes.searchsorted(low, "left") @@ -153,15 +152,11 @@ def find_indices_between(index_ranges, low, up, datacube, offset, method=None): start = indexes.index(low) end = indexes.index(up) if start-1 < 0: - start_offset_indicator = "need_offset" index_val_found = indexes[-1:][0] - indexes_between_before = [start_offset_indicator, new_offset, index_val_found] - indexes_between_ranges.append(indexes_between_before) + indexes_between_ranges.append([index_val_found]) if end+1 > len(indexes): - start_offset_indicator = "need_offset" index_val_found = indexes[:2][0] - indexes_between_after = [start_offset_indicator, new_offset, index_val_found] - indexes_between_ranges.append(indexes_between_after) + indexes_between_ranges.append([index_val_found]) start = max(start-1, 0) end = min(end+1, len(indexes)) if cls.name in datacube.complete_axes: @@ -269,7 +264,7 @@ def unmap_total_path_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, offset, method=None): + def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: @@ -352,7 +347,7 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, offset, method=None): + def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: @@ -408,7 +403,7 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, offset, method=None): + def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: @@ -517,7 +512,7 @@ def unmap_to_datacube(path, unmapped_path): def remap_to_requested(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, offset, method=None): + def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: @@ -622,7 +617,7 @@ def unmap_total_path_to_datacube(self, path, unmapped_path): def remap_to_requeest(path, unmapped_path): return (path, unmapped_path) - def find_indices_between(self, index_ranges, low, up, datacube, offset, method=None): + def find_indices_between(self, index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] for indexes in index_ranges: From f82c8e84c03f5a24337b8fbdaebaa4e068560985 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Sep 2023 11:01:59 +0200 Subject: [PATCH 141/332] black --- polytope/datacube/backends/FDB_datacube.py | 1 - polytope/datacube/backends/datacube.py | 10 +++-- polytope/datacube/datacube_axis.py | 42 ++++++++++--------- polytope/datacube/index_tree.py | 4 +- .../datacube_transformations.py | 12 +++--- polytope/engine/hullslicer.py | 2 +- polytope/shapes.py | 1 - tests/test_cyclic_axis_over_negative_vals.py | 7 ++-- tests/test_cyclic_axis_slicer_not_0.py | 12 +----- tests/test_cyclic_axis_slicing.py | 4 +- tests/test_cyclic_simple.py | 12 +++--- tests/test_cyclic_snapping.py | 4 +- tests/test_datacube_axes_init.py | 36 +++++++++------- tests/test_snapping_real_data.py | 6 ++- 14 files changed, 76 insertions(+), 77 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 46e2c673c..764ae56c8 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -19,7 +19,6 @@ def update_fdb_dataarray(fdb_dataarray): class FDBDatacube(Datacube): - def __init__(self, config={}, axis_options={}): self.axis_options = axis_options self.grid_mapper = None diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 96554550f..37e7faa8f 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -29,10 +29,12 @@ def validate(self, axes): def _create_axes(self, name, values, transformation_type_key, transformation_options): # first check what the final axes are for this axis name given transformations - final_axis_names = DatacubeAxisTransformation.get_final_axes(name, transformation_type_key, - transformation_options) - transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key, - transformation_options) + final_axis_names = DatacubeAxisTransformation.get_final_axes( + name, transformation_type_key, transformation_options + ) + transformation = DatacubeAxisTransformation.create_transform( + name, transformation_type_key, transformation_options + ) for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) for axis_name in final_axis_names: diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 736349d22..9582bd0b5 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -151,14 +151,14 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): else: start = indexes.index(low) end = indexes.index(up) - if start-1 < 0: + if start - 1 < 0: index_val_found = indexes[-1:][0] indexes_between_ranges.append([index_val_found]) - if end+1 > len(indexes): + if end + 1 > len(indexes): index_val_found = indexes[:2][0] indexes_between_ranges.append([index_val_found]) - start = max(start-1, 0) - end = min(end+1, len(indexes)) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) if cls.name in datacube.complete_axes: indexes_between = indexes[start:end].to_list() else: @@ -190,6 +190,7 @@ def mapper(cls): from .transformations.datacube_mappers import DatacubeMapper if cls.has_mapper: + def find_indexes(path, datacube): # first, find the relevant transformation object that is a mapping in the cls.transformation dico for transform in cls.transformations: @@ -275,8 +276,8 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): if method == "surrounding": start = idxs.index(low) end = idxs.index(up) - start = max(start-1, 0) - end = min(end+1, len(idxs)) + start = max(start - 1, 0) + end = min(end + 1, len(idxs)) # indexes_between = [i for i in indexes if low <= i <= up] indexes_between = idxs[start:end] indexes_between_ranges.append(indexes_between) @@ -306,6 +307,7 @@ def merge(cls): from .transformations.datacube_merger import DatacubeAxisMerger if cls.has_merger: + def find_indexes(path, datacube): # first, find the relevant transformation object that is a mapping in the cls.transformation dico for transform in cls.transformations: @@ -358,8 +360,8 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): if method == "surrounding": start = indexes.index(low) end = indexes.index(up) - start = max(start-1, 0) - end = min(end+1, len(indexes)) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) # indexes_between = [i for i in indexes if low <= i <= up] indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) @@ -387,6 +389,7 @@ def reverse(cls): from .transformations.datacube_reverse import DatacubeAxisReverse if cls.reorder: + def find_indexes(path, datacube): # first, find the relevant transformation object that is a mapping in the cls.transformation dico subarray = datacube.dataarray.sel(path, method="nearest") @@ -419,8 +422,8 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): if method == "surrounding": start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") - start = max(start-1, 0) - end = min(end+1, len(indexes)) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) indexes_between = indexes[start:end].to_list() indexes_between_ranges.append(indexes_between) else: @@ -432,8 +435,8 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): if method == "surrounding": start = indexes.index(low) end = indexes.index(up) - start = max(start-1, 0) - end = min(end+1, len(indexes)) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) # indexes_between = [i for i in indexes if low <= i <= up] indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) @@ -471,7 +474,6 @@ def type_change(cls): from .transformations.datacube_type_change import DatacubeAxisTypeChange if cls.type_change: - old_find_indexes = cls.find_indexes def find_indexes(path, datacube): @@ -523,8 +525,8 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): if method == "surrounding": start = indexes.index(low) end = indexes.index(up) - start = max(start-1, 0) - end = min(end+1, len(indexes)) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) # indexes_between = [i for i in indexes if low <= i <= up] indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) @@ -628,8 +630,8 @@ def find_indices_between(self, index_ranges, low, up, datacube, method=None): if method == "surrounding": start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") - start = max(start-1, 0) - end = min(end+1, len(indexes)) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) indexes_between = indexes[start:end].to_list() indexes_between_ranges.append(indexes_between) else: @@ -641,8 +643,8 @@ def find_indices_between(self, index_ranges, low, up, datacube, method=None): if method == "surrounding": start = indexes.index(low) end = indexes.index(up) - start = max(start-1, 0) - end = min(end+1, len(indexes)) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) # indexes_between = [i for i in indexes if low <= i <= up] indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) @@ -699,7 +701,7 @@ class FloatDatacubeAxis(DatacubeAxis): tol = 1e-12 range = None transformations = [] - type = 0. + type = 0.0 def parse(self, value: Any) -> Any: return float(value) diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index 2d285327e..b7a37aff5 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -63,9 +63,9 @@ def __eq__(self, other): else: if other.value == self.value: return True - if other.value - 2*other.axis.tol <= self.value <= other.value + 2*other.axis.tol: + if other.value - 2 * other.axis.tol <= self.value <= other.value + 2 * other.axis.tol: return True - elif self.value - 2*self.axis.tol <= other.value <= self.value + 2*self.axis.tol: + elif self.value - 2 * self.axis.tol <= other.value <= self.value + 2 * self.axis.tol: return True else: return False diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index d0af6dcec..4afac6ce8 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -4,7 +4,6 @@ class DatacubeAxisTransformation(ABC): - @staticmethod def create_transform(name, transformation_type_key, transformation_options): transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] @@ -20,8 +19,9 @@ def create_transform(name, transformation_type_key, transformation_options): @staticmethod def get_final_axes(name, transformation_type_key, transformation_options): - new_transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key, - transformation_options) + new_transformation = DatacubeAxisTransformation.create_transform( + name, transformation_type_key, transformation_options + ) transformation_axis_names = new_transformation.transformation_axes_final() return transformation_axis_names @@ -102,8 +102,8 @@ def change_val_type(self, axis_name, values): has_transform = { "mapper": "has_mapper", - "cyclic" : "is_cyclic", - "merge" : "has_merger", + "cyclic": "is_cyclic", + "merge": "has_merger", "reverse": "reorder", - "type_change": "type_change" + "type_change": "type_change", } diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 39a88d350..5ea0c951e 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -56,7 +56,7 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex # remapped_val = (ax.remap([value, value])[0][0] + ax.remap([value, value])[0][1])/2 remapped_val = value if ax.is_cyclic: - remapped_val = (ax.remap([value, value])[0][0] + ax.remap([value, value])[0][1])/2 + remapped_val = (ax.remap([value, value])[0][0] + ax.remap([value, value])[0][1]) / 2 remapped_val = round(remapped_val, int(-math.log10(ax.tol))) # child = node.create_child(ax, value) child = node.create_child(ax, remapped_val) diff --git a/polytope/shapes.py b/polytope/shapes.py index e0d0b0bd4..94562abc5 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -23,7 +23,6 @@ def axes(self) -> List[str]: class ConvexPolytope(Shape): - def __init__(self, axes, points, method=None): self._axes = list(axes) self.points = points diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index a67333cce..ccb0bbaa0 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -58,7 +58,7 @@ def test_cyclic_float_axis_across_seam(self): -0.5, -0.4, -0.3, - -0.2 + -0.2, ] def test_cyclic_float_axis_inside_cyclic_range(self): @@ -94,8 +94,7 @@ def test_cyclic_float_axis_above_axis_range(self): result = self.API.retrieve(request) # result.pprint() assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, - -0.7, -0.6, -0.5, -0.4, -0.3] + assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, -0.3] def test_cyclic_float_axis_two_range_loops(self): request = Request( @@ -124,7 +123,7 @@ def test_cyclic_float_axis_two_range_loops(self): -0.5, -0.4, -0.3, - -0.2 + -0.2, ] def test_cyclic_float_axis_below_axis_range(self): diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 33ef062e6..daba54a86 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -92,17 +92,7 @@ def test_cyclic_float_axis_above_axis_range(self): ) result = self.API.retrieve(request) assert len(result.leaves) == 10 - assert [leaf.value for leaf in result.leaves] == [ - -0.7, - -0.6, - -0.5, - -0.4, - -0.3, - -0.7, - -0.6, - -0.5, - -0.4, - -0.3] + assert [leaf.value for leaf in result.leaves] == [-0.7, -0.6, -0.5, -0.4, -0.3, -0.7, -0.6, -0.5, -0.4, -0.3] def test_cyclic_float_axis_two_range_loops(self): request = Request( diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index 3d3c68465..554ae7a85 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -45,7 +45,7 @@ def test_cyclic_float_axis_across_seam(self): 0.7, 0.8, 0.9, - 1., + 1.0, 0.1, 0.2, 0.3, @@ -55,7 +55,7 @@ def test_cyclic_float_axis_across_seam(self): 0.7, 0.8, 0.9, - 1., + 1.0, ] def test_cyclic_float_axis_across_seam_repeated(self): diff --git a/tests/test_cyclic_simple.py b/tests/test_cyclic_simple.py index f7dda6dd8..bf6660ee7 100644 --- a/tests/test_cyclic_simple.py +++ b/tests/test_cyclic_simple.py @@ -40,27 +40,27 @@ def test_cyclic_float_axis_across_seam(self): def test_cyclic_float_surrounding(self): request = Request( Select("step", [0]), - Select("long", [1.], method="surrounding"), + Select("long", [1.0], method="surrounding"), Select("date", ["2000-01-01"]), - Select("level", [128]) + Select("level", [128]), ) result = self.API.retrieve(request) result.pprint() for leaf in result.leaves: path = leaf.flatten() lon_val = path["long"] - assert lon_val in [0., 0.1, 0.9, 1.] + assert lon_val in [0.0, 0.1, 0.9, 1.0] def test_cyclic_float_surrounding_below_seam(self): request = Request( Select("step", [0]), - Select("long", [0.], method="surrounding"), + Select("long", [0.0], method="surrounding"), Select("date", ["2000-01-01"]), - Select("level", [128]) + Select("level", [128]), ) result = self.API.retrieve(request) result.pprint() for leaf in result.leaves: path = leaf.flatten() lon_val = path["long"] - assert lon_val in [0., 0.1, 0.9, 1.] + assert lon_val in [0.0, 0.1, 0.9, 1.0] diff --git a/tests/test_cyclic_snapping.py b/tests/test_cyclic_snapping.py index b0a6580ba..ab6189ba3 100644 --- a/tests/test_cyclic_snapping.py +++ b/tests/test_cyclic_snapping.py @@ -24,9 +24,7 @@ def setup_method(self, method): # Testing different shapes def test_cyclic_float_axis_across_seam(self): - request = Request( - Select("long", [-0.2], method="surrounding") - ) + request = Request(Select("long", [-0.2], method="surrounding")) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 2 diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index bea08086b..decee5bb5 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -29,27 +29,35 @@ def test_created_axes(self): assert self.datacube._axes["longitude"].has_mapper assert type(self.datacube._axes["longitude"]) == FloatDatacubeAxis assert not ("values" in self.datacube._axes.keys()) - assert self.datacube._axes["latitude"].find_indexes({}, - self.datacube)[:5] == [89.94618771566562, - 89.87647835333229, - 89.80635731954224, - 89.73614327160958, - 89.6658939412157] - assert self.datacube._axes["longitude"].find_indexes({"latitude": 89.94618771566562}, - self.datacube)[:8] == [0.0, 18.0, 36.0, 54.0, - 72.0, 90.0, 108.0, 126.0] - assert len(self.datacube._axes["longitude"].find_indexes({"latitude": 89.94618771566562}, - self.datacube)) == 20 + assert self.datacube._axes["latitude"].find_indexes({}, self.datacube)[:5] == [ + 89.94618771566562, + 89.87647835333229, + 89.80635731954224, + 89.73614327160958, + 89.6658939412157, + ] + assert self.datacube._axes["longitude"].find_indexes({"latitude": 89.94618771566562}, self.datacube)[:8] == [ + 0.0, + 18.0, + 36.0, + 54.0, + 72.0, + 90.0, + 108.0, + 126.0, + ] + assert len(self.datacube._axes["longitude"].find_indexes({"latitude": 89.94618771566562}, self.datacube)) == 20 lon_ax = self.datacube._axes["longitude"] lat_ax = self.datacube._axes["latitude"] (path, unmapped_path) = lat_ax.unmap_to_datacube({"latitude": 89.94618771566562}, {}) assert path == {} - assert unmapped_path == {'latitude': 89.94618771566562} + assert unmapped_path == {"latitude": 89.94618771566562} (path, unmapped_path) = lon_ax.unmap_to_datacube({"longitude": 0.0}, {"latitude": 89.94618771566562}) assert path == {} assert unmapped_path == {"values": 0} - assert lat_ax.find_indices_between([[89.94618771566562, 89.87647835333229]], - 89.87, 90, self.datacube, 0) == [[89.94618771566562, 89.87647835333229]] + assert lat_ax.find_indices_between([[89.94618771566562, 89.87647835333229]], 89.87, 90, self.datacube, 0) == [ + [89.94618771566562, 89.87647835333229] + ] def test_mapper_transformation_request(self): request = Request( diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index bfe32b21d..4cf4e01f7 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -15,8 +15,10 @@ def setup_method(self, method): array = ds.to_xarray().isel(step=0).t self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - options = {"latitude": {"transformation": {"reverse": {True}}}, - "longitude": {"transformation": {"cyclic": [0, 360.0]}}} + options = { + "latitude": {"transformation": {"reverse": {True}}}, + "longitude": {"transformation": {"cyclic": [0, 360.0]}}, + } self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) def test_surrounding_on_grid_point(self): From dbf3895c31daf3061a3a9b756b546a7af1383b16 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Sep 2023 16:52:00 +0200 Subject: [PATCH 142/332] add healpix mapper --- .../transformations/datacube_mappers.py | 87 ++++++++++++++++++- tests/data/healpix.grib | 3 + 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 tests/data/healpix.grib diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 16fd1b5df..1db0354f0 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -69,6 +69,90 @@ def unmap(self, first_val, second_val): return final_transformation.unmap(first_val, second_val) +class HealpixGridMapper(DatacubeMapper): + def __init__(self, base_axis, mapped_axes, resolution): + self._mapped_axes = mapped_axes + self._base_axis = base_axis + self._resolution = resolution + + def first_axis_vals(self): + rad2deg = 180/math.pi + vals = [0] * (4*self._resolution - 1) + + # Polar caps + for i in range(1, self._resolution): + val = 90 - (rad2deg * math.acos(1-(i*i/(3 * self._resolution * self._resolution)))) + vals[i-1] = val + vals[4 * self._resolution - 1 - i] = - val + # Equatorial belts + for i in range(self._resolution , 2*self._resolution): + val = 90 - (rad2deg * math.acos((4*self._resolution - 2*i)/(3*self._resolution))) + vals[i-1] = val + vals[4 * self._resolution - 1 - i] = - val + # Equator + vals[2 * self._resolution - 1] = 0 + + return vals + + def map_first_axis(self, lower, upper): + axis_lines = self.first_axis_vals() + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def second_axis_vals(self, first_val): + idx = self.first_axis_vals().index(first_val) + + # Polar caps + if idx < self._resolution - 1 or 3 * self._resolution - 1 < idx <= 4 * self._resolution - 2: + start = 45 / (idx+1) + vals = [start + i * (360 / (4*(idx + 1))) for i in range(4 * (idx + 1))] + return vals + # Equatorial belts + start = 45/self._resolution + if self._resolution - 1 <= idx < 2 * self._resolution - 1 or 2*self._resolution <= idx < 3 * self._resolution: + r_start = start * (2 - (((idx + 1) - self._resolution + 1) % 2)) + vals = [r_start + i * (360 / (4 * self._resolution)) for i in range(4 * self._resolution)] + return vals + # Equator + temp_val = 1 if self._resolution % 2 else 0 + r_start = start * (1 - temp_val) + if idx == 2 * self._resolution - 1: + vals = [r_start + i * (360 / (4 * self._resolution)) for i in range(4 * self._resolution)] + return vals + + def map_second_axis(self, first_val, lower, upper): + axis_lines = self.second_axis_vals(first_val) + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def axes_idx_to_healpix_idx(self, first_idx, second_idx): + idx = 0 + for i in range(self._resolution - 1): + if i != first_idx: + idx += 4 * (i + 1) + else: + idx += second_idx + for i in range(self._resolution - 1, 3 * self._resolution): + if i != first_idx: + idx += 4 * self._resolution + else: + idx += second_idx + for i in range(3 * self._resolution, 4 * self._resolution - 1): + if i != first_idx: + idx += 4 * (4 * self._resolution - 1 - i + 1) + else: + idx += second_idx + return idx + + def unmap(self, first_val, second_val): + first_axis_vals = self.first_axis_vals() + first_idx = first_axis_vals.index(first_val) + second_axis_vals = self.second_axis_vals(first_val) + second_idx = second_axis_vals.index(second_val) + healpix_index = self.axes_idx_to_healpix_idx(first_idx, second_idx) + return healpix_index + + class OctahedralGridMapper(DatacubeMapper): def __init__(self, base_axis, mapped_axes, resolution): self._mapped_axes = mapped_axes @@ -2785,4 +2869,5 @@ def unmap(self, first_val, second_val): return octahedral_index -_type_to_datacube_mapper_lookup = {"octahedral": "OctahedralGridMapper"} +_type_to_datacube_mapper_lookup = {"octahedral": "OctahedralGridMapper", + "healpix": "HealpixGridMapper"} diff --git a/tests/data/healpix.grib b/tests/data/healpix.grib new file mode 100644 index 000000000..693c98c12 --- /dev/null +++ b/tests/data/healpix.grib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df57790df280627b2883c01fa4b56986d9938ed69b7434fe09c4bede99d2895f +size 37030 From 85a314ae83bffcf34a6f5f38ff347b5c9bc1a12b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Sep 2023 16:59:52 +0200 Subject: [PATCH 143/332] add healpix test --- tests/test_healpix_mapper.py | 80 ++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 tests/test_healpix_mapper.py diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py new file mode 100644 index 000000000..6d3edd37f --- /dev/null +++ b/tests/test_healpix_mapper.py @@ -0,0 +1,80 @@ +from earthkit import data +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestOctahedralGrid: + def setup_method(self, method): + ds = data.from_source("file", "./tests/data/healpix.grib") + self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) + self.latlon_array = self.latlon_array.t2m + self.xarraydatacube = XArrayDatacube(self.latlon_array) + self.options = { + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 32, "axes": ["latitude", "longitude"]} + } + } + } + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) + + def find_nearest_latlon(self, grib_file, target_lat, target_lon): + # Open the GRIB file + f = open(grib_file) + + # Load the GRIB messages from the file + messages = [] + while True: + message = codes_grib_new_from_file(f) + if message is None: + break + messages.append(message) + + # Find the nearest grid points + nearest_points = [] + for message in messages: + nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) + nearest_points.append(nearest_index) + + # Close the GRIB file + f.close() + + return nearest_points + + def test_octahedral_grid(self): + # request = Request( + # Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + # Select("number", [0]), + # Select("time", ["2023-06-25T12:00:00"]), + # Select("step", ["00:00:00"]), + # Select("surface", [0]), + # Select("valid_time", ["2023-06-25T12:00:00"]), + # ) + # result = self.API.retrieve(request) + # assert len(result.leaves) == 9 + + # lats = [] + # lons = [] + # eccodes_lats = [] + # tol = 1e-8 + # for i in range(len(result.leaves)): + # cubepath = result.leaves[i].flatten() + # lat = cubepath["latitude"] + # lon = cubepath["longitude"] + # lats.append(lat) + # lons.append(lon) + # nearest_points = self.find_nearest_latlon("./tests/data/foo.grib", lat, lon) + # eccodes_lat = nearest_points[0][0]["lat"] + # eccodes_lon = nearest_points[0][0]["lon"] + # eccodes_lats.append(eccodes_lat) + # assert eccodes_lat - tol <= lat + # assert lat <= eccodes_lat + tol + # assert eccodes_lon - tol <= lon + # assert lon <= eccodes_lon + tol + # assert len(eccodes_lats) == 9 + pass From 4c426b372341afba4b0b2cd933f012518514adf2 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Sep 2023 13:50:40 +0200 Subject: [PATCH 144/332] test healpix mapper --- polytope/datacube/backends/datacube.py | 1 + polytope/datacube/backends/xarray.py | 1 + .../transformations/datacube_mappers.py | 40 +++++------ polytope/engine/hullslicer.py | 2 - tests/test_healpix_mapper.py | 66 +++++++++---------- 5 files changed, 54 insertions(+), 56 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index f6ffb041f..e48b219eb 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -38,6 +38,7 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) for axis_name in final_axis_names: + print(axis_name) self.complete_axes.append(axis_name) # if axis does not yet exist, create it diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 321f0f644..e43a7ad41 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -46,6 +46,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self._check_and_add_axes(options, name, val) def get(self, requests: IndexTree): + requests.pprint() for r in requests.leaves: path = r.flatten() path = self.remap_path(path) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 1db0354f0..bac6bcf92 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -76,19 +76,19 @@ def __init__(self, base_axis, mapped_axes, resolution): self._resolution = resolution def first_axis_vals(self): - rad2deg = 180/math.pi - vals = [0] * (4*self._resolution - 1) + rad2deg = 180 / math.pi + vals = [0] * (4 * self._resolution - 1) # Polar caps for i in range(1, self._resolution): - val = 90 - (rad2deg * math.acos(1-(i*i/(3 * self._resolution * self._resolution)))) - vals[i-1] = val - vals[4 * self._resolution - 1 - i] = - val + val = 90 - (rad2deg * math.acos(1 - (i * i / (3 * self._resolution * self._resolution)))) + vals[i - 1] = val + vals[4 * self._resolution - 1 - i] = -val # Equatorial belts - for i in range(self._resolution , 2*self._resolution): - val = 90 - (rad2deg * math.acos((4*self._resolution - 2*i)/(3*self._resolution))) - vals[i-1] = val - vals[4 * self._resolution - 1 - i] = - val + for i in range(self._resolution, 2 * self._resolution): + val = 90 - (rad2deg * math.acos((4 * self._resolution - 2 * i) / (3 * self._resolution))) + vals[i - 1] = val + vals[4 * self._resolution - 1 - i] = -val # Equator vals[2 * self._resolution - 1] = 0 @@ -100,16 +100,18 @@ def map_first_axis(self, lower, upper): return return_vals def second_axis_vals(self, first_val): + tol = 1e-8 + first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] idx = self.first_axis_vals().index(first_val) # Polar caps if idx < self._resolution - 1 or 3 * self._resolution - 1 < idx <= 4 * self._resolution - 2: - start = 45 / (idx+1) - vals = [start + i * (360 / (4*(idx + 1))) for i in range(4 * (idx + 1))] + start = 45 / (idx + 1) + vals = [start + i * (360 / (4 * (idx + 1))) for i in range(4 * (idx + 1))] return vals # Equatorial belts - start = 45/self._resolution - if self._resolution - 1 <= idx < 2 * self._resolution - 1 or 2*self._resolution <= idx < 3 * self._resolution: + start = 45 / self._resolution + if self._resolution - 1 <= idx < 2 * self._resolution - 1 or 2 * self._resolution <= idx < 3 * self._resolution: r_start = start * (2 - (((idx + 1) - self._resolution + 1) % 2)) vals = [r_start + i * (360 / (4 * self._resolution)) for i in range(4 * self._resolution)] return vals @@ -145,10 +147,11 @@ def axes_idx_to_healpix_idx(self, first_idx, second_idx): return idx def unmap(self, first_val, second_val): - first_axis_vals = self.first_axis_vals() - first_idx = first_axis_vals.index(first_val) - second_axis_vals = self.second_axis_vals(first_val) - second_idx = second_axis_vals.index(second_val) + tol = 1e-8 + first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] + first_idx = self.first_axis_vals().index(first_val) + second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] + second_idx = self.second_axis_vals(first_val).index(second_val) healpix_index = self.axes_idx_to_healpix_idx(first_idx, second_idx) return healpix_index @@ -2869,5 +2872,4 @@ def unmap(self, first_val, second_val): return octahedral_index -_type_to_datacube_mapper_lookup = {"octahedral": "OctahedralGridMapper", - "healpix": "HealpixGridMapper"} +_type_to_datacube_mapper_lookup = {"octahedral": "OctahedralGridMapper", "healpix": "HealpixGridMapper"} diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 5ea0c951e..f39f8b0c0 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -53,12 +53,10 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex fvalue = ax.to_float(value) new_polytope = slice(polytope, ax.name, fvalue) # store the native type - # remapped_val = (ax.remap([value, value])[0][0] + ax.remap([value, value])[0][1])/2 remapped_val = value if ax.is_cyclic: remapped_val = (ax.remap([value, value])[0][0] + ax.remap([value, value])[0][1]) / 2 remapped_val = round(remapped_val, int(-math.log10(ax.tol))) - # child = node.create_child(ax, value) child = node.create_child(ax, remapped_val) child["unsliced_polytopes"] = copy(node["unsliced_polytopes"]) child["unsliced_polytopes"].remove(polytope) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index 6d3edd37f..c743736cf 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -10,14 +10,11 @@ class TestOctahedralGrid: def setup_method(self, method): ds = data.from_source("file", "./tests/data/healpix.grib") - self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) - self.latlon_array = self.latlon_array.t2m + self.latlon_array = ds.to_xarray().isel(step=0).isel(time=0).isel(isobaricInhPa=0).z self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 32, "axes": ["latitude", "longitude"]} - } + "transformation": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}} } } self.slicer = HullSlicer() @@ -47,34 +44,33 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points def test_octahedral_grid(self): - # request = Request( - # Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), - # Select("number", [0]), - # Select("time", ["2023-06-25T12:00:00"]), - # Select("step", ["00:00:00"]), - # Select("surface", [0]), - # Select("valid_time", ["2023-06-25T12:00:00"]), - # ) - # result = self.API.retrieve(request) - # assert len(result.leaves) == 9 + request = Request( + Box(["latitude", "longitude"], [-2, -2], [5, 5]), + Select("time", ["2022-12-14T12:00:00"]), + Select("step", ["01:00:00"]), + Select("isobaricInhPa", [500]), + Select("valid_time", ["2022-12-14T13:00:00"]), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 9 - # lats = [] - # lons = [] - # eccodes_lats = [] - # tol = 1e-8 - # for i in range(len(result.leaves)): - # cubepath = result.leaves[i].flatten() - # lat = cubepath["latitude"] - # lon = cubepath["longitude"] - # lats.append(lat) - # lons.append(lon) - # nearest_points = self.find_nearest_latlon("./tests/data/foo.grib", lat, lon) - # eccodes_lat = nearest_points[0][0]["lat"] - # eccodes_lon = nearest_points[0][0]["lon"] - # eccodes_lats.append(eccodes_lat) - # assert eccodes_lat - tol <= lat - # assert lat <= eccodes_lat + tol - # assert eccodes_lon - tol <= lon - # assert lon <= eccodes_lon + tol - # assert len(eccodes_lats) == 9 - pass + lats = [] + lons = [] + eccodes_lats = [] + tol = 1e-8 + for i in range(len(result.leaves)): + cubepath = result.leaves[i].flatten() + lat = cubepath["latitude"] + lon = cubepath["longitude"] + lats.append(lat) + lons.append(lon) + nearest_points = self.find_nearest_latlon("./tests/data/healpix.grib", lat, lon) + eccodes_lat = nearest_points[0][0]["lat"] + eccodes_lon = nearest_points[0][0]["lon"] + eccodes_lats.append(eccodes_lat) + assert eccodes_lat - tol <= lat + assert lat <= eccodes_lat + tol + assert eccodes_lon - tol <= lon + assert lon <= eccodes_lon + tol + assert len(eccodes_lats) == 9 From 01c974683538b3f1782165689aa715eba68eecf7 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Sep 2023 13:58:25 +0200 Subject: [PATCH 145/332] add plotting of healpix mapper --- tests/test_healpix_mapper.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index c743736cf..325314845 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -1,3 +1,5 @@ +import geopandas as gpd +import matplotlib.pyplot as plt from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file @@ -45,7 +47,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): def test_octahedral_grid(self): request = Request( - Box(["latitude", "longitude"], [-2, -2], [5, 5]), + Box(["latitude", "longitude"], [-2, -2], [10, 10]), Select("time", ["2022-12-14T12:00:00"]), Select("step", ["01:00:00"]), Select("isobaricInhPa", [500]), @@ -53,11 +55,12 @@ def test_octahedral_grid(self): ) result = self.API.retrieve(request) result.pprint() - assert len(result.leaves) == 9 + assert len(result.leaves) == 35 lats = [] lons = [] eccodes_lats = [] + eccodes_lons = [] tol = 1e-8 for i in range(len(result.leaves)): cubepath = result.leaves[i].flatten() @@ -69,8 +72,17 @@ def test_octahedral_grid(self): eccodes_lat = nearest_points[0][0]["lat"] eccodes_lon = nearest_points[0][0]["lon"] eccodes_lats.append(eccodes_lat) + eccodes_lons.append(eccodes_lon) assert eccodes_lat - tol <= lat assert lat <= eccodes_lat + tol assert eccodes_lon - tol <= lon assert lon <= eccodes_lon + tol - assert len(eccodes_lats) == 9 + assert len(eccodes_lats) == 35 + worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + fig, ax = plt.subplots(figsize=(12, 6)) + worldmap.plot(color="darkgrey", ax=ax) + + plt.scatter(eccodes_lons, eccodes_lats, c="blue", marker="s", s=20) + plt.scatter(lons, lats, s=16, c="red", cmap="YlOrRd") + plt.colorbar(label="Temperature") + plt.show() From 54aaf0cd3626bdbfcb165bcc18176c08a1632ebc Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Sep 2023 14:04:06 +0200 Subject: [PATCH 146/332] clean up --- polytope/datacube/backends/datacube.py | 1 - polytope/datacube/backends/xarray.py | 1 - tests/test_healpix_mapper.py | 1 - 3 files changed, 3 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index e48b219eb..f6ffb041f 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -38,7 +38,6 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) for axis_name in final_axis_names: - print(axis_name) self.complete_axes.append(axis_name) # if axis does not yet exist, create it diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index e43a7ad41..321f0f644 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -46,7 +46,6 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self._check_and_add_axes(options, name, val) def get(self, requests: IndexTree): - requests.pprint() for r in requests.leaves: path = r.flatten() path = self.remap_path(path) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index 325314845..8e676fa3b 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -54,7 +54,6 @@ def test_octahedral_grid(self): Select("valid_time", ["2022-12-14T13:00:00"]), ) result = self.API.retrieve(request) - result.pprint() assert len(result.leaves) == 35 lats = [] From 623591de79416ae975ed2632067ef9ea5825abfb Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Sep 2023 14:06:41 +0200 Subject: [PATCH 147/332] add plotting to example folder and test without plotting for healpix mapper --- examples/healpix_grid_box_example.py | 87 ++++++++++++++++++++++++++++ tests/test_healpix_mapper.py | 12 ---- 2 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 examples/healpix_grid_box_example.py diff --git a/examples/healpix_grid_box_example.py b/examples/healpix_grid_box_example.py new file mode 100644 index 000000000..8e676fa3b --- /dev/null +++ b/examples/healpix_grid_box_example.py @@ -0,0 +1,87 @@ +import geopandas as gpd +import matplotlib.pyplot as plt +from earthkit import data +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestOctahedralGrid: + def setup_method(self, method): + ds = data.from_source("file", "./tests/data/healpix.grib") + self.latlon_array = ds.to_xarray().isel(step=0).isel(time=0).isel(isobaricInhPa=0).z + self.xarraydatacube = XArrayDatacube(self.latlon_array) + self.options = { + "values": { + "transformation": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}} + } + } + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) + + def find_nearest_latlon(self, grib_file, target_lat, target_lon): + # Open the GRIB file + f = open(grib_file) + + # Load the GRIB messages from the file + messages = [] + while True: + message = codes_grib_new_from_file(f) + if message is None: + break + messages.append(message) + + # Find the nearest grid points + nearest_points = [] + for message in messages: + nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) + nearest_points.append(nearest_index) + + # Close the GRIB file + f.close() + + return nearest_points + + def test_octahedral_grid(self): + request = Request( + Box(["latitude", "longitude"], [-2, -2], [10, 10]), + Select("time", ["2022-12-14T12:00:00"]), + Select("step", ["01:00:00"]), + Select("isobaricInhPa", [500]), + Select("valid_time", ["2022-12-14T13:00:00"]), + ) + result = self.API.retrieve(request) + assert len(result.leaves) == 35 + + lats = [] + lons = [] + eccodes_lats = [] + eccodes_lons = [] + tol = 1e-8 + for i in range(len(result.leaves)): + cubepath = result.leaves[i].flatten() + lat = cubepath["latitude"] + lon = cubepath["longitude"] + lats.append(lat) + lons.append(lon) + nearest_points = self.find_nearest_latlon("./tests/data/healpix.grib", lat, lon) + eccodes_lat = nearest_points[0][0]["lat"] + eccodes_lon = nearest_points[0][0]["lon"] + eccodes_lats.append(eccodes_lat) + eccodes_lons.append(eccodes_lon) + assert eccodes_lat - tol <= lat + assert lat <= eccodes_lat + tol + assert eccodes_lon - tol <= lon + assert lon <= eccodes_lon + tol + assert len(eccodes_lats) == 35 + worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + fig, ax = plt.subplots(figsize=(12, 6)) + worldmap.plot(color="darkgrey", ax=ax) + + plt.scatter(eccodes_lons, eccodes_lats, c="blue", marker="s", s=20) + plt.scatter(lons, lats, s=16, c="red", cmap="YlOrRd") + plt.colorbar(label="Temperature") + plt.show() diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index 8e676fa3b..a54f197cb 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -1,5 +1,3 @@ -import geopandas as gpd -import matplotlib.pyplot as plt from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file @@ -59,7 +57,6 @@ def test_octahedral_grid(self): lats = [] lons = [] eccodes_lats = [] - eccodes_lons = [] tol = 1e-8 for i in range(len(result.leaves)): cubepath = result.leaves[i].flatten() @@ -71,17 +68,8 @@ def test_octahedral_grid(self): eccodes_lat = nearest_points[0][0]["lat"] eccodes_lon = nearest_points[0][0]["lon"] eccodes_lats.append(eccodes_lat) - eccodes_lons.append(eccodes_lon) assert eccodes_lat - tol <= lat assert lat <= eccodes_lat + tol assert eccodes_lon - tol <= lon assert lon <= eccodes_lon + tol assert len(eccodes_lats) == 35 - worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) - fig, ax = plt.subplots(figsize=(12, 6)) - worldmap.plot(color="darkgrey", ax=ax) - - plt.scatter(eccodes_lons, eccodes_lats, c="blue", marker="s", s=20) - plt.scatter(lons, lats, s=16, c="red", cmap="YlOrRd") - plt.colorbar(label="Temperature") - plt.show() From 15efd14345e9baf9b16f6077d55de16345bffa13 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Sep 2023 14:12:29 +0200 Subject: [PATCH 148/332] update CI to work with eccodes develop branch --- .github/workflows/ci.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 43b2d82b7..7a5337095 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -110,8 +110,16 @@ jobs: - name: Download Git LFS file run: | git lfs pull - - name: Install eccodes - run: sudo apt-get install -y libeccodes-dev + # - name: Install eccodes + # run: sudo apt-get install -y libeccodes-dev + - name: Install eccodes and Dependencies + id: install-dependencies + uses: ecmwf-actions/build-package@v2 + with: + self_build: false + dependencies: | + ecmwf/ecbuild@develop + ecmwf/eccodes@develop - name: Setup Python uses: actions/setup-python@v4 From 0fbe0add3934873c0d3774c948a9c99b98af301d Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Sep 2023 14:23:07 +0200 Subject: [PATCH 149/332] update CI to work with eccodes develop branch --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7a5337095..a43bce28b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -119,6 +119,7 @@ jobs: self_build: false dependencies: | ecmwf/ecbuild@develop + MathisRosenhauer/libaec@master ecmwf/eccodes@develop - name: Setup Python From 4b0650b8738aa8c9d93c07ca7a6deb020d0c3241 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 9 Oct 2023 15:35:50 +0100 Subject: [PATCH 150/332] download test data from nexus --- .github/workflows/ci.yaml | 11 ----------- tests/test_datacube_axes_init.py | 23 ++++++++++++++++++++++- tests/test_healpix_mapper.py | 24 +++++++++++++++++++++++- tests/test_merge_octahedral_one_axis.py | 23 ++++++++++++++++++++++- tests/test_octahedral_grid.py | 23 ++++++++++++++++++++++- tests/test_slicer_era5.py | 23 ++++++++++++++++++++++- tests/test_snapping_real_data.py | 23 ++++++++++++++++++++++- 7 files changed, 133 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a43bce28b..8980ee39e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -99,19 +99,8 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - with: - fetch-depth: 3 # enable full clone to download Git LFS files - name: Change write permission run: chmod -R 777 ./tests/data/ - - name: Install Git LFS - run: | - sudo apt-get install git-lfs - git lfs install - - name: Download Git LFS file - run: | - git lfs pull - # - name: Install eccodes - # run: sudo apt-get install -y libeccodes-dev - name: Install eccodes and Dependencies id: install-dependencies uses: ecmwf-actions/build-package@v2 diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 21412c4f1..68e99aa7f 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -1,3 +1,6 @@ +import os + +import requests from earthkit import data from polytope.datacube.backends.xarray import XArrayDatacube @@ -9,7 +12,25 @@ class TestInitDatacubeAxes: def setup_method(self, method): - ds = data.from_source("file", "./tests/data/foo.grib") + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" + + local_directory = "./tests/data_tests" + + if not os.path.exists(local_directory): + os.makedirs(local_directory) + + # Construct the full path for the local file + local_file_path = os.path.join(local_directory, "foo.grib") + + if not os.path.exists(local_file_path): + session = requests.Session() + response = session.get(nexus_url) + if response.status_code == 200: + # Save the downloaded data to the local file + with open(local_file_path, "wb") as f: + f.write(response.content) + + ds = data.from_source("file", "./tests/data_tests/foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m self.xarraydatacube = XArrayDatacube(latlon_array) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index a54f197cb..e50e4af3f 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -1,3 +1,6 @@ +import os + +import requests from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file @@ -9,7 +12,26 @@ class TestOctahedralGrid: def setup_method(self, method): - ds = data.from_source("file", "./tests/data/healpix.grib") + + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/healpix.grib" + + local_directory = "./tests/data_tests" + + if not os.path.exists(local_directory): + os.makedirs(local_directory) + + # Construct the full path for the local file + local_file_path = os.path.join(local_directory, "healpix.grib") + + if not os.path.exists(local_file_path): + session = requests.Session() + response = session.get(nexus_url) + if response.status_code == 200: + # Save the downloaded data to the local file + with open(local_file_path, "wb") as f: + f.write(response.content) + + ds = data.from_source("file", "./tests/data_tests/healpix.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(time=0).isel(isobaricInhPa=0).z self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index 534f00bb3..d05f25f46 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -1,3 +1,6 @@ +import os + +import requests from earthkit import data from polytope.datacube.backends.xarray import XArrayDatacube @@ -8,7 +11,25 @@ class TestSlicingMultipleTransformationsOneAxis: def setup_method(self, method): - ds = data.from_source("file", "./tests/data/foo.grib") + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" + + local_directory = "./tests/data_tests" + + if not os.path.exists(local_directory): + os.makedirs(local_directory) + + # Construct the full path for the local file + local_file_path = os.path.join(local_directory, "foo.grib") + + if not os.path.exists(local_file_path): + session = requests.Session() + response = session.get(nexus_url) + if response.status_code == 200: + # Save the downloaded data to the local file + with open(local_file_path, "wb") as f: + f.write(response.content) + + ds = data.from_source("file", "./tests/data_tests/foo.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) self.latlon_array = self.latlon_array.t2m self.xarraydatacube = XArrayDatacube(self.latlon_array) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 15177f4e7..c98722dfc 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -1,3 +1,6 @@ +import os + +import requests from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file @@ -9,7 +12,25 @@ class TestOctahedralGrid: def setup_method(self, method): - ds = data.from_source("file", "./tests/data/foo.grib") + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" + + local_directory = "./tests/data_tests" + + if not os.path.exists(local_directory): + os.makedirs(local_directory) + + # Construct the full path for the local file + local_file_path = os.path.join(local_directory, "foo.grib") + + if not os.path.exists(local_file_path): + session = requests.Session() + response = session.get(nexus_url) + if response.status_code == 200: + # Save the downloaded data to the local file + with open(local_file_path, "wb") as f: + f.write(response.content) + + ds = data.from_source("file", "./tests/data_tests/foo.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) self.latlon_array = self.latlon_array.t2m self.xarraydatacube = XArrayDatacube(self.latlon_array) diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 27346227e..edb2d5d8a 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -1,4 +1,7 @@ +import os + import numpy as np +import requests from earthkit import data from polytope.datacube.backends.xarray import XArrayDatacube @@ -9,7 +12,25 @@ class TestSlicingEra5Data: def setup_method(self, method): - ds = data.from_source("file", "./tests/data/era5-levels-members.grib") + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" + + local_directory = "./tests/data_tests" + + if not os.path.exists(local_directory): + os.makedirs(local_directory) + + # Construct the full path for the local file + local_file_path = os.path.join(local_directory, "era5-levels-members.grib") + + if not os.path.exists(local_file_path): + session = requests.Session() + response = session.get(nexus_url) + if response.status_code == 200: + # Save the downloaded data to the local file + with open(local_file_path, "wb") as f: + f.write(response.content) + + ds = data.from_source("file", "./tests/data_tests/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index 4cf4e01f7..e4ec1b4f3 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -1,6 +1,9 @@ # import geopandas as gpd # import matplotlib.pyplot as plt +import os + import numpy as np +import requests from earthkit import data from polytope.datacube.backends.xarray import XArrayDatacube @@ -11,7 +14,25 @@ class TestSlicingEra5Data: def setup_method(self, method): - ds = data.from_source("file", "./tests/data/era5-levels-members.grib") + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" + + local_directory = "./tests/data_tests" + + if not os.path.exists(local_directory): + os.makedirs(local_directory) + + # Construct the full path for the local file + local_file_path = os.path.join(local_directory, "era5-levels-members.grib") + + if not os.path.exists(local_file_path): + session = requests.Session() + response = session.get(nexus_url) + if response.status_code == 200: + # Save the downloaded data to the local file + with open(local_file_path, "wb") as f: + f.write(response.content) + + ds = data.from_source("file", "./tests/data_tests/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() From 3b770756b794e98b786a63d7835d94b5fd2cab22 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 9 Oct 2023 15:46:00 +0100 Subject: [PATCH 151/332] download nexus data to tests/data --- tests/data/era5-levels-members.grib | 3 --- tests/data/foo.grib | 3 --- tests/data/healpix.grib | 3 --- tests/test_datacube_axes_init.py | 4 ++-- tests/test_healpix_mapper.py | 4 ++-- tests/test_merge_octahedral_one_axis.py | 4 ++-- tests/test_octahedral_grid.py | 4 ++-- tests/test_slicer_era5.py | 4 ++-- tests/test_snapping_real_data.py | 4 ++-- 9 files changed, 12 insertions(+), 21 deletions(-) delete mode 100644 tests/data/era5-levels-members.grib delete mode 100644 tests/data/foo.grib delete mode 100644 tests/data/healpix.grib diff --git a/tests/data/era5-levels-members.grib b/tests/data/era5-levels-members.grib deleted file mode 100644 index 90d45deed..000000000 --- a/tests/data/era5-levels-members.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c144fde61ca5d53702bf6f212775ef2cc783bdd66b6865160bf597c1b35ed898 -size 2361600 diff --git a/tests/data/foo.grib b/tests/data/foo.grib deleted file mode 100644 index 9c5efa68b..000000000 --- a/tests/data/foo.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:12533bae9b3aa078a3298a82f289c7dbba6dbdb9a21e4e5e24e84b8695a75dee -size 26409360 diff --git a/tests/data/healpix.grib b/tests/data/healpix.grib deleted file mode 100644 index 693c98c12..000000000 --- a/tests/data/healpix.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:df57790df280627b2883c01fa4b56986d9938ed69b7434fe09c4bede99d2895f -size 37030 diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 68e99aa7f..10dce63a2 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -14,7 +14,7 @@ class TestInitDatacubeAxes: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" - local_directory = "./tests/data_tests" + local_directory = "./tests/data" if not os.path.exists(local_directory): os.makedirs(local_directory) @@ -30,7 +30,7 @@ def setup_method(self, method): with open(local_file_path, "wb") as f: f.write(response.content) - ds = data.from_source("file", "./tests/data_tests/foo.grib") + ds = data.from_source("file", "./tests/data/foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m self.xarraydatacube = XArrayDatacube(latlon_array) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index e50e4af3f..709d9dbd9 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -15,7 +15,7 @@ def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/healpix.grib" - local_directory = "./tests/data_tests" + local_directory = "./tests/data" if not os.path.exists(local_directory): os.makedirs(local_directory) @@ -31,7 +31,7 @@ def setup_method(self, method): with open(local_file_path, "wb") as f: f.write(response.content) - ds = data.from_source("file", "./tests/data_tests/healpix.grib") + ds = data.from_source("file", "./tests/data/healpix.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(time=0).isel(isobaricInhPa=0).z self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index d05f25f46..d7cddedd5 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -13,7 +13,7 @@ class TestSlicingMultipleTransformationsOneAxis: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" - local_directory = "./tests/data_tests" + local_directory = "./tests/data" if not os.path.exists(local_directory): os.makedirs(local_directory) @@ -29,7 +29,7 @@ def setup_method(self, method): with open(local_file_path, "wb") as f: f.write(response.content) - ds = data.from_source("file", "./tests/data_tests/foo.grib") + ds = data.from_source("file", "./tests/data/foo.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) self.latlon_array = self.latlon_array.t2m self.xarraydatacube = XArrayDatacube(self.latlon_array) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index c98722dfc..eda0bb1f8 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -14,7 +14,7 @@ class TestOctahedralGrid: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" - local_directory = "./tests/data_tests" + local_directory = "./tests/data" if not os.path.exists(local_directory): os.makedirs(local_directory) @@ -30,7 +30,7 @@ def setup_method(self, method): with open(local_file_path, "wb") as f: f.write(response.content) - ds = data.from_source("file", "./tests/data_tests/foo.grib") + ds = data.from_source("file", "./tests/data/foo.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) self.latlon_array = self.latlon_array.t2m self.xarraydatacube = XArrayDatacube(self.latlon_array) diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index edb2d5d8a..f02d1ce64 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -14,7 +14,7 @@ class TestSlicingEra5Data: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" - local_directory = "./tests/data_tests" + local_directory = "./tests/data" if not os.path.exists(local_directory): os.makedirs(local_directory) @@ -30,7 +30,7 @@ def setup_method(self, method): with open(local_file_path, "wb") as f: f.write(response.content) - ds = data.from_source("file", "./tests/data_tests/era5-levels-members.grib") + ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index e4ec1b4f3..5df9b9273 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -16,7 +16,7 @@ class TestSlicingEra5Data: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" - local_directory = "./tests/data_tests" + local_directory = "./tests/data" if not os.path.exists(local_directory): os.makedirs(local_directory) @@ -32,7 +32,7 @@ def setup_method(self, method): with open(local_file_path, "wb") as f: f.write(response.content) - ds = data.from_source("file", "./tests/data_tests/era5-levels-members.grib") + ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() From fec89c80f19cee5dd9bb743e14fac1e7df05a2c0 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 9 Oct 2023 15:47:50 +0100 Subject: [PATCH 152/332] black --- tests/test_healpix_mapper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index 709d9dbd9..c8ec4e92f 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -12,7 +12,6 @@ class TestOctahedralGrid: def setup_method(self, method): - nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/healpix.grib" local_directory = "./tests/data" From ffbe4e15958dc9dc3e5fb256d853f9673cf5c771 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 9 Oct 2023 15:49:53 +0100 Subject: [PATCH 153/332] remove change write permissions to test data folder in CI --- .github/workflows/ci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8980ee39e..031a8330e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -99,8 +99,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - name: Change write permission - run: chmod -R 777 ./tests/data/ - name: Install eccodes and Dependencies id: install-dependencies uses: ecmwf-actions/build-package@v2 From b9bfdd4440427f9bfd2f36650c2303ffb1261409 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 10 Oct 2023 09:37:26 +0100 Subject: [PATCH 154/332] add internet connection warning in tests --- .github/workflows/ci.yaml | 2 +- tests/test_datacube_axes_init.py | 2 ++ tests/test_healpix_mapper.py | 2 ++ tests/test_merge_octahedral_one_axis.py | 2 ++ tests/test_octahedral_grid.py | 2 ++ tests/test_slicer_era5.py | 2 ++ tests/test_snapping_real_data.py | 2 ++ 7 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 031a8330e..017d4bf0e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -98,7 +98,7 @@ jobs: matrix: ${{ fromJson(needs.setup.outputs.matrix) }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install eccodes and Dependencies id: install-dependencies uses: ecmwf-actions/build-package@v2 diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 10dce63a2..8a2f274ac 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -11,6 +11,8 @@ class TestInitDatacubeAxes: + # This test requires an internet connection + def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index c8ec4e92f..44ddd9b20 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -11,6 +11,8 @@ class TestOctahedralGrid: + # This test requires an internet connection + def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/healpix.grib" diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index d7cddedd5..0f8c3ec78 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -10,6 +10,8 @@ class TestSlicingMultipleTransformationsOneAxis: + # This test requires an internet connection + def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index eda0bb1f8..2bd8905fa 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -11,6 +11,8 @@ class TestOctahedralGrid: + # This test requires an internet connection + def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index f02d1ce64..9c5afb4c9 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -11,6 +11,8 @@ class TestSlicingEra5Data: + # This test requires an internet connection + def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index 5df9b9273..2f1bdbfa4 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -13,6 +13,8 @@ class TestSlicingEra5Data: + # This test requires an internet connection + def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" From dd4b58b6c993feb6868665adf0d855507beeba1c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 10 Oct 2023 10:00:18 +0100 Subject: [PATCH 155/332] mark tests with internet connection --- pyproject.toml | 4 +++- tests/test_datacube_axes_init.py | 5 +++-- tests/test_healpix_mapper.py | 4 ++-- tests/test_merge_octahedral_one_axis.py | 4 ++-- tests/test_octahedral_grid.py | 4 ++-- tests/test_slicer_era5.py | 4 ++-- tests/test_snapping_real_data.py | 4 ++-- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 89b8eb763..28f4ec600 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,4 +2,6 @@ line-length = 120 [build-system] requires = ["setuptools>=61.0"] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" +[tool.pytest.ini_options] +markers = ["internet: downloads test data from the internet (deselect with '-m \"not internet\"')",] \ No newline at end of file diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 8a2f274ac..526368bb5 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -1,5 +1,6 @@ import os +import pytest import requests from earthkit import data @@ -11,8 +12,6 @@ class TestInitDatacubeAxes: - # This test requires an internet connection - def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" @@ -47,6 +46,7 @@ def setup_method(self, method): self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=self.options) self.datacube = self.API.datacube + @pytest.mark.internet def test_created_axes(self): assert self.datacube._axes["latitude"].has_mapper assert self.datacube._axes["longitude"].has_mapper @@ -101,6 +101,7 @@ def test_created_axes(self): [89.94618771566562, 89.87647835333229] ] + @pytest.mark.internet def test_mapper_transformation_request(self): request = Request( Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index 44ddd9b20..0702a18ab 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -1,5 +1,6 @@ import os +import pytest import requests from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file @@ -11,8 +12,6 @@ class TestOctahedralGrid: - # This test requires an internet connection - def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/healpix.grib" @@ -66,6 +65,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points + @pytest.mark.internet def test_octahedral_grid(self): request = Request( Box(["latitude", "longitude"], [-2, -2], [10, 10]), diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index 0f8c3ec78..92ce76d54 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -1,5 +1,6 @@ import os +import pytest import requests from earthkit import data @@ -10,8 +11,6 @@ class TestSlicingMultipleTransformationsOneAxis: - # This test requires an internet connection - def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" @@ -46,6 +45,7 @@ def setup_method(self, method): self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) + @pytest.mark.internet def test_merge_axis(self): request = Request( Select("number", [0]), diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 2bd8905fa..25eb7172e 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -1,5 +1,6 @@ import os +import pytest import requests from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file @@ -11,8 +12,6 @@ class TestOctahedralGrid: - # This test requires an internet connection - def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" @@ -69,6 +68,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points + @pytest.mark.internet def test_octahedral_grid(self): request = Request( Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 9c5afb4c9..cbd88e509 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -1,6 +1,7 @@ import os import numpy as np +import pytest import requests from earthkit import data @@ -11,8 +12,6 @@ class TestSlicingEra5Data: - # This test requires an internet connection - def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" @@ -39,6 +38,7 @@ def setup_method(self, method): options = {"lat": {"transformation": {"reverse": {True}}}} self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + @pytest.mark.internet def test_2D_box(self): request = Request( Box(["number", "isobaricInhPa"], [3, 0.0], [6, 1000.0]), diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index 2f1bdbfa4..43b1e6e3f 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -3,6 +3,7 @@ import os import numpy as np +import pytest import requests from earthkit import data @@ -13,8 +14,6 @@ class TestSlicingEra5Data: - # This test requires an internet connection - def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" @@ -44,6 +43,7 @@ def setup_method(self, method): } self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) + @pytest.mark.internet def test_surrounding_on_grid_point(self): requested_lat = 0 requested_lon = -720 From f44af33cbbab494e59669b816132a4ac302dcaaf Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 10 Oct 2023 10:16:20 +0100 Subject: [PATCH 156/332] refactor download data function --- tests/helper_functions.py | 21 +++++++++++++++++++++ tests/test_datacube_axes_init.py | 22 +++------------------- tests/test_healpix_mapper.py | 22 +++------------------- tests/test_merge_octahedral_one_axis.py | 22 +++------------------- tests/test_octahedral_grid.py | 22 +++------------------- tests/test_slicer_era5.py | 22 +++------------------- tests/test_snapping_real_data.py | 21 +++------------------ 7 files changed, 39 insertions(+), 113 deletions(-) create mode 100644 tests/helper_functions.py diff --git a/tests/helper_functions.py b/tests/helper_functions.py new file mode 100644 index 000000000..97268632f --- /dev/null +++ b/tests/helper_functions.py @@ -0,0 +1,21 @@ +import os + +import requests + + +def download_test_data(nexus_url, filename): + local_directory = "./tests/data" + + if not os.path.exists(local_directory): + os.makedirs(local_directory) + + # Construct the full path for the local file + local_file_path = os.path.join(local_directory, filename) + + if not os.path.exists(local_file_path): + session = requests.Session() + response = session.get(nexus_url) + if response.status_code == 200: + # Save the downloaded data to the local file + with open(local_file_path, "wb") as f: + f.write(response.content) diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 526368bb5..bd48a8102 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -1,7 +1,4 @@ -import os - import pytest -import requests from earthkit import data from polytope.datacube.backends.xarray import XArrayDatacube @@ -10,26 +7,13 @@ from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select +from .helper_functions import download_test_data + class TestInitDatacubeAxes: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" - - local_directory = "./tests/data" - - if not os.path.exists(local_directory): - os.makedirs(local_directory) - - # Construct the full path for the local file - local_file_path = os.path.join(local_directory, "foo.grib") - - if not os.path.exists(local_file_path): - session = requests.Session() - response = session.get(nexus_url) - if response.status_code == 200: - # Save the downloaded data to the local file - with open(local_file_path, "wb") as f: - f.write(response.content) + download_test_data(nexus_url, "foo.grib") ds = data.from_source("file", "./tests/data/foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index 0702a18ab..423ff2440 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -1,7 +1,4 @@ -import os - import pytest -import requests from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file @@ -10,26 +7,13 @@ from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select +from .helper_functions import download_test_data + class TestOctahedralGrid: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/healpix.grib" - - local_directory = "./tests/data" - - if not os.path.exists(local_directory): - os.makedirs(local_directory) - - # Construct the full path for the local file - local_file_path = os.path.join(local_directory, "healpix.grib") - - if not os.path.exists(local_file_path): - session = requests.Session() - response = session.get(nexus_url) - if response.status_code == 200: - # Save the downloaded data to the local file - with open(local_file_path, "wb") as f: - f.write(response.content) + download_test_data(nexus_url, "healpix.grib") ds = data.from_source("file", "./tests/data/healpix.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(time=0).isel(isobaricInhPa=0).z diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index 92ce76d54..ed6b35fe9 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -1,7 +1,4 @@ -import os - import pytest -import requests from earthkit import data from polytope.datacube.backends.xarray import XArrayDatacube @@ -9,26 +6,13 @@ from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select +from .helper_functions import download_test_data + class TestSlicingMultipleTransformationsOneAxis: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" - - local_directory = "./tests/data" - - if not os.path.exists(local_directory): - os.makedirs(local_directory) - - # Construct the full path for the local file - local_file_path = os.path.join(local_directory, "foo.grib") - - if not os.path.exists(local_file_path): - session = requests.Session() - response = session.get(nexus_url) - if response.status_code == 200: - # Save the downloaded data to the local file - with open(local_file_path, "wb") as f: - f.write(response.content) + download_test_data(nexus_url, "foo.grib") ds = data.from_source("file", "./tests/data/foo.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 25eb7172e..484ad45a8 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -1,7 +1,4 @@ -import os - import pytest -import requests from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file @@ -10,26 +7,13 @@ from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select +from .helper_functions import download_test_data + class TestOctahedralGrid: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" - - local_directory = "./tests/data" - - if not os.path.exists(local_directory): - os.makedirs(local_directory) - - # Construct the full path for the local file - local_file_path = os.path.join(local_directory, "foo.grib") - - if not os.path.exists(local_file_path): - session = requests.Session() - response = session.get(nexus_url) - if response.status_code == 200: - # Save the downloaded data to the local file - with open(local_file_path, "wb") as f: - f.write(response.content) + download_test_data(nexus_url, "foo.grib") ds = data.from_source("file", "./tests/data/foo.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index cbd88e509..6bfcc63a3 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -1,8 +1,5 @@ -import os - import numpy as np import pytest -import requests from earthkit import data from polytope.datacube.backends.xarray import XArrayDatacube @@ -10,26 +7,13 @@ from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select +from .helper_functions import download_test_data + class TestSlicingEra5Data: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" - - local_directory = "./tests/data" - - if not os.path.exists(local_directory): - os.makedirs(local_directory) - - # Construct the full path for the local file - local_file_path = os.path.join(local_directory, "era5-levels-members.grib") - - if not os.path.exists(local_file_path): - session = requests.Session() - response = session.get(nexus_url) - if response.status_code == 200: - # Save the downloaded data to the local file - with open(local_file_path, "wb") as f: - f.write(response.content) + download_test_data(nexus_url, "era5-levels-members.grib") ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index 43b1e6e3f..342620511 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -1,10 +1,8 @@ # import geopandas as gpd # import matplotlib.pyplot as plt -import os import numpy as np import pytest -import requests from earthkit import data from polytope.datacube.backends.xarray import XArrayDatacube @@ -12,26 +10,13 @@ from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select +from .helper_functions import download_test_data + class TestSlicingEra5Data: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" - - local_directory = "./tests/data" - - if not os.path.exists(local_directory): - os.makedirs(local_directory) - - # Construct the full path for the local file - local_file_path = os.path.join(local_directory, "era5-levels-members.grib") - - if not os.path.exists(local_file_path): - session = requests.Session() - response = session.get(nexus_url) - if response.status_code == 200: - # Save the downloaded data to the local file - with open(local_file_path, "wb") as f: - f.write(response.content) + download_test_data(nexus_url, "era5-levels-members.grib") ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t From 6503e27c2acd74ce1ffcfaa4df873fe1fec3d9bd Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 10 Oct 2023 10:23:13 +0100 Subject: [PATCH 157/332] fix relative import error in tests --- tests/test_datacube_axes_init.py | 3 +-- tests/test_healpix_mapper.py | 3 +-- tests/test_merge_octahedral_one_axis.py | 3 +-- tests/test_octahedral_grid.py | 3 +-- tests/test_slicer_era5.py | 3 +-- tests/test_snapping_real_data.py | 3 +-- 6 files changed, 6 insertions(+), 12 deletions(-) diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index bd48a8102..9089de0c7 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -1,5 +1,6 @@ import pytest from earthkit import data +from helper_functions import download_test_data from polytope.datacube.backends.xarray import XArrayDatacube from polytope.datacube.datacube_axis import FloatDatacubeAxis @@ -7,8 +8,6 @@ from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select -from .helper_functions import download_test_data - class TestInitDatacubeAxes: def setup_method(self, method): diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index 423ff2440..c80d51042 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -1,14 +1,13 @@ import pytest from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file +from helper_functions import download_test_data from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select -from .helper_functions import download_test_data - class TestOctahedralGrid: def setup_method(self, method): diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index ed6b35fe9..5ff5dae91 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -1,13 +1,12 @@ import pytest from earthkit import data +from helper_functions import download_test_data from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select -from .helper_functions import download_test_data - class TestSlicingMultipleTransformationsOneAxis: def setup_method(self, method): diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 484ad45a8..56b21ee53 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -1,14 +1,13 @@ import pytest from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file +from helper_functions import download_test_data from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select -from .helper_functions import download_test_data - class TestOctahedralGrid: def setup_method(self, method): diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 6bfcc63a3..1cd0fc897 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -1,14 +1,13 @@ import numpy as np import pytest from earthkit import data +from helper_functions import download_test_data from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select -from .helper_functions import download_test_data - class TestSlicingEra5Data: def setup_method(self, method): diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index 342620511..c0d2a7a7d 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -4,14 +4,13 @@ import numpy as np import pytest from earthkit import data +from helper_functions import download_test_data from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select -from .helper_functions import download_test_data - class TestSlicingEra5Data: def setup_method(self, method): From c906f9c1112f59e57229a8a890f77aab32ee9d27 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 10 Oct 2023 10:31:58 +0100 Subject: [PATCH 158/332] raise exception in helper functions --- tests/data/era5-levels-members.grib | 3 +++ tests/helper_functions.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 tests/data/era5-levels-members.grib diff --git a/tests/data/era5-levels-members.grib b/tests/data/era5-levels-members.grib new file mode 100644 index 000000000..90d45deed --- /dev/null +++ b/tests/data/era5-levels-members.grib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c144fde61ca5d53702bf6f212775ef2cc783bdd66b6865160bf597c1b35ed898 +size 2361600 diff --git a/tests/helper_functions.py b/tests/helper_functions.py index 97268632f..5cc0dbf69 100644 --- a/tests/helper_functions.py +++ b/tests/helper_functions.py @@ -15,7 +15,7 @@ def download_test_data(nexus_url, filename): if not os.path.exists(local_file_path): session = requests.Session() response = session.get(nexus_url) - if response.status_code == 200: - # Save the downloaded data to the local file - with open(local_file_path, "wb") as f: - f.write(response.content) + response.raise_for_status() + # Save the downloaded data to the local file + with open(local_file_path, "wb") as f: + f.write(response.content) From 3a3113a42c15a298d4729a8fcaced59558962841 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 10 Oct 2023 11:12:56 +0100 Subject: [PATCH 159/332] remove exception handling in helper functions --- tests/data/era5-levels-members.grib | 3 --- tests/helper_functions.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 tests/data/era5-levels-members.grib diff --git a/tests/data/era5-levels-members.grib b/tests/data/era5-levels-members.grib deleted file mode 100644 index 90d45deed..000000000 --- a/tests/data/era5-levels-members.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c144fde61ca5d53702bf6f212775ef2cc783bdd66b6865160bf597c1b35ed898 -size 2361600 diff --git a/tests/helper_functions.py b/tests/helper_functions.py index 5cc0dbf69..7b579e4fd 100644 --- a/tests/helper_functions.py +++ b/tests/helper_functions.py @@ -15,7 +15,7 @@ def download_test_data(nexus_url, filename): if not os.path.exists(local_file_path): session = requests.Session() response = session.get(nexus_url) - response.raise_for_status() + # response.raise_for_status() # Save the downloaded data to the local file with open(local_file_path, "wb") as f: f.write(response.content) From c95438fbfd0d9da9cc7aa72b2525de96dc794091 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 10 Oct 2023 11:17:38 +0100 Subject: [PATCH 160/332] add exception handling in helper functions --- tests/helper_functions.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/helper_functions.py b/tests/helper_functions.py index 7b579e4fd..4e82bbe1a 100644 --- a/tests/helper_functions.py +++ b/tests/helper_functions.py @@ -3,6 +3,13 @@ import requests +class HTTPError(Exception): + def __init__(self, status_code, message): + self.status_code = status_code + self.message = message + super().__init__(f"HTTPError {status_code}: {message}") + + def download_test_data(nexus_url, filename): local_directory = "./tests/data" @@ -15,7 +22,8 @@ def download_test_data(nexus_url, filename): if not os.path.exists(local_file_path): session = requests.Session() response = session.get(nexus_url) - # response.raise_for_status() + if response.status_code != 200: + raise HTTPError(response.status_code, "Failed to download data.") # Save the downloaded data to the local file with open(local_file_path, "wb") as f: f.write(response.content) From 7090e0677fbf21aa67338b00e1f9b0b4bf65383b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 12 Oct 2023 14:10:00 +0100 Subject: [PATCH 161/332] make first fdb extraction work --- polytope/datacube/backends/FDB_datacube.py | 35 +++++++++++++++------ tests/.DS_Store | Bin 8196 -> 8196 bytes tests/test_fdb_datacube.py | 12 +++---- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 846ac840e..4343217c4 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -1,3 +1,4 @@ +import math from copy import deepcopy import pyfdb @@ -5,10 +6,6 @@ from .datacube import Datacube, IndexTree -def glue(path, unmap_path): - return {"t": 0} - - def update_fdb_dataarray(fdb_dataarray): fdb_dataarray["values"] = [0.0] return fdb_dataarray @@ -30,8 +27,8 @@ def __init__(self, config={}, axis_options={}): partial_request = config # Find values in the level 3 FDB datacube # Will be in the form of a dictionary? {axis_name:values_available, ...} - fdb = pyfdb.FDB() - fdb_dataarray = fdb.axes(partial_request).as_dict() + self.fdb = pyfdb.FDB() + fdb_dataarray = self.fdb.axes(partial_request).as_dict() dataarray = update_fdb_dataarray(fdb_dataarray) self.dataarray = dataarray @@ -61,10 +58,28 @@ def get(self, requests: IndexTree): axis = self._axes[key] (path, unmapped_path) = axis.unmap_total_path_to_datacube(path, unmapped_path) path = self.fit_path(path) - subxarray = glue(path, unmapped_path) - key = list(subxarray.keys())[0] - value = subxarray[key] - r.result = (key, value) + # merge path and unmapped path into a single path + path.update(unmapped_path) + + # fit request into something for pyfdb + fdb_request_val = path["values"] + path.pop("values") + fdb_request_key = path + + # TODO: should do this in the merge transformation, if it doesn't break xarray backend + fdb_request_key["date"] = fdb_request_key["date"].replace('-', '') + fdb_request_key["time"] = fdb_request_key["time"].replace(":", "") + + fdb_requests = [(fdb_request_key, [(fdb_request_val, fdb_request_val+1)])] + + # need to request data from the fdb + + subxarray = self.fdb.extract(fdb_requests) + subxarray_output_tuple = subxarray[0][0] + output_value = subxarray_output_tuple[0][0][0] + + if not math.isnan(output_value): + r.result = output_value else: r.remove_branch() diff --git a/tests/.DS_Store b/tests/.DS_Store index a3976a99336542543e657a5a6e812cda6ef38fb3..28aa9c59f3e8cfd1d76bfcb4e242dc62551a9375 100644 GIT binary patch delta 66 zcmV-I0KNZ&K!iY$PXQ0HP`eKS50eZKJCh9+3X|~=K9dU-43m2iK9hkGPLppD46{2E Y<^i*76>0&qgA9ZNvj-UW1hIJr1Fsbn%m4rY delta 356 zcmZp1XmOa}&nUhzU^hRb_+%adYvp8ye1=?x0)|9}B8E(cVjwS%A)djKAsZ+f&tM3m zCmRTfhOcK}V9@&y20#`A0}De6P)`X%qGwKia#Buy5>QNlfq_vLNF%Fb2ddFyNC%sq z#9)M~nkjv9gMjqpKmj{OfyonqtcijmtRNHSO>PkgWmE!+$+CmoR>F`mIZjAVh$$V- zc(9Vmi9!?Go(z`6Ov*A>OUtC$*2G_Q?`g9m7$QK6i5~W%_#x8Bozn|%EeiM zeq-d@EF<`rkvE Date: Thu, 12 Oct 2023 15:55:26 +0100 Subject: [PATCH 162/332] fix unmerging of date and time --- polytope/datacube/backends/FDB_datacube.py | 4 ---- polytope/datacube/backends/xarray.py | 3 +++ polytope/datacube/transformations/datacube_merger.py | 4 ++++ tests/test_merge_cyclic_octahedral.py | 6 +++--- tests/test_merge_transformation.py | 6 +++--- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 4343217c4..2525d5057 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -66,10 +66,6 @@ def get(self, requests: IndexTree): path.pop("values") fdb_request_key = path - # TODO: should do this in the merge transformation, if it doesn't break xarray backend - fdb_request_key["date"] = fdb_request_key["date"].replace('-', '') - fdb_request_key["time"] = fdb_request_key["time"].replace(":", "") - fdb_requests = [(fdb_request_key, [(fdb_request_val, fdb_request_val+1)])] # need to request data from the fdb diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 321f0f644..47847deba 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -76,6 +76,9 @@ def datacube_natural_indexes(self, axis, subarray): return indexes def select(self, path, unmapped_path): + print("now") + print(path) + print(self.dataarray) subarray = self.dataarray.sel(path, method="nearest") subarray = subarray.sel(unmapped_path) return subarray diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index eb404498d..e38a57d8c 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -44,6 +44,10 @@ def unmerge(self, merged_val): first_linker_size = len(self._linkers[0]) second_linked_size = len(self._linkers[1]) second_val = merged_val[first_idx + first_linker_size : -second_linked_size] + + # TODO: maybe replacing like this is too specific to time/dates? + first_val = str(first_val).replace('-', '') + second_val = second_val.replace(":", "") return (first_val, second_val) def change_val_type(self, axis_name, values): diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index ac5d5abc5..7f633c0be 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -15,14 +15,14 @@ def setup_method(self, method): np.random.randn(1, 1, 4289589, 3), dims=("date", "time", "values", "step"), coords={ - "date": ["2000-01-01"], - "time": ["06:00"], + "date": ["20000101"], + "time": ["0600"], "values": list(range(4289589)), "step": [0, 1, 2], }, ) self.options = { - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", ":00"]}}}, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, "values": { "transformation": { "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index af3b0d79c..fe15cf987 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -15,11 +15,11 @@ def setup_method(self, method): np.random.randn(1, 1), dims=("date", "time"), coords={ - "date": ["2000-01-01"], - "time": ["06:00"], + "date": ["20000101"], + "time": ["0600"], }, ) - self.options = {"date": {"transformation": {"merge": {"with": "time", "linkers": [" ", ":00"]}}}} + self.options = {"date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}} self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) From c6385aba9f483711a9ac7524442a20cdbc261d32 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 13 Oct 2023 11:15:47 +0100 Subject: [PATCH 163/332] add regular grid mapping for fdb and test, which does not work yet --- polytope/datacube/backends/xarray.py | 3 - .../transformations/datacube_mappers.py | 44 +++++++++- tests/test_regular_grid.py | 86 +++++++++++++++++++ 3 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 tests/test_regular_grid.py diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 47847deba..321f0f644 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -76,9 +76,6 @@ def datacube_natural_indexes(self, axis, subarray): return indexes def select(self, path, unmapped_path): - print("now") - print(path) - print(self.dataarray) subarray = self.dataarray.sel(path, method="nearest") subarray = subarray.sel(unmapped_path) return subarray diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index bac6bcf92..a8e56aaf5 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -69,6 +69,46 @@ def unmap(self, first_val, second_val): return final_transformation.unmap(first_val, second_val) +class RegularGridMapper(DatacubeMapper): + def __init__(self, base_axis, mapped_axes, resolution): + self._mapped_axes = mapped_axes + self._base_axis = base_axis + self._resolution = resolution + self.deg_increment = 90/self._resolution + + def first_axis_vals(self): + first_ax_vals = [-90 + i * self.deg_increment for i in range(2*self._resolution)] + return first_ax_vals + + def map_first_axis(self, lower, upper): + axis_lines = self.first_axis_vals() + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def second_axis_vals(self, first_val): + second_ax_vals = [i * self.deg_increment for i in range(4*self._resolution)] + return second_ax_vals + + def map_second_axis(self, first_val, lower, upper): + axis_lines = self.second_axis_vals(first_val) + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def axes_idx_to_regular_idx(self, first_idx, second_idx): + final_idx = first_idx * 4 * self._resolution + second_idx + return final_idx + + def unmap(self, first_val, second_val): + print("used unmap for regular grid !!!!!") + tol = 1e-8 + first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] + first_idx = self.first_axis_vals().index(first_val) + second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] + second_idx = self.second_axis_vals(first_val).index(second_val) + final_index = self.axes_idx_to_regular_idx(first_idx, second_idx) + return final_index + + class HealpixGridMapper(DatacubeMapper): def __init__(self, base_axis, mapped_axes, resolution): self._mapped_axes = mapped_axes @@ -2872,4 +2912,6 @@ def unmap(self, first_val, second_val): return octahedral_index -_type_to_datacube_mapper_lookup = {"octahedral": "OctahedralGridMapper", "healpix": "HealpixGridMapper"} +_type_to_datacube_mapper_lookup = {"octahedral": "OctahedralGridMapper", + "healpix": "HealpixGridMapper", + "regular": "RegularGridMapper"} diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py new file mode 100644 index 000000000..5e7ea358f --- /dev/null +++ b/tests/test_regular_grid.py @@ -0,0 +1,86 @@ +import numpy as np +import pytest +from earthkit import data +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file +from helper_functions import download_test_data + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestRegularGrid: + def setup_method(self, method): + # TODO: for this test, need to fdb write era5 data and use fdb backend + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" + download_test_data(nexus_url, "era5-levels-members.grib") + + ds = data.from_source("file", "./tests/data/era5-levels-members.grib") + self.latlon_array = ds.to_xarray().isel(step=0).t + self.xarraydatacube = XArrayDatacube(self.latlon_array) + self.options = { + "values": { + "transformation": { + "mapper": {"type": "regular", "resolution": 640, "axes": ["latitude", "longitude"]} + } + }, + "isobaricInhPa": {"transformation": {"reverse": {True}}} + } + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) + + def find_nearest_latlon(self, grib_file, target_lat, target_lon): + # Open the GRIB file + f = open(grib_file) + + # Load the GRIB messages from the file + messages = [] + while True: + message = codes_grib_new_from_file(f) + if message is None: + break + messages.append(message) + + # Find the nearest grid points + nearest_points = [] + for message in messages: + nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) + nearest_points.append(nearest_index) + + # Close the GRIB file + f.close() + + return nearest_points + + @pytest.mark.internet + def test_regular_grid(self): + request = Request( + Box(["number", "isobaricInhPa"], [3, 0.0], [3, 500.0]), + Select("time", ["2017-01-02T12:00:00"]), + Box(["latitude", "longitude"], lower_corner=[0.0, 0.0], upper_corner=[3.0, 3.0]), + Select("step", [np.timedelta64(0, "s")]), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 4 + + # lats = [] + # lons = [] + # eccodes_lats = [] + # tol = 1e-8 + # for i in range(len(result.leaves)): + # cubepath = result.leaves[i].flatten() + # lat = cubepath["latitude"] + # lon = cubepath["longitude"] + # lats.append(lat) + # lons.append(lon) + # nearest_points = self.find_nearest_latlon("./tests/data/foo.grib", lat, lon) + # eccodes_lat = nearest_points[0][0]["lat"] + # eccodes_lon = nearest_points[0][0]["lon"] + # eccodes_lats.append(eccodes_lat) + # assert eccodes_lat - tol <= lat + # assert lat <= eccodes_lat + tol + # assert eccodes_lon - tol <= lon + # assert lon <= eccodes_lon + tol + # assert len(eccodes_lats) == 9 From a6f938ccc466dcd005ff2929639bad70ebe794e9 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 13 Oct 2023 12:48:16 +0100 Subject: [PATCH 164/332] fix regular grid test --- polytope/datacube/backends/FDB_datacube.py | 1 + tests/test_regular_grid.py | 73 ++++++++++++---------- 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 2525d5057..63f5cc701 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -47,6 +47,7 @@ def __init__(self, config={}, axis_options={}): self._check_and_add_axes(options, name, val) def get(self, requests: IndexTree): + requests.pprint() for r in requests.leaves: path = r.flatten() path = self.remap_path(path) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 5e7ea358f..4ee3adc23 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -1,10 +1,9 @@ -import numpy as np +import pandas as pd import pytest -from earthkit import data from eccodes import codes_grib_find_nearest, codes_grib_new_from_file from helper_functions import download_test_data -from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.datacube.backends.FDB_datacube import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -12,23 +11,21 @@ class TestRegularGrid: def setup_method(self, method): - # TODO: for this test, need to fdb write era5 data and use fdb backend nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") - - ds = data.from_source("file", "./tests/data/era5-levels-members.grib") - self.latlon_array = ds.to_xarray().isel(step=0).t - self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { "values": { "transformation": { - "mapper": {"type": "regular", "resolution": 640, "axes": ["latitude", "longitude"]} + "mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]} } }, - "isobaricInhPa": {"transformation": {"reverse": {True}}} + "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, } + self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() - self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) def find_nearest_latlon(self, grib_file, target_lat, target_lon): # Open the GRIB file @@ -56,31 +53,39 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): @pytest.mark.internet def test_regular_grid(self): request = Request( - Box(["number", "isobaricInhPa"], [3, 0.0], [3, 500.0]), - Select("time", ["2017-01-02T12:00:00"]), - Box(["latitude", "longitude"], lower_corner=[0.0, 0.0], upper_corner=[3.0, 3.0]), - Select("step", [np.timedelta64(0, "s")]), + Select("step", [0]), + Select("levtype", ["pl"]), + Select("date", [pd.Timestamp("20170102T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["129"]), + Select("class", ["ea"]), + Select("stream", ["enda"]), + Select("type", ["an"]), + Box(["latitude", "longitude"], [0, 0], [3, 3]), + Select("levelist", ["500"]), + Select("number", ["0"]) ) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 4 - # lats = [] - # lons = [] - # eccodes_lats = [] - # tol = 1e-8 - # for i in range(len(result.leaves)): - # cubepath = result.leaves[i].flatten() - # lat = cubepath["latitude"] - # lon = cubepath["longitude"] - # lats.append(lat) - # lons.append(lon) - # nearest_points = self.find_nearest_latlon("./tests/data/foo.grib", lat, lon) - # eccodes_lat = nearest_points[0][0]["lat"] - # eccodes_lon = nearest_points[0][0]["lon"] - # eccodes_lats.append(eccodes_lat) - # assert eccodes_lat - tol <= lat - # assert lat <= eccodes_lat + tol - # assert eccodes_lon - tol <= lon - # assert lon <= eccodes_lon + tol - # assert len(eccodes_lats) == 9 + lats = [] + lons = [] + eccodes_lats = [] + tol = 1e-8 + for i in range(len(result.leaves)): + cubepath = result.leaves[i].flatten() + lat = cubepath["latitude"] + lon = cubepath["longitude"] + lats.append(lat) + lons.append(lon) + nearest_points = self.find_nearest_latlon("./tests/data/era5-levels-members.grib", lat, lon) + eccodes_lat = nearest_points[0][0]["lat"] + eccodes_lon = nearest_points[0][0]["lon"] + eccodes_lats.append(eccodes_lat) + assert eccodes_lat - tol <= lat + assert lat <= eccodes_lat + tol + assert eccodes_lon - tol <= lon + assert lon <= eccodes_lon + tol + assert len(eccodes_lats) == 4 From 17776a885ef89d4c48ae6feabfa1e60bbd5d6772 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 13 Oct 2023 13:46:31 +0100 Subject: [PATCH 165/332] small fixes and first performance test --- performance/fdb_performance.py | 47 ++++++++++++++++++++++ polytope/datacube/backends/FDB_datacube.py | 1 - tests/test_fdb_datacube.py | 21 ++++++++++ tests/test_regular_grid.py | 23 ++++++++--- 4 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 performance/fdb_performance.py diff --git a/performance/fdb_performance.py b/performance/fdb_performance.py new file mode 100644 index 000000000..57969bade --- /dev/null +++ b/performance/fdb_performance.py @@ -0,0 +1,47 @@ +import time + +import pandas as pd +import pytest + +from polytope.datacube.backends.FDB_datacube import FDBDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + } + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.skip(reason="can't install fdb branch on CI") + def test_fdb_datacube(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20230625T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Box(["latitude", "longitude"], [0, 0], [10, 10]), + ) + time1 = time.time() + result = self.API.retrieve(request) + print(time.time() - time1) + assert len(result.leaves) == 19226 diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 63f5cc701..2525d5057 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -47,7 +47,6 @@ def __init__(self, config={}, axis_options={}): self._check_and_add_axes(options, name, val) def get(self, requests: IndexTree): - requests.pprint() for r in requests.leaves: path = r.flatten() path = self.remap_path(path) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 4e48613d9..952b0bb7b 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -6,6 +6,9 @@ from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select +# import geopandas as gpd +# import matplotlib.pyplot as plt + class TestSlicingFDBDatacube: def setup_method(self, method): @@ -42,3 +45,21 @@ def test_fdb_datacube(self): result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 9 + + # lats = [] + # lons = [] + # tol = 1e-8 + # for i in range(len(result.leaves)): + # cubepath = result.leaves[i].flatten() + # lat = cubepath["latitude"] + # lon = cubepath["longitude"] + # lats.append(lat) + # lons.append(lon) + + # worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + # fig, ax = plt.subplots(figsize=(12, 6)) + # worldmap.plot(color="darkgrey", ax=ax) + + # plt.scatter(lons, lats, s=16, c="red", cmap="YlOrRd") + # plt.colorbar(label="Temperature") + # plt.show() diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 4ee3adc23..7247f73f6 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -6,7 +6,10 @@ from polytope.datacube.backends.FDB_datacube import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Box, Select +from polytope.shapes import Disk, Select + +# import geopandas as gpd +# import matplotlib.pyplot as plt class TestRegularGrid: @@ -51,6 +54,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_regular_grid(self): request = Request( Select("step", [0]), @@ -62,13 +66,13 @@ def test_regular_grid(self): Select("class", ["ea"]), Select("stream", ["enda"]), Select("type", ["an"]), - Box(["latitude", "longitude"], [0, 0], [3, 3]), + Disk(["latitude", "longitude"], [0, 0], [15, 15]), Select("levelist", ["500"]), - Select("number", ["0"]) + Select("number", ["0", "1"]) ) result = self.API.retrieve(request) result.pprint() - assert len(result.leaves) == 4 + assert len(result.leaves) == 46*2 lats = [] lons = [] @@ -88,4 +92,13 @@ def test_regular_grid(self): assert lat <= eccodes_lat + tol assert eccodes_lon - tol <= lon assert lon <= eccodes_lon + tol - assert len(eccodes_lats) == 4 + + # worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + # fig, ax = plt.subplots(figsize=(12, 6)) + # worldmap.plot(color="darkgrey", ax=ax) + + # plt.scatter(lons, lats, s=16, c="red", cmap="YlOrRd") + # plt.colorbar(label="Temperature") + # plt.show() + + assert len(eccodes_lats) == 46*2 From bb76ea4a88180bfe6b9385dfaf7e2d34002b4037 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 18 Oct 2023 11:37:45 +0200 Subject: [PATCH 166/332] make octahedral grid mapping faster and first fdb performance --- performance/fdb_performance.py | 2 +- polytope/datacube/backends/FDB_datacube.py | 20 ++++- polytope/datacube/backends/__init__.py | 1 + polytope/datacube/datacube_axis.py | 18 +++- .../transformations/datacube_mappers.py | 89 ++++++++++++------- .../transformations/datacube_merger.py | 2 +- .../transformations/datacube_type_change.py | 13 +-- polytope/utility/list_tools.py | 22 +++++ 8 files changed, 120 insertions(+), 47 deletions(-) create mode 100644 polytope/datacube/backends/__init__.py create mode 100644 polytope/utility/list_tools.py diff --git a/performance/fdb_performance.py b/performance/fdb_performance.py index 57969bade..9bcac25b7 100644 --- a/performance/fdb_performance.py +++ b/performance/fdb_performance.py @@ -27,7 +27,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 2525d5057..61404cd6f 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -1,4 +1,5 @@ import math +import time from copy import deepcopy import pyfdb @@ -47,16 +48,22 @@ def __init__(self, config={}, axis_options={}): self._check_and_add_axes(options, name, val) def get(self, requests: IndexTree): + time0 = time.time() + time_changing_path = 0 + accumulated_fdb_time = 0 for r in requests.leaves: path = r.flatten() path = self.remap_path(path) if len(path.items()) == self.axis_counter: # first, find the grid mapper transform + unmapped_path = {} path_copy = deepcopy(path) + time2 = time.time() for key in path_copy: axis = self._axes[key] (path, unmapped_path) = axis.unmap_total_path_to_datacube(path, unmapped_path) + time_changing_path += time.time() - time2 path = self.fit_path(path) # merge path and unmapped path into a single path path.update(unmapped_path) @@ -66,18 +73,23 @@ def get(self, requests: IndexTree): path.pop("values") fdb_request_key = path - fdb_requests = [(fdb_request_key, [(fdb_request_val, fdb_request_val+1)])] - + fdb_requests = [(fdb_request_key, [(fdb_request_val, fdb_request_val + 1)])] # need to request data from the fdb - + time1 = time.time() subxarray = self.fdb.extract(fdb_requests) + accumulated_fdb_time += time.time() - time1 subxarray_output_tuple = subxarray[0][0] output_value = subxarray_output_tuple[0][0][0] - if not math.isnan(output_value): r.result = output_value else: r.remove_branch() + print("FDB TIME") + print(accumulated_fdb_time) + print("GET TIME") + print(time.time() - time0) + print("TIME CHANGING PATH") + print(time_changing_path) def datacube_natural_indexes(self, axis, subarray): indexes = subarray[axis.name] diff --git a/polytope/datacube/backends/__init__.py b/polytope/datacube/backends/__init__.py new file mode 100644 index 000000000..63902115a --- /dev/null +++ b/polytope/datacube/backends/__init__.py @@ -0,0 +1 @@ +from ..backends.datacube import * diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 77a223710..91b5c6b61 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod from copy import deepcopy from typing import Any, List +import time import numpy as np import pandas as pd @@ -118,6 +119,7 @@ def find_indexes(path, datacube): old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube def unmap_total_path_to_datacube(path, unmapped_path): + time1 = time.time() for transform in cls.transformations: if isinstance(transform, DatacubeAxisCyclic): transformation = transform @@ -127,6 +129,8 @@ def unmap_total_path_to_datacube(path, unmapped_path): new_val = _remap_val_to_axis_range(old_val) path[cls.name] = new_val (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) + print("CYCLIC UNMAP TIME") + print(time.time() - time1) return (path, unmapped_path) old_unmap_to_datacube = cls.unmap_to_datacube @@ -235,11 +239,15 @@ def unmap_to_datacube(path, unmapped_path): old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube def unmap_total_path_to_datacube(path, unmapped_path): + # TODO: to be faster, could just compute the first lat unmapped idx, and do +1 for each subsequent lat idx? + # time1 = time.time() (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeMapper): + # time2 = time.time() transformation = transform if cls.name == transformation._mapped_axes()[0]: + # axis = cls.name # if we are on the first axis, then need to add the first val to unmapped_path first_val = path.get(cls.name, None) path.pop(cls.name, None) @@ -251,6 +259,7 @@ def unmap_total_path_to_datacube(path, unmapped_path): if cls.name == transformation._mapped_axes()[1]: # if we are on the second axis, then the val of the first axis is stored # inside unmapped_path so can get it from there + # axis = cls.name second_val = path.get(cls.name, None) path.pop(cls.name, None) first_val = unmapped_path.get(transformation._mapped_axes()[0], None) @@ -259,9 +268,16 @@ def unmap_total_path_to_datacube(path, unmapped_path): if first_val is None: first_val = path.get(transformation._mapped_axes()[0], None) path.pop(transformation._mapped_axes()[0], None) - if first_val is not None and second_val is not None: + if second_val is not None: unmapped_idx = transformation.unmap(first_val, second_val) unmapped_path[transformation.old_axis] = unmapped_idx + # time3 = time.time() + # print("MAPPER UNMAP TIME") + # print(time3 - time1) + # print("AXIS THIS IS FOR") + # print(axis) + # print("MAPPER TIME ONCE CHOSEN THE MAPPING") + # print(time3 - time2) return (path, unmapped_path) def remap_to_requested(path, unmapped_path): diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index a8e56aaf5..d3b6958df 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -1,6 +1,10 @@ import math from copy import deepcopy from importlib import import_module +from ...utility.list_tools import bisect_left_cmp, bisect_right_cmp +import bisect +import time +import array from .datacube_transformations import DatacubeAxisTransformation @@ -14,6 +18,7 @@ def __init__(self, name, mapper_options): self.grid_resolution = mapper_options["resolution"] self.grid_axes = mapper_options["axes"] self.old_axis = name + self._final_transformation = self.generate_final_transformation() def generate_final_transformation(self): map_type = _type_to_datacube_mapper_lookup[self.grid_type] @@ -26,8 +31,8 @@ def blocked_axes(self): return [] def transformation_axes_final(self): - final_transformation = self.generate_final_transformation() - final_axes = final_transformation._mapped_axes + # final_transformation = self.generate_final_transformation() + final_axes = self._final_transformation._mapped_axes return final_axes # Needs to also implement its own methods @@ -38,8 +43,8 @@ def change_val_type(self, axis_name, values): def _mapped_axes(self): # NOTE: Each of the mapper method needs to call it's sub mapper method - final_transformation = self.generate_final_transformation() - final_axes = final_transformation._mapped_axes + # final_transformation = self.generate_final_transformation() + final_axes = self._final_transformation._mapped_axes return final_axes def _base_axis(self): @@ -49,24 +54,24 @@ def _resolution(self): pass def first_axis_vals(self): - final_transformation = self.generate_final_transformation() - return final_transformation.first_axis_vals() + # final_transformation = self.generate_final_transformation() + return self._final_transformation.first_axis_vals() def second_axis_vals(self, first_val): - final_transformation = self.generate_final_transformation() - return final_transformation.second_axis_vals(first_val) + # final_transformation = self.generate_final_transformation() + return self._final_transformation.second_axis_vals(first_val) def map_first_axis(self, lower, upper): - final_transformation = self.generate_final_transformation() - return final_transformation.map_first_axis(lower, upper) + # final_transformation = self.generate_final_transformation() + return self._final_transformation.map_first_axis(lower, upper) def map_second_axis(self, first_val, lower, upper): - final_transformation = self.generate_final_transformation() - return final_transformation.map_second_axis(first_val, lower, upper) + # final_transformation = self.generate_final_transformation() + return self._final_transformation.map_second_axis(first_val, lower, upper) def unmap(self, first_val, second_val): - final_transformation = self.generate_final_transformation() - return final_transformation.unmap(first_val, second_val) + # final_transformation = self.generate_final_transformation() + return self._final_transformation.unmap(first_val, second_val) class RegularGridMapper(DatacubeMapper): @@ -74,10 +79,10 @@ def __init__(self, base_axis, mapped_axes, resolution): self._mapped_axes = mapped_axes self._base_axis = base_axis self._resolution = resolution - self.deg_increment = 90/self._resolution + self.deg_increment = 90 / self._resolution def first_axis_vals(self): - first_ax_vals = [-90 + i * self.deg_increment for i in range(2*self._resolution)] + first_ax_vals = [-90 + i * self.deg_increment for i in range(2 * self._resolution)] return first_ax_vals def map_first_axis(self, lower, upper): @@ -86,7 +91,7 @@ def map_first_axis(self, lower, upper): return return_vals def second_axis_vals(self, first_val): - second_ax_vals = [i * self.deg_increment for i in range(4*self._resolution)] + second_ax_vals = [i * self.deg_increment for i in range(4 * self._resolution)] return second_ax_vals def map_second_axis(self, first_val, lower, upper): @@ -99,7 +104,6 @@ def axes_idx_to_regular_idx(self, first_idx, second_idx): return final_idx def unmap(self, first_val, second_val): - print("used unmap for regular grid !!!!!") tol = 1e-8 first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] first_idx = self.first_axis_vals().index(first_val) @@ -201,6 +205,7 @@ def __init__(self, base_axis, mapped_axes, resolution): self._mapped_axes = mapped_axes self._base_axis = base_axis self._resolution = resolution + self._first_axis_vals = self.first_axis_vals() def gauss_first_guess(self): i = 0 @@ -2859,30 +2864,41 @@ def first_axis_vals(self): return new_vals def map_first_axis(self, lower, upper): - axis_lines = self.first_axis_vals() - return_vals = [val for val in axis_lines if lower <= val <= upper] + axis_lines = self._first_axis_vals + # return_vals = [val for val in axis_lines if lower <= val <= upper] + end_idx = bisect_left_cmp(axis_lines, lower, cmp=lambda x, y: x > y) + 1 + start_idx = bisect_right_cmp(axis_lines, upper, cmp=lambda x, y: x > y) + return_vals = axis_lines[start_idx:end_idx] return return_vals def second_axis_vals(self, first_val): - first_axis_vals = self.first_axis_vals() + first_axis_vals = self._first_axis_vals tol = 1e-10 - first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] - first_idx = first_axis_vals.index(first_val) + # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] + # first_idx = first_axis_vals.index(first_val) + first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) if first_idx >= self._resolution: first_idx = (2 * self._resolution) - 1 - first_idx first_idx = first_idx + 1 npoints = 4 * first_idx + 16 second_axis_spacing = 360 / npoints - second_axis_start = 0 - second_axis_vals = [second_axis_start + i * second_axis_spacing for i in range(int(npoints))] + second_axis_vals = [i * second_axis_spacing for i in range(npoints)] return second_axis_vals def map_second_axis(self, first_val, lower, upper): second_axis_vals = self.second_axis_vals(first_val) - return_vals = [val for val in second_axis_vals if lower <= val <= upper] + # NOTE: here this seems faster than the bisect.bisect? + # return_vals = [val for val in second_axis_vals if lower <= val <= upper] + start_idx = bisect_left_cmp(second_axis_vals, lower, cmp=lambda x, y: x < y) + 1 + end_idx = bisect_right_cmp(second_axis_vals, upper, cmp=lambda x, y: x < y) + 1 + return_vals = second_axis_vals[start_idx:end_idx] + # start_idx = bisect.bisect_left(second_axis_vals, lower) + # end_idx = bisect.bisect_right(second_axis_vals, upper) + # return_vals = second_axis_vals[start_idx:end_idx] return return_vals def axes_idx_to_octahedral_idx(self, first_idx, second_idx): + # NOTE: for now this takes ~2e-4s per point, so taking significant time octa_idx = 0 if first_idx == 1: octa_idx = second_idx @@ -2901,17 +2917,22 @@ def axes_idx_to_octahedral_idx(self, first_idx, second_idx): return octa_idx def unmap(self, first_val, second_val): - first_axis_vals = self.first_axis_vals() + first_axis_vals = self._first_axis_vals tol = 1e-10 - first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] - first_idx = first_axis_vals.index(first_val) + 1 + # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] + # first_idx = first_axis_vals.index(first_val) + 1 + # first_idx = len(first_axis_vals) - bisect.bisect_left(first_axis_vals[::-1], first_val - tol) + first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + 1 second_axis_vals = self.second_axis_vals(first_val) - second_val = [val for val in second_axis_vals if second_val - tol < val < second_val + tol][0] - second_idx = second_axis_vals.index(second_val) + # second_val = [val for val in second_axis_vals if second_val - tol < val < second_val + tol][0] + # second_idx = second_axis_vals.index(second_val) + second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) octahedral_index = self.axes_idx_to_octahedral_idx(first_idx, second_idx) return octahedral_index -_type_to_datacube_mapper_lookup = {"octahedral": "OctahedralGridMapper", - "healpix": "HealpixGridMapper", - "regular": "RegularGridMapper"} +_type_to_datacube_mapper_lookup = { + "octahedral": "OctahedralGridMapper", + "healpix": "HealpixGridMapper", + "regular": "RegularGridMapper", +} diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index e38a57d8c..e8ff67145 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -46,7 +46,7 @@ def unmerge(self, merged_val): second_val = merged_val[first_idx + first_linker_size : -second_linked_size] # TODO: maybe replacing like this is too specific to time/dates? - first_val = str(first_val).replace('-', '') + first_val = str(first_val).replace("-", "") second_val = second_val.replace(":", "") return (first_val, second_val) diff --git a/polytope/datacube/transformations/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change.py index e3e8a682c..0f4cedd21 100644 --- a/polytope/datacube/transformations/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change.py @@ -11,6 +11,7 @@ def __init__(self, name, type_options): self.name = name self.transformation_options = type_options self.new_type = type_options + self._final_transformation = self.generate_final_transformation() def generate_final_transformation(self): map_type = _type_to_datacube_type_change_lookup[self.new_type] @@ -20,16 +21,16 @@ def generate_final_transformation(self): return transformation def transformation_axes_final(self): - final_transformation = self.generate_final_transformation() - return [final_transformation.axis_name] + # final_transformation = self.generate_final_transformation() + return [self._final_transformation.axis_name] def change_val_type(self, axis_name, values): - transformation = self.generate_final_transformation() - return [transformation.transform_type(val) for val in values] + # transformation = self.generate_final_transformation() + return [self._final_transformation.transform_type(val) for val in values] def make_str(self, value): - transformation = self.generate_final_transformation() - return transformation.make_str(value) + # transformation = self.generate_final_transformation() + return self._final_transformation.make_str(value) def blocked_axes(self): return [] diff --git a/polytope/utility/list_tools.py b/polytope/utility/list_tools.py new file mode 100644 index 000000000..2d18917c1 --- /dev/null +++ b/polytope/utility/list_tools.py @@ -0,0 +1,22 @@ +def bisect_left_cmp(arr, val, cmp): + left = -1 + r = len(arr) + while r - left > 1: + e = (left + r) >> 1 + if cmp(arr[e], val): + left = e + else: + r = e + return left + + +def bisect_right_cmp(arr, val, cmp): + left = -1 + r = len(arr) + while r - left > 1: + e = (left + r) >> 1 + if cmp(arr[e], val): + left = e + else: + r = e + return r From 6f2dbd8299c6aeff7f7921c4f4034a96f63d2fad Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 18 Oct 2023 15:35:46 +0200 Subject: [PATCH 167/332] create dictionary of latitude lines in octahedral mapping --- polytope/datacube/datacube_axis.py | 14 +++ .../transformations/datacube_mappers.py | 115 +++++++++++++++++- 2 files changed, 125 insertions(+), 4 deletions(-) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 91b5c6b61..07edd8d40 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -260,17 +260,31 @@ def unmap_total_path_to_datacube(path, unmapped_path): # if we are on the second axis, then the val of the first axis is stored # inside unmapped_path so can get it from there # axis = cls.name + # print("TIME time handling path") + # time2 = time.time() second_val = path.get(cls.name, None) path.pop(cls.name, None) first_val = unmapped_path.get(transformation._mapped_axes()[0], None) + unmapped_path.pop(transformation._mapped_axes()[0], None) + # print(time.time() - time2) + # NOTE: here we first calculate the starting idx of the first_val grid line + # and then append the second_idx to get the final unmapped_idx + # To do this, also need to find second_idx from second_val... + # second_idx = transformation.find_second_idx(first_val, second_val) + # first_line_idx = transformation.unmap_first_val_to_start_line_idx(first_val) + # if the first_val was not in the unmapped_path, then it's still in path + # print("AAAAAND TIME TAKEN DOING UNMAP") if first_val is None: first_val = path.get(transformation._mapped_axes()[0], None) path.pop(transformation._mapped_axes()[0], None) if second_val is not None: + # time4 = time.time() + # unmapped_idx = first_line_idx + second_idx unmapped_idx = transformation.unmap(first_val, second_val) unmapped_path[transformation.old_axis] = unmapped_idx + # print(time.time() - time4) # time3 = time.time() # print("MAPPER UNMAP TIME") # print(time3 - time1) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index d3b6958df..06b7b57d1 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -69,6 +69,12 @@ def map_second_axis(self, first_val, lower, upper): # final_transformation = self.generate_final_transformation() return self._final_transformation.map_second_axis(first_val, lower, upper) + def find_second_idx(self, first_val, second_val): + return self._final_transformation.find_second_idx(first_val, second_val) + + def unmap_first_val_to_start_line_idx(self, first_val): + return self._final_transformation.unmap_first_val_to_start_line_idx(first_val) + def unmap(self, first_val, second_val): # final_transformation = self.generate_final_transformation() return self._final_transformation.unmap(first_val, second_val) @@ -103,6 +109,18 @@ def axes_idx_to_regular_idx(self, first_idx, second_idx): final_idx = first_idx * 4 * self._resolution + second_idx return final_idx + def find_second_idx(self, first_val, second_val): + tol = 1e-10 + second_axis_vals = self.second_axis_vals(first_val) + second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) + return second_idx + + def unmap_first_val_to_start_line_idx(self, first_val): + tol = 1e-8 + first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] + first_idx = self.first_axis_vals().index(first_val) + return first_idx * 4 * self._resolution + def unmap(self, first_val, second_val): tol = 1e-8 first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] @@ -190,6 +208,33 @@ def axes_idx_to_healpix_idx(self, first_idx, second_idx): idx += second_idx return idx + def find_second_idx(self, first_val, second_val): + tol = 1e-10 + second_axis_vals = self.second_axis_vals(first_val) + second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) + return second_idx + + def unmap_first_val_to_start_line_idx(self, first_val): + tol = 1e-8 + first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] + first_idx = self.first_axis_vals().index(first_val) + idx = 0 + for i in range(self._resolution - 1): + if i != first_idx: + idx += 4 * (i + 1) + else: + return idx + for i in range(self._resolution - 1, 3 * self._resolution): + if i != first_idx: + idx += 4 * self._resolution + else: + return idx + for i in range(3 * self._resolution, 4 * self._resolution - 1): + if i != first_idx: + idx += 4 * (4 * self._resolution - 1 - i + 1) + else: + return idx + def unmap(self, first_val, second_val): tol = 1e-8 first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] @@ -206,6 +251,7 @@ def __init__(self, base_axis, mapped_axes, resolution): self._base_axis = base_axis self._resolution = resolution self._first_axis_vals = self.first_axis_vals() + self._first_idx_map = self.create_first_idx_map() def gauss_first_guess(self): i = 0 @@ -2898,10 +2944,67 @@ def map_second_axis(self, first_val, lower, upper): return return_vals def axes_idx_to_octahedral_idx(self, first_idx, second_idx): - # NOTE: for now this takes ~2e-4s per point, so taking significant time + # NOTE: for now this takes ~2e-4s per point, so taking significant time -> for 20k points, takes 4s + # Would it be better to store a dictionary of first_idx with cumulative number of points on that idx? + # Because this is what we are doing here, but we are calculating for each point... + # But then this would only work for special grid resolutions, so need to do like a O1280 version of this + + # NOTE: OR somehow cache this for a given first_idx and then only modify the axis idx for second_idx when the + # first_idx changes + # time1 = time.time() + octa_idx = self._first_idx_map[first_idx-1] + second_idx + # octa_idx = 0 + # if first_idx == 1: + # octa_idx = second_idx + # else: + # for i in range(first_idx - 1): + # if i <= self._resolution - 1: + # octa_idx += 20 + 4 * i + # else: + # i = i - self._resolution + 1 + # if i == 1: + # octa_idx += 16 + 4 * self._resolution + # else: + # i = i - 1 + # octa_idx += 16 + 4 * (self._resolution - i) + # octa_idx += second_idx + # print("TIME UNMAPPING TO OCT IDX") + # print(time.time() - time1) + return octa_idx + + def find_second_idx(self, first_val, second_val): + tol = 1e-10 + second_axis_vals = self.second_axis_vals(first_val) + second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) + return second_idx + + def create_first_idx_map(self): + # first_idx_list = [0] * (2*self._resolution) + first_idx_list = {} + idx = 0 + for i in range(2*self._resolution): + first_idx_list[i] = idx + if i <= self._resolution - 1: + idx += 20 + 4 * i + else: + i = i - self._resolution + 1 + if i == 1: + idx += 16 + 4 * self._resolution + else: + i = i - 1 + idx += 16 + 4 * (self._resolution - i) + return first_idx_list + + def unmap_first_val_to_start_line_idx(self, first_val): + first_axis_vals = self._first_axis_vals + tol = 1e-10 + # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] + # first_idx = first_axis_vals.index(first_val) + 1 + # first_idx = len(first_axis_vals) - bisect.bisect_left(first_axis_vals[::-1], first_val - tol) + first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + 1 octa_idx = 0 if first_idx == 1: - octa_idx = second_idx + return octa_idx else: for i in range(first_idx - 1): if i <= self._resolution - 1: @@ -2913,10 +3016,10 @@ def axes_idx_to_octahedral_idx(self, first_idx, second_idx): else: i = i - 1 octa_idx += 16 + 4 * (self._resolution - i) - octa_idx += second_idx - return octa_idx + return octa_idx def unmap(self, first_val, second_val): + time1 = time.time() first_axis_vals = self._first_axis_vals tol = 1e-10 # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] @@ -2927,7 +3030,11 @@ def unmap(self, first_val, second_val): # second_val = [val for val in second_axis_vals if second_val - tol < val < second_val + tol][0] # second_idx = second_axis_vals.index(second_val) second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) + print("TIME SPENT DOING VAL TO IDX") + print(time.time() - time1) octahedral_index = self.axes_idx_to_octahedral_idx(first_idx, second_idx) + print("OCTAHEDRAL UNMAP TIME") + print(time.time() - time1) return octahedral_index From 7b18dce42a45c0265430f6c2a3243e703b3676bc Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 19 Oct 2023 13:11:49 +0200 Subject: [PATCH 168/332] add some sort of caching for longitude values in unmap --- .../transformations/datacube_mappers.py | 2598 ++++++++++++++++- 1 file changed, 2591 insertions(+), 7 deletions(-) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 06b7b57d1..40870f1ac 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -5,6 +5,8 @@ import bisect import time import array +import numpy as np +from sortedcontainers import SortedDict, SortedList from .datacube_transformations import DatacubeAxisTransformation @@ -251,7 +253,10 @@ def __init__(self, base_axis, mapped_axes, resolution): self._base_axis = base_axis self._resolution = resolution self._first_axis_vals = self.first_axis_vals() + self._inv_first_axis_vals = self._first_axis_vals[::-1] + # self._inv_first_axis_vals = {v:k for k,v in self._first_axis_vals.items()} self._first_idx_map = self.create_first_idx_map() + self.treated_first_vals = dict() def gauss_first_guess(self): i = 0 @@ -319,6 +324,2572 @@ def gauss_first_guess(self): def get_precomputed_values_N1280(self): lats = [0] * 2560 + # lats = SortedList() + # lats = {} + lats[0] = 89.946187715665616 + lats[1] = 89.876478353332288 + lats[2] = 89.806357319542244 + lats[3] = 89.736143271609578 + lats[4] = 89.6658939412157 + lats[5] = 89.595627537554492 + lats[6] = 89.525351592371393 + lats[7] = 89.45506977912261 + lats[8] = 89.3847841013921 + lats[9] = 89.314495744374256 + lats[10] = 89.24420545380525 + lats[11] = 89.173913722284126 + lats[12] = 89.103620888238879 + lats[13] = 89.033327191845927 + lats[14] = 88.96303280826325 + lats[15] = 88.892737868230952 + lats[16] = 88.822442471310097 + lats[17] = 88.752146694650691 + lats[18] = 88.681850598961759 + lats[19] = 88.611554232668382 + lats[20] = 88.541257634868515 + lats[21] = 88.470960837474877 + lats[22] = 88.40066386679355 + lats[23] = 88.330366744702559 + lats[24] = 88.26006948954614 + lats[25] = 88.189772116820762 + lats[26] = 88.119474639706425 + lats[27] = 88.049177069484486 + lats[28] = 87.978879415867283 + lats[29] = 87.908581687261687 + lats[30] = 87.838283890981543 + lats[31] = 87.767986033419561 + lats[32] = 87.697688120188062 + lats[33] = 87.627390156234085 + lats[34] = 87.557092145935584 + lats[35] = 87.486794093180748 + lats[36] = 87.416496001434894 + lats[37] = 87.346197873795816 + lats[38] = 87.275899713041966 + lats[39] = 87.205601521672108 + lats[40] = 87.135303301939786 + lats[41] = 87.065005055882821 + lats[42] = 86.994706785348129 + lats[43] = 86.924408492014166 + lats[44] = 86.854110177408927 + lats[45] = 86.783811842927179 + lats[46] = 86.713513489844246 + lats[47] = 86.643215119328573 + lats[48] = 86.572916732453024 + lats[49] = 86.502618330203831 + lats[50] = 86.432319913489792 + lats[51] = 86.362021483149363 + lats[52] = 86.291723039957418 + lats[53] = 86.221424584631109 + lats[54] = 86.151126117835304 + lats[55] = 86.080827640187209 + lats[56] = 86.010529152260403 + lats[57] = 85.940230654588888 + lats[58] = 85.869932147670127 + lats[59] = 85.799633631968391 + lats[60] = 85.729335107917464 + lats[61] = 85.659036575922883 + lats[62] = 85.588738036364362 + lats[63] = 85.518439489597966 + lats[64] = 85.448140935957483 + lats[65] = 85.377842375756586 + lats[66] = 85.307543809290152 + lats[67] = 85.237245236835548 + lats[68] = 85.16694665865414 + lats[69] = 85.09664807499216 + lats[70] = 85.026349486081983 + lats[71] = 84.95605089214304 + lats[72] = 84.885752293382765 + lats[73] = 84.81545368999717 + lats[74] = 84.745155082171991 + lats[75] = 84.674856470082915 + lats[76] = 84.604557853896708 + lats[77] = 84.534259233771479 + lats[78] = 84.463960609857125 + lats[79] = 84.393661982296322 + lats[80] = 84.323363351224444 + lats[81] = 84.253064716770425 + lats[82] = 84.18276607905679 + lats[83] = 84.112467438200326 + lats[84] = 84.042168794312317 + lats[85] = 83.971870147498763 + lats[86] = 83.901571497860914 + lats[87] = 83.831272845495249 + lats[88] = 83.760974190494011 + lats[89] = 83.690675532945292 + lats[90] = 83.620376872933264 + lats[91] = 83.550078210538487 + lats[92] = 83.479779545838113 + lats[93] = 83.409480878905782 + lats[94] = 83.339182209812321 + lats[95] = 83.268883538625232 + lats[96] = 83.198584865409657 + lats[97] = 83.128286190227698 + lats[98] = 83.057987513139125 + lats[99] = 82.987688834201322 + lats[100] = 82.917390153469313 + lats[101] = 82.84709147099602 + lats[102] = 82.77679278683226 + lats[103] = 82.706494101026948 + lats[104] = 82.63619541362705 + lats[105] = 82.56589672467787 + lats[106] = 82.495598034222837 + lats[107] = 82.425299342304029 + lats[108] = 82.355000648961692 + lats[109] = 82.284701954234833 + lats[110] = 82.214403258160871 + lats[111] = 82.144104560776 + lats[112] = 82.073805862115165 + lats[113] = 82.003507162211946 + lats[114] = 81.933208461098829 + lats[115] = 81.862909758807191 + lats[116] = 81.792611055367345 + lats[117] = 81.722312350808508 + lats[118] = 81.652013645158945 + lats[119] = 81.581714938445955 + lats[120] = 81.511416230696042 + lats[121] = 81.441117521934686 + lats[122] = 81.370818812186627 + lats[123] = 81.300520101475826 + lats[124] = 81.230221389825374 + lats[125] = 81.159922677257711 + lats[126] = 81.089623963794551 + lats[127] = 81.019325249456955 + lats[128] = 80.949026534265244 + lats[129] = 80.878727818239184 + lats[130] = 80.808429101397948 + lats[131] = 80.73813038376008 + lats[132] = 80.667831665343556 + lats[133] = 80.59753294616587 + lats[134] = 80.527234226243991 + lats[135] = 80.456935505594302 + lats[136] = 80.386636784232863 + lats[137] = 80.316338062175078 + lats[138] = 80.246039339436052 + lats[139] = 80.175740616030438 + lats[140] = 80.105441891972376 + lats[141] = 80.035143167275749 + lats[142] = 79.9648444419539 + lats[143] = 79.894545716019948 + lats[144] = 79.824246989486554 + lats[145] = 79.753948262366038 + lats[146] = 79.683649534670437 + lats[147] = 79.61335080641139 + lats[148] = 79.543052077600308 + lats[149] = 79.472753348248219 + lats[150] = 79.402454618365894 + lats[151] = 79.332155887963822 + lats[152] = 79.261857157052191 + lats[153] = 79.191558425640977 + lats[154] = 79.121259693739859 + lats[155] = 79.050960961358285 + lats[156] = 78.980662228505423 + lats[157] = 78.910363495190211 + lats[158] = 78.840064761421445 + lats[159] = 78.769766027207638 + lats[160] = 78.699467292557102 + lats[161] = 78.629168557477882 + lats[162] = 78.558869821977908 + lats[163] = 78.488571086064923 + lats[164] = 78.418272349746417 + lats[165] = 78.347973613029708 + lats[166] = 78.277674875922045 + lats[167] = 78.207376138430348 + lats[168] = 78.137077400561424 + lats[169] = 78.066778662322022 + lats[170] = 77.996479923718596 + lats[171] = 77.926181184757539 + lats[172] = 77.855882445445019 + lats[173] = 77.785583705787161 + lats[174] = 77.71528496578982 + lats[175] = 77.644986225458879 + lats[176] = 77.574687484799924 + lats[177] = 77.504388743818524 + lats[178] = 77.434090002520122 + lats[179] = 77.363791260909963 + lats[180] = 77.293492518993247 + lats[181] = 77.22319377677502 + lats[182] = 77.15289503426024 + lats[183] = 77.082596291453768 + lats[184] = 77.012297548360323 + lats[185] = 76.941998804984564 + lats[186] = 76.871700061330955 + lats[187] = 76.801401317404 + lats[188] = 76.731102573208048 + lats[189] = 76.660803828747362 + lats[190] = 76.59050508402602 + lats[191] = 76.520206339048215 + lats[192] = 76.449907593817869 + lats[193] = 76.379608848338933 + lats[194] = 76.3093101026152 + lats[195] = 76.239011356650423 + lats[196] = 76.16871261044831 + lats[197] = 76.098413864012443 + lats[198] = 76.028115117346374 + lats[199] = 75.957816370453543 + lats[200] = 75.887517623337317 + lats[201] = 75.81721887600105 + lats[202] = 75.746920128447996 + lats[203] = 75.67662138068134 + lats[204] = 75.60632263270422 + lats[205] = 75.536023884519707 + lats[206] = 75.465725136130786 + lats[207] = 75.395426387540439 + lats[208] = 75.325127638751567 + lats[209] = 75.254828889766983 + lats[210] = 75.184530140589501 + lats[211] = 75.114231391221821 + lats[212] = 75.043932641666672 + lats[213] = 74.973633891926625 + lats[214] = 74.903335142004323 + lats[215] = 74.833036391902269 + lats[216] = 74.762737641622991 + lats[217] = 74.692438891168877 + lats[218] = 74.622140140542356 + lats[219] = 74.551841389745761 + lats[220] = 74.481542638781434 + lats[221] = 74.411243887651622 + lats[222] = 74.340945136358584 + lats[223] = 74.270646384904481 + lats[224] = 74.200347633291472 + lats[225] = 74.13004888152166 + lats[226] = 74.059750129597163 + lats[227] = 73.98945137751997 + lats[228] = 73.919152625292114 + lats[229] = 73.848853872915541 + lats[230] = 73.778555120392184 + lats[231] = 73.70825636772399 + lats[232] = 73.637957614912779 + lats[233] = 73.567658861960396 + lats[234] = 73.497360108868662 + lats[235] = 73.427061355639339 + lats[236] = 73.356762602274188 + lats[237] = 73.2864638487749 + lats[238] = 73.216165095143182 + lats[239] = 73.145866341380668 + lats[240] = 73.075567587489019 + lats[241] = 73.005268833469799 + lats[242] = 72.934970079324657 + lats[243] = 72.864671325055056 + lats[244] = 72.794372570662574 + lats[245] = 72.724073816148703 + lats[246] = 72.653775061514935 + lats[247] = 72.583476306762691 + lats[248] = 72.513177551893421 + lats[249] = 72.442878796908545 + lats[250] = 72.3725800418094 + lats[251] = 72.302281286597392 + lats[252] = 72.231982531273843 + lats[253] = 72.161683775840089 + lats[254] = 72.091385020297409 + lats[255] = 72.02108626464711 + lats[256] = 71.950787508890414 + lats[257] = 71.880488753028587 + lats[258] = 71.810189997062835 + lats[259] = 71.739891240994368 + lats[260] = 71.669592484824364 + lats[261] = 71.599293728553988 + lats[262] = 71.528994972184378 + lats[263] = 71.458696215716685 + lats[264] = 71.388397459152031 + lats[265] = 71.318098702491469 + lats[266] = 71.247799945736105 + lats[267] = 71.177501188887007 + lats[268] = 71.107202431945211 + lats[269] = 71.036903674911756 + lats[270] = 70.966604917787635 + lats[271] = 70.896306160573886 + lats[272] = 70.826007403271475 + lats[273] = 70.755708645881384 + lats[274] = 70.685409888404578 + lats[275] = 70.615111130841967 + lats[276] = 70.544812373194532 + lats[277] = 70.474513615463138 + lats[278] = 70.404214857648739 + lats[279] = 70.333916099752187 + lats[280] = 70.263617341774406 + lats[281] = 70.193318583716191 + lats[282] = 70.123019825578467 + lats[283] = 70.052721067362043 + lats[284] = 69.982422309067744 + lats[285] = 69.912123550696421 + lats[286] = 69.841824792248843 + lats[287] = 69.771526033725834 + lats[288] = 69.701227275128161 + lats[289] = 69.630928516456592 + lats[290] = 69.560629757711908 + lats[291] = 69.490330998894862 + lats[292] = 69.420032240006194 + lats[293] = 69.349733481046613 + lats[294] = 69.279434722016902 + lats[295] = 69.209135962917699 + lats[296] = 69.138837203749759 + lats[297] = 69.068538444513763 + lats[298] = 68.998239685210365 + lats[299] = 68.927940925840304 + lats[300] = 68.85764216640419 + lats[301] = 68.787343406902693 + lats[302] = 68.717044647336493 + lats[303] = 68.646745887706189 + lats[304] = 68.576447128012447 + lats[305] = 68.506148368255865 + lats[306] = 68.435849608437067 + lats[307] = 68.365550848556666 + lats[308] = 68.295252088615257 + lats[309] = 68.224953328613438 + lats[310] = 68.154654568551791 + lats[311] = 68.084355808430871 + lats[312] = 68.014057048251274 + lats[313] = 67.943758288013555 + lats[314] = 67.873459527718282 + lats[315] = 67.803160767365966 + lats[316] = 67.732862006957205 + lats[317] = 67.662563246492482 + lats[318] = 67.592264485972336 + lats[319] = 67.521965725397308 + lats[320] = 67.451666964767895 + lats[321] = 67.381368204084609 + lats[322] = 67.311069443347961 + lats[323] = 67.240770682558434 + lats[324] = 67.170471921716526 + lats[325] = 67.100173160822706 + lats[326] = 67.029874399877471 + lats[327] = 66.95957563888129 + lats[328] = 66.889276877834618 + lats[329] = 66.818978116737924 + lats[330] = 66.748679355591662 + lats[331] = 66.678380594396273 + lats[332] = 66.608081833152212 + lats[333] = 66.537783071859891 + lats[334] = 66.467484310519808 + lats[335] = 66.397185549132331 + lats[336] = 66.326886787697887 + lats[337] = 66.256588026216932 + lats[338] = 66.186289264689833 + lats[339] = 66.115990503117033 + lats[340] = 66.045691741498899 + lats[341] = 65.975392979835888 + lats[342] = 65.905094218128355 + lats[343] = 65.834795456376696 + lats[344] = 65.764496694581283 + lats[345] = 65.694197932742526 + lats[346] = 65.623899170860767 + lats[347] = 65.553600408936404 + lats[348] = 65.483301646969792 + lats[349] = 65.413002884961315 + lats[350] = 65.342704122911286 + lats[351] = 65.272405360820116 + lats[352] = 65.202106598688133 + lats[353] = 65.131807836515677 + lats[354] = 65.061509074303089 + lats[355] = 64.991210312050711 + lats[356] = 64.920911549758912 + lats[357] = 64.850612787427963 + lats[358] = 64.780314025058246 + lats[359] = 64.710015262650074 + lats[360] = 64.639716500203733 + lats[361] = 64.569417737719576 + lats[362] = 64.499118975197902 + lats[363] = 64.428820212639039 + lats[364] = 64.358521450043284 + lats[365] = 64.288222687410922 + lats[366] = 64.21792392474228 + lats[367] = 64.147625162037642 + lats[368] = 64.07732639929732 + lats[369] = 64.00702763652157 + lats[370] = 63.93672887371072 + lats[371] = 63.866430110865004 + lats[372] = 63.796131347984762 + lats[373] = 63.725832585070251 + lats[374] = 63.655533822121711 + lats[375] = 63.585235059139464 + lats[376] = 63.514936296123757 + lats[377] = 63.444637533074854 + lats[378] = 63.374338769993031 + lats[379] = 63.304040006878537 + lats[380] = 63.23374124373165 + lats[381] = 63.163442480552604 + lats[382] = 63.093143717341647 + lats[383] = 63.022844954099064 + lats[384] = 62.952546190825068 + lats[385] = 62.882247427519928 + lats[386] = 62.811948664183866 + lats[387] = 62.741649900817137 + lats[388] = 62.67135113741999 + lats[389] = 62.60105237399263 + lats[390] = 62.530753610535321 + lats[391] = 62.460454847048261 + lats[392] = 62.3901560835317 + lats[393] = 62.319857319985871 + lats[394] = 62.249558556410982 + lats[395] = 62.179259792807258 + lats[396] = 62.108961029174914 + lats[397] = 62.038662265514176 + lats[398] = 61.968363501825259 + lats[399] = 61.898064738108381 + lats[400] = 61.827765974363729 + lats[401] = 61.757467210591535 + lats[402] = 61.687168446791986 + lats[403] = 61.616869682965287 + lats[404] = 61.546570919111666 + lats[405] = 61.476272155231321 + lats[406] = 61.405973391324409 + lats[407] = 61.335674627391185 + lats[408] = 61.265375863431785 + lats[409] = 61.195077099446451 + lats[410] = 61.124778335435344 + lats[411] = 61.054479571398652 + lats[412] = 60.984180807336578 + lats[413] = 60.913882043249295 + lats[414] = 60.843583279137007 + lats[415] = 60.773284514999872 + lats[416] = 60.702985750838074 + lats[417] = 60.632686986651805 + lats[418] = 60.562388222441243 + lats[419] = 60.492089458206543 + lats[420] = 60.421790693947884 + lats[421] = 60.35149192966545 + lats[422] = 60.28119316535939 + lats[423] = 60.21089440102989 + lats[424] = 60.140595636677112 + lats[425] = 60.070296872301235 + lats[426] = 59.999998107902378 + lats[427] = 59.929699343480763 + lats[428] = 59.859400579036503 + lats[429] = 59.78910181456979 + lats[430] = 59.718803050080759 + lats[431] = 59.64850428556958 + lats[432] = 59.578205521036402 + lats[433] = 59.507906756481383 + lats[434] = 59.43760799190467 + lats[435] = 59.3673092273064 + lats[436] = 59.29701046268675 + lats[437] = 59.226711698045854 + lats[438] = 59.156412933383855 + lats[439] = 59.086114168700909 + lats[440] = 59.015815403997145 + lats[441] = 58.945516639272725 + lats[442] = 58.875217874527763 + lats[443] = 58.804919109762423 + lats[444] = 58.73462034497684 + lats[445] = 58.664321580171141 + lats[446] = 58.594022815345468 + lats[447] = 58.523724050499972 + lats[448] = 58.453425285634758 + lats[449] = 58.383126520749968 + lats[450] = 58.312827755845746 + lats[451] = 58.242528990922203 + lats[452] = 58.172230225979497 + lats[453] = 58.101931461017728 + lats[454] = 58.031632696037022 + lats[455] = 57.961333931037537 + lats[456] = 57.891035166019364 + lats[457] = 57.820736400982646 + lats[458] = 57.75043763592749 + lats[459] = 57.680138870854037 + lats[460] = 57.60984010576238 + lats[461] = 57.539541340652676 + lats[462] = 57.469242575525016 + lats[463] = 57.398943810379521 + lats[464] = 57.328645045216312 + lats[465] = 57.258346280035504 + lats[466] = 57.188047514837208 + lats[467] = 57.117748749621541 + lats[468] = 57.047449984388614 + lats[469] = 56.977151219138541 + lats[470] = 56.90685245387143 + lats[471] = 56.836553688587379 + lats[472] = 56.766254923286517 + lats[473] = 56.695956157968951 + lats[474] = 56.625657392634771 + lats[475] = 56.555358627284086 + lats[476] = 56.485059861917016 + lats[477] = 56.41476109653366 + lats[478] = 56.34446233113411 + lats[479] = 56.274163565718467 + lats[480] = 56.203864800286865 + lats[481] = 56.133566034839362 + lats[482] = 56.063267269376091 + lats[483] = 55.992968503897131 + lats[484] = 55.922669738402583 + lats[485] = 55.852370972892551 + lats[486] = 55.782072207367136 + lats[487] = 55.711773441826416 + lats[488] = 55.641474676270505 + lats[489] = 55.571175910699488 + lats[490] = 55.500877145113449 + lats[491] = 55.430578379512511 + lats[492] = 55.360279613896743 + lats[493] = 55.289980848266232 + lats[494] = 55.219682082621084 + lats[495] = 55.149383316961377 + lats[496] = 55.07908455128721 + lats[497] = 55.008785785598668 + lats[498] = 54.938487019895831 + lats[499] = 54.868188254178797 + lats[500] = 54.797889488447652 + lats[501] = 54.727590722702473 + lats[502] = 54.657291956943347 + lats[503] = 54.586993191170357 + lats[504] = 54.516694425383605 + lats[505] = 54.446395659583146 + lats[506] = 54.376096893769081 + lats[507] = 54.305798127941479 + lats[508] = 54.235499362100448 + lats[509] = 54.165200596246031 + lats[510] = 54.094901830378333 + lats[511] = 54.024603064497434 + lats[512] = 53.954304298603383 + lats[513] = 53.884005532696307 + lats[514] = 53.813706766776235 + lats[515] = 53.743408000843282 + lats[516] = 53.673109234897495 + lats[517] = 53.602810468938962 + lats[518] = 53.53251170296776 + lats[519] = 53.462212936983953 + lats[520] = 53.391914170987633 + lats[521] = 53.321615404978871 + lats[522] = 53.251316638957725 + lats[523] = 53.181017872924265 + lats[524] = 53.110719106878584 + lats[525] = 53.040420340820731 + lats[526] = 52.970121574750792 + lats[527] = 52.899822808668837 + lats[528] = 52.829524042574917 + lats[529] = 52.759225276469131 + lats[530] = 52.688926510351514 + lats[531] = 52.618627744222159 + lats[532] = 52.548328978081123 + lats[533] = 52.478030211928477 + lats[534] = 52.407731445764284 + lats[535] = 52.337432679588609 + lats[536] = 52.26713391340153 + lats[537] = 52.196835147203096 + lats[538] = 52.126536380993372 + lats[539] = 52.056237614772435 + lats[540] = 51.985938848540336 + lats[541] = 51.915640082297152 + lats[542] = 51.845341316042933 + lats[543] = 51.775042549777737 + lats[544] = 51.704743783501634 + lats[545] = 51.634445017214695 + lats[546] = 51.56414625091697 + lats[547] = 51.493847484608516 + lats[548] = 51.423548718289396 + lats[549] = 51.353249951959683 + lats[550] = 51.282951185619417 + lats[551] = 51.21265241926865 + lats[552] = 51.14235365290746 + lats[553] = 51.072054886535909 + lats[554] = 51.001756120154049 + lats[555] = 50.931457353761914 + lats[556] = 50.86115858735959 + lats[557] = 50.790859820947119 + lats[558] = 50.720561054524559 + lats[559] = 50.650262288091959 + lats[560] = 50.579963521649397 + lats[561] = 50.509664755196901 + lats[562] = 50.439365988734544 + lats[563] = 50.369067222262359 + lats[564] = 50.298768455780426 + lats[565] = 50.228469689288779 + lats[566] = 50.158170922787484 + lats[567] = 50.087872156276575 + lats[568] = 50.017573389756123 + lats[569] = 49.947274623226157 + lats[570] = 49.876975856686762 + lats[571] = 49.80667709013796 + lats[572] = 49.736378323579807 + lats[573] = 49.66607955701236 + lats[574] = 49.595780790435676 + lats[575] = 49.525482023849783 + lats[576] = 49.455183257254745 + lats[577] = 49.384884490650613 + lats[578] = 49.314585724037435 + lats[579] = 49.244286957415234 + lats[580] = 49.173988190784094 + lats[581] = 49.103689424144044 + lats[582] = 49.03339065749514 + lats[583] = 48.963091890837418 + lats[584] = 48.892793124170929 + lats[585] = 48.822494357495721 + lats[586] = 48.752195590811837 + lats[587] = 48.681896824119335 + lats[588] = 48.611598057418242 + lats[589] = 48.541299290708608 + lats[590] = 48.47100052399049 + lats[591] = 48.400701757263917 + lats[592] = 48.330402990528938 + lats[593] = 48.260104223785596 + lats[594] = 48.189805457033941 + lats[595] = 48.119506690274015 + lats[596] = 48.049207923505868 + lats[597] = 47.978909156729507 + lats[598] = 47.908610389945018 + lats[599] = 47.838311623152421 + lats[600] = 47.76801285635176 + lats[601] = 47.697714089543084 + lats[602] = 47.627415322726435 + lats[603] = 47.557116555901828 + lats[604] = 47.486817789069342 + lats[605] = 47.416519022228997 + lats[606] = 47.346220255380835 + lats[607] = 47.275921488524894 + lats[608] = 47.205622721661214 + lats[609] = 47.13532395478984 + lats[610] = 47.065025187910805 + lats[611] = 46.994726421024154 + lats[612] = 46.924427654129929 + lats[613] = 46.85412888722815 + lats[614] = 46.783830120318882 + lats[615] = 46.713531353402139 + lats[616] = 46.643232586477971 + lats[617] = 46.572933819546414 + lats[618] = 46.502635052607502 + lats[619] = 46.432336285661272 + lats[620] = 46.362037518707766 + lats[621] = 46.291738751747012 + lats[622] = 46.221439984779053 + lats[623] = 46.151141217803925 + lats[624] = 46.080842450821663 + lats[625] = 46.01054368383231 + lats[626] = 45.94024491683588 + lats[627] = 45.869946149832437 + lats[628] = 45.799647382821995 + lats[629] = 45.729348615804589 + lats[630] = 45.659049848780263 + lats[631] = 45.588751081749038 + lats[632] = 45.51845231471097 + lats[633] = 45.448153547666081 + lats[634] = 45.377854780614399 + lats[635] = 45.30755601355596 + lats[636] = 45.237257246490813 + lats[637] = 45.166958479418959 + lats[638] = 45.096659712340461 + lats[639] = 45.026360945255341 + lats[640] = 44.956062178163634 + lats[641] = 44.885763411065362 + lats[642] = 44.81546464396056 + lats[643] = 44.745165876849271 + lats[644] = 44.674867109731515 + lats[645] = 44.604568342607337 + lats[646] = 44.534269575476756 + lats[647] = 44.463970808339802 + lats[648] = 44.39367204119651 + lats[649] = 44.323373274046915 + lats[650] = 44.253074506891046 + lats[651] = 44.182775739728925 + lats[652] = 44.112476972560586 + lats[653] = 44.042178205386072 + lats[654] = 43.971879438205391 + lats[655] = 43.9015806710186 + lats[656] = 43.831281903825705 + lats[657] = 43.760983136626741 + lats[658] = 43.690684369421732 + lats[659] = 43.620385602210717 + lats[660] = 43.550086834993728 + lats[661] = 43.479788067770777 + lats[662] = 43.409489300541907 + lats[663] = 43.339190533307139 + lats[664] = 43.26889176606651 + lats[665] = 43.19859299882004 + lats[666] = 43.128294231567757 + lats[667] = 43.057995464309691 + lats[668] = 42.987696697045862 + lats[669] = 42.917397929776307 + lats[670] = 42.847099162501053 + lats[671] = 42.776800395220121 + lats[672] = 42.706501627933541 + lats[673] = 42.63620286064134 + lats[674] = 42.565904093343548 + lats[675] = 42.495605326040177 + lats[676] = 42.425306558731272 + lats[677] = 42.355007791416853 + lats[678] = 42.284709024096927 + lats[679] = 42.214410256771551 + lats[680] = 42.144111489440725 + lats[681] = 42.073812722104492 + lats[682] = 42.003513954762873 + lats[683] = 41.933215187415882 + lats[684] = 41.862916420063563 + lats[685] = 41.792617652705921 + lats[686] = 41.722318885343 + lats[687] = 41.6520201179748 + lats[688] = 41.581721350601363 + lats[689] = 41.511422583222718 + lats[690] = 41.441123815838885 + lats[691] = 41.370825048449873 + lats[692] = 41.300526281055724 + lats[693] = 41.230227513656445 + lats[694] = 41.159928746252085 + lats[695] = 41.089629978842645 + lats[696] = 41.01933121142816 + lats[697] = 40.949032444008644 + lats[698] = 40.878733676584126 + lats[699] = 40.808434909154634 + lats[700] = 40.738136141720176 + lats[701] = 40.667837374280786 + lats[702] = 40.597538606836487 + lats[703] = 40.527239839387299 + lats[704] = 40.456941071933244 + lats[705] = 40.386642304474343 + lats[706] = 40.316343537010617 + lats[707] = 40.246044769542102 + lats[708] = 40.175746002068806 + lats[709] = 40.105447234590748 + lats[710] = 40.035148467107952 + lats[711] = 39.964849699620437 + lats[712] = 39.894550932128247 + lats[713] = 39.824252164631375 + lats[714] = 39.753953397129855 + lats[715] = 39.683654629623703 + lats[716] = 39.613355862112947 + lats[717] = 39.543057094597607 + lats[718] = 39.472758327077692 + lats[719] = 39.402459559553229 + lats[720] = 39.332160792024254 + lats[721] = 39.261862024490775 + lats[722] = 39.191563256952804 + lats[723] = 39.121264489410365 + lats[724] = 39.050965721863491 + lats[725] = 38.980666954312184 + lats[726] = 38.910368186756479 + lats[727] = 38.840069419196389 + lats[728] = 38.769770651631937 + lats[729] = 38.699471884063136 + lats[730] = 38.629173116490001 + lats[731] = 38.558874348912568 + lats[732] = 38.488575581330842 + lats[733] = 38.418276813744846 + lats[734] = 38.347978046154608 + lats[735] = 38.277679278560143 + lats[736] = 38.20738051096145 + lats[737] = 38.137081743358586 + lats[738] = 38.066782975751536 + lats[739] = 37.99648420814033 + lats[740] = 37.926185440524989 + lats[741] = 37.855886672905527 + lats[742] = 37.785587905281965 + lats[743] = 37.715289137654317 + lats[744] = 37.644990370022605 + lats[745] = 37.574691602386856 + lats[746] = 37.504392834747065 + lats[747] = 37.434094067103274 + lats[748] = 37.363795299455489 + lats[749] = 37.293496531803719 + lats[750] = 37.223197764147997 + lats[751] = 37.152898996488332 + lats[752] = 37.082600228824752 + lats[753] = 37.012301461157264 + lats[754] = 36.942002693485883 + lats[755] = 36.871703925810628 + lats[756] = 36.801405158131523 + lats[757] = 36.731106390448581 + lats[758] = 36.660807622761808 + lats[759] = 36.590508855071242 + lats[760] = 36.520210087376888 + lats[761] = 36.449911319678755 + lats[762] = 36.379612551976876 + lats[763] = 36.309313784271254 + lats[764] = 36.239015016561908 + lats[765] = 36.16871624884886 + lats[766] = 36.098417481132117 + lats[767] = 36.028118713411708 + lats[768] = 35.957819945687639 + lats[769] = 35.887521177959933 + lats[770] = 35.817222410228595 + lats[771] = 35.746923642493655 + lats[772] = 35.676624874755113 + lats[773] = 35.606326107012997 + lats[774] = 35.536027339267314 + lats[775] = 35.465728571518085 + lats[776] = 35.395429803765317 + lats[777] = 35.325131036009047 + lats[778] = 35.254832268249267 + lats[779] = 35.184533500486005 + lats[780] = 35.114234732719261 + lats[781] = 35.043935964949064 + lats[782] = 34.973637197175435 + lats[783] = 34.903338429398374 + lats[784] = 34.833039661617903 + lats[785] = 34.762740893834028 + lats[786] = 34.692442126046771 + lats[787] = 34.622143358256153 + lats[788] = 34.551844590462188 + lats[789] = 34.481545822664863 + lats[790] = 34.411247054864234 + lats[791] = 34.340948287060286 + lats[792] = 34.270649519253041 + lats[793] = 34.200350751442521 + lats[794] = 34.130051983628725 + lats[795] = 34.059753215811682 + lats[796] = 33.989454447991392 + lats[797] = 33.919155680167876 + lats[798] = 33.848856912341155 + lats[799] = 33.778558144511237 + lats[800] = 33.708259376678136 + lats[801] = 33.637960608841851 + lats[802] = 33.567661841002426 + lats[803] = 33.497363073159853 + lats[804] = 33.42706430531414 + lats[805] = 33.356765537465314 + lats[806] = 33.286466769613391 + lats[807] = 33.216168001758369 + lats[808] = 33.145869233900278 + lats[809] = 33.075570466039117 + lats[810] = 33.005271698174909 + lats[811] = 32.934972930307666 + lats[812] = 32.864674162437396 + lats[813] = 32.794375394564113 + lats[814] = 32.724076626687825 + lats[815] = 32.653777858808567 + lats[816] = 32.583479090926325 + lats[817] = 32.513180323041112 + lats[818] = 32.442881555152965 + lats[819] = 32.372582787261891 + lats[820] = 32.302284019367875 + lats[821] = 32.231985251470959 + lats[822] = 32.161686483571145 + lats[823] = 32.091387715668439 + lats[824] = 32.021088947762863 + lats[825] = 31.950790179854422 + lats[826] = 31.880491411943137 + lats[827] = 31.810192644029012 + lats[828] = 31.739893876112063 + lats[829] = 31.669595108192297 + lats[830] = 31.599296340269738 + lats[831] = 31.528997572344384 + lats[832] = 31.458698804416255 + lats[833] = 31.388400036485361 + lats[834] = 31.318101268551715 + lats[835] = 31.247802500615318 + lats[836] = 31.177503732676204 + lats[837] = 31.107204964734358 + lats[838] = 31.036906196789811 + lats[839] = 30.966607428842572 + lats[840] = 30.896308660892647 + lats[841] = 30.826009892940046 + lats[842] = 30.755711124984781 + lats[843] = 30.685412357026873 + lats[844] = 30.615113589066322 + lats[845] = 30.544814821103138 + lats[846] = 30.47451605313735 + lats[847] = 30.404217285168947 + lats[848] = 30.333918517197947 + lats[849] = 30.263619749224372 + lats[850] = 30.19332098124822 + lats[851] = 30.123022213269511 + lats[852] = 30.052723445288244 + lats[853] = 29.98242467730444 + lats[854] = 29.91212590931811 + lats[855] = 29.841827141329258 + lats[856] = 29.771528373337894 + lats[857] = 29.701229605344039 + lats[858] = 29.630930837347698 + lats[859] = 29.560632069348884 + lats[860] = 29.490333301347597 + lats[861] = 29.420034533343859 + lats[862] = 29.349735765337677 + lats[863] = 29.279436997329057 + lats[864] = 29.209138229318015 + lats[865] = 29.138839461304556 + lats[866] = 29.068540693288696 + lats[867] = 28.998241925270449 + lats[868] = 28.927943157249814 + lats[869] = 28.857644389226806 + lats[870] = 28.787345621201432 + lats[871] = 28.717046853173709 + lats[872] = 28.646748085143642 + lats[873] = 28.576449317111244 + lats[874] = 28.506150549076519 + lats[875] = 28.435851781039485 + lats[876] = 28.365553013000145 + lats[877] = 28.29525424495851 + lats[878] = 28.224955476914594 + lats[879] = 28.154656708868405 + lats[880] = 28.084357940819952 + lats[881] = 28.014059172769244 + lats[882] = 27.94376040471629 + lats[883] = 27.873461636661098 + lats[884] = 27.803162868603682 + lats[885] = 27.732864100544052 + lats[886] = 27.662565332482213 + lats[887] = 27.592266564418171 + lats[888] = 27.521967796351948 + lats[889] = 27.451669028283543 + lats[890] = 27.381370260212968 + lats[891] = 27.311071492140236 + lats[892] = 27.240772724065348 + lats[893] = 27.170473955988321 + lats[894] = 27.100175187909159 + lats[895] = 27.029876419827872 + lats[896] = 26.959577651744471 + lats[897] = 26.889278883658971 + lats[898] = 26.818980115571364 + lats[899] = 26.748681347481678 + lats[900] = 26.678382579389908 + lats[901] = 26.608083811296069 + lats[902] = 26.53778504320017 + lats[903] = 26.467486275102218 + lats[904] = 26.397187507002222 + lats[905] = 26.326888738900195 + lats[906] = 26.256589970796135 + lats[907] = 26.186291202690064 + lats[908] = 26.115992434581983 + lats[909] = 26.045693666471902 + lats[910] = 25.975394898359827 + lats[911] = 25.90509613024577 + lats[912] = 25.834797362129745 + lats[913] = 25.764498594011751 + lats[914] = 25.694199825891793 + lats[915] = 25.623901057769892 + lats[916] = 25.553602289646051 + lats[917] = 25.483303521520277 + lats[918] = 25.413004753392578 + lats[919] = 25.342705985262967 + lats[920] = 25.272407217131445 + lats[921] = 25.202108448998025 + lats[922] = 25.13180968086272 + lats[923] = 25.061510912725527 + lats[924] = 24.991212144586456 + lats[925] = 24.920913376445526 + lats[926] = 24.850614608302738 + lats[927] = 24.780315840158096 + lats[928] = 24.710017072011613 + lats[929] = 24.639718303863294 + lats[930] = 24.569419535713152 + lats[931] = 24.499120767561195 + lats[932] = 24.428821999407425 + lats[933] = 24.358523231251851 + lats[934] = 24.288224463094483 + lats[935] = 24.217925694935328 + lats[936] = 24.1476269267744 + lats[937] = 24.077328158611696 + lats[938] = 24.007029390447226 + lats[939] = 23.936730622281004 + lats[940] = 23.866431854113038 + lats[941] = 23.796133085943328 + lats[942] = 23.725834317771888 + lats[943] = 23.655535549598721 + lats[944] = 23.585236781423838 + lats[945] = 23.514938013247242 + lats[946] = 23.444639245068949 + lats[947] = 23.374340476888957 + lats[948] = 23.304041708707278 + lats[949] = 23.233742940523921 + lats[950] = 23.163444172338895 + lats[951] = 23.0931454041522 + lats[952] = 23.022846635963852 + lats[953] = 22.952547867773848 + lats[954] = 22.882249099582204 + lats[955] = 22.811950331388925 + lats[956] = 22.741651563194019 + lats[957] = 22.671352794997489 + lats[958] = 22.60105402679935 + lats[959] = 22.530755258599601 + lats[960] = 22.460456490398254 + lats[961] = 22.390157722195315 + lats[962] = 22.319858953990789 + lats[963] = 22.249560185784691 + lats[964] = 22.179261417577013 + lats[965] = 22.108962649367779 + lats[966] = 22.038663881156989 + lats[967] = 21.968365112944642 + lats[968] = 21.898066344730758 + lats[969] = 21.827767576515338 + lats[970] = 21.757468808298391 + lats[971] = 21.687170040079913 + lats[972] = 21.616871271859928 + lats[973] = 21.546572503638437 + lats[974] = 21.47627373541544 + lats[975] = 21.40597496719095 + lats[976] = 21.335676198964972 + lats[977] = 21.265377430737512 + lats[978] = 21.195078662508585 + lats[979] = 21.124779894278181 + lats[980] = 21.054481126046323 + lats[981] = 20.984182357813012 + lats[982] = 20.913883589578251 + lats[983] = 20.843584821342048 + lats[984] = 20.773286053104417 + lats[985] = 20.702987284865355 + lats[986] = 20.632688516624874 + lats[987] = 20.562389748382977 + lats[988] = 20.492090980139672 + lats[989] = 20.421792211894967 + lats[990] = 20.35149344364887 + lats[991] = 20.28119467540138 + lats[992] = 20.210895907152516 + lats[993] = 20.140597138902272 + lats[994] = 20.070298370650661 + lats[995] = 19.999999602397686 + lats[996] = 19.929700834143357 + lats[997] = 19.859402065887682 + lats[998] = 19.789103297630657 + lats[999] = 19.718804529372303 + lats[1000] = 19.648505761112613 + lats[1001] = 19.578206992851602 + lats[1002] = 19.507908224589269 + lats[1003] = 19.437609456325632 + lats[1004] = 19.367310688060684 + lats[1005] = 19.297011919794439 + lats[1006] = 19.226713151526898 + lats[1007] = 19.15641438325807 + lats[1008] = 19.086115614987968 + lats[1009] = 19.015816846716586 + lats[1010] = 18.945518078443939 + lats[1011] = 18.875219310170031 + lats[1012] = 18.804920541894862 + lats[1013] = 18.734621773618446 + lats[1014] = 18.664323005340787 + lats[1015] = 18.594024237061891 + lats[1016] = 18.523725468781763 + lats[1017] = 18.453426700500408 + lats[1018] = 18.383127932217832 + lats[1019] = 18.312829163934047 + lats[1020] = 18.242530395649048 + lats[1021] = 18.172231627362851 + lats[1022] = 18.101932859075458 + lats[1023] = 18.031634090786874 + lats[1024] = 17.96133532249711 + lats[1025] = 17.89103655420616 + lats[1026] = 17.820737785914044 + lats[1027] = 17.75043901762076 + lats[1028] = 17.680140249326314 + lats[1029] = 17.60984148103071 + lats[1030] = 17.539542712733962 + lats[1031] = 17.469243944436066 + lats[1032] = 17.39894517613704 + lats[1033] = 17.328646407836878 + lats[1034] = 17.258347639535586 + lats[1035] = 17.188048871233182 + lats[1036] = 17.117750102929655 + lats[1037] = 17.04745133462502 + lats[1038] = 16.977152566319283 + lats[1039] = 16.906853798012452 + lats[1040] = 16.836555029704527 + lats[1041] = 16.766256261395515 + lats[1042] = 16.69595749308542 + lats[1043] = 16.625658724774254 + lats[1044] = 16.555359956462013 + lats[1045] = 16.485061188148713 + lats[1046] = 16.41476241983435 + lats[1047] = 16.344463651518936 + lats[1048] = 16.274164883202477 + lats[1049] = 16.203866114884974 + lats[1050] = 16.133567346566434 + lats[1051] = 16.063268578246863 + lats[1052] = 15.992969809926265 + lats[1053] = 15.922671041604652 + lats[1054] = 15.852372273282016 + lats[1055] = 15.78207350495838 + lats[1056] = 15.711774736633735 + lats[1057] = 15.641475968308091 + lats[1058] = 15.571177199981456 + lats[1059] = 15.500878431653829 + lats[1060] = 15.430579663325226 + lats[1061] = 15.360280894995643 + lats[1062] = 15.289982126665089 + lats[1063] = 15.219683358333569 + lats[1064] = 15.149384590001089 + lats[1065] = 15.07908582166765 + lats[1066] = 15.008787053333259 + lats[1067] = 14.938488284997929 + lats[1068] = 14.868189516661655 + lats[1069] = 14.797890748324447 + lats[1070] = 14.727591979986309 + lats[1071] = 14.657293211647247 + lats[1072] = 14.586994443307265 + lats[1073] = 14.516695674966371 + lats[1074] = 14.446396906624567 + lats[1075] = 14.376098138281863 + lats[1076] = 14.305799369938256 + lats[1077] = 14.23550060159376 + lats[1078] = 14.165201833248371 + lats[1079] = 14.0949030649021 + lats[1080] = 14.024604296554955 + lats[1081] = 13.954305528206934 + lats[1082] = 13.884006759858046 + lats[1083] = 13.813707991508297 + lats[1084] = 13.743409223157688 + lats[1085] = 13.673110454806226 + lats[1086] = 13.602811686453919 + lats[1087] = 13.532512918100766 + lats[1088] = 13.46221414974678 + lats[1089] = 13.391915381391959 + lats[1090] = 13.32161661303631 + lats[1091] = 13.251317844679837 + lats[1092] = 13.181019076322551 + lats[1093] = 13.110720307964451 + lats[1094] = 13.040421539605545 + lats[1095] = 12.970122771245832 + lats[1096] = 12.899824002885323 + lats[1097] = 12.829525234524022 + lats[1098] = 12.759226466161934 + lats[1099] = 12.688927697799061 + lats[1100] = 12.618628929435411 + lats[1101] = 12.548330161070988 + lats[1102] = 12.478031392705796 + lats[1103] = 12.407732624339841 + lats[1104] = 12.337433855973126 + lats[1105] = 12.267135087605659 + lats[1106] = 12.196836319237443 + lats[1107] = 12.126537550868482 + lats[1108] = 12.056238782498781 + lats[1109] = 11.985940014128348 + lats[1110] = 11.915641245757183 + lats[1111] = 11.845342477385294 + lats[1112] = 11.775043709012685 + lats[1113] = 11.704744940639358 + lats[1114] = 11.634446172265324 + lats[1115] = 11.564147403890583 + lats[1116] = 11.493848635515141 + lats[1117] = 11.423549867139002 + lats[1118] = 11.35325109876217 + lats[1119] = 11.282952330384653 + lats[1120] = 11.212653562006453 + lats[1121] = 11.142354793627575 + lats[1122] = 11.072056025248026 + lats[1123] = 11.001757256867807 + lats[1124] = 10.931458488486923 + lats[1125] = 10.861159720105382 + lats[1126] = 10.790860951723188 + lats[1127] = 10.720562183340341 + lats[1128] = 10.65026341495685 + lats[1129] = 10.579964646572719 + lats[1130] = 10.509665878187954 + lats[1131] = 10.439367109802557 + lats[1132] = 10.369068341416533 + lats[1133] = 10.298769573029887 + lats[1134] = 10.228470804642624 + lats[1135] = 10.158172036254747 + lats[1136] = 10.087873267866264 + lats[1137] = 10.017574499477174 + lats[1138] = 9.9472757310874869 + lats[1139] = 9.8769769626972046 + lats[1140] = 9.8066781943063344 + lats[1141] = 9.7363794259148779 + lats[1142] = 9.6660806575228388 + lats[1143] = 9.5957818891302242 + lats[1144] = 9.5254831207370376 + lats[1145] = 9.4551843523432826 + lats[1146] = 9.3848855839489662 + lats[1147] = 9.3145868155540921 + lats[1148] = 9.2442880471586619 + lats[1149] = 9.1739892787626829 + lats[1150] = 9.1036905103661585 + lats[1151] = 9.0333917419690941 + lats[1152] = 8.963092973571495 + lats[1153] = 8.8927942051733631 + lats[1154] = 8.8224954367747017 + lats[1155] = 8.7521966683755217 + lats[1156] = 8.6818978999758194 + lats[1157] = 8.6115991315756055 + lats[1158] = 8.5413003631748801 + lats[1159] = 8.4710015947736537 + lats[1160] = 8.4007028263719228 + lats[1161] = 8.3304040579696963 + lats[1162] = 8.2601052895669778 + lats[1163] = 8.1898065211637725 + lats[1164] = 8.1195077527600841 + lats[1165] = 8.049208984355916 + lats[1166] = 7.9789102159512737 + lats[1167] = 7.9086114475461606 + lats[1168] = 7.8383126791405831 + lats[1169] = 7.7680139107345463 + lats[1170] = 7.6977151423280494 + lats[1171] = 7.6274163739210996 + lats[1172] = 7.557117605513703 + lats[1173] = 7.4868188371058624 + lats[1174] = 7.4165200686975803 + lats[1175] = 7.3462213002888648 + lats[1176] = 7.2759225318797176 + lats[1177] = 7.2056237634701441 + lats[1178] = 7.1353249950601469 + lats[1179] = 7.0650262266497315 + lats[1180] = 6.994727458238903 + lats[1181] = 6.924428689827665 + lats[1182] = 6.8541299214160212 + lats[1183] = 6.7838311530039768 + lats[1184] = 6.7135323845915353 + lats[1185] = 6.6432336161787013 + lats[1186] = 6.5729348477654792 + lats[1187] = 6.5026360793518734 + lats[1188] = 6.4323373109378874 + lats[1189] = 6.3620385425235257 + lats[1190] = 6.2917397741087928 + lats[1191] = 6.2214410056936931 + lats[1192] = 6.151142237278231 + lats[1193] = 6.0808434688624091 + lats[1194] = 6.0105447004462347 + lats[1195] = 5.9402459320297085 + lats[1196] = 5.869947163612836 + lats[1197] = 5.7996483951956233 + lats[1198] = 5.729349626778073 + lats[1199] = 5.6590508583601888 + lats[1200] = 5.5887520899419751 + lats[1201] = 5.5184533215234373 + lats[1202] = 5.4481545531045787 + lats[1203] = 5.3778557846854023 + lats[1204] = 5.3075570162659149 + lats[1205] = 5.2372582478461194 + lats[1206] = 5.1669594794260192 + lats[1207] = 5.0966607110056197 + lats[1208] = 5.0263619425849244 + lats[1209] = 4.9560631741639369 + lats[1210] = 4.8857644057426626 + lats[1211] = 4.8154656373211049 + lats[1212] = 4.7451668688992683 + lats[1213] = 4.6748681004771564 + lats[1214] = 4.6045693320547736 + lats[1215] = 4.5342705636321252 + lats[1216] = 4.4639717952092139 + lats[1217] = 4.3936730267860451 + lats[1218] = 4.3233742583626205 + lats[1219] = 4.2530754899389471 + lats[1220] = 4.1827767215150269 + lats[1221] = 4.1124779530908659 + lats[1222] = 4.0421791846664661 + lats[1223] = 3.9718804162418326 + lats[1224] = 3.90158164781697 + lats[1225] = 3.8312828793918823 + lats[1226] = 3.7609841109665734 + lats[1227] = 3.6906853425410477 + lats[1228] = 3.6203865741153085 + lats[1229] = 3.5500878056893601 + lats[1230] = 3.4797890372632065 + lats[1231] = 3.4094902688368531 + lats[1232] = 3.339191500410303 + lats[1233] = 3.2688927319835597 + lats[1234] = 3.1985939635566285 + lats[1235] = 3.1282951951295126 + lats[1236] = 3.0579964267022164 + lats[1237] = 2.9876976582747439 + lats[1238] = 2.9173988898470999 + lats[1239] = 2.8471001214192873 + lats[1240] = 2.7768013529913107 + lats[1241] = 2.7065025845631743 + lats[1242] = 2.6362038161348824 + lats[1243] = 2.5659050477064382 + lats[1244] = 2.4956062792778466 + lats[1245] = 2.4253075108491116 + lats[1246] = 2.3550087424202366 + lats[1247] = 2.2847099739912267 + lats[1248] = 2.2144112055620848 + lats[1249] = 2.1441124371328155 + lats[1250] = 2.0738136687034232 + lats[1251] = 2.0035149002739114 + lats[1252] = 1.9332161318442849 + lats[1253] = 1.8629173634145471 + lats[1254] = 1.792618594984702 + lats[1255] = 1.7223198265547539 + lats[1256] = 1.6520210581247066 + lats[1257] = 1.5817222896945646 + lats[1258] = 1.5114235212643317 + lats[1259] = 1.4411247528340119 + lats[1260] = 1.3708259844036093 + lats[1261] = 1.300527215973128 + lats[1262] = 1.2302284475425722 + lats[1263] = 1.1599296791119456 + lats[1264] = 1.0896309106812523 + lats[1265] = 1.0193321422504964 + lats[1266] = 0.949033373819682 + lats[1267] = 0.87873460538881287 + lats[1268] = 0.80843583695789356 + lats[1269] = 0.73813706852692773 + lats[1270] = 0.66783830009591949 + lats[1271] = 0.59753953166487306 + lats[1272] = 0.52724076323379232 + lats[1273] = 0.45694199480268116 + lats[1274] = 0.3866432263715438 + lats[1275] = 0.31634445794038429 + lats[1276] = 0.24604568950920663 + lats[1277] = 0.17574692107801482 + lats[1278] = 0.10544815264681295 + lats[1279] = 0.035149384215604956 + lats[1280] = -0.035149384215604956 + lats[1281] = -0.10544815264681295 + lats[1282] = -0.17574692107801482 + lats[1283] = -0.24604568950920663 + lats[1284] = -0.31634445794038429 + lats[1285] = -0.3866432263715438 + lats[1286] = -0.45694199480268116 + lats[1287] = -0.52724076323379232 + lats[1288] = -0.59753953166487306 + lats[1289] = -0.66783830009591949 + lats[1290] = -0.73813706852692773 + lats[1291] = -0.80843583695789356 + lats[1292] = -0.87873460538881287 + lats[1293] = -0.949033373819682 + lats[1294] = -1.0193321422504964 + lats[1295] = -1.0896309106812523 + lats[1296] = -1.1599296791119456 + lats[1297] = -1.2302284475425722 + lats[1298] = -1.300527215973128 + lats[1299] = -1.3708259844036093 + lats[1300] = -1.4411247528340119 + lats[1301] = -1.5114235212643317 + lats[1302] = -1.5817222896945646 + lats[1303] = -1.6520210581247066 + lats[1304] = -1.7223198265547539 + lats[1305] = -1.792618594984702 + lats[1306] = -1.8629173634145471 + lats[1307] = -1.9332161318442849 + lats[1308] = -2.0035149002739114 + lats[1309] = -2.0738136687034232 + lats[1310] = -2.1441124371328155 + lats[1311] = -2.2144112055620848 + lats[1312] = -2.2847099739912267 + lats[1313] = -2.3550087424202366 + lats[1314] = -2.4253075108491116 + lats[1315] = -2.4956062792778466 + lats[1316] = -2.5659050477064382 + lats[1317] = -2.6362038161348824 + lats[1318] = -2.7065025845631743 + lats[1319] = -2.7768013529913107 + lats[1320] = -2.8471001214192873 + lats[1321] = -2.9173988898470999 + lats[1322] = -2.9876976582747439 + lats[1323] = -3.0579964267022164 + lats[1324] = -3.1282951951295126 + lats[1325] = -3.1985939635566285 + lats[1326] = -3.2688927319835597 + lats[1327] = -3.339191500410303 + lats[1328] = -3.4094902688368531 + lats[1329] = -3.4797890372632065 + lats[1330] = -3.5500878056893601 + lats[1331] = -3.6203865741153085 + lats[1332] = -3.6906853425410477 + lats[1333] = -3.7609841109665734 + lats[1334] = -3.8312828793918823 + lats[1335] = -3.90158164781697 + lats[1336] = -3.9718804162418326 + lats[1337] = -4.0421791846664661 + lats[1338] = -4.1124779530908659 + lats[1339] = -4.1827767215150269 + lats[1340] = -4.2530754899389471 + lats[1341] = -4.3233742583626205 + lats[1342] = -4.3936730267860451 + lats[1343] = -4.4639717952092139 + lats[1344] = -4.5342705636321252 + lats[1345] = -4.6045693320547736 + lats[1346] = -4.6748681004771564 + lats[1347] = -4.7451668688992683 + lats[1348] = -4.8154656373211049 + lats[1349] = -4.8857644057426626 + lats[1350] = -4.9560631741639369 + lats[1351] = -5.0263619425849244 + lats[1352] = -5.0966607110056197 + lats[1353] = -5.1669594794260192 + lats[1354] = -5.2372582478461194 + lats[1355] = -5.3075570162659149 + lats[1356] = -5.3778557846854023 + lats[1357] = -5.4481545531045787 + lats[1358] = -5.5184533215234373 + lats[1359] = -5.5887520899419751 + lats[1360] = -5.6590508583601888 + lats[1361] = -5.729349626778073 + lats[1362] = -5.7996483951956233 + lats[1363] = -5.869947163612836 + lats[1364] = -5.9402459320297085 + lats[1365] = -6.0105447004462347 + lats[1366] = -6.0808434688624091 + lats[1367] = -6.151142237278231 + lats[1368] = -6.2214410056936931 + lats[1369] = -6.2917397741087928 + lats[1370] = -6.3620385425235257 + lats[1371] = -6.4323373109378874 + lats[1372] = -6.5026360793518734 + lats[1373] = -6.5729348477654792 + lats[1374] = -6.6432336161787013 + lats[1375] = -6.7135323845915353 + lats[1376] = -6.7838311530039768 + lats[1377] = -6.8541299214160212 + lats[1378] = -6.924428689827665 + lats[1379] = -6.994727458238903 + lats[1380] = -7.0650262266497315 + lats[1381] = -7.1353249950601469 + lats[1382] = -7.2056237634701441 + lats[1383] = -7.2759225318797176 + lats[1384] = -7.3462213002888648 + lats[1385] = -7.4165200686975803 + lats[1386] = -7.4868188371058624 + lats[1387] = -7.557117605513703 + lats[1388] = -7.6274163739210996 + lats[1389] = -7.6977151423280494 + lats[1390] = -7.7680139107345463 + lats[1391] = -7.8383126791405831 + lats[1392] = -7.9086114475461606 + lats[1393] = -7.9789102159512737 + lats[1394] = -8.049208984355916 + lats[1395] = -8.1195077527600841 + lats[1396] = -8.1898065211637725 + lats[1397] = -8.2601052895669778 + lats[1398] = -8.3304040579696963 + lats[1399] = -8.4007028263719228 + lats[1400] = -8.4710015947736537 + lats[1401] = -8.5413003631748801 + lats[1402] = -8.6115991315756055 + lats[1403] = -8.6818978999758194 + lats[1404] = -8.7521966683755217 + lats[1405] = -8.8224954367747017 + lats[1406] = -8.8927942051733631 + lats[1407] = -8.963092973571495 + lats[1408] = -9.0333917419690941 + lats[1409] = -9.1036905103661585 + lats[1410] = -9.1739892787626829 + lats[1411] = -9.2442880471586619 + lats[1412] = -9.3145868155540921 + lats[1413] = -9.3848855839489662 + lats[1414] = -9.4551843523432826 + lats[1415] = -9.5254831207370376 + lats[1416] = -9.5957818891302242 + lats[1417] = -9.6660806575228388 + lats[1418] = -9.7363794259148779 + lats[1419] = -9.8066781943063344 + lats[1420] = -9.8769769626972046 + lats[1421] = -9.9472757310874869 + lats[1422] = -10.017574499477174 + lats[1423] = -10.087873267866264 + lats[1424] = -10.158172036254747 + lats[1425] = -10.228470804642624 + lats[1426] = -10.298769573029887 + lats[1427] = -10.369068341416533 + lats[1428] = -10.439367109802557 + lats[1429] = -10.509665878187954 + lats[1430] = -10.579964646572719 + lats[1431] = -10.65026341495685 + lats[1432] = -10.720562183340341 + lats[1433] = -10.790860951723188 + lats[1434] = -10.861159720105382 + lats[1435] = -10.931458488486923 + lats[1436] = -11.001757256867807 + lats[1437] = -11.072056025248026 + lats[1438] = -11.142354793627575 + lats[1439] = -11.212653562006453 + lats[1440] = -11.282952330384653 + lats[1441] = -11.35325109876217 + lats[1442] = -11.423549867139002 + lats[1443] = -11.493848635515141 + lats[1444] = -11.564147403890583 + lats[1445] = -11.634446172265324 + lats[1446] = -11.704744940639358 + lats[1447] = -11.775043709012685 + lats[1448] = -11.845342477385294 + lats[1449] = -11.915641245757183 + lats[1450] = -11.985940014128348 + lats[1451] = -12.056238782498781 + lats[1452] = -12.126537550868482 + lats[1453] = -12.196836319237443 + lats[1454] = -12.267135087605659 + lats[1455] = -12.337433855973126 + lats[1456] = -12.407732624339841 + lats[1457] = -12.478031392705796 + lats[1458] = -12.548330161070988 + lats[1459] = -12.618628929435411 + lats[1460] = -12.688927697799061 + lats[1461] = -12.759226466161934 + lats[1462] = -12.829525234524022 + lats[1463] = -12.899824002885323 + lats[1464] = -12.970122771245832 + lats[1465] = -13.040421539605545 + lats[1466] = -13.110720307964451 + lats[1467] = -13.181019076322551 + lats[1468] = -13.251317844679837 + lats[1469] = -13.32161661303631 + lats[1470] = -13.391915381391959 + lats[1471] = -13.46221414974678 + lats[1472] = -13.532512918100766 + lats[1473] = -13.602811686453919 + lats[1474] = -13.673110454806226 + lats[1475] = -13.743409223157688 + lats[1476] = -13.813707991508297 + lats[1477] = -13.884006759858046 + lats[1478] = -13.954305528206934 + lats[1479] = -14.024604296554955 + lats[1480] = -14.0949030649021 + lats[1481] = -14.165201833248371 + lats[1482] = -14.23550060159376 + lats[1483] = -14.305799369938256 + lats[1484] = -14.376098138281863 + lats[1485] = -14.446396906624567 + lats[1486] = -14.516695674966371 + lats[1487] = -14.586994443307265 + lats[1488] = -14.657293211647247 + lats[1489] = -14.727591979986309 + lats[1490] = -14.797890748324447 + lats[1491] = -14.868189516661655 + lats[1492] = -14.938488284997929 + lats[1493] = -15.008787053333259 + lats[1494] = -15.07908582166765 + lats[1495] = -15.149384590001089 + lats[1496] = -15.219683358333569 + lats[1497] = -15.289982126665089 + lats[1498] = -15.360280894995643 + lats[1499] = -15.430579663325226 + lats[1500] = -15.500878431653829 + lats[1501] = -15.571177199981456 + lats[1502] = -15.641475968308091 + lats[1503] = -15.711774736633735 + lats[1504] = -15.78207350495838 + lats[1505] = -15.852372273282016 + lats[1506] = -15.922671041604652 + lats[1507] = -15.992969809926265 + lats[1508] = -16.063268578246863 + lats[1509] = -16.133567346566434 + lats[1510] = -16.203866114884974 + lats[1511] = -16.274164883202477 + lats[1512] = -16.344463651518936 + lats[1513] = -16.41476241983435 + lats[1514] = -16.485061188148713 + lats[1515] = -16.555359956462013 + lats[1516] = -16.625658724774254 + lats[1517] = -16.69595749308542 + lats[1518] = -16.766256261395515 + lats[1519] = -16.836555029704527 + lats[1520] = -16.906853798012452 + lats[1521] = -16.977152566319283 + lats[1522] = -17.04745133462502 + lats[1523] = -17.117750102929655 + lats[1524] = -17.188048871233182 + lats[1525] = -17.258347639535586 + lats[1526] = -17.328646407836878 + lats[1527] = -17.39894517613704 + lats[1528] = -17.469243944436066 + lats[1529] = -17.539542712733962 + lats[1530] = -17.60984148103071 + lats[1531] = -17.680140249326314 + lats[1532] = -17.75043901762076 + lats[1533] = -17.820737785914044 + lats[1534] = -17.89103655420616 + lats[1535] = -17.96133532249711 + lats[1536] = -18.031634090786874 + lats[1537] = -18.101932859075458 + lats[1538] = -18.172231627362851 + lats[1539] = -18.242530395649048 + lats[1540] = -18.312829163934047 + lats[1541] = -18.383127932217832 + lats[1542] = -18.453426700500408 + lats[1543] = -18.523725468781763 + lats[1544] = -18.594024237061891 + lats[1545] = -18.664323005340787 + lats[1546] = -18.734621773618446 + lats[1547] = -18.804920541894862 + lats[1548] = -18.875219310170031 + lats[1549] = -18.945518078443939 + lats[1550] = -19.015816846716586 + lats[1551] = -19.086115614987968 + lats[1552] = -19.15641438325807 + lats[1553] = -19.226713151526898 + lats[1554] = -19.297011919794439 + lats[1555] = -19.367310688060684 + lats[1556] = -19.437609456325632 + lats[1557] = -19.507908224589269 + lats[1558] = -19.578206992851602 + lats[1559] = -19.648505761112613 + lats[1560] = -19.718804529372303 + lats[1561] = -19.789103297630657 + lats[1562] = -19.859402065887682 + lats[1563] = -19.929700834143357 + lats[1564] = -19.999999602397686 + lats[1565] = -20.070298370650661 + lats[1566] = -20.140597138902272 + lats[1567] = -20.210895907152516 + lats[1568] = -20.28119467540138 + lats[1569] = -20.35149344364887 + lats[1570] = -20.421792211894967 + lats[1571] = -20.492090980139672 + lats[1572] = -20.562389748382977 + lats[1573] = -20.632688516624874 + lats[1574] = -20.702987284865355 + lats[1575] = -20.773286053104417 + lats[1576] = -20.843584821342048 + lats[1577] = -20.913883589578251 + lats[1578] = -20.984182357813012 + lats[1579] = -21.054481126046323 + lats[1580] = -21.124779894278181 + lats[1581] = -21.195078662508585 + lats[1582] = -21.265377430737512 + lats[1583] = -21.335676198964972 + lats[1584] = -21.40597496719095 + lats[1585] = -21.47627373541544 + lats[1586] = -21.546572503638437 + lats[1587] = -21.616871271859928 + lats[1588] = -21.687170040079913 + lats[1589] = -21.757468808298391 + lats[1590] = -21.827767576515338 + lats[1591] = -21.898066344730758 + lats[1592] = -21.968365112944642 + lats[1593] = -22.038663881156989 + lats[1594] = -22.108962649367779 + lats[1595] = -22.179261417577013 + lats[1596] = -22.249560185784691 + lats[1597] = -22.319858953990789 + lats[1598] = -22.390157722195315 + lats[1599] = -22.460456490398254 + lats[1600] = -22.530755258599601 + lats[1601] = -22.60105402679935 + lats[1602] = -22.671352794997489 + lats[1603] = -22.741651563194019 + lats[1604] = -22.811950331388925 + lats[1605] = -22.882249099582204 + lats[1606] = -22.952547867773848 + lats[1607] = -23.022846635963852 + lats[1608] = -23.0931454041522 + lats[1609] = -23.163444172338895 + lats[1610] = -23.233742940523921 + lats[1611] = -23.304041708707278 + lats[1612] = -23.374340476888957 + lats[1613] = -23.444639245068949 + lats[1614] = -23.514938013247242 + lats[1615] = -23.585236781423838 + lats[1616] = -23.655535549598721 + lats[1617] = -23.725834317771888 + lats[1618] = -23.796133085943328 + lats[1619] = -23.866431854113038 + lats[1620] = -23.936730622281004 + lats[1621] = -24.007029390447226 + lats[1622] = -24.077328158611696 + lats[1623] = -24.1476269267744 + lats[1624] = -24.217925694935328 + lats[1625] = -24.288224463094483 + lats[1626] = -24.358523231251851 + lats[1627] = -24.428821999407425 + lats[1628] = -24.499120767561195 + lats[1629] = -24.569419535713152 + lats[1630] = -24.639718303863294 + lats[1631] = -24.710017072011613 + lats[1632] = -24.780315840158096 + lats[1633] = -24.850614608302738 + lats[1634] = -24.920913376445526 + lats[1635] = -24.991212144586456 + lats[1636] = -25.061510912725527 + lats[1637] = -25.13180968086272 + lats[1638] = -25.202108448998025 + lats[1639] = -25.272407217131445 + lats[1640] = -25.342705985262967 + lats[1641] = -25.413004753392578 + lats[1642] = -25.483303521520277 + lats[1643] = -25.553602289646051 + lats[1644] = -25.623901057769892 + lats[1645] = -25.694199825891793 + lats[1646] = -25.764498594011751 + lats[1647] = -25.834797362129745 + lats[1648] = -25.90509613024577 + lats[1649] = -25.975394898359827 + lats[1650] = -26.045693666471902 + lats[1651] = -26.115992434581983 + lats[1652] = -26.186291202690064 + lats[1653] = -26.256589970796135 + lats[1654] = -26.326888738900195 + lats[1655] = -26.397187507002222 + lats[1656] = -26.467486275102218 + lats[1657] = -26.53778504320017 + lats[1658] = -26.608083811296069 + lats[1659] = -26.678382579389908 + lats[1660] = -26.748681347481678 + lats[1661] = -26.818980115571364 + lats[1662] = -26.889278883658971 + lats[1663] = -26.959577651744471 + lats[1664] = -27.029876419827872 + lats[1665] = -27.100175187909159 + lats[1666] = -27.170473955988321 + lats[1667] = -27.240772724065348 + lats[1668] = -27.311071492140236 + lats[1669] = -27.381370260212968 + lats[1670] = -27.451669028283543 + lats[1671] = -27.521967796351948 + lats[1672] = -27.592266564418171 + lats[1673] = -27.662565332482213 + lats[1674] = -27.732864100544052 + lats[1675] = -27.803162868603682 + lats[1676] = -27.873461636661098 + lats[1677] = -27.94376040471629 + lats[1678] = -28.014059172769244 + lats[1679] = -28.084357940819952 + lats[1680] = -28.154656708868405 + lats[1681] = -28.224955476914594 + lats[1682] = -28.29525424495851 + lats[1683] = -28.365553013000145 + lats[1684] = -28.435851781039485 + lats[1685] = -28.506150549076519 + lats[1686] = -28.576449317111244 + lats[1687] = -28.646748085143642 + lats[1688] = -28.717046853173709 + lats[1689] = -28.787345621201432 + lats[1690] = -28.857644389226806 + lats[1691] = -28.927943157249814 + lats[1692] = -28.998241925270449 + lats[1693] = -29.068540693288696 + lats[1694] = -29.138839461304556 + lats[1695] = -29.209138229318015 + lats[1696] = -29.279436997329057 + lats[1697] = -29.349735765337677 + lats[1698] = -29.420034533343859 + lats[1699] = -29.490333301347597 + lats[1700] = -29.560632069348884 + lats[1701] = -29.630930837347698 + lats[1702] = -29.701229605344039 + lats[1703] = -29.771528373337894 + lats[1704] = -29.841827141329258 + lats[1705] = -29.91212590931811 + lats[1706] = -29.98242467730444 + lats[1707] = -30.052723445288244 + lats[1708] = -30.123022213269511 + lats[1709] = -30.19332098124822 + lats[1710] = -30.263619749224372 + lats[1711] = -30.333918517197947 + lats[1712] = -30.404217285168947 + lats[1713] = -30.47451605313735 + lats[1714] = -30.544814821103138 + lats[1715] = -30.615113589066322 + lats[1716] = -30.685412357026873 + lats[1717] = -30.755711124984781 + lats[1718] = -30.826009892940046 + lats[1719] = -30.896308660892647 + lats[1720] = -30.966607428842572 + lats[1721] = -31.036906196789811 + lats[1722] = -31.107204964734358 + lats[1723] = -31.177503732676204 + lats[1724] = -31.247802500615318 + lats[1725] = -31.318101268551715 + lats[1726] = -31.388400036485361 + lats[1727] = -31.458698804416255 + lats[1728] = -31.528997572344384 + lats[1729] = -31.599296340269738 + lats[1730] = -31.669595108192297 + lats[1731] = -31.739893876112063 + lats[1732] = -31.810192644029012 + lats[1733] = -31.880491411943137 + lats[1734] = -31.950790179854422 + lats[1735] = -32.021088947762863 + lats[1736] = -32.091387715668439 + lats[1737] = -32.161686483571145 + lats[1738] = -32.231985251470959 + lats[1739] = -32.302284019367875 + lats[1740] = -32.372582787261891 + lats[1741] = -32.442881555152965 + lats[1742] = -32.513180323041112 + lats[1743] = -32.583479090926325 + lats[1744] = -32.653777858808567 + lats[1745] = -32.724076626687825 + lats[1746] = -32.794375394564113 + lats[1747] = -32.864674162437396 + lats[1748] = -32.934972930307666 + lats[1749] = -33.005271698174909 + lats[1750] = -33.075570466039117 + lats[1751] = -33.145869233900278 + lats[1752] = -33.216168001758369 + lats[1753] = -33.286466769613391 + lats[1754] = -33.356765537465314 + lats[1755] = -33.42706430531414 + lats[1756] = -33.497363073159853 + lats[1757] = -33.567661841002426 + lats[1758] = -33.637960608841851 + lats[1759] = -33.708259376678136 + lats[1760] = -33.778558144511237 + lats[1761] = -33.848856912341155 + lats[1762] = -33.919155680167876 + lats[1763] = -33.989454447991392 + lats[1764] = -34.059753215811682 + lats[1765] = -34.130051983628725 + lats[1766] = -34.200350751442521 + lats[1767] = -34.270649519253041 + lats[1768] = -34.340948287060286 + lats[1769] = -34.411247054864234 + lats[1770] = -34.481545822664863 + lats[1771] = -34.551844590462188 + lats[1772] = -34.622143358256153 + lats[1773] = -34.692442126046771 + lats[1774] = -34.762740893834028 + lats[1775] = -34.833039661617903 + lats[1776] = -34.903338429398374 + lats[1777] = -34.973637197175435 + lats[1778] = -35.043935964949064 + lats[1779] = -35.114234732719261 + lats[1780] = -35.184533500486005 + lats[1781] = -35.254832268249267 + lats[1782] = -35.325131036009047 + lats[1783] = -35.395429803765317 + lats[1784] = -35.465728571518085 + lats[1785] = -35.536027339267314 + lats[1786] = -35.606326107012997 + lats[1787] = -35.676624874755113 + lats[1788] = -35.746923642493655 + lats[1789] = -35.817222410228595 + lats[1790] = -35.887521177959933 + lats[1791] = -35.957819945687639 + lats[1792] = -36.028118713411708 + lats[1793] = -36.098417481132117 + lats[1794] = -36.16871624884886 + lats[1795] = -36.239015016561908 + lats[1796] = -36.309313784271254 + lats[1797] = -36.379612551976876 + lats[1798] = -36.449911319678755 + lats[1799] = -36.520210087376888 + lats[1800] = -36.590508855071242 + lats[1801] = -36.660807622761808 + lats[1802] = -36.731106390448581 + lats[1803] = -36.801405158131523 + lats[1804] = -36.871703925810628 + lats[1805] = -36.942002693485883 + lats[1806] = -37.012301461157264 + lats[1807] = -37.082600228824752 + lats[1808] = -37.152898996488332 + lats[1809] = -37.223197764147997 + lats[1810] = -37.293496531803719 + lats[1811] = -37.363795299455489 + lats[1812] = -37.434094067103274 + lats[1813] = -37.504392834747065 + lats[1814] = -37.574691602386856 + lats[1815] = -37.644990370022605 + lats[1816] = -37.715289137654317 + lats[1817] = -37.785587905281965 + lats[1818] = -37.855886672905527 + lats[1819] = -37.926185440524989 + lats[1820] = -37.99648420814033 + lats[1821] = -38.066782975751536 + lats[1822] = -38.137081743358586 + lats[1823] = -38.20738051096145 + lats[1824] = -38.277679278560143 + lats[1825] = -38.347978046154608 + lats[1826] = -38.418276813744846 + lats[1827] = -38.488575581330842 + lats[1828] = -38.558874348912568 + lats[1829] = -38.629173116490001 + lats[1830] = -38.699471884063136 + lats[1831] = -38.769770651631937 + lats[1832] = -38.840069419196389 + lats[1833] = -38.910368186756479 + lats[1834] = -38.980666954312184 + lats[1835] = -39.050965721863491 + lats[1836] = -39.121264489410365 + lats[1837] = -39.191563256952804 + lats[1838] = -39.261862024490775 + lats[1839] = -39.332160792024254 + lats[1840] = -39.402459559553229 + lats[1841] = -39.472758327077692 + lats[1842] = -39.543057094597607 + lats[1843] = -39.613355862112947 + lats[1844] = -39.683654629623703 + lats[1845] = -39.753953397129855 + lats[1846] = -39.824252164631375 + lats[1847] = -39.894550932128247 + lats[1848] = -39.964849699620437 + lats[1849] = -40.035148467107952 + lats[1850] = -40.105447234590748 + lats[1851] = -40.175746002068806 + lats[1852] = -40.246044769542102 + lats[1853] = -40.316343537010617 + lats[1854] = -40.386642304474343 + lats[1855] = -40.456941071933244 + lats[1856] = -40.527239839387299 + lats[1857] = -40.597538606836487 + lats[1858] = -40.667837374280786 + lats[1859] = -40.738136141720176 + lats[1860] = -40.808434909154634 + lats[1861] = -40.878733676584126 + lats[1862] = -40.949032444008644 + lats[1863] = -41.01933121142816 + lats[1864] = -41.089629978842645 + lats[1865] = -41.159928746252085 + lats[1866] = -41.230227513656445 + lats[1867] = -41.300526281055724 + lats[1868] = -41.370825048449873 + lats[1869] = -41.441123815838885 + lats[1870] = -41.511422583222718 + lats[1871] = -41.581721350601363 + lats[1872] = -41.6520201179748 + lats[1873] = -41.722318885343 + lats[1874] = -41.792617652705921 + lats[1875] = -41.862916420063563 + lats[1876] = -41.933215187415882 + lats[1877] = -42.003513954762873 + lats[1878] = -42.073812722104492 + lats[1879] = -42.144111489440725 + lats[1880] = -42.214410256771551 + lats[1881] = -42.284709024096927 + lats[1882] = -42.355007791416853 + lats[1883] = -42.425306558731272 + lats[1884] = -42.495605326040177 + lats[1885] = -42.565904093343548 + lats[1886] = -42.63620286064134 + lats[1887] = -42.706501627933541 + lats[1888] = -42.776800395220121 + lats[1889] = -42.847099162501053 + lats[1890] = -42.917397929776307 + lats[1891] = -42.987696697045862 + lats[1892] = -43.057995464309691 + lats[1893] = -43.128294231567757 + lats[1894] = -43.19859299882004 + lats[1895] = -43.26889176606651 + lats[1896] = -43.339190533307139 + lats[1897] = -43.409489300541907 + lats[1898] = -43.479788067770777 + lats[1899] = -43.550086834993728 + lats[1900] = -43.620385602210717 + lats[1901] = -43.690684369421732 + lats[1902] = -43.760983136626741 + lats[1903] = -43.831281903825705 + lats[1904] = -43.9015806710186 + lats[1905] = -43.971879438205391 + lats[1906] = -44.042178205386072 + lats[1907] = -44.112476972560586 + lats[1908] = -44.182775739728925 + lats[1909] = -44.253074506891046 + lats[1910] = -44.323373274046915 + lats[1911] = -44.39367204119651 + lats[1912] = -44.463970808339802 + lats[1913] = -44.534269575476756 + lats[1914] = -44.604568342607337 + lats[1915] = -44.674867109731515 + lats[1916] = -44.745165876849271 + lats[1917] = -44.81546464396056 + lats[1918] = -44.885763411065362 + lats[1919] = -44.956062178163634 + lats[1920] = -45.026360945255341 + lats[1921] = -45.096659712340461 + lats[1922] = -45.166958479418959 + lats[1923] = -45.237257246490813 + lats[1924] = -45.30755601355596 + lats[1925] = -45.377854780614399 + lats[1926] = -45.448153547666081 + lats[1927] = -45.51845231471097 + lats[1928] = -45.588751081749038 + lats[1929] = -45.659049848780263 + lats[1930] = -45.729348615804589 + lats[1931] = -45.799647382821995 + lats[1932] = -45.869946149832437 + lats[1933] = -45.94024491683588 + lats[1934] = -46.01054368383231 + lats[1935] = -46.080842450821663 + lats[1936] = -46.151141217803925 + lats[1937] = -46.221439984779053 + lats[1938] = -46.291738751747012 + lats[1939] = -46.362037518707766 + lats[1940] = -46.432336285661272 + lats[1941] = -46.502635052607502 + lats[1942] = -46.572933819546414 + lats[1943] = -46.643232586477971 + lats[1944] = -46.713531353402139 + lats[1945] = -46.783830120318882 + lats[1946] = -46.85412888722815 + lats[1947] = -46.924427654129929 + lats[1948] = -46.994726421024154 + lats[1949] = -47.065025187910805 + lats[1950] = -47.13532395478984 + lats[1951] = -47.205622721661214 + lats[1952] = -47.275921488524894 + lats[1953] = -47.346220255380835 + lats[1954] = -47.416519022228997 + lats[1955] = -47.486817789069342 + lats[1956] = -47.557116555901828 + lats[1957] = -47.627415322726435 + lats[1958] = -47.697714089543084 + lats[1959] = -47.76801285635176 + lats[1960] = -47.838311623152421 + lats[1961] = -47.908610389945018 + lats[1962] = -47.978909156729507 + lats[1963] = -48.049207923505868 + lats[1964] = -48.119506690274015 + lats[1965] = -48.189805457033941 + lats[1966] = -48.260104223785596 + lats[1967] = -48.330402990528938 + lats[1968] = -48.400701757263917 + lats[1969] = -48.47100052399049 + lats[1970] = -48.541299290708608 + lats[1971] = -48.611598057418242 + lats[1972] = -48.681896824119335 + lats[1973] = -48.752195590811837 + lats[1974] = -48.822494357495721 + lats[1975] = -48.892793124170929 + lats[1976] = -48.963091890837418 + lats[1977] = -49.03339065749514 + lats[1978] = -49.103689424144044 + lats[1979] = -49.173988190784094 + lats[1980] = -49.244286957415234 + lats[1981] = -49.314585724037435 + lats[1982] = -49.384884490650613 + lats[1983] = -49.455183257254745 + lats[1984] = -49.525482023849783 + lats[1985] = -49.595780790435676 + lats[1986] = -49.66607955701236 + lats[1987] = -49.736378323579807 + lats[1988] = -49.80667709013796 + lats[1989] = -49.876975856686762 + lats[1990] = -49.947274623226157 + lats[1991] = -50.017573389756123 + lats[1992] = -50.087872156276575 + lats[1993] = -50.158170922787484 + lats[1994] = -50.228469689288779 + lats[1995] = -50.298768455780426 + lats[1996] = -50.369067222262359 + lats[1997] = -50.439365988734544 + lats[1998] = -50.509664755196901 + lats[1999] = -50.579963521649397 + lats[2000] = -50.650262288091959 + lats[2001] = -50.720561054524559 + lats[2002] = -50.790859820947119 + lats[2003] = -50.86115858735959 + lats[2004] = -50.931457353761914 + lats[2005] = -51.001756120154049 + lats[2006] = -51.072054886535909 + lats[2007] = -51.14235365290746 + lats[2008] = -51.21265241926865 + lats[2009] = -51.282951185619417 + lats[2010] = -51.353249951959683 + lats[2011] = -51.423548718289396 + lats[2012] = -51.493847484608516 + lats[2013] = -51.56414625091697 + lats[2014] = -51.634445017214695 + lats[2015] = -51.704743783501634 + lats[2016] = -51.775042549777737 + lats[2017] = -51.845341316042933 + lats[2018] = -51.915640082297152 + lats[2019] = -51.985938848540336 + lats[2020] = -52.056237614772435 + lats[2021] = -52.126536380993372 + lats[2022] = -52.196835147203096 + lats[2023] = -52.26713391340153 + lats[2024] = -52.337432679588609 + lats[2025] = -52.407731445764284 + lats[2026] = -52.478030211928477 + lats[2027] = -52.548328978081123 + lats[2028] = -52.618627744222159 + lats[2029] = -52.688926510351514 + lats[2030] = -52.759225276469131 + lats[2031] = -52.829524042574917 + lats[2032] = -52.899822808668837 + lats[2033] = -52.970121574750792 + lats[2034] = -53.040420340820731 + lats[2035] = -53.110719106878584 + lats[2036] = -53.181017872924265 + lats[2037] = -53.251316638957725 + lats[2038] = -53.321615404978871 + lats[2039] = -53.391914170987633 + lats[2040] = -53.462212936983953 + lats[2041] = -53.53251170296776 + lats[2042] = -53.602810468938962 + lats[2043] = -53.673109234897495 + lats[2044] = -53.743408000843282 + lats[2045] = -53.813706766776235 + lats[2046] = -53.884005532696307 + lats[2047] = -53.954304298603383 + lats[2048] = -54.024603064497434 + lats[2049] = -54.094901830378333 + lats[2050] = -54.165200596246031 + lats[2051] = -54.235499362100448 + lats[2052] = -54.305798127941479 + lats[2053] = -54.376096893769081 + lats[2054] = -54.446395659583146 + lats[2055] = -54.516694425383605 + lats[2056] = -54.586993191170357 + lats[2057] = -54.657291956943347 + lats[2058] = -54.727590722702473 + lats[2059] = -54.797889488447652 + lats[2060] = -54.868188254178797 + lats[2061] = -54.938487019895831 + lats[2062] = -55.008785785598668 + lats[2063] = -55.07908455128721 + lats[2064] = -55.149383316961377 + lats[2065] = -55.219682082621084 + lats[2066] = -55.289980848266232 + lats[2067] = -55.360279613896743 + lats[2068] = -55.430578379512511 + lats[2069] = -55.500877145113449 + lats[2070] = -55.571175910699488 + lats[2071] = -55.641474676270505 + lats[2072] = -55.711773441826416 + lats[2073] = -55.782072207367136 + lats[2074] = -55.852370972892551 + lats[2075] = -55.922669738402583 + lats[2076] = -55.992968503897131 + lats[2077] = -56.063267269376091 + lats[2078] = -56.133566034839362 + lats[2079] = -56.203864800286865 + lats[2080] = -56.274163565718467 + lats[2081] = -56.34446233113411 + lats[2082] = -56.41476109653366 + lats[2083] = -56.485059861917016 + lats[2084] = -56.555358627284086 + lats[2085] = -56.625657392634771 + lats[2086] = -56.695956157968951 + lats[2087] = -56.766254923286517 + lats[2088] = -56.836553688587379 + lats[2089] = -56.90685245387143 + lats[2090] = -56.977151219138541 + lats[2091] = -57.047449984388614 + lats[2092] = -57.117748749621541 + lats[2093] = -57.188047514837208 + lats[2094] = -57.258346280035504 + lats[2095] = -57.328645045216312 + lats[2096] = -57.398943810379521 + lats[2097] = -57.469242575525016 + lats[2098] = -57.539541340652676 + lats[2099] = -57.60984010576238 + lats[2100] = -57.680138870854037 + lats[2101] = -57.75043763592749 + lats[2102] = -57.820736400982646 + lats[2103] = -57.891035166019364 + lats[2104] = -57.961333931037537 + lats[2105] = -58.031632696037022 + lats[2106] = -58.101931461017728 + lats[2107] = -58.172230225979497 + lats[2108] = -58.242528990922203 + lats[2109] = -58.312827755845746 + lats[2110] = -58.383126520749968 + lats[2111] = -58.453425285634758 + lats[2112] = -58.523724050499972 + lats[2113] = -58.594022815345468 + lats[2114] = -58.664321580171141 + lats[2115] = -58.73462034497684 + lats[2116] = -58.804919109762423 + lats[2117] = -58.875217874527763 + lats[2118] = -58.945516639272725 + lats[2119] = -59.015815403997145 + lats[2120] = -59.086114168700909 + lats[2121] = -59.156412933383855 + lats[2122] = -59.226711698045854 + lats[2123] = -59.29701046268675 + lats[2124] = -59.3673092273064 + lats[2125] = -59.43760799190467 + lats[2126] = -59.507906756481383 + lats[2127] = -59.578205521036402 + lats[2128] = -59.64850428556958 + lats[2129] = -59.718803050080759 + lats[2130] = -59.78910181456979 + lats[2131] = -59.859400579036503 + lats[2132] = -59.929699343480763 + lats[2133] = -59.999998107902378 + lats[2134] = -60.070296872301235 + lats[2135] = -60.140595636677112 + lats[2136] = -60.21089440102989 + lats[2137] = -60.28119316535939 + lats[2138] = -60.35149192966545 + lats[2139] = -60.421790693947884 + lats[2140] = -60.492089458206543 + lats[2141] = -60.562388222441243 + lats[2142] = -60.632686986651805 + lats[2143] = -60.702985750838074 + lats[2144] = -60.773284514999872 + lats[2145] = -60.843583279137007 + lats[2146] = -60.913882043249295 + lats[2147] = -60.984180807336578 + lats[2148] = -61.054479571398652 + lats[2149] = -61.124778335435344 + lats[2150] = -61.195077099446451 + lats[2151] = -61.265375863431785 + lats[2152] = -61.335674627391185 + lats[2153] = -61.405973391324409 + lats[2154] = -61.476272155231321 + lats[2155] = -61.546570919111666 + lats[2156] = -61.616869682965287 + lats[2157] = -61.687168446791986 + lats[2158] = -61.757467210591535 + lats[2159] = -61.827765974363729 + lats[2160] = -61.898064738108381 + lats[2161] = -61.968363501825259 + lats[2162] = -62.038662265514176 + lats[2163] = -62.108961029174914 + lats[2164] = -62.179259792807258 + lats[2165] = -62.249558556410982 + lats[2166] = -62.319857319985871 + lats[2167] = -62.3901560835317 + lats[2168] = -62.460454847048261 + lats[2169] = -62.530753610535321 + lats[2170] = -62.60105237399263 + lats[2171] = -62.67135113741999 + lats[2172] = -62.741649900817137 + lats[2173] = -62.811948664183866 + lats[2174] = -62.882247427519928 + lats[2175] = -62.952546190825068 + lats[2176] = -63.022844954099064 + lats[2177] = -63.093143717341647 + lats[2178] = -63.163442480552604 + lats[2179] = -63.23374124373165 + lats[2180] = -63.304040006878537 + lats[2181] = -63.374338769993031 + lats[2182] = -63.444637533074854 + lats[2183] = -63.514936296123757 + lats[2184] = -63.585235059139464 + lats[2185] = -63.655533822121711 + lats[2186] = -63.725832585070251 + lats[2187] = -63.796131347984762 + lats[2188] = -63.866430110865004 + lats[2189] = -63.93672887371072 + lats[2190] = -64.00702763652157 + lats[2191] = -64.07732639929732 + lats[2192] = -64.147625162037642 + lats[2193] = -64.21792392474228 + lats[2194] = -64.288222687410922 + lats[2195] = -64.358521450043284 + lats[2196] = -64.428820212639039 + lats[2197] = -64.499118975197902 + lats[2198] = -64.569417737719576 + lats[2199] = -64.639716500203733 + lats[2200] = -64.710015262650074 + lats[2201] = -64.780314025058246 + lats[2202] = -64.850612787427963 + lats[2203] = -64.920911549758912 + lats[2204] = -64.991210312050711 + lats[2205] = -65.061509074303089 + lats[2206] = -65.131807836515677 + lats[2207] = -65.202106598688133 + lats[2208] = -65.272405360820116 + lats[2209] = -65.342704122911286 + lats[2210] = -65.413002884961315 + lats[2211] = -65.483301646969792 + lats[2212] = -65.553600408936404 + lats[2213] = -65.623899170860767 + lats[2214] = -65.694197932742526 + lats[2215] = -65.764496694581283 + lats[2216] = -65.834795456376696 + lats[2217] = -65.905094218128355 + lats[2218] = -65.975392979835888 + lats[2219] = -66.045691741498899 + lats[2220] = -66.115990503117033 + lats[2221] = -66.186289264689833 + lats[2222] = -66.256588026216932 + lats[2223] = -66.326886787697887 + lats[2224] = -66.397185549132331 + lats[2225] = -66.467484310519808 + lats[2226] = -66.537783071859891 + lats[2227] = -66.608081833152212 + lats[2228] = -66.678380594396273 + lats[2229] = -66.748679355591662 + lats[2230] = -66.818978116737924 + lats[2231] = -66.889276877834618 + lats[2232] = -66.95957563888129 + lats[2233] = -67.029874399877471 + lats[2234] = -67.100173160822706 + lats[2235] = -67.170471921716526 + lats[2236] = -67.240770682558434 + lats[2237] = -67.311069443347961 + lats[2238] = -67.381368204084609 + lats[2239] = -67.451666964767895 + lats[2240] = -67.521965725397308 + lats[2241] = -67.592264485972336 + lats[2242] = -67.662563246492482 + lats[2243] = -67.732862006957205 + lats[2244] = -67.803160767365966 + lats[2245] = -67.873459527718282 + lats[2246] = -67.943758288013555 + lats[2247] = -68.014057048251274 + lats[2248] = -68.084355808430871 + lats[2249] = -68.154654568551791 + lats[2250] = -68.224953328613438 + lats[2251] = -68.295252088615257 + lats[2252] = -68.365550848556666 + lats[2253] = -68.435849608437067 + lats[2254] = -68.506148368255865 + lats[2255] = -68.576447128012447 + lats[2256] = -68.646745887706189 + lats[2257] = -68.717044647336493 + lats[2258] = -68.787343406902693 + lats[2259] = -68.85764216640419 + lats[2260] = -68.927940925840304 + lats[2261] = -68.998239685210365 + lats[2262] = -69.068538444513763 + lats[2263] = -69.138837203749759 + lats[2264] = -69.209135962917699 + lats[2265] = -69.279434722016902 + lats[2266] = -69.349733481046613 + lats[2267] = -69.420032240006194 + lats[2268] = -69.490330998894862 + lats[2269] = -69.560629757711908 + lats[2270] = -69.630928516456592 + lats[2271] = -69.701227275128161 + lats[2272] = -69.771526033725834 + lats[2273] = -69.841824792248843 + lats[2274] = -69.912123550696421 + lats[2275] = -69.982422309067744 + lats[2276] = -70.052721067362043 + lats[2277] = -70.123019825578467 + lats[2278] = -70.193318583716191 + lats[2279] = -70.263617341774406 + lats[2280] = -70.333916099752187 + lats[2281] = -70.404214857648739 + lats[2282] = -70.474513615463138 + lats[2283] = -70.544812373194532 + lats[2284] = -70.615111130841967 + lats[2285] = -70.685409888404578 + lats[2286] = -70.755708645881384 + lats[2287] = -70.826007403271475 + lats[2288] = -70.896306160573886 + lats[2289] = -70.966604917787635 + lats[2290] = -71.036903674911756 + lats[2291] = -71.107202431945211 + lats[2292] = -71.177501188887007 + lats[2293] = -71.247799945736105 + lats[2294] = -71.318098702491469 + lats[2295] = -71.388397459152031 + lats[2296] = -71.458696215716685 + lats[2297] = -71.528994972184378 + lats[2298] = -71.599293728553988 + lats[2299] = -71.669592484824364 + lats[2300] = -71.739891240994368 + lats[2301] = -71.810189997062835 + lats[2302] = -71.880488753028587 + lats[2303] = -71.950787508890414 + lats[2304] = -72.02108626464711 + lats[2305] = -72.091385020297409 + lats[2306] = -72.161683775840089 + lats[2307] = -72.231982531273843 + lats[2308] = -72.302281286597392 + lats[2309] = -72.3725800418094 + lats[2310] = -72.442878796908545 + lats[2311] = -72.513177551893421 + lats[2312] = -72.583476306762691 + lats[2313] = -72.653775061514935 + lats[2314] = -72.724073816148703 + lats[2315] = -72.794372570662574 + lats[2316] = -72.864671325055056 + lats[2317] = -72.934970079324657 + lats[2318] = -73.005268833469799 + lats[2319] = -73.075567587489019 + lats[2320] = -73.145866341380668 + lats[2321] = -73.216165095143182 + lats[2322] = -73.2864638487749 + lats[2323] = -73.356762602274188 + lats[2324] = -73.427061355639339 + lats[2325] = -73.497360108868662 + lats[2326] = -73.567658861960396 + lats[2327] = -73.637957614912779 + lats[2328] = -73.70825636772399 + lats[2329] = -73.778555120392184 + lats[2330] = -73.848853872915541 + lats[2331] = -73.919152625292114 + lats[2332] = -73.98945137751997 + lats[2333] = -74.059750129597163 + lats[2334] = -74.13004888152166 + lats[2335] = -74.200347633291472 + lats[2336] = -74.270646384904481 + lats[2337] = -74.340945136358584 + lats[2338] = -74.411243887651622 + lats[2339] = -74.481542638781434 + lats[2340] = -74.551841389745761 + lats[2341] = -74.622140140542356 + lats[2342] = -74.692438891168877 + lats[2343] = -74.762737641622991 + lats[2344] = -74.833036391902269 + lats[2345] = -74.903335142004323 + lats[2346] = -74.973633891926625 + lats[2347] = -75.043932641666672 + lats[2348] = -75.114231391221821 + lats[2349] = -75.184530140589501 + lats[2350] = -75.254828889766983 + lats[2351] = -75.325127638751567 + lats[2352] = -75.395426387540439 + lats[2353] = -75.465725136130786 + lats[2354] = -75.536023884519707 + lats[2355] = -75.60632263270422 + lats[2356] = -75.67662138068134 + lats[2357] = -75.746920128447996 + lats[2358] = -75.81721887600105 + lats[2359] = -75.887517623337317 + lats[2360] = -75.957816370453543 + lats[2361] = -76.028115117346374 + lats[2362] = -76.098413864012443 + lats[2363] = -76.16871261044831 + lats[2364] = -76.239011356650423 + lats[2365] = -76.3093101026152 + lats[2366] = -76.379608848338933 + lats[2367] = -76.449907593817869 + lats[2368] = -76.520206339048215 + lats[2369] = -76.59050508402602 + lats[2370] = -76.660803828747362 + lats[2371] = -76.731102573208048 + lats[2372] = -76.801401317404 + lats[2373] = -76.871700061330955 + lats[2374] = -76.941998804984564 + lats[2375] = -77.012297548360323 + lats[2376] = -77.082596291453768 + lats[2377] = -77.15289503426024 + lats[2378] = -77.22319377677502 + lats[2379] = -77.293492518993247 + lats[2380] = -77.363791260909963 + lats[2381] = -77.434090002520122 + lats[2382] = -77.504388743818524 + lats[2383] = -77.574687484799924 + lats[2384] = -77.644986225458879 + lats[2385] = -77.71528496578982 + lats[2386] = -77.785583705787161 + lats[2387] = -77.855882445445019 + lats[2388] = -77.926181184757539 + lats[2389] = -77.996479923718596 + lats[2390] = -78.066778662322022 + lats[2391] = -78.137077400561424 + lats[2392] = -78.207376138430348 + lats[2393] = -78.277674875922045 + lats[2394] = -78.347973613029708 + lats[2395] = -78.418272349746417 + lats[2396] = -78.488571086064923 + lats[2397] = -78.558869821977908 + lats[2398] = -78.629168557477882 + lats[2399] = -78.699467292557102 + lats[2400] = -78.769766027207638 + lats[2401] = -78.840064761421445 + lats[2402] = -78.910363495190211 + lats[2403] = -78.980662228505423 + lats[2404] = -79.050960961358285 + lats[2405] = -79.121259693739859 + lats[2406] = -79.191558425640977 + lats[2407] = -79.261857157052191 + lats[2408] = -79.332155887963822 + lats[2409] = -79.402454618365894 + lats[2410] = -79.472753348248219 + lats[2411] = -79.543052077600308 + lats[2412] = -79.61335080641139 + lats[2413] = -79.683649534670437 + lats[2414] = -79.753948262366038 + lats[2415] = -79.824246989486554 + lats[2416] = -79.894545716019948 + lats[2417] = -79.9648444419539 + lats[2418] = -80.035143167275749 + lats[2419] = -80.105441891972376 + lats[2420] = -80.175740616030438 + lats[2421] = -80.246039339436052 + lats[2422] = -80.316338062175078 + lats[2423] = -80.386636784232863 + lats[2424] = -80.456935505594302 + lats[2425] = -80.527234226243991 + lats[2426] = -80.59753294616587 + lats[2427] = -80.667831665343556 + lats[2428] = -80.73813038376008 + lats[2429] = -80.808429101397948 + lats[2430] = -80.878727818239184 + lats[2431] = -80.949026534265244 + lats[2432] = -81.019325249456955 + lats[2433] = -81.089623963794551 + lats[2434] = -81.159922677257711 + lats[2435] = -81.230221389825374 + lats[2436] = -81.300520101475826 + lats[2437] = -81.370818812186627 + lats[2438] = -81.441117521934686 + lats[2439] = -81.511416230696042 + lats[2440] = -81.581714938445955 + lats[2441] = -81.652013645158945 + lats[2442] = -81.722312350808508 + lats[2443] = -81.792611055367345 + lats[2444] = -81.862909758807191 + lats[2445] = -81.933208461098829 + lats[2446] = -82.003507162211946 + lats[2447] = -82.073805862115165 + lats[2448] = -82.144104560776 + lats[2449] = -82.214403258160871 + lats[2450] = -82.284701954234833 + lats[2451] = -82.355000648961692 + lats[2452] = -82.425299342304029 + lats[2453] = -82.495598034222837 + lats[2454] = -82.56589672467787 + lats[2455] = -82.63619541362705 + lats[2456] = -82.706494101026948 + lats[2457] = -82.77679278683226 + lats[2458] = -82.84709147099602 + lats[2459] = -82.917390153469313 + lats[2460] = -82.987688834201322 + lats[2461] = -83.057987513139125 + lats[2462] = -83.128286190227698 + lats[2463] = -83.198584865409657 + lats[2464] = -83.268883538625232 + lats[2465] = -83.339182209812321 + lats[2466] = -83.409480878905782 + lats[2467] = -83.479779545838113 + lats[2468] = -83.550078210538487 + lats[2469] = -83.620376872933264 + lats[2470] = -83.690675532945292 + lats[2471] = -83.760974190494011 + lats[2472] = -83.831272845495249 + lats[2473] = -83.901571497860914 + lats[2474] = -83.971870147498763 + lats[2475] = -84.042168794312317 + lats[2476] = -84.112467438200326 + lats[2477] = -84.18276607905679 + lats[2478] = -84.253064716770425 + lats[2479] = -84.323363351224444 + lats[2480] = -84.393661982296322 + lats[2481] = -84.463960609857125 + lats[2482] = -84.534259233771479 + lats[2483] = -84.604557853896708 + lats[2484] = -84.674856470082915 + lats[2485] = -84.745155082171991 + lats[2486] = -84.81545368999717 + lats[2487] = -84.885752293382765 + lats[2488] = -84.95605089214304 + lats[2489] = -85.026349486081983 + lats[2490] = -85.09664807499216 + lats[2491] = -85.16694665865414 + lats[2492] = -85.237245236835548 + lats[2493] = -85.307543809290152 + lats[2494] = -85.377842375756586 + lats[2495] = -85.448140935957483 + lats[2496] = -85.518439489597966 + lats[2497] = -85.588738036364362 + lats[2498] = -85.659036575922883 + lats[2499] = -85.729335107917464 + lats[2500] = -85.799633631968391 + lats[2501] = -85.869932147670127 + lats[2502] = -85.940230654588888 + lats[2503] = -86.010529152260403 + lats[2504] = -86.080827640187209 + lats[2505] = -86.151126117835304 + lats[2506] = -86.221424584631109 + lats[2507] = -86.291723039957418 + lats[2508] = -86.362021483149363 + lats[2509] = -86.432319913489792 + lats[2510] = -86.502618330203831 + lats[2511] = -86.572916732453024 + lats[2512] = -86.643215119328573 + lats[2513] = -86.713513489844246 + lats[2514] = -86.783811842927179 + lats[2515] = -86.854110177408927 + lats[2516] = -86.924408492014166 + lats[2517] = -86.994706785348129 + lats[2518] = -87.065005055882821 + lats[2519] = -87.135303301939786 + lats[2520] = -87.205601521672108 + lats[2521] = -87.275899713041966 + lats[2522] = -87.346197873795816 + lats[2523] = -87.416496001434894 + lats[2524] = -87.486794093180748 + lats[2525] = -87.557092145935584 + lats[2526] = -87.627390156234085 + lats[2527] = -87.697688120188062 + lats[2528] = -87.767986033419561 + lats[2529] = -87.838283890981543 + lats[2530] = -87.908581687261687 + lats[2531] = -87.978879415867283 + lats[2532] = -88.049177069484486 + lats[2533] = -88.119474639706425 + lats[2534] = -88.189772116820762 + lats[2535] = -88.26006948954614 + lats[2536] = -88.330366744702559 + lats[2537] = -88.40066386679355 + lats[2538] = -88.470960837474877 + lats[2539] = -88.541257634868515 + lats[2540] = -88.611554232668382 + lats[2541] = -88.681850598961759 + lats[2542] = -88.752146694650691 + lats[2543] = -88.822442471310097 + lats[2544] = -88.892737868230952 + lats[2545] = -88.96303280826325 + lats[2546] = -89.033327191845927 + lats[2547] = -89.103620888238879 + lats[2548] = -89.173913722284126 + lats[2549] = -89.24420545380525 + lats[2550] = -89.314495744374256 + lats[2551] = -89.3847841013921 + lats[2552] = -89.45506977912261 + lats[2553] = -89.525351592371393 + lats[2554] = -89.595627537554492 + lats[2555] = -89.6658939412157 + lats[2556] = -89.736143271609578 + lats[2557] = -89.806357319542244 + lats[2558] = -89.876478353332288 + lats[2559] = -89.946187715665616 + return lats + + def get_precomputed_values_N1280_new(self): + lats = np.empty([1280*2]) lats[0] = 89.946187715665616 lats[1] = 89.876478353332288 lats[2] = 89.806357319542244 @@ -2952,6 +5523,7 @@ def axes_idx_to_octahedral_idx(self, first_idx, second_idx): # NOTE: OR somehow cache this for a given first_idx and then only modify the axis idx for second_idx when the # first_idx changes # time1 = time.time() + # octa_idx = self._first_idx_map[first_idx-1] + second_idx octa_idx = self._first_idx_map[first_idx-1] + second_idx # octa_idx = 0 # if first_idx == 1: @@ -2983,6 +5555,7 @@ def create_first_idx_map(self): first_idx_list = {} idx = 0 for i in range(2*self._resolution): + # first_idx_list[i] = idx first_idx_list[i] = idx if i <= self._resolution - 1: idx += 20 + 4 * i @@ -3020,21 +5593,32 @@ def unmap_first_val_to_start_line_idx(self, first_val): def unmap(self, first_val, second_val): time1 = time.time() - first_axis_vals = self._first_axis_vals + # first_axis_vals = self._first_axis_vals + # inv_first_axis_vals = self._inv_first_axis_vals tol = 1e-10 # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] # first_idx = first_axis_vals.index(first_val) + 1 # first_idx = len(first_axis_vals) - bisect.bisect_left(first_axis_vals[::-1], first_val - tol) - first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + 1 - second_axis_vals = self.second_axis_vals(first_val) + # first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + 1 + first_idx = bisect.bisect_left(self._inv_first_axis_vals, - (first_val - tol)) + # print(inv_first_axis_vals) + # print(first_val) + # first_idx = inv_first_axis_vals[first_val] + # first_idx = np.searchsorted(-first_axis_vals, - (first_val - tol), side="right") + if first_val not in self.treated_first_vals: + second_axis_vals = self.second_axis_vals(first_val) + self.treated_first_vals[first_val] = second_axis_vals + else: + second_axis_vals = self.treated_first_vals[first_val] # second_val = [val for val in second_axis_vals if second_val - tol < val < second_val + tol][0] # second_idx = second_axis_vals.index(second_val) second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) - print("TIME SPENT DOING VAL TO IDX") - print(time.time() - time1) + # second_idx = np.searchsorted(second_axis_vals, second_val - tol) + # print("TIME SPENT DOING VAL TO IDX") + # print(time.time() - time1) octahedral_index = self.axes_idx_to_octahedral_idx(first_idx, second_idx) - print("OCTAHEDRAL UNMAP TIME") - print(time.time() - time1) + # print("OCTAHEDRAL UNMAP TIME ") + # print(time.time() - time1) return octahedral_index From a898711ae58f6290292a05352a327e8c9e8b55b9 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 19 Oct 2023 14:55:51 +0200 Subject: [PATCH 169/332] optimise second idx search --- .../transformations/datacube_mappers.py | 2713 +---------------- 1 file changed, 93 insertions(+), 2620 deletions(-) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 40870f1ac..a26a0df6f 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -253,10 +253,11 @@ def __init__(self, base_axis, mapped_axes, resolution): self._base_axis = base_axis self._resolution = resolution self._first_axis_vals = self.first_axis_vals() - self._inv_first_axis_vals = self._first_axis_vals[::-1] + # self._inv_first_axis_vals = self._first_axis_vals[::-1] # self._inv_first_axis_vals = {v:k for k,v in self._first_axis_vals.items()} self._first_idx_map = self.create_first_idx_map() - self.treated_first_vals = dict() + self._second_axis_spacing = dict() + # self.treated_first_vals = dict() def gauss_first_guess(self): i = 0 @@ -2888,2570 +2889,6 @@ def get_precomputed_values_N1280(self): lats[2559] = -89.946187715665616 return lats - def get_precomputed_values_N1280_new(self): - lats = np.empty([1280*2]) - lats[0] = 89.946187715665616 - lats[1] = 89.876478353332288 - lats[2] = 89.806357319542244 - lats[3] = 89.736143271609578 - lats[4] = 89.6658939412157 - lats[5] = 89.595627537554492 - lats[6] = 89.525351592371393 - lats[7] = 89.45506977912261 - lats[8] = 89.3847841013921 - lats[9] = 89.314495744374256 - lats[10] = 89.24420545380525 - lats[11] = 89.173913722284126 - lats[12] = 89.103620888238879 - lats[13] = 89.033327191845927 - lats[14] = 88.96303280826325 - lats[15] = 88.892737868230952 - lats[16] = 88.822442471310097 - lats[17] = 88.752146694650691 - lats[18] = 88.681850598961759 - lats[19] = 88.611554232668382 - lats[20] = 88.541257634868515 - lats[21] = 88.470960837474877 - lats[22] = 88.40066386679355 - lats[23] = 88.330366744702559 - lats[24] = 88.26006948954614 - lats[25] = 88.189772116820762 - lats[26] = 88.119474639706425 - lats[27] = 88.049177069484486 - lats[28] = 87.978879415867283 - lats[29] = 87.908581687261687 - lats[30] = 87.838283890981543 - lats[31] = 87.767986033419561 - lats[32] = 87.697688120188062 - lats[33] = 87.627390156234085 - lats[34] = 87.557092145935584 - lats[35] = 87.486794093180748 - lats[36] = 87.416496001434894 - lats[37] = 87.346197873795816 - lats[38] = 87.275899713041966 - lats[39] = 87.205601521672108 - lats[40] = 87.135303301939786 - lats[41] = 87.065005055882821 - lats[42] = 86.994706785348129 - lats[43] = 86.924408492014166 - lats[44] = 86.854110177408927 - lats[45] = 86.783811842927179 - lats[46] = 86.713513489844246 - lats[47] = 86.643215119328573 - lats[48] = 86.572916732453024 - lats[49] = 86.502618330203831 - lats[50] = 86.432319913489792 - lats[51] = 86.362021483149363 - lats[52] = 86.291723039957418 - lats[53] = 86.221424584631109 - lats[54] = 86.151126117835304 - lats[55] = 86.080827640187209 - lats[56] = 86.010529152260403 - lats[57] = 85.940230654588888 - lats[58] = 85.869932147670127 - lats[59] = 85.799633631968391 - lats[60] = 85.729335107917464 - lats[61] = 85.659036575922883 - lats[62] = 85.588738036364362 - lats[63] = 85.518439489597966 - lats[64] = 85.448140935957483 - lats[65] = 85.377842375756586 - lats[66] = 85.307543809290152 - lats[67] = 85.237245236835548 - lats[68] = 85.16694665865414 - lats[69] = 85.09664807499216 - lats[70] = 85.026349486081983 - lats[71] = 84.95605089214304 - lats[72] = 84.885752293382765 - lats[73] = 84.81545368999717 - lats[74] = 84.745155082171991 - lats[75] = 84.674856470082915 - lats[76] = 84.604557853896708 - lats[77] = 84.534259233771479 - lats[78] = 84.463960609857125 - lats[79] = 84.393661982296322 - lats[80] = 84.323363351224444 - lats[81] = 84.253064716770425 - lats[82] = 84.18276607905679 - lats[83] = 84.112467438200326 - lats[84] = 84.042168794312317 - lats[85] = 83.971870147498763 - lats[86] = 83.901571497860914 - lats[87] = 83.831272845495249 - lats[88] = 83.760974190494011 - lats[89] = 83.690675532945292 - lats[90] = 83.620376872933264 - lats[91] = 83.550078210538487 - lats[92] = 83.479779545838113 - lats[93] = 83.409480878905782 - lats[94] = 83.339182209812321 - lats[95] = 83.268883538625232 - lats[96] = 83.198584865409657 - lats[97] = 83.128286190227698 - lats[98] = 83.057987513139125 - lats[99] = 82.987688834201322 - lats[100] = 82.917390153469313 - lats[101] = 82.84709147099602 - lats[102] = 82.77679278683226 - lats[103] = 82.706494101026948 - lats[104] = 82.63619541362705 - lats[105] = 82.56589672467787 - lats[106] = 82.495598034222837 - lats[107] = 82.425299342304029 - lats[108] = 82.355000648961692 - lats[109] = 82.284701954234833 - lats[110] = 82.214403258160871 - lats[111] = 82.144104560776 - lats[112] = 82.073805862115165 - lats[113] = 82.003507162211946 - lats[114] = 81.933208461098829 - lats[115] = 81.862909758807191 - lats[116] = 81.792611055367345 - lats[117] = 81.722312350808508 - lats[118] = 81.652013645158945 - lats[119] = 81.581714938445955 - lats[120] = 81.511416230696042 - lats[121] = 81.441117521934686 - lats[122] = 81.370818812186627 - lats[123] = 81.300520101475826 - lats[124] = 81.230221389825374 - lats[125] = 81.159922677257711 - lats[126] = 81.089623963794551 - lats[127] = 81.019325249456955 - lats[128] = 80.949026534265244 - lats[129] = 80.878727818239184 - lats[130] = 80.808429101397948 - lats[131] = 80.73813038376008 - lats[132] = 80.667831665343556 - lats[133] = 80.59753294616587 - lats[134] = 80.527234226243991 - lats[135] = 80.456935505594302 - lats[136] = 80.386636784232863 - lats[137] = 80.316338062175078 - lats[138] = 80.246039339436052 - lats[139] = 80.175740616030438 - lats[140] = 80.105441891972376 - lats[141] = 80.035143167275749 - lats[142] = 79.9648444419539 - lats[143] = 79.894545716019948 - lats[144] = 79.824246989486554 - lats[145] = 79.753948262366038 - lats[146] = 79.683649534670437 - lats[147] = 79.61335080641139 - lats[148] = 79.543052077600308 - lats[149] = 79.472753348248219 - lats[150] = 79.402454618365894 - lats[151] = 79.332155887963822 - lats[152] = 79.261857157052191 - lats[153] = 79.191558425640977 - lats[154] = 79.121259693739859 - lats[155] = 79.050960961358285 - lats[156] = 78.980662228505423 - lats[157] = 78.910363495190211 - lats[158] = 78.840064761421445 - lats[159] = 78.769766027207638 - lats[160] = 78.699467292557102 - lats[161] = 78.629168557477882 - lats[162] = 78.558869821977908 - lats[163] = 78.488571086064923 - lats[164] = 78.418272349746417 - lats[165] = 78.347973613029708 - lats[166] = 78.277674875922045 - lats[167] = 78.207376138430348 - lats[168] = 78.137077400561424 - lats[169] = 78.066778662322022 - lats[170] = 77.996479923718596 - lats[171] = 77.926181184757539 - lats[172] = 77.855882445445019 - lats[173] = 77.785583705787161 - lats[174] = 77.71528496578982 - lats[175] = 77.644986225458879 - lats[176] = 77.574687484799924 - lats[177] = 77.504388743818524 - lats[178] = 77.434090002520122 - lats[179] = 77.363791260909963 - lats[180] = 77.293492518993247 - lats[181] = 77.22319377677502 - lats[182] = 77.15289503426024 - lats[183] = 77.082596291453768 - lats[184] = 77.012297548360323 - lats[185] = 76.941998804984564 - lats[186] = 76.871700061330955 - lats[187] = 76.801401317404 - lats[188] = 76.731102573208048 - lats[189] = 76.660803828747362 - lats[190] = 76.59050508402602 - lats[191] = 76.520206339048215 - lats[192] = 76.449907593817869 - lats[193] = 76.379608848338933 - lats[194] = 76.3093101026152 - lats[195] = 76.239011356650423 - lats[196] = 76.16871261044831 - lats[197] = 76.098413864012443 - lats[198] = 76.028115117346374 - lats[199] = 75.957816370453543 - lats[200] = 75.887517623337317 - lats[201] = 75.81721887600105 - lats[202] = 75.746920128447996 - lats[203] = 75.67662138068134 - lats[204] = 75.60632263270422 - lats[205] = 75.536023884519707 - lats[206] = 75.465725136130786 - lats[207] = 75.395426387540439 - lats[208] = 75.325127638751567 - lats[209] = 75.254828889766983 - lats[210] = 75.184530140589501 - lats[211] = 75.114231391221821 - lats[212] = 75.043932641666672 - lats[213] = 74.973633891926625 - lats[214] = 74.903335142004323 - lats[215] = 74.833036391902269 - lats[216] = 74.762737641622991 - lats[217] = 74.692438891168877 - lats[218] = 74.622140140542356 - lats[219] = 74.551841389745761 - lats[220] = 74.481542638781434 - lats[221] = 74.411243887651622 - lats[222] = 74.340945136358584 - lats[223] = 74.270646384904481 - lats[224] = 74.200347633291472 - lats[225] = 74.13004888152166 - lats[226] = 74.059750129597163 - lats[227] = 73.98945137751997 - lats[228] = 73.919152625292114 - lats[229] = 73.848853872915541 - lats[230] = 73.778555120392184 - lats[231] = 73.70825636772399 - lats[232] = 73.637957614912779 - lats[233] = 73.567658861960396 - lats[234] = 73.497360108868662 - lats[235] = 73.427061355639339 - lats[236] = 73.356762602274188 - lats[237] = 73.2864638487749 - lats[238] = 73.216165095143182 - lats[239] = 73.145866341380668 - lats[240] = 73.075567587489019 - lats[241] = 73.005268833469799 - lats[242] = 72.934970079324657 - lats[243] = 72.864671325055056 - lats[244] = 72.794372570662574 - lats[245] = 72.724073816148703 - lats[246] = 72.653775061514935 - lats[247] = 72.583476306762691 - lats[248] = 72.513177551893421 - lats[249] = 72.442878796908545 - lats[250] = 72.3725800418094 - lats[251] = 72.302281286597392 - lats[252] = 72.231982531273843 - lats[253] = 72.161683775840089 - lats[254] = 72.091385020297409 - lats[255] = 72.02108626464711 - lats[256] = 71.950787508890414 - lats[257] = 71.880488753028587 - lats[258] = 71.810189997062835 - lats[259] = 71.739891240994368 - lats[260] = 71.669592484824364 - lats[261] = 71.599293728553988 - lats[262] = 71.528994972184378 - lats[263] = 71.458696215716685 - lats[264] = 71.388397459152031 - lats[265] = 71.318098702491469 - lats[266] = 71.247799945736105 - lats[267] = 71.177501188887007 - lats[268] = 71.107202431945211 - lats[269] = 71.036903674911756 - lats[270] = 70.966604917787635 - lats[271] = 70.896306160573886 - lats[272] = 70.826007403271475 - lats[273] = 70.755708645881384 - lats[274] = 70.685409888404578 - lats[275] = 70.615111130841967 - lats[276] = 70.544812373194532 - lats[277] = 70.474513615463138 - lats[278] = 70.404214857648739 - lats[279] = 70.333916099752187 - lats[280] = 70.263617341774406 - lats[281] = 70.193318583716191 - lats[282] = 70.123019825578467 - lats[283] = 70.052721067362043 - lats[284] = 69.982422309067744 - lats[285] = 69.912123550696421 - lats[286] = 69.841824792248843 - lats[287] = 69.771526033725834 - lats[288] = 69.701227275128161 - lats[289] = 69.630928516456592 - lats[290] = 69.560629757711908 - lats[291] = 69.490330998894862 - lats[292] = 69.420032240006194 - lats[293] = 69.349733481046613 - lats[294] = 69.279434722016902 - lats[295] = 69.209135962917699 - lats[296] = 69.138837203749759 - lats[297] = 69.068538444513763 - lats[298] = 68.998239685210365 - lats[299] = 68.927940925840304 - lats[300] = 68.85764216640419 - lats[301] = 68.787343406902693 - lats[302] = 68.717044647336493 - lats[303] = 68.646745887706189 - lats[304] = 68.576447128012447 - lats[305] = 68.506148368255865 - lats[306] = 68.435849608437067 - lats[307] = 68.365550848556666 - lats[308] = 68.295252088615257 - lats[309] = 68.224953328613438 - lats[310] = 68.154654568551791 - lats[311] = 68.084355808430871 - lats[312] = 68.014057048251274 - lats[313] = 67.943758288013555 - lats[314] = 67.873459527718282 - lats[315] = 67.803160767365966 - lats[316] = 67.732862006957205 - lats[317] = 67.662563246492482 - lats[318] = 67.592264485972336 - lats[319] = 67.521965725397308 - lats[320] = 67.451666964767895 - lats[321] = 67.381368204084609 - lats[322] = 67.311069443347961 - lats[323] = 67.240770682558434 - lats[324] = 67.170471921716526 - lats[325] = 67.100173160822706 - lats[326] = 67.029874399877471 - lats[327] = 66.95957563888129 - lats[328] = 66.889276877834618 - lats[329] = 66.818978116737924 - lats[330] = 66.748679355591662 - lats[331] = 66.678380594396273 - lats[332] = 66.608081833152212 - lats[333] = 66.537783071859891 - lats[334] = 66.467484310519808 - lats[335] = 66.397185549132331 - lats[336] = 66.326886787697887 - lats[337] = 66.256588026216932 - lats[338] = 66.186289264689833 - lats[339] = 66.115990503117033 - lats[340] = 66.045691741498899 - lats[341] = 65.975392979835888 - lats[342] = 65.905094218128355 - lats[343] = 65.834795456376696 - lats[344] = 65.764496694581283 - lats[345] = 65.694197932742526 - lats[346] = 65.623899170860767 - lats[347] = 65.553600408936404 - lats[348] = 65.483301646969792 - lats[349] = 65.413002884961315 - lats[350] = 65.342704122911286 - lats[351] = 65.272405360820116 - lats[352] = 65.202106598688133 - lats[353] = 65.131807836515677 - lats[354] = 65.061509074303089 - lats[355] = 64.991210312050711 - lats[356] = 64.920911549758912 - lats[357] = 64.850612787427963 - lats[358] = 64.780314025058246 - lats[359] = 64.710015262650074 - lats[360] = 64.639716500203733 - lats[361] = 64.569417737719576 - lats[362] = 64.499118975197902 - lats[363] = 64.428820212639039 - lats[364] = 64.358521450043284 - lats[365] = 64.288222687410922 - lats[366] = 64.21792392474228 - lats[367] = 64.147625162037642 - lats[368] = 64.07732639929732 - lats[369] = 64.00702763652157 - lats[370] = 63.93672887371072 - lats[371] = 63.866430110865004 - lats[372] = 63.796131347984762 - lats[373] = 63.725832585070251 - lats[374] = 63.655533822121711 - lats[375] = 63.585235059139464 - lats[376] = 63.514936296123757 - lats[377] = 63.444637533074854 - lats[378] = 63.374338769993031 - lats[379] = 63.304040006878537 - lats[380] = 63.23374124373165 - lats[381] = 63.163442480552604 - lats[382] = 63.093143717341647 - lats[383] = 63.022844954099064 - lats[384] = 62.952546190825068 - lats[385] = 62.882247427519928 - lats[386] = 62.811948664183866 - lats[387] = 62.741649900817137 - lats[388] = 62.67135113741999 - lats[389] = 62.60105237399263 - lats[390] = 62.530753610535321 - lats[391] = 62.460454847048261 - lats[392] = 62.3901560835317 - lats[393] = 62.319857319985871 - lats[394] = 62.249558556410982 - lats[395] = 62.179259792807258 - lats[396] = 62.108961029174914 - lats[397] = 62.038662265514176 - lats[398] = 61.968363501825259 - lats[399] = 61.898064738108381 - lats[400] = 61.827765974363729 - lats[401] = 61.757467210591535 - lats[402] = 61.687168446791986 - lats[403] = 61.616869682965287 - lats[404] = 61.546570919111666 - lats[405] = 61.476272155231321 - lats[406] = 61.405973391324409 - lats[407] = 61.335674627391185 - lats[408] = 61.265375863431785 - lats[409] = 61.195077099446451 - lats[410] = 61.124778335435344 - lats[411] = 61.054479571398652 - lats[412] = 60.984180807336578 - lats[413] = 60.913882043249295 - lats[414] = 60.843583279137007 - lats[415] = 60.773284514999872 - lats[416] = 60.702985750838074 - lats[417] = 60.632686986651805 - lats[418] = 60.562388222441243 - lats[419] = 60.492089458206543 - lats[420] = 60.421790693947884 - lats[421] = 60.35149192966545 - lats[422] = 60.28119316535939 - lats[423] = 60.21089440102989 - lats[424] = 60.140595636677112 - lats[425] = 60.070296872301235 - lats[426] = 59.999998107902378 - lats[427] = 59.929699343480763 - lats[428] = 59.859400579036503 - lats[429] = 59.78910181456979 - lats[430] = 59.718803050080759 - lats[431] = 59.64850428556958 - lats[432] = 59.578205521036402 - lats[433] = 59.507906756481383 - lats[434] = 59.43760799190467 - lats[435] = 59.3673092273064 - lats[436] = 59.29701046268675 - lats[437] = 59.226711698045854 - lats[438] = 59.156412933383855 - lats[439] = 59.086114168700909 - lats[440] = 59.015815403997145 - lats[441] = 58.945516639272725 - lats[442] = 58.875217874527763 - lats[443] = 58.804919109762423 - lats[444] = 58.73462034497684 - lats[445] = 58.664321580171141 - lats[446] = 58.594022815345468 - lats[447] = 58.523724050499972 - lats[448] = 58.453425285634758 - lats[449] = 58.383126520749968 - lats[450] = 58.312827755845746 - lats[451] = 58.242528990922203 - lats[452] = 58.172230225979497 - lats[453] = 58.101931461017728 - lats[454] = 58.031632696037022 - lats[455] = 57.961333931037537 - lats[456] = 57.891035166019364 - lats[457] = 57.820736400982646 - lats[458] = 57.75043763592749 - lats[459] = 57.680138870854037 - lats[460] = 57.60984010576238 - lats[461] = 57.539541340652676 - lats[462] = 57.469242575525016 - lats[463] = 57.398943810379521 - lats[464] = 57.328645045216312 - lats[465] = 57.258346280035504 - lats[466] = 57.188047514837208 - lats[467] = 57.117748749621541 - lats[468] = 57.047449984388614 - lats[469] = 56.977151219138541 - lats[470] = 56.90685245387143 - lats[471] = 56.836553688587379 - lats[472] = 56.766254923286517 - lats[473] = 56.695956157968951 - lats[474] = 56.625657392634771 - lats[475] = 56.555358627284086 - lats[476] = 56.485059861917016 - lats[477] = 56.41476109653366 - lats[478] = 56.34446233113411 - lats[479] = 56.274163565718467 - lats[480] = 56.203864800286865 - lats[481] = 56.133566034839362 - lats[482] = 56.063267269376091 - lats[483] = 55.992968503897131 - lats[484] = 55.922669738402583 - lats[485] = 55.852370972892551 - lats[486] = 55.782072207367136 - lats[487] = 55.711773441826416 - lats[488] = 55.641474676270505 - lats[489] = 55.571175910699488 - lats[490] = 55.500877145113449 - lats[491] = 55.430578379512511 - lats[492] = 55.360279613896743 - lats[493] = 55.289980848266232 - lats[494] = 55.219682082621084 - lats[495] = 55.149383316961377 - lats[496] = 55.07908455128721 - lats[497] = 55.008785785598668 - lats[498] = 54.938487019895831 - lats[499] = 54.868188254178797 - lats[500] = 54.797889488447652 - lats[501] = 54.727590722702473 - lats[502] = 54.657291956943347 - lats[503] = 54.586993191170357 - lats[504] = 54.516694425383605 - lats[505] = 54.446395659583146 - lats[506] = 54.376096893769081 - lats[507] = 54.305798127941479 - lats[508] = 54.235499362100448 - lats[509] = 54.165200596246031 - lats[510] = 54.094901830378333 - lats[511] = 54.024603064497434 - lats[512] = 53.954304298603383 - lats[513] = 53.884005532696307 - lats[514] = 53.813706766776235 - lats[515] = 53.743408000843282 - lats[516] = 53.673109234897495 - lats[517] = 53.602810468938962 - lats[518] = 53.53251170296776 - lats[519] = 53.462212936983953 - lats[520] = 53.391914170987633 - lats[521] = 53.321615404978871 - lats[522] = 53.251316638957725 - lats[523] = 53.181017872924265 - lats[524] = 53.110719106878584 - lats[525] = 53.040420340820731 - lats[526] = 52.970121574750792 - lats[527] = 52.899822808668837 - lats[528] = 52.829524042574917 - lats[529] = 52.759225276469131 - lats[530] = 52.688926510351514 - lats[531] = 52.618627744222159 - lats[532] = 52.548328978081123 - lats[533] = 52.478030211928477 - lats[534] = 52.407731445764284 - lats[535] = 52.337432679588609 - lats[536] = 52.26713391340153 - lats[537] = 52.196835147203096 - lats[538] = 52.126536380993372 - lats[539] = 52.056237614772435 - lats[540] = 51.985938848540336 - lats[541] = 51.915640082297152 - lats[542] = 51.845341316042933 - lats[543] = 51.775042549777737 - lats[544] = 51.704743783501634 - lats[545] = 51.634445017214695 - lats[546] = 51.56414625091697 - lats[547] = 51.493847484608516 - lats[548] = 51.423548718289396 - lats[549] = 51.353249951959683 - lats[550] = 51.282951185619417 - lats[551] = 51.21265241926865 - lats[552] = 51.14235365290746 - lats[553] = 51.072054886535909 - lats[554] = 51.001756120154049 - lats[555] = 50.931457353761914 - lats[556] = 50.86115858735959 - lats[557] = 50.790859820947119 - lats[558] = 50.720561054524559 - lats[559] = 50.650262288091959 - lats[560] = 50.579963521649397 - lats[561] = 50.509664755196901 - lats[562] = 50.439365988734544 - lats[563] = 50.369067222262359 - lats[564] = 50.298768455780426 - lats[565] = 50.228469689288779 - lats[566] = 50.158170922787484 - lats[567] = 50.087872156276575 - lats[568] = 50.017573389756123 - lats[569] = 49.947274623226157 - lats[570] = 49.876975856686762 - lats[571] = 49.80667709013796 - lats[572] = 49.736378323579807 - lats[573] = 49.66607955701236 - lats[574] = 49.595780790435676 - lats[575] = 49.525482023849783 - lats[576] = 49.455183257254745 - lats[577] = 49.384884490650613 - lats[578] = 49.314585724037435 - lats[579] = 49.244286957415234 - lats[580] = 49.173988190784094 - lats[581] = 49.103689424144044 - lats[582] = 49.03339065749514 - lats[583] = 48.963091890837418 - lats[584] = 48.892793124170929 - lats[585] = 48.822494357495721 - lats[586] = 48.752195590811837 - lats[587] = 48.681896824119335 - lats[588] = 48.611598057418242 - lats[589] = 48.541299290708608 - lats[590] = 48.47100052399049 - lats[591] = 48.400701757263917 - lats[592] = 48.330402990528938 - lats[593] = 48.260104223785596 - lats[594] = 48.189805457033941 - lats[595] = 48.119506690274015 - lats[596] = 48.049207923505868 - lats[597] = 47.978909156729507 - lats[598] = 47.908610389945018 - lats[599] = 47.838311623152421 - lats[600] = 47.76801285635176 - lats[601] = 47.697714089543084 - lats[602] = 47.627415322726435 - lats[603] = 47.557116555901828 - lats[604] = 47.486817789069342 - lats[605] = 47.416519022228997 - lats[606] = 47.346220255380835 - lats[607] = 47.275921488524894 - lats[608] = 47.205622721661214 - lats[609] = 47.13532395478984 - lats[610] = 47.065025187910805 - lats[611] = 46.994726421024154 - lats[612] = 46.924427654129929 - lats[613] = 46.85412888722815 - lats[614] = 46.783830120318882 - lats[615] = 46.713531353402139 - lats[616] = 46.643232586477971 - lats[617] = 46.572933819546414 - lats[618] = 46.502635052607502 - lats[619] = 46.432336285661272 - lats[620] = 46.362037518707766 - lats[621] = 46.291738751747012 - lats[622] = 46.221439984779053 - lats[623] = 46.151141217803925 - lats[624] = 46.080842450821663 - lats[625] = 46.01054368383231 - lats[626] = 45.94024491683588 - lats[627] = 45.869946149832437 - lats[628] = 45.799647382821995 - lats[629] = 45.729348615804589 - lats[630] = 45.659049848780263 - lats[631] = 45.588751081749038 - lats[632] = 45.51845231471097 - lats[633] = 45.448153547666081 - lats[634] = 45.377854780614399 - lats[635] = 45.30755601355596 - lats[636] = 45.237257246490813 - lats[637] = 45.166958479418959 - lats[638] = 45.096659712340461 - lats[639] = 45.026360945255341 - lats[640] = 44.956062178163634 - lats[641] = 44.885763411065362 - lats[642] = 44.81546464396056 - lats[643] = 44.745165876849271 - lats[644] = 44.674867109731515 - lats[645] = 44.604568342607337 - lats[646] = 44.534269575476756 - lats[647] = 44.463970808339802 - lats[648] = 44.39367204119651 - lats[649] = 44.323373274046915 - lats[650] = 44.253074506891046 - lats[651] = 44.182775739728925 - lats[652] = 44.112476972560586 - lats[653] = 44.042178205386072 - lats[654] = 43.971879438205391 - lats[655] = 43.9015806710186 - lats[656] = 43.831281903825705 - lats[657] = 43.760983136626741 - lats[658] = 43.690684369421732 - lats[659] = 43.620385602210717 - lats[660] = 43.550086834993728 - lats[661] = 43.479788067770777 - lats[662] = 43.409489300541907 - lats[663] = 43.339190533307139 - lats[664] = 43.26889176606651 - lats[665] = 43.19859299882004 - lats[666] = 43.128294231567757 - lats[667] = 43.057995464309691 - lats[668] = 42.987696697045862 - lats[669] = 42.917397929776307 - lats[670] = 42.847099162501053 - lats[671] = 42.776800395220121 - lats[672] = 42.706501627933541 - lats[673] = 42.63620286064134 - lats[674] = 42.565904093343548 - lats[675] = 42.495605326040177 - lats[676] = 42.425306558731272 - lats[677] = 42.355007791416853 - lats[678] = 42.284709024096927 - lats[679] = 42.214410256771551 - lats[680] = 42.144111489440725 - lats[681] = 42.073812722104492 - lats[682] = 42.003513954762873 - lats[683] = 41.933215187415882 - lats[684] = 41.862916420063563 - lats[685] = 41.792617652705921 - lats[686] = 41.722318885343 - lats[687] = 41.6520201179748 - lats[688] = 41.581721350601363 - lats[689] = 41.511422583222718 - lats[690] = 41.441123815838885 - lats[691] = 41.370825048449873 - lats[692] = 41.300526281055724 - lats[693] = 41.230227513656445 - lats[694] = 41.159928746252085 - lats[695] = 41.089629978842645 - lats[696] = 41.01933121142816 - lats[697] = 40.949032444008644 - lats[698] = 40.878733676584126 - lats[699] = 40.808434909154634 - lats[700] = 40.738136141720176 - lats[701] = 40.667837374280786 - lats[702] = 40.597538606836487 - lats[703] = 40.527239839387299 - lats[704] = 40.456941071933244 - lats[705] = 40.386642304474343 - lats[706] = 40.316343537010617 - lats[707] = 40.246044769542102 - lats[708] = 40.175746002068806 - lats[709] = 40.105447234590748 - lats[710] = 40.035148467107952 - lats[711] = 39.964849699620437 - lats[712] = 39.894550932128247 - lats[713] = 39.824252164631375 - lats[714] = 39.753953397129855 - lats[715] = 39.683654629623703 - lats[716] = 39.613355862112947 - lats[717] = 39.543057094597607 - lats[718] = 39.472758327077692 - lats[719] = 39.402459559553229 - lats[720] = 39.332160792024254 - lats[721] = 39.261862024490775 - lats[722] = 39.191563256952804 - lats[723] = 39.121264489410365 - lats[724] = 39.050965721863491 - lats[725] = 38.980666954312184 - lats[726] = 38.910368186756479 - lats[727] = 38.840069419196389 - lats[728] = 38.769770651631937 - lats[729] = 38.699471884063136 - lats[730] = 38.629173116490001 - lats[731] = 38.558874348912568 - lats[732] = 38.488575581330842 - lats[733] = 38.418276813744846 - lats[734] = 38.347978046154608 - lats[735] = 38.277679278560143 - lats[736] = 38.20738051096145 - lats[737] = 38.137081743358586 - lats[738] = 38.066782975751536 - lats[739] = 37.99648420814033 - lats[740] = 37.926185440524989 - lats[741] = 37.855886672905527 - lats[742] = 37.785587905281965 - lats[743] = 37.715289137654317 - lats[744] = 37.644990370022605 - lats[745] = 37.574691602386856 - lats[746] = 37.504392834747065 - lats[747] = 37.434094067103274 - lats[748] = 37.363795299455489 - lats[749] = 37.293496531803719 - lats[750] = 37.223197764147997 - lats[751] = 37.152898996488332 - lats[752] = 37.082600228824752 - lats[753] = 37.012301461157264 - lats[754] = 36.942002693485883 - lats[755] = 36.871703925810628 - lats[756] = 36.801405158131523 - lats[757] = 36.731106390448581 - lats[758] = 36.660807622761808 - lats[759] = 36.590508855071242 - lats[760] = 36.520210087376888 - lats[761] = 36.449911319678755 - lats[762] = 36.379612551976876 - lats[763] = 36.309313784271254 - lats[764] = 36.239015016561908 - lats[765] = 36.16871624884886 - lats[766] = 36.098417481132117 - lats[767] = 36.028118713411708 - lats[768] = 35.957819945687639 - lats[769] = 35.887521177959933 - lats[770] = 35.817222410228595 - lats[771] = 35.746923642493655 - lats[772] = 35.676624874755113 - lats[773] = 35.606326107012997 - lats[774] = 35.536027339267314 - lats[775] = 35.465728571518085 - lats[776] = 35.395429803765317 - lats[777] = 35.325131036009047 - lats[778] = 35.254832268249267 - lats[779] = 35.184533500486005 - lats[780] = 35.114234732719261 - lats[781] = 35.043935964949064 - lats[782] = 34.973637197175435 - lats[783] = 34.903338429398374 - lats[784] = 34.833039661617903 - lats[785] = 34.762740893834028 - lats[786] = 34.692442126046771 - lats[787] = 34.622143358256153 - lats[788] = 34.551844590462188 - lats[789] = 34.481545822664863 - lats[790] = 34.411247054864234 - lats[791] = 34.340948287060286 - lats[792] = 34.270649519253041 - lats[793] = 34.200350751442521 - lats[794] = 34.130051983628725 - lats[795] = 34.059753215811682 - lats[796] = 33.989454447991392 - lats[797] = 33.919155680167876 - lats[798] = 33.848856912341155 - lats[799] = 33.778558144511237 - lats[800] = 33.708259376678136 - lats[801] = 33.637960608841851 - lats[802] = 33.567661841002426 - lats[803] = 33.497363073159853 - lats[804] = 33.42706430531414 - lats[805] = 33.356765537465314 - lats[806] = 33.286466769613391 - lats[807] = 33.216168001758369 - lats[808] = 33.145869233900278 - lats[809] = 33.075570466039117 - lats[810] = 33.005271698174909 - lats[811] = 32.934972930307666 - lats[812] = 32.864674162437396 - lats[813] = 32.794375394564113 - lats[814] = 32.724076626687825 - lats[815] = 32.653777858808567 - lats[816] = 32.583479090926325 - lats[817] = 32.513180323041112 - lats[818] = 32.442881555152965 - lats[819] = 32.372582787261891 - lats[820] = 32.302284019367875 - lats[821] = 32.231985251470959 - lats[822] = 32.161686483571145 - lats[823] = 32.091387715668439 - lats[824] = 32.021088947762863 - lats[825] = 31.950790179854422 - lats[826] = 31.880491411943137 - lats[827] = 31.810192644029012 - lats[828] = 31.739893876112063 - lats[829] = 31.669595108192297 - lats[830] = 31.599296340269738 - lats[831] = 31.528997572344384 - lats[832] = 31.458698804416255 - lats[833] = 31.388400036485361 - lats[834] = 31.318101268551715 - lats[835] = 31.247802500615318 - lats[836] = 31.177503732676204 - lats[837] = 31.107204964734358 - lats[838] = 31.036906196789811 - lats[839] = 30.966607428842572 - lats[840] = 30.896308660892647 - lats[841] = 30.826009892940046 - lats[842] = 30.755711124984781 - lats[843] = 30.685412357026873 - lats[844] = 30.615113589066322 - lats[845] = 30.544814821103138 - lats[846] = 30.47451605313735 - lats[847] = 30.404217285168947 - lats[848] = 30.333918517197947 - lats[849] = 30.263619749224372 - lats[850] = 30.19332098124822 - lats[851] = 30.123022213269511 - lats[852] = 30.052723445288244 - lats[853] = 29.98242467730444 - lats[854] = 29.91212590931811 - lats[855] = 29.841827141329258 - lats[856] = 29.771528373337894 - lats[857] = 29.701229605344039 - lats[858] = 29.630930837347698 - lats[859] = 29.560632069348884 - lats[860] = 29.490333301347597 - lats[861] = 29.420034533343859 - lats[862] = 29.349735765337677 - lats[863] = 29.279436997329057 - lats[864] = 29.209138229318015 - lats[865] = 29.138839461304556 - lats[866] = 29.068540693288696 - lats[867] = 28.998241925270449 - lats[868] = 28.927943157249814 - lats[869] = 28.857644389226806 - lats[870] = 28.787345621201432 - lats[871] = 28.717046853173709 - lats[872] = 28.646748085143642 - lats[873] = 28.576449317111244 - lats[874] = 28.506150549076519 - lats[875] = 28.435851781039485 - lats[876] = 28.365553013000145 - lats[877] = 28.29525424495851 - lats[878] = 28.224955476914594 - lats[879] = 28.154656708868405 - lats[880] = 28.084357940819952 - lats[881] = 28.014059172769244 - lats[882] = 27.94376040471629 - lats[883] = 27.873461636661098 - lats[884] = 27.803162868603682 - lats[885] = 27.732864100544052 - lats[886] = 27.662565332482213 - lats[887] = 27.592266564418171 - lats[888] = 27.521967796351948 - lats[889] = 27.451669028283543 - lats[890] = 27.381370260212968 - lats[891] = 27.311071492140236 - lats[892] = 27.240772724065348 - lats[893] = 27.170473955988321 - lats[894] = 27.100175187909159 - lats[895] = 27.029876419827872 - lats[896] = 26.959577651744471 - lats[897] = 26.889278883658971 - lats[898] = 26.818980115571364 - lats[899] = 26.748681347481678 - lats[900] = 26.678382579389908 - lats[901] = 26.608083811296069 - lats[902] = 26.53778504320017 - lats[903] = 26.467486275102218 - lats[904] = 26.397187507002222 - lats[905] = 26.326888738900195 - lats[906] = 26.256589970796135 - lats[907] = 26.186291202690064 - lats[908] = 26.115992434581983 - lats[909] = 26.045693666471902 - lats[910] = 25.975394898359827 - lats[911] = 25.90509613024577 - lats[912] = 25.834797362129745 - lats[913] = 25.764498594011751 - lats[914] = 25.694199825891793 - lats[915] = 25.623901057769892 - lats[916] = 25.553602289646051 - lats[917] = 25.483303521520277 - lats[918] = 25.413004753392578 - lats[919] = 25.342705985262967 - lats[920] = 25.272407217131445 - lats[921] = 25.202108448998025 - lats[922] = 25.13180968086272 - lats[923] = 25.061510912725527 - lats[924] = 24.991212144586456 - lats[925] = 24.920913376445526 - lats[926] = 24.850614608302738 - lats[927] = 24.780315840158096 - lats[928] = 24.710017072011613 - lats[929] = 24.639718303863294 - lats[930] = 24.569419535713152 - lats[931] = 24.499120767561195 - lats[932] = 24.428821999407425 - lats[933] = 24.358523231251851 - lats[934] = 24.288224463094483 - lats[935] = 24.217925694935328 - lats[936] = 24.1476269267744 - lats[937] = 24.077328158611696 - lats[938] = 24.007029390447226 - lats[939] = 23.936730622281004 - lats[940] = 23.866431854113038 - lats[941] = 23.796133085943328 - lats[942] = 23.725834317771888 - lats[943] = 23.655535549598721 - lats[944] = 23.585236781423838 - lats[945] = 23.514938013247242 - lats[946] = 23.444639245068949 - lats[947] = 23.374340476888957 - lats[948] = 23.304041708707278 - lats[949] = 23.233742940523921 - lats[950] = 23.163444172338895 - lats[951] = 23.0931454041522 - lats[952] = 23.022846635963852 - lats[953] = 22.952547867773848 - lats[954] = 22.882249099582204 - lats[955] = 22.811950331388925 - lats[956] = 22.741651563194019 - lats[957] = 22.671352794997489 - lats[958] = 22.60105402679935 - lats[959] = 22.530755258599601 - lats[960] = 22.460456490398254 - lats[961] = 22.390157722195315 - lats[962] = 22.319858953990789 - lats[963] = 22.249560185784691 - lats[964] = 22.179261417577013 - lats[965] = 22.108962649367779 - lats[966] = 22.038663881156989 - lats[967] = 21.968365112944642 - lats[968] = 21.898066344730758 - lats[969] = 21.827767576515338 - lats[970] = 21.757468808298391 - lats[971] = 21.687170040079913 - lats[972] = 21.616871271859928 - lats[973] = 21.546572503638437 - lats[974] = 21.47627373541544 - lats[975] = 21.40597496719095 - lats[976] = 21.335676198964972 - lats[977] = 21.265377430737512 - lats[978] = 21.195078662508585 - lats[979] = 21.124779894278181 - lats[980] = 21.054481126046323 - lats[981] = 20.984182357813012 - lats[982] = 20.913883589578251 - lats[983] = 20.843584821342048 - lats[984] = 20.773286053104417 - lats[985] = 20.702987284865355 - lats[986] = 20.632688516624874 - lats[987] = 20.562389748382977 - lats[988] = 20.492090980139672 - lats[989] = 20.421792211894967 - lats[990] = 20.35149344364887 - lats[991] = 20.28119467540138 - lats[992] = 20.210895907152516 - lats[993] = 20.140597138902272 - lats[994] = 20.070298370650661 - lats[995] = 19.999999602397686 - lats[996] = 19.929700834143357 - lats[997] = 19.859402065887682 - lats[998] = 19.789103297630657 - lats[999] = 19.718804529372303 - lats[1000] = 19.648505761112613 - lats[1001] = 19.578206992851602 - lats[1002] = 19.507908224589269 - lats[1003] = 19.437609456325632 - lats[1004] = 19.367310688060684 - lats[1005] = 19.297011919794439 - lats[1006] = 19.226713151526898 - lats[1007] = 19.15641438325807 - lats[1008] = 19.086115614987968 - lats[1009] = 19.015816846716586 - lats[1010] = 18.945518078443939 - lats[1011] = 18.875219310170031 - lats[1012] = 18.804920541894862 - lats[1013] = 18.734621773618446 - lats[1014] = 18.664323005340787 - lats[1015] = 18.594024237061891 - lats[1016] = 18.523725468781763 - lats[1017] = 18.453426700500408 - lats[1018] = 18.383127932217832 - lats[1019] = 18.312829163934047 - lats[1020] = 18.242530395649048 - lats[1021] = 18.172231627362851 - lats[1022] = 18.101932859075458 - lats[1023] = 18.031634090786874 - lats[1024] = 17.96133532249711 - lats[1025] = 17.89103655420616 - lats[1026] = 17.820737785914044 - lats[1027] = 17.75043901762076 - lats[1028] = 17.680140249326314 - lats[1029] = 17.60984148103071 - lats[1030] = 17.539542712733962 - lats[1031] = 17.469243944436066 - lats[1032] = 17.39894517613704 - lats[1033] = 17.328646407836878 - lats[1034] = 17.258347639535586 - lats[1035] = 17.188048871233182 - lats[1036] = 17.117750102929655 - lats[1037] = 17.04745133462502 - lats[1038] = 16.977152566319283 - lats[1039] = 16.906853798012452 - lats[1040] = 16.836555029704527 - lats[1041] = 16.766256261395515 - lats[1042] = 16.69595749308542 - lats[1043] = 16.625658724774254 - lats[1044] = 16.555359956462013 - lats[1045] = 16.485061188148713 - lats[1046] = 16.41476241983435 - lats[1047] = 16.344463651518936 - lats[1048] = 16.274164883202477 - lats[1049] = 16.203866114884974 - lats[1050] = 16.133567346566434 - lats[1051] = 16.063268578246863 - lats[1052] = 15.992969809926265 - lats[1053] = 15.922671041604652 - lats[1054] = 15.852372273282016 - lats[1055] = 15.78207350495838 - lats[1056] = 15.711774736633735 - lats[1057] = 15.641475968308091 - lats[1058] = 15.571177199981456 - lats[1059] = 15.500878431653829 - lats[1060] = 15.430579663325226 - lats[1061] = 15.360280894995643 - lats[1062] = 15.289982126665089 - lats[1063] = 15.219683358333569 - lats[1064] = 15.149384590001089 - lats[1065] = 15.07908582166765 - lats[1066] = 15.008787053333259 - lats[1067] = 14.938488284997929 - lats[1068] = 14.868189516661655 - lats[1069] = 14.797890748324447 - lats[1070] = 14.727591979986309 - lats[1071] = 14.657293211647247 - lats[1072] = 14.586994443307265 - lats[1073] = 14.516695674966371 - lats[1074] = 14.446396906624567 - lats[1075] = 14.376098138281863 - lats[1076] = 14.305799369938256 - lats[1077] = 14.23550060159376 - lats[1078] = 14.165201833248371 - lats[1079] = 14.0949030649021 - lats[1080] = 14.024604296554955 - lats[1081] = 13.954305528206934 - lats[1082] = 13.884006759858046 - lats[1083] = 13.813707991508297 - lats[1084] = 13.743409223157688 - lats[1085] = 13.673110454806226 - lats[1086] = 13.602811686453919 - lats[1087] = 13.532512918100766 - lats[1088] = 13.46221414974678 - lats[1089] = 13.391915381391959 - lats[1090] = 13.32161661303631 - lats[1091] = 13.251317844679837 - lats[1092] = 13.181019076322551 - lats[1093] = 13.110720307964451 - lats[1094] = 13.040421539605545 - lats[1095] = 12.970122771245832 - lats[1096] = 12.899824002885323 - lats[1097] = 12.829525234524022 - lats[1098] = 12.759226466161934 - lats[1099] = 12.688927697799061 - lats[1100] = 12.618628929435411 - lats[1101] = 12.548330161070988 - lats[1102] = 12.478031392705796 - lats[1103] = 12.407732624339841 - lats[1104] = 12.337433855973126 - lats[1105] = 12.267135087605659 - lats[1106] = 12.196836319237443 - lats[1107] = 12.126537550868482 - lats[1108] = 12.056238782498781 - lats[1109] = 11.985940014128348 - lats[1110] = 11.915641245757183 - lats[1111] = 11.845342477385294 - lats[1112] = 11.775043709012685 - lats[1113] = 11.704744940639358 - lats[1114] = 11.634446172265324 - lats[1115] = 11.564147403890583 - lats[1116] = 11.493848635515141 - lats[1117] = 11.423549867139002 - lats[1118] = 11.35325109876217 - lats[1119] = 11.282952330384653 - lats[1120] = 11.212653562006453 - lats[1121] = 11.142354793627575 - lats[1122] = 11.072056025248026 - lats[1123] = 11.001757256867807 - lats[1124] = 10.931458488486923 - lats[1125] = 10.861159720105382 - lats[1126] = 10.790860951723188 - lats[1127] = 10.720562183340341 - lats[1128] = 10.65026341495685 - lats[1129] = 10.579964646572719 - lats[1130] = 10.509665878187954 - lats[1131] = 10.439367109802557 - lats[1132] = 10.369068341416533 - lats[1133] = 10.298769573029887 - lats[1134] = 10.228470804642624 - lats[1135] = 10.158172036254747 - lats[1136] = 10.087873267866264 - lats[1137] = 10.017574499477174 - lats[1138] = 9.9472757310874869 - lats[1139] = 9.8769769626972046 - lats[1140] = 9.8066781943063344 - lats[1141] = 9.7363794259148779 - lats[1142] = 9.6660806575228388 - lats[1143] = 9.5957818891302242 - lats[1144] = 9.5254831207370376 - lats[1145] = 9.4551843523432826 - lats[1146] = 9.3848855839489662 - lats[1147] = 9.3145868155540921 - lats[1148] = 9.2442880471586619 - lats[1149] = 9.1739892787626829 - lats[1150] = 9.1036905103661585 - lats[1151] = 9.0333917419690941 - lats[1152] = 8.963092973571495 - lats[1153] = 8.8927942051733631 - lats[1154] = 8.8224954367747017 - lats[1155] = 8.7521966683755217 - lats[1156] = 8.6818978999758194 - lats[1157] = 8.6115991315756055 - lats[1158] = 8.5413003631748801 - lats[1159] = 8.4710015947736537 - lats[1160] = 8.4007028263719228 - lats[1161] = 8.3304040579696963 - lats[1162] = 8.2601052895669778 - lats[1163] = 8.1898065211637725 - lats[1164] = 8.1195077527600841 - lats[1165] = 8.049208984355916 - lats[1166] = 7.9789102159512737 - lats[1167] = 7.9086114475461606 - lats[1168] = 7.8383126791405831 - lats[1169] = 7.7680139107345463 - lats[1170] = 7.6977151423280494 - lats[1171] = 7.6274163739210996 - lats[1172] = 7.557117605513703 - lats[1173] = 7.4868188371058624 - lats[1174] = 7.4165200686975803 - lats[1175] = 7.3462213002888648 - lats[1176] = 7.2759225318797176 - lats[1177] = 7.2056237634701441 - lats[1178] = 7.1353249950601469 - lats[1179] = 7.0650262266497315 - lats[1180] = 6.994727458238903 - lats[1181] = 6.924428689827665 - lats[1182] = 6.8541299214160212 - lats[1183] = 6.7838311530039768 - lats[1184] = 6.7135323845915353 - lats[1185] = 6.6432336161787013 - lats[1186] = 6.5729348477654792 - lats[1187] = 6.5026360793518734 - lats[1188] = 6.4323373109378874 - lats[1189] = 6.3620385425235257 - lats[1190] = 6.2917397741087928 - lats[1191] = 6.2214410056936931 - lats[1192] = 6.151142237278231 - lats[1193] = 6.0808434688624091 - lats[1194] = 6.0105447004462347 - lats[1195] = 5.9402459320297085 - lats[1196] = 5.869947163612836 - lats[1197] = 5.7996483951956233 - lats[1198] = 5.729349626778073 - lats[1199] = 5.6590508583601888 - lats[1200] = 5.5887520899419751 - lats[1201] = 5.5184533215234373 - lats[1202] = 5.4481545531045787 - lats[1203] = 5.3778557846854023 - lats[1204] = 5.3075570162659149 - lats[1205] = 5.2372582478461194 - lats[1206] = 5.1669594794260192 - lats[1207] = 5.0966607110056197 - lats[1208] = 5.0263619425849244 - lats[1209] = 4.9560631741639369 - lats[1210] = 4.8857644057426626 - lats[1211] = 4.8154656373211049 - lats[1212] = 4.7451668688992683 - lats[1213] = 4.6748681004771564 - lats[1214] = 4.6045693320547736 - lats[1215] = 4.5342705636321252 - lats[1216] = 4.4639717952092139 - lats[1217] = 4.3936730267860451 - lats[1218] = 4.3233742583626205 - lats[1219] = 4.2530754899389471 - lats[1220] = 4.1827767215150269 - lats[1221] = 4.1124779530908659 - lats[1222] = 4.0421791846664661 - lats[1223] = 3.9718804162418326 - lats[1224] = 3.90158164781697 - lats[1225] = 3.8312828793918823 - lats[1226] = 3.7609841109665734 - lats[1227] = 3.6906853425410477 - lats[1228] = 3.6203865741153085 - lats[1229] = 3.5500878056893601 - lats[1230] = 3.4797890372632065 - lats[1231] = 3.4094902688368531 - lats[1232] = 3.339191500410303 - lats[1233] = 3.2688927319835597 - lats[1234] = 3.1985939635566285 - lats[1235] = 3.1282951951295126 - lats[1236] = 3.0579964267022164 - lats[1237] = 2.9876976582747439 - lats[1238] = 2.9173988898470999 - lats[1239] = 2.8471001214192873 - lats[1240] = 2.7768013529913107 - lats[1241] = 2.7065025845631743 - lats[1242] = 2.6362038161348824 - lats[1243] = 2.5659050477064382 - lats[1244] = 2.4956062792778466 - lats[1245] = 2.4253075108491116 - lats[1246] = 2.3550087424202366 - lats[1247] = 2.2847099739912267 - lats[1248] = 2.2144112055620848 - lats[1249] = 2.1441124371328155 - lats[1250] = 2.0738136687034232 - lats[1251] = 2.0035149002739114 - lats[1252] = 1.9332161318442849 - lats[1253] = 1.8629173634145471 - lats[1254] = 1.792618594984702 - lats[1255] = 1.7223198265547539 - lats[1256] = 1.6520210581247066 - lats[1257] = 1.5817222896945646 - lats[1258] = 1.5114235212643317 - lats[1259] = 1.4411247528340119 - lats[1260] = 1.3708259844036093 - lats[1261] = 1.300527215973128 - lats[1262] = 1.2302284475425722 - lats[1263] = 1.1599296791119456 - lats[1264] = 1.0896309106812523 - lats[1265] = 1.0193321422504964 - lats[1266] = 0.949033373819682 - lats[1267] = 0.87873460538881287 - lats[1268] = 0.80843583695789356 - lats[1269] = 0.73813706852692773 - lats[1270] = 0.66783830009591949 - lats[1271] = 0.59753953166487306 - lats[1272] = 0.52724076323379232 - lats[1273] = 0.45694199480268116 - lats[1274] = 0.3866432263715438 - lats[1275] = 0.31634445794038429 - lats[1276] = 0.24604568950920663 - lats[1277] = 0.17574692107801482 - lats[1278] = 0.10544815264681295 - lats[1279] = 0.035149384215604956 - lats[1280] = -0.035149384215604956 - lats[1281] = -0.10544815264681295 - lats[1282] = -0.17574692107801482 - lats[1283] = -0.24604568950920663 - lats[1284] = -0.31634445794038429 - lats[1285] = -0.3866432263715438 - lats[1286] = -0.45694199480268116 - lats[1287] = -0.52724076323379232 - lats[1288] = -0.59753953166487306 - lats[1289] = -0.66783830009591949 - lats[1290] = -0.73813706852692773 - lats[1291] = -0.80843583695789356 - lats[1292] = -0.87873460538881287 - lats[1293] = -0.949033373819682 - lats[1294] = -1.0193321422504964 - lats[1295] = -1.0896309106812523 - lats[1296] = -1.1599296791119456 - lats[1297] = -1.2302284475425722 - lats[1298] = -1.300527215973128 - lats[1299] = -1.3708259844036093 - lats[1300] = -1.4411247528340119 - lats[1301] = -1.5114235212643317 - lats[1302] = -1.5817222896945646 - lats[1303] = -1.6520210581247066 - lats[1304] = -1.7223198265547539 - lats[1305] = -1.792618594984702 - lats[1306] = -1.8629173634145471 - lats[1307] = -1.9332161318442849 - lats[1308] = -2.0035149002739114 - lats[1309] = -2.0738136687034232 - lats[1310] = -2.1441124371328155 - lats[1311] = -2.2144112055620848 - lats[1312] = -2.2847099739912267 - lats[1313] = -2.3550087424202366 - lats[1314] = -2.4253075108491116 - lats[1315] = -2.4956062792778466 - lats[1316] = -2.5659050477064382 - lats[1317] = -2.6362038161348824 - lats[1318] = -2.7065025845631743 - lats[1319] = -2.7768013529913107 - lats[1320] = -2.8471001214192873 - lats[1321] = -2.9173988898470999 - lats[1322] = -2.9876976582747439 - lats[1323] = -3.0579964267022164 - lats[1324] = -3.1282951951295126 - lats[1325] = -3.1985939635566285 - lats[1326] = -3.2688927319835597 - lats[1327] = -3.339191500410303 - lats[1328] = -3.4094902688368531 - lats[1329] = -3.4797890372632065 - lats[1330] = -3.5500878056893601 - lats[1331] = -3.6203865741153085 - lats[1332] = -3.6906853425410477 - lats[1333] = -3.7609841109665734 - lats[1334] = -3.8312828793918823 - lats[1335] = -3.90158164781697 - lats[1336] = -3.9718804162418326 - lats[1337] = -4.0421791846664661 - lats[1338] = -4.1124779530908659 - lats[1339] = -4.1827767215150269 - lats[1340] = -4.2530754899389471 - lats[1341] = -4.3233742583626205 - lats[1342] = -4.3936730267860451 - lats[1343] = -4.4639717952092139 - lats[1344] = -4.5342705636321252 - lats[1345] = -4.6045693320547736 - lats[1346] = -4.6748681004771564 - lats[1347] = -4.7451668688992683 - lats[1348] = -4.8154656373211049 - lats[1349] = -4.8857644057426626 - lats[1350] = -4.9560631741639369 - lats[1351] = -5.0263619425849244 - lats[1352] = -5.0966607110056197 - lats[1353] = -5.1669594794260192 - lats[1354] = -5.2372582478461194 - lats[1355] = -5.3075570162659149 - lats[1356] = -5.3778557846854023 - lats[1357] = -5.4481545531045787 - lats[1358] = -5.5184533215234373 - lats[1359] = -5.5887520899419751 - lats[1360] = -5.6590508583601888 - lats[1361] = -5.729349626778073 - lats[1362] = -5.7996483951956233 - lats[1363] = -5.869947163612836 - lats[1364] = -5.9402459320297085 - lats[1365] = -6.0105447004462347 - lats[1366] = -6.0808434688624091 - lats[1367] = -6.151142237278231 - lats[1368] = -6.2214410056936931 - lats[1369] = -6.2917397741087928 - lats[1370] = -6.3620385425235257 - lats[1371] = -6.4323373109378874 - lats[1372] = -6.5026360793518734 - lats[1373] = -6.5729348477654792 - lats[1374] = -6.6432336161787013 - lats[1375] = -6.7135323845915353 - lats[1376] = -6.7838311530039768 - lats[1377] = -6.8541299214160212 - lats[1378] = -6.924428689827665 - lats[1379] = -6.994727458238903 - lats[1380] = -7.0650262266497315 - lats[1381] = -7.1353249950601469 - lats[1382] = -7.2056237634701441 - lats[1383] = -7.2759225318797176 - lats[1384] = -7.3462213002888648 - lats[1385] = -7.4165200686975803 - lats[1386] = -7.4868188371058624 - lats[1387] = -7.557117605513703 - lats[1388] = -7.6274163739210996 - lats[1389] = -7.6977151423280494 - lats[1390] = -7.7680139107345463 - lats[1391] = -7.8383126791405831 - lats[1392] = -7.9086114475461606 - lats[1393] = -7.9789102159512737 - lats[1394] = -8.049208984355916 - lats[1395] = -8.1195077527600841 - lats[1396] = -8.1898065211637725 - lats[1397] = -8.2601052895669778 - lats[1398] = -8.3304040579696963 - lats[1399] = -8.4007028263719228 - lats[1400] = -8.4710015947736537 - lats[1401] = -8.5413003631748801 - lats[1402] = -8.6115991315756055 - lats[1403] = -8.6818978999758194 - lats[1404] = -8.7521966683755217 - lats[1405] = -8.8224954367747017 - lats[1406] = -8.8927942051733631 - lats[1407] = -8.963092973571495 - lats[1408] = -9.0333917419690941 - lats[1409] = -9.1036905103661585 - lats[1410] = -9.1739892787626829 - lats[1411] = -9.2442880471586619 - lats[1412] = -9.3145868155540921 - lats[1413] = -9.3848855839489662 - lats[1414] = -9.4551843523432826 - lats[1415] = -9.5254831207370376 - lats[1416] = -9.5957818891302242 - lats[1417] = -9.6660806575228388 - lats[1418] = -9.7363794259148779 - lats[1419] = -9.8066781943063344 - lats[1420] = -9.8769769626972046 - lats[1421] = -9.9472757310874869 - lats[1422] = -10.017574499477174 - lats[1423] = -10.087873267866264 - lats[1424] = -10.158172036254747 - lats[1425] = -10.228470804642624 - lats[1426] = -10.298769573029887 - lats[1427] = -10.369068341416533 - lats[1428] = -10.439367109802557 - lats[1429] = -10.509665878187954 - lats[1430] = -10.579964646572719 - lats[1431] = -10.65026341495685 - lats[1432] = -10.720562183340341 - lats[1433] = -10.790860951723188 - lats[1434] = -10.861159720105382 - lats[1435] = -10.931458488486923 - lats[1436] = -11.001757256867807 - lats[1437] = -11.072056025248026 - lats[1438] = -11.142354793627575 - lats[1439] = -11.212653562006453 - lats[1440] = -11.282952330384653 - lats[1441] = -11.35325109876217 - lats[1442] = -11.423549867139002 - lats[1443] = -11.493848635515141 - lats[1444] = -11.564147403890583 - lats[1445] = -11.634446172265324 - lats[1446] = -11.704744940639358 - lats[1447] = -11.775043709012685 - lats[1448] = -11.845342477385294 - lats[1449] = -11.915641245757183 - lats[1450] = -11.985940014128348 - lats[1451] = -12.056238782498781 - lats[1452] = -12.126537550868482 - lats[1453] = -12.196836319237443 - lats[1454] = -12.267135087605659 - lats[1455] = -12.337433855973126 - lats[1456] = -12.407732624339841 - lats[1457] = -12.478031392705796 - lats[1458] = -12.548330161070988 - lats[1459] = -12.618628929435411 - lats[1460] = -12.688927697799061 - lats[1461] = -12.759226466161934 - lats[1462] = -12.829525234524022 - lats[1463] = -12.899824002885323 - lats[1464] = -12.970122771245832 - lats[1465] = -13.040421539605545 - lats[1466] = -13.110720307964451 - lats[1467] = -13.181019076322551 - lats[1468] = -13.251317844679837 - lats[1469] = -13.32161661303631 - lats[1470] = -13.391915381391959 - lats[1471] = -13.46221414974678 - lats[1472] = -13.532512918100766 - lats[1473] = -13.602811686453919 - lats[1474] = -13.673110454806226 - lats[1475] = -13.743409223157688 - lats[1476] = -13.813707991508297 - lats[1477] = -13.884006759858046 - lats[1478] = -13.954305528206934 - lats[1479] = -14.024604296554955 - lats[1480] = -14.0949030649021 - lats[1481] = -14.165201833248371 - lats[1482] = -14.23550060159376 - lats[1483] = -14.305799369938256 - lats[1484] = -14.376098138281863 - lats[1485] = -14.446396906624567 - lats[1486] = -14.516695674966371 - lats[1487] = -14.586994443307265 - lats[1488] = -14.657293211647247 - lats[1489] = -14.727591979986309 - lats[1490] = -14.797890748324447 - lats[1491] = -14.868189516661655 - lats[1492] = -14.938488284997929 - lats[1493] = -15.008787053333259 - lats[1494] = -15.07908582166765 - lats[1495] = -15.149384590001089 - lats[1496] = -15.219683358333569 - lats[1497] = -15.289982126665089 - lats[1498] = -15.360280894995643 - lats[1499] = -15.430579663325226 - lats[1500] = -15.500878431653829 - lats[1501] = -15.571177199981456 - lats[1502] = -15.641475968308091 - lats[1503] = -15.711774736633735 - lats[1504] = -15.78207350495838 - lats[1505] = -15.852372273282016 - lats[1506] = -15.922671041604652 - lats[1507] = -15.992969809926265 - lats[1508] = -16.063268578246863 - lats[1509] = -16.133567346566434 - lats[1510] = -16.203866114884974 - lats[1511] = -16.274164883202477 - lats[1512] = -16.344463651518936 - lats[1513] = -16.41476241983435 - lats[1514] = -16.485061188148713 - lats[1515] = -16.555359956462013 - lats[1516] = -16.625658724774254 - lats[1517] = -16.69595749308542 - lats[1518] = -16.766256261395515 - lats[1519] = -16.836555029704527 - lats[1520] = -16.906853798012452 - lats[1521] = -16.977152566319283 - lats[1522] = -17.04745133462502 - lats[1523] = -17.117750102929655 - lats[1524] = -17.188048871233182 - lats[1525] = -17.258347639535586 - lats[1526] = -17.328646407836878 - lats[1527] = -17.39894517613704 - lats[1528] = -17.469243944436066 - lats[1529] = -17.539542712733962 - lats[1530] = -17.60984148103071 - lats[1531] = -17.680140249326314 - lats[1532] = -17.75043901762076 - lats[1533] = -17.820737785914044 - lats[1534] = -17.89103655420616 - lats[1535] = -17.96133532249711 - lats[1536] = -18.031634090786874 - lats[1537] = -18.101932859075458 - lats[1538] = -18.172231627362851 - lats[1539] = -18.242530395649048 - lats[1540] = -18.312829163934047 - lats[1541] = -18.383127932217832 - lats[1542] = -18.453426700500408 - lats[1543] = -18.523725468781763 - lats[1544] = -18.594024237061891 - lats[1545] = -18.664323005340787 - lats[1546] = -18.734621773618446 - lats[1547] = -18.804920541894862 - lats[1548] = -18.875219310170031 - lats[1549] = -18.945518078443939 - lats[1550] = -19.015816846716586 - lats[1551] = -19.086115614987968 - lats[1552] = -19.15641438325807 - lats[1553] = -19.226713151526898 - lats[1554] = -19.297011919794439 - lats[1555] = -19.367310688060684 - lats[1556] = -19.437609456325632 - lats[1557] = -19.507908224589269 - lats[1558] = -19.578206992851602 - lats[1559] = -19.648505761112613 - lats[1560] = -19.718804529372303 - lats[1561] = -19.789103297630657 - lats[1562] = -19.859402065887682 - lats[1563] = -19.929700834143357 - lats[1564] = -19.999999602397686 - lats[1565] = -20.070298370650661 - lats[1566] = -20.140597138902272 - lats[1567] = -20.210895907152516 - lats[1568] = -20.28119467540138 - lats[1569] = -20.35149344364887 - lats[1570] = -20.421792211894967 - lats[1571] = -20.492090980139672 - lats[1572] = -20.562389748382977 - lats[1573] = -20.632688516624874 - lats[1574] = -20.702987284865355 - lats[1575] = -20.773286053104417 - lats[1576] = -20.843584821342048 - lats[1577] = -20.913883589578251 - lats[1578] = -20.984182357813012 - lats[1579] = -21.054481126046323 - lats[1580] = -21.124779894278181 - lats[1581] = -21.195078662508585 - lats[1582] = -21.265377430737512 - lats[1583] = -21.335676198964972 - lats[1584] = -21.40597496719095 - lats[1585] = -21.47627373541544 - lats[1586] = -21.546572503638437 - lats[1587] = -21.616871271859928 - lats[1588] = -21.687170040079913 - lats[1589] = -21.757468808298391 - lats[1590] = -21.827767576515338 - lats[1591] = -21.898066344730758 - lats[1592] = -21.968365112944642 - lats[1593] = -22.038663881156989 - lats[1594] = -22.108962649367779 - lats[1595] = -22.179261417577013 - lats[1596] = -22.249560185784691 - lats[1597] = -22.319858953990789 - lats[1598] = -22.390157722195315 - lats[1599] = -22.460456490398254 - lats[1600] = -22.530755258599601 - lats[1601] = -22.60105402679935 - lats[1602] = -22.671352794997489 - lats[1603] = -22.741651563194019 - lats[1604] = -22.811950331388925 - lats[1605] = -22.882249099582204 - lats[1606] = -22.952547867773848 - lats[1607] = -23.022846635963852 - lats[1608] = -23.0931454041522 - lats[1609] = -23.163444172338895 - lats[1610] = -23.233742940523921 - lats[1611] = -23.304041708707278 - lats[1612] = -23.374340476888957 - lats[1613] = -23.444639245068949 - lats[1614] = -23.514938013247242 - lats[1615] = -23.585236781423838 - lats[1616] = -23.655535549598721 - lats[1617] = -23.725834317771888 - lats[1618] = -23.796133085943328 - lats[1619] = -23.866431854113038 - lats[1620] = -23.936730622281004 - lats[1621] = -24.007029390447226 - lats[1622] = -24.077328158611696 - lats[1623] = -24.1476269267744 - lats[1624] = -24.217925694935328 - lats[1625] = -24.288224463094483 - lats[1626] = -24.358523231251851 - lats[1627] = -24.428821999407425 - lats[1628] = -24.499120767561195 - lats[1629] = -24.569419535713152 - lats[1630] = -24.639718303863294 - lats[1631] = -24.710017072011613 - lats[1632] = -24.780315840158096 - lats[1633] = -24.850614608302738 - lats[1634] = -24.920913376445526 - lats[1635] = -24.991212144586456 - lats[1636] = -25.061510912725527 - lats[1637] = -25.13180968086272 - lats[1638] = -25.202108448998025 - lats[1639] = -25.272407217131445 - lats[1640] = -25.342705985262967 - lats[1641] = -25.413004753392578 - lats[1642] = -25.483303521520277 - lats[1643] = -25.553602289646051 - lats[1644] = -25.623901057769892 - lats[1645] = -25.694199825891793 - lats[1646] = -25.764498594011751 - lats[1647] = -25.834797362129745 - lats[1648] = -25.90509613024577 - lats[1649] = -25.975394898359827 - lats[1650] = -26.045693666471902 - lats[1651] = -26.115992434581983 - lats[1652] = -26.186291202690064 - lats[1653] = -26.256589970796135 - lats[1654] = -26.326888738900195 - lats[1655] = -26.397187507002222 - lats[1656] = -26.467486275102218 - lats[1657] = -26.53778504320017 - lats[1658] = -26.608083811296069 - lats[1659] = -26.678382579389908 - lats[1660] = -26.748681347481678 - lats[1661] = -26.818980115571364 - lats[1662] = -26.889278883658971 - lats[1663] = -26.959577651744471 - lats[1664] = -27.029876419827872 - lats[1665] = -27.100175187909159 - lats[1666] = -27.170473955988321 - lats[1667] = -27.240772724065348 - lats[1668] = -27.311071492140236 - lats[1669] = -27.381370260212968 - lats[1670] = -27.451669028283543 - lats[1671] = -27.521967796351948 - lats[1672] = -27.592266564418171 - lats[1673] = -27.662565332482213 - lats[1674] = -27.732864100544052 - lats[1675] = -27.803162868603682 - lats[1676] = -27.873461636661098 - lats[1677] = -27.94376040471629 - lats[1678] = -28.014059172769244 - lats[1679] = -28.084357940819952 - lats[1680] = -28.154656708868405 - lats[1681] = -28.224955476914594 - lats[1682] = -28.29525424495851 - lats[1683] = -28.365553013000145 - lats[1684] = -28.435851781039485 - lats[1685] = -28.506150549076519 - lats[1686] = -28.576449317111244 - lats[1687] = -28.646748085143642 - lats[1688] = -28.717046853173709 - lats[1689] = -28.787345621201432 - lats[1690] = -28.857644389226806 - lats[1691] = -28.927943157249814 - lats[1692] = -28.998241925270449 - lats[1693] = -29.068540693288696 - lats[1694] = -29.138839461304556 - lats[1695] = -29.209138229318015 - lats[1696] = -29.279436997329057 - lats[1697] = -29.349735765337677 - lats[1698] = -29.420034533343859 - lats[1699] = -29.490333301347597 - lats[1700] = -29.560632069348884 - lats[1701] = -29.630930837347698 - lats[1702] = -29.701229605344039 - lats[1703] = -29.771528373337894 - lats[1704] = -29.841827141329258 - lats[1705] = -29.91212590931811 - lats[1706] = -29.98242467730444 - lats[1707] = -30.052723445288244 - lats[1708] = -30.123022213269511 - lats[1709] = -30.19332098124822 - lats[1710] = -30.263619749224372 - lats[1711] = -30.333918517197947 - lats[1712] = -30.404217285168947 - lats[1713] = -30.47451605313735 - lats[1714] = -30.544814821103138 - lats[1715] = -30.615113589066322 - lats[1716] = -30.685412357026873 - lats[1717] = -30.755711124984781 - lats[1718] = -30.826009892940046 - lats[1719] = -30.896308660892647 - lats[1720] = -30.966607428842572 - lats[1721] = -31.036906196789811 - lats[1722] = -31.107204964734358 - lats[1723] = -31.177503732676204 - lats[1724] = -31.247802500615318 - lats[1725] = -31.318101268551715 - lats[1726] = -31.388400036485361 - lats[1727] = -31.458698804416255 - lats[1728] = -31.528997572344384 - lats[1729] = -31.599296340269738 - lats[1730] = -31.669595108192297 - lats[1731] = -31.739893876112063 - lats[1732] = -31.810192644029012 - lats[1733] = -31.880491411943137 - lats[1734] = -31.950790179854422 - lats[1735] = -32.021088947762863 - lats[1736] = -32.091387715668439 - lats[1737] = -32.161686483571145 - lats[1738] = -32.231985251470959 - lats[1739] = -32.302284019367875 - lats[1740] = -32.372582787261891 - lats[1741] = -32.442881555152965 - lats[1742] = -32.513180323041112 - lats[1743] = -32.583479090926325 - lats[1744] = -32.653777858808567 - lats[1745] = -32.724076626687825 - lats[1746] = -32.794375394564113 - lats[1747] = -32.864674162437396 - lats[1748] = -32.934972930307666 - lats[1749] = -33.005271698174909 - lats[1750] = -33.075570466039117 - lats[1751] = -33.145869233900278 - lats[1752] = -33.216168001758369 - lats[1753] = -33.286466769613391 - lats[1754] = -33.356765537465314 - lats[1755] = -33.42706430531414 - lats[1756] = -33.497363073159853 - lats[1757] = -33.567661841002426 - lats[1758] = -33.637960608841851 - lats[1759] = -33.708259376678136 - lats[1760] = -33.778558144511237 - lats[1761] = -33.848856912341155 - lats[1762] = -33.919155680167876 - lats[1763] = -33.989454447991392 - lats[1764] = -34.059753215811682 - lats[1765] = -34.130051983628725 - lats[1766] = -34.200350751442521 - lats[1767] = -34.270649519253041 - lats[1768] = -34.340948287060286 - lats[1769] = -34.411247054864234 - lats[1770] = -34.481545822664863 - lats[1771] = -34.551844590462188 - lats[1772] = -34.622143358256153 - lats[1773] = -34.692442126046771 - lats[1774] = -34.762740893834028 - lats[1775] = -34.833039661617903 - lats[1776] = -34.903338429398374 - lats[1777] = -34.973637197175435 - lats[1778] = -35.043935964949064 - lats[1779] = -35.114234732719261 - lats[1780] = -35.184533500486005 - lats[1781] = -35.254832268249267 - lats[1782] = -35.325131036009047 - lats[1783] = -35.395429803765317 - lats[1784] = -35.465728571518085 - lats[1785] = -35.536027339267314 - lats[1786] = -35.606326107012997 - lats[1787] = -35.676624874755113 - lats[1788] = -35.746923642493655 - lats[1789] = -35.817222410228595 - lats[1790] = -35.887521177959933 - lats[1791] = -35.957819945687639 - lats[1792] = -36.028118713411708 - lats[1793] = -36.098417481132117 - lats[1794] = -36.16871624884886 - lats[1795] = -36.239015016561908 - lats[1796] = -36.309313784271254 - lats[1797] = -36.379612551976876 - lats[1798] = -36.449911319678755 - lats[1799] = -36.520210087376888 - lats[1800] = -36.590508855071242 - lats[1801] = -36.660807622761808 - lats[1802] = -36.731106390448581 - lats[1803] = -36.801405158131523 - lats[1804] = -36.871703925810628 - lats[1805] = -36.942002693485883 - lats[1806] = -37.012301461157264 - lats[1807] = -37.082600228824752 - lats[1808] = -37.152898996488332 - lats[1809] = -37.223197764147997 - lats[1810] = -37.293496531803719 - lats[1811] = -37.363795299455489 - lats[1812] = -37.434094067103274 - lats[1813] = -37.504392834747065 - lats[1814] = -37.574691602386856 - lats[1815] = -37.644990370022605 - lats[1816] = -37.715289137654317 - lats[1817] = -37.785587905281965 - lats[1818] = -37.855886672905527 - lats[1819] = -37.926185440524989 - lats[1820] = -37.99648420814033 - lats[1821] = -38.066782975751536 - lats[1822] = -38.137081743358586 - lats[1823] = -38.20738051096145 - lats[1824] = -38.277679278560143 - lats[1825] = -38.347978046154608 - lats[1826] = -38.418276813744846 - lats[1827] = -38.488575581330842 - lats[1828] = -38.558874348912568 - lats[1829] = -38.629173116490001 - lats[1830] = -38.699471884063136 - lats[1831] = -38.769770651631937 - lats[1832] = -38.840069419196389 - lats[1833] = -38.910368186756479 - lats[1834] = -38.980666954312184 - lats[1835] = -39.050965721863491 - lats[1836] = -39.121264489410365 - lats[1837] = -39.191563256952804 - lats[1838] = -39.261862024490775 - lats[1839] = -39.332160792024254 - lats[1840] = -39.402459559553229 - lats[1841] = -39.472758327077692 - lats[1842] = -39.543057094597607 - lats[1843] = -39.613355862112947 - lats[1844] = -39.683654629623703 - lats[1845] = -39.753953397129855 - lats[1846] = -39.824252164631375 - lats[1847] = -39.894550932128247 - lats[1848] = -39.964849699620437 - lats[1849] = -40.035148467107952 - lats[1850] = -40.105447234590748 - lats[1851] = -40.175746002068806 - lats[1852] = -40.246044769542102 - lats[1853] = -40.316343537010617 - lats[1854] = -40.386642304474343 - lats[1855] = -40.456941071933244 - lats[1856] = -40.527239839387299 - lats[1857] = -40.597538606836487 - lats[1858] = -40.667837374280786 - lats[1859] = -40.738136141720176 - lats[1860] = -40.808434909154634 - lats[1861] = -40.878733676584126 - lats[1862] = -40.949032444008644 - lats[1863] = -41.01933121142816 - lats[1864] = -41.089629978842645 - lats[1865] = -41.159928746252085 - lats[1866] = -41.230227513656445 - lats[1867] = -41.300526281055724 - lats[1868] = -41.370825048449873 - lats[1869] = -41.441123815838885 - lats[1870] = -41.511422583222718 - lats[1871] = -41.581721350601363 - lats[1872] = -41.6520201179748 - lats[1873] = -41.722318885343 - lats[1874] = -41.792617652705921 - lats[1875] = -41.862916420063563 - lats[1876] = -41.933215187415882 - lats[1877] = -42.003513954762873 - lats[1878] = -42.073812722104492 - lats[1879] = -42.144111489440725 - lats[1880] = -42.214410256771551 - lats[1881] = -42.284709024096927 - lats[1882] = -42.355007791416853 - lats[1883] = -42.425306558731272 - lats[1884] = -42.495605326040177 - lats[1885] = -42.565904093343548 - lats[1886] = -42.63620286064134 - lats[1887] = -42.706501627933541 - lats[1888] = -42.776800395220121 - lats[1889] = -42.847099162501053 - lats[1890] = -42.917397929776307 - lats[1891] = -42.987696697045862 - lats[1892] = -43.057995464309691 - lats[1893] = -43.128294231567757 - lats[1894] = -43.19859299882004 - lats[1895] = -43.26889176606651 - lats[1896] = -43.339190533307139 - lats[1897] = -43.409489300541907 - lats[1898] = -43.479788067770777 - lats[1899] = -43.550086834993728 - lats[1900] = -43.620385602210717 - lats[1901] = -43.690684369421732 - lats[1902] = -43.760983136626741 - lats[1903] = -43.831281903825705 - lats[1904] = -43.9015806710186 - lats[1905] = -43.971879438205391 - lats[1906] = -44.042178205386072 - lats[1907] = -44.112476972560586 - lats[1908] = -44.182775739728925 - lats[1909] = -44.253074506891046 - lats[1910] = -44.323373274046915 - lats[1911] = -44.39367204119651 - lats[1912] = -44.463970808339802 - lats[1913] = -44.534269575476756 - lats[1914] = -44.604568342607337 - lats[1915] = -44.674867109731515 - lats[1916] = -44.745165876849271 - lats[1917] = -44.81546464396056 - lats[1918] = -44.885763411065362 - lats[1919] = -44.956062178163634 - lats[1920] = -45.026360945255341 - lats[1921] = -45.096659712340461 - lats[1922] = -45.166958479418959 - lats[1923] = -45.237257246490813 - lats[1924] = -45.30755601355596 - lats[1925] = -45.377854780614399 - lats[1926] = -45.448153547666081 - lats[1927] = -45.51845231471097 - lats[1928] = -45.588751081749038 - lats[1929] = -45.659049848780263 - lats[1930] = -45.729348615804589 - lats[1931] = -45.799647382821995 - lats[1932] = -45.869946149832437 - lats[1933] = -45.94024491683588 - lats[1934] = -46.01054368383231 - lats[1935] = -46.080842450821663 - lats[1936] = -46.151141217803925 - lats[1937] = -46.221439984779053 - lats[1938] = -46.291738751747012 - lats[1939] = -46.362037518707766 - lats[1940] = -46.432336285661272 - lats[1941] = -46.502635052607502 - lats[1942] = -46.572933819546414 - lats[1943] = -46.643232586477971 - lats[1944] = -46.713531353402139 - lats[1945] = -46.783830120318882 - lats[1946] = -46.85412888722815 - lats[1947] = -46.924427654129929 - lats[1948] = -46.994726421024154 - lats[1949] = -47.065025187910805 - lats[1950] = -47.13532395478984 - lats[1951] = -47.205622721661214 - lats[1952] = -47.275921488524894 - lats[1953] = -47.346220255380835 - lats[1954] = -47.416519022228997 - lats[1955] = -47.486817789069342 - lats[1956] = -47.557116555901828 - lats[1957] = -47.627415322726435 - lats[1958] = -47.697714089543084 - lats[1959] = -47.76801285635176 - lats[1960] = -47.838311623152421 - lats[1961] = -47.908610389945018 - lats[1962] = -47.978909156729507 - lats[1963] = -48.049207923505868 - lats[1964] = -48.119506690274015 - lats[1965] = -48.189805457033941 - lats[1966] = -48.260104223785596 - lats[1967] = -48.330402990528938 - lats[1968] = -48.400701757263917 - lats[1969] = -48.47100052399049 - lats[1970] = -48.541299290708608 - lats[1971] = -48.611598057418242 - lats[1972] = -48.681896824119335 - lats[1973] = -48.752195590811837 - lats[1974] = -48.822494357495721 - lats[1975] = -48.892793124170929 - lats[1976] = -48.963091890837418 - lats[1977] = -49.03339065749514 - lats[1978] = -49.103689424144044 - lats[1979] = -49.173988190784094 - lats[1980] = -49.244286957415234 - lats[1981] = -49.314585724037435 - lats[1982] = -49.384884490650613 - lats[1983] = -49.455183257254745 - lats[1984] = -49.525482023849783 - lats[1985] = -49.595780790435676 - lats[1986] = -49.66607955701236 - lats[1987] = -49.736378323579807 - lats[1988] = -49.80667709013796 - lats[1989] = -49.876975856686762 - lats[1990] = -49.947274623226157 - lats[1991] = -50.017573389756123 - lats[1992] = -50.087872156276575 - lats[1993] = -50.158170922787484 - lats[1994] = -50.228469689288779 - lats[1995] = -50.298768455780426 - lats[1996] = -50.369067222262359 - lats[1997] = -50.439365988734544 - lats[1998] = -50.509664755196901 - lats[1999] = -50.579963521649397 - lats[2000] = -50.650262288091959 - lats[2001] = -50.720561054524559 - lats[2002] = -50.790859820947119 - lats[2003] = -50.86115858735959 - lats[2004] = -50.931457353761914 - lats[2005] = -51.001756120154049 - lats[2006] = -51.072054886535909 - lats[2007] = -51.14235365290746 - lats[2008] = -51.21265241926865 - lats[2009] = -51.282951185619417 - lats[2010] = -51.353249951959683 - lats[2011] = -51.423548718289396 - lats[2012] = -51.493847484608516 - lats[2013] = -51.56414625091697 - lats[2014] = -51.634445017214695 - lats[2015] = -51.704743783501634 - lats[2016] = -51.775042549777737 - lats[2017] = -51.845341316042933 - lats[2018] = -51.915640082297152 - lats[2019] = -51.985938848540336 - lats[2020] = -52.056237614772435 - lats[2021] = -52.126536380993372 - lats[2022] = -52.196835147203096 - lats[2023] = -52.26713391340153 - lats[2024] = -52.337432679588609 - lats[2025] = -52.407731445764284 - lats[2026] = -52.478030211928477 - lats[2027] = -52.548328978081123 - lats[2028] = -52.618627744222159 - lats[2029] = -52.688926510351514 - lats[2030] = -52.759225276469131 - lats[2031] = -52.829524042574917 - lats[2032] = -52.899822808668837 - lats[2033] = -52.970121574750792 - lats[2034] = -53.040420340820731 - lats[2035] = -53.110719106878584 - lats[2036] = -53.181017872924265 - lats[2037] = -53.251316638957725 - lats[2038] = -53.321615404978871 - lats[2039] = -53.391914170987633 - lats[2040] = -53.462212936983953 - lats[2041] = -53.53251170296776 - lats[2042] = -53.602810468938962 - lats[2043] = -53.673109234897495 - lats[2044] = -53.743408000843282 - lats[2045] = -53.813706766776235 - lats[2046] = -53.884005532696307 - lats[2047] = -53.954304298603383 - lats[2048] = -54.024603064497434 - lats[2049] = -54.094901830378333 - lats[2050] = -54.165200596246031 - lats[2051] = -54.235499362100448 - lats[2052] = -54.305798127941479 - lats[2053] = -54.376096893769081 - lats[2054] = -54.446395659583146 - lats[2055] = -54.516694425383605 - lats[2056] = -54.586993191170357 - lats[2057] = -54.657291956943347 - lats[2058] = -54.727590722702473 - lats[2059] = -54.797889488447652 - lats[2060] = -54.868188254178797 - lats[2061] = -54.938487019895831 - lats[2062] = -55.008785785598668 - lats[2063] = -55.07908455128721 - lats[2064] = -55.149383316961377 - lats[2065] = -55.219682082621084 - lats[2066] = -55.289980848266232 - lats[2067] = -55.360279613896743 - lats[2068] = -55.430578379512511 - lats[2069] = -55.500877145113449 - lats[2070] = -55.571175910699488 - lats[2071] = -55.641474676270505 - lats[2072] = -55.711773441826416 - lats[2073] = -55.782072207367136 - lats[2074] = -55.852370972892551 - lats[2075] = -55.922669738402583 - lats[2076] = -55.992968503897131 - lats[2077] = -56.063267269376091 - lats[2078] = -56.133566034839362 - lats[2079] = -56.203864800286865 - lats[2080] = -56.274163565718467 - lats[2081] = -56.34446233113411 - lats[2082] = -56.41476109653366 - lats[2083] = -56.485059861917016 - lats[2084] = -56.555358627284086 - lats[2085] = -56.625657392634771 - lats[2086] = -56.695956157968951 - lats[2087] = -56.766254923286517 - lats[2088] = -56.836553688587379 - lats[2089] = -56.90685245387143 - lats[2090] = -56.977151219138541 - lats[2091] = -57.047449984388614 - lats[2092] = -57.117748749621541 - lats[2093] = -57.188047514837208 - lats[2094] = -57.258346280035504 - lats[2095] = -57.328645045216312 - lats[2096] = -57.398943810379521 - lats[2097] = -57.469242575525016 - lats[2098] = -57.539541340652676 - lats[2099] = -57.60984010576238 - lats[2100] = -57.680138870854037 - lats[2101] = -57.75043763592749 - lats[2102] = -57.820736400982646 - lats[2103] = -57.891035166019364 - lats[2104] = -57.961333931037537 - lats[2105] = -58.031632696037022 - lats[2106] = -58.101931461017728 - lats[2107] = -58.172230225979497 - lats[2108] = -58.242528990922203 - lats[2109] = -58.312827755845746 - lats[2110] = -58.383126520749968 - lats[2111] = -58.453425285634758 - lats[2112] = -58.523724050499972 - lats[2113] = -58.594022815345468 - lats[2114] = -58.664321580171141 - lats[2115] = -58.73462034497684 - lats[2116] = -58.804919109762423 - lats[2117] = -58.875217874527763 - lats[2118] = -58.945516639272725 - lats[2119] = -59.015815403997145 - lats[2120] = -59.086114168700909 - lats[2121] = -59.156412933383855 - lats[2122] = -59.226711698045854 - lats[2123] = -59.29701046268675 - lats[2124] = -59.3673092273064 - lats[2125] = -59.43760799190467 - lats[2126] = -59.507906756481383 - lats[2127] = -59.578205521036402 - lats[2128] = -59.64850428556958 - lats[2129] = -59.718803050080759 - lats[2130] = -59.78910181456979 - lats[2131] = -59.859400579036503 - lats[2132] = -59.929699343480763 - lats[2133] = -59.999998107902378 - lats[2134] = -60.070296872301235 - lats[2135] = -60.140595636677112 - lats[2136] = -60.21089440102989 - lats[2137] = -60.28119316535939 - lats[2138] = -60.35149192966545 - lats[2139] = -60.421790693947884 - lats[2140] = -60.492089458206543 - lats[2141] = -60.562388222441243 - lats[2142] = -60.632686986651805 - lats[2143] = -60.702985750838074 - lats[2144] = -60.773284514999872 - lats[2145] = -60.843583279137007 - lats[2146] = -60.913882043249295 - lats[2147] = -60.984180807336578 - lats[2148] = -61.054479571398652 - lats[2149] = -61.124778335435344 - lats[2150] = -61.195077099446451 - lats[2151] = -61.265375863431785 - lats[2152] = -61.335674627391185 - lats[2153] = -61.405973391324409 - lats[2154] = -61.476272155231321 - lats[2155] = -61.546570919111666 - lats[2156] = -61.616869682965287 - lats[2157] = -61.687168446791986 - lats[2158] = -61.757467210591535 - lats[2159] = -61.827765974363729 - lats[2160] = -61.898064738108381 - lats[2161] = -61.968363501825259 - lats[2162] = -62.038662265514176 - lats[2163] = -62.108961029174914 - lats[2164] = -62.179259792807258 - lats[2165] = -62.249558556410982 - lats[2166] = -62.319857319985871 - lats[2167] = -62.3901560835317 - lats[2168] = -62.460454847048261 - lats[2169] = -62.530753610535321 - lats[2170] = -62.60105237399263 - lats[2171] = -62.67135113741999 - lats[2172] = -62.741649900817137 - lats[2173] = -62.811948664183866 - lats[2174] = -62.882247427519928 - lats[2175] = -62.952546190825068 - lats[2176] = -63.022844954099064 - lats[2177] = -63.093143717341647 - lats[2178] = -63.163442480552604 - lats[2179] = -63.23374124373165 - lats[2180] = -63.304040006878537 - lats[2181] = -63.374338769993031 - lats[2182] = -63.444637533074854 - lats[2183] = -63.514936296123757 - lats[2184] = -63.585235059139464 - lats[2185] = -63.655533822121711 - lats[2186] = -63.725832585070251 - lats[2187] = -63.796131347984762 - lats[2188] = -63.866430110865004 - lats[2189] = -63.93672887371072 - lats[2190] = -64.00702763652157 - lats[2191] = -64.07732639929732 - lats[2192] = -64.147625162037642 - lats[2193] = -64.21792392474228 - lats[2194] = -64.288222687410922 - lats[2195] = -64.358521450043284 - lats[2196] = -64.428820212639039 - lats[2197] = -64.499118975197902 - lats[2198] = -64.569417737719576 - lats[2199] = -64.639716500203733 - lats[2200] = -64.710015262650074 - lats[2201] = -64.780314025058246 - lats[2202] = -64.850612787427963 - lats[2203] = -64.920911549758912 - lats[2204] = -64.991210312050711 - lats[2205] = -65.061509074303089 - lats[2206] = -65.131807836515677 - lats[2207] = -65.202106598688133 - lats[2208] = -65.272405360820116 - lats[2209] = -65.342704122911286 - lats[2210] = -65.413002884961315 - lats[2211] = -65.483301646969792 - lats[2212] = -65.553600408936404 - lats[2213] = -65.623899170860767 - lats[2214] = -65.694197932742526 - lats[2215] = -65.764496694581283 - lats[2216] = -65.834795456376696 - lats[2217] = -65.905094218128355 - lats[2218] = -65.975392979835888 - lats[2219] = -66.045691741498899 - lats[2220] = -66.115990503117033 - lats[2221] = -66.186289264689833 - lats[2222] = -66.256588026216932 - lats[2223] = -66.326886787697887 - lats[2224] = -66.397185549132331 - lats[2225] = -66.467484310519808 - lats[2226] = -66.537783071859891 - lats[2227] = -66.608081833152212 - lats[2228] = -66.678380594396273 - lats[2229] = -66.748679355591662 - lats[2230] = -66.818978116737924 - lats[2231] = -66.889276877834618 - lats[2232] = -66.95957563888129 - lats[2233] = -67.029874399877471 - lats[2234] = -67.100173160822706 - lats[2235] = -67.170471921716526 - lats[2236] = -67.240770682558434 - lats[2237] = -67.311069443347961 - lats[2238] = -67.381368204084609 - lats[2239] = -67.451666964767895 - lats[2240] = -67.521965725397308 - lats[2241] = -67.592264485972336 - lats[2242] = -67.662563246492482 - lats[2243] = -67.732862006957205 - lats[2244] = -67.803160767365966 - lats[2245] = -67.873459527718282 - lats[2246] = -67.943758288013555 - lats[2247] = -68.014057048251274 - lats[2248] = -68.084355808430871 - lats[2249] = -68.154654568551791 - lats[2250] = -68.224953328613438 - lats[2251] = -68.295252088615257 - lats[2252] = -68.365550848556666 - lats[2253] = -68.435849608437067 - lats[2254] = -68.506148368255865 - lats[2255] = -68.576447128012447 - lats[2256] = -68.646745887706189 - lats[2257] = -68.717044647336493 - lats[2258] = -68.787343406902693 - lats[2259] = -68.85764216640419 - lats[2260] = -68.927940925840304 - lats[2261] = -68.998239685210365 - lats[2262] = -69.068538444513763 - lats[2263] = -69.138837203749759 - lats[2264] = -69.209135962917699 - lats[2265] = -69.279434722016902 - lats[2266] = -69.349733481046613 - lats[2267] = -69.420032240006194 - lats[2268] = -69.490330998894862 - lats[2269] = -69.560629757711908 - lats[2270] = -69.630928516456592 - lats[2271] = -69.701227275128161 - lats[2272] = -69.771526033725834 - lats[2273] = -69.841824792248843 - lats[2274] = -69.912123550696421 - lats[2275] = -69.982422309067744 - lats[2276] = -70.052721067362043 - lats[2277] = -70.123019825578467 - lats[2278] = -70.193318583716191 - lats[2279] = -70.263617341774406 - lats[2280] = -70.333916099752187 - lats[2281] = -70.404214857648739 - lats[2282] = -70.474513615463138 - lats[2283] = -70.544812373194532 - lats[2284] = -70.615111130841967 - lats[2285] = -70.685409888404578 - lats[2286] = -70.755708645881384 - lats[2287] = -70.826007403271475 - lats[2288] = -70.896306160573886 - lats[2289] = -70.966604917787635 - lats[2290] = -71.036903674911756 - lats[2291] = -71.107202431945211 - lats[2292] = -71.177501188887007 - lats[2293] = -71.247799945736105 - lats[2294] = -71.318098702491469 - lats[2295] = -71.388397459152031 - lats[2296] = -71.458696215716685 - lats[2297] = -71.528994972184378 - lats[2298] = -71.599293728553988 - lats[2299] = -71.669592484824364 - lats[2300] = -71.739891240994368 - lats[2301] = -71.810189997062835 - lats[2302] = -71.880488753028587 - lats[2303] = -71.950787508890414 - lats[2304] = -72.02108626464711 - lats[2305] = -72.091385020297409 - lats[2306] = -72.161683775840089 - lats[2307] = -72.231982531273843 - lats[2308] = -72.302281286597392 - lats[2309] = -72.3725800418094 - lats[2310] = -72.442878796908545 - lats[2311] = -72.513177551893421 - lats[2312] = -72.583476306762691 - lats[2313] = -72.653775061514935 - lats[2314] = -72.724073816148703 - lats[2315] = -72.794372570662574 - lats[2316] = -72.864671325055056 - lats[2317] = -72.934970079324657 - lats[2318] = -73.005268833469799 - lats[2319] = -73.075567587489019 - lats[2320] = -73.145866341380668 - lats[2321] = -73.216165095143182 - lats[2322] = -73.2864638487749 - lats[2323] = -73.356762602274188 - lats[2324] = -73.427061355639339 - lats[2325] = -73.497360108868662 - lats[2326] = -73.567658861960396 - lats[2327] = -73.637957614912779 - lats[2328] = -73.70825636772399 - lats[2329] = -73.778555120392184 - lats[2330] = -73.848853872915541 - lats[2331] = -73.919152625292114 - lats[2332] = -73.98945137751997 - lats[2333] = -74.059750129597163 - lats[2334] = -74.13004888152166 - lats[2335] = -74.200347633291472 - lats[2336] = -74.270646384904481 - lats[2337] = -74.340945136358584 - lats[2338] = -74.411243887651622 - lats[2339] = -74.481542638781434 - lats[2340] = -74.551841389745761 - lats[2341] = -74.622140140542356 - lats[2342] = -74.692438891168877 - lats[2343] = -74.762737641622991 - lats[2344] = -74.833036391902269 - lats[2345] = -74.903335142004323 - lats[2346] = -74.973633891926625 - lats[2347] = -75.043932641666672 - lats[2348] = -75.114231391221821 - lats[2349] = -75.184530140589501 - lats[2350] = -75.254828889766983 - lats[2351] = -75.325127638751567 - lats[2352] = -75.395426387540439 - lats[2353] = -75.465725136130786 - lats[2354] = -75.536023884519707 - lats[2355] = -75.60632263270422 - lats[2356] = -75.67662138068134 - lats[2357] = -75.746920128447996 - lats[2358] = -75.81721887600105 - lats[2359] = -75.887517623337317 - lats[2360] = -75.957816370453543 - lats[2361] = -76.028115117346374 - lats[2362] = -76.098413864012443 - lats[2363] = -76.16871261044831 - lats[2364] = -76.239011356650423 - lats[2365] = -76.3093101026152 - lats[2366] = -76.379608848338933 - lats[2367] = -76.449907593817869 - lats[2368] = -76.520206339048215 - lats[2369] = -76.59050508402602 - lats[2370] = -76.660803828747362 - lats[2371] = -76.731102573208048 - lats[2372] = -76.801401317404 - lats[2373] = -76.871700061330955 - lats[2374] = -76.941998804984564 - lats[2375] = -77.012297548360323 - lats[2376] = -77.082596291453768 - lats[2377] = -77.15289503426024 - lats[2378] = -77.22319377677502 - lats[2379] = -77.293492518993247 - lats[2380] = -77.363791260909963 - lats[2381] = -77.434090002520122 - lats[2382] = -77.504388743818524 - lats[2383] = -77.574687484799924 - lats[2384] = -77.644986225458879 - lats[2385] = -77.71528496578982 - lats[2386] = -77.785583705787161 - lats[2387] = -77.855882445445019 - lats[2388] = -77.926181184757539 - lats[2389] = -77.996479923718596 - lats[2390] = -78.066778662322022 - lats[2391] = -78.137077400561424 - lats[2392] = -78.207376138430348 - lats[2393] = -78.277674875922045 - lats[2394] = -78.347973613029708 - lats[2395] = -78.418272349746417 - lats[2396] = -78.488571086064923 - lats[2397] = -78.558869821977908 - lats[2398] = -78.629168557477882 - lats[2399] = -78.699467292557102 - lats[2400] = -78.769766027207638 - lats[2401] = -78.840064761421445 - lats[2402] = -78.910363495190211 - lats[2403] = -78.980662228505423 - lats[2404] = -79.050960961358285 - lats[2405] = -79.121259693739859 - lats[2406] = -79.191558425640977 - lats[2407] = -79.261857157052191 - lats[2408] = -79.332155887963822 - lats[2409] = -79.402454618365894 - lats[2410] = -79.472753348248219 - lats[2411] = -79.543052077600308 - lats[2412] = -79.61335080641139 - lats[2413] = -79.683649534670437 - lats[2414] = -79.753948262366038 - lats[2415] = -79.824246989486554 - lats[2416] = -79.894545716019948 - lats[2417] = -79.9648444419539 - lats[2418] = -80.035143167275749 - lats[2419] = -80.105441891972376 - lats[2420] = -80.175740616030438 - lats[2421] = -80.246039339436052 - lats[2422] = -80.316338062175078 - lats[2423] = -80.386636784232863 - lats[2424] = -80.456935505594302 - lats[2425] = -80.527234226243991 - lats[2426] = -80.59753294616587 - lats[2427] = -80.667831665343556 - lats[2428] = -80.73813038376008 - lats[2429] = -80.808429101397948 - lats[2430] = -80.878727818239184 - lats[2431] = -80.949026534265244 - lats[2432] = -81.019325249456955 - lats[2433] = -81.089623963794551 - lats[2434] = -81.159922677257711 - lats[2435] = -81.230221389825374 - lats[2436] = -81.300520101475826 - lats[2437] = -81.370818812186627 - lats[2438] = -81.441117521934686 - lats[2439] = -81.511416230696042 - lats[2440] = -81.581714938445955 - lats[2441] = -81.652013645158945 - lats[2442] = -81.722312350808508 - lats[2443] = -81.792611055367345 - lats[2444] = -81.862909758807191 - lats[2445] = -81.933208461098829 - lats[2446] = -82.003507162211946 - lats[2447] = -82.073805862115165 - lats[2448] = -82.144104560776 - lats[2449] = -82.214403258160871 - lats[2450] = -82.284701954234833 - lats[2451] = -82.355000648961692 - lats[2452] = -82.425299342304029 - lats[2453] = -82.495598034222837 - lats[2454] = -82.56589672467787 - lats[2455] = -82.63619541362705 - lats[2456] = -82.706494101026948 - lats[2457] = -82.77679278683226 - lats[2458] = -82.84709147099602 - lats[2459] = -82.917390153469313 - lats[2460] = -82.987688834201322 - lats[2461] = -83.057987513139125 - lats[2462] = -83.128286190227698 - lats[2463] = -83.198584865409657 - lats[2464] = -83.268883538625232 - lats[2465] = -83.339182209812321 - lats[2466] = -83.409480878905782 - lats[2467] = -83.479779545838113 - lats[2468] = -83.550078210538487 - lats[2469] = -83.620376872933264 - lats[2470] = -83.690675532945292 - lats[2471] = -83.760974190494011 - lats[2472] = -83.831272845495249 - lats[2473] = -83.901571497860914 - lats[2474] = -83.971870147498763 - lats[2475] = -84.042168794312317 - lats[2476] = -84.112467438200326 - lats[2477] = -84.18276607905679 - lats[2478] = -84.253064716770425 - lats[2479] = -84.323363351224444 - lats[2480] = -84.393661982296322 - lats[2481] = -84.463960609857125 - lats[2482] = -84.534259233771479 - lats[2483] = -84.604557853896708 - lats[2484] = -84.674856470082915 - lats[2485] = -84.745155082171991 - lats[2486] = -84.81545368999717 - lats[2487] = -84.885752293382765 - lats[2488] = -84.95605089214304 - lats[2489] = -85.026349486081983 - lats[2490] = -85.09664807499216 - lats[2491] = -85.16694665865414 - lats[2492] = -85.237245236835548 - lats[2493] = -85.307543809290152 - lats[2494] = -85.377842375756586 - lats[2495] = -85.448140935957483 - lats[2496] = -85.518439489597966 - lats[2497] = -85.588738036364362 - lats[2498] = -85.659036575922883 - lats[2499] = -85.729335107917464 - lats[2500] = -85.799633631968391 - lats[2501] = -85.869932147670127 - lats[2502] = -85.940230654588888 - lats[2503] = -86.010529152260403 - lats[2504] = -86.080827640187209 - lats[2505] = -86.151126117835304 - lats[2506] = -86.221424584631109 - lats[2507] = -86.291723039957418 - lats[2508] = -86.362021483149363 - lats[2509] = -86.432319913489792 - lats[2510] = -86.502618330203831 - lats[2511] = -86.572916732453024 - lats[2512] = -86.643215119328573 - lats[2513] = -86.713513489844246 - lats[2514] = -86.783811842927179 - lats[2515] = -86.854110177408927 - lats[2516] = -86.924408492014166 - lats[2517] = -86.994706785348129 - lats[2518] = -87.065005055882821 - lats[2519] = -87.135303301939786 - lats[2520] = -87.205601521672108 - lats[2521] = -87.275899713041966 - lats[2522] = -87.346197873795816 - lats[2523] = -87.416496001434894 - lats[2524] = -87.486794093180748 - lats[2525] = -87.557092145935584 - lats[2526] = -87.627390156234085 - lats[2527] = -87.697688120188062 - lats[2528] = -87.767986033419561 - lats[2529] = -87.838283890981543 - lats[2530] = -87.908581687261687 - lats[2531] = -87.978879415867283 - lats[2532] = -88.049177069484486 - lats[2533] = -88.119474639706425 - lats[2534] = -88.189772116820762 - lats[2535] = -88.26006948954614 - lats[2536] = -88.330366744702559 - lats[2537] = -88.40066386679355 - lats[2538] = -88.470960837474877 - lats[2539] = -88.541257634868515 - lats[2540] = -88.611554232668382 - lats[2541] = -88.681850598961759 - lats[2542] = -88.752146694650691 - lats[2543] = -88.822442471310097 - lats[2544] = -88.892737868230952 - lats[2545] = -88.96303280826325 - lats[2546] = -89.033327191845927 - lats[2547] = -89.103620888238879 - lats[2548] = -89.173913722284126 - lats[2549] = -89.24420545380525 - lats[2550] = -89.314495744374256 - lats[2551] = -89.3847841013921 - lats[2552] = -89.45506977912261 - lats[2553] = -89.525351592371393 - lats[2554] = -89.595627537554492 - lats[2555] = -89.6658939412157 - lats[2556] = -89.736143271609578 - lats[2557] = -89.806357319542244 - lats[2558] = -89.876478353332288 - lats[2559] = -89.946187715665616 - return lats - def first_axis_vals(self): if self._resolution == 1280: return self.get_precomputed_values_N1280() @@ -5502,16 +2939,40 @@ def second_axis_vals(self, first_val): second_axis_vals = [i * second_axis_spacing for i in range(npoints)] return second_axis_vals + def second_axis_spacing(self, first_val): + first_axis_vals = self._first_axis_vals + tol = 1e-10 + # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] + # first_idx = first_axis_vals.index(first_val) + _first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + first_idx = _first_idx + if first_idx >= self._resolution: + first_idx = (2 * self._resolution) - 1 - first_idx + first_idx = first_idx + 1 + npoints = 4 * first_idx + 16 + second_axis_spacing = 360 / npoints + return (second_axis_spacing, _first_idx + 1) + def map_second_axis(self, first_val, lower, upper): - second_axis_vals = self.second_axis_vals(first_val) - # NOTE: here this seems faster than the bisect.bisect? - # return_vals = [val for val in second_axis_vals if lower <= val <= upper] - start_idx = bisect_left_cmp(second_axis_vals, lower, cmp=lambda x, y: x < y) + 1 - end_idx = bisect_right_cmp(second_axis_vals, upper, cmp=lambda x, y: x < y) + 1 - return_vals = second_axis_vals[start_idx:end_idx] - # start_idx = bisect.bisect_left(second_axis_vals, lower) - # end_idx = bisect.bisect_right(second_axis_vals, upper) + second_axis_spacing, first_idx = self.second_axis_spacing(first_val) + # if first_val not in self._second_axis_spacing: + # (second_axis_spacing, first_idx) = self.second_axis_spacing(first_val) + # self._second_axis_spacing[first_val] = (second_axis_spacing, first_idx) + # else: + # (second_axis_spacing, first_idx) = self._second_axis_spacing[first_val] + start_idx = int(lower/second_axis_spacing) + end_idx = int(upper/second_axis_spacing) + 1 + return_vals = [i * second_axis_spacing for i in range(start_idx, end_idx)] + + # second_axis_vals = self.second_axis_vals(first_val) + # # NOTE: here this seems faster than the bisect.bisect? + # # return_vals = [val for val in second_axis_vals if lower <= val <= upper] + # start_idx = bisect_left_cmp(second_axis_vals, lower, cmp=lambda x, y: x < y) + 1 + # end_idx = bisect_right_cmp(second_axis_vals, upper, cmp=lambda x, y: x < y) + 1 # return_vals = second_axis_vals[start_idx:end_idx] + # # start_idx = bisect.bisect_left(second_axis_vals, lower) + # # end_idx = bisect.bisect_right(second_axis_vals, upper) + # # return_vals = second_axis_vals[start_idx:end_idx] return return_vals def axes_idx_to_octahedral_idx(self, first_idx, second_idx): @@ -5544,11 +3005,11 @@ def axes_idx_to_octahedral_idx(self, first_idx, second_idx): # print(time.time() - time1) return octa_idx - def find_second_idx(self, first_val, second_val): - tol = 1e-10 - second_axis_vals = self.second_axis_vals(first_val) - second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) - return second_idx + # def find_second_idx(self, first_val, second_val): + # tol = 1e-10 + # second_axis_vals = self.second_axis_vals(first_val) + # second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) + # return second_idx def create_first_idx_map(self): # first_idx_list = [0] * (2*self._resolution) @@ -5568,55 +3029,67 @@ def create_first_idx_map(self): idx += 16 + 4 * (self._resolution - i) return first_idx_list - def unmap_first_val_to_start_line_idx(self, first_val): - first_axis_vals = self._first_axis_vals - tol = 1e-10 - # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] - # first_idx = first_axis_vals.index(first_val) + 1 - # first_idx = len(first_axis_vals) - bisect.bisect_left(first_axis_vals[::-1], first_val - tol) - first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + 1 - octa_idx = 0 - if first_idx == 1: - return octa_idx - else: - for i in range(first_idx - 1): - if i <= self._resolution - 1: - octa_idx += 20 + 4 * i - else: - i = i - self._resolution + 1 - if i == 1: - octa_idx += 16 + 4 * self._resolution - else: - i = i - 1 - octa_idx += 16 + 4 * (self._resolution - i) - return octa_idx + # def unmap_first_val_to_start_line_idx(self, first_val): + # first_axis_vals = self._first_axis_vals + # tol = 1e-10 + # # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] + # # first_idx = first_axis_vals.index(first_val) + 1 + # # first_idx = len(first_axis_vals) - bisect.bisect_left(first_axis_vals[::-1], first_val - tol) + # first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + 1 + # octa_idx = 0 + # if first_idx == 1: + # return octa_idx + # else: + # for i in range(first_idx - 1): + # if i <= self._resolution - 1: + # octa_idx += 20 + 4 * i + # else: + # i = i - self._resolution + 1 + # if i == 1: + # octa_idx += 16 + 4 * self._resolution + # else: + # i = i - 1 + # octa_idx += 16 + 4 * (self._resolution - i) + # return octa_idx + + def find_second_axis_idx(self, first_val, second_val): + (second_axis_spacing, first_idx) = self.second_axis_spacing(first_val) + # if first_val not in self._second_axis_spacing: + # (second_axis_spacing, first_idx) = self.second_axis_spacing(first_val) + # self._second_axis_spacing[first_val] = (second_axis_spacing, first_idx) + # else: + # (second_axis_spacing, first_idx) = self._second_axis_spacing[first_val] + second_idx = int(second_val/second_axis_spacing) + return (first_idx, second_idx) def unmap(self, first_val, second_val): - time1 = time.time() + # time1 = time.time() # first_axis_vals = self._first_axis_vals # inv_first_axis_vals = self._inv_first_axis_vals - tol = 1e-10 - # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] - # first_idx = first_axis_vals.index(first_val) + 1 - # first_idx = len(first_axis_vals) - bisect.bisect_left(first_axis_vals[::-1], first_val - tol) - # first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + 1 - first_idx = bisect.bisect_left(self._inv_first_axis_vals, - (first_val - tol)) - # print(inv_first_axis_vals) - # print(first_val) - # first_idx = inv_first_axis_vals[first_val] - # first_idx = np.searchsorted(-first_axis_vals, - (first_val - tol), side="right") - if first_val not in self.treated_first_vals: - second_axis_vals = self.second_axis_vals(first_val) - self.treated_first_vals[first_val] = second_axis_vals - else: - second_axis_vals = self.treated_first_vals[first_val] - # second_val = [val for val in second_axis_vals if second_val - tol < val < second_val + tol][0] - # second_idx = second_axis_vals.index(second_val) - second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) + # tol = 1e-10 + # # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] + # # first_idx = first_axis_vals.index(first_val) + 1 + # # first_idx = len(first_axis_vals) - bisect.bisect_left(first_axis_vals[::-1], first_val - tol) + # # first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + 1 + # first_idx = bisect.bisect_left(self._inv_first_axis_vals, - (first_val - tol)) + # # print(inv_first_axis_vals) + # # print(first_val) + # # first_idx = inv_first_axis_vals[first_val] + # # first_idx = np.searchsorted(-first_axis_vals, - (first_val - tol), side="right") + # if first_val not in self.treated_first_vals: + # second_axis_vals = self.second_axis_vals(first_val) + # self.treated_first_vals[first_val] = second_axis_vals + # else: + # second_axis_vals = self.treated_first_vals[first_val] + # # second_val = [val for val in second_axis_vals if second_val - tol < val < second_val + tol][0] + # # second_idx = second_axis_vals.index(second_val) + # second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) + (first_idx, second_idx) = self.find_second_axis_idx(first_val, second_val) # second_idx = np.searchsorted(second_axis_vals, second_val - tol) # print("TIME SPENT DOING VAL TO IDX") # print(time.time() - time1) octahedral_index = self.axes_idx_to_octahedral_idx(first_idx, second_idx) + # octahedral_index = int(octahedral_index) # print("OCTAHEDRAL UNMAP TIME ") # print(time.time() - time1) return octahedral_index From 9faf711f483a9b539b5f03da2b783276266ff66b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 19 Oct 2023 16:52:27 +0200 Subject: [PATCH 170/332] optimise small bits --- polytope/datacube/backends/FDB_datacube.py | 18 +++++++++++++----- polytope/datacube/index_tree.py | 1 + .../transformations/datacube_merger.py | 18 ++++++++++++++++-- polytope/engine/hullslicer.py | 1 + 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 61404cd6f..ba119d9a6 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -8,7 +8,7 @@ def update_fdb_dataarray(fdb_dataarray): - fdb_dataarray["values"] = [0.0] + fdb_dataarray["values"] = [] return fdb_dataarray @@ -32,7 +32,6 @@ def __init__(self, config={}, axis_options={}): fdb_dataarray = self.fdb.axes(partial_request).as_dict() dataarray = update_fdb_dataarray(fdb_dataarray) self.dataarray = dataarray - for name, values in dataarray.items(): values.sort() options = axis_options.get(name, {}) @@ -48,11 +47,17 @@ def __init__(self, config={}, axis_options={}): self._check_and_add_axes(options, name, val) def get(self, requests: IndexTree): + # NOTE: this will do all the transformation unmappings for all the points + # It doesn't use the tree structure of the result to do the unmapping transformations anymore time0 = time.time() time_changing_path = 0 accumulated_fdb_time = 0 + time_change_path = 0 for r in requests.leaves: + time5 = time.time() + # NOTE: Accumulated time in flatten is 0.14s... could be better? path = r.flatten() + time_change_path += time.time() - time5 path = self.remap_path(path) if len(path.items()) == self.axis_counter: # first, find the grid mapper transform @@ -88,6 +93,8 @@ def get(self, requests: IndexTree): print(accumulated_fdb_time) print("GET TIME") print(time.time() - time0) + print("TIME FLATTEN PATH AND CHANGE PATH") + print(time_change_path) print("TIME CHANGING PATH") print(time_changing_path) @@ -99,6 +106,7 @@ def select(self, path, unmapped_path): return self.dataarray def ax_vals(self, name): - for _name, values in self.dataarray.items(): - if _name == name: - return values + # for _name, values in self.dataarray.items(): + # if _name == name: + # return values + return self.dataarray.get(name, None) diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index b7a37aff5..20f82f1c2 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -33,6 +33,7 @@ def __init__(self, axis=root, value=None): @property def leaves(self): + # TODO: could store ancestors directly in leaves? leaves = [] self._collect_leaf_nodes(leaves) return leaves diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index e8ff67145..288875135 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd +import time from .datacube_transformations import DatacubeAxisTransformation @@ -16,19 +17,29 @@ def blocked_axes(self): return [self._second_axis] def merged_values(self, datacube): + # time1 = time.time() first_ax_vals = datacube.ax_vals(self.name) second_ax_name = self._second_axis second_ax_vals = datacube.ax_vals(second_ax_name) linkers = self._linkers merged_values = [] - for first_val in first_ax_vals: - for second_val in second_ax_vals: + # merged_values = np.empty(len(first_ax_vals)*len(second_ax_vals)) + # for first_val in first_ax_vals: + for i in range(len(first_ax_vals)): + first_val = first_ax_vals[i] + # for second_val in second_ax_vals: + for j in range(len(second_ax_vals)): + second_val = second_ax_vals[j] # TODO: check that the first and second val are strings val_to_add = pd.to_datetime(first_val + linkers[0] + second_val + linkers[1]) val_to_add = val_to_add.to_numpy() val_to_add = val_to_add.astype("datetime64[s]") merged_values.append(val_to_add) + # print(val_to_add) + # merged_values[i*len(second_ax_vals) + j] = val_to_add merged_values = np.array(merged_values) + # print("MERGED VALUES TIME") + # print(time.time() - time1) return merged_values def transformation_axes_final(self): @@ -38,6 +49,7 @@ def generate_final_transformation(self): return self def unmerge(self, merged_val): + # time1 = time.time() merged_val = str(merged_val) first_idx = merged_val.find(self._linkers[0]) first_val = merged_val[:first_idx] @@ -48,6 +60,8 @@ def unmerge(self, merged_val): # TODO: maybe replacing like this is too specific to time/dates? first_val = str(first_val).replace("-", "") second_val = second_val.replace(":", "") + # print("UNMERGE TIME") + # print(time.time() - time1) return (first_val, second_val) def change_val_type(self, axis_name, values): diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index f39f8b0c0..7b386cae8 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -2,6 +2,7 @@ from copy import copy from itertools import chain from typing import List +import time import scipy.spatial From fb079544348904116a79d4d754fa670edd5df95e Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 20 Oct 2023 13:36:26 +0200 Subject: [PATCH 171/332] calculate leaves' ancestors while finding leaves optimisation --- polytope/datacube/backends/FDB_datacube.py | 5 ++-- polytope/datacube/index_tree.py | 29 +++++++++++++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index ba119d9a6..fcecda8cc 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -53,10 +53,11 @@ def get(self, requests: IndexTree): time_changing_path = 0 accumulated_fdb_time = 0 time_change_path = 0 - for r in requests.leaves: + for r in requests.leaves_with_ancestors: time5 = time.time() # NOTE: Accumulated time in flatten is 0.14s... could be better? - path = r.flatten() + path = r.flatten_with_ancestors() + # path = r.flatten() time_change_path += time.time() - time5 path = self.remap_path(path) if len(path.items()) == self.axis_counter: diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index 20f82f1c2..9054cd0c5 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -30,18 +30,38 @@ def __init__(self, axis=root, value=None): self._parent = None self.result = None self.axis = axis + self.ancestors = [] @property def leaves(self): - # TODO: could store ancestors directly in leaves? leaves = [] self._collect_leaf_nodes(leaves) return leaves + @property + def leaves_with_ancestors(self): + # TODO: could store ancestors directly in leaves? Change here + leaves = [] + self._collect_leaf_nodes(leaves) + return leaves + + def _collect_leaf_nodes_old(self, leaves): + if len(self.children) == 0: + leaves.append(self) + for n in self.children: + n._collect_leaf_nodes(leaves) + def _collect_leaf_nodes(self, leaves): + # NOTE: leaves_and_ancestors is going to be a list of tuples, where first entry is leaf and second entry is a + # list of its ancestors if len(self.children) == 0: leaves.append(self) + self.ancestors.append(self) for n in self.children: + for ancestor in self.ancestors: + n.ancestors.append(ancestor) + if self.axis != IndexTree.root: + n.ancestors.append(self) n._collect_leaf_nodes(leaves) def __setitem__(self, key, value): @@ -165,6 +185,13 @@ def flatten(self): path[ancestor.axis.name] = ancestor.value return path + def flatten_with_ancestors(self): + path = DatacubePath() + ancestors = self.ancestors + for ancestor in ancestors: + path[ancestor.axis.name] = ancestor.value + return path + def get_ancestors(self): ancestors = [] current_node = self From 03f5b17aaf29c498355a489dda6a36af03265119 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 20 Oct 2023 15:14:38 +0200 Subject: [PATCH 172/332] small optimisation --- polytope/engine/hullslicer.py | 9 +++++---- polytope/polytope.py | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 7b386cae8..1975f75b8 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -20,7 +20,7 @@ def __init__(self): pass def _unique_continuous_points(self, p: ConvexPolytope, datacube: Datacube): - for i, ax in enumerate(p.axes()): + for i, ax in enumerate(p._axes): mapper = datacube.get_mapper(ax) if isinstance(mapper, UnsliceableDatacubeAxis): break @@ -31,7 +31,7 @@ def _unique_continuous_points(self, p: ConvexPolytope, datacube: Datacube): unique(p.points) def _build_unsliceable_child(self, polytope, ax, node, datacube, lower, next_nodes): - if polytope.axes() != [ax.name]: + if polytope._axes != [ax.name]: raise UnsliceableShapeError(ax) path = node.flatten() if datacube.has_index(path, ax, lower): @@ -56,7 +56,8 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex # store the native type remapped_val = value if ax.is_cyclic: - remapped_val = (ax.remap([value, value])[0][0] + ax.remap([value, value])[0][1]) / 2 + remapped_val_interm = ax.remap([value, value])[0] + remapped_val = (remapped_val_interm[0] + remapped_val_interm[1]) / 2 remapped_val = round(remapped_val, int(-math.log10(ax.tol))) child = node.create_child(ax, remapped_val) child["unsliced_polytopes"] = copy(node["unsliced_polytopes"]) @@ -67,7 +68,7 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex def _build_branch(self, ax, node, datacube, next_nodes): for polytope in node["unsliced_polytopes"]: - if ax.name in polytope.axes(): + if ax.name in polytope._axes: lower, upper = polytope.extents(ax.name) # here, first check if the axis is an unsliceable axis and directly build node if it is if isinstance(ax, UnsliceableDatacubeAxis): diff --git a/polytope/polytope.py b/polytope/polytope.py index 5e8fc7133..8dea584d5 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -1,4 +1,5 @@ from typing import List +import time from .shapes import ConvexPolytope from .utility.exceptions import AxisOverdefinedError @@ -44,6 +45,9 @@ def slice(self, polytopes: List[ConvexPolytope]): def retrieve(self, request: Request, method="standard"): """Higher-level API which takes a request and uses it to slice the datacube""" + print("TIME IN POLYTOPE EXTRACT") + time0 = time.time() request_tree = self.engine.extract(self.datacube, request.polytopes()) + print(time.time() - time0) self.datacube.get(request_tree) return request_tree From 517591985aa55cd8807a6e9b89c48f44d4523f97 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 20 Oct 2023 15:23:14 +0200 Subject: [PATCH 173/332] store slice_axis_idx in hullslicer --- polytope/engine/hullslicer.py | 18 +++++++++--------- polytope/shapes.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 1975f75b8..00db17bbb 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -30,7 +30,7 @@ def _unique_continuous_points(self, p: ConvexPolytope, datacube: Datacube): # Remove duplicate points unique(p.points) - def _build_unsliceable_child(self, polytope, ax, node, datacube, lower, next_nodes): + def _build_unsliceable_child(self, polytope, ax, node, datacube, lower, next_nodes, slice_axis_idx): if polytope._axes != [ax.name]: raise UnsliceableShapeError(ax) path = node.flatten() @@ -43,7 +43,7 @@ def _build_unsliceable_child(self, polytope, ax, node, datacube, lower, next_nod # raise a value not found error raise ValueError() - def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, next_nodes): + def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, next_nodes, slice_axis_idx): tol = ax.tol lower = ax.from_float(lower - tol) upper = ax.from_float(upper + tol) @@ -52,7 +52,7 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex for value in datacube.get_indices(flattened, ax, lower, upper, method): # convert to float for slicing fvalue = ax.to_float(value) - new_polytope = slice(polytope, ax.name, fvalue) + new_polytope = slice(polytope, ax.name, fvalue, slice_axis_idx) # store the native type remapped_val = value if ax.is_cyclic: @@ -69,12 +69,12 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex def _build_branch(self, ax, node, datacube, next_nodes): for polytope in node["unsliced_polytopes"]: if ax.name in polytope._axes: - lower, upper = polytope.extents(ax.name) + lower, upper, slice_axis_idx = polytope.extents(ax.name) # here, first check if the axis is an unsliceable axis and directly build node if it is if isinstance(ax, UnsliceableDatacubeAxis): - self._build_unsliceable_child(polytope, ax, node, datacube, lower, next_nodes) + self._build_unsliceable_child(polytope, ax, node, datacube, lower, next_nodes, slice_axis_idx) else: - self._build_sliceable_child(polytope, ax, node, datacube, lower, upper, next_nodes) + self._build_sliceable_child(polytope, ax, node, datacube, lower, upper, next_nodes, slice_axis_idx) del node["unsliced_polytopes"] def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): @@ -129,8 +129,8 @@ def _reduce_dimension(intersects, slice_axis_idx): return temp_intersects -def slice(polytope: ConvexPolytope, axis, value): - slice_axis_idx = polytope._axes.index(axis) +def slice(polytope: ConvexPolytope, axis, value, slice_axis_idx): + # slice_axis_idx = polytope._axes.index(axis) if len(polytope.points[0]) == 1: # Note that in this case, we do not need to do linear interpolation so we can save time @@ -147,7 +147,7 @@ def slice(polytope: ConvexPolytope, axis, value): # Reduce dimension of intersection points, removing slice axis intersects = _reduce_dimension(intersects, slice_axis_idx) - axes = [ax for ax in polytope.axes() if ax != axis] + axes = [ax for ax in polytope._axes if ax != axis] if len(intersects) < len(intersects[0]) + 1: return ConvexPolytope(axes, intersects) diff --git a/polytope/shapes.py b/polytope/shapes.py index 94562abc5..59ac39fd0 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -33,7 +33,7 @@ def extents(self, axis): axis_values = [point[slice_axis_idx] for point in self.points] lower = min(axis_values) upper = max(axis_values) - return (lower, upper) + return (lower, upper, slice_axis_idx) def __str__(self): return f"Polytope in {self.axes} with points {self.points}" From fb2b1104fb78b6265db7974af7e1fc00aebd341f Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 24 Oct 2023 11:56:09 +0200 Subject: [PATCH 174/332] try to make faster --- polytope/datacube/backends/FDB_datacube.py | 15 +++++++++++++ polytope/engine/hullslicer.py | 25 ++++++++++++++++++---- polytope/utility/combinatorics.py | 2 +- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index fcecda8cc..7e3f5ecc2 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -53,6 +53,9 @@ def get(self, requests: IndexTree): time_changing_path = 0 accumulated_fdb_time = 0 time_change_path = 0 + time_removing_branch = 0 + time_is_nan = 0 + interm_time = 0 for r in requests.leaves_with_ancestors: time5 = time.time() # NOTE: Accumulated time in flatten is 0.14s... could be better? @@ -70,6 +73,7 @@ def get(self, requests: IndexTree): axis = self._axes[key] (path, unmapped_path) = axis.unmap_total_path_to_datacube(path, unmapped_path) time_changing_path += time.time() - time2 + time8 = time.time() path = self.fit_path(path) # merge path and unmapped path into a single path path.update(unmapped_path) @@ -80,16 +84,21 @@ def get(self, requests: IndexTree): fdb_request_key = path fdb_requests = [(fdb_request_key, [(fdb_request_val, fdb_request_val + 1)])] + interm_time += time.time() - time8 # need to request data from the fdb time1 = time.time() subxarray = self.fdb.extract(fdb_requests) accumulated_fdb_time += time.time() - time1 subxarray_output_tuple = subxarray[0][0] output_value = subxarray_output_tuple[0][0][0] + time7 = time.time() if not math.isnan(output_value): r.result = output_value + time_is_nan += time.time() - time7 else: + time6 = time.time() r.remove_branch() + time_removing_branch += time.time() - time6 print("FDB TIME") print(accumulated_fdb_time) print("GET TIME") @@ -98,6 +107,12 @@ def get(self, requests: IndexTree): print(time_change_path) print("TIME CHANGING PATH") print(time_changing_path) + print("TIME REMOVING BRANCHES") + print(time_removing_branch) + print("TIME IS NAN") + print(time_is_nan) + print("INTERM TIME") + print(interm_time) def datacube_natural_indexes(self, axis, subarray): indexes = subarray[axis.name] diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 00db17bbb..158f09273 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -1,6 +1,6 @@ import math from copy import copy -from itertools import chain +from itertools import chain, product from typing import List import time @@ -9,7 +9,7 @@ from ..datacube.backends.datacube import Datacube, IndexTree from ..datacube.datacube_axis import UnsliceableDatacubeAxis from ..shapes import ConvexPolytope -from ..utility.combinatorics import argmax, argmin, group, product, unique +from ..utility.combinatorics import argmax, argmin, group, tensor_product, unique from ..utility.exceptions import UnsliceableShapeError from ..utility.geometry import lerp from .engine import Engine @@ -85,7 +85,7 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): groups, input_axes = group(polytopes) datacube.validate(input_axes) request = IndexTree() - combinations = product(groups) + combinations = tensor_product(groups) for c in combinations: r = IndexTree() @@ -103,6 +103,12 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): def _find_intersects(polytope, slice_axis_idx, value): intersects = [] # Find all points above and below slice axis + # above_slice, below_slice = [], [] + # for p in polytope.points: + # if p[slice_axis_idx] >= value: + # above_slice.append(p) + # if p[slice_axis_idx] <= value: + # below_slice.append(p) above_slice = [p for p in polytope.points if p[slice_axis_idx] >= value] below_slice = [p for p in polytope.points if p[slice_axis_idx] <= value] @@ -118,6 +124,16 @@ def _find_intersects(polytope, slice_axis_idx, value): interp_coeff = (value - b[slice_axis_idx]) / (a[slice_axis_idx] - b[slice_axis_idx]) intersect = lerp(a, b, interp_coeff) intersects.append(intersect) + # for (a, b) in product(above_slice, below_slice): + # # edge is incident with slice plane, don't need these points + # if a[slice_axis_idx] == b[slice_axis_idx]: + # intersects.append(b) + # continue + + # # Linearly interpolate all coordinates of two points (a,b) of the polytope + # interp_coeff = (value - b[slice_axis_idx]) / (a[slice_axis_idx] - b[slice_axis_idx]) + # intersect = lerp(a, b, interp_coeff) + # intersects.append(intersect) return intersects @@ -164,7 +180,8 @@ def slice(polytope: ConvexPolytope, axis, value, slice_axis_idx): vertices = hull.vertices except scipy.spatial.qhull.QhullError as e: - if "input is less than" or "simplex is flat" in str(e): + # if "input is less than" or "simplex is flat" in str(e): + if "less than" or "flat" in str(e): return ConvexPolytope(axes, intersects) # Sliced result is simply the convex hull return ConvexPolytope(axes, [intersects[i] for i in vertices]) diff --git a/polytope/utility/combinatorics.py b/polytope/utility/combinatorics.py index 1a4a24a5b..9dc641084 100644 --- a/polytope/utility/combinatorics.py +++ b/polytope/utility/combinatorics.py @@ -18,7 +18,7 @@ def group(polytopes: List[ConvexPolytope]): return groups, concatenation -def product(groups): +def tensor_product(groups): # Compute the tensor product of polytope groups return list(itertools.product(*groups.values())) From 722d0f087819656b44486f1431c1e615d9d83ffa Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 24 Oct 2023 13:38:21 +0200 Subject: [PATCH 175/332] add a null transformation and see effect on performance --- performance/fdb_performance.py | 5 +++ polytope/datacube/datacube_axis.py | 39 +++++++++++++++++++ .../datacube_null_transformation.py | 19 +++++++++ .../datacube_transformations.py | 3 ++ 4 files changed, 66 insertions(+) create mode 100644 polytope/datacube/transformations/datacube_null_transformation.py diff --git a/performance/fdb_performance.py b/performance/fdb_performance.py index 9bcac25b7..d4fd0ac5a 100644 --- a/performance/fdb_performance.py +++ b/performance/fdb_performance.py @@ -20,6 +20,11 @@ def setup_method(self, method): }, "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, + # "latitude": {"transformation": {"null": []}}, + # "longitude": {"transformation": {"null": []}}, + # "class": {"transformation": {"null": []}}, + # "param": {"transformation": {"null": []}}, + # "stream": {"transformation": {"null": []}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 07edd8d40..0efbefa9b 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -559,6 +559,45 @@ def remap(range): return cls +def null(cls): + + if cls.type_change: + old_find_indexes = cls.find_indexes + + def find_indexes(path, datacube): + return old_find_indexes(path, datacube) + + old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube + + def unmap_total_path_to_datacube(path, unmapped_path): + return old_unmap_total_path_to_datacube(path, unmapped_path) + + def unmap_to_datacube(path, unmapped_path): + return (path, unmapped_path) + + def remap_to_requested(path, unmapped_path): + return (path, unmapped_path) + + def find_indices_between(index_ranges, low, up, datacube, method=None): + indexes_between_ranges = [] + for indexes in index_ranges: + indexes_between = [i for i in indexes if low <= i <= up] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + def remap(range): + return [range] + + cls.remap = remap + cls.find_indexes = find_indexes + cls.unmap_to_datacube = unmap_to_datacube + cls.remap_to_requested = remap_to_requested + cls.find_indices_between = find_indices_between + cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube + + return cls + + class DatacubeAxis(ABC): is_cyclic = False has_mapper = False diff --git a/polytope/datacube/transformations/datacube_null_transformation.py b/polytope/datacube/transformations/datacube_null_transformation.py new file mode 100644 index 000000000..ef026428c --- /dev/null +++ b/polytope/datacube/transformations/datacube_null_transformation.py @@ -0,0 +1,19 @@ +from .datacube_transformations import DatacubeAxisTransformation + + +class DatacubeNullTransformation(DatacubeAxisTransformation): + def __init__(self, name, mapper_options): + self.name = name + self.transformation_options = mapper_options + + def generate_final_transformation(self): + return self + + def transformation_axes_final(self): + return [self.name] + + def change_val_type(self, axis_name, values): + return values + + def blocked_axes(self): + return [] diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index f4ce357a9..900ad16b6 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -50,6 +50,7 @@ def change_val_type(self, axis_name, values): "merge": "DatacubeAxisMerger", "reverse": "DatacubeAxisReverse", "type_change": "DatacubeAxisTypeChange", + "null": "DatacubeNullTransformation", } _type_to_transformation_file_lookup = { @@ -58,6 +59,7 @@ def change_val_type(self, axis_name, values): "merge": "merger", "reverse": "reverse", "type_change": "type_change", + "null": "null_transformation", } has_transform = { @@ -66,4 +68,5 @@ def change_val_type(self, axis_name, values): "merge": "has_merger", "reverse": "reorder", "type_change": "type_change", + "null": "null", } From a225c41cc005f105f67fa0c0c2f59d4582d95d86 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 25 Oct 2023 13:32:27 +0200 Subject: [PATCH 176/332] small optimisation --- polytope/datacube/backends/datacube.py | 1 + polytope/datacube/backends/xarray.py | 2 + polytope/datacube/datacube_axis.py | 62 +++++++++++++------ .../transformations/datacube_mappers.py | 11 ++-- .../transformations/datacube_merger.py | 4 +- tests/test_combinatorics.py | 4 +- tests/test_datacube_axes_init.py | 6 +- tests/test_hull_slicer.py | 16 ++--- 8 files changed, 69 insertions(+), 37 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index f6ffb041f..c9e8854b6 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -39,6 +39,7 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt self.blocked_axes.append(blocked_axis) for axis_name in final_axis_names: self.complete_axes.append(axis_name) + self.fake_axes.append(axis_name) # if axis does not yet exist, create it # first need to change the values so that we have right type diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 321f0f644..18ec06aa2 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -66,6 +66,8 @@ def get(self, requests: IndexTree): r.remove_branch() def datacube_natural_indexes(self, axis, subarray): + # if axis.name in self.fake_axes: + # indexes = subarray[axis.name].values if axis.name in self.complete_axes: indexes = next(iter(subarray.xindexes.values())).to_pandas_index() else: diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 0efbefa9b..2505c6f09 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -2,6 +2,7 @@ from copy import deepcopy from typing import Any, List import time +import bisect import numpy as np import pandas as pd @@ -249,8 +250,14 @@ def unmap_total_path_to_datacube(path, unmapped_path): if cls.name == transformation._mapped_axes()[0]: # axis = cls.name # if we are on the first axis, then need to add the first val to unmapped_path - first_val = path.get(cls.name, None) - path.pop(cls.name, None) + # if cls.name in path: + # first_val = path.get(cls.name, None) + # path.pop(cls.name, None) + first_val = path[cls.name] + del path[cls.name] + # else: + # first_val = None + if unmapped_path is None: unmapped_path[cls.name] = first_val elif cls.name not in unmapped_path: @@ -262,11 +269,14 @@ def unmap_total_path_to_datacube(path, unmapped_path): # axis = cls.name # print("TIME time handling path") # time2 = time.time() - second_val = path.get(cls.name, None) - path.pop(cls.name, None) + # second_val = path.get(cls.name, None) + # path.pop(cls.name, None) + second_val = path[cls.name] + del path[cls.name] first_val = unmapped_path.get(transformation._mapped_axes()[0], None) unmapped_path.pop(transformation._mapped_axes()[0], None) + # del unmapped_path[transformation._mapped_axes()[0]] # print(time.time() - time2) # NOTE: here we first calculate the starting idx of the first_val grid line # and then append the second_idx to get the final unmapped_idx @@ -279,11 +289,11 @@ def unmap_total_path_to_datacube(path, unmapped_path): if first_val is None: first_val = path.get(transformation._mapped_axes()[0], None) path.pop(transformation._mapped_axes()[0], None) - if second_val is not None: + # if second_val is not None: # time4 = time.time() # unmapped_idx = first_line_idx + second_idx - unmapped_idx = transformation.unmap(first_val, second_val) - unmapped_path[transformation.old_axis] = unmapped_idx + unmapped_idx = transformation.unmap(first_val, second_val) + unmapped_path[transformation.old_axis] = unmapped_idx # print(time.time() - time4) # time3 = time.time() # print("MAPPER UNMAP TIME") @@ -313,6 +323,10 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between = idxs[start:end] indexes_between_ranges.append(indexes_between) else: + # lower_idx = bisect.bisect_left(idxs, low) + # upper_idx = bisect.bisect_right(idxs, up) + # indexes_between = idxs[lower_idx: upper_idx] + # print(indexes_between) indexes_between = [i for i in idxs if low <= i <= up] indexes_between_ranges.append(indexes_between) return indexes_between_ranges @@ -337,12 +351,14 @@ def merge(cls): if cls.has_merger: + cls_name = cls.name + def find_indexes(path, datacube): # first, find the relevant transformation object that is a mapping in the cls.transformation dico for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): transformation = transform - if cls.name == transformation._first_axis: + if cls_name == transformation._first_axis: return transformation.merged_values(datacube) old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube @@ -352,10 +368,11 @@ def unmap_total_path_to_datacube(path, unmapped_path): for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): transformation = transform - if cls.name == transformation._first_axis: - old_val = path.get(cls.name, None) + if cls_name == transformation._first_axis: + # old_val = path.get(cls.name, None) + old_val = path[cls_name] (first_val, second_val) = transformation.unmerge(old_val) - path.pop(cls.name, None) + # path.pop(cls.name, None) path[transformation._first_axis] = first_val path[transformation._second_axis] = second_val return (path, unmapped_path) @@ -367,10 +384,10 @@ def unmap_to_datacube(path, unmapped_path): for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): transformation = transform - if cls.name == transformation._first_axis: - old_val = path.get(cls.name, None) + if cls_name == transformation._first_axis: + old_val = path.get(cls_name, None) (first_val, second_val) = transformation.unmerge(old_val) - path.pop(cls.name, None) + path.pop(cls_name, None) path[transformation._first_axis] = first_val path[transformation._second_axis] = second_val return (path, unmapped_path) @@ -384,7 +401,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): transformation = transform - if cls.name in transformation._mapped_axes(): + if cls_name in transformation._mapped_axes(): for indexes in index_ranges: if method == "surrounding": start = indexes.index(low) @@ -394,7 +411,10 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) else: - indexes_between = [i for i in indexes if low <= i <= up] + lower_idx = bisect.bisect_left(indexes, low) + upper_idx = bisect.bisect_right(indexes, up) + indexes_between = indexes[lower_idx: upper_idx] + # indexes_between = [i for i in indexes if low <= i <= up] indexes_between_ranges.append(indexes_between) return indexes_between_ranges @@ -466,7 +486,10 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) else: - indexes_between = [i for i in indexes if low <= i <= up] + # indexes_between = [i for i in indexes if low <= i <= up] + lower_idx = bisect.bisect_left(indexes, low) + upper_idx = bisect.bisect_right(indexes, up) + indexes_between = indexes[lower_idx: upper_idx] indexes_between_ranges.append(indexes_between) return indexes_between_ranges @@ -542,7 +565,10 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) else: - indexes_between = [i for i in indexes if low <= i <= up] + # indexes_between = [i for i in indexes if low <= i <= up] + lower_idx = bisect.bisect_left(indexes, low) + upper_idx = bisect.bisect_right(indexes, up) + indexes_between = indexes[lower_idx: upper_idx] indexes_between_ranges.append(indexes_between) return indexes_between_ranges diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index a26a0df6f..ec4b7d70b 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -3,10 +3,6 @@ from importlib import import_module from ...utility.list_tools import bisect_left_cmp, bisect_right_cmp import bisect -import time -import array -import numpy as np -from sortedcontainers import SortedDict, SortedList from .datacube_transformations import DatacubeAxisTransformation @@ -21,6 +17,7 @@ def __init__(self, name, mapper_options): self.grid_axes = mapper_options["axes"] self.old_axis = name self._final_transformation = self.generate_final_transformation() + self._final_mapped_axes = self._final_transformation._mapped_axes def generate_final_transformation(self): map_type = _type_to_datacube_mapper_lookup[self.grid_type] @@ -34,7 +31,8 @@ def blocked_axes(self): def transformation_axes_final(self): # final_transformation = self.generate_final_transformation() - final_axes = self._final_transformation._mapped_axes + # final_axes = self._final_transformation._mapped_axes + final_axes = self._final_mapped_axes return final_axes # Needs to also implement its own methods @@ -46,7 +44,8 @@ def change_val_type(self, axis_name, values): def _mapped_axes(self): # NOTE: Each of the mapper method needs to call it's sub mapper method # final_transformation = self.generate_final_transformation() - final_axes = self._final_transformation._mapped_axes + # final_axes = self._final_transformation._mapped_axes + final_axes = self._final_mapped_axes return final_axes def _base_axis(self): diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 288875135..4326d202b 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -1,6 +1,6 @@ import numpy as np import pandas as pd -import time +# import time from .datacube_transformations import DatacubeAxisTransformation @@ -31,7 +31,7 @@ def merged_values(self, datacube): for j in range(len(second_ax_vals)): second_val = second_ax_vals[j] # TODO: check that the first and second val are strings - val_to_add = pd.to_datetime(first_val + linkers[0] + second_val + linkers[1]) + val_to_add = pd.to_datetime("".join([first_val, linkers[0], second_val, linkers[1]])) val_to_add = val_to_add.to_numpy() val_to_add = val_to_add.astype("datetime64[s]") merged_values.append(val_to_add) diff --git a/tests/test_combinatorics.py b/tests/test_combinatorics.py index fab2fb2ca..980ad8e2c 100644 --- a/tests/test_combinatorics.py +++ b/tests/test_combinatorics.py @@ -2,7 +2,7 @@ import pytest from polytope import ConvexPolytope -from polytope.utility.combinatorics import group, product, validate_axes +from polytope.utility.combinatorics import group, tensor_product, validate_axes from polytope.utility.exceptions import ( AxisNotFoundError, AxisOverdefinedError, @@ -28,7 +28,7 @@ def test_group_and_product(self): assert len(groups[("a", "b")]) == 4 assert len(all_axes) == 4 - combinations = product(groups) + combinations = tensor_product(groups) assert len(combinations) == 4 for c in combinations: diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 9089de0c7..248b59ff4 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -23,7 +23,8 @@ def setup_method(self, method): "transformation": { "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} } - } + }, + # "latitude": {"transformation": {"reverse": {True}}}, } self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=self.options) @@ -35,7 +36,8 @@ def test_created_axes(self): assert self.datacube._axes["longitude"].has_mapper assert isinstance(self.datacube._axes["longitude"], FloatDatacubeAxis) assert not ("values" in self.datacube._axes.keys()) - assert self.datacube._axes["latitude"].find_indexes({}, self.datacube)[:5] == [ + print(list(self.datacube._axes["latitude"].find_indexes({}, self.datacube)[:5])) + assert list(self.datacube._axes["latitude"].find_indexes({}, self.datacube)[:5]) == [ 89.94618771566562, 89.87647835333229, 89.80635731954224, diff --git a/tests/test_hull_slicer.py b/tests/test_hull_slicer.py index 4e9404e23..e733612f7 100644 --- a/tests/test_hull_slicer.py +++ b/tests/test_hull_slicer.py @@ -20,38 +20,40 @@ def construct_nd_cube(self, dimension, lower=-1, upper=1): def test_3D(self): p3 = self.construct_nd_cube(3) print(p3) - p2 = polytope.engine.hullslicer.slice(p3, "c", 0.5) + p2 = polytope.engine.hullslicer.slice(p3, "c", 0.5, 2) print(p2) - p1 = polytope.engine.hullslicer.slice(p2, "b", 0.5) + p1 = polytope.engine.hullslicer.slice(p2, "b", 0.5, 1) print(p1) + @pytest.mark.skip(reason="This is too slow.") def test_4D(self): p = self.construct_nd_cube(4) print(p) while len(p.axes()) > 1: - p = polytope.engine.hullslicer.slice(p, p._axes[-1], 0.5) + p = polytope.engine.hullslicer.slice(p, p._axes[-1], 0.5, -1) print(p) + @pytest.mark.skip(reason="This is too slow.") def test_ND(self): with benchmark("4D"): p = self.construct_nd_cube(4) while len(p.axes()) > 1: - p = polytope.engine.hullslicer.slice(p, p._axes[-1], 0.5) + p = polytope.engine.hullslicer.slice(p, p._axes[-1], 0.5, -1) with benchmark("5D"): p = self.construct_nd_cube(5) while len(p.axes()) > 1: - p = polytope.engine.hullslicer.slice(p, p._axes[-1], 0.5) + p = polytope.engine.hullslicer.slice(p, p._axes[-1], 0.5, -1) with benchmark("6D"): p = self.construct_nd_cube(6) while len(p.axes()) > 1: - p = polytope.engine.hullslicer.slice(p, p._axes[-1], 0.5) + p = polytope.engine.hullslicer.slice(p, p._axes[-1], 0.5, -1) with benchmark("7D"): p = self.construct_nd_cube(7) while len(p.axes()) > 1: - p = polytope.engine.hullslicer.slice(p, p._axes[-1], 0.5) + p = polytope.engine.hullslicer.slice(p, p._axes[-1], 0.5, -1) # QHull is not performant above 7D as per its documentation # with benchmark("8D"): From dd5c2498052501bd48f2a499a558dfefea2712f4 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 26 Oct 2023 10:01:14 +0200 Subject: [PATCH 177/332] small optimisations --- polytope/datacube/backends/FDB_datacube.py | 9 +- polytope/datacube/backends/datacube.py | 1 + polytope/datacube/backends/xarray.py | 2 +- polytope/datacube/datacube_axis.py | 85 +++++++++---------- .../transformations/datacube_mappers.py | 7 +- 5 files changed, 52 insertions(+), 52 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 7e3f5ecc2..d25029e3d 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -49,20 +49,19 @@ def __init__(self, config={}, axis_options={}): def get(self, requests: IndexTree): # NOTE: this will do all the transformation unmappings for all the points # It doesn't use the tree structure of the result to do the unmapping transformations anymore - time0 = time.time() time_changing_path = 0 accumulated_fdb_time = 0 time_change_path = 0 - time_removing_branch = 0 time_is_nan = 0 interm_time = 0 + time0 = time.time() for r in requests.leaves_with_ancestors: time5 = time.time() # NOTE: Accumulated time in flatten is 0.14s... could be better? path = r.flatten_with_ancestors() # path = r.flatten() time_change_path += time.time() - time5 - path = self.remap_path(path) + # path = self.remap_path(path) if len(path.items()) == self.axis_counter: # first, find the grid mapper transform @@ -96,9 +95,7 @@ def get(self, requests: IndexTree): r.result = output_value time_is_nan += time.time() - time7 else: - time6 = time.time() r.remove_branch() - time_removing_branch += time.time() - time6 print("FDB TIME") print(accumulated_fdb_time) print("GET TIME") @@ -107,8 +104,6 @@ def get(self, requests: IndexTree): print(time_change_path) print("TIME CHANGING PATH") print(time_changing_path) - print("TIME REMOVING BRANCHES") - print(time_removing_branch) print("TIME IS NAN") print(time_is_nan) print("INTERM TIME") diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index c9e8854b6..b9bf40d54 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -2,6 +2,7 @@ import math from abc import ABC, abstractmethod from typing import Any +import time import xarray as xr diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 18ec06aa2..fdb7f087e 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -48,7 +48,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() - path = self.remap_path(path) + # path = self.remap_path(path) if len(path.items()) == self.axis_counter: # first, find the grid mapper transform unmapped_path = {} diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 2505c6f09..87181888e 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -3,6 +3,7 @@ from typing import Any, List import time import bisect +from ..utility.list_tools import bisect_left_cmp, bisect_right_cmp import numpy as np import pandas as pd @@ -213,28 +214,27 @@ def unmap_to_datacube(path, unmapped_path): (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeMapper): - transformation = transform - if cls.name == transformation._mapped_axes()[0]: + if cls.name == transform._mapped_axes()[0]: # if we are on the first axis, then need to add the first val to unmapped_path first_val = path.get(cls.name, None) path.pop(cls.name, None) if cls.name not in unmapped_path: # if for some reason, the unmapped_path already has the first axis val, then don't update unmapped_path[cls.name] = first_val - if cls.name == transformation._mapped_axes()[1]: + if cls.name == transform._mapped_axes()[1]: # if we are on the second axis, then the val of the first axis is stored # inside unmapped_path so can get it from there second_val = path.get(cls.name, None) path.pop(cls.name, None) - first_val = unmapped_path.get(transformation._mapped_axes()[0], None) - unmapped_path.pop(transformation._mapped_axes()[0], None) + first_val = unmapped_path.get(transform._mapped_axes()[0], None) + unmapped_path.pop(transform._mapped_axes()[0], None) # if the first_val was not in the unmapped_path, then it's still in path if first_val is None: - first_val = path.get(transformation._mapped_axes()[0], None) - path.pop(transformation._mapped_axes()[0], None) + first_val = path.get(transform._mapped_axes()[0], None) + path.pop(transform._mapped_axes()[0], None) if first_val is not None and second_val is not None: - unmapped_idx = transformation.unmap(first_val, second_val) - unmapped_path[transformation.old_axis] = unmapped_idx + unmapped_idx = transform.unmap(first_val, second_val) + unmapped_path[transform.old_axis] = unmapped_idx return (path, unmapped_path) old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube @@ -246,13 +246,9 @@ def unmap_total_path_to_datacube(path, unmapped_path): for transform in cls.transformations: if isinstance(transform, DatacubeMapper): # time2 = time.time() - transformation = transform - if cls.name == transformation._mapped_axes()[0]: + if cls.name == transform._mapped_axes()[0]: # axis = cls.name # if we are on the first axis, then need to add the first val to unmapped_path - # if cls.name in path: - # first_val = path.get(cls.name, None) - # path.pop(cls.name, None) first_val = path[cls.name] del path[cls.name] # else: @@ -263,7 +259,7 @@ def unmap_total_path_to_datacube(path, unmapped_path): elif cls.name not in unmapped_path: # if for some reason, the unmapped_path already has the first axis val, then don't update unmapped_path[cls.name] = first_val - if cls.name == transformation._mapped_axes()[1]: + if cls.name == transform._mapped_axes()[1]: # if we are on the second axis, then the val of the first axis is stored # inside unmapped_path so can get it from there # axis = cls.name @@ -273,30 +269,25 @@ def unmap_total_path_to_datacube(path, unmapped_path): # path.pop(cls.name, None) second_val = path[cls.name] del path[cls.name] - first_val = unmapped_path.get(transformation._mapped_axes()[0], None) + first_val = unmapped_path.get(transform._mapped_axes()[0], None) - unmapped_path.pop(transformation._mapped_axes()[0], None) + unmapped_path.pop(transform._mapped_axes()[0], None) # del unmapped_path[transformation._mapped_axes()[0]] # print(time.time() - time2) # NOTE: here we first calculate the starting idx of the first_val grid line # and then append the second_idx to get the final unmapped_idx # To do this, also need to find second_idx from second_val... - # second_idx = transformation.find_second_idx(first_val, second_val) - # first_line_idx = transformation.unmap_first_val_to_start_line_idx(first_val) # if the first_val was not in the unmapped_path, then it's still in path # print("AAAAAND TIME TAKEN DOING UNMAP") if first_val is None: - first_val = path.get(transformation._mapped_axes()[0], None) - path.pop(transformation._mapped_axes()[0], None) - # if second_val is not None: - # time4 = time.time() - # unmapped_idx = first_line_idx + second_idx - unmapped_idx = transformation.unmap(first_val, second_val) - unmapped_path[transformation.old_axis] = unmapped_idx - # print(time.time() - time4) + first_val = path.get(transform._mapped_axes()[0], None) + path.pop(transform._mapped_axes()[0], None) + unmapped_idx = transform.unmap(first_val, second_val) + unmapped_path[transform.old_axis] = unmapped_idx # time3 = time.time() # print("MAPPER UNMAP TIME") + # print(time.time() - time1) # print(time3 - time1) # print("AXIS THIS IS FOR") # print(axis) @@ -323,11 +314,17 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between = idxs[start:end] indexes_between_ranges.append(indexes_between) else: - # lower_idx = bisect.bisect_left(idxs, low) - # upper_idx = bisect.bisect_right(idxs, up) - # indexes_between = idxs[lower_idx: upper_idx] - # print(indexes_between) - indexes_between = [i for i in idxs if low <= i <= up] + axis_reversed = transform._axis_reversed[cls.name] + if not axis_reversed: + lower_idx = bisect.bisect_left(idxs, low) + upper_idx = bisect.bisect_right(idxs, up) + indexes_between = idxs[lower_idx: upper_idx] + else: + # TODO: do the custom bisect + end_idx = bisect_left_cmp(idxs, low, cmp=lambda x, y: x > y) + 1 + start_idx = bisect_right_cmp(idxs, up, cmp=lambda x, y: x > y) + indexes_between = idxs[start_idx:end_idx] + # indexes_between = [i for i in idxs if low <= i <= up] indexes_between_ranges.append(indexes_between) return indexes_between_ranges @@ -351,30 +348,29 @@ def merge(cls): if cls.has_merger: - cls_name = cls.name - def find_indexes(path, datacube): # first, find the relevant transformation object that is a mapping in the cls.transformation dico for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): transformation = transform - if cls_name == transformation._first_axis: + if cls.name == transformation._first_axis: return transformation.merged_values(datacube) old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube def unmap_total_path_to_datacube(path, unmapped_path): + # time1 = time.time() (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): transformation = transform - if cls_name == transformation._first_axis: - # old_val = path.get(cls.name, None) - old_val = path[cls_name] + if cls.name == transformation._first_axis: + old_val = path[cls.name] (first_val, second_val) = transformation.unmerge(old_val) - # path.pop(cls.name, None) path[transformation._first_axis] = first_val path[transformation._second_axis] = second_val + # print("UNMAPPER TIME INSIDE MERGE") + # print(time.time() - time1) return (path, unmapped_path) old_unmap_to_datacube = cls.unmap_to_datacube @@ -384,10 +380,10 @@ def unmap_to_datacube(path, unmapped_path): for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): transformation = transform - if cls_name == transformation._first_axis: - old_val = path.get(cls_name, None) + if cls.name == transformation._first_axis: + old_val = path.get(cls.name, None) (first_val, second_val) = transformation.unmerge(old_val) - path.pop(cls_name, None) + path.pop(cls.name, None) path[transformation._first_axis] = first_val path[transformation._second_axis] = second_val return (path, unmapped_path) @@ -401,7 +397,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): transformation = transform - if cls_name in transformation._mapped_axes(): + if cls.name in transformation._mapped_axes(): for indexes in index_ranges: if method == "surrounding": start = indexes.index(low) @@ -522,6 +518,7 @@ def find_indexes(path, datacube): old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube def unmap_total_path_to_datacube(path, unmapped_path): + # time1 = time.time() (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): @@ -532,6 +529,8 @@ def unmap_total_path_to_datacube(path, unmapped_path): if cls.name in path: path.pop(cls.name, None) unmapped_path[cls.name] = unchanged_val + # print("UNMAPPER TIME INSIDE TYPE_CHANGE") + # print(time.time() - time1) return (path, unmapped_path) def unmap_to_datacube(path, unmapped_path): diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index ec4b7d70b..947920e4e 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -18,6 +18,7 @@ def __init__(self, name, mapper_options): self.old_axis = name self._final_transformation = self.generate_final_transformation() self._final_mapped_axes = self._final_transformation._mapped_axes + self._axis_reversed = self._final_transformation._axis_reversed def generate_final_transformation(self): map_type = _type_to_datacube_mapper_lookup[self.grid_type] @@ -87,6 +88,7 @@ def __init__(self, base_axis, mapped_axes, resolution): self._base_axis = base_axis self._resolution = resolution self.deg_increment = 90 / self._resolution + self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} def first_axis_vals(self): first_ax_vals = [-90 + i * self.deg_increment for i in range(2 * self._resolution)] @@ -137,6 +139,7 @@ def __init__(self, base_axis, mapped_axes, resolution): self._mapped_axes = mapped_axes self._base_axis = base_axis self._resolution = resolution + self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} def first_axis_vals(self): rad2deg = 180 / math.pi @@ -255,8 +258,10 @@ def __init__(self, base_axis, mapped_axes, resolution): # self._inv_first_axis_vals = self._first_axis_vals[::-1] # self._inv_first_axis_vals = {v:k for k,v in self._first_axis_vals.items()} self._first_idx_map = self.create_first_idx_map() - self._second_axis_spacing = dict() + # self._second_axis_spacing = dict() + self._second_axis_spacing = {} # self.treated_first_vals = dict() + self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} def gauss_first_guess(self): i = 0 From b32ff76996436908802e09d45435ec82744afc2e Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 27 Oct 2023 12:57:27 +0200 Subject: [PATCH 178/332] make new recursive get --- polytope/datacube/backends/FDB_datacube.py | 58 ++++++++++++++++- polytope/datacube/backends/datacube.py | 2 + polytope/datacube/datacube_axis.py | 64 +++++++++++++++++++ .../transformations/datacube_cyclic.py | 3 + .../transformations/datacube_mappers.py | 3 + .../transformations/datacube_merger.py | 3 + .../datacube_null_transformation.py | 3 + .../transformations/datacube_reverse.py | 3 + .../transformations/datacube_type_change.py | 3 + 9 files changed, 141 insertions(+), 1 deletion(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index d25029e3d..7fdff8e8c 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -22,6 +22,7 @@ def __init__(self, config={}, axis_options={}): self.non_complete_axes = [] self.complete_axes = [] self.blocked_axes = [] + self.unwanted_axes = [] self.transformation = None self.fake_axes = [] @@ -46,7 +47,7 @@ def __init__(self, config={}, axis_options={}): val = self._axes[name].type self._check_and_add_axes(options, name, val) - def get(self, requests: IndexTree): + def get_old(self, requests: IndexTree): # NOTE: this will do all the transformation unmappings for all the points # It doesn't use the tree structure of the result to do the unmapping transformations anymore time_changing_path = 0 @@ -109,6 +110,61 @@ def get(self, requests: IndexTree): print("INTERM TIME") print(interm_time) + def remove_unwanted_axes(self, leaf_path): + for axis in self.unwanted_axes: + leaf_path.pop(axis) + return leaf_path + + def get(self, requests: IndexTree, leaf_path={}): + # NOTE: this will do all the transformation unmappings for all the points + # It doesn't use the tree structure of the result to do the unmapping transformations anymore + + # SECOND when request node is root, go to its children + # if requests.is_root(): + if requests.axis.name == "root": + if len(requests.children) == 0: + pass + else: + for c in requests.children: + self.get(c) + + # FIRST if request node has no children, we have a leaf so need to assign fdb values to it + # of course, before assigning results, we need to remap this last key too + else: + if len(requests.children) == 0: + # remap this last key + key_value_path = {requests.axis.name: requests.value} + ax = self._axes[requests.axis.name] + (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) + leaf_path.update(key_value_path) + leaf_path_copy = deepcopy(leaf_path) + leaf_path_copy = self.remove_unwanted_axes(leaf_path_copy) + # print("FINAL LEAF PATH") + # print(leaf_path) + output_value = self.find_fdb_values(leaf_path_copy) + if not math.isnan(output_value): + requests.result = output_value + + # THIRD TODO: otherwise remap the path for this key and iterate again over children + # NOTE: how do we remap for keys that are inside a mapping for multiple axes? + else: + key_value_path = {requests.axis.name: requests.value} + ax = self._axes[requests.axis.name] + (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) + leaf_path.update(key_value_path) + for c in requests.children: + self.get(c, leaf_path) + + def find_fdb_values(self, path): + fdb_request_val = path["values"] + path.pop("values") + fdb_request_key = path + fdb_requests = [(fdb_request_key, [(fdb_request_val, fdb_request_val + 1)])] + subxarray = self.fdb.extract(fdb_requests) + subxarray_output_tuple = subxarray[0][0] + output_value = subxarray_output_tuple[0][0][0] + return output_value + def datacube_natural_indexes(self, axis, subarray): indexes = subarray[axis.name] return indexes diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index b9bf40d54..a31bb947b 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -38,6 +38,8 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt ) for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) + for unwanted_axis in transformation.unwanted_axes(): + self.unwanted_axes.append(unwanted_axis) for axis_name in final_axis_names: self.complete_axes.append(axis_name) self.fake_axes.append(axis_name) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 87181888e..a09798862 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -134,6 +134,18 @@ def unmap_total_path_to_datacube(path, unmapped_path): print("CYCLIC UNMAP TIME") print(time.time() - time1) return (path, unmapped_path) + + old_unmap_path_key = cls.unmap_path_key + + def unmap_path_key(key_value_path, leaf_path): + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisCyclic): + if cls.name == transform.name: + new_val = _remap_val_to_axis_range(value) + key_value_path[cls.name] = new_val + key_value_path, leaf_path = old_unmap_path_key(key_value_path, leaf_path) + return (key_value_path, leaf_path) old_unmap_to_datacube = cls.unmap_to_datacube @@ -188,6 +200,7 @@ def offset(range): cls.unmap_to_datacube = unmap_to_datacube cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube cls.find_indices_between = find_indices_between + cls.unmap_path_key = unmap_path_key return cls @@ -294,6 +307,25 @@ def unmap_total_path_to_datacube(path, unmapped_path): # print("MAPPER TIME ONCE CHOSEN THE MAPPING") # print(time3 - time2) return (path, unmapped_path) + + old_unmap_path_key = cls.unmap_path_key + + def unmap_path_key(key_value_path, leaf_path): + key_value_path, leaf_path = old_unmap_path_key(key_value_path, leaf_path) + value = key_value_path[cls.name] + # print(cls.name) + # print(key_value_path) + for transform in cls.transformations: + if isinstance(transform, DatacubeMapper): + # if cls.name == transform._mapped_axes()[0]: + # unmapping_path[cls.name] = value + if cls.name == transform._mapped_axes()[1]: + first_val = leaf_path[transform._mapped_axes()[0]] + unmapped_idx = transform.unmap(first_val, value) + # leaf_path.pop(transform._mapped_axes()[0]) + key_value_path.pop(cls.name) + key_value_path[transform.old_axis] = unmapped_idx + return (key_value_path, leaf_path) def remap_to_requested(path, unmapped_path): return (path, unmapped_path) @@ -339,6 +371,7 @@ def remap(range): cls.remap_to_requested = remap_to_requested cls.find_indices_between = find_indices_between cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube + cls.unmap_path_key = unmap_path_key return cls @@ -372,6 +405,20 @@ def unmap_total_path_to_datacube(path, unmapped_path): # print("UNMAPPER TIME INSIDE MERGE") # print(time.time() - time1) return (path, unmapped_path) + + old_unmap_path_key = cls.unmap_path_key + + def unmap_path_key(key_value_path, leaf_path): + key_value_path, leaf_path = old_unmap_path_key(key_value_path, leaf_path) + new_key_value_path = {} + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisMerger): + if cls.name == transform._first_axis: + (first_val, second_val) = transform.unmerge(value) + new_key_value_path[transform._first_axis] = first_val + new_key_value_path[transform._second_axis] = second_val + return (new_key_value_path, leaf_path) old_unmap_to_datacube = cls.unmap_to_datacube @@ -423,6 +470,7 @@ def remap(range): cls.remap_to_requested = remap_to_requested cls.find_indices_between = find_indices_between cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube + cls.unmap_path_key = unmap_path_key return cls @@ -533,6 +581,18 @@ def unmap_total_path_to_datacube(path, unmapped_path): # print(time.time() - time1) return (path, unmapped_path) + old_unmap_path_key = cls.unmap_path_key + + def unmap_path_key(key_value_path, leaf_path): + key_value_path, leaf_path = old_unmap_path_key(key_value_path, leaf_path) + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisTypeChange): + if cls.name == transform.name: + unchanged_val = transform.make_str(value) + key_value_path[cls.name] = unchanged_val + return (key_value_path, leaf_path) + def unmap_to_datacube(path, unmapped_path): for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): @@ -580,6 +640,7 @@ def remap(range): cls.remap_to_requested = remap_to_requested cls.find_indices_between = find_indices_between cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube + cls.unmap_path_key = unmap_path_key return cls @@ -677,6 +738,9 @@ def offset(self, value): def unmap_total_path_to_datacube(self, path, unmapped_path): return (path, unmapped_path) + def unmap_path_key(self, key_value_path, leaf_path): + return (key_value_path, leaf_path) + def remap_to_requeest(path, unmapped_path): return (path, unmapped_path) diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index a48dedf3d..284a2f6ad 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -20,3 +20,6 @@ def change_val_type(self, axis_name, values): def blocked_axes(self): return [] + + def unwanted_axes(self): + return [] diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 947920e4e..f9a55a7ef 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -30,6 +30,9 @@ def generate_final_transformation(self): def blocked_axes(self): return [] + def unwanted_axes(self): + return [self._final_mapped_axes[0]] + def transformation_axes_final(self): # final_transformation = self.generate_final_transformation() # final_axes = self._final_transformation._mapped_axes diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 4326d202b..48250b20e 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -15,6 +15,9 @@ def __init__(self, name, merge_options): def blocked_axes(self): return [self._second_axis] + + def unwanted_axes(self): + return [] def merged_values(self, datacube): # time1 = time.time() diff --git a/polytope/datacube/transformations/datacube_null_transformation.py b/polytope/datacube/transformations/datacube_null_transformation.py index ef026428c..26f9ffaa4 100644 --- a/polytope/datacube/transformations/datacube_null_transformation.py +++ b/polytope/datacube/transformations/datacube_null_transformation.py @@ -17,3 +17,6 @@ def change_val_type(self, axis_name, values): def blocked_axes(self): return [] + + def unwanted_axes(self): + return [] diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index bda03c4a0..f0b1415b5 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -17,3 +17,6 @@ def change_val_type(self, axis_name, values): def blocked_axes(self): return [] + + def unwanted_axes(self): + return [] diff --git a/polytope/datacube/transformations/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change.py index 0f4cedd21..dde8ee346 100644 --- a/polytope/datacube/transformations/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change.py @@ -34,6 +34,9 @@ def make_str(self, value): def blocked_axes(self): return [] + + def unwanted_axes(self): + return [] class TypeChangeStrToInt(DatacubeAxisTypeChange): From 0deab2efdcd8b5a836f3b3f37c6dbc79b95f85c5 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 30 Oct 2023 09:20:55 +0100 Subject: [PATCH 179/332] time recursive get function --- polytope/datacube/backends/FDB_datacube.py | 3 +++ polytope/polytope.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 7fdff8e8c..76130bf0a 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -25,6 +25,7 @@ def __init__(self, config={}, axis_options={}): self.unwanted_axes = [] self.transformation = None self.fake_axes = [] + self.time_fdb = 0 partial_request = config # Find values in the level 3 FDB datacube @@ -141,7 +142,9 @@ def get(self, requests: IndexTree, leaf_path={}): leaf_path_copy = self.remove_unwanted_axes(leaf_path_copy) # print("FINAL LEAF PATH") # print(leaf_path) + time0 = time.time() output_value = self.find_fdb_values(leaf_path_copy) + self.time_fdb += time.time() - time0 if not math.isnan(output_value): requests.result = output_value diff --git a/polytope/polytope.py b/polytope/polytope.py index 8dea584d5..4e6a93645 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -49,5 +49,10 @@ def retrieve(self, request: Request, method="standard"): time0 = time.time() request_tree = self.engine.extract(self.datacube, request.polytopes()) print(time.time() - time0) + print("TIME INSIDE OF GET") + time1 = time.time() self.datacube.get(request_tree) + print(time.time() - time1) + print("TIME INSIDE FDB") + print(self.datacube.time_fdb) return request_tree From 9a3f3bc5fa10defe1df6c811b0a753fc1878baaf Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 30 Oct 2023 14:23:59 +0100 Subject: [PATCH 180/332] add scalability_plot --- performance/fdb_performance.py | 3 +- performance/fdb_scalability_plot.py | 12 ++++++++ polytope/datacube/backends/FDB_datacube.py | 33 +++++++++++++++------- polytope/datacube/backends/xarray.py | 3 ++ polytope/polytope.py | 4 +++ 5 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 performance/fdb_scalability_plot.py diff --git a/performance/fdb_performance.py b/performance/fdb_performance.py index d4fd0ac5a..37b3cb500 100644 --- a/performance/fdb_performance.py +++ b/performance/fdb_performance.py @@ -49,4 +49,5 @@ def test_fdb_datacube(self): time1 = time.time() result = self.API.retrieve(request) print(time.time() - time1) - assert len(result.leaves) == 19226 + print(len(result.leaves)) + # assert len(result.leaves) == 19226 diff --git a/performance/fdb_scalability_plot.py b/performance/fdb_scalability_plot.py new file mode 100644 index 000000000..a5c7bcb4f --- /dev/null +++ b/performance/fdb_scalability_plot.py @@ -0,0 +1,12 @@ +import matplotlib.pyplot as plt + +fdb_time = [7.6377081871032715 - 7.558288812637329, 73.57192325592041 - 72.99611115455627, 733.2706120014191 - 727.7059993743896, 4808.3157522678375 - 4770.814565420151] +num_extracted_points = [1986, 19226, 191543, 1267134] + +# for the 1.3M points, we used 100 latitudes too...., maybe that's why it's not as linear... + +# plt.xscale("log") +plt.plot(num_extracted_points, fdb_time, marker="o") +plt.xlabel("Number of extracted points") +plt.ylabel("Polytope extraction time (in s)") +plt.show() diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 76130bf0a..72b6c719c 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -26,6 +26,8 @@ def __init__(self, config={}, axis_options={}): self.transformation = None self.fake_axes = [] self.time_fdb = 0 + self.time_unmap_key = 0 + self.other_time = 0 partial_request = config # Find values in the level 3 FDB datacube @@ -132,29 +134,38 @@ def get(self, requests: IndexTree, leaf_path={}): # FIRST if request node has no children, we have a leaf so need to assign fdb values to it # of course, before assigning results, we need to remap this last key too else: + key_value_path = {requests.axis.name: requests.value} + # ax = self._axes[requests.axis.name] + ax = requests.axis + time1 = time.time() + (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) + self.time_unmap_key += time.time() - time1 + leaf_path.update(key_value_path) if len(requests.children) == 0: # remap this last key - key_value_path = {requests.axis.name: requests.value} - ax = self._axes[requests.axis.name] - (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) - leaf_path.update(key_value_path) + # key_value_path = {requests.axis.name: requests.value} + # ax = self._axes[requests.axis.name] + # (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) + # leaf_path.update(key_value_path) + time2 = time.time() leaf_path_copy = deepcopy(leaf_path) + self.other_time += time.time() - time2 leaf_path_copy = self.remove_unwanted_axes(leaf_path_copy) # print("FINAL LEAF PATH") # print(leaf_path) - time0 = time.time() + # time0 = time.time() output_value = self.find_fdb_values(leaf_path_copy) - self.time_fdb += time.time() - time0 + # self.time_fdb += time.time() - time0 if not math.isnan(output_value): requests.result = output_value # THIRD TODO: otherwise remap the path for this key and iterate again over children # NOTE: how do we remap for keys that are inside a mapping for multiple axes? else: - key_value_path = {requests.axis.name: requests.value} - ax = self._axes[requests.axis.name] - (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) - leaf_path.update(key_value_path) + # key_value_path = {requests.axis.name: requests.value} + # ax = self._axes[requests.axis.name] + # (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) + # leaf_path.update(key_value_path) for c in requests.children: self.get(c, leaf_path) @@ -163,7 +174,9 @@ def find_fdb_values(self, path): path.pop("values") fdb_request_key = path fdb_requests = [(fdb_request_key, [(fdb_request_val, fdb_request_val + 1)])] + time0 = time.time() subxarray = self.fdb.extract(fdb_requests) + self.time_fdb += time.time() - time0 subxarray_output_tuple = subxarray[0][0] output_value = subxarray_output_tuple[0][0][0] return output_value diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index fdb7f087e..84a4b793b 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -20,6 +20,9 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.blocked_axes = [] self.transformation = None self.fake_axes = [] + self.time_fdb = 0 + self.time_unmap_key = 0 + self.unwanted_axes = [] for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: options = axis_options.get(name, {}) diff --git a/polytope/polytope.py b/polytope/polytope.py index 4e6a93645..1f20beef3 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -55,4 +55,8 @@ def retrieve(self, request: Request, method="standard"): print(time.time() - time1) print("TIME INSIDE FDB") print(self.datacube.time_fdb) + print("TIME UNMAP KEY") + print(self.datacube.time_unmap_key) + print("TIME SPENT REMOVE UNNECESSARY PATH KEYS") + print(self.datacube.other_time) return request_tree From f2aa886e53fcd818c60a29bfe0772df9776550b7 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 2 Nov 2023 11:22:50 +0100 Subject: [PATCH 181/332] retrieve ranges of longitudes from FDB --- polytope/datacube/backends/FDB_datacube.py | 235 +++++++++++++++--- polytope/datacube/datacube_axis.py | 131 ++++++++++ .../transformations/datacube_mappers.py | 6 +- 3 files changed, 335 insertions(+), 37 deletions(-) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 72b6c719c..f95ddd146 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -28,6 +28,10 @@ def __init__(self, config={}, axis_options={}): self.time_fdb = 0 self.time_unmap_key = 0 self.other_time = 0 + self.final_path = {"class" : 0, "date": 0, "domain": 0, "expver": 0, "levtype": 0, "param": 0, + "step" : 0, "stream": 0, "time": 0, "type": 0, "values": 0} + # self.unwanted_path = {"latitude": 0} + self.unwanted_path = {} partial_request = config # Find values in the level 3 FDB datacube @@ -118,12 +122,8 @@ def remove_unwanted_axes(self, leaf_path): leaf_path.pop(axis) return leaf_path - def get(self, requests: IndexTree, leaf_path={}): - # NOTE: this will do all the transformation unmappings for all the points - # It doesn't use the tree structure of the result to do the unmapping transformations anymore - - # SECOND when request node is root, go to its children - # if requests.is_root(): + def older_get(self, requests: IndexTree, leaf_path={}): + # First when request node is root, go to its children if requests.axis.name == "root": if len(requests.children) == 0: pass @@ -131,55 +131,218 @@ def get(self, requests: IndexTree, leaf_path={}): for c in requests.children: self.get(c) - # FIRST if request node has no children, we have a leaf so need to assign fdb values to it - # of course, before assigning results, we need to remap this last key too + # Second if request node has no children, we have a leaf so need to assign fdb values to it else: + # time2 = time.time() key_value_path = {requests.axis.name: requests.value} - # ax = self._axes[requests.axis.name] + # self.other_time += time.time() - time2 ax = requests.axis time1 = time.time() - (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) + # (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) self.time_unmap_key += time.time() - time1 - leaf_path.update(key_value_path) + time2 = time.time() + leaf_path |= key_value_path + self.other_time += time.time() - time2 if len(requests.children) == 0: # remap this last key - # key_value_path = {requests.axis.name: requests.value} - # ax = self._axes[requests.axis.name] - # (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) - # leaf_path.update(key_value_path) - time2 = time.time() - leaf_path_copy = deepcopy(leaf_path) - self.other_time += time.time() - time2 - leaf_path_copy = self.remove_unwanted_axes(leaf_path_copy) - # print("FINAL LEAF PATH") - # print(leaf_path) - # time0 = time.time() - output_value = self.find_fdb_values(leaf_path_copy) - # self.time_fdb += time.time() - time0 + output_value = self.find_fdb_values(leaf_path) if not math.isnan(output_value): requests.result = output_value - # THIRD TODO: otherwise remap the path for this key and iterate again over children - # NOTE: how do we remap for keys that are inside a mapping for multiple axes? + # THIRD otherwise remap the path for this key and iterate again over children + else: + for c in requests.children: + self.get(c, leaf_path) + + def get(self, requests: IndexTree, leaf_path={}): + # First when request node is root, go to its children + if requests.axis.name == "root": + if len(requests.children) == 0: + pass + else: + for c in requests.children: + self.get(c) + + # Second if request node has no children, we have a leaf so need to assign fdb values to it + else: + # time2 = time.time() + key_value_path = {requests.axis.name: requests.value} + # self.other_time += time.time() - time2 + ax = requests.axis + time1 = time.time() + # (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) + self.time_unmap_key += time.time() - time1 + time2 = time.time() + leaf_path |= key_value_path + self.other_time += time.time() - time2 + if len(requests.children[0].children) == 0: + # remap this last key + self.get_last_layer_before_leaf(requests, leaf_path) + + # THIRD otherwise remap the path for this key and iterate again over children else: - # key_value_path = {requests.axis.name: requests.value} - # ax = self._axes[requests.axis.name] - # (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) - # leaf_path.update(key_value_path) for c in requests.children: self.get(c, leaf_path) + # def get_last_layer_before_leaf(self, requests, leaf_path={}): + # range_length = 1 + # current_start_idx = None + # fdb_range_nodes = [IndexTree.root] * 200 + # for c in requests.children: + # # now c are the leaves of the initial tree + # key_value_path = {c.axis.name: c.value} + # print(key_value_path) + # ax = c.axis + # (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) + # leaf_path |= key_value_path + # last_idx = key_value_path["values"] + # if current_start_idx is None: + # current_start_idx = last_idx + # else: + # if last_idx == current_start_idx + 1: + # range_length += 1 + # fdb_range_nodes[range_length-1] = c + # else: + # # here, we jump to another range, so we first extract the old values from the fdb, and then we reset range_length etc... + # self.give_fdb_val_to_node(leaf_path, range_length, current_start_idx, fdb_range_nodes) + # key_value_path = {c.axis.name: c.value} + # ax = c.axis + # (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) + # leaf_path |= key_value_path + # current_start_idx = key_value_path["values"] + # range_length = 1 + # fdb_range_nodes = [c] * 200 + # # need to extract the last ranges + # self.give_fdb_val_to_node(leaf_path, range_length, current_start_idx, fdb_range_nodes) + + def get_last_layer_before_leaf(self, requests, leaf_path={}): + range_length = 1 + current_start_idx = None + fdb_range_nodes = [IndexTree.root] * 200 + for c in requests.children: + # now c are the leaves of the initial tree + key_value_path = {c.axis.name: c.value} + # print(key_value_path) + ax = c.axis + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) + leaf_path |= key_value_path + last_idx = key_value_path["values"] + if current_start_idx is None: + current_start_idx = last_idx + fdb_range_nodes[range_length-1] = c + else: + # if last_idx == current_start_idx + 1: + # print((last_idx, current_start_idx+range_length)) + if last_idx == current_start_idx + range_length: + range_length += 1 + fdb_range_nodes[range_length-1] = c + else: + # here, we jump to another range, so we first extract the old values from the fdb, and then we reset range_length etc... + # print(range_length) + # print(current_start_idx) + self.give_fdb_val_to_node(leaf_path, range_length, current_start_idx, fdb_range_nodes) + key_value_path = {c.axis.name: c.value} + ax = c.axis + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) + leaf_path |= key_value_path + current_start_idx = key_value_path["values"] + range_length = 1 + fdb_range_nodes = [IndexTree.root] * 200 + # need to extract the last ranges + self.give_fdb_val_to_node(leaf_path, range_length, current_start_idx, fdb_range_nodes) + + def give_fdb_val_to_node(self, leaf_path, range_length, current_start_idx, fdb_range_nodes): + output_values = self.new_find_fdb_values(leaf_path, range_length, current_start_idx) + for i in range(len(fdb_range_nodes[:range_length])): + n = fdb_range_nodes[i] + n.result = output_values[i] + + # def get_last_layer_before_leaf(self, requests, leaf_path={}): + # range_lengths = [[1]*200]*200 + # current_start_idx = [[None]*200]*200 + # fdb_range_nodes = [[[IndexTree.root] * 200]*200]*200 + # requests_length = len(requests.children) + # j=0 + # # for c in requests.children: + # for i in range(len(requests.children)): + # c = requests.children[i] + # # now c are the leaves of the initial tree + # key_value_path = {c.axis.name: c.value} + # ax = c.axis + # (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) + # leaf_path |= key_value_path + # last_idx = key_value_path["values"] + # # print(last_idx) + # if current_start_idx[i][j] is None: + # current_start_idx[i][j] = last_idx + # # print("HERE") + # else: + # if last_idx == current_start_idx[i][j] + 1: + # range_lengths[i][j] += 1 + # fdb_range_nodes[i][j][range_lengths[i][j]-1] = c + # else: + # # here, we jump to another range, so we first extract the old values from the fdb, and then we reset range_length etc... + # # self.give_fdb_val_to_node(leaf_path, range_lengths, current_start_idx, fdb_range_nodes, requests_length) + # key_value_path = {c.axis.name: c.value} + # ax = c.axis + # (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) + # j += 1 + # leaf_path |= key_value_path + # current_start_idx[i][j] = key_value_path["values"] + # range_lengths[i][j] = 1 + # fdb_range_nodes[i][j] = [c] * 200 + # # need to extract the last ranges + # self.give_fdb_val_to_node(leaf_path, range_lengths, current_start_idx, fdb_range_nodes, requests_length) + + # def give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_range_nodes, requests_length): + # # print("RANGE LENGTHS") + # # print(range_lengths) + # # print("CURRENT START IDX") + # # print(current_start_idx) + # # TODO: change this to accommodate for several requests at once + # output_values = self.new_find_fdb_values(leaf_path, range_lengths, current_start_idx, requests_length) + # for j in range(requests_length): + # for i in range(range_lengths[j]): + # n = fdb_range_nodes[j][i] + # n.result = output_values[j][i] # TODO: is this true?? + def find_fdb_values(self, path): - fdb_request_val = path["values"] - path.pop("values") - fdb_request_key = path - fdb_requests = [(fdb_request_key, [(fdb_request_val, fdb_request_val + 1)])] + fdb_request_val = path.pop("values") + fdb_requests = [(path, [(fdb_request_val, fdb_request_val + 1)])] time0 = time.time() subxarray = self.fdb.extract(fdb_requests) self.time_fdb += time.time() - time0 - subxarray_output_tuple = subxarray[0][0] - output_value = subxarray_output_tuple[0][0][0] + output_value = subxarray[0][0][0][0][0] return output_value + + def new_find_fdb_values(self, path, range_length, current_start_idx): + fdb_request_val = path.pop("values") + # print((current_start_idx, current_start_idx + range_length + 1)) + fdb_requests = [(path, [(current_start_idx, current_start_idx + range_length + 1)])] + # fdb_requests = [(path, new_reqs)] + time0 = time.time() + subxarray = self.fdb.extract(fdb_requests) + self.time_fdb += time.time() - time0 + # output_value = subxarray[0][0][0][0][0] + output_values = subxarray[0][0][0][0] + return output_values + + # def new_find_fdb_values(self, path, range_lengths, current_start_idx, requests_length): + # fdb_request_val = path.pop("values") + # fdb_requests = [(path, [])] + # for j in range(requests_length): + # current_request_ranges = (current_start_idx[j], current_start_idx[j] + range_lengths[j]+1) + # # print(current_request_ranges) + # # fdb_requests = [(path, [(current_start_idx, current_start_idx + range_length + 1)])] + # fdb_requests[0][1].append(current_request_ranges) + # time0 = time.time() + # subxarray = self.fdb.extract(fdb_requests) + # self.time_fdb += time.time() - time0 + # # output_value = subxarray[0][0][0][0][0] + # output_values = subxarray[0][0][0] + # return output_values def datacube_natural_indexes(self, axis, subarray): indexes = subarray[axis.name] diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index a09798862..197e59a78 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -146,6 +146,30 @@ def unmap_path_key(key_value_path, leaf_path): key_value_path[cls.name] = new_val key_value_path, leaf_path = old_unmap_path_key(key_value_path, leaf_path) return (key_value_path, leaf_path) + + old_n_unmap_path_key = cls.n_unmap_path_key + + def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisCyclic): + if cls.name == transform.name: + new_val = _remap_val_to_axis_range(value) + key_value_path[cls.name] = new_val + key_value_path, leaf_path, unwanted_path = old_n_unmap_path_key(key_value_path, leaf_path, unwanted_path) + return (key_value_path, leaf_path, unwanted_path) + + old_new_unmap_path_key = cls.new_unmap_path_key + + def new_unmap_path_key(key_value_path, unwanted_path, final_path): + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisCyclic): + if cls.name == transform.name: + new_val = _remap_val_to_axis_range(value) + key_value_path[cls.name] = new_val + key_value_path, unwanted_path, final_path = old_new_unmap_path_key(key_value_path, unwanted_path, final_path) + return (key_value_path, unwanted_path, final_path) old_unmap_to_datacube = cls.unmap_to_datacube @@ -201,6 +225,8 @@ def offset(range): cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube cls.find_indices_between = find_indices_between cls.unmap_path_key = unmap_path_key + cls.new_unmap_path_key = new_unmap_path_key + cls.n_unmap_path_key = n_unmap_path_key return cls @@ -326,6 +352,46 @@ def unmap_path_key(key_value_path, leaf_path): key_value_path.pop(cls.name) key_value_path[transform.old_axis] = unmapped_idx return (key_value_path, leaf_path) + + old_n_unmap_path_key = cls.n_unmap_path_key + + def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): + key_value_path, leaf_path, unwanted_path = old_n_unmap_path_key(key_value_path, leaf_path, unwanted_path) + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeMapper): + if cls.name == transform._mapped_axes()[0]: + unwanted_val = key_value_path[transform._mapped_axes()[0]] + unwanted_path[cls.name] = unwanted_val + # leaf_path.pop(transform._mapped_axes()[0]) + if cls.name == transform._mapped_axes()[1]: + # first_val = leaf_path[transform._mapped_axes()[0]] + first_val = unwanted_path[transform._mapped_axes()[0]] + unmapped_idx = transform.unmap(first_val, value) + leaf_path.pop(transform._mapped_axes()[0], None) + key_value_path.pop(cls.name) + key_value_path[transform.old_axis] = unmapped_idx + return (key_value_path, leaf_path, unwanted_path) + + old_new_unmap_path_key = cls.new_unmap_path_key + + def new_unmap_path_key(key_value_path, unwanted_path, final_path): + # NOTE: think this doesn't work recursively because we update the final path already, which is sort of duplicated from the key-value_path... + key_value_path, unwanted_path, final_path = old_new_unmap_path_key(key_value_path, unwanted_path, final_path) + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeMapper): + if cls.name == transform._mapped_axes()[0]: + unwanted_val = key_value_path[transform._mapped_axes()[0]] + unwanted_path[cls.name] = unwanted_val + if cls.name == transform._mapped_axes()[1]: + # first_val = leaf_path[transform._mapped_axes()[0]] + first_val = unwanted_path[transform._mapped_axes()[0]] + unmapped_idx = transform.unmap(first_val, value) + # key_value_path.pop(cls.name) + # key_value_path[transform.old_axis] = unmapped_idx + final_path[transform.old_axis] = unmapped_idx + return (key_value_path, unwanted_path, final_path) def remap_to_requested(path, unmapped_path): return (path, unmapped_path) @@ -372,6 +438,8 @@ def remap(range): cls.find_indices_between = find_indices_between cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube cls.unmap_path_key = unmap_path_key + cls.new_unmap_path_key = new_unmap_path_key + cls.n_unmap_path_key = n_unmap_path_key return cls @@ -419,6 +487,36 @@ def unmap_path_key(key_value_path, leaf_path): new_key_value_path[transform._first_axis] = first_val new_key_value_path[transform._second_axis] = second_val return (new_key_value_path, leaf_path) + + old_n_unmap_path_key = cls.n_unmap_path_key + + def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): + key_value_path, leaf_path, unwanted_path = old_n_unmap_path_key(key_value_path, leaf_path, unwanted_path) + new_key_value_path = {} + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisMerger): + if cls.name == transform._first_axis: + (first_val, second_val) = transform.unmerge(value) + new_key_value_path[transform._first_axis] = first_val + new_key_value_path[transform._second_axis] = second_val + return (new_key_value_path, leaf_path, unwanted_path) + + old_new_unmap_path_key = cls.new_unmap_path_key + + def new_unmap_path_key(key_value_path, unwanted_path, final_path): + key_value_path, unwanted_path, final_path = old_new_unmap_path_key(key_value_path, unwanted_path, final_path) + # new_key_value_path = {} + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisMerger): + if cls.name == transform._first_axis: + (first_val, second_val) = transform.unmerge(value) + # new_key_value_path[transform._first_axis] = first_val + final_path[transform._first_axis] = first_val + # new_key_value_path[transform._second_axis] = second_val + final_path[transform._second_axis] = second_val + return (key_value_path, unwanted_path, final_path) old_unmap_to_datacube = cls.unmap_to_datacube @@ -471,6 +569,8 @@ def remap(range): cls.find_indices_between = find_indices_between cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube cls.unmap_path_key = unmap_path_key + cls.new_unmap_path_key = new_unmap_path_key + cls.n_unmap_path_key = n_unmap_path_key return cls @@ -592,6 +692,30 @@ def unmap_path_key(key_value_path, leaf_path): unchanged_val = transform.make_str(value) key_value_path[cls.name] = unchanged_val return (key_value_path, leaf_path) + + old_n_unmap_path_key = cls.n_unmap_path_key + + def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): + key_value_path, leaf_path, unwanted_path = old_n_unmap_path_key(key_value_path, leaf_path, unwanted_path) + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisTypeChange): + if cls.name == transform.name: + unchanged_val = transform.make_str(value) + key_value_path[cls.name] = unchanged_val + return (key_value_path, leaf_path, unwanted_path) + + old_new_unmap_path_key = cls.new_unmap_path_key + + def new_unmap_path_key(key_value_path, unwanted_path, final_path): + key_value_path, unwanted_path, final_path = old_new_unmap_path_key(key_value_path, unwanted_path, final_path) + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisTypeChange): + if cls.name == transform.name: + unchanged_val = transform.make_str(value) + key_value_path[cls.name] = unchanged_val + return (key_value_path, unwanted_path, final_path) def unmap_to_datacube(path, unmapped_path): for transform in cls.transformations: @@ -641,6 +765,7 @@ def remap(range): cls.find_indices_between = find_indices_between cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube cls.unmap_path_key = unmap_path_key + cls.n_unmap_path_key = n_unmap_path_key return cls @@ -740,6 +865,12 @@ def unmap_total_path_to_datacube(self, path, unmapped_path): def unmap_path_key(self, key_value_path, leaf_path): return (key_value_path, leaf_path) + + def n_unmap_path_key(self, key_value_path, leaf_path, unwanted_path): + return (key_value_path, leaf_path, unwanted_path) + + def new_unmap_path_key(self, key_value_path, leaf_path, unwanted_path): + return (key_value_path, leaf_path, unwanted_path) def remap_to_requeest(path, unmapped_path): return (path, unmapped_path) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index f9a55a7ef..07a53336d 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -3066,7 +3066,11 @@ def find_second_axis_idx(self, first_val, second_val): # self._second_axis_spacing[first_val] = (second_axis_spacing, first_idx) # else: # (second_axis_spacing, first_idx) = self._second_axis_spacing[first_val] - second_idx = int(second_val/second_axis_spacing) + tol = 1e-8 + if second_val/second_axis_spacing > int(second_val/second_axis_spacing) + 1 - tol: + second_idx = int(second_val/second_axis_spacing) + 1 + else: + second_idx = int(second_val/second_axis_spacing) return (first_idx, second_idx) def unmap(self, first_val, second_val): From 245d0b4f0c3cb1e0c0916fd54eab54939bb4f2cc Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 7 Nov 2023 10:27:46 +0000 Subject: [PATCH 182/332] make latlon requests to the fdb in a single path --- performance/fdb_performance.py | 4 + polytope/datacube/backends/FDB_datacube.py | 150 ++++++++++++++++++--- polytope/datacube/backends/xarray.py | 1 + polytope/datacube/datacube_axis.py | 4 +- polytope/polytope.py | 24 ++-- 5 files changed, 154 insertions(+), 29 deletions(-) diff --git a/performance/fdb_performance.py b/performance/fdb_performance.py index 37b3cb500..48853b231 100644 --- a/performance/fdb_performance.py +++ b/performance/fdb_performance.py @@ -45,9 +45,13 @@ def test_fdb_datacube(self): Select("stream", ["oper"]), Select("type", ["an"]), Box(["latitude", "longitude"], [0, 0], [10, 10]), + # Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) time1 = time.time() result = self.API.retrieve(request) + print("ENTIRE TIME") print(time.time() - time1) + print("FDB TIME") + print(self.fdbdatacube.time_fdb) print(len(result.leaves)) # assert len(result.leaves) == 19226 diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index f95ddd146..cde83b521 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -155,7 +155,7 @@ def older_get(self, requests: IndexTree, leaf_path={}): for c in requests.children: self.get(c, leaf_path) - def get(self, requests: IndexTree, leaf_path={}): + def lon_get(self, requests: IndexTree, leaf_path={}): # First when request node is root, go to its children if requests.axis.name == "root": if len(requests.children) == 0: @@ -186,6 +186,37 @@ def get(self, requests: IndexTree, leaf_path={}): for c in requests.children: self.get(c, leaf_path) + def get(self, requests: IndexTree, leaf_path={}): + # First when request node is root, go to its children + if requests.axis.name == "root": + if len(requests.children) == 0: + pass + else: + for c in requests.children: + self.get(c) + + # Second if request node has no children, we have a leaf so need to assign fdb values to it + else: + # time2 = time.time() + key_value_path = {requests.axis.name: requests.value} + # self.other_time += time.time() - time2 + ax = requests.axis + time1 = time.time() + # (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) + self.time_unmap_key += time.time() - time1 + time2 = time.time() + leaf_path |= key_value_path + self.other_time += time.time() - time2 + if len(requests.children[0].children[0].children) == 0: + # remap this last key + self.handle_last_before_last_layer(requests, leaf_path) + + # THIRD otherwise remap the path for this key and iterate again over children + else: + for c in requests.children: + self.get(c, leaf_path) + # def get_last_layer_before_leaf(self, requests, leaf_path={}): # range_length = 1 # current_start_idx = None @@ -217,41 +248,86 @@ def get(self, requests: IndexTree, leaf_path={}): # # need to extract the last ranges # self.give_fdb_val_to_node(leaf_path, range_length, current_start_idx, fdb_range_nodes) - def get_last_layer_before_leaf(self, requests, leaf_path={}): - range_length = 1 - current_start_idx = None - fdb_range_nodes = [IndexTree.root] * 200 + def handle_last_before_last_layer(self, requests, leaf_path={}): + range_lengths = [[1]*200]*200 + current_start_idxs = [[None]*200]*200 + fdb_node_ranges = [[[IndexTree.root]*200]*200]*200 + lat_length = len(requests.children) + # requests.pprint() + for i in range(len(requests.children)): + lat_child = requests.children[i] + range_length = deepcopy(range_lengths[i]) + current_start_idx = deepcopy(current_start_idxs[i]) + fdb_range_nodes = deepcopy(fdb_node_ranges[i]) + key_value_path = {lat_child.axis.name: lat_child.value} + # print(key_value_path) + ax = lat_child.axis + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) + leaf_path |= key_value_path + # leaf_path.pop("values", None) + (range_lengths[i], + current_start_idxs[i], + fdb_node_ranges[i]) = self.get_last_last_layer_before_leaf(lat_child, + leaf_path, + range_length, + current_start_idx, + fdb_range_nodes) + # print(current_start_idxs[i]) + # print(lat_length) + # TODO: now call get_last_last_layer_before_layer on lat_child and add the range_lengths etc to the big array + # TODO: fetch data from fdb using big arrays + self.new_give_fdb_val_to_node(leaf_path, range_lengths, current_start_idxs, fdb_node_ranges, lat_length) + + def get_last_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, fdb_range_n): + i = 0 + # range_length = range_l[i] + # current_start_idx = current_idx[i] + # fdb_range_nodes = fdb_range_n[i] + # TODO: need to build the range_lengths etc ranges... for c in requests.children: # now c are the leaves of the initial tree key_value_path = {c.axis.name: c.value} # print(key_value_path) + # print(key_value_path) ax = c.axis (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) + # print(key_value_path) + # print(current_idx[i]) leaf_path |= key_value_path + # print("INSIDE SECOND FUNCTION") + # print(leaf_path) last_idx = key_value_path["values"] - if current_start_idx is None: - current_start_idx = last_idx - fdb_range_nodes[range_length-1] = c + if current_idx[i] is None: + current_idx[i] = last_idx + fdb_range_n[i][range_l[i]-1] = c else: # if last_idx == current_start_idx + 1: # print((last_idx, current_start_idx+range_length)) - if last_idx == current_start_idx + range_length: - range_length += 1 - fdb_range_nodes[range_length-1] = c + if last_idx == current_idx[i] + range_l[i]: + range_l[i] += 1 + fdb_range_n[i][range_l[i]-1] = c else: + # print(key_value_path) + # print(last_idx) + # print(current_idx[i] + range_l[i]) # here, we jump to another range, so we first extract the old values from the fdb, and then we reset range_length etc... # print(range_length) # print(current_start_idx) - self.give_fdb_val_to_node(leaf_path, range_length, current_start_idx, fdb_range_nodes) + # self.give_fdb_val_to_node(leaf_path, range_l[i], current_idx[i], fdb_range_n[i]) key_value_path = {c.axis.name: c.value} ax = c.axis (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) leaf_path |= key_value_path + i += 1 current_start_idx = key_value_path["values"] - range_length = 1 - fdb_range_nodes = [IndexTree.root] * 200 + current_idx[i] = current_start_idx + # range_length = 1 + # fdb_range_nodes = [IndexTree.root] * 200 # need to extract the last ranges - self.give_fdb_val_to_node(leaf_path, range_length, current_start_idx, fdb_range_nodes) + # print(range_l) + # print(current_idx) + return (range_l, current_idx, fdb_range_n) + # self.give_fdb_val_to_node(leaf_path, range_length, current_start_idx, fdb_range_nodes) def give_fdb_val_to_node(self, leaf_path, range_length, current_start_idx, fdb_range_nodes): output_values = self.new_find_fdb_values(leaf_path, range_length, current_start_idx) @@ -308,6 +384,22 @@ def give_fdb_val_to_node(self, leaf_path, range_length, current_start_idx, fdb_r # n = fdb_range_nodes[j][i] # n.result = output_values[j][i] # TODO: is this true?? + def new_give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_range_nodes, lat_length): + # TODO: change this to accommodate for several requests at once + output_values = self.newest_find_fdb_values(leaf_path, range_lengths, current_start_idx, lat_length) + for j in range(lat_length - 1): + for i in range(len(range_lengths[j])): + if current_start_idx[j][i] is not None: + for k in range(range_lengths[j][i]): + # print(output_values) + n = fdb_range_nodes[j][i][k] + # print("NOW") + # print(i) + # print(j) + # print(k) + # print(output_values[j][0][0][i][k]) + n.result = output_values[j][0][0][i][k] # TODO: is this true?? + def find_fdb_values(self, path): fdb_request_val = path.pop("values") fdb_requests = [(path, [(fdb_request_val, fdb_request_val + 1)])] @@ -344,6 +436,34 @@ def new_find_fdb_values(self, path, range_length, current_start_idx): # output_values = subxarray[0][0][0] # return output_values + def newest_find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): + fdb_request_val = path.pop("values") + # fdb_requests = [(path, [])] + # fdb_requests = list(zip([path]*lat_length, [[]]*lat_length)) + fdb_requests = [] + # print(fdb_requests) + for i in range(lat_length): + # fdb_requests[i][1] = [] + interm_request_ranges = [] + for j in range(200): + if current_start_idx[i][j] is not None: + current_request_ranges = (current_start_idx[i][j], current_start_idx[i][j] + range_lengths[i][j]) + # print(current_request_ranges) + # fdb_requests = [(path, [(current_start_idx, current_start_idx + range_length + 1)])] + # print(current_request_ranges) + interm_request_ranges.append(current_request_ranges) + fdb_requests.append(tuple((path, interm_request_ranges))) + print(fdb_requests) + # print(fdb_requests) + # print("TIME EXTRACT") + time0 = time.time() + subxarray = self.fdb.extract(fdb_requests) + self.time_fdb += time.time() - time0 + # output_value = subxarray[0][0][0][0][0] + # print(subxarray) + output_values = subxarray + return output_values + def datacube_natural_indexes(self, axis, subarray): indexes = subarray[axis.name] return indexes diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 84a4b793b..3a2dd83cd 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -21,6 +21,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.transformation = None self.fake_axes = [] self.time_fdb = 0 + self.other_time = 0 self.time_unmap_key = 0 self.unwanted_axes = [] for name, values in dataarray.coords.variables.items(): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 197e59a78..090e0e326 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -131,8 +131,8 @@ def unmap_total_path_to_datacube(path, unmapped_path): new_val = _remap_val_to_axis_range(old_val) path[cls.name] = new_val (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) - print("CYCLIC UNMAP TIME") - print(time.time() - time1) + # print("CYCLIC UNMAP TIME") + # print(time.time() - time1) return (path, unmapped_path) old_unmap_path_key = cls.unmap_path_key diff --git a/polytope/polytope.py b/polytope/polytope.py index 1f20beef3..dcd46f407 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -45,18 +45,18 @@ def slice(self, polytopes: List[ConvexPolytope]): def retrieve(self, request: Request, method="standard"): """Higher-level API which takes a request and uses it to slice the datacube""" - print("TIME IN POLYTOPE EXTRACT") - time0 = time.time() + # print("TIME IN POLYTOPE EXTRACT") + # time0 = time.time() request_tree = self.engine.extract(self.datacube, request.polytopes()) - print(time.time() - time0) - print("TIME INSIDE OF GET") - time1 = time.time() + # print(time.time() - time0) + # print("TIME INSIDE OF GET") + # time1 = time.time() self.datacube.get(request_tree) - print(time.time() - time1) - print("TIME INSIDE FDB") - print(self.datacube.time_fdb) - print("TIME UNMAP KEY") - print(self.datacube.time_unmap_key) - print("TIME SPENT REMOVE UNNECESSARY PATH KEYS") - print(self.datacube.other_time) + # print(time.time() - time1) + # print("TIME INSIDE FDB") + # print(self.datacube.time_fdb) + # print("TIME UNMAP KEY") + # print(self.datacube.time_unmap_key) + # print("TIME SPENT REMOVE UNNECESSARY PATH KEYS") + # print(self.datacube.other_time) return request_tree From 15de427d32fd363ab917077bd86de7c1ea4c2f93 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 8 Nov 2023 09:07:34 +0000 Subject: [PATCH 183/332] request only 1 request to pyfdb --- performance/fdb_performance_3D.py | 56 ++++++++++++++++++++++ polytope/datacube/backends/FDB_datacube.py | 52 ++++++++++++++------ 2 files changed, 93 insertions(+), 15 deletions(-) create mode 100644 performance/fdb_performance_3D.py diff --git a/performance/fdb_performance_3D.py b/performance/fdb_performance_3D.py new file mode 100644 index 000000000..11f6ed197 --- /dev/null +++ b/performance/fdb_performance_3D.py @@ -0,0 +1,56 @@ +import time + +import pandas as pd +import pytest + +from polytope.datacube.backends.FDB_datacube import FDBDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select, Span + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + } + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + "levelist": {"transformation": {"type_change": "int"}}, + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + # print(self.fdbdatacube) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + # @pytest.mark.skip(reason="can't install fdb branch on CI") + def test_fdb_datacube(self): + request = Request( + # Select("step", [0]), + Span("step", 1, 15), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231102T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Box(["latitude", "longitude"], [0, 0], [3, 5]), + # Span("levelist", 1, 15) + # Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) + time1 = time.time() + result = self.API.retrieve(request) + print("ENTIRE TIME") + print(time.time() - time1) + print("FDB TIME") + print(self.fdbdatacube.time_fdb) + print(len(result.leaves)) + # assert len(result.leaves) == 19226 diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index cde83b521..3db63e7dd 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -386,19 +386,33 @@ def give_fdb_val_to_node(self, leaf_path, range_length, current_start_idx, fdb_r def new_give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_range_nodes, lat_length): # TODO: change this to accommodate for several requests at once - output_values = self.newest_find_fdb_values(leaf_path, range_lengths, current_start_idx, lat_length) - for j in range(lat_length - 1): + (output_values, original_indices) = self.newest_find_fdb_values(leaf_path, range_lengths, current_start_idx, lat_length) + new_fdb_range_nodes = [] + new_range_lengths = [] + for j in range(lat_length): for i in range(len(range_lengths[j])): if current_start_idx[j][i] is not None: - for k in range(range_lengths[j][i]): - # print(output_values) - n = fdb_range_nodes[j][i][k] - # print("NOW") - # print(i) - # print(j) - # print(k) - # print(output_values[j][0][0][i][k]) - n.result = output_values[j][0][0][i][k] # TODO: is this true?? + new_fdb_range_nodes.append(fdb_range_nodes[j][i]) + new_range_lengths.append(range_lengths[j][i]) + sorted_fdb_range_nodes = [new_fdb_range_nodes[i] for i in original_indices] + sorted_range_lengths = [new_range_lengths[i] for i in original_indices] + for i in range(len(sorted_fdb_range_nodes)): + for k in range(sorted_range_lengths[i]): + n = sorted_fdb_range_nodes[i][k] + n.result = output_values[0][0][0][i][k] + # for j in range(lat_length - 1): + # for i in range(len(range_lengths[j])): + # if current_start_idx[j][i] is not None: + # # TODO: need to use original_indices to unmap j and i to the right idxs + # for k in range(range_lengths[j][i]): + # # print(output_values) + # n = fdb_range_nodes[j][i][k] + # # print("NOW") + # # print(i) + # # print(j) + # # print(k) + # # print(output_values[j][0][0][i][k]) + # n.result = output_values[j][0][0][i][k] # TODO: is this true?? def find_fdb_values(self, path): fdb_request_val = path.pop("values") @@ -438,13 +452,15 @@ def new_find_fdb_values(self, path, range_length, current_start_idx): def newest_find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): fdb_request_val = path.pop("values") + # print("NOW INSIDE NEWEST FIND FDB VAL") + # print(path) # fdb_requests = [(path, [])] # fdb_requests = list(zip([path]*lat_length, [[]]*lat_length)) fdb_requests = [] # print(fdb_requests) + interm_request_ranges = [] for i in range(lat_length): # fdb_requests[i][1] = [] - interm_request_ranges = [] for j in range(200): if current_start_idx[i][j] is not None: current_request_ranges = (current_start_idx[i][j], current_start_idx[i][j] + range_lengths[i][j]) @@ -452,17 +468,23 @@ def newest_find_fdb_values(self, path, range_lengths, current_start_idx, lat_len # fdb_requests = [(path, [(current_start_idx, current_start_idx + range_length + 1)])] # print(current_request_ranges) interm_request_ranges.append(current_request_ranges) - fdb_requests.append(tuple((path, interm_request_ranges))) - print(fdb_requests) + request_ranges_with_idx = list(enumerate(interm_request_ranges)) + sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) + sorted_request_ranges = [item[1] for item in sorted_list] + original_indices = [item[0] for item in sorted_list] + fdb_requests.append(tuple((path, sorted_request_ranges))) + # print(fdb_requests) # print(fdb_requests) # print("TIME EXTRACT") + # print(fdb_requests) time0 = time.time() subxarray = self.fdb.extract(fdb_requests) self.time_fdb += time.time() - time0 # output_value = subxarray[0][0][0][0][0] # print(subxarray) output_values = subxarray - return output_values + # output_values = [[[[[0]*1000]*1000]*1000]] + return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): indexes = subarray[axis.name] From 56d978a0984d001be935ad94fcb15909bf0a21fd Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 10 Nov 2023 11:14:20 +0000 Subject: [PATCH 184/332] clean up --- performance/fdb_performance.py | 8 -------- performance/fdb_performance_3D.py | 6 ------ performance/fdb_scalability_plot.py | 4 ++-- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/performance/fdb_performance.py b/performance/fdb_performance.py index 48853b231..01d57b7c6 100644 --- a/performance/fdb_performance.py +++ b/performance/fdb_performance.py @@ -1,7 +1,6 @@ import time import pandas as pd -import pytest from polytope.datacube.backends.FDB_datacube import FDBDatacube from polytope.engine.hullslicer import HullSlicer @@ -20,11 +19,6 @@ def setup_method(self, method): }, "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, - # "latitude": {"transformation": {"null": []}}, - # "longitude": {"transformation": {"null": []}}, - # "class": {"transformation": {"null": []}}, - # "param": {"transformation": {"null": []}}, - # "stream": {"transformation": {"null": []}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -45,7 +39,6 @@ def test_fdb_datacube(self): Select("stream", ["oper"]), Select("type", ["an"]), Box(["latitude", "longitude"], [0, 0], [10, 10]), - # Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) time1 = time.time() result = self.API.retrieve(request) @@ -54,4 +47,3 @@ def test_fdb_datacube(self): print("FDB TIME") print(self.fdbdatacube.time_fdb) print(len(result.leaves)) - # assert len(result.leaves) == 19226 diff --git a/performance/fdb_performance_3D.py b/performance/fdb_performance_3D.py index 11f6ed197..b2f4a2bbe 100644 --- a/performance/fdb_performance_3D.py +++ b/performance/fdb_performance_3D.py @@ -1,7 +1,6 @@ import time import pandas as pd -import pytest from polytope.datacube.backends.FDB_datacube import FDBDatacube from polytope.engine.hullslicer import HullSlicer @@ -24,7 +23,6 @@ def setup_method(self, method): } self.config = {"class": "od", "expver": "0001", "levtype": "sfc"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) - # print(self.fdbdatacube) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) @@ -32,7 +30,6 @@ def setup_method(self, method): # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( - # Select("step", [0]), Span("step", 1, 15), Select("levtype", ["sfc"]), Select("date", [pd.Timestamp("20231102T000000")]), @@ -43,8 +40,6 @@ def test_fdb_datacube(self): Select("stream", ["oper"]), Select("type", ["fc"]), Box(["latitude", "longitude"], [0, 0], [3, 5]), - # Span("levelist", 1, 15) - # Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) time1 = time.time() result = self.API.retrieve(request) @@ -53,4 +48,3 @@ def test_fdb_datacube(self): print("FDB TIME") print(self.fdbdatacube.time_fdb) print(len(result.leaves)) - # assert len(result.leaves) == 19226 diff --git a/performance/fdb_scalability_plot.py b/performance/fdb_scalability_plot.py index a5c7bcb4f..13f2bd9b7 100644 --- a/performance/fdb_scalability_plot.py +++ b/performance/fdb_scalability_plot.py @@ -1,11 +1,11 @@ import matplotlib.pyplot as plt -fdb_time = [7.6377081871032715 - 7.558288812637329, 73.57192325592041 - 72.99611115455627, 733.2706120014191 - 727.7059993743896, 4808.3157522678375 - 4770.814565420151] +fdb_time = [7.6377081871032715 - 7.558288812637329, 73.57192325592041 - 72.99611115455627, + 733.2706120014191 - 727.7059993743896, 4808.3157522678375 - 4770.814565420151] num_extracted_points = [1986, 19226, 191543, 1267134] # for the 1.3M points, we used 100 latitudes too...., maybe that's why it's not as linear... -# plt.xscale("log") plt.plot(num_extracted_points, fdb_time, marker="o") plt.xlabel("Number of extracted points") plt.ylabel("Polytope extraction time (in s)") From 7a7a2ace2fbece5dffea6ba944be6157cb057688 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 10 Nov 2023 12:02:52 +0000 Subject: [PATCH 185/332] clean up --- performance/fdb_performance.py | 2 - performance/fdb_performance_3D.py | 2 - polytope/datacube/backends/FDB_datacube.py | 361 +----------------- polytope/datacube/backends/datacube.py | 1 - polytope/datacube/backends/xarray.py | 3 - polytope/datacube/datacube_axis.py | 195 +--------- .../transformations/datacube_cyclic.py | 2 +- .../transformations/datacube_mappers.py | 119 +----- .../transformations/datacube_merger.py | 14 +- .../datacube_null_transformation.py | 2 +- .../transformations/datacube_reverse.py | 2 +- .../transformations/datacube_type_change.py | 5 +- polytope/engine/hullslicer.py | 21 +- polytope/polytope.py | 13 - requirements_example.txt | 1 + tests/data/era5-levels-members.grib | 3 + tests/data/foo.grib | 3 + tests/data/healpix.grib | 3 + 18 files changed, 38 insertions(+), 714 deletions(-) create mode 100644 requirements_example.txt create mode 100644 tests/data/era5-levels-members.grib create mode 100644 tests/data/foo.grib create mode 100644 tests/data/healpix.grib diff --git a/performance/fdb_performance.py b/performance/fdb_performance.py index 01d57b7c6..4ee8715be 100644 --- a/performance/fdb_performance.py +++ b/performance/fdb_performance.py @@ -44,6 +44,4 @@ def test_fdb_datacube(self): result = self.API.retrieve(request) print("ENTIRE TIME") print(time.time() - time1) - print("FDB TIME") - print(self.fdbdatacube.time_fdb) print(len(result.leaves)) diff --git a/performance/fdb_performance_3D.py b/performance/fdb_performance_3D.py index b2f4a2bbe..2cfec94cf 100644 --- a/performance/fdb_performance_3D.py +++ b/performance/fdb_performance_3D.py @@ -45,6 +45,4 @@ def test_fdb_datacube(self): result = self.API.retrieve(request) print("ENTIRE TIME") print(time.time() - time1) - print("FDB TIME") - print(self.fdbdatacube.time_fdb) print(len(result.leaves)) diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 3db63e7dd..7326d6e4d 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -1,5 +1,3 @@ -import math -import time from copy import deepcopy import pyfdb @@ -25,12 +23,8 @@ def __init__(self, config={}, axis_options={}): self.unwanted_axes = [] self.transformation = None self.fake_axes = [] - self.time_fdb = 0 - self.time_unmap_key = 0 - self.other_time = 0 self.final_path = {"class" : 0, "date": 0, "domain": 0, "expver": 0, "levtype": 0, "param": 0, "step" : 0, "stream": 0, "time": 0, "type": 0, "values": 0} - # self.unwanted_path = {"latitude": 0} self.unwanted_path = {} partial_request = config @@ -54,138 +48,11 @@ def __init__(self, config={}, axis_options={}): val = self._axes[name].type self._check_and_add_axes(options, name, val) - def get_old(self, requests: IndexTree): - # NOTE: this will do all the transformation unmappings for all the points - # It doesn't use the tree structure of the result to do the unmapping transformations anymore - time_changing_path = 0 - accumulated_fdb_time = 0 - time_change_path = 0 - time_is_nan = 0 - interm_time = 0 - time0 = time.time() - for r in requests.leaves_with_ancestors: - time5 = time.time() - # NOTE: Accumulated time in flatten is 0.14s... could be better? - path = r.flatten_with_ancestors() - # path = r.flatten() - time_change_path += time.time() - time5 - # path = self.remap_path(path) - if len(path.items()) == self.axis_counter: - # first, find the grid mapper transform - - unmapped_path = {} - path_copy = deepcopy(path) - time2 = time.time() - for key in path_copy: - axis = self._axes[key] - (path, unmapped_path) = axis.unmap_total_path_to_datacube(path, unmapped_path) - time_changing_path += time.time() - time2 - time8 = time.time() - path = self.fit_path(path) - # merge path and unmapped path into a single path - path.update(unmapped_path) - - # fit request into something for pyfdb - fdb_request_val = path["values"] - path.pop("values") - fdb_request_key = path - - fdb_requests = [(fdb_request_key, [(fdb_request_val, fdb_request_val + 1)])] - interm_time += time.time() - time8 - # need to request data from the fdb - time1 = time.time() - subxarray = self.fdb.extract(fdb_requests) - accumulated_fdb_time += time.time() - time1 - subxarray_output_tuple = subxarray[0][0] - output_value = subxarray_output_tuple[0][0][0] - time7 = time.time() - if not math.isnan(output_value): - r.result = output_value - time_is_nan += time.time() - time7 - else: - r.remove_branch() - print("FDB TIME") - print(accumulated_fdb_time) - print("GET TIME") - print(time.time() - time0) - print("TIME FLATTEN PATH AND CHANGE PATH") - print(time_change_path) - print("TIME CHANGING PATH") - print(time_changing_path) - print("TIME IS NAN") - print(time_is_nan) - print("INTERM TIME") - print(interm_time) - def remove_unwanted_axes(self, leaf_path): for axis in self.unwanted_axes: leaf_path.pop(axis) return leaf_path - def older_get(self, requests: IndexTree, leaf_path={}): - # First when request node is root, go to its children - if requests.axis.name == "root": - if len(requests.children) == 0: - pass - else: - for c in requests.children: - self.get(c) - - # Second if request node has no children, we have a leaf so need to assign fdb values to it - else: - # time2 = time.time() - key_value_path = {requests.axis.name: requests.value} - # self.other_time += time.time() - time2 - ax = requests.axis - time1 = time.time() - # (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) - self.time_unmap_key += time.time() - time1 - time2 = time.time() - leaf_path |= key_value_path - self.other_time += time.time() - time2 - if len(requests.children) == 0: - # remap this last key - output_value = self.find_fdb_values(leaf_path) - if not math.isnan(output_value): - requests.result = output_value - - # THIRD otherwise remap the path for this key and iterate again over children - else: - for c in requests.children: - self.get(c, leaf_path) - - def lon_get(self, requests: IndexTree, leaf_path={}): - # First when request node is root, go to its children - if requests.axis.name == "root": - if len(requests.children) == 0: - pass - else: - for c in requests.children: - self.get(c) - - # Second if request node has no children, we have a leaf so need to assign fdb values to it - else: - # time2 = time.time() - key_value_path = {requests.axis.name: requests.value} - # self.other_time += time.time() - time2 - ax = requests.axis - time1 = time.time() - # (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) - self.time_unmap_key += time.time() - time1 - time2 = time.time() - leaf_path |= key_value_path - self.other_time += time.time() - time2 - if len(requests.children[0].children) == 0: - # remap this last key - self.get_last_layer_before_leaf(requests, leaf_path) - - # THIRD otherwise remap the path for this key and iterate again over children - else: - for c in requests.children: - self.get(c, leaf_path) - def get(self, requests: IndexTree, leaf_path={}): # First when request node is root, go to its children if requests.axis.name == "root": @@ -197,17 +64,11 @@ def get(self, requests: IndexTree, leaf_path={}): # Second if request node has no children, we have a leaf so need to assign fdb values to it else: - # time2 = time.time() key_value_path = {requests.axis.name: requests.value} - # self.other_time += time.time() - time2 ax = requests.axis - time1 = time.time() - # (key_value_path, leaf_path) = ax.unmap_path_key(key_value_path, leaf_path) - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) - self.time_unmap_key += time.time() - time1 - time2 = time.time() + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, + self.unwanted_path) leaf_path |= key_value_path - self.other_time += time.time() - time2 if len(requests.children[0].children[0].children) == 0: # remap this last key self.handle_last_before_last_layer(requests, leaf_path) @@ -217,176 +78,58 @@ def get(self, requests: IndexTree, leaf_path={}): for c in requests.children: self.get(c, leaf_path) - # def get_last_layer_before_leaf(self, requests, leaf_path={}): - # range_length = 1 - # current_start_idx = None - # fdb_range_nodes = [IndexTree.root] * 200 - # for c in requests.children: - # # now c are the leaves of the initial tree - # key_value_path = {c.axis.name: c.value} - # print(key_value_path) - # ax = c.axis - # (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) - # leaf_path |= key_value_path - # last_idx = key_value_path["values"] - # if current_start_idx is None: - # current_start_idx = last_idx - # else: - # if last_idx == current_start_idx + 1: - # range_length += 1 - # fdb_range_nodes[range_length-1] = c - # else: - # # here, we jump to another range, so we first extract the old values from the fdb, and then we reset range_length etc... - # self.give_fdb_val_to_node(leaf_path, range_length, current_start_idx, fdb_range_nodes) - # key_value_path = {c.axis.name: c.value} - # ax = c.axis - # (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) - # leaf_path |= key_value_path - # current_start_idx = key_value_path["values"] - # range_length = 1 - # fdb_range_nodes = [c] * 200 - # # need to extract the last ranges - # self.give_fdb_val_to_node(leaf_path, range_length, current_start_idx, fdb_range_nodes) - def handle_last_before_last_layer(self, requests, leaf_path={}): range_lengths = [[1]*200]*200 current_start_idxs = [[None]*200]*200 fdb_node_ranges = [[[IndexTree.root]*200]*200]*200 lat_length = len(requests.children) - # requests.pprint() for i in range(len(requests.children)): lat_child = requests.children[i] range_length = deepcopy(range_lengths[i]) current_start_idx = deepcopy(current_start_idxs[i]) fdb_range_nodes = deepcopy(fdb_node_ranges[i]) key_value_path = {lat_child.axis.name: lat_child.value} - # print(key_value_path) ax = lat_child.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, + self.unwanted_path) leaf_path |= key_value_path - # leaf_path.pop("values", None) (range_lengths[i], current_start_idxs[i], - fdb_node_ranges[i]) = self.get_last_last_layer_before_leaf(lat_child, - leaf_path, - range_length, - current_start_idx, - fdb_range_nodes) - # print(current_start_idxs[i]) - # print(lat_length) - # TODO: now call get_last_last_layer_before_layer on lat_child and add the range_lengths etc to the big array - # TODO: fetch data from fdb using big arrays - self.new_give_fdb_val_to_node(leaf_path, range_lengths, current_start_idxs, fdb_node_ranges, lat_length) + fdb_node_ranges[i]) = self.get_last_layer_before_leaf(lat_child, leaf_path, range_length, + current_start_idx, fdb_range_nodes) + self.give_fdb_val_to_node(leaf_path, range_lengths, current_start_idxs, fdb_node_ranges, lat_length) - def get_last_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, fdb_range_n): + def get_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, fdb_range_n): i = 0 - # range_length = range_l[i] - # current_start_idx = current_idx[i] - # fdb_range_nodes = fdb_range_n[i] - # TODO: need to build the range_lengths etc ranges... for c in requests.children: # now c are the leaves of the initial tree key_value_path = {c.axis.name: c.value} - # print(key_value_path) - # print(key_value_path) ax = c.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) - # print(key_value_path) - # print(current_idx[i]) + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, + self.unwanted_path) leaf_path |= key_value_path - # print("INSIDE SECOND FUNCTION") - # print(leaf_path) last_idx = key_value_path["values"] if current_idx[i] is None: current_idx[i] = last_idx fdb_range_n[i][range_l[i]-1] = c else: - # if last_idx == current_start_idx + 1: - # print((last_idx, current_start_idx+range_length)) if last_idx == current_idx[i] + range_l[i]: range_l[i] += 1 fdb_range_n[i][range_l[i]-1] = c else: - # print(key_value_path) - # print(last_idx) - # print(current_idx[i] + range_l[i]) - # here, we jump to another range, so we first extract the old values from the fdb, and then we reset range_length etc... - # print(range_length) - # print(current_start_idx) - # self.give_fdb_val_to_node(leaf_path, range_l[i], current_idx[i], fdb_range_n[i]) key_value_path = {c.axis.name: c.value} ax = c.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, + self.unwanted_path) leaf_path |= key_value_path i += 1 current_start_idx = key_value_path["values"] current_idx[i] = current_start_idx - # range_length = 1 - # fdb_range_nodes = [IndexTree.root] * 200 - # need to extract the last ranges - # print(range_l) - # print(current_idx) return (range_l, current_idx, fdb_range_n) - # self.give_fdb_val_to_node(leaf_path, range_length, current_start_idx, fdb_range_nodes) - - def give_fdb_val_to_node(self, leaf_path, range_length, current_start_idx, fdb_range_nodes): - output_values = self.new_find_fdb_values(leaf_path, range_length, current_start_idx) - for i in range(len(fdb_range_nodes[:range_length])): - n = fdb_range_nodes[i] - n.result = output_values[i] - # def get_last_layer_before_leaf(self, requests, leaf_path={}): - # range_lengths = [[1]*200]*200 - # current_start_idx = [[None]*200]*200 - # fdb_range_nodes = [[[IndexTree.root] * 200]*200]*200 - # requests_length = len(requests.children) - # j=0 - # # for c in requests.children: - # for i in range(len(requests.children)): - # c = requests.children[i] - # # now c are the leaves of the initial tree - # key_value_path = {c.axis.name: c.value} - # ax = c.axis - # (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) - # leaf_path |= key_value_path - # last_idx = key_value_path["values"] - # # print(last_idx) - # if current_start_idx[i][j] is None: - # current_start_idx[i][j] = last_idx - # # print("HERE") - # else: - # if last_idx == current_start_idx[i][j] + 1: - # range_lengths[i][j] += 1 - # fdb_range_nodes[i][j][range_lengths[i][j]-1] = c - # else: - # # here, we jump to another range, so we first extract the old values from the fdb, and then we reset range_length etc... - # # self.give_fdb_val_to_node(leaf_path, range_lengths, current_start_idx, fdb_range_nodes, requests_length) - # key_value_path = {c.axis.name: c.value} - # ax = c.axis - # (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, self.unwanted_path) - # j += 1 - # leaf_path |= key_value_path - # current_start_idx[i][j] = key_value_path["values"] - # range_lengths[i][j] = 1 - # fdb_range_nodes[i][j] = [c] * 200 - # # need to extract the last ranges - # self.give_fdb_val_to_node(leaf_path, range_lengths, current_start_idx, fdb_range_nodes, requests_length) - - # def give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_range_nodes, requests_length): - # # print("RANGE LENGTHS") - # # print(range_lengths) - # # print("CURRENT START IDX") - # # print(current_start_idx) - # # TODO: change this to accommodate for several requests at once - # output_values = self.new_find_fdb_values(leaf_path, range_lengths, current_start_idx, requests_length) - # for j in range(requests_length): - # for i in range(range_lengths[j]): - # n = fdb_range_nodes[j][i] - # n.result = output_values[j][i] # TODO: is this true?? - - def new_give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_range_nodes, lat_length): - # TODO: change this to accommodate for several requests at once - (output_values, original_indices) = self.newest_find_fdb_values(leaf_path, range_lengths, current_start_idx, lat_length) + def give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_range_nodes, lat_length): + (output_values, original_indices) = self.find_fdb_values(leaf_path, range_lengths, current_start_idx, + lat_length) new_fdb_range_nodes = [] new_range_lengths = [] for j in range(lat_length): @@ -400,90 +143,23 @@ def new_give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, for k in range(sorted_range_lengths[i]): n = sorted_fdb_range_nodes[i][k] n.result = output_values[0][0][0][i][k] - # for j in range(lat_length - 1): - # for i in range(len(range_lengths[j])): - # if current_start_idx[j][i] is not None: - # # TODO: need to use original_indices to unmap j and i to the right idxs - # for k in range(range_lengths[j][i]): - # # print(output_values) - # n = fdb_range_nodes[j][i][k] - # # print("NOW") - # # print(i) - # # print(j) - # # print(k) - # # print(output_values[j][0][0][i][k]) - # n.result = output_values[j][0][0][i][k] # TODO: is this true?? - - def find_fdb_values(self, path): - fdb_request_val = path.pop("values") - fdb_requests = [(path, [(fdb_request_val, fdb_request_val + 1)])] - time0 = time.time() - subxarray = self.fdb.extract(fdb_requests) - self.time_fdb += time.time() - time0 - output_value = subxarray[0][0][0][0][0] - return output_value - - def new_find_fdb_values(self, path, range_length, current_start_idx): - fdb_request_val = path.pop("values") - # print((current_start_idx, current_start_idx + range_length + 1)) - fdb_requests = [(path, [(current_start_idx, current_start_idx + range_length + 1)])] - # fdb_requests = [(path, new_reqs)] - time0 = time.time() - subxarray = self.fdb.extract(fdb_requests) - self.time_fdb += time.time() - time0 - # output_value = subxarray[0][0][0][0][0] - output_values = subxarray[0][0][0][0] - return output_values - - # def new_find_fdb_values(self, path, range_lengths, current_start_idx, requests_length): - # fdb_request_val = path.pop("values") - # fdb_requests = [(path, [])] - # for j in range(requests_length): - # current_request_ranges = (current_start_idx[j], current_start_idx[j] + range_lengths[j]+1) - # # print(current_request_ranges) - # # fdb_requests = [(path, [(current_start_idx, current_start_idx + range_length + 1)])] - # fdb_requests[0][1].append(current_request_ranges) - # time0 = time.time() - # subxarray = self.fdb.extract(fdb_requests) - # self.time_fdb += time.time() - time0 - # # output_value = subxarray[0][0][0][0][0] - # output_values = subxarray[0][0][0] - # return output_values - def newest_find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): - fdb_request_val = path.pop("values") - # print("NOW INSIDE NEWEST FIND FDB VAL") - # print(path) - # fdb_requests = [(path, [])] - # fdb_requests = list(zip([path]*lat_length, [[]]*lat_length)) + def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): + path.pop("values") fdb_requests = [] - # print(fdb_requests) interm_request_ranges = [] for i in range(lat_length): - # fdb_requests[i][1] = [] for j in range(200): if current_start_idx[i][j] is not None: current_request_ranges = (current_start_idx[i][j], current_start_idx[i][j] + range_lengths[i][j]) - # print(current_request_ranges) - # fdb_requests = [(path, [(current_start_idx, current_start_idx + range_length + 1)])] - # print(current_request_ranges) interm_request_ranges.append(current_request_ranges) request_ranges_with_idx = list(enumerate(interm_request_ranges)) sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) sorted_request_ranges = [item[1] for item in sorted_list] original_indices = [item[0] for item in sorted_list] fdb_requests.append(tuple((path, sorted_request_ranges))) - # print(fdb_requests) - # print(fdb_requests) - # print("TIME EXTRACT") - # print(fdb_requests) - time0 = time.time() subxarray = self.fdb.extract(fdb_requests) - self.time_fdb += time.time() - time0 - # output_value = subxarray[0][0][0][0][0] - # print(subxarray) output_values = subxarray - # output_values = [[[[[0]*1000]*1000]*1000]] return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): @@ -494,7 +170,4 @@ def select(self, path, unmapped_path): return self.dataarray def ax_vals(self, name): - # for _name, values in self.dataarray.items(): - # if _name == name: - # return values return self.dataarray.get(name, None) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index a31bb947b..10b5a6613 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -2,7 +2,6 @@ import math from abc import ABC, abstractmethod from typing import Any -import time import xarray as xr diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 3a2dd83cd..0d244764e 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -20,9 +20,6 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.blocked_axes = [] self.transformation = None self.fake_axes = [] - self.time_fdb = 0 - self.other_time = 0 - self.time_unmap_key = 0 self.unwanted_axes = [] for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 090e0e326..57a971bcb 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -1,13 +1,13 @@ +import bisect from abc import ABC, abstractmethod from copy import deepcopy from typing import Any, List -import time -import bisect -from ..utility.list_tools import bisect_left_cmp, bisect_right_cmp import numpy as np import pandas as pd +from ..utility.list_tools import bisect_left_cmp, bisect_right_cmp + def cyclic(cls): if cls.is_cyclic: @@ -121,7 +121,6 @@ def find_indexes(path, datacube): old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube def unmap_total_path_to_datacube(path, unmapped_path): - time1 = time.time() for transform in cls.transformations: if isinstance(transform, DatacubeAxisCyclic): transformation = transform @@ -131,22 +130,8 @@ def unmap_total_path_to_datacube(path, unmapped_path): new_val = _remap_val_to_axis_range(old_val) path[cls.name] = new_val (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) - # print("CYCLIC UNMAP TIME") - # print(time.time() - time1) return (path, unmapped_path) - - old_unmap_path_key = cls.unmap_path_key - def unmap_path_key(key_value_path, leaf_path): - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisCyclic): - if cls.name == transform.name: - new_val = _remap_val_to_axis_range(value) - key_value_path[cls.name] = new_val - key_value_path, leaf_path = old_unmap_path_key(key_value_path, leaf_path) - return (key_value_path, leaf_path) - old_n_unmap_path_key = cls.n_unmap_path_key def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): @@ -158,18 +143,6 @@ def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): key_value_path[cls.name] = new_val key_value_path, leaf_path, unwanted_path = old_n_unmap_path_key(key_value_path, leaf_path, unwanted_path) return (key_value_path, leaf_path, unwanted_path) - - old_new_unmap_path_key = cls.new_unmap_path_key - - def new_unmap_path_key(key_value_path, unwanted_path, final_path): - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisCyclic): - if cls.name == transform.name: - new_val = _remap_val_to_axis_range(value) - key_value_path[cls.name] = new_val - key_value_path, unwanted_path, final_path = old_new_unmap_path_key(key_value_path, unwanted_path, final_path) - return (key_value_path, unwanted_path, final_path) old_unmap_to_datacube = cls.unmap_to_datacube @@ -224,8 +197,6 @@ def offset(range): cls.unmap_to_datacube = unmap_to_datacube cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube cls.find_indices_between = find_indices_between - cls.unmap_path_key = unmap_path_key - cls.new_unmap_path_key = new_unmap_path_key cls.n_unmap_path_key = n_unmap_path_key return cls @@ -280,18 +251,13 @@ def unmap_to_datacube(path, unmapped_path): def unmap_total_path_to_datacube(path, unmapped_path): # TODO: to be faster, could just compute the first lat unmapped idx, and do +1 for each subsequent lat idx? - # time1 = time.time() (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeMapper): - # time2 = time.time() if cls.name == transform._mapped_axes()[0]: - # axis = cls.name # if we are on the first axis, then need to add the first val to unmapped_path first_val = path[cls.name] del path[cls.name] - # else: - # first_val = None if unmapped_path is None: unmapped_path[cls.name] = first_val @@ -301,58 +267,23 @@ def unmap_total_path_to_datacube(path, unmapped_path): if cls.name == transform._mapped_axes()[1]: # if we are on the second axis, then the val of the first axis is stored # inside unmapped_path so can get it from there - # axis = cls.name - # print("TIME time handling path") - # time2 = time.time() - # second_val = path.get(cls.name, None) - # path.pop(cls.name, None) second_val = path[cls.name] del path[cls.name] first_val = unmapped_path.get(transform._mapped_axes()[0], None) unmapped_path.pop(transform._mapped_axes()[0], None) - # del unmapped_path[transformation._mapped_axes()[0]] - # print(time.time() - time2) # NOTE: here we first calculate the starting idx of the first_val grid line # and then append the second_idx to get the final unmapped_idx # To do this, also need to find second_idx from second_val... # if the first_val was not in the unmapped_path, then it's still in path - # print("AAAAAND TIME TAKEN DOING UNMAP") if first_val is None: first_val = path.get(transform._mapped_axes()[0], None) path.pop(transform._mapped_axes()[0], None) unmapped_idx = transform.unmap(first_val, second_val) unmapped_path[transform.old_axis] = unmapped_idx - # time3 = time.time() - # print("MAPPER UNMAP TIME") - # print(time.time() - time1) - # print(time3 - time1) - # print("AXIS THIS IS FOR") - # print(axis) - # print("MAPPER TIME ONCE CHOSEN THE MAPPING") - # print(time3 - time2) return (path, unmapped_path) - - old_unmap_path_key = cls.unmap_path_key - def unmap_path_key(key_value_path, leaf_path): - key_value_path, leaf_path = old_unmap_path_key(key_value_path, leaf_path) - value = key_value_path[cls.name] - # print(cls.name) - # print(key_value_path) - for transform in cls.transformations: - if isinstance(transform, DatacubeMapper): - # if cls.name == transform._mapped_axes()[0]: - # unmapping_path[cls.name] = value - if cls.name == transform._mapped_axes()[1]: - first_val = leaf_path[transform._mapped_axes()[0]] - unmapped_idx = transform.unmap(first_val, value) - # leaf_path.pop(transform._mapped_axes()[0]) - key_value_path.pop(cls.name) - key_value_path[transform.old_axis] = unmapped_idx - return (key_value_path, leaf_path) - old_n_unmap_path_key = cls.n_unmap_path_key def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): @@ -363,38 +294,13 @@ def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): if cls.name == transform._mapped_axes()[0]: unwanted_val = key_value_path[transform._mapped_axes()[0]] unwanted_path[cls.name] = unwanted_val - # leaf_path.pop(transform._mapped_axes()[0]) if cls.name == transform._mapped_axes()[1]: - # first_val = leaf_path[transform._mapped_axes()[0]] first_val = unwanted_path[transform._mapped_axes()[0]] unmapped_idx = transform.unmap(first_val, value) leaf_path.pop(transform._mapped_axes()[0], None) key_value_path.pop(cls.name) key_value_path[transform.old_axis] = unmapped_idx return (key_value_path, leaf_path, unwanted_path) - - old_new_unmap_path_key = cls.new_unmap_path_key - - def new_unmap_path_key(key_value_path, unwanted_path, final_path): - # NOTE: think this doesn't work recursively because we update the final path already, which is sort of duplicated from the key-value_path... - key_value_path, unwanted_path, final_path = old_new_unmap_path_key(key_value_path, unwanted_path, final_path) - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeMapper): - if cls.name == transform._mapped_axes()[0]: - unwanted_val = key_value_path[transform._mapped_axes()[0]] - unwanted_path[cls.name] = unwanted_val - if cls.name == transform._mapped_axes()[1]: - # first_val = leaf_path[transform._mapped_axes()[0]] - first_val = unwanted_path[transform._mapped_axes()[0]] - unmapped_idx = transform.unmap(first_val, value) - # key_value_path.pop(cls.name) - # key_value_path[transform.old_axis] = unmapped_idx - final_path[transform.old_axis] = unmapped_idx - return (key_value_path, unwanted_path, final_path) - - def remap_to_requested(path, unmapped_path): - return (path, unmapped_path) def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping @@ -422,7 +328,6 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): end_idx = bisect_left_cmp(idxs, low, cmp=lambda x, y: x > y) + 1 start_idx = bisect_right_cmp(idxs, up, cmp=lambda x, y: x > y) indexes_between = idxs[start_idx:end_idx] - # indexes_between = [i for i in idxs if low <= i <= up] indexes_between_ranges.append(indexes_between) return indexes_between_ranges @@ -434,11 +339,8 @@ def remap(range): cls.remap = remap cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube - cls.remap_to_requested = remap_to_requested cls.find_indices_between = find_indices_between cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube - cls.unmap_path_key = unmap_path_key - cls.new_unmap_path_key = new_unmap_path_key cls.n_unmap_path_key = n_unmap_path_key return cls @@ -460,7 +362,6 @@ def find_indexes(path, datacube): old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube def unmap_total_path_to_datacube(path, unmapped_path): - # time1 = time.time() (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): @@ -470,24 +371,8 @@ def unmap_total_path_to_datacube(path, unmapped_path): (first_val, second_val) = transformation.unmerge(old_val) path[transformation._first_axis] = first_val path[transformation._second_axis] = second_val - # print("UNMAPPER TIME INSIDE MERGE") - # print(time.time() - time1) return (path, unmapped_path) - - old_unmap_path_key = cls.unmap_path_key - def unmap_path_key(key_value_path, leaf_path): - key_value_path, leaf_path = old_unmap_path_key(key_value_path, leaf_path) - new_key_value_path = {} - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - if cls.name == transform._first_axis: - (first_val, second_val) = transform.unmerge(value) - new_key_value_path[transform._first_axis] = first_val - new_key_value_path[transform._second_axis] = second_val - return (new_key_value_path, leaf_path) - old_n_unmap_path_key = cls.n_unmap_path_key def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): @@ -502,22 +387,6 @@ def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): new_key_value_path[transform._second_axis] = second_val return (new_key_value_path, leaf_path, unwanted_path) - old_new_unmap_path_key = cls.new_unmap_path_key - - def new_unmap_path_key(key_value_path, unwanted_path, final_path): - key_value_path, unwanted_path, final_path = old_new_unmap_path_key(key_value_path, unwanted_path, final_path) - # new_key_value_path = {} - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - if cls.name == transform._first_axis: - (first_val, second_val) = transform.unmerge(value) - # new_key_value_path[transform._first_axis] = first_val - final_path[transform._first_axis] = first_val - # new_key_value_path[transform._second_axis] = second_val - final_path[transform._second_axis] = second_val - return (key_value_path, unwanted_path, final_path) - old_unmap_to_datacube = cls.unmap_to_datacube def unmap_to_datacube(path, unmapped_path): @@ -533,9 +402,6 @@ def unmap_to_datacube(path, unmapped_path): path[transformation._second_axis] = second_val return (path, unmapped_path) - def remap_to_requested(path, unmapped_path): - return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] @@ -555,7 +421,6 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): lower_idx = bisect.bisect_left(indexes, low) upper_idx = bisect.bisect_right(indexes, up) indexes_between = indexes[lower_idx: upper_idx] - # indexes_between = [i for i in indexes if low <= i <= up] indexes_between_ranges.append(indexes_between) return indexes_between_ranges @@ -565,11 +430,8 @@ def remap(range): cls.remap = remap cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube - cls.remap_to_requested = remap_to_requested cls.find_indices_between = find_indices_between cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube - cls.unmap_path_key = unmap_path_key - cls.new_unmap_path_key = new_unmap_path_key cls.n_unmap_path_key = n_unmap_path_key return cls @@ -593,9 +455,6 @@ def find_indexes(path, datacube): def unmap_to_datacube(path, unmapped_path): return (path, unmapped_path) - def remap_to_requested(path, unmapped_path): - return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] @@ -630,7 +489,6 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) else: - # indexes_between = [i for i in indexes if low <= i <= up] lower_idx = bisect.bisect_left(indexes, low) upper_idx = bisect.bisect_right(indexes, up) indexes_between = indexes[lower_idx: upper_idx] @@ -643,7 +501,6 @@ def remap(range): cls.remap = remap cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube - cls.remap_to_requested = remap_to_requested cls.find_indices_between = find_indices_between return cls @@ -666,7 +523,6 @@ def find_indexes(path, datacube): old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube def unmap_total_path_to_datacube(path, unmapped_path): - # time1 = time.time() (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): @@ -677,22 +533,8 @@ def unmap_total_path_to_datacube(path, unmapped_path): if cls.name in path: path.pop(cls.name, None) unmapped_path[cls.name] = unchanged_val - # print("UNMAPPER TIME INSIDE TYPE_CHANGE") - # print(time.time() - time1) return (path, unmapped_path) - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path): - key_value_path, leaf_path = old_unmap_path_key(key_value_path, leaf_path) - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - if cls.name == transform.name: - unchanged_val = transform.make_str(value) - key_value_path[cls.name] = unchanged_val - return (key_value_path, leaf_path) - old_n_unmap_path_key = cls.n_unmap_path_key def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): @@ -704,18 +546,6 @@ def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): unchanged_val = transform.make_str(value) key_value_path[cls.name] = unchanged_val return (key_value_path, leaf_path, unwanted_path) - - old_new_unmap_path_key = cls.new_unmap_path_key - - def new_unmap_path_key(key_value_path, unwanted_path, final_path): - key_value_path, unwanted_path, final_path = old_new_unmap_path_key(key_value_path, unwanted_path, final_path) - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - if cls.name == transform.name: - unchanged_val = transform.make_str(value) - key_value_path[cls.name] = unchanged_val - return (key_value_path, unwanted_path, final_path) def unmap_to_datacube(path, unmapped_path): for transform in cls.transformations: @@ -729,9 +559,6 @@ def unmap_to_datacube(path, unmapped_path): unmapped_path[cls.name] = unchanged_val return (path, unmapped_path) - def remap_to_requested(path, unmapped_path): - return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] @@ -748,7 +575,6 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) else: - # indexes_between = [i for i in indexes if low <= i <= up] lower_idx = bisect.bisect_left(indexes, low) upper_idx = bisect.bisect_right(indexes, up) indexes_between = indexes[lower_idx: upper_idx] @@ -761,10 +587,8 @@ def remap(range): cls.remap = remap cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube - cls.remap_to_requested = remap_to_requested cls.find_indices_between = find_indices_between cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube - cls.unmap_path_key = unmap_path_key cls.n_unmap_path_key = n_unmap_path_key return cls @@ -786,9 +610,6 @@ def unmap_total_path_to_datacube(path, unmapped_path): def unmap_to_datacube(path, unmapped_path): return (path, unmapped_path) - def remap_to_requested(path, unmapped_path): - return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between_ranges = [] for indexes in index_ranges: @@ -802,7 +623,6 @@ def remap(range): cls.remap = remap cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube - cls.remap_to_requested = remap_to_requested cls.find_indices_between = find_indices_between cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube @@ -863,17 +683,8 @@ def offset(self, value): def unmap_total_path_to_datacube(self, path, unmapped_path): return (path, unmapped_path) - def unmap_path_key(self, key_value_path, leaf_path): - return (key_value_path, leaf_path) - def n_unmap_path_key(self, key_value_path, leaf_path, unwanted_path): return (key_value_path, leaf_path, unwanted_path) - - def new_unmap_path_key(self, key_value_path, leaf_path, unwanted_path): - return (key_value_path, leaf_path, unwanted_path) - - def remap_to_requeest(path, unmapped_path): - return (path, unmapped_path) def find_indices_between(self, index_ranges, low, up, datacube, method=None): # TODO: add method for snappping diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index 284a2f6ad..802285c32 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -20,6 +20,6 @@ def change_val_type(self, axis_name, values): def blocked_axes(self): return [] - + def unwanted_axes(self): return [] diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 07a53336d..fa6f4fd8c 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -1,9 +1,9 @@ +import bisect import math from copy import deepcopy from importlib import import_module -from ...utility.list_tools import bisect_left_cmp, bisect_right_cmp -import bisect +from ...utility.list_tools import bisect_left_cmp, bisect_right_cmp from .datacube_transformations import DatacubeAxisTransformation @@ -34,8 +34,6 @@ def unwanted_axes(self): return [self._final_mapped_axes[0]] def transformation_axes_final(self): - # final_transformation = self.generate_final_transformation() - # final_axes = self._final_transformation._mapped_axes final_axes = self._final_mapped_axes return final_axes @@ -47,8 +45,6 @@ def change_val_type(self, axis_name, values): def _mapped_axes(self): # NOTE: Each of the mapper method needs to call it's sub mapper method - # final_transformation = self.generate_final_transformation() - # final_axes = self._final_transformation._mapped_axes final_axes = self._final_mapped_axes return final_axes @@ -59,19 +55,15 @@ def _resolution(self): pass def first_axis_vals(self): - # final_transformation = self.generate_final_transformation() return self._final_transformation.first_axis_vals() def second_axis_vals(self, first_val): - # final_transformation = self.generate_final_transformation() return self._final_transformation.second_axis_vals(first_val) def map_first_axis(self, lower, upper): - # final_transformation = self.generate_final_transformation() return self._final_transformation.map_first_axis(lower, upper) def map_second_axis(self, first_val, lower, upper): - # final_transformation = self.generate_final_transformation() return self._final_transformation.map_second_axis(first_val, lower, upper) def find_second_idx(self, first_val, second_val): @@ -81,7 +73,6 @@ def unmap_first_val_to_start_line_idx(self, first_val): return self._final_transformation.unmap_first_val_to_start_line_idx(first_val) def unmap(self, first_val, second_val): - # final_transformation = self.generate_final_transformation() return self._final_transformation.unmap(first_val, second_val) @@ -258,12 +249,8 @@ def __init__(self, base_axis, mapped_axes, resolution): self._base_axis = base_axis self._resolution = resolution self._first_axis_vals = self.first_axis_vals() - # self._inv_first_axis_vals = self._first_axis_vals[::-1] - # self._inv_first_axis_vals = {v:k for k,v in self._first_axis_vals.items()} self._first_idx_map = self.create_first_idx_map() - # self._second_axis_spacing = dict() self._second_axis_spacing = {} - # self.treated_first_vals = dict() self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} def gauss_first_guess(self): @@ -2926,7 +2913,6 @@ def first_axis_vals(self): def map_first_axis(self, lower, upper): axis_lines = self._first_axis_vals - # return_vals = [val for val in axis_lines if lower <= val <= upper] end_idx = bisect_left_cmp(axis_lines, lower, cmp=lambda x, y: x > y) + 1 start_idx = bisect_right_cmp(axis_lines, upper, cmp=lambda x, y: x > y) return_vals = axis_lines[start_idx:end_idx] @@ -2935,8 +2921,6 @@ def map_first_axis(self, lower, upper): def second_axis_vals(self, first_val): first_axis_vals = self._first_axis_vals tol = 1e-10 - # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] - # first_idx = first_axis_vals.index(first_val) first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) if first_idx >= self._resolution: first_idx = (2 * self._resolution) - 1 - first_idx @@ -2949,8 +2933,6 @@ def second_axis_vals(self, first_val): def second_axis_spacing(self, first_val): first_axis_vals = self._first_axis_vals tol = 1e-10 - # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] - # first_idx = first_axis_vals.index(first_val) _first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) first_idx = _first_idx if first_idx >= self._resolution: @@ -2962,24 +2944,9 @@ def second_axis_spacing(self, first_val): def map_second_axis(self, first_val, lower, upper): second_axis_spacing, first_idx = self.second_axis_spacing(first_val) - # if first_val not in self._second_axis_spacing: - # (second_axis_spacing, first_idx) = self.second_axis_spacing(first_val) - # self._second_axis_spacing[first_val] = (second_axis_spacing, first_idx) - # else: - # (second_axis_spacing, first_idx) = self._second_axis_spacing[first_val] start_idx = int(lower/second_axis_spacing) end_idx = int(upper/second_axis_spacing) + 1 return_vals = [i * second_axis_spacing for i in range(start_idx, end_idx)] - - # second_axis_vals = self.second_axis_vals(first_val) - # # NOTE: here this seems faster than the bisect.bisect? - # # return_vals = [val for val in second_axis_vals if lower <= val <= upper] - # start_idx = bisect_left_cmp(second_axis_vals, lower, cmp=lambda x, y: x < y) + 1 - # end_idx = bisect_right_cmp(second_axis_vals, upper, cmp=lambda x, y: x < y) + 1 - # return_vals = second_axis_vals[start_idx:end_idx] - # # start_idx = bisect.bisect_left(second_axis_vals, lower) - # # end_idx = bisect.bisect_right(second_axis_vals, upper) - # # return_vals = second_axis_vals[start_idx:end_idx] return return_vals def axes_idx_to_octahedral_idx(self, first_idx, second_idx): @@ -2990,40 +2957,13 @@ def axes_idx_to_octahedral_idx(self, first_idx, second_idx): # NOTE: OR somehow cache this for a given first_idx and then only modify the axis idx for second_idx when the # first_idx changes - # time1 = time.time() - # octa_idx = self._first_idx_map[first_idx-1] + second_idx octa_idx = self._first_idx_map[first_idx-1] + second_idx - # octa_idx = 0 - # if first_idx == 1: - # octa_idx = second_idx - # else: - # for i in range(first_idx - 1): - # if i <= self._resolution - 1: - # octa_idx += 20 + 4 * i - # else: - # i = i - self._resolution + 1 - # if i == 1: - # octa_idx += 16 + 4 * self._resolution - # else: - # i = i - 1 - # octa_idx += 16 + 4 * (self._resolution - i) - # octa_idx += second_idx - # print("TIME UNMAPPING TO OCT IDX") - # print(time.time() - time1) return octa_idx - # def find_second_idx(self, first_val, second_val): - # tol = 1e-10 - # second_axis_vals = self.second_axis_vals(first_val) - # second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) - # return second_idx - def create_first_idx_map(self): - # first_idx_list = [0] * (2*self._resolution) first_idx_list = {} idx = 0 for i in range(2*self._resolution): - # first_idx_list[i] = idx first_idx_list[i] = idx if i <= self._resolution - 1: idx += 20 + 4 * i @@ -3036,36 +2976,8 @@ def create_first_idx_map(self): idx += 16 + 4 * (self._resolution - i) return first_idx_list - # def unmap_first_val_to_start_line_idx(self, first_val): - # first_axis_vals = self._first_axis_vals - # tol = 1e-10 - # # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] - # # first_idx = first_axis_vals.index(first_val) + 1 - # # first_idx = len(first_axis_vals) - bisect.bisect_left(first_axis_vals[::-1], first_val - tol) - # first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + 1 - # octa_idx = 0 - # if first_idx == 1: - # return octa_idx - # else: - # for i in range(first_idx - 1): - # if i <= self._resolution - 1: - # octa_idx += 20 + 4 * i - # else: - # i = i - self._resolution + 1 - # if i == 1: - # octa_idx += 16 + 4 * self._resolution - # else: - # i = i - 1 - # octa_idx += 16 + 4 * (self._resolution - i) - # return octa_idx - def find_second_axis_idx(self, first_val, second_val): (second_axis_spacing, first_idx) = self.second_axis_spacing(first_val) - # if first_val not in self._second_axis_spacing: - # (second_axis_spacing, first_idx) = self.second_axis_spacing(first_val) - # self._second_axis_spacing[first_val] = (second_axis_spacing, first_idx) - # else: - # (second_axis_spacing, first_idx) = self._second_axis_spacing[first_val] tol = 1e-8 if second_val/second_axis_spacing > int(second_val/second_axis_spacing) + 1 - tol: second_idx = int(second_val/second_axis_spacing) + 1 @@ -3074,35 +2986,8 @@ def find_second_axis_idx(self, first_val, second_val): return (first_idx, second_idx) def unmap(self, first_val, second_val): - # time1 = time.time() - # first_axis_vals = self._first_axis_vals - # inv_first_axis_vals = self._inv_first_axis_vals - # tol = 1e-10 - # # first_val = [val for val in first_axis_vals if first_val - tol < val < first_val + tol][0] - # # first_idx = first_axis_vals.index(first_val) + 1 - # # first_idx = len(first_axis_vals) - bisect.bisect_left(first_axis_vals[::-1], first_val - tol) - # # first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + 1 - # first_idx = bisect.bisect_left(self._inv_first_axis_vals, - (first_val - tol)) - # # print(inv_first_axis_vals) - # # print(first_val) - # # first_idx = inv_first_axis_vals[first_val] - # # first_idx = np.searchsorted(-first_axis_vals, - (first_val - tol), side="right") - # if first_val not in self.treated_first_vals: - # second_axis_vals = self.second_axis_vals(first_val) - # self.treated_first_vals[first_val] = second_axis_vals - # else: - # second_axis_vals = self.treated_first_vals[first_val] - # # second_val = [val for val in second_axis_vals if second_val - tol < val < second_val + tol][0] - # # second_idx = second_axis_vals.index(second_val) - # second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) (first_idx, second_idx) = self.find_second_axis_idx(first_val, second_val) - # second_idx = np.searchsorted(second_axis_vals, second_val - tol) - # print("TIME SPENT DOING VAL TO IDX") - # print(time.time() - time1) octahedral_index = self.axes_idx_to_octahedral_idx(first_idx, second_idx) - # octahedral_index = int(octahedral_index) - # print("OCTAHEDRAL UNMAP TIME ") - # print(time.time() - time1) return octahedral_index diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index 48250b20e..e91354c4f 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -1,6 +1,5 @@ import numpy as np import pandas as pd -# import time from .datacube_transformations import DatacubeAxisTransformation @@ -15,22 +14,18 @@ def __init__(self, name, merge_options): def blocked_axes(self): return [self._second_axis] - + def unwanted_axes(self): return [] def merged_values(self, datacube): - # time1 = time.time() first_ax_vals = datacube.ax_vals(self.name) second_ax_name = self._second_axis second_ax_vals = datacube.ax_vals(second_ax_name) linkers = self._linkers merged_values = [] - # merged_values = np.empty(len(first_ax_vals)*len(second_ax_vals)) - # for first_val in first_ax_vals: for i in range(len(first_ax_vals)): first_val = first_ax_vals[i] - # for second_val in second_ax_vals: for j in range(len(second_ax_vals)): second_val = second_ax_vals[j] # TODO: check that the first and second val are strings @@ -38,11 +33,7 @@ def merged_values(self, datacube): val_to_add = val_to_add.to_numpy() val_to_add = val_to_add.astype("datetime64[s]") merged_values.append(val_to_add) - # print(val_to_add) - # merged_values[i*len(second_ax_vals) + j] = val_to_add merged_values = np.array(merged_values) - # print("MERGED VALUES TIME") - # print(time.time() - time1) return merged_values def transformation_axes_final(self): @@ -52,7 +43,6 @@ def generate_final_transformation(self): return self def unmerge(self, merged_val): - # time1 = time.time() merged_val = str(merged_val) first_idx = merged_val.find(self._linkers[0]) first_val = merged_val[:first_idx] @@ -63,8 +53,6 @@ def unmerge(self, merged_val): # TODO: maybe replacing like this is too specific to time/dates? first_val = str(first_val).replace("-", "") second_val = second_val.replace(":", "") - # print("UNMERGE TIME") - # print(time.time() - time1) return (first_val, second_val) def change_val_type(self, axis_name, values): diff --git a/polytope/datacube/transformations/datacube_null_transformation.py b/polytope/datacube/transformations/datacube_null_transformation.py index 26f9ffaa4..55c94277e 100644 --- a/polytope/datacube/transformations/datacube_null_transformation.py +++ b/polytope/datacube/transformations/datacube_null_transformation.py @@ -17,6 +17,6 @@ def change_val_type(self, axis_name, values): def blocked_axes(self): return [] - + def unwanted_axes(self): return [] diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index f0b1415b5..6a556907a 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -17,6 +17,6 @@ def change_val_type(self, axis_name, values): def blocked_axes(self): return [] - + def unwanted_axes(self): return [] diff --git a/polytope/datacube/transformations/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change.py index dde8ee346..8dee75341 100644 --- a/polytope/datacube/transformations/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change.py @@ -21,20 +21,17 @@ def generate_final_transformation(self): return transformation def transformation_axes_final(self): - # final_transformation = self.generate_final_transformation() return [self._final_transformation.axis_name] def change_val_type(self, axis_name, values): - # transformation = self.generate_final_transformation() return [self._final_transformation.transform_type(val) for val in values] def make_str(self, value): - # transformation = self.generate_final_transformation() return self._final_transformation.make_str(value) def blocked_axes(self): return [] - + def unwanted_axes(self): return [] diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 158f09273..10c3bd5f4 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -1,8 +1,7 @@ import math from copy import copy -from itertools import chain, product +from itertools import chain from typing import List -import time import scipy.spatial @@ -103,12 +102,6 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): def _find_intersects(polytope, slice_axis_idx, value): intersects = [] # Find all points above and below slice axis - # above_slice, below_slice = [], [] - # for p in polytope.points: - # if p[slice_axis_idx] >= value: - # above_slice.append(p) - # if p[slice_axis_idx] <= value: - # below_slice.append(p) above_slice = [p for p in polytope.points if p[slice_axis_idx] >= value] below_slice = [p for p in polytope.points if p[slice_axis_idx] <= value] @@ -124,16 +117,6 @@ def _find_intersects(polytope, slice_axis_idx, value): interp_coeff = (value - b[slice_axis_idx]) / (a[slice_axis_idx] - b[slice_axis_idx]) intersect = lerp(a, b, interp_coeff) intersects.append(intersect) - # for (a, b) in product(above_slice, below_slice): - # # edge is incident with slice plane, don't need these points - # if a[slice_axis_idx] == b[slice_axis_idx]: - # intersects.append(b) - # continue - - # # Linearly interpolate all coordinates of two points (a,b) of the polytope - # interp_coeff = (value - b[slice_axis_idx]) / (a[slice_axis_idx] - b[slice_axis_idx]) - # intersect = lerp(a, b, interp_coeff) - # intersects.append(intersect) return intersects @@ -146,7 +129,6 @@ def _reduce_dimension(intersects, slice_axis_idx): def slice(polytope: ConvexPolytope, axis, value, slice_axis_idx): - # slice_axis_idx = polytope._axes.index(axis) if len(polytope.points[0]) == 1: # Note that in this case, we do not need to do linear interpolation so we can save time @@ -180,7 +162,6 @@ def slice(polytope: ConvexPolytope, axis, value, slice_axis_idx): vertices = hull.vertices except scipy.spatial.qhull.QhullError as e: - # if "input is less than" or "simplex is flat" in str(e): if "less than" or "flat" in str(e): return ConvexPolytope(axes, intersects) # Sliced result is simply the convex hull diff --git a/polytope/polytope.py b/polytope/polytope.py index dcd46f407..5e8fc7133 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -1,5 +1,4 @@ from typing import List -import time from .shapes import ConvexPolytope from .utility.exceptions import AxisOverdefinedError @@ -45,18 +44,6 @@ def slice(self, polytopes: List[ConvexPolytope]): def retrieve(self, request: Request, method="standard"): """Higher-level API which takes a request and uses it to slice the datacube""" - # print("TIME IN POLYTOPE EXTRACT") - # time0 = time.time() request_tree = self.engine.extract(self.datacube, request.polytopes()) - # print(time.time() - time0) - # print("TIME INSIDE OF GET") - # time1 = time.time() self.datacube.get(request_tree) - # print(time.time() - time1) - # print("TIME INSIDE FDB") - # print(self.datacube.time_fdb) - # print("TIME UNMAP KEY") - # print(self.datacube.time_unmap_key) - # print("TIME SPENT REMOVE UNNECESSARY PATH KEYS") - # print(self.datacube.other_time) return request_tree diff --git a/requirements_example.txt b/requirements_example.txt new file mode 100644 index 000000000..19717dacf --- /dev/null +++ b/requirements_example.txt @@ -0,0 +1 @@ +cfgrib==0.9.10.3 \ No newline at end of file diff --git a/tests/data/era5-levels-members.grib b/tests/data/era5-levels-members.grib new file mode 100644 index 000000000..90d45deed --- /dev/null +++ b/tests/data/era5-levels-members.grib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c144fde61ca5d53702bf6f212775ef2cc783bdd66b6865160bf597c1b35ed898 +size 2361600 diff --git a/tests/data/foo.grib b/tests/data/foo.grib new file mode 100644 index 000000000..9c5efa68b --- /dev/null +++ b/tests/data/foo.grib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12533bae9b3aa078a3298a82f289c7dbba6dbdb9a21e4e5e24e84b8695a75dee +size 26409360 diff --git a/tests/data/healpix.grib b/tests/data/healpix.grib new file mode 100644 index 000000000..693c98c12 --- /dev/null +++ b/tests/data/healpix.grib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df57790df280627b2883c01fa4b56986d9938ed69b7434fe09c4bede99d2895f +size 37030 From 2090029856d8214d20907cc4c813dd9ef4f35f74 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 10 Nov 2023 14:50:26 +0000 Subject: [PATCH 186/332] black --- performance/fdb_scalability_plot.py | 8 ++- polytope/datacube/backends/FDB_datacube.py | 57 ++++++++++++------- polytope/datacube/datacube_axis.py | 9 ++- .../transformations/datacube_mappers.py | 14 ++--- polytope/engine/hullslicer.py | 1 - tests/test_regular_grid.py | 10 ++-- 6 files changed, 57 insertions(+), 42 deletions(-) diff --git a/performance/fdb_scalability_plot.py b/performance/fdb_scalability_plot.py index 13f2bd9b7..7230fd47b 100644 --- a/performance/fdb_scalability_plot.py +++ b/performance/fdb_scalability_plot.py @@ -1,7 +1,11 @@ import matplotlib.pyplot as plt -fdb_time = [7.6377081871032715 - 7.558288812637329, 73.57192325592041 - 72.99611115455627, - 733.2706120014191 - 727.7059993743896, 4808.3157522678375 - 4770.814565420151] +fdb_time = [ + 7.6377081871032715 - 7.558288812637329, + 73.57192325592041 - 72.99611115455627, + 733.2706120014191 - 727.7059993743896, + 4808.3157522678375 - 4770.814565420151, +] num_extracted_points = [1986, 19226, 191543, 1267134] # for the 1.3M points, we used 100 latitudes too...., maybe that's why it's not as linear... diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/FDB_datacube.py index 7326d6e4d..70a867cf0 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/FDB_datacube.py @@ -23,8 +23,19 @@ def __init__(self, config={}, axis_options={}): self.unwanted_axes = [] self.transformation = None self.fake_axes = [] - self.final_path = {"class" : 0, "date": 0, "domain": 0, "expver": 0, "levtype": 0, "param": 0, - "step" : 0, "stream": 0, "time": 0, "type": 0, "values": 0} + self.final_path = { + "class": 0, + "date": 0, + "domain": 0, + "expver": 0, + "levtype": 0, + "param": 0, + "step": 0, + "stream": 0, + "time": 0, + "type": 0, + "values": 0, + } self.unwanted_path = {} partial_request = config @@ -66,8 +77,9 @@ def get(self, requests: IndexTree, leaf_path={}): else: key_value_path = {requests.axis.name: requests.value} ax = requests.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, - self.unwanted_path) + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key( + key_value_path, leaf_path, self.unwanted_path + ) leaf_path |= key_value_path if len(requests.children[0].children[0].children) == 0: # remap this last key @@ -79,9 +91,9 @@ def get(self, requests: IndexTree, leaf_path={}): self.get(c, leaf_path) def handle_last_before_last_layer(self, requests, leaf_path={}): - range_lengths = [[1]*200]*200 - current_start_idxs = [[None]*200]*200 - fdb_node_ranges = [[[IndexTree.root]*200]*200]*200 + range_lengths = [[1] * 200] * 200 + current_start_idxs = [[None] * 200] * 200 + fdb_node_ranges = [[[IndexTree.root] * 200] * 200] * 200 lat_length = len(requests.children) for i in range(len(requests.children)): lat_child = requests.children[i] @@ -90,13 +102,13 @@ def handle_last_before_last_layer(self, requests, leaf_path={}): fdb_range_nodes = deepcopy(fdb_node_ranges[i]) key_value_path = {lat_child.axis.name: lat_child.value} ax = lat_child.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, - self.unwanted_path) + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key( + key_value_path, leaf_path, self.unwanted_path + ) leaf_path |= key_value_path - (range_lengths[i], - current_start_idxs[i], - fdb_node_ranges[i]) = self.get_last_layer_before_leaf(lat_child, leaf_path, range_length, - current_start_idx, fdb_range_nodes) + (range_lengths[i], current_start_idxs[i], fdb_node_ranges[i]) = self.get_last_layer_before_leaf( + lat_child, leaf_path, range_length, current_start_idx, fdb_range_nodes + ) self.give_fdb_val_to_node(leaf_path, range_lengths, current_start_idxs, fdb_node_ranges, lat_length) def get_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, fdb_range_n): @@ -105,22 +117,24 @@ def get_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, # now c are the leaves of the initial tree key_value_path = {c.axis.name: c.value} ax = c.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, - self.unwanted_path) + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key( + key_value_path, leaf_path, self.unwanted_path + ) leaf_path |= key_value_path last_idx = key_value_path["values"] if current_idx[i] is None: current_idx[i] = last_idx - fdb_range_n[i][range_l[i]-1] = c + fdb_range_n[i][range_l[i] - 1] = c else: if last_idx == current_idx[i] + range_l[i]: range_l[i] += 1 - fdb_range_n[i][range_l[i]-1] = c + fdb_range_n[i][range_l[i] - 1] = c else: key_value_path = {c.axis.name: c.value} ax = c.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key(key_value_path, leaf_path, - self.unwanted_path) + (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key( + key_value_path, leaf_path, self.unwanted_path + ) leaf_path |= key_value_path i += 1 current_start_idx = key_value_path["values"] @@ -128,8 +142,9 @@ def get_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, return (range_l, current_idx, fdb_range_n) def give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_range_nodes, lat_length): - (output_values, original_indices) = self.find_fdb_values(leaf_path, range_lengths, current_start_idx, - lat_length) + (output_values, original_indices) = self.find_fdb_values( + leaf_path, range_lengths, current_start_idx, lat_length + ) new_fdb_range_nodes = [] new_range_lengths = [] for j in range(lat_length): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 57a971bcb..00102800d 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -322,7 +322,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): if not axis_reversed: lower_idx = bisect.bisect_left(idxs, low) upper_idx = bisect.bisect_right(idxs, up) - indexes_between = idxs[lower_idx: upper_idx] + indexes_between = idxs[lower_idx:upper_idx] else: # TODO: do the custom bisect end_idx = bisect_left_cmp(idxs, low, cmp=lambda x, y: x > y) + 1 @@ -420,7 +420,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): else: lower_idx = bisect.bisect_left(indexes, low) upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx: upper_idx] + indexes_between = indexes[lower_idx:upper_idx] indexes_between_ranges.append(indexes_between) return indexes_between_ranges @@ -491,7 +491,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): else: lower_idx = bisect.bisect_left(indexes, low) upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx: upper_idx] + indexes_between = indexes[lower_idx:upper_idx] indexes_between_ranges.append(indexes_between) return indexes_between_ranges @@ -577,7 +577,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): else: lower_idx = bisect.bisect_left(indexes, low) upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx: upper_idx] + indexes_between = indexes[lower_idx:upper_idx] indexes_between_ranges.append(indexes_between) return indexes_between_ranges @@ -595,7 +595,6 @@ def remap(range): def null(cls): - if cls.type_change: old_find_indexes = cls.find_indexes diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index fa6f4fd8c..34af16bdc 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -2944,8 +2944,8 @@ def second_axis_spacing(self, first_val): def map_second_axis(self, first_val, lower, upper): second_axis_spacing, first_idx = self.second_axis_spacing(first_val) - start_idx = int(lower/second_axis_spacing) - end_idx = int(upper/second_axis_spacing) + 1 + start_idx = int(lower / second_axis_spacing) + end_idx = int(upper / second_axis_spacing) + 1 return_vals = [i * second_axis_spacing for i in range(start_idx, end_idx)] return return_vals @@ -2957,13 +2957,13 @@ def axes_idx_to_octahedral_idx(self, first_idx, second_idx): # NOTE: OR somehow cache this for a given first_idx and then only modify the axis idx for second_idx when the # first_idx changes - octa_idx = self._first_idx_map[first_idx-1] + second_idx + octa_idx = self._first_idx_map[first_idx - 1] + second_idx return octa_idx def create_first_idx_map(self): first_idx_list = {} idx = 0 - for i in range(2*self._resolution): + for i in range(2 * self._resolution): first_idx_list[i] = idx if i <= self._resolution - 1: idx += 20 + 4 * i @@ -2979,10 +2979,10 @@ def create_first_idx_map(self): def find_second_axis_idx(self, first_val, second_val): (second_axis_spacing, first_idx) = self.second_axis_spacing(first_val) tol = 1e-8 - if second_val/second_axis_spacing > int(second_val/second_axis_spacing) + 1 - tol: - second_idx = int(second_val/second_axis_spacing) + 1 + if second_val / second_axis_spacing > int(second_val / second_axis_spacing) + 1 - tol: + second_idx = int(second_val / second_axis_spacing) + 1 else: - second_idx = int(second_val/second_axis_spacing) + second_idx = int(second_val / second_axis_spacing) return (first_idx, second_idx) def unmap(self, first_val, second_val): diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 10c3bd5f4..d361fb13c 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -129,7 +129,6 @@ def _reduce_dimension(intersects, slice_axis_idx): def slice(polytope: ConvexPolytope, axis, value, slice_axis_idx): - if len(polytope.points[0]) == 1: # Note that in this case, we do not need to do linear interpolation so we can save time if value in chain(*polytope.points): diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 7247f73f6..c32a7760f 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -18,9 +18,7 @@ def setup_method(self, method): download_test_data(nexus_url, "era5-levels-members.grib") self.options = { "values": { - "transformation": { - "mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]} - } + "transformation": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}} }, "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, @@ -68,11 +66,11 @@ def test_regular_grid(self): Select("type", ["an"]), Disk(["latitude", "longitude"], [0, 0], [15, 15]), Select("levelist", ["500"]), - Select("number", ["0", "1"]) + Select("number", ["0", "1"]), ) result = self.API.retrieve(request) result.pprint() - assert len(result.leaves) == 46*2 + assert len(result.leaves) == 46 * 2 lats = [] lons = [] @@ -101,4 +99,4 @@ def test_regular_grid(self): # plt.colorbar(label="Temperature") # plt.show() - assert len(eccodes_lats) == 46*2 + assert len(eccodes_lats) == 46 * 2 From f8922df8c9fcc26645ddb9714bd45781b7d098b5 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 10 Nov 2023 14:58:50 +0000 Subject: [PATCH 187/332] remove data --- tests/data/era5-levels-members.grib | 3 --- tests/data/foo.grib | 3 --- tests/data/healpix.grib | 3 --- 3 files changed, 9 deletions(-) delete mode 100644 tests/data/era5-levels-members.grib delete mode 100644 tests/data/foo.grib delete mode 100644 tests/data/healpix.grib diff --git a/tests/data/era5-levels-members.grib b/tests/data/era5-levels-members.grib deleted file mode 100644 index 90d45deed..000000000 --- a/tests/data/era5-levels-members.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c144fde61ca5d53702bf6f212775ef2cc783bdd66b6865160bf597c1b35ed898 -size 2361600 diff --git a/tests/data/foo.grib b/tests/data/foo.grib deleted file mode 100644 index 9c5efa68b..000000000 --- a/tests/data/foo.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:12533bae9b3aa078a3298a82f289c7dbba6dbdb9a21e4e5e24e84b8695a75dee -size 26409360 diff --git a/tests/data/healpix.grib b/tests/data/healpix.grib deleted file mode 100644 index 693c98c12..000000000 --- a/tests/data/healpix.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:df57790df280627b2883c01fa4b56986d9938ed69b7434fe09c4bede99d2895f -size 37030 From 5842a1be774bfcde98ce7faf71deecc07712dd33 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 10 Nov 2023 15:37:53 +0000 Subject: [PATCH 188/332] remove unnecessary code --- .gitignore | 6 +- polytope/datacube/backends/xarray.py | 5 +- polytope/datacube/datacube_axis.py | 101 --------------------------- 3 files changed, 6 insertions(+), 106 deletions(-) diff --git a/.gitignore b/.gitignore index 037872e66..525662725 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,8 @@ polytope.egg-info .pytest_cache *.prof -*.idx \ No newline at end of file +*.idx +*.grib +*.xml +site +.coverage \ No newline at end of file diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 0d244764e..01d6a2915 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -49,14 +49,13 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): def get(self, requests: IndexTree): for r in requests.leaves: path = r.flatten() - # path = self.remap_path(path) if len(path.items()) == self.axis_counter: # first, find the grid mapper transform unmapped_path = {} path_copy = deepcopy(path) for key in path_copy: axis = self._axes[key] - (path, unmapped_path) = axis.unmap_total_path_to_datacube(path, unmapped_path) + (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) path = self.fit_path(path) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmapped_path) @@ -67,8 +66,6 @@ def get(self, requests: IndexTree): r.remove_branch() def datacube_natural_indexes(self, axis, subarray): - # if axis.name in self.fake_axes: - # indexes = subarray[axis.name].values if axis.name in self.complete_axes: indexes = next(iter(subarray.xindexes.values())).to_pandas_index() else: diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 00102800d..2d3160354 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -118,20 +118,6 @@ def remap(range: List): def find_indexes(path, datacube): return old_find_indexes(path, datacube) - old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube - - def unmap_total_path_to_datacube(path, unmapped_path): - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisCyclic): - transformation = transform - if cls.name == transformation.name: - old_val = path.get(cls.name, None) - path.pop(cls.name, None) - new_val = _remap_val_to_axis_range(old_val) - path[cls.name] = new_val - (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) - return (path, unmapped_path) - old_n_unmap_path_key = cls.n_unmap_path_key def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): @@ -195,7 +181,6 @@ def offset(range): cls.offset = offset cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube - cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube cls.find_indices_between = find_indices_between cls.n_unmap_path_key = n_unmap_path_key @@ -247,43 +232,6 @@ def unmap_to_datacube(path, unmapped_path): unmapped_path[transform.old_axis] = unmapped_idx return (path, unmapped_path) - old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube - - def unmap_total_path_to_datacube(path, unmapped_path): - # TODO: to be faster, could just compute the first lat unmapped idx, and do +1 for each subsequent lat idx? - (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) - for transform in cls.transformations: - if isinstance(transform, DatacubeMapper): - if cls.name == transform._mapped_axes()[0]: - # if we are on the first axis, then need to add the first val to unmapped_path - first_val = path[cls.name] - del path[cls.name] - - if unmapped_path is None: - unmapped_path[cls.name] = first_val - elif cls.name not in unmapped_path: - # if for some reason, the unmapped_path already has the first axis val, then don't update - unmapped_path[cls.name] = first_val - if cls.name == transform._mapped_axes()[1]: - # if we are on the second axis, then the val of the first axis is stored - # inside unmapped_path so can get it from there - second_val = path[cls.name] - del path[cls.name] - first_val = unmapped_path.get(transform._mapped_axes()[0], None) - - unmapped_path.pop(transform._mapped_axes()[0], None) - # NOTE: here we first calculate the starting idx of the first_val grid line - # and then append the second_idx to get the final unmapped_idx - # To do this, also need to find second_idx from second_val... - - # if the first_val was not in the unmapped_path, then it's still in path - if first_val is None: - first_val = path.get(transform._mapped_axes()[0], None) - path.pop(transform._mapped_axes()[0], None) - unmapped_idx = transform.unmap(first_val, second_val) - unmapped_path[transform.old_axis] = unmapped_idx - return (path, unmapped_path) - old_n_unmap_path_key = cls.n_unmap_path_key def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): @@ -340,7 +288,6 @@ def remap(range): cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between - cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube cls.n_unmap_path_key = n_unmap_path_key return cls @@ -359,20 +306,6 @@ def find_indexes(path, datacube): if cls.name == transformation._first_axis: return transformation.merged_values(datacube) - old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube - - def unmap_total_path_to_datacube(path, unmapped_path): - (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name == transformation._first_axis: - old_val = path[cls.name] - (first_val, second_val) = transformation.unmerge(old_val) - path[transformation._first_axis] = first_val - path[transformation._second_axis] = second_val - return (path, unmapped_path) - old_n_unmap_path_key = cls.n_unmap_path_key def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): @@ -431,7 +364,6 @@ def remap(range): cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between - cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube cls.n_unmap_path_key = n_unmap_path_key return cls @@ -452,9 +384,6 @@ def find_indexes(path, datacube): ordered_indices = unordered_indices return ordered_indices - def unmap_to_datacube(path, unmapped_path): - return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] @@ -500,7 +429,6 @@ def remap(range): cls.remap = remap cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between return cls @@ -520,21 +448,6 @@ def find_indexes(path, datacube): original_vals = old_find_indexes(path, datacube) return transformation.change_val_type(cls.name, original_vals) - old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube - - def unmap_total_path_to_datacube(path, unmapped_path): - (path, unmapped_path) = old_unmap_total_path_to_datacube(path, unmapped_path) - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - changed_val = path.get(cls.name, None) - unchanged_val = transformation.make_str(changed_val) - if cls.name in path: - path.pop(cls.name, None) - unmapped_path[cls.name] = unchanged_val - return (path, unmapped_path) - old_n_unmap_path_key = cls.n_unmap_path_key def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): @@ -588,7 +501,6 @@ def remap(range): cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between - cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube cls.n_unmap_path_key = n_unmap_path_key return cls @@ -601,14 +513,6 @@ def null(cls): def find_indexes(path, datacube): return old_find_indexes(path, datacube) - old_unmap_total_path_to_datacube = cls.unmap_total_path_to_datacube - - def unmap_total_path_to_datacube(path, unmapped_path): - return old_unmap_total_path_to_datacube(path, unmapped_path) - - def unmap_to_datacube(path, unmapped_path): - return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between_ranges = [] for indexes in index_ranges: @@ -621,9 +525,7 @@ def remap(range): cls.remap = remap cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between - cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube return cls @@ -679,9 +581,6 @@ def find_indexes(self, path, datacube): def offset(self, value): return 0 - def unmap_total_path_to_datacube(self, path, unmapped_path): - return (path, unmapped_path) - def n_unmap_path_key(self, key_value_path, leaf_path, unwanted_path): return (key_value_path, leaf_path, unwanted_path) From 1065c722ee8756f83f1e447c543ce8e68e4b4747 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 14 Nov 2023 11:44:47 +0100 Subject: [PATCH 189/332] remove unnecessary code --- polytope/datacube/datacube_axis.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 2d3160354..77ad107e0 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -279,12 +279,6 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between_ranges.append(indexes_between) return indexes_between_ranges - old_remap = cls.remap - - def remap(range): - return old_remap(range) - - cls.remap = remap cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between From 6fffbffd6b6e702c41566c08dc6c9ba5dc05929e Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 15 Nov 2023 15:00:25 +0100 Subject: [PATCH 190/332] renaming and small fixes --- performance/fdb_performance.py | 2 +- performance/fdb_performance_3D.py | 2 +- .../backends/{FDB_datacube.py => fdb.py} | 51 +++++++------------ polytope/datacube/backends/xarray.py | 2 - polytope/datacube/datacube_axis.py | 34 ++++++------- tests/test_fdb_datacube.py | 2 +- tests/test_regular_grid.py | 2 +- 7 files changed, 38 insertions(+), 57 deletions(-) rename polytope/datacube/backends/{FDB_datacube.py => fdb.py} (87%) diff --git a/performance/fdb_performance.py b/performance/fdb_performance.py index 4ee8715be..78819d462 100644 --- a/performance/fdb_performance.py +++ b/performance/fdb_performance.py @@ -2,7 +2,7 @@ import pandas as pd -from polytope.datacube.backends.FDB_datacube import FDBDatacube +from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select diff --git a/performance/fdb_performance_3D.py b/performance/fdb_performance_3D.py index 2cfec94cf..547d865b0 100644 --- a/performance/fdb_performance_3D.py +++ b/performance/fdb_performance_3D.py @@ -2,7 +2,7 @@ import pandas as pd -from polytope.datacube.backends.FDB_datacube import FDBDatacube +from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select, Span diff --git a/polytope/datacube/backends/FDB_datacube.py b/polytope/datacube/backends/fdb.py similarity index 87% rename from polytope/datacube/backends/FDB_datacube.py rename to polytope/datacube/backends/fdb.py index 70a867cf0..34b455364 100644 --- a/polytope/datacube/backends/FDB_datacube.py +++ b/polytope/datacube/backends/fdb.py @@ -5,15 +5,9 @@ from .datacube import Datacube, IndexTree -def update_fdb_dataarray(fdb_dataarray): - fdb_dataarray["values"] = [] - return fdb_dataarray - - class FDBDatacube(Datacube): def __init__(self, config={}, axis_options={}): self.axis_options = axis_options - self.grid_mapper = None self.axis_counter = 0 self._axes = None treated_axes = [] @@ -21,31 +15,16 @@ def __init__(self, config={}, axis_options={}): self.complete_axes = [] self.blocked_axes = [] self.unwanted_axes = [] - self.transformation = None self.fake_axes = [] - self.final_path = { - "class": 0, - "date": 0, - "domain": 0, - "expver": 0, - "levtype": 0, - "param": 0, - "step": 0, - "stream": 0, - "time": 0, - "type": 0, - "values": 0, - } self.unwanted_path = {} partial_request = config # Find values in the level 3 FDB datacube # Will be in the form of a dictionary? {axis_name:values_available, ...} self.fdb = pyfdb.FDB() - fdb_dataarray = self.fdb.axes(partial_request).as_dict() - dataarray = update_fdb_dataarray(fdb_dataarray) - self.dataarray = dataarray - for name, values in dataarray.items(): + self.fdb_coordinates = self.fdb.axes(partial_request).as_dict() + self.fdb_coordinates["values"] = [] + for name, values in self.fdb_coordinates.items(): values.sort() options = axis_options.get(name, {}) self._check_and_add_axes(options, name, values) @@ -77,7 +56,7 @@ def get(self, requests: IndexTree, leaf_path={}): else: key_value_path = {requests.axis.name: requests.value} ax = requests.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key( + (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path ) leaf_path |= key_value_path @@ -91,18 +70,22 @@ def get(self, requests: IndexTree, leaf_path={}): self.get(c, leaf_path) def handle_last_before_last_layer(self, requests, leaf_path={}): - range_lengths = [[1] * 200] * 200 - current_start_idxs = [[None] * 200] * 200 - fdb_node_ranges = [[[IndexTree.root] * 200] * 200] * 200 lat_length = len(requests.children) + range_lengths = [False] * lat_length + current_start_idxs = [False] * lat_length + fdb_node_ranges = [False] * lat_length for i in range(len(requests.children)): lat_child = requests.children[i] + lon_length = len(lat_child.children) + range_lengths[i] = [1] * lon_length + current_start_idxs[i] = [None] * lon_length + fdb_node_ranges[i] = [[IndexTree.root] * lon_length] * lon_length range_length = deepcopy(range_lengths[i]) current_start_idx = deepcopy(current_start_idxs[i]) fdb_range_nodes = deepcopy(fdb_node_ranges[i]) key_value_path = {lat_child.axis.name: lat_child.value} ax = lat_child.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key( + (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path ) leaf_path |= key_value_path @@ -117,7 +100,7 @@ def get_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, # now c are the leaves of the initial tree key_value_path = {c.axis.name: c.value} ax = c.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key( + (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path ) leaf_path |= key_value_path @@ -132,7 +115,7 @@ def get_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, else: key_value_path = {c.axis.name: c.value} ax = c.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.n_unmap_path_key( + (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path ) leaf_path |= key_value_path @@ -164,7 +147,7 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): fdb_requests = [] interm_request_ranges = [] for i in range(lat_length): - for j in range(200): + for j in range(len(range_lengths[i])): if current_start_idx[i][j] is not None: current_request_ranges = (current_start_idx[i][j], current_start_idx[i][j] + range_lengths[i][j]) interm_request_ranges.append(current_request_ranges) @@ -182,7 +165,7 @@ def datacube_natural_indexes(self, axis, subarray): return indexes def select(self, path, unmapped_path): - return self.dataarray + return self.fdb_coordinates def ax_vals(self, name): - return self.dataarray.get(name, None) + return self.fdb_coordinates.get(name, None) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 01d6a2915..650038e05 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -10,7 +10,6 @@ class XArrayDatacube(Datacube): def __init__(self, dataarray: xr.DataArray, axis_options={}): self.axis_options = axis_options - self.grid_mapper = None self.axis_counter = 0 self._axes = None self.dataarray = dataarray @@ -18,7 +17,6 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.non_complete_axes = [] self.complete_axes = [] self.blocked_axes = [] - self.transformation = None self.fake_axes = [] self.unwanted_axes = [] for name, values in dataarray.coords.variables.items(): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 77ad107e0..508caa849 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -118,16 +118,16 @@ def remap(range: List): def find_indexes(path, datacube): return old_find_indexes(path, datacube) - old_n_unmap_path_key = cls.n_unmap_path_key + old_unmap_path_key = cls.unmap_path_key - def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): + def unmap_path_key(key_value_path, leaf_path, unwanted_path): value = key_value_path[cls.name] for transform in cls.transformations: if isinstance(transform, DatacubeAxisCyclic): if cls.name == transform.name: new_val = _remap_val_to_axis_range(value) key_value_path[cls.name] = new_val - key_value_path, leaf_path, unwanted_path = old_n_unmap_path_key(key_value_path, leaf_path, unwanted_path) + key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) return (key_value_path, leaf_path, unwanted_path) old_unmap_to_datacube = cls.unmap_to_datacube @@ -182,7 +182,7 @@ def offset(range): cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between - cls.n_unmap_path_key = n_unmap_path_key + cls.unmap_path_key = unmap_path_key return cls @@ -232,10 +232,10 @@ def unmap_to_datacube(path, unmapped_path): unmapped_path[transform.old_axis] = unmapped_idx return (path, unmapped_path) - old_n_unmap_path_key = cls.n_unmap_path_key + old_unmap_path_key = cls.unmap_path_key - def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): - key_value_path, leaf_path, unwanted_path = old_n_unmap_path_key(key_value_path, leaf_path, unwanted_path) + def unmap_path_key(key_value_path, leaf_path, unwanted_path): + key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) value = key_value_path[cls.name] for transform in cls.transformations: if isinstance(transform, DatacubeMapper): @@ -282,7 +282,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between - cls.n_unmap_path_key = n_unmap_path_key + cls.unmap_path_key = unmap_path_key return cls @@ -300,10 +300,10 @@ def find_indexes(path, datacube): if cls.name == transformation._first_axis: return transformation.merged_values(datacube) - old_n_unmap_path_key = cls.n_unmap_path_key + old_unmap_path_key = cls.unmap_path_key - def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): - key_value_path, leaf_path, unwanted_path = old_n_unmap_path_key(key_value_path, leaf_path, unwanted_path) + def unmap_path_key(key_value_path, leaf_path, unwanted_path): + key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) new_key_value_path = {} value = key_value_path[cls.name] for transform in cls.transformations: @@ -358,7 +358,7 @@ def remap(range): cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between - cls.n_unmap_path_key = n_unmap_path_key + cls.unmap_path_key = unmap_path_key return cls @@ -442,10 +442,10 @@ def find_indexes(path, datacube): original_vals = old_find_indexes(path, datacube) return transformation.change_val_type(cls.name, original_vals) - old_n_unmap_path_key = cls.n_unmap_path_key + old_unmap_path_key = cls.unmap_path_key - def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): - key_value_path, leaf_path, unwanted_path = old_n_unmap_path_key(key_value_path, leaf_path, unwanted_path) + def unmap_path_key(key_value_path, leaf_path, unwanted_path): + key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) value = key_value_path[cls.name] for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): @@ -495,7 +495,7 @@ def remap(range): cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between - cls.n_unmap_path_key = n_unmap_path_key + cls.unmap_path_key = unmap_path_key return cls @@ -575,7 +575,7 @@ def find_indexes(self, path, datacube): def offset(self, value): return 0 - def n_unmap_path_key(self, key_value_path, leaf_path, unwanted_path): + def unmap_path_key(self, key_value_path, leaf_path, unwanted_path): return (key_value_path, leaf_path, unwanted_path) def find_indices_between(self, index_ranges, low, up, datacube, method=None): diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 952b0bb7b..84b3b2059 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -1,7 +1,7 @@ import pandas as pd import pytest -from polytope.datacube.backends.FDB_datacube import FDBDatacube +from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index c32a7760f..c4d16baf7 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -3,7 +3,7 @@ from eccodes import codes_grib_find_nearest, codes_grib_new_from_file from helper_functions import download_test_data -from polytope.datacube.backends.FDB_datacube import FDBDatacube +from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Disk, Select From eef415dea5ca80e59730e7ba126514d55bb421c5 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 16 Nov 2023 10:37:36 +0100 Subject: [PATCH 191/332] renaming --- polytope/datacube/backends/fdb.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 34b455364..30f3d2a9c 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -51,8 +51,7 @@ def get(self, requests: IndexTree, leaf_path={}): else: for c in requests.children: self.get(c) - - # Second if request node has no children, we have a leaf so need to assign fdb values to it + # If request node has no children, we have a leaf so need to assign fdb values to it else: key_value_path = {requests.axis.name: requests.value} ax = requests.axis @@ -62,14 +61,16 @@ def get(self, requests: IndexTree, leaf_path={}): leaf_path |= key_value_path if len(requests.children[0].children[0].children) == 0: # remap this last key - self.handle_last_before_last_layer(requests, leaf_path) + self.get_2nd_last_values(requests, leaf_path) - # THIRD otherwise remap the path for this key and iterate again over children + # Otherwise remap the path for this key and iterate again over children else: for c in requests.children: self.get(c, leaf_path) - def handle_last_before_last_layer(self, requests, leaf_path={}): + def get_2nd_last_values(self, requests, leaf_path={}): + # In this function, we recursively loop over the last two layers of the tree and store the indices of the + # request ranges in those layers lat_length = len(requests.children) range_lengths = [False] * lat_length current_start_idxs = [False] * lat_length From 0dfabef4ec6847d3cf064f466c58f9b8ef0f6df3 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 20 Nov 2023 12:13:18 +0100 Subject: [PATCH 192/332] fix regular grid problem with too many points found --- .../transformations/datacube_mappers.py | 30 +++++++++++-------- polytope/shapes.py | 1 - tests/test_healpix_mapper.py | 6 ++-- tests/test_regular_grid.py | 8 +++-- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 34af16bdc..22478f217 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -83,13 +83,14 @@ def __init__(self, base_axis, mapped_axes, resolution): self._resolution = resolution self.deg_increment = 90 / self._resolution self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} + self._first_axis_vals = self.first_axis_vals() def first_axis_vals(self): - first_ax_vals = [-90 + i * self.deg_increment for i in range(2 * self._resolution)] + first_ax_vals = [90 - i * self.deg_increment for i in range(2 * self._resolution)] return first_ax_vals def map_first_axis(self, lower, upper): - axis_lines = self.first_axis_vals() + axis_lines = self._first_axis_vals return_vals = [val for val in axis_lines if lower <= val <= upper] return return_vals @@ -114,14 +115,14 @@ def find_second_idx(self, first_val, second_val): def unmap_first_val_to_start_line_idx(self, first_val): tol = 1e-8 - first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] - first_idx = self.first_axis_vals().index(first_val) + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + first_idx = self._first_axis_vals.index(first_val) return first_idx * 4 * self._resolution def unmap(self, first_val, second_val): tol = 1e-8 - first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] - first_idx = self.first_axis_vals().index(first_val) + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + first_idx = self._first_axis_vals.index(first_val) second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] second_idx = self.second_axis_vals(first_val).index(second_val) final_index = self.axes_idx_to_regular_idx(first_idx, second_idx) @@ -134,6 +135,7 @@ def __init__(self, base_axis, mapped_axes, resolution): self._base_axis = base_axis self._resolution = resolution self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} + self._first_axis_vals = self.first_axis_vals() def first_axis_vals(self): rad2deg = 180 / math.pi @@ -155,14 +157,14 @@ def first_axis_vals(self): return vals def map_first_axis(self, lower, upper): - axis_lines = self.first_axis_vals() + axis_lines = self._first_axis_vals return_vals = [val for val in axis_lines if lower <= val <= upper] return return_vals def second_axis_vals(self, first_val): tol = 1e-8 - first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] - idx = self.first_axis_vals().index(first_val) + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + idx = self._first_axis_vals.index(first_val) # Polar caps if idx < self._resolution - 1 or 3 * self._resolution - 1 < idx <= 4 * self._resolution - 2: @@ -214,8 +216,8 @@ def find_second_idx(self, first_val, second_val): def unmap_first_val_to_start_line_idx(self, first_val): tol = 1e-8 - first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] - first_idx = self.first_axis_vals().index(first_val) + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + first_idx = self._first_axis_vals.index(first_val) idx = 0 for i in range(self._resolution - 1): if i != first_idx: @@ -235,8 +237,10 @@ def unmap_first_val_to_start_line_idx(self, first_val): def unmap(self, first_val, second_val): tol = 1e-8 - first_val = [i for i in self.first_axis_vals() if first_val - tol <= i <= first_val + tol][0] - first_idx = self.first_axis_vals().index(first_val) + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + first_idx = self._first_axis_vals.index(first_val) + print(self.second_axis_vals(first_val)) + print(second_val) second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] second_idx = self.second_axis_vals(first_val).index(second_val) healpix_index = self.axes_idx_to_healpix_idx(first_idx, second_idx) diff --git a/polytope/shapes.py b/polytope/shapes.py index 59ac39fd0..5ec6e10a0 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -110,7 +110,6 @@ def __init__(self, axes, lower_corner=None, upper_corner=None): if i >> d & 1: vertex[d] = upper_corner[d] self.vertices.append(vertex) - assert lower_corner in self.vertices assert upper_corner in self.vertices assert len(self.vertices) == 2**dimension diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index c80d51042..6487bd9ba 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -20,7 +20,8 @@ def setup_method(self, method): self.options = { "values": { "transformation": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}} - } + }, + "longitude": {"transformation": {"cyclic": [0, 360]}} } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) @@ -49,7 +50,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - def test_octahedral_grid(self): + def test_healpix_grid(self): request = Request( Box(["latitude", "longitude"], [-2, -2], [10, 10]), Select("time", ["2022-12-14T12:00:00"]), @@ -58,6 +59,7 @@ def test_octahedral_grid(self): Select("valid_time", ["2022-12-14T13:00:00"]), ) result = self.API.retrieve(request) + result.pprint() assert len(result.leaves) == 35 lats = [] diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index c4d16baf7..86337b6f0 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -22,6 +22,8 @@ def setup_method(self, method): }, "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, + "number": {"transformation": {"type_change": "int"}}, + "longitude": {"transformation": {"cyclic": [0, 360]}}, } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -64,13 +66,13 @@ def test_regular_grid(self): Select("class", ["ea"]), Select("stream", ["enda"]), Select("type", ["an"]), - Disk(["latitude", "longitude"], [0, 0], [15, 15]), + Disk(["latitude", "longitude"], [0, 0], [3, 3]), Select("levelist", ["500"]), Select("number", ["0", "1"]), ) result = self.API.retrieve(request) result.pprint() - assert len(result.leaves) == 46 * 2 + assert len(result.leaves) == 10 lats = [] lons = [] @@ -99,4 +101,4 @@ def test_regular_grid(self): # plt.colorbar(label="Temperature") # plt.show() - assert len(eccodes_lats) == 46 * 2 + assert len(eccodes_lats) == 10 From 458531d9749cf392c2c0d8d4eddb1169bdeda791 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 20 Nov 2023 15:49:20 +0100 Subject: [PATCH 193/332] make healpix grid work with cyclic axes --- polytope/datacube/transformations/datacube_mappers.py | 9 +++++---- tests/test_healpix_mapper.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 22478f217..f759a00bf 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -153,7 +153,6 @@ def first_axis_vals(self): vals[4 * self._resolution - 1 - i] = -val # Equator vals[2 * self._resolution - 1] = 0 - return vals def map_first_axis(self, lower, upper): @@ -176,6 +175,8 @@ def second_axis_vals(self, first_val): if self._resolution - 1 <= idx < 2 * self._resolution - 1 or 2 * self._resolution <= idx < 3 * self._resolution: r_start = start * (2 - (((idx + 1) - self._resolution + 1) % 2)) vals = [r_start + i * (360 / (4 * self._resolution)) for i in range(4 * self._resolution)] + if vals[-1] == 360: + vals[-1] = 0 return vals # Equator temp_val = 1 if self._resolution % 2 else 0 @@ -196,17 +197,19 @@ def axes_idx_to_healpix_idx(self, first_idx, second_idx): idx += 4 * (i + 1) else: idx += second_idx + return idx for i in range(self._resolution - 1, 3 * self._resolution): if i != first_idx: idx += 4 * self._resolution else: idx += second_idx + return idx for i in range(3 * self._resolution, 4 * self._resolution - 1): if i != first_idx: idx += 4 * (4 * self._resolution - 1 - i + 1) else: idx += second_idx - return idx + return idx def find_second_idx(self, first_val, second_val): tol = 1e-10 @@ -239,8 +242,6 @@ def unmap(self, first_val, second_val): tol = 1e-8 first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] first_idx = self._first_axis_vals.index(first_val) - print(self.second_axis_vals(first_val)) - print(second_val) second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] second_idx = self.second_axis_vals(first_val).index(second_val) healpix_index = self.axes_idx_to_healpix_idx(first_idx, second_idx) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index 6487bd9ba..bcdc7e3e5 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -60,7 +60,7 @@ def test_healpix_grid(self): ) result = self.API.retrieve(request) result.pprint() - assert len(result.leaves) == 35 + assert len(result.leaves) == 40 lats = [] lons = [] @@ -80,4 +80,4 @@ def test_healpix_grid(self): assert lat <= eccodes_lat + tol assert eccodes_lon - tol <= lon assert lon <= eccodes_lon + tol - assert len(eccodes_lats) == 35 + assert len(eccodes_lats) == 40 From ec07170deae56dfeb6323a6d16b05608904e42ce Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 20 Nov 2023 15:52:15 +0100 Subject: [PATCH 194/332] black --- tests/test_healpix_mapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index bcdc7e3e5..9014e5e3a 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -21,7 +21,7 @@ def setup_method(self, method): "values": { "transformation": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}} }, - "longitude": {"transformation": {"cyclic": [0, 360]}} + "longitude": {"transformation": {"cyclic": [0, 360]}}, } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) From d4697b85e1cb8e278018d0debe52861f3469838c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 21 Nov 2023 10:00:01 +0100 Subject: [PATCH 195/332] fdb axes indices are not always in sorted order --- polytope/datacube/transformations/datacube_type_change.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/polytope/datacube/transformations/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change.py index 8dee75341..cdc046b76 100644 --- a/polytope/datacube/transformations/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change.py @@ -24,7 +24,9 @@ def transformation_axes_final(self): return [self._final_transformation.axis_name] def change_val_type(self, axis_name, values): - return [self._final_transformation.transform_type(val) for val in values] + return_idx = [self._final_transformation.transform_type(val) for val in values] + return_idx.sort() + return return_idx def make_str(self, value): return self._final_transformation.make_str(value) From 4f72f1fad5659e32ca15c282700e224458d7dcab Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 22 Nov 2023 09:39:28 +0100 Subject: [PATCH 196/332] small fixes --- polytope/datacube/backends/fdb.py | 18 ++--- polytope/engine/hullslicer.py | 7 +- tests/test_datacube_axes_init.py | 1 - tests/test_fdb_datacube.py | 2 +- tests/test_incomplete_tree_fdb.py | 97 +++++++++++++++++++++++++++ tests/test_merge_cyclic_octahedral.py | 1 - tests/test_merge_transformation.py | 1 - tests/test_regular_grid.py | 2 +- 8 files changed, 114 insertions(+), 15 deletions(-) create mode 100644 tests/test_incomplete_tree_fdb.py diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 30f3d2a9c..fd4bacfdf 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -4,6 +4,8 @@ from .datacube import Datacube, IndexTree +import time + class FDBDatacube(Datacube): def __init__(self, config={}, axis_options={}): @@ -44,13 +46,11 @@ def remove_unwanted_axes(self, leaf_path): return leaf_path def get(self, requests: IndexTree, leaf_path={}): + time1 = time.time() # First when request node is root, go to its children if requests.axis.name == "root": - if len(requests.children) == 0: - pass - else: - for c in requests.children: - self.get(c) + for c in requests.children: + self.get(c) # If request node has no children, we have a leaf so need to assign fdb values to it else: key_value_path = {requests.axis.name: requests.value} @@ -67,6 +67,8 @@ def get(self, requests: IndexTree, leaf_path={}): else: for c in requests.children: self.get(c, leaf_path) + print("TOTAL GET TIME") + print(time.time() - time1) def get_2nd_last_values(self, requests, leaf_path={}): # In this function, we recursively loop over the last two layers of the tree and store the indices of the @@ -154,11 +156,9 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): interm_request_ranges.append(current_request_ranges) request_ranges_with_idx = list(enumerate(interm_request_ranges)) sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) - sorted_request_ranges = [item[1] for item in sorted_list] - original_indices = [item[0] for item in sorted_list] + original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) - subxarray = self.fdb.extract(fdb_requests) - output_values = subxarray + output_values = self.fdb.extract(fdb_requests) return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index d361fb13c..6b0306ed2 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -48,7 +48,12 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex upper = ax.from_float(upper + tol) flattened = node.flatten() method = polytope.method - for value in datacube.get_indices(flattened, ax, lower, upper, method): + values = datacube.get_indices(flattened, ax, lower, upper, method) + + if len(values) == 0: + node.remove_branch() + + for value in values: # convert to float for slicing fvalue = ax.to_float(value) new_polytope = slice(polytope, ax.name, fvalue, slice_axis_idx) diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 248b59ff4..6f3d59bbf 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -36,7 +36,6 @@ def test_created_axes(self): assert self.datacube._axes["longitude"].has_mapper assert isinstance(self.datacube._axes["longitude"], FloatDatacubeAxis) assert not ("values" in self.datacube._axes.keys()) - print(list(self.datacube._axes["latitude"].find_indexes({}, self.datacube)[:5])) assert list(self.datacube._axes["latitude"].find_indexes({}, self.datacube)[:5]) == [ 89.94618771566562, 89.87647835333229, diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 84b3b2059..9bbfa9bf1 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -28,7 +28,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py new file mode 100644 index 000000000..28c357465 --- /dev/null +++ b/tests/test_incomplete_tree_fdb.py @@ -0,0 +1,97 @@ +import pandas as pd +import pytest +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file +from helper_functions import download_test_data + +from polytope.datacube.backends.fdb import FDBDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Select + + +class TestRegularGrid: + def setup_method(self, method): + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" + download_test_data(nexus_url, "era5-levels-members.grib") + self.options = { + "values": { + "transformation": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}} + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + "number": {"transformation": {"type_change": "int"}}, + "longitude": {"transformation": {"cyclic": [0, 360]}}, + } + self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + def find_nearest_latlon(self, grib_file, target_lat, target_lon): + # Open the GRIB file + f = open(grib_file) + + # Load the GRIB messages from the file + messages = [] + while True: + message = codes_grib_new_from_file(f) + if message is None: + break + messages.append(message) + + # Find the nearest grid points + nearest_points = [] + for message in messages: + nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) + nearest_points.append(nearest_index) + + # Close the GRIB file + f.close() + + return nearest_points + + @pytest.mark.internet + # @pytest.mark.skip(reason="can't install fdb branch on CI") + def test_incomplete_fdb_branch(self): + request = Request( + Select("step", [0]), + Select("levtype", ["pl"]), + Select("date", [pd.Timestamp("20170102T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["129"]), + Select("class", ["ea"]), + Select("stream", ["enda"]), + Select("type", ["an"]), + Select("latitude", [0]), + Select("longitude", [1]), + Select("levelist", ["500"]), + Select("number", ["0"]), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + assert result.is_root() + + @pytest.mark.internet + # @pytest.mark.skip(reason="can't install fdb branch on CI") + def test_incomplete_fdb_branch_2(self): + request = Request( + Select("step", [0]), + Select("levtype", ["pl"]), + Select("date", [pd.Timestamp("20170102T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["129"]), + Select("class", ["ea"]), + Select("stream", ["enda"]), + Select("type", ["an"]), + Select("latitude", [1]), + Select("longitude", [0]), + Select("levelist", ["500"]), + Select("number", ["0"]), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + assert result.is_root() diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index 7f633c0be..6fcf16323 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -34,7 +34,6 @@ def setup_method(self, method): self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) - # @pytest.mark.skip(reason="Need date time to not be strings") def test_merge_axis(self): # NOTE: does not work because the date is a string in the merge option... date = np.datetime64("2000-01-01T06:00:00") diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index fe15cf987..63b7a39cd 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -27,5 +27,4 @@ def setup_method(self, method): def test_merge_axis(self): request = Request(Select("date", [pd.Timestamp("2000-01-01T06:00:00")])) result = self.API.retrieve(request) - # assert result.leaves[0].flatten()["date"] == np.datetime64("2000-01-01T06:00:00") assert result.leaves[0].flatten()["date"] == pd.Timestamp("2000-01-01T06:00:00") diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 86337b6f0..e811a9c15 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -54,7 +54,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_regular_grid(self): request = Request( Select("step", [0]), From 27470ce6171b181a47dfa3832915ff8e2684bca2 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 22 Nov 2023 10:46:12 +0100 Subject: [PATCH 197/332] add repr for shapes and Point shape which supports surrounding method --- polytope/polytope.py | 6 +++ polytope/shapes.py | 66 +++++++++++++++++++++++++++++ tests/data/era5-levels-members.grib | 3 ++ tests/data/foo.grib | 3 ++ tests/data/healpix.grib | 3 ++ tests/test_point_shape.py | 36 ++++++++++++++++ 6 files changed, 117 insertions(+) create mode 100644 tests/data/era5-levels-members.grib create mode 100644 tests/data/foo.grib create mode 100644 tests/data/healpix.grib create mode 100644 tests/test_point_shape.py diff --git a/polytope/polytope.py b/polytope/polytope.py index 5e8fc7133..f6d4a723e 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -29,6 +29,12 @@ def polytopes(self): polytopes.extend(shape.polytope()) return polytopes + def __repr__(self): + return_str = "" + for shape in self.shapes: + return_str += shape.__repr__() + "\n" + return return_str + class Polytope: def __init__(self, datacube, engine=None, axis_options={}): diff --git a/polytope/shapes.py b/polytope/shapes.py index 94562abc5..5a824533a 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -38,6 +38,9 @@ def extents(self, axis): def __str__(self): return f"Polytope in {self.axes} with points {self.points}" + def __repr__(self): + return f"Polytope in {self.axes} with points {self.points}" + def axes(self): return self._axes @@ -60,6 +63,31 @@ def axes(self): def polytope(self): return [ConvexPolytope([self.axis], [[v]], self.method) for v in self.values] + def __repr__(self): + return f"Select in {self.axis} with points {self.values}" + + +class Point(Shape): + """Matches several discrete value""" + + def __init__(self, axes, values, method=None): + self._axes = axes + self.values = values + self.method = method + self.polytopes = [] + for i in range(len(axes)): + polytope_points = [v[i] for v in self.values] + self.polytopes.append(ConvexPolytope([axes[i]], [polytope_points], method)) + + def axes(self): + return self._axes + + def polytope(self): + return self.polytopes + + def __repr__(self): + return f"Point in {self._axes} with points {self.values}" + class Span(Shape): """1-D range along a single axis""" @@ -77,6 +105,9 @@ def axes(self): def polytope(self): return [ConvexPolytope([self.axis], [[self.lower], [self.upper]])] + def __repr__(self): + return f"Span in {self.axis} with range from {self.lower} to {self.upper}" + class All(Span): """Matches all indices in an axis""" @@ -84,12 +115,17 @@ class All(Span): def __init__(self, axis): super().__init__(axis) + def __repr__(self): + return f"All in {self.axis}" + class Box(Shape): """N-D axis-aligned bounding box (AABB), specified by two opposite corners""" def __init__(self, axes, lower_corner=None, upper_corner=None): dimension = len(axes) + self._lower_corner = lower_corner + self._upper_corner = upper_corner self._axes = axes assert len(lower_corner) == dimension assert len(upper_corner) == dimension @@ -121,6 +157,9 @@ def axes(self): def polytope(self): return [ConvexPolytope(self.axes(), self.vertices)] + def __repr__(self): + return f"Box in {self._axes} with with lower corner {self._lower_corner} and upper corner{self._upper_corner}" + class Disk(Shape): """2-D shape bounded by an ellipse""" @@ -159,6 +198,9 @@ def axes(self): def polytope(self): return [ConvexPolytope(self.axes(), self.points)] + def __repr__(self): + return f"Disk in {self._axes} with centred at {self.centre} and with radius {self.radius}" + class Ellipsoid(Shape): # Here we use the formula for the inscribed circle in an icosahedron @@ -211,12 +253,18 @@ def _icosahedron_edge_length_coeff(self): def polytope(self): return [ConvexPolytope(self.axes(), self.points)] + def __repr__(self): + return f"Ellipsoid in {self._axes} with centred at {self.centre} and with radius {self.radius}" + class PathSegment(Shape): """N-D polytope defined by a shape which is swept along a straight line between two points""" def __init__(self, axes, shape: Shape, start: List, end: List): self._axes = axes + self._start = start + self._end = end + self._shape = shape assert shape.axes() == self.axes() assert len(start) == len(self.axes()) @@ -240,12 +288,18 @@ def axes(self): def polytope(self): return self.polytopes + def __repr__(self): + return f"PathSegment in {self._axes} obtained by sweeping a {self._shape.__repr__()} \ + between the points {self._start} and {self._end}" + class Path(Shape): """N-D polytope defined by a shape which is swept along a polyline defined by multiple points""" def __init__(self, axes, shape, *points, closed=False): self._axes = axes + self._shape = shape + self._points = points assert shape.axes() == self.axes() for p in points: @@ -267,6 +321,10 @@ def axes(self): def polytope(self): return self.union.polytope() + def __repr__(self): + return f"Path in {self._axes} obtained by sweeping a {self._shape.__repr__()} \ + between the points {self._points}" + class Union(Shape): """N-D union of two shapes with the same axes""" @@ -277,6 +335,7 @@ def __init__(self, axes, *shapes): assert s.axes() == self.axes() self.polytopes = [] + self._shapes = shapes for s in shapes: self.polytopes.extend(s.polytope()) @@ -287,6 +346,9 @@ def axes(self): def polytope(self): return self.polytopes + def __repr__(self): + return f"Union in {self._axes} of the shapes {self._shapes}" + class Polygon(Shape): """2-D polygon defined by a set of exterior points""" @@ -297,6 +359,7 @@ def __init__(self, axes, points): for p in points: assert len(p) == 2 + self._points = points triangles = tripy.earclip(points) self.polytopes = [] @@ -313,3 +376,6 @@ def axes(self): def polytope(self): return self.polytopes + + def __repr__(self): + return f"Polygon in {self._axes} with points {self._points}" diff --git a/tests/data/era5-levels-members.grib b/tests/data/era5-levels-members.grib new file mode 100644 index 000000000..90d45deed --- /dev/null +++ b/tests/data/era5-levels-members.grib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c144fde61ca5d53702bf6f212775ef2cc783bdd66b6865160bf597c1b35ed898 +size 2361600 diff --git a/tests/data/foo.grib b/tests/data/foo.grib new file mode 100644 index 000000000..9c5efa68b --- /dev/null +++ b/tests/data/foo.grib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12533bae9b3aa078a3298a82f289c7dbba6dbdb9a21e4e5e24e84b8695a75dee +size 26409360 diff --git a/tests/data/healpix.grib b/tests/data/healpix.grib new file mode 100644 index 000000000..693c98c12 --- /dev/null +++ b/tests/data/healpix.grib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df57790df280627b2883c01fa4b56986d9938ed69b7434fe09c4bede99d2895f +size 37030 diff --git a/tests/test_point_shape.py b/tests/test_point_shape.py new file mode 100644 index 000000000..0bc203d61 --- /dev/null +++ b/tests/test_point_shape.py @@ -0,0 +1,36 @@ +import numpy as np +import pandas as pd +import xarray as xr + +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Point, Select + + +class TestSlicing3DXarrayDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + array = xr.DataArray( + np.random.randn(3, 6, 129), + dims=("date", "step", "level"), + coords={ + "date": pd.date_range("2000-01-01", "2000-01-03", 3), + "step": [0, 3, 6, 9, 12, 15], + "level": range(1, 130), + }, + ) + self.xarraydatacube = XArrayDatacube(array) + self.slicer = HullSlicer() + self.API = Polytope(datacube=array, engine=self.slicer) + + def test_point(self): + request = Request(Point(["step", "level"], [[3, 10]]), Select("date", ["2000-01-01"])) + result = self.API.retrieve(request) + assert len(result.leaves) == 1 + assert result.leaves[0].axis.name == "level" + + def test_point_surrounding_step(self): + request = Request(Point(["step", "level"], [[2, 10]], method="surrounding"), Select("date", ["2000-01-01"])) + result = self.API.retrieve(request) + assert len(result.leaves) == 6 From 7cffad10e0909ea26dd3ebf6d65f9ed5896e4323 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 22 Nov 2023 10:50:34 +0100 Subject: [PATCH 198/332] add grib to gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 037872e66..7b8e19f62 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ polytope.egg-info .pytest_cache *.prof -*.idx \ No newline at end of file +*.idx +*.grib \ No newline at end of file From fe730ea4268b25a0a146b56c1784ee38d46aaaa2 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 22 Nov 2023 10:51:18 +0100 Subject: [PATCH 199/332] remove data from tests --- tests/data/era5-levels-members.grib | 3 --- tests/data/foo.grib | 3 --- tests/data/healpix.grib | 3 --- 3 files changed, 9 deletions(-) delete mode 100644 tests/data/era5-levels-members.grib delete mode 100644 tests/data/foo.grib delete mode 100644 tests/data/healpix.grib diff --git a/tests/data/era5-levels-members.grib b/tests/data/era5-levels-members.grib deleted file mode 100644 index 90d45deed..000000000 --- a/tests/data/era5-levels-members.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c144fde61ca5d53702bf6f212775ef2cc783bdd66b6865160bf597c1b35ed898 -size 2361600 diff --git a/tests/data/foo.grib b/tests/data/foo.grib deleted file mode 100644 index 9c5efa68b..000000000 --- a/tests/data/foo.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:12533bae9b3aa078a3298a82f289c7dbba6dbdb9a21e4e5e24e84b8695a75dee -size 26409360 diff --git a/tests/data/healpix.grib b/tests/data/healpix.grib deleted file mode 100644 index 693c98c12..000000000 --- a/tests/data/healpix.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:df57790df280627b2883c01fa4b56986d9938ed69b7434fe09c4bede99d2895f -size 37030 From 8a77ed32c894877d35bb50951ffe7014258355c8 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 22 Nov 2023 10:56:53 +0100 Subject: [PATCH 200/332] remove double print for convex polytope --- polytope/shapes.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/polytope/shapes.py b/polytope/shapes.py index 5a824533a..306d1a64a 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -38,9 +38,6 @@ def extents(self, axis): def __str__(self): return f"Polytope in {self.axes} with points {self.points}" - def __repr__(self): - return f"Polytope in {self.axes} with points {self.points}" - def axes(self): return self._axes From 487e6e9e848e6bc2288237cc2f62375382545aaa Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 22 Nov 2023 14:49:41 +0100 Subject: [PATCH 201/332] make All shape work --- polytope/datacube/datacube_axis.py | 5 ++ polytope/shapes.py | 2 +- tests/test_shapes.py | 74 ++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 tests/test_shapes.py diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 77a223710..625051f7c 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -1,3 +1,4 @@ +import math from abc import ABC, abstractmethod from copy import deepcopy from typing import Any, List @@ -18,6 +19,10 @@ def update_range(): def to_intervals(range): update_range() + if range[0] == -math.inf: + range[0] = cls.range[0] + if range[1] == math.inf: + range[1] = cls.range[1] axis_lower = cls.range[0] axis_upper = cls.range[1] axis_range = axis_upper - axis_lower diff --git a/polytope/shapes.py b/polytope/shapes.py index 306d1a64a..cc77273a7 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -89,7 +89,7 @@ def __repr__(self): class Span(Shape): """1-D range along a single axis""" - def __init__(self, axis, lower=None, upper=None): + def __init__(self, axis, lower=-math.inf, upper=math.inf): assert not isinstance(lower, list) assert not isinstance(upper, list) self.axis = axis diff --git a/tests/test_shapes.py b/tests/test_shapes.py new file mode 100644 index 000000000..ba355dd4c --- /dev/null +++ b/tests/test_shapes.py @@ -0,0 +1,74 @@ +import numpy as np +import pandas as pd +import pytest +import xarray as xr + +from polytope.datacube.backends.FDB_datacube import FDBDatacube +from polytope.datacube.backends.xarray import XArrayDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import All, Select, Span + + +class TestSlicing3DXarrayDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + array = xr.DataArray( + np.random.randn(3, 6, 129, 360), + dims=("date", "step", "level", "longitude"), + coords={ + "date": pd.date_range("2000-01-01", "2000-01-03", 3), + "step": [0, 3, 6, 9, 12, 15], + "level": range(1, 130), + "longitude": range(0, 360), + }, + ) + self.xarraydatacube = XArrayDatacube(array) + self.options = {"longitude": {"transformation": {"cyclic": [0, 360]}}} + self.slicer = HullSlicer() + self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) + + def test_all(self): + request = Request(Select("step", [3]), Select("date", ["2000-01-01"]), All("level"), Select("longitude", [1])) + result = self.API.retrieve(request) + assert len(result.leaves) == 129 + + def test_all_cyclic(self): + request = Request(Select("step", [3]), Select("date", ["2000-01-01"]), Select("level", [1]), All("longitude")) + result = self.API.retrieve(request) + # result.pprint() + assert len(result.leaves) == 360 + + @pytest.mark.skip(reason="can't install fdb branch on CI") + def test_all_mapper_cyclic(self): + self.options = { + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + } + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + "longitude": {"transformation": {"cyclic": [0, 360]}}, + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + request = Request( + Select("step", [11]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20230710T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["151130"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Span("latitude", 89.9, 90), + All("longitude"), + ) + result = self.API.retrieve(request) + # result.pprint() + assert len(result.leaves) == 20 From 086b343c65a8f0ac698b651d19fcef1635ce6087 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 22 Nov 2023 10:46:12 +0100 Subject: [PATCH 202/332] add repr for shapes and Point shape which supports surrounding method --- polytope/shapes.py | 3 +++ tests/data/era5-levels-members.grib | 3 +++ tests/data/foo.grib | 3 +++ tests/data/healpix.grib | 3 +++ 4 files changed, 12 insertions(+) create mode 100644 tests/data/era5-levels-members.grib create mode 100644 tests/data/foo.grib create mode 100644 tests/data/healpix.grib diff --git a/polytope/shapes.py b/polytope/shapes.py index cc77273a7..d79ccd7af 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -38,6 +38,9 @@ def extents(self, axis): def __str__(self): return f"Polytope in {self.axes} with points {self.points}" + def __repr__(self): + return f"Polytope in {self.axes} with points {self.points}" + def axes(self): return self._axes diff --git a/tests/data/era5-levels-members.grib b/tests/data/era5-levels-members.grib new file mode 100644 index 000000000..90d45deed --- /dev/null +++ b/tests/data/era5-levels-members.grib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c144fde61ca5d53702bf6f212775ef2cc783bdd66b6865160bf597c1b35ed898 +size 2361600 diff --git a/tests/data/foo.grib b/tests/data/foo.grib new file mode 100644 index 000000000..9c5efa68b --- /dev/null +++ b/tests/data/foo.grib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12533bae9b3aa078a3298a82f289c7dbba6dbdb9a21e4e5e24e84b8695a75dee +size 26409360 diff --git a/tests/data/healpix.grib b/tests/data/healpix.grib new file mode 100644 index 000000000..693c98c12 --- /dev/null +++ b/tests/data/healpix.grib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df57790df280627b2883c01fa4b56986d9938ed69b7434fe09c4bede99d2895f +size 37030 From a14f976bed5cf4339edbcf23b7eeb1d18271dcf7 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 22 Nov 2023 10:51:18 +0100 Subject: [PATCH 203/332] remove data from tests --- tests/data/era5-levels-members.grib | 3 --- tests/data/foo.grib | 3 --- tests/data/healpix.grib | 3 --- 3 files changed, 9 deletions(-) delete mode 100644 tests/data/era5-levels-members.grib delete mode 100644 tests/data/foo.grib delete mode 100644 tests/data/healpix.grib diff --git a/tests/data/era5-levels-members.grib b/tests/data/era5-levels-members.grib deleted file mode 100644 index 90d45deed..000000000 --- a/tests/data/era5-levels-members.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c144fde61ca5d53702bf6f212775ef2cc783bdd66b6865160bf597c1b35ed898 -size 2361600 diff --git a/tests/data/foo.grib b/tests/data/foo.grib deleted file mode 100644 index 9c5efa68b..000000000 --- a/tests/data/foo.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:12533bae9b3aa078a3298a82f289c7dbba6dbdb9a21e4e5e24e84b8695a75dee -size 26409360 diff --git a/tests/data/healpix.grib b/tests/data/healpix.grib deleted file mode 100644 index 693c98c12..000000000 --- a/tests/data/healpix.grib +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:df57790df280627b2883c01fa4b56986d9938ed69b7434fe09c4bede99d2895f -size 37030 From 1b8facba15b0d59ff10d0d447789b418b64f210f Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 22 Nov 2023 10:56:53 +0100 Subject: [PATCH 204/332] remove double print for convex polytope --- polytope/shapes.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/polytope/shapes.py b/polytope/shapes.py index d79ccd7af..cc77273a7 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -38,9 +38,6 @@ def extents(self, axis): def __str__(self): return f"Polytope in {self.axes} with points {self.points}" - def __repr__(self): - return f"Polytope in {self.axes} with points {self.points}" - def axes(self): return self._axes From 3312772dda1b2e4e1de1f53d5568d86c32a82a68 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 23 Nov 2023 09:43:18 +0100 Subject: [PATCH 205/332] clean up branch --- polytope/datacube/backends/fdb.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index fd4bacfdf..0679dbdd0 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -4,8 +4,6 @@ from .datacube import Datacube, IndexTree -import time - class FDBDatacube(Datacube): def __init__(self, config={}, axis_options={}): @@ -46,7 +44,6 @@ def remove_unwanted_axes(self, leaf_path): return leaf_path def get(self, requests: IndexTree, leaf_path={}): - time1 = time.time() # First when request node is root, go to its children if requests.axis.name == "root": for c in requests.children: @@ -67,8 +64,6 @@ def get(self, requests: IndexTree, leaf_path={}): else: for c in requests.children: self.get(c, leaf_path) - print("TOTAL GET TIME") - print(time.time() - time1) def get_2nd_last_values(self, requests, leaf_path={}): # In this function, we recursively loop over the last two layers of the tree and store the indices of the From 2d4ea3f1e8d6cbd23dadf42ca4a49694dd5e5a72 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 23 Nov 2023 10:45:46 +0100 Subject: [PATCH 206/332] make date axis a pandas datetime axis --- .../transformations/datacube_merger.py | 6 ++- tests/test_fdb_datacube.py | 4 +- tests/test_merge_transformation.py | 4 +- tests/test_regular_grid.py | 4 +- tests/test_slice_date_range_fdb.py | 44 +++++++++++++++++++ 5 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 tests/test_slice_date_range_fdb.py diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index e91354c4f..d60278671 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -18,6 +18,9 @@ def blocked_axes(self): def unwanted_axes(self): return [] + def _mapped_axes(self): + return self._first_axis + def merged_values(self, datacube): first_ax_vals = datacube.ax_vals(self.name) second_ax_name = self._second_axis @@ -56,4 +59,5 @@ def unmerge(self, merged_val): return (first_val, second_val) def change_val_type(self, axis_name, values): - return values + new_values = pd.to_datetime(values) + return new_values diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 9bbfa9bf1..ff56f6709 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -19,7 +19,7 @@ def setup_method(self, method): "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} } }, - "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} @@ -28,7 +28,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index 63b7a39cd..ac3abf5bb 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -19,12 +19,12 @@ def setup_method(self, method): "time": ["0600"], }, ) - self.options = {"date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}} + self.options = {"date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}} self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) def test_merge_axis(self): - request = Request(Select("date", [pd.Timestamp("2000-01-01T06:00:00")])) + request = Request(Select("date", [pd.Timestamp("20000101T060000")])) result = self.API.retrieve(request) assert result.leaves[0].flatten()["date"] == pd.Timestamp("2000-01-01T06:00:00") diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index e811a9c15..1de8a8705 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -20,7 +20,7 @@ def setup_method(self, method): "values": { "transformation": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}} }, - "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, "number": {"transformation": {"type_change": "int"}}, "longitude": {"transformation": {"cyclic": [0, 360]}}, @@ -54,7 +54,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_regular_grid(self): request = Request( Select("step", [0]), diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py new file mode 100644 index 000000000..567230687 --- /dev/null +++ b/tests/test_slice_date_range_fdb.py @@ -0,0 +1,44 @@ +import pandas as pd +import pytest + +from polytope.datacube.backends.fdb import FDBDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select, Span + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + } + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.skip(reason="can't install fdb branch on CI") + def test_fdb_datacube(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Span("date", pd.Timestamp("20230625T120000"), pd.Timestamp("20230626T120000")), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 9 From 80d56c090097dd6867f6886877ffeb59e761c65a Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 23 Nov 2023 11:55:57 +0100 Subject: [PATCH 207/332] fix naming problem in tests --- tests/test_shapes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_shapes.py b/tests/test_shapes.py index ba355dd4c..ebc55b527 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -3,7 +3,7 @@ import pytest import xarray as xr -from polytope.datacube.backends.FDB_datacube import FDBDatacube +from polytope.datacube.backends.fdb import FDBDatacube from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request From 0086059555ac63174140cee1ed513f7edc5786fc Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 23 Nov 2023 11:59:14 +0100 Subject: [PATCH 208/332] remove tests that depend on fdb --- tests/test_incomplete_tree_fdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 28c357465..87d55aef1 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -51,7 +51,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch(self): request = Request( Select("step", [0]), @@ -74,7 +74,7 @@ def test_incomplete_fdb_branch(self): assert result.is_root() @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch_2(self): request = Request( Select("step", [0]), From ce0c4c536f22195a01fbca4682316f530f05410e Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Dec 2023 10:13:54 +0100 Subject: [PATCH 209/332] quick fix to surrounding method --- .../fdb_slice_many_numbers_timeseries.py | 56 +++++++++++++++++++ polytope/datacube/backends/datacube.py | 3 +- polytope/datacube/datacube_axis.py | 15 +++-- polytope/shapes.py | 2 +- 4 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 performance/fdb_slice_many_numbers_timeseries.py diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py new file mode 100644 index 000000000..29a4dd1ed --- /dev/null +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -0,0 +1,56 @@ +import time + +import pandas as pd +import pytest + +from polytope.datacube.backends.fdb import FDBDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import All, Point, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + } + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + "number": {"transformation": {"type_change": "int"}}, + "longitude": {"transformation": {"cyclic": [0, 360]}} + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.skip(reason="can't install fdb branch on CI") + def test_fdb_datacube(self): + request = Request( + # Select("step", [0]), + All("step"), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231205T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["enfo"]), + Select("type", ["pf"]), + # Select("latitude", [0.035149384216], method="surrounding"), + # Select("latitude", [0.04], method="surrounding"), + # Select("longitude", [0], method="surrounding"), + Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"), + All("number") + ) + time1 = time.time() + result = self.API.retrieve(request) + print(time.time() - time1) + print(len(result.leaves)) + # result.pprint() + # assert len(result.leaves) == 9 diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 10b5a6613..9d1ba9f3e 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -40,7 +40,6 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt for unwanted_axis in transformation.unwanted_axes(): self.unwanted_axes.append(unwanted_axis) for axis_name in final_axis_names: - self.complete_axes.append(axis_name) self.fake_axes.append(axis_name) # if axis does not yet exist, create it @@ -84,7 +83,7 @@ def has_index(self, path: DatacubePath, axis, index): def fit_path(self, path): for key in path.keys(): - if key not in self.complete_axes: + if key not in self.complete_axes and key not in self.fake_axes: path.pop(key) return path diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index a84bfd303..5c78c22f6 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -155,8 +155,9 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") else: - start = indexes.index(low) - end = indexes.index(up) + start = bisect.bisect_left(indexes, low) + end = bisect.bisect_right(indexes, up) + if start - 1 < 0: index_val_found = indexes[-1:][0] indexes_between_ranges.append([index_val_found]) @@ -264,8 +265,14 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): if cls.name in transformation._mapped_axes(): for idxs in index_ranges: if method == "surrounding": - start = idxs.index(low) - end = idxs.index(up) + axis_reversed = transform._axis_reversed[cls.name] + if not axis_reversed: + start = bisect.bisect_left(idxs, low) + end = bisect.bisect_right(idxs, up) + else: + # TODO: do the custom bisect + end = bisect_left_cmp(idxs, low, cmp=lambda x, y: x > y) + 1 + start = bisect_right_cmp(idxs, up, cmp=lambda x, y: x > y) start = max(start - 1, 0) end = min(end + 1, len(idxs)) indexes_between = idxs[start:end] diff --git a/polytope/shapes.py b/polytope/shapes.py index 5653f521e..c10b12e4b 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -74,7 +74,7 @@ def __init__(self, axes, values, method=None): self.polytopes = [] for i in range(len(axes)): polytope_points = [v[i] for v in self.values] - self.polytopes.append(ConvexPolytope([axes[i]], [polytope_points], method)) + self.polytopes.append(ConvexPolytope([axes[i]], [polytope_points], self.method)) def axes(self): return self._axes From 34da743bdb20592c3aa3fbcf1f1e6fbd12db0915 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Dec 2023 10:46:02 +0100 Subject: [PATCH 210/332] black --- performance/fdb_slice_many_numbers_timeseries.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index 29a4dd1ed..c6174f918 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -21,7 +21,7 @@ def setup_method(self, method): "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, "number": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}} + "longitude": {"transformation": {"cyclic": [0, 360]}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -46,7 +46,7 @@ def test_fdb_datacube(self): # Select("latitude", [0.04], method="surrounding"), # Select("longitude", [0], method="surrounding"), Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"), - All("number") + All("number"), ) time1 = time.time() result = self.API.retrieve(request) From dd5911f28308a3e674477fa026105caa8a953f7a Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Dec 2023 11:04:30 +0100 Subject: [PATCH 211/332] time components in example --- .../fdb_slice_many_numbers_timeseries.py | 80 ++++++++----------- 1 file changed, 34 insertions(+), 46 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index c6174f918..c65ce2084 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -1,56 +1,44 @@ import time import pandas as pd -import pytest from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import All, Point, Select +time1 = time.time() +# Create a dataarray with 3 labelled axes using different index types +options = { + "values": { + "transformation": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + "number": {"transformation": {"type_change": "int"}}, + "longitude": {"transformation": {"cyclic": [0, 360]}}, +} +config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"} +fdbdatacube = FDBDatacube(config, axis_options=options) +slicer = HullSlicer() +self_API = Polytope(datacube=fdbdatacube, engine=slicer, axis_options=options) -class TestSlicingFDBDatacube: - def setup_method(self, method): - # Create a dataarray with 3 labelled axes using different index types - self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, - "number": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}}, - } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"} - self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) - self.slicer = HullSlicer() - self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) - - # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") - def test_fdb_datacube(self): - request = Request( - # Select("step", [0]), - All("step"), - Select("levtype", ["sfc"]), - Select("date", [pd.Timestamp("20231205T000000")]), - Select("domain", ["g"]), - Select("expver", ["0001"]), - Select("param", ["167"]), - Select("class", ["od"]), - Select("stream", ["enfo"]), - Select("type", ["pf"]), - # Select("latitude", [0.035149384216], method="surrounding"), - # Select("latitude", [0.04], method="surrounding"), - # Select("longitude", [0], method="surrounding"), - Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"), - All("number"), - ) - time1 = time.time() - result = self.API.retrieve(request) - print(time.time() - time1) - print(len(result.leaves)) - # result.pprint() - # assert len(result.leaves) == 9 +time2 = time.time() +request = Request( + All("step"), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231205T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["enfo"]), + Select("type", ["pf"]), + # Select("latitude", [0.035149384216], method="surrounding"), + Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"), + All("number"), +) +result = self_API.retrieve(request) +print(time.time() - time1) +print(time.time() - time2) +print(len(result.leaves)) From 583b0f0540cf8db849830bf61cbe2c4fa8b4cf14 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Dec 2023 13:57:58 +0100 Subject: [PATCH 212/332] remove the transformation keyword in datacube options --- .../fdb_slice_many_numbers_timeseries.py | 12 +++++------- polytope/datacube/backends/datacube.py | 9 +++------ polytope/datacube/backends/fdb.py | 11 ++--------- polytope/datacube/backends/xarray.py | 12 +++++------- tests/test_cyclic_axis_over_negative_vals.py | 4 ++-- tests/test_cyclic_axis_slicer_not_0.py | 4 ++-- tests/test_cyclic_axis_slicing.py | 4 ++-- tests/test_cyclic_simple.py | 2 +- tests/test_cyclic_snapping.py | 2 +- tests/test_datacube_axes_init.py | 6 +----- tests/test_fdb_datacube.py | 15 +++++++-------- tests/test_healpix_mapper.py | 6 ++---- tests/test_incomplete_tree_fdb.py | 16 +++++++--------- tests/test_merge_cyclic_octahedral.py | 10 +++------- tests/test_merge_octahedral_one_axis.py | 8 ++------ tests/test_merge_transformation.py | 2 +- tests/test_octahedral_grid.py | 6 +----- tests/test_regular_grid.py | 14 ++++++-------- tests/test_reverse_transformation.py | 2 +- tests/test_shapes.py | 18 ++++++++---------- tests/test_slice_date_range_fdb.py | 14 ++++++-------- tests/test_slicer_era5.py | 2 +- tests/test_snapping_real_data.py | 4 ++-- tests/test_type_change_transformation.py | 2 +- 24 files changed, 72 insertions(+), 113 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index c65ce2084..45f67699f 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -10,13 +10,11 @@ time1 = time.time() # Create a dataarray with 3 labelled axes using different index types options = { - "values": { - "transformation": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, - "number": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}}, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "longitude": {"cyclic": [0, 360]}, } config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"} fdbdatacube = FDBDatacube(config, axis_options=options) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 9d1ba9f3e..efa20e93d 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -37,8 +37,6 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt ) for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) - for unwanted_axis in transformation.unwanted_axes(): - self.unwanted_axes.append(unwanted_axis) for axis_name in final_axis_names: self.fake_axes.append(axis_name) # if axis does not yet exist, create it @@ -61,12 +59,11 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt self._axes[axis_name].transformations.append(transformation) def _add_all_transformation_axes(self, options, name, values): - transformation_options = options["transformation"] - for transformation_type_key in transformation_options.keys(): - self._create_axes(name, values, transformation_type_key, transformation_options) + for transformation_type_key in options.keys(): + self._create_axes(name, values, transformation_type_key, options) def _check_and_add_axes(self, options, name, values): - if "transformation" in options: + if options is not None: self._add_all_transformation_axes(options, name, values) else: if name not in self.blocked_axes: diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 0679dbdd0..bab3fc2a6 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -11,10 +11,8 @@ def __init__(self, config={}, axis_options={}): self.axis_counter = 0 self._axes = None treated_axes = [] - self.non_complete_axes = [] self.complete_axes = [] self.blocked_axes = [] - self.unwanted_axes = [] self.fake_axes = [] self.unwanted_path = {} @@ -26,7 +24,7 @@ def __init__(self, config={}, axis_options={}): self.fdb_coordinates["values"] = [] for name, values in self.fdb_coordinates.items(): values.sort() - options = axis_options.get(name, {}) + options = axis_options.get(name, None) self._check_and_add_axes(options, name, values) treated_axes.append(name) self.complete_axes.append(name) @@ -34,15 +32,10 @@ def __init__(self, config={}, axis_options={}): # add other options to axis which were just created above like "lat" for the mapper transformations for eg for name in self._axes: if name not in treated_axes: - options = axis_options.get(name, {}) + options = axis_options.get(name, None) val = self._axes[name].type self._check_and_add_axes(options, name, val) - def remove_unwanted_axes(self, leaf_path): - for axis in self.unwanted_axes: - leaf_path.pop(axis) - return leaf_path - def get(self, requests: IndexTree, leaf_path={}): # First when request node is root, go to its children if requests.axis.name == "root": diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 650038e05..f8ca1c2e2 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -14,33 +14,31 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self._axes = None self.dataarray = dataarray treated_axes = [] - self.non_complete_axes = [] self.complete_axes = [] self.blocked_axes = [] self.fake_axes = [] - self.unwanted_axes = [] + for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: - options = axis_options.get(name, {}) + options = axis_options.get(name, None) self._check_and_add_axes(options, name, values) treated_axes.append(name) self.complete_axes.append(name) else: if self.dataarray[name].dims == (): - options = axis_options.get(name, {}) + options = axis_options.get(name, None) self._check_and_add_axes(options, name, values) treated_axes.append(name) - self.non_complete_axes.append(name) for name in dataarray.dims: if name not in treated_axes: - options = axis_options.get(name, {}) + options = axis_options.get(name, None) val = dataarray[name].values[0] self._check_and_add_axes(options, name, val) treated_axes.append(name) # add other options to axis which were just created above like "lat" for the mapper transformations for eg for name in self._axes: if name not in treated_axes: - options = axis_options.get(name, {}) + options = axis_options.get(name, None) val = self._axes[name].type self._check_and_add_axes(options, name, val) diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index 9008f9b82..52cd41c21 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -22,8 +22,8 @@ def setup_method(self, method): }, ) options = { - "long": {"transformation": {"cyclic": [-1.1, -0.1]}}, - "level": {"transformation": {"cyclic": [1, 129]}}, + "long": {"cyclic": [-1.1, -0.1]}, + "level": {"cyclic": [1, 129]}, } self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 12d39b566..8266c0a09 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -22,8 +22,8 @@ def setup_method(self, method): }, ) self.options = { - "long": {"transformation": {"cyclic": [-1.1, -0.1]}}, - "level": {"transformation": {"cyclic": [1, 129]}}, + "long": {"cyclic": [-1.1, -0.1]}, + "level": {"cyclic": [1, 129]}, } self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index 34c2cf6c8..c0ac914f5 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -22,8 +22,8 @@ def setup_method(self, method): }, ) self.options = { - "long": {"transformation": {"cyclic": [0, 1.0]}}, - "level": {"transformation": {"cyclic": [1, 129]}}, + "long": {"cyclic": [0, 1.0]}, + "level": {"cyclic": [1, 129]}, } self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() diff --git a/tests/test_cyclic_simple.py b/tests/test_cyclic_simple.py index bf6660ee7..f3bd1e28b 100644 --- a/tests/test_cyclic_simple.py +++ b/tests/test_cyclic_simple.py @@ -21,7 +21,7 @@ def setup_method(self, method): "long": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], }, ) - options = {"long": {"transformation": {"cyclic": [0, 1.0]}}, "level": {"transformation": {"cyclic": [1, 129]}}} + options = {"long": {"cyclic": [0, 1.0]}, "level": {"cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_cyclic_snapping.py b/tests/test_cyclic_snapping.py index ab6189ba3..adb3a7959 100644 --- a/tests/test_cyclic_snapping.py +++ b/tests/test_cyclic_snapping.py @@ -16,7 +16,7 @@ def setup_method(self, method): "long": [0, 0.5, 1.0], }, ) - options = {"long": {"transformation": {"cyclic": [0, 1.0]}}} + options = {"long": {"cyclic": [0, 1.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 6f3d59bbf..f2a7f3eb4 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -19,11 +19,7 @@ def setup_method(self, method): latlon_array = latlon_array.t2m self.xarraydatacube = XArrayDatacube(latlon_array) self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, # "latitude": {"transformation": {"reverse": {True}}}, } self.slicer = HullSlicer() diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index ff56f6709..03bc1084c 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -14,21 +14,19 @@ class TestSlicingFDBDatacube: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + print(self.fdbdatacube.fdb_coordinates) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), @@ -40,6 +38,7 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), + Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index 9014e5e3a..043f1ea12 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -18,10 +18,8 @@ def setup_method(self, method): self.latlon_array = ds.to_xarray().isel(step=0).isel(time=0).isel(isobaricInhPa=0).z self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { - "values": { - "transformation": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}} - }, - "longitude": {"transformation": {"cyclic": [0, 360]}}, + "values": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}}, + "longitude": {"cyclic": [0, 360]}, } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 87d55aef1..6f5f11ecf 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -14,13 +14,11 @@ def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") self.options = { - "values": { - "transformation": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}} - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, - "number": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}}, + "values": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": [" ", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "longitude": {"cyclic": [0, 360]}, } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -51,7 +49,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch(self): request = Request( Select("step", [0]), @@ -74,7 +72,7 @@ def test_incomplete_fdb_branch(self): assert result.is_root() @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch_2(self): request = Request( Select("step", [0]), diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index 6fcf16323..fd50b6b1a 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -22,13 +22,9 @@ def setup_method(self, method): }, ) self.options = { - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, - "step": {"transformation": {"cyclic": [0, 2]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "step": {"cyclic": [0, 2]}, } self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index 5ff5dae91..7c3764904 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -18,12 +18,8 @@ def setup_method(self, method): self.latlon_array = self.latlon_array.t2m self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, - "longitude": {"transformation": {"cyclic": [0, 360.0]}}, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "longitude": {"cyclic": [0, 360.0]}, } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index ac3abf5bb..28f075d56 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -19,7 +19,7 @@ def setup_method(self, method): "time": ["0600"], }, ) - self.options = {"date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}} + self.options = {"date": {"merge": {"with": "time", "linkers": ["T", "00"]}}} self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 56b21ee53..bf1ace37b 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -19,11 +19,7 @@ def setup_method(self, method): self.latlon_array = self.latlon_array.t2m self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - } + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 1de8a8705..5e5fa3878 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -17,13 +17,11 @@ def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") self.options = { - "values": { - "transformation": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}} - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, - "number": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}}, + "values": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "longitude": {"cyclic": [0, 360]}, } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -54,7 +52,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_regular_grid(self): request = Request( Select("step", [0]), diff --git a/tests/test_reverse_transformation.py b/tests/test_reverse_transformation.py index 858f2f29f..e501dfdfc 100644 --- a/tests/test_reverse_transformation.py +++ b/tests/test_reverse_transformation.py @@ -17,7 +17,7 @@ def setup_method(self, method): "lat": [4, 3, 2, 1], }, ) - options = {"lat": {"transformation": {"reverse": {True}}}} + options = {"lat": {"reverse": {True}}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_shapes.py b/tests/test_shapes.py index ebc55b527..3d9d8b202 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -24,7 +24,7 @@ def setup_method(self, method): }, ) self.xarraydatacube = XArrayDatacube(array) - self.options = {"longitude": {"transformation": {"cyclic": [0, 360]}}} + self.options = {"longitude": {"cyclic": [0, 360]}} self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) @@ -39,17 +39,14 @@ def test_all_cyclic(self): # result.pprint() assert len(result.leaves) == 360 - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_all_mapper_cyclic(self): self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}}, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "longitude": {"cyclic": [0, 360]}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -58,6 +55,7 @@ def test_all_mapper_cyclic(self): request = Request( Select("step", [11]), + Select("number", [1]), Select("levtype", ["sfc"]), Select("date", [pd.Timestamp("20230710T120000")]), Select("domain", ["g"]), diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 567230687..fb4255db8 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -11,13 +11,10 @@ class TestSlicingFDBDatacube: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -25,10 +22,11 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), + Select("number", [1]), Select("levtype", ["sfc"]), Span("date", pd.Timestamp("20230625T120000"), pd.Timestamp("20230626T120000")), Select("domain", ["g"]), diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 1cd0fc897..15d355cb1 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -18,7 +18,7 @@ def setup_method(self, method): array = ds.to_xarray().isel(step=0).t self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - options = {"lat": {"transformation": {"reverse": {True}}}} + options = {"lat": {"reverse": {True}}} self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) @pytest.mark.internet diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index c0d2a7a7d..acad24fbc 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -22,8 +22,8 @@ def setup_method(self, method): self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() options = { - "latitude": {"transformation": {"reverse": {True}}}, - "longitude": {"transformation": {"cyclic": [0, 360.0]}}, + "latitude": {"reverse": {True}}, + "longitude": {"cyclic": [0, 360.0]}, } self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_type_change_transformation.py b/tests/test_type_change_transformation.py index 80f5755aa..d88a03aca 100644 --- a/tests/test_type_change_transformation.py +++ b/tests/test_type_change_transformation.py @@ -18,7 +18,7 @@ def setup_method(self, method): }, ) self.array = array - options = {"step": {"transformation": {"type_change": "int"}}} + options = {"step": {"type_change": "int"}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) From c918b6b0d9280000f3dd5e53b0c9023d7310c412 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Dec 2023 14:11:46 +0100 Subject: [PATCH 213/332] show fdb time --- performance/fdb_slice_many_numbers_timeseries.py | 1 + polytope/datacube/backends/fdb.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index 45f67699f..630b48f43 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -39,4 +39,5 @@ result = self_API.retrieve(request) print(time.time() - time1) print(time.time() - time2) +print(fdbdatacube.fdb_time) print(len(result.leaves)) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index bab3fc2a6..fad3ca4a7 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -4,6 +4,8 @@ from .datacube import Datacube, IndexTree +import time + class FDBDatacube(Datacube): def __init__(self, config={}, axis_options={}): @@ -15,6 +17,7 @@ def __init__(self, config={}, axis_options={}): self.blocked_axes = [] self.fake_axes = [] self.unwanted_path = {} + self.fdb_time = 0 partial_request = config # Find values in the level 3 FDB datacube @@ -146,7 +149,9 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) + time1 = time.time() output_values = self.fdb.extract(fdb_requests) + self.fdb_time += time.time() - time1 return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): From 0d334cee9fd80ac502fd6aa86795ca12e1a5e257 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Dec 2023 14:59:09 +0100 Subject: [PATCH 214/332] time different functions --- performance/fdb_slice_many_numbers_timeseries.py | 8 +++++--- polytope/engine/hullslicer.py | 5 +++++ polytope/polytope.py | 8 ++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index 630b48f43..0b592de78 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -18,10 +18,12 @@ } config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"} fdbdatacube = FDBDatacube(config, axis_options=options) -slicer = HullSlicer() -self_API = Polytope(datacube=fdbdatacube, engine=slicer, axis_options=options) +self_API = Polytope(datacube=fdbdatacube, axis_options=options) + +print(time.time() - time1) time2 = time.time() + request = Request( All("step"), Select("levtype", ["sfc"]), @@ -39,5 +41,5 @@ result = self_API.retrieve(request) print(time.time() - time1) print(time.time() - time2) -print(fdbdatacube.fdb_time) +print(time.time()-time2 - fdbdatacube.fdb_time) print(len(result.leaves)) diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 6b0306ed2..51a622535 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -13,6 +13,8 @@ from ..utility.geometry import lerp from .engine import Engine +import time + class HullSlicer(Engine): def __init__(self): @@ -91,6 +93,7 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): request = IndexTree() combinations = tensor_product(groups) + time0 = time.time() for c in combinations: r = IndexTree() r["unsliced_polytopes"] = set(c) @@ -101,6 +104,8 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): self._build_branch(ax, node, datacube, next_nodes) current_nodes = next_nodes request.merge(r) + print("time spent inside extract building tree") + print(time.time() - time0) return request diff --git a/polytope/polytope.py b/polytope/polytope.py index f6d4a723e..5e4fd0914 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -3,6 +3,8 @@ from .shapes import ConvexPolytope from .utility.exceptions import AxisOverdefinedError +import time + class Request: """Encapsulates a request for data""" @@ -50,6 +52,12 @@ def slice(self, polytopes: List[ConvexPolytope]): def retrieve(self, request: Request, method="standard"): """Higher-level API which takes a request and uses it to slice the datacube""" + time1 = time.time() request_tree = self.engine.extract(self.datacube, request.polytopes()) + print("extract time") + print(time.time() - time1) + time0 = time.time() self.datacube.get(request_tree) + print("get time") + print(time.time() - time0 - self.datacube.fdb_time) return request_tree From dc47ba983b298d88c8d1aafca4080850f37c4579 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 09:52:23 +0100 Subject: [PATCH 215/332] add shortcut if polytope is flat --- polytope/shapes.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/polytope/shapes.py b/polytope/shapes.py index c10b12e4b..6af698ee2 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -25,14 +25,22 @@ def axes(self) -> List[str]: class ConvexPolytope(Shape): def __init__(self, axes, points, method=None): self._axes = list(axes) + self.is_flat = False + if len(self._axes) == 1: + self.is_flat = True self.points = points self.method = method def extents(self, axis): - slice_axis_idx = self.axes().index(axis) - axis_values = [point[slice_axis_idx] for point in self.points] - lower = min(axis_values) - upper = max(axis_values) + if self.is_flat: + slice_axis_idx = 1 + lower = min(self.points)[0] + upper = max(self.points)[0] + else: + slice_axis_idx = self.axes().index(axis) + axis_values = [point[slice_axis_idx] for point in self.points] + lower = min(axis_values) + upper = max(axis_values) return (lower, upper, slice_axis_idx) def __str__(self): From 9b34b4a8d59b3ad65d414c791abf5bbba53137fb Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 09:54:45 +0100 Subject: [PATCH 216/332] add fdb_time to xarray to make tests work --- polytope/datacube/backends/xarray.py | 1 + 1 file changed, 1 insertion(+) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index f8ca1c2e2..9ce40135b 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -17,6 +17,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.complete_axes = [] self.blocked_axes = [] self.fake_axes = [] + self.fdb_time = 0 for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: From 4f65e7ab055675f7779f72162efe39232cf85648 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 14:02:46 +0100 Subject: [PATCH 217/332] small optimisations --- polytope/datacube/backends/datacube.py | 2 ++ polytope/datacube/backends/fdb.py | 1 + polytope/datacube/backends/mock.py | 1 + polytope/datacube/backends/xarray.py | 1 + polytope/engine/hullslicer.py | 48 +++++++++++++++++++++----- 5 files changed, 45 insertions(+), 8 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index efa20e93d..62620eb0b 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -37,6 +37,8 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt ) for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) + if len(final_axis_names) > 1: + self.coupled_axes.append(final_axis_names) for axis_name in final_axis_names: self.fake_axes.append(axis_name) # if axis does not yet exist, create it diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index fad3ca4a7..2182a708b 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -18,6 +18,7 @@ def __init__(self, config={}, axis_options={}): self.fake_axes = [] self.unwanted_path = {} self.fdb_time = 0 + self.coupled_axes = [] partial_request = config # Find values in the level 3 FDB datacube diff --git a/polytope/datacube/backends/mock.py b/polytope/datacube/backends/mock.py index b6bc32d3e..6281c9cf3 100644 --- a/polytope/datacube/backends/mock.py +++ b/polytope/datacube/backends/mock.py @@ -22,6 +22,7 @@ def __init__(self, dimensions): for k, v in reversed(dimensions.items()): self.stride[k] = stride_cumulative stride_cumulative *= self.dimensions[k] + self.coupled_axes = [] def get(self, requests: IndexTree): # Takes in a datacube and verifies the leaves of the tree are complete diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 9ce40135b..26b71dbb9 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -18,6 +18,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.blocked_axes = [] self.fake_axes = [] self.fdb_time = 0 + self.coupled_axes = [] for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 51a622535..a8b1dce87 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -18,21 +18,25 @@ class HullSlicer(Engine): def __init__(self): - pass + self.ax_is_unsliceable = {} + self.axis_values_between = {} def _unique_continuous_points(self, p: ConvexPolytope, datacube: Datacube): for i, ax in enumerate(p._axes): mapper = datacube.get_mapper(ax) - if isinstance(mapper, UnsliceableDatacubeAxis): + if self.ax_is_unsliceable.get(ax, None) is None: + self.ax_is_unsliceable[ax] = isinstance(mapper, UnsliceableDatacubeAxis) + # if isinstance(mapper, UnsliceableDatacubeAxis): + if self.ax_is_unsliceable[ax]: break for j, val in enumerate(p.points): - p.points[j] = list(p.points[j]) + # p.points[j] = list(p.points[j]) p.points[j][i] = mapper.to_float(mapper.parse(p.points[j][i])) # Remove duplicate points unique(p.points) def _build_unsliceable_child(self, polytope, ax, node, datacube, lower, next_nodes, slice_axis_idx): - if polytope._axes != [ax.name]: + if not polytope.is_flat: raise UnsliceableShapeError(ax) path = node.flatten() if datacube.has_index(path, ax, lower): @@ -50,6 +54,25 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex upper = ax.from_float(upper + tol) flattened = node.flatten() method = polytope.method + + # TODO: this hashing doesn't work because we need to know the latitude val for finding longitude values + # TODO: Maybe create a coupled_axes list inside of datacube and add to it during axis formation, then here + # do something like if ax is in second place of coupled_axes, then take the flattened part of the array that corresponds + # to the first place of cooupled_axes in the hashing + # Else, if we do not need the flattened bit in the hash, can just put an empty string instead? + + # if len(datacube.coupled_axes) > 0: + # if flattened.get(datacube.coupled_axes[0][0], None) is not None: + # flattened = {datacube.coupled_axes[0][0] : flattened.get(datacube.coupled_axes[0][0], None)} + # else: + # flattened = {} + + # print(datacube.coupled_axes) + # print(flattened.get("latitude")) + # flattened = interm_flattened + # if self.axis_values_between.get((ax.name, lower, upper, method), None) is None: + # self.axis_values_between[(ax.name, lower, upper, method)] = datacube.get_indices(flattened, ax, lower, upper, method) + # values = self.axis_values_between[(ax.name, lower, upper, method)] values = datacube.get_indices(flattened, ax, lower, upper, method) if len(values) == 0: @@ -77,7 +100,12 @@ def _build_branch(self, ax, node, datacube, next_nodes): if ax.name in polytope._axes: lower, upper, slice_axis_idx = polytope.extents(ax.name) # here, first check if the axis is an unsliceable axis and directly build node if it is - if isinstance(ax, UnsliceableDatacubeAxis): + # if isinstance(ax, UnsliceableDatacubeAxis): + + # NOTE: we should have already created the ax_is_unsliceable cache before + # if self.ax_is_unsliceable.get(ax.name, None) is None: + # self.ax_is_unsliceable[ax.name] = isinstance(ax, UnsliceableDatacubeAxis) + if self.ax_is_unsliceable[ax.name]: self._build_unsliceable_child(polytope, ax, node, datacube, lower, next_nodes, slice_axis_idx) else: self._build_sliceable_child(polytope, ax, node, datacube, lower, upper, next_nodes, slice_axis_idx) @@ -93,6 +121,10 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): request = IndexTree() combinations = tensor_product(groups) + # NOTE: could optimise here if we know combinations will always be for one request. + # Then we do not need to create a new index tree and merge it to request, but can just + # directly work on request and return it... + time0 = time.time() for c in combinations: r = IndexTree() @@ -139,8 +171,7 @@ def _reduce_dimension(intersects, slice_axis_idx): def slice(polytope: ConvexPolytope, axis, value, slice_axis_idx): - if len(polytope.points[0]) == 1: - # Note that in this case, we do not need to do linear interpolation so we can save time + if polytope.is_flat: if value in chain(*polytope.points): intersects = [[value]] else: @@ -154,7 +185,8 @@ def slice(polytope: ConvexPolytope, axis, value, slice_axis_idx): # Reduce dimension of intersection points, removing slice axis intersects = _reduce_dimension(intersects, slice_axis_idx) - axes = [ax for ax in polytope._axes if ax != axis] + axes = copy(polytope._axes) + axes.remove(axis) if len(intersects) < len(intersects[0]) + 1: return ConvexPolytope(axes, intersects) From 7bf5b17908d7ec606752daed6b59353866b7f4e5 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 14:11:24 +0100 Subject: [PATCH 218/332] clean up branch --- .../fdb_slice_many_numbers_timeseries.py | 2 -- polytope/datacube/backends/fdb.py | 6 ----- polytope/datacube/backends/xarray.py | 2 -- polytope/engine/hullslicer.py | 26 ++----------------- polytope/polytope.py | 8 ------ tests/test_datacube_axes_init.py | 1 - tests/test_fdb_datacube.py | 3 +-- tests/test_incomplete_tree_fdb.py | 4 +-- tests/test_regular_grid.py | 2 +- tests/test_shapes.py | 2 +- tests/test_slice_date_range_fdb.py | 2 +- 11 files changed, 8 insertions(+), 50 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index 0b592de78..13dc00f79 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -3,7 +3,6 @@ import pandas as pd from polytope.datacube.backends.fdb import FDBDatacube -from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import All, Point, Select @@ -41,5 +40,4 @@ result = self_API.retrieve(request) print(time.time() - time1) print(time.time() - time2) -print(time.time()-time2 - fdbdatacube.fdb_time) print(len(result.leaves)) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 2182a708b..bab3fc2a6 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -4,8 +4,6 @@ from .datacube import Datacube, IndexTree -import time - class FDBDatacube(Datacube): def __init__(self, config={}, axis_options={}): @@ -17,8 +15,6 @@ def __init__(self, config={}, axis_options={}): self.blocked_axes = [] self.fake_axes = [] self.unwanted_path = {} - self.fdb_time = 0 - self.coupled_axes = [] partial_request = config # Find values in the level 3 FDB datacube @@ -150,9 +146,7 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) - time1 = time.time() output_values = self.fdb.extract(fdb_requests) - self.fdb_time += time.time() - time1 return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 26b71dbb9..f8ca1c2e2 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -17,8 +17,6 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.complete_axes = [] self.blocked_axes = [] self.fake_axes = [] - self.fdb_time = 0 - self.coupled_axes = [] for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index a8b1dce87..e6c8e3eb0 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -13,8 +13,6 @@ from ..utility.geometry import lerp from .engine import Engine -import time - class HullSlicer(Engine): def __init__(self): @@ -26,11 +24,9 @@ def _unique_continuous_points(self, p: ConvexPolytope, datacube: Datacube): mapper = datacube.get_mapper(ax) if self.ax_is_unsliceable.get(ax, None) is None: self.ax_is_unsliceable[ax] = isinstance(mapper, UnsliceableDatacubeAxis) - # if isinstance(mapper, UnsliceableDatacubeAxis): if self.ax_is_unsliceable[ax]: break for j, val in enumerate(p.points): - # p.points[j] = list(p.points[j]) p.points[j][i] = mapper.to_float(mapper.parse(p.points[j][i])) # Remove duplicate points unique(p.points) @@ -57,22 +53,10 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex # TODO: this hashing doesn't work because we need to know the latitude val for finding longitude values # TODO: Maybe create a coupled_axes list inside of datacube and add to it during axis formation, then here - # do something like if ax is in second place of coupled_axes, then take the flattened part of the array that corresponds - # to the first place of cooupled_axes in the hashing + # do something like if ax is in second place of coupled_axes, then take the flattened part of the array that + # corresponds to the first place of cooupled_axes in the hashing # Else, if we do not need the flattened bit in the hash, can just put an empty string instead? - # if len(datacube.coupled_axes) > 0: - # if flattened.get(datacube.coupled_axes[0][0], None) is not None: - # flattened = {datacube.coupled_axes[0][0] : flattened.get(datacube.coupled_axes[0][0], None)} - # else: - # flattened = {} - - # print(datacube.coupled_axes) - # print(flattened.get("latitude")) - # flattened = interm_flattened - # if self.axis_values_between.get((ax.name, lower, upper, method), None) is None: - # self.axis_values_between[(ax.name, lower, upper, method)] = datacube.get_indices(flattened, ax, lower, upper, method) - # values = self.axis_values_between[(ax.name, lower, upper, method)] values = datacube.get_indices(flattened, ax, lower, upper, method) if len(values) == 0: @@ -100,11 +84,8 @@ def _build_branch(self, ax, node, datacube, next_nodes): if ax.name in polytope._axes: lower, upper, slice_axis_idx = polytope.extents(ax.name) # here, first check if the axis is an unsliceable axis and directly build node if it is - # if isinstance(ax, UnsliceableDatacubeAxis): # NOTE: we should have already created the ax_is_unsliceable cache before - # if self.ax_is_unsliceable.get(ax.name, None) is None: - # self.ax_is_unsliceable[ax.name] = isinstance(ax, UnsliceableDatacubeAxis) if self.ax_is_unsliceable[ax.name]: self._build_unsliceable_child(polytope, ax, node, datacube, lower, next_nodes, slice_axis_idx) else: @@ -125,7 +106,6 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): # Then we do not need to create a new index tree and merge it to request, but can just # directly work on request and return it... - time0 = time.time() for c in combinations: r = IndexTree() r["unsliced_polytopes"] = set(c) @@ -136,8 +116,6 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): self._build_branch(ax, node, datacube, next_nodes) current_nodes = next_nodes request.merge(r) - print("time spent inside extract building tree") - print(time.time() - time0) return request diff --git a/polytope/polytope.py b/polytope/polytope.py index 5e4fd0914..f6d4a723e 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -3,8 +3,6 @@ from .shapes import ConvexPolytope from .utility.exceptions import AxisOverdefinedError -import time - class Request: """Encapsulates a request for data""" @@ -52,12 +50,6 @@ def slice(self, polytopes: List[ConvexPolytope]): def retrieve(self, request: Request, method="standard"): """Higher-level API which takes a request and uses it to slice the datacube""" - time1 = time.time() request_tree = self.engine.extract(self.datacube, request.polytopes()) - print("extract time") - print(time.time() - time1) - time0 = time.time() self.datacube.get(request_tree) - print("get time") - print(time.time() - time0 - self.datacube.fdb_time) return request_tree diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index f2a7f3eb4..f3739e5bf 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -20,7 +20,6 @@ def setup_method(self, method): self.xarraydatacube = XArrayDatacube(latlon_array) self.options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - # "latitude": {"transformation": {"reverse": {True}}}, } self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 03bc1084c..d846b91fb 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -21,12 +21,11 @@ def setup_method(self, method): } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) - print(self.fdbdatacube.fdb_coordinates) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 6f5f11ecf..40fd5933e 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -49,7 +49,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch(self): request = Request( Select("step", [0]), @@ -72,7 +72,7 @@ def test_incomplete_fdb_branch(self): assert result.is_root() @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch_2(self): request = Request( Select("step", [0]), diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 5e5fa3878..506cd0cf2 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -52,7 +52,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_regular_grid(self): request = Request( Select("step", [0]), diff --git a/tests/test_shapes.py b/tests/test_shapes.py index 3d9d8b202..29c6b35f6 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -39,7 +39,7 @@ def test_all_cyclic(self): # result.pprint() assert len(result.leaves) == 360 - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_all_mapper_cyclic(self): self.options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index fb4255db8..a233e2c26 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -22,7 +22,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), From 8ce60e57807fda55a71c7aa713996ac98d7918e9 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 14:16:54 +0100 Subject: [PATCH 219/332] clean up --- polytope/datacube/backends/datacube.py | 2 -- polytope/datacube/backends/mock.py | 1 - 2 files changed, 3 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 62620eb0b..efa20e93d 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -37,8 +37,6 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt ) for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) - if len(final_axis_names) > 1: - self.coupled_axes.append(final_axis_names) for axis_name in final_axis_names: self.fake_axes.append(axis_name) # if axis does not yet exist, create it diff --git a/polytope/datacube/backends/mock.py b/polytope/datacube/backends/mock.py index 6281c9cf3..b6bc32d3e 100644 --- a/polytope/datacube/backends/mock.py +++ b/polytope/datacube/backends/mock.py @@ -22,7 +22,6 @@ def __init__(self, dimensions): for k, v in reversed(dimensions.items()): self.stride[k] = stride_cumulative stride_cumulative *= self.dimensions[k] - self.coupled_axes = [] def get(self, requests: IndexTree): # Takes in a datacube and verifies the leaves of the tree are complete From 07bf141cee37a17c3cfb90575efd17810bb2cad5 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 16:52:27 +0100 Subject: [PATCH 220/332] make caching for the sliceable values in hullslicer work --- polytope/engine/hullslicer.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index a8b1dce87..f83524b9b 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -61,19 +61,21 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex # to the first place of cooupled_axes in the hashing # Else, if we do not need the flattened bit in the hash, can just put an empty string instead? - # if len(datacube.coupled_axes) > 0: - # if flattened.get(datacube.coupled_axes[0][0], None) is not None: - # flattened = {datacube.coupled_axes[0][0] : flattened.get(datacube.coupled_axes[0][0], None)} - # else: - # flattened = {} - + flattened_tuple = tuple() + if len(datacube.coupled_axes) > 0: + if flattened.get(datacube.coupled_axes[0][0], None) is not None: + flattened_tuple = (datacube.coupled_axes[0][0], flattened.get(datacube.coupled_axes[0][0], None)) + flattened = {flattened_tuple[0]: flattened_tuple[1]} + else: + flattened_tuple = tuple() + flattened = {} # print(datacube.coupled_axes) # print(flattened.get("latitude")) # flattened = interm_flattened - # if self.axis_values_between.get((ax.name, lower, upper, method), None) is None: - # self.axis_values_between[(ax.name, lower, upper, method)] = datacube.get_indices(flattened, ax, lower, upper, method) - # values = self.axis_values_between[(ax.name, lower, upper, method)] - values = datacube.get_indices(flattened, ax, lower, upper, method) + if self.axis_values_between.get((flattened_tuple, ax.name, lower, upper, method), None) is None: + self.axis_values_between[(flattened_tuple, ax.name, lower, upper, method)] = datacube.get_indices(flattened, ax, lower, upper, method) + values = self.axis_values_between[(flattened_tuple, ax.name, lower, upper, method)] + # values = datacube.get_indices(flattened, ax, lower, upper, method) if len(values) == 0: node.remove_branch() From 158c3e74839657d17aad9b0e1116738180544893 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 11 Dec 2023 11:01:48 +0100 Subject: [PATCH 221/332] skip test with non-formatted datacube and clean up --- .../fdb_slice_many_numbers_timeseries.py | 2 -- polytope/datacube/backends/fdb.py | 5 ----- polytope/datacube/backends/xarray.py | 1 - polytope/engine/hullslicer.py | 22 +++++-------------- polytope/polytope.py | 8 ------- tests/test_merge_cyclic_octahedral.py | 4 +++- 6 files changed, 9 insertions(+), 33 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index 0b592de78..13dc00f79 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -3,7 +3,6 @@ import pandas as pd from polytope.datacube.backends.fdb import FDBDatacube -from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import All, Point, Select @@ -41,5 +40,4 @@ result = self_API.retrieve(request) print(time.time() - time1) print(time.time() - time2) -print(time.time()-time2 - fdbdatacube.fdb_time) print(len(result.leaves)) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 2182a708b..4d2796623 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -4,8 +4,6 @@ from .datacube import Datacube, IndexTree -import time - class FDBDatacube(Datacube): def __init__(self, config={}, axis_options={}): @@ -17,7 +15,6 @@ def __init__(self, config={}, axis_options={}): self.blocked_axes = [] self.fake_axes = [] self.unwanted_path = {} - self.fdb_time = 0 self.coupled_axes = [] partial_request = config @@ -150,9 +147,7 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) - time1 = time.time() output_values = self.fdb.extract(fdb_requests) - self.fdb_time += time.time() - time1 return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 26b71dbb9..4b3b13117 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -17,7 +17,6 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.complete_axes = [] self.blocked_axes = [] self.fake_axes = [] - self.fdb_time = 0 self.coupled_axes = [] for name, values in dataarray.coords.variables.items(): diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index f83524b9b..a8fac6c26 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -13,8 +13,6 @@ from ..utility.geometry import lerp from .engine import Engine -import time - class HullSlicer(Engine): def __init__(self): @@ -26,11 +24,9 @@ def _unique_continuous_points(self, p: ConvexPolytope, datacube: Datacube): mapper = datacube.get_mapper(ax) if self.ax_is_unsliceable.get(ax, None) is None: self.ax_is_unsliceable[ax] = isinstance(mapper, UnsliceableDatacubeAxis) - # if isinstance(mapper, UnsliceableDatacubeAxis): if self.ax_is_unsliceable[ax]: break for j, val in enumerate(p.points): - # p.points[j] = list(p.points[j]) p.points[j][i] = mapper.to_float(mapper.parse(p.points[j][i])) # Remove duplicate points unique(p.points) @@ -57,8 +53,8 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex # TODO: this hashing doesn't work because we need to know the latitude val for finding longitude values # TODO: Maybe create a coupled_axes list inside of datacube and add to it during axis formation, then here - # do something like if ax is in second place of coupled_axes, then take the flattened part of the array that corresponds - # to the first place of cooupled_axes in the hashing + # do something like if ax is in second place of coupled_axes, then take the flattened part of the array that + # corresponds to the first place of cooupled_axes in the hashing # Else, if we do not need the flattened bit in the hash, can just put an empty string instead? flattened_tuple = tuple() @@ -69,13 +65,11 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex else: flattened_tuple = tuple() flattened = {} - # print(datacube.coupled_axes) - # print(flattened.get("latitude")) - # flattened = interm_flattened if self.axis_values_between.get((flattened_tuple, ax.name, lower, upper, method), None) is None: - self.axis_values_between[(flattened_tuple, ax.name, lower, upper, method)] = datacube.get_indices(flattened, ax, lower, upper, method) + self.axis_values_between[(flattened_tuple, ax.name, lower, upper, method)] = datacube.get_indices( + flattened, ax, lower, upper, method + ) values = self.axis_values_between[(flattened_tuple, ax.name, lower, upper, method)] - # values = datacube.get_indices(flattened, ax, lower, upper, method) if len(values) == 0: node.remove_branch() @@ -105,8 +99,7 @@ def _build_branch(self, ax, node, datacube, next_nodes): # if isinstance(ax, UnsliceableDatacubeAxis): # NOTE: we should have already created the ax_is_unsliceable cache before - # if self.ax_is_unsliceable.get(ax.name, None) is None: - # self.ax_is_unsliceable[ax.name] = isinstance(ax, UnsliceableDatacubeAxis) + if self.ax_is_unsliceable[ax.name]: self._build_unsliceable_child(polytope, ax, node, datacube, lower, next_nodes, slice_axis_idx) else: @@ -127,7 +120,6 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): # Then we do not need to create a new index tree and merge it to request, but can just # directly work on request and return it... - time0 = time.time() for c in combinations: r = IndexTree() r["unsliced_polytopes"] = set(c) @@ -138,8 +130,6 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): self._build_branch(ax, node, datacube, next_nodes) current_nodes = next_nodes request.merge(r) - print("time spent inside extract building tree") - print(time.time() - time0) return request diff --git a/polytope/polytope.py b/polytope/polytope.py index 5e4fd0914..f6d4a723e 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -3,8 +3,6 @@ from .shapes import ConvexPolytope from .utility.exceptions import AxisOverdefinedError -import time - class Request: """Encapsulates a request for data""" @@ -52,12 +50,6 @@ def slice(self, polytopes: List[ConvexPolytope]): def retrieve(self, request: Request, method="standard"): """Higher-level API which takes a request and uses it to slice the datacube""" - time1 = time.time() request_tree = self.engine.extract(self.datacube, request.polytopes()) - print("extract time") - print(time.time() - time1) - time0 = time.time() self.datacube.get(request_tree) - print("get time") - print(time.time() - time0 - self.datacube.fdb_time) return request_tree diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index fd50b6b1a..2714c36c6 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -1,4 +1,5 @@ import numpy as np +import pytest import xarray as xr from polytope.datacube.backends.xarray import XArrayDatacube @@ -10,7 +11,6 @@ class TestMultipleTransformations: @classmethod def setup_method(self, method): - # Create a dataarray with 4 labelled axes using different index types self.array = xr.DataArray( np.random.randn(1, 1, 4289589, 3), dims=("date", "time", "values", "step"), @@ -21,6 +21,7 @@ def setup_method(self, method): "step": [0, 1, 2], }, ) + self.options = { "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, @@ -30,6 +31,7 @@ def setup_method(self, method): self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) + @pytest.mark.skip(reason="Datacube not formatted right.") def test_merge_axis(self): # NOTE: does not work because the date is a string in the merge option... date = np.datetime64("2000-01-01T06:00:00") From 1cc38edeb315dfbd170f344f3f68ba9b4a2f8b50 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 11 Dec 2023 11:06:11 +0100 Subject: [PATCH 222/332] flake 8 tests --- tests/test_fdb_datacube.py | 3 +-- tests/test_shapes.py | 3 +-- tests/test_slice_date_range_fdb.py | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 03bc1084c..d846b91fb 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -21,12 +21,11 @@ def setup_method(self, method): } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) - print(self.fdbdatacube.fdb_coordinates) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), diff --git a/tests/test_shapes.py b/tests/test_shapes.py index 3d9d8b202..13f7b4297 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -36,10 +36,9 @@ def test_all(self): def test_all_cyclic(self): request = Request(Select("step", [3]), Select("date", ["2000-01-01"]), Select("level", [1]), All("longitude")) result = self.API.retrieve(request) - # result.pprint() assert len(result.leaves) == 360 - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_all_mapper_cyclic(self): self.options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index fb4255db8..1b8a89d37 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -21,8 +21,7 @@ def setup_method(self, method): self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) - # Testing different shapes - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), From 457bd764359b84944ff633d7065e1d8a3cee94dc Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 11 Dec 2023 11:10:59 +0100 Subject: [PATCH 223/332] skip tests that depend on fdb --- tests/test_incomplete_tree_fdb.py | 4 ++-- tests/test_regular_grid.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 6f5f11ecf..40fd5933e 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -49,7 +49,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch(self): request = Request( Select("step", [0]), @@ -72,7 +72,7 @@ def test_incomplete_fdb_branch(self): assert result.is_root() @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch_2(self): request = Request( Select("step", [0]), diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 5e5fa3878..506cd0cf2 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -52,7 +52,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_regular_grid(self): request = Request( Select("step", [0]), From 3239323303978f13ace2456cd619fde155ec430b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 11 Dec 2023 11:39:10 +0100 Subject: [PATCH 224/332] add caching for unsliceable values --- polytope/engine/hullslicer.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index f83524b9b..7c9f78145 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -20,6 +20,7 @@ class HullSlicer(Engine): def __init__(self): self.ax_is_unsliceable = {} self.axis_values_between = {} + self.has_value = {} def _unique_continuous_points(self, p: ConvexPolytope, datacube: Datacube): for i, ax in enumerate(p._axes): @@ -39,7 +40,21 @@ def _build_unsliceable_child(self, polytope, ax, node, datacube, lower, next_nod if not polytope.is_flat: raise UnsliceableShapeError(ax) path = node.flatten() - if datacube.has_index(path, ax, lower): + + flattened_tuple = tuple() + if len(datacube.coupled_axes) > 0: + if path.get(datacube.coupled_axes[0][0], None) is not None: + flattened_tuple = (datacube.coupled_axes[0][0], path.get(datacube.coupled_axes[0][0], None)) + path = {flattened_tuple[0]: flattened_tuple[1]} + else: + flattened_tuple = tuple() + path = {} + + if self.axis_values_between.get((flattened_tuple, ax.name, lower), None) is None: + self.axis_values_between[(flattened_tuple, ax.name, lower)] = datacube.has_index(path, ax, lower) + datacube_has_index = self.axis_values_between[(flattened_tuple, ax.name, lower)] + + if datacube_has_index: child = node.create_child(ax, lower) child["unsliced_polytopes"] = copy(node["unsliced_polytopes"]) child["unsliced_polytopes"].remove(polytope) From fbfc9144b40156c6a6a2bde4fd3c5fcf56335e67 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 12 Dec 2023 11:20:53 +0100 Subject: [PATCH 225/332] add more caching of functions in extract --- .../fdb_slice_many_numbers_timeseries.py | 41 +++++++++-------- polytope/engine/hullslicer.py | 44 ++++++++++++------- polytope/polytope.py | 6 --- 3 files changed, 51 insertions(+), 40 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index 0b592de78..8b31a0d35 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -22,24 +22,27 @@ print(time.time() - time1) -time2 = time.time() +total_polytope_time = 0 +for i in range(10): + time2 = time.time() -request = Request( - All("step"), - Select("levtype", ["sfc"]), - Select("date", [pd.Timestamp("20231205T000000")]), - Select("domain", ["g"]), - Select("expver", ["0001"]), - Select("param", ["167"]), - Select("class", ["od"]), - Select("stream", ["enfo"]), - Select("type", ["pf"]), - # Select("latitude", [0.035149384216], method="surrounding"), - Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"), - All("number"), -) -result = self_API.retrieve(request) -print(time.time() - time1) -print(time.time() - time2) -print(time.time()-time2 - fdbdatacube.fdb_time) + request = Request( + All("step"), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231205T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["enfo"]), + Select("type", ["pf"]), + # Select("latitude", [0.035149384216], method="surrounding"), + Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"), + All("number"), + ) + result = self_API.retrieve(request) + # print(time.time() - time1) + total_polytope_time += time.time()-time2 +print('polytope time') +print((total_polytope_time - fdbdatacube.fdb_time)/10) print(len(result.leaves)) diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 7c9f78145..5f943138a 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -13,14 +13,14 @@ from ..utility.geometry import lerp from .engine import Engine -import time - class HullSlicer(Engine): def __init__(self): self.ax_is_unsliceable = {} self.axis_values_between = {} self.has_value = {} + self.sliced_polytopes = {} + self.remapped_vals = {} def _unique_continuous_points(self, p: ConvexPolytope, datacube: Datacube): for i, ax in enumerate(p._axes): @@ -47,7 +47,7 @@ def _build_unsliceable_child(self, polytope, ax, node, datacube, lower, next_nod flattened_tuple = (datacube.coupled_axes[0][0], path.get(datacube.coupled_axes[0][0], None)) path = {flattened_tuple[0]: flattened_tuple[1]} else: - flattened_tuple = tuple() + # flattened_tuple = tuple() path = {} if self.axis_values_between.get((flattened_tuple, ax.name, lower), None) is None: @@ -82,14 +82,16 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex flattened_tuple = (datacube.coupled_axes[0][0], flattened.get(datacube.coupled_axes[0][0], None)) flattened = {flattened_tuple[0]: flattened_tuple[1]} else: - flattened_tuple = tuple() + # flattened_tuple = tuple() flattened = {} # print(datacube.coupled_axes) # print(flattened.get("latitude")) # flattened = interm_flattened + values = self.axis_values_between.get((flattened_tuple, ax.name, lower, upper, method), None) if self.axis_values_between.get((flattened_tuple, ax.name, lower, upper, method), None) is None: - self.axis_values_between[(flattened_tuple, ax.name, lower, upper, method)] = datacube.get_indices(flattened, ax, lower, upper, method) - values = self.axis_values_between[(flattened_tuple, ax.name, lower, upper, method)] + values = datacube.get_indices(flattened, ax, lower, upper, method) + self.axis_values_between[(flattened_tuple, ax.name, lower, upper, method)] = values + # values = self.axis_values_between[(flattened_tuple, ax.name, lower, upper, method)] # values = datacube.get_indices(flattened, ax, lower, upper, method) if len(values) == 0: @@ -98,13 +100,28 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex for value in values: # convert to float for slicing fvalue = ax.to_float(value) - new_polytope = slice(polytope, ax.name, fvalue, slice_axis_idx) + new_polytope = self.sliced_polytopes.get((polytope, ax.name, fvalue, slice_axis_idx), False) + if new_polytope is False: + new_polytope = slice(polytope, ax.name, fvalue, slice_axis_idx) + self.sliced_polytopes[(polytope, ax.name, fvalue, slice_axis_idx)] = new_polytope + # new_polytope = self.sliced_polytopes[(polytope, ax.name, fvalue, slice_axis_idx)] + # new_polytope = slice(polytope, ax.name, fvalue, slice_axis_idx) # store the native type - remapped_val = value - if ax.is_cyclic: - remapped_val_interm = ax.remap([value, value])[0] - remapped_val = (remapped_val_interm[0] + remapped_val_interm[1]) / 2 - remapped_val = round(remapped_val, int(-math.log10(ax.tol))) + + remapped_val = self.remapped_vals.get((value, ax.name), None) + if remapped_val is None: + remapped_val = value + if ax.is_cyclic: + remapped_val_interm = ax.remap([value, value])[0] + remapped_val = (remapped_val_interm[0] + remapped_val_interm[1]) / 2 + remapped_val = round(remapped_val, int(-math.log10(ax.tol))) + self.remapped_vals[(value, ax.name)] = remapped_val + + # remapped_val = value + # if ax.is_cyclic: + # remapped_val_interm = ax.remap([value, value])[0] + # remapped_val = (remapped_val_interm[0] + remapped_val_interm[1]) / 2 + # remapped_val = round(remapped_val, int(-math.log10(ax.tol))) child = node.create_child(ax, remapped_val) child["unsliced_polytopes"] = copy(node["unsliced_polytopes"]) child["unsliced_polytopes"].remove(polytope) @@ -142,7 +159,6 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): # Then we do not need to create a new index tree and merge it to request, but can just # directly work on request and return it... - time0 = time.time() for c in combinations: r = IndexTree() r["unsliced_polytopes"] = set(c) @@ -153,8 +169,6 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): self._build_branch(ax, node, datacube, next_nodes) current_nodes = next_nodes request.merge(r) - print("time spent inside extract building tree") - print(time.time() - time0) return request diff --git a/polytope/polytope.py b/polytope/polytope.py index 5e4fd0914..8089b96dd 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -52,12 +52,6 @@ def slice(self, polytopes: List[ConvexPolytope]): def retrieve(self, request: Request, method="standard"): """Higher-level API which takes a request and uses it to slice the datacube""" - time1 = time.time() request_tree = self.engine.extract(self.datacube, request.polytopes()) - print("extract time") - print(time.time() - time1) - time0 = time.time() self.datacube.get(request_tree) - print("get time") - print(time.time() - time0 - self.datacube.fdb_time) return request_tree From a354ec877b954aeea79a99917a77125646d0d0dc Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 12 Dec 2023 15:07:08 +0100 Subject: [PATCH 226/332] clean up --- .../fdb_slice_many_numbers_timeseries.py | 7 ++--- polytope/datacube/backends/fdb.py | 5 ---- polytope/datacube/backends/xarray.py | 1 - polytope/engine/hullslicer.py | 29 ++++--------------- polytope/polytope.py | 2 -- 5 files changed, 8 insertions(+), 36 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index 8b31a0d35..db91553d0 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -3,7 +3,6 @@ import pandas as pd from polytope.datacube.backends.fdb import FDBDatacube -from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import All, Point, Select @@ -42,7 +41,7 @@ ) result = self_API.retrieve(request) # print(time.time() - time1) - total_polytope_time += time.time()-time2 -print('polytope time') -print((total_polytope_time - fdbdatacube.fdb_time)/10) + total_polytope_time += time.time() - time2 +# print('polytope time') +# print((total_polytope_time - fdbdatacube.fdb_time)/10) print(len(result.leaves)) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 2182a708b..4d2796623 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -4,8 +4,6 @@ from .datacube import Datacube, IndexTree -import time - class FDBDatacube(Datacube): def __init__(self, config={}, axis_options={}): @@ -17,7 +15,6 @@ def __init__(self, config={}, axis_options={}): self.blocked_axes = [] self.fake_axes = [] self.unwanted_path = {} - self.fdb_time = 0 self.coupled_axes = [] partial_request = config @@ -150,9 +147,7 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) - time1 = time.time() output_values = self.fdb.extract(fdb_requests) - self.fdb_time += time.time() - time1 return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 26b71dbb9..4b3b13117 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -17,7 +17,6 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.complete_axes = [] self.blocked_axes = [] self.fake_axes = [] - self.fdb_time = 0 self.coupled_axes = [] for name, values in dataarray.coords.variables.items(): diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 5f943138a..0a4e57910 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -27,11 +27,9 @@ def _unique_continuous_points(self, p: ConvexPolytope, datacube: Datacube): mapper = datacube.get_mapper(ax) if self.ax_is_unsliceable.get(ax, None) is None: self.ax_is_unsliceable[ax] = isinstance(mapper, UnsliceableDatacubeAxis) - # if isinstance(mapper, UnsliceableDatacubeAxis): if self.ax_is_unsliceable[ax]: break for j, val in enumerate(p.points): - # p.points[j] = list(p.points[j]) p.points[j][i] = mapper.to_float(mapper.parse(p.points[j][i])) # Remove duplicate points unique(p.points) @@ -47,7 +45,6 @@ def _build_unsliceable_child(self, polytope, ax, node, datacube, lower, next_nod flattened_tuple = (datacube.coupled_axes[0][0], path.get(datacube.coupled_axes[0][0], None)) path = {flattened_tuple[0]: flattened_tuple[1]} else: - # flattened_tuple = tuple() path = {} if self.axis_values_between.get((flattened_tuple, ax.name, lower), None) is None: @@ -70,10 +67,9 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex flattened = node.flatten() method = polytope.method - # TODO: this hashing doesn't work because we need to know the latitude val for finding longitude values - # TODO: Maybe create a coupled_axes list inside of datacube and add to it during axis formation, then here - # do something like if ax is in second place of coupled_axes, then take the flattened part of the array that corresponds - # to the first place of cooupled_axes in the hashing + # NOTE: Here we create a coupled_axes list inside of datacube and add to it during axis formation, then here + # do something like if ax is in second place of coupled_axes, then take the flattened part of the array that + # corresponds to the first place of cooupled_axes in the hashing # Else, if we do not need the flattened bit in the hash, can just put an empty string instead? flattened_tuple = tuple() @@ -82,17 +78,12 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex flattened_tuple = (datacube.coupled_axes[0][0], flattened.get(datacube.coupled_axes[0][0], None)) flattened = {flattened_tuple[0]: flattened_tuple[1]} else: - # flattened_tuple = tuple() flattened = {} - # print(datacube.coupled_axes) - # print(flattened.get("latitude")) - # flattened = interm_flattened + values = self.axis_values_between.get((flattened_tuple, ax.name, lower, upper, method), None) if self.axis_values_between.get((flattened_tuple, ax.name, lower, upper, method), None) is None: values = datacube.get_indices(flattened, ax, lower, upper, method) self.axis_values_between[(flattened_tuple, ax.name, lower, upper, method)] = values - # values = self.axis_values_between[(flattened_tuple, ax.name, lower, upper, method)] - # values = datacube.get_indices(flattened, ax, lower, upper, method) if len(values) == 0: node.remove_branch() @@ -104,10 +95,8 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex if new_polytope is False: new_polytope = slice(polytope, ax.name, fvalue, slice_axis_idx) self.sliced_polytopes[(polytope, ax.name, fvalue, slice_axis_idx)] = new_polytope - # new_polytope = self.sliced_polytopes[(polytope, ax.name, fvalue, slice_axis_idx)] - # new_polytope = slice(polytope, ax.name, fvalue, slice_axis_idx) - # store the native type + # store the native type remapped_val = self.remapped_vals.get((value, ax.name), None) if remapped_val is None: remapped_val = value @@ -117,11 +106,6 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex remapped_val = round(remapped_val, int(-math.log10(ax.tol))) self.remapped_vals[(value, ax.name)] = remapped_val - # remapped_val = value - # if ax.is_cyclic: - # remapped_val_interm = ax.remap([value, value])[0] - # remapped_val = (remapped_val_interm[0] + remapped_val_interm[1]) / 2 - # remapped_val = round(remapped_val, int(-math.log10(ax.tol))) child = node.create_child(ax, remapped_val) child["unsliced_polytopes"] = copy(node["unsliced_polytopes"]) child["unsliced_polytopes"].remove(polytope) @@ -134,11 +118,8 @@ def _build_branch(self, ax, node, datacube, next_nodes): if ax.name in polytope._axes: lower, upper, slice_axis_idx = polytope.extents(ax.name) # here, first check if the axis is an unsliceable axis and directly build node if it is - # if isinstance(ax, UnsliceableDatacubeAxis): # NOTE: we should have already created the ax_is_unsliceable cache before - # if self.ax_is_unsliceable.get(ax.name, None) is None: - # self.ax_is_unsliceable[ax.name] = isinstance(ax, UnsliceableDatacubeAxis) if self.ax_is_unsliceable[ax.name]: self._build_unsliceable_child(polytope, ax, node, datacube, lower, next_nodes, slice_axis_idx) else: diff --git a/polytope/polytope.py b/polytope/polytope.py index 8089b96dd..f6d4a723e 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -3,8 +3,6 @@ from .shapes import ConvexPolytope from .utility.exceptions import AxisOverdefinedError -import time - class Request: """Encapsulates a request for data""" From de8ae1941c70e87965cc75edab0e1bdd039c9118 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 12 Dec 2023 15:19:27 +0100 Subject: [PATCH 227/332] ignore tests which need fdb --- tests/helper_functions.py | 25 ++++++++++++++++++++++ tests/test_datacube_axes_init.py | 1 - tests/test_fdb_datacube.py | 2 +- tests/test_healpix_mapper.py | 28 ++----------------------- tests/test_incomplete_tree_fdb.py | 28 ++----------------------- tests/test_merge_cyclic_octahedral.py | 2 ++ tests/test_octahedral_grid.py | 28 ++----------------------- tests/test_regular_grid.py | 30 +++------------------------ tests/test_shapes.py | 2 +- tests/test_slice_date_range_fdb.py | 2 +- 10 files changed, 39 insertions(+), 109 deletions(-) diff --git a/tests/helper_functions.py b/tests/helper_functions.py index 4e82bbe1a..936fab35d 100644 --- a/tests/helper_functions.py +++ b/tests/helper_functions.py @@ -1,6 +1,7 @@ import os import requests +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file class HTTPError(Exception): @@ -27,3 +28,27 @@ def download_test_data(nexus_url, filename): # Save the downloaded data to the local file with open(local_file_path, "wb") as f: f.write(response.content) + + +def find_nearest_latlon(grib_file, target_lat, target_lon): + # Open the GRIB file + f = open(grib_file) + + # Load the GRIB messages from the file + messages = [] + while True: + message = codes_grib_new_from_file(f) + if message is None: + break + messages.append(message) + + # Find the nearest grid points + nearest_points = [] + for message in messages: + nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) + nearest_points.append(nearest_index) + + # Close the GRIB file + f.close() + + return nearest_points diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index f2a7f3eb4..f3739e5bf 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -20,7 +20,6 @@ def setup_method(self, method): self.xarraydatacube = XArrayDatacube(latlon_array) self.options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - # "latitude": {"transformation": {"reverse": {True}}}, } self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 03bc1084c..490cb8f03 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -26,7 +26,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index 043f1ea12..ea6c4e6f9 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -1,7 +1,6 @@ import pytest from earthkit import data -from eccodes import codes_grib_find_nearest, codes_grib_new_from_file -from helper_functions import download_test_data +from helper_functions import download_test_data, find_nearest_latlon from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer @@ -24,29 +23,6 @@ def setup_method(self, method): self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) - def find_nearest_latlon(self, grib_file, target_lat, target_lon): - # Open the GRIB file - f = open(grib_file) - - # Load the GRIB messages from the file - messages = [] - while True: - message = codes_grib_new_from_file(f) - if message is None: - break - messages.append(message) - - # Find the nearest grid points - nearest_points = [] - for message in messages: - nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) - nearest_points.append(nearest_index) - - # Close the GRIB file - f.close() - - return nearest_points - @pytest.mark.internet def test_healpix_grid(self): request = Request( @@ -70,7 +46,7 @@ def test_healpix_grid(self): lon = cubepath["longitude"] lats.append(lat) lons.append(lon) - nearest_points = self.find_nearest_latlon("./tests/data/healpix.grib", lat, lon) + nearest_points = find_nearest_latlon("./tests/data/healpix.grib", lat, lon) eccodes_lat = nearest_points[0][0]["lat"] eccodes_lon = nearest_points[0][0]["lon"] eccodes_lats.append(eccodes_lat) diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 6f5f11ecf..2633e7b43 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -1,6 +1,5 @@ import pandas as pd import pytest -from eccodes import codes_grib_find_nearest, codes_grib_new_from_file from helper_functions import download_test_data from polytope.datacube.backends.fdb import FDBDatacube @@ -25,31 +24,8 @@ def setup_method(self, method): self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) - def find_nearest_latlon(self, grib_file, target_lat, target_lon): - # Open the GRIB file - f = open(grib_file) - - # Load the GRIB messages from the file - messages = [] - while True: - message = codes_grib_new_from_file(f) - if message is None: - break - messages.append(message) - - # Find the nearest grid points - nearest_points = [] - for message in messages: - nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) - nearest_points.append(nearest_index) - - # Close the GRIB file - f.close() - - return nearest_points - @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch(self): request = Request( Select("step", [0]), @@ -72,7 +48,7 @@ def test_incomplete_fdb_branch(self): assert result.is_root() @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch_2(self): request = Request( Select("step", [0]), diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index fd50b6b1a..efbab37ec 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -1,4 +1,5 @@ import numpy as np +import pytest import xarray as xr from polytope.datacube.backends.xarray import XArrayDatacube @@ -30,6 +31,7 @@ def setup_method(self, method): self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) + @pytest.mark.skip(reason="Wrong datacube format.") def test_merge_axis(self): # NOTE: does not work because the date is a string in the merge option... date = np.datetime64("2000-01-01T06:00:00") diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index bf1ace37b..adc251684 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -1,7 +1,6 @@ import pytest from earthkit import data -from eccodes import codes_grib_find_nearest, codes_grib_new_from_file -from helper_functions import download_test_data +from helper_functions import download_test_data, find_nearest_latlon from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer @@ -24,29 +23,6 @@ def setup_method(self, method): self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) - def find_nearest_latlon(self, grib_file, target_lat, target_lon): - # Open the GRIB file - f = open(grib_file) - - # Load the GRIB messages from the file - messages = [] - while True: - message = codes_grib_new_from_file(f) - if message is None: - break - messages.append(message) - - # Find the nearest grid points - nearest_points = [] - for message in messages: - nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) - nearest_points.append(nearest_index) - - # Close the GRIB file - f.close() - - return nearest_points - @pytest.mark.internet def test_octahedral_grid(self): request = Request( @@ -70,7 +46,7 @@ def test_octahedral_grid(self): lon = cubepath["longitude"] lats.append(lat) lons.append(lon) - nearest_points = self.find_nearest_latlon("./tests/data/foo.grib", lat, lon) + nearest_points = find_nearest_latlon("./tests/data/foo.grib", lat, lon) eccodes_lat = nearest_points[0][0]["lat"] eccodes_lon = nearest_points[0][0]["lon"] eccodes_lats.append(eccodes_lat) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 5e5fa3878..934384782 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -1,7 +1,6 @@ import pandas as pd import pytest -from eccodes import codes_grib_find_nearest, codes_grib_new_from_file -from helper_functions import download_test_data +from helper_functions import download_test_data, find_nearest_latlon from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer @@ -28,31 +27,8 @@ def setup_method(self, method): self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) - def find_nearest_latlon(self, grib_file, target_lat, target_lon): - # Open the GRIB file - f = open(grib_file) - - # Load the GRIB messages from the file - messages = [] - while True: - message = codes_grib_new_from_file(f) - if message is None: - break - messages.append(message) - - # Find the nearest grid points - nearest_points = [] - for message in messages: - nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) - nearest_points.append(nearest_index) - - # Close the GRIB file - f.close() - - return nearest_points - @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_regular_grid(self): request = Request( Select("step", [0]), @@ -82,7 +58,7 @@ def test_regular_grid(self): lon = cubepath["longitude"] lats.append(lat) lons.append(lon) - nearest_points = self.find_nearest_latlon("./tests/data/era5-levels-members.grib", lat, lon) + nearest_points = find_nearest_latlon("./tests/data/era5-levels-members.grib", lat, lon) eccodes_lat = nearest_points[0][0]["lat"] eccodes_lon = nearest_points[0][0]["lon"] eccodes_lats.append(eccodes_lat) diff --git a/tests/test_shapes.py b/tests/test_shapes.py index 3d9d8b202..29c6b35f6 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -39,7 +39,7 @@ def test_all_cyclic(self): # result.pprint() assert len(result.leaves) == 360 - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_all_mapper_cyclic(self): self.options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index fb4255db8..a233e2c26 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -22,7 +22,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), From 191fb4aec85ad120377d82b20ec21211c0d70e7b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 13 Dec 2023 19:22:41 +0100 Subject: [PATCH 228/332] fix tests which didn't work --- performance/fdb_slice_many_numbers_timeseries.py | 2 +- tests/test_fdb_datacube.py | 4 +++- tests/test_incomplete_tree_fdb.py | 4 ++-- tests/test_regular_grid.py | 2 +- tests/test_shapes.py | 4 +++- tests/test_slice_date_range_fdb.py | 4 +++- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index c6174f918..46fd0abe4 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -29,7 +29,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( # Select("step", [0]), diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index ff56f6709..733988ac5 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -21,6 +21,7 @@ def setup_method(self, method): }, "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, + "number": {"transformation": {"type_change": "int"}} } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -28,7 +29,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), @@ -40,6 +41,7 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), + Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 87d55aef1..28c357465 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -51,7 +51,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch(self): request = Request( Select("step", [0]), @@ -74,7 +74,7 @@ def test_incomplete_fdb_branch(self): assert result.is_root() @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch_2(self): request = Request( Select("step", [0]), diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 1de8a8705..9e3fb7986 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -54,7 +54,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_regular_grid(self): request = Request( Select("step", [0]), diff --git a/tests/test_shapes.py b/tests/test_shapes.py index ebc55b527..a04c7bfd5 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -39,7 +39,7 @@ def test_all_cyclic(self): # result.pprint() assert len(result.leaves) == 360 - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_all_mapper_cyclic(self): self.options = { "values": { @@ -50,6 +50,7 @@ def test_all_mapper_cyclic(self): "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, "longitude": {"transformation": {"cyclic": [0, 360]}}, + "number": {"transformation": {"type_change": "int"}} } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -66,6 +67,7 @@ def test_all_mapper_cyclic(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["fc"]), + Select("number", [1]), Span("latitude", 89.9, 90), All("longitude"), ) diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 567230687..56d2d5290 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -18,6 +18,7 @@ def setup_method(self, method): }, "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, + "number": {"transformation": {"type_change": "int"}} } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -25,7 +26,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), @@ -37,6 +38,7 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), + Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) From da2044d7461fb7a15db0ddf91d59735bfbcbfedd Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 13 Dec 2023 19:27:50 +0100 Subject: [PATCH 229/332] mark fdb tests --- .github/workflows/ci.yaml | 2 +- performance/fdb_slice_many_numbers_timeseries.py | 2 -- pyproject.toml | 3 ++- tests/test_fdb_datacube.py | 4 ++-- tests/test_incomplete_tree_fdb.py | 4 ++-- tests/test_regular_grid.py | 2 +- tests/test_shapes.py | 4 ++-- tests/test_slice_date_range_fdb.py | 4 ++-- 8 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 017d4bf0e..56b8e7ee0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -131,7 +131,7 @@ jobs: LD_LIBRARY_PATH: ${{ steps.install-dependencies.outputs.lib_path }} shell: bash -eux {0} run: | - DYLD_LIBRARY_PATH=${{ env.LD_LIBRARY_PATH }} python -m pytest tests --cov=./ --cov-report=xml + DYLD_LIBRARY_PATH=${{ env.LD_LIBRARY_PATH }} python -m pytest -m "not fdb" tests --cov=./ --cov-report=xml python -m coverage report - name: Upload coverage to Codecov diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index 46fd0abe4..d7975d6da 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -1,7 +1,6 @@ import time import pandas as pd -import pytest from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer @@ -29,7 +28,6 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( # Select("step", [0]), diff --git a/pyproject.toml b/pyproject.toml index 28f4ec600..ced7b03b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,4 +4,5 @@ line-length = 120 requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -markers = ["internet: downloads test data from the internet (deselect with '-m \"not internet\"')",] \ No newline at end of file +markers = ["internet: downloads test data from the internet (deselect with '-m \"not internet\"')", + "fdb: test which uses fdb (deselect with '-m \"not fdb\"')",] \ No newline at end of file diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 733988ac5..5c4f3aa4b 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -21,7 +21,7 @@ def setup_method(self, method): }, "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, - "number": {"transformation": {"type_change": "int"}} + "number": {"transformation": {"type_change": "int"}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -29,7 +29,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.fdb def test_fdb_datacube(self): request = Request( Select("step", [0]), diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 28c357465..d41146611 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -51,7 +51,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.fdb def test_incomplete_fdb_branch(self): request = Request( Select("step", [0]), @@ -74,7 +74,7 @@ def test_incomplete_fdb_branch(self): assert result.is_root() @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.fdb def test_incomplete_fdb_branch_2(self): request = Request( Select("step", [0]), diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 9e3fb7986..b6fd81930 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -54,7 +54,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.fdb def test_regular_grid(self): request = Request( Select("step", [0]), diff --git a/tests/test_shapes.py b/tests/test_shapes.py index a04c7bfd5..a46410974 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -39,7 +39,7 @@ def test_all_cyclic(self): # result.pprint() assert len(result.leaves) == 360 - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.fdb def test_all_mapper_cyclic(self): self.options = { "values": { @@ -50,7 +50,7 @@ def test_all_mapper_cyclic(self): "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, "longitude": {"transformation": {"cyclic": [0, 360]}}, - "number": {"transformation": {"type_change": "int"}} + "number": {"transformation": {"type_change": "int"}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 56d2d5290..2ecccb7ee 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -18,7 +18,7 @@ def setup_method(self, method): }, "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, - "number": {"transformation": {"type_change": "int"}} + "number": {"transformation": {"type_change": "int"}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -26,7 +26,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.fdb def test_fdb_datacube(self): request = Request( Select("step", [0]), From 5abd67f8740795373518aafda85687946e16cea3 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 14 Dec 2023 16:54:58 +0100 Subject: [PATCH 230/332] make fdb backend work with gribjump instead of fdb --- polytope/datacube/backends/fdb.py | 19 +++++++++++++--- tests/.DS_Store | Bin 8196 -> 8196 bytes tests/test_fdb_datacube.py | 3 ++- tests/test_incomplete_tree_fdb.py | 2 +- tests/test_regular_grid.py | 35 +++++++++++++++++++++++++++-- tests/test_shapes.py | 2 +- tests/test_slice_date_range_fdb.py | 2 +- 7 files changed, 54 insertions(+), 9 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 0679dbdd0..8355c61b7 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -1,5 +1,6 @@ from copy import deepcopy +import pygribjump as pygj import pyfdb from .datacube import Datacube, IndexTree @@ -21,8 +22,14 @@ def __init__(self, config={}, axis_options={}): partial_request = config # Find values in the level 3 FDB datacube # Will be in the form of a dictionary? {axis_name:values_available, ...} - self.fdb = pyfdb.FDB() - self.fdb_coordinates = self.fdb.axes(partial_request).as_dict() + # self.fdb = pyfdb.FDB() + self.fdb = pygj.GribJump() + self.fdb_coordinates = self.fdb.axes(partial_request) + # self.fdb_coordinates = self.fdb.axes("class=od") + # print("NOW") + # print(self.fdb_coordinates) + # self.fdb_coordinates = self.fdb.axes(partial_request).as_dict() + # self.fdb_coordinates["number"] = self.fdb_coordinates["number"][::-1] self.fdb_coordinates["values"] = [] for name, values in self.fdb_coordinates.items(): values.sort() @@ -126,6 +133,7 @@ def give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_ (output_values, original_indices) = self.find_fdb_values( leaf_path, range_lengths, current_start_idx, lat_length ) + print(output_values) new_fdb_range_nodes = [] new_range_lengths = [] for j in range(lat_length): @@ -138,7 +146,9 @@ def give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_ for i in range(len(sorted_fdb_range_nodes)): for k in range(sorted_range_lengths[i]): n = sorted_fdb_range_nodes[i][k] - n.result = output_values[0][0][0][i][k] + print(output_values[0][0][i][0][k]) + # n.result = output_values[0][0][0][i][k] + n.result = output_values[0][0][i][0][k] def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): path.pop("values") @@ -153,6 +163,9 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) + print("BEFORE EXTRACT") + # print(path) + print(fdb_requests) output_values = self.fdb.extract(fdb_requests) return (output_values, original_indices) diff --git a/tests/.DS_Store b/tests/.DS_Store index 28aa9c59f3e8cfd1d76bfcb4e242dc62551a9375..6669a9c325d6a62f2dbbed0a0af1e2777a9de4a8 100644 GIT binary patch delta 308 zcmZp1XmOa}&nUJrU^hRb*km37Yvp8ye1=?x0)|9}B8E(cVjwS%A)djKAsZ+f&tM3m zCmRTf>aS;DV9@&y20#`A13NQ5ffq^M~a)W^MJJC?vri4|GsI zL&{`5At@%H{&ND6j0zw#Ws4Y6844Lnfn+hzoD!f*Qh^YmT$~l?W=6ivGJ<~@dE*&C j9xVl$1yndmP>KmC@k6Maabrak<7Re=Z!DbHWy2W(^hZ!5 delta 62 zcmV-E0Kxx+K!iY$PXQ0HP`eKS50eZKJCnZSER$IhPO}FS@&U6$ U6>0&qgA9ZNvj-UV1hIGr19$}!?f?J) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 5c4f3aa4b..692a327d0 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -23,7 +23,8 @@ def setup_method(self, method): "step": {"transformation": {"type_change": "int"}}, "number": {"transformation": {"type_change": "int"}}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc"} + # self.config = "class=od, expver=001, levtype=sfc, step=0" self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index d41146611..852cd7bd1 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -22,7 +22,7 @@ def setup_method(self, method): "number": {"transformation": {"type_change": "int"}}, "longitude": {"transformation": {"cyclic": [0, 360]}}, } - self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} + self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": "0"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index b6fd81930..a0b6b04ea 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -6,7 +6,7 @@ from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Disk, Select +from polytope.shapes import Disk, Select, Span # import geopandas as gpd # import matplotlib.pyplot as plt @@ -25,10 +25,25 @@ def setup_method(self, method): "number": {"transformation": {"type_change": "int"}}, "longitude": {"transformation": {"cyclic": [0, 360]}}, } - self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} + self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": "0"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + print("NOW") + print(self.fdbdatacube.fdb_coordinates) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + # self.options = { + # "values": { + # "transformation": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}} + # }, + # "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, + # "step": {"transformation": {"type_change": "int"}}, + # "number": {"transformation": {"type_change": "int"}}, + # "longitude": {"transformation": {"cyclic": [0, 360]}}, + # } + # self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": "0"} + # self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + # self.slicer = HullSlicer() + # self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) def find_nearest_latlon(self, grib_file, target_lat, target_lon): # Open the GRIB file @@ -55,6 +70,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): @pytest.mark.internet @pytest.mark.fdb + # @pytest.mark.skip("FDB issue") def test_regular_grid(self): request = Request( Select("step", [0]), @@ -70,6 +86,21 @@ def test_regular_grid(self): Select("levelist", ["500"]), Select("number", ["0", "1"]), ) + # request = Request( + # Select("step", [0]), + # Select("levtype", ["pl"]), + # Select("date", [pd.Timestamp("20170102T120000")]), + # Select("domain", ["g"]), + # Select("expver", ["0001"]), + # Select("param", ["129"]), + # Select("class", ["ea"]), + # Select("stream", ["enda"]), + # Select("type", ["an"]), + # Select("latitude", [0]), + # Select("longitude", [0]), + # Select("levelist", ["500"]), + # Select("number", ["0"]), + # ) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 10 diff --git a/tests/test_shapes.py b/tests/test_shapes.py index a46410974..2c87f1211 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -52,7 +52,7 @@ def test_all_mapper_cyclic(self): "longitude": {"transformation": {"cyclic": [0, 360]}}, "number": {"transformation": {"type_change": "int"}}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": "11"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 2ecccb7ee..e700ef551 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -20,7 +20,7 @@ def setup_method(self, method): "step": {"transformation": {"type_change": "int"}}, "number": {"transformation": {"type_change": "int"}}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": "0"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) From c4af86b5d121260ced80b40e1c15359b63ebbee0 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 14 Dec 2023 16:58:02 +0100 Subject: [PATCH 231/332] clean up --- polytope/datacube/backends/fdb.py | 15 +------------- tests/test_regular_grid.py | 33 +------------------------------ 2 files changed, 2 insertions(+), 46 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 8355c61b7..b00010fcd 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -1,7 +1,6 @@ from copy import deepcopy import pygribjump as pygj -import pyfdb from .datacube import Datacube, IndexTree @@ -21,15 +20,9 @@ def __init__(self, config={}, axis_options={}): partial_request = config # Find values in the level 3 FDB datacube - # Will be in the form of a dictionary? {axis_name:values_available, ...} - # self.fdb = pyfdb.FDB() + self.fdb = pygj.GribJump() self.fdb_coordinates = self.fdb.axes(partial_request) - # self.fdb_coordinates = self.fdb.axes("class=od") - # print("NOW") - # print(self.fdb_coordinates) - # self.fdb_coordinates = self.fdb.axes(partial_request).as_dict() - # self.fdb_coordinates["number"] = self.fdb_coordinates["number"][::-1] self.fdb_coordinates["values"] = [] for name, values in self.fdb_coordinates.items(): values.sort() @@ -133,7 +126,6 @@ def give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_ (output_values, original_indices) = self.find_fdb_values( leaf_path, range_lengths, current_start_idx, lat_length ) - print(output_values) new_fdb_range_nodes = [] new_range_lengths = [] for j in range(lat_length): @@ -146,8 +138,6 @@ def give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_ for i in range(len(sorted_fdb_range_nodes)): for k in range(sorted_range_lengths[i]): n = sorted_fdb_range_nodes[i][k] - print(output_values[0][0][i][0][k]) - # n.result = output_values[0][0][0][i][k] n.result = output_values[0][0][i][0][k] def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): @@ -163,9 +153,6 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) - print("BEFORE EXTRACT") - # print(path) - print(fdb_requests) output_values = self.fdb.extract(fdb_requests) return (output_values, original_indices) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index a0b6b04ea..14bd45fb4 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -6,7 +6,7 @@ from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Disk, Select, Span +from polytope.shapes import Disk, Select # import geopandas as gpd # import matplotlib.pyplot as plt @@ -27,23 +27,8 @@ def setup_method(self, method): } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": "0"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) - print("NOW") - print(self.fdbdatacube.fdb_coordinates) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) - # self.options = { - # "values": { - # "transformation": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}} - # }, - # "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, - # "step": {"transformation": {"type_change": "int"}}, - # "number": {"transformation": {"type_change": "int"}}, - # "longitude": {"transformation": {"cyclic": [0, 360]}}, - # } - # self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": "0"} - # self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) - # self.slicer = HullSlicer() - # self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) def find_nearest_latlon(self, grib_file, target_lat, target_lon): # Open the GRIB file @@ -70,7 +55,6 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): @pytest.mark.internet @pytest.mark.fdb - # @pytest.mark.skip("FDB issue") def test_regular_grid(self): request = Request( Select("step", [0]), @@ -86,21 +70,6 @@ def test_regular_grid(self): Select("levelist", ["500"]), Select("number", ["0", "1"]), ) - # request = Request( - # Select("step", [0]), - # Select("levtype", ["pl"]), - # Select("date", [pd.Timestamp("20170102T120000")]), - # Select("domain", ["g"]), - # Select("expver", ["0001"]), - # Select("param", ["129"]), - # Select("class", ["ea"]), - # Select("stream", ["enda"]), - # Select("type", ["an"]), - # Select("latitude", [0]), - # Select("longitude", [0]), - # Select("levelist", ["500"]), - # Select("number", ["0"]), - # ) result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 10 From 70fa4b0e1a0a9131255d1da628e2a5b9296819dd Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 14 Dec 2023 17:10:20 +0100 Subject: [PATCH 232/332] clean up --- tests/test_fdb_datacube.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 692a327d0..bb76ed00e 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -24,7 +24,6 @@ def setup_method(self, method): "number": {"transformation": {"type_change": "int"}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc"} - # self.config = "class=od, expver=001, levtype=sfc, step=0" self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) From 8ffa3cef01c8b0a2f422727386a83fffe7ea68ce Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 14 Dec 2023 17:14:24 +0100 Subject: [PATCH 233/332] remove gribjump test --- tests/test_regular_grid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 14bd45fb4..0a7fb24c6 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -55,6 +55,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): @pytest.mark.internet @pytest.mark.fdb + @pytest.mark.skip("Depends on GribJump") def test_regular_grid(self): request = Request( Select("step", [0]), From fda49c655777f2b96eb81603e27c390ec31d9b1f Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 14 Dec 2023 17:22:51 +0100 Subject: [PATCH 234/332] don't skip gribjump test --- tests/test_regular_grid.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 0a7fb24c6..14bd45fb4 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -55,7 +55,6 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): @pytest.mark.internet @pytest.mark.fdb - @pytest.mark.skip("Depends on GribJump") def test_regular_grid(self): request = Request( Select("step", [0]), From c4ca9d2d00b7251ec143b49ace6e75a1925b35b1 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 18 Dec 2023 11:10:29 +0100 Subject: [PATCH 235/332] reformat pytest markers --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ced7b03b7..937b5e9ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,4 +5,4 @@ requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] markers = ["internet: downloads test data from the internet (deselect with '-m \"not internet\"')", - "fdb: test which uses fdb (deselect with '-m \"not fdb\"')",] \ No newline at end of file + "fdb: test which uses fdb (deselect with '-m \"not fdb\"')",] \ No newline at end of file From 4df602a637250669d35b2b28e70aa1993422c170 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 18 Dec 2023 11:19:05 +0100 Subject: [PATCH 236/332] add pytest mark --- tests/test_regular_grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 14bd45fb4..f6541fe6d 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -12,6 +12,7 @@ # import matplotlib.pyplot as plt +@pytest.mark.fdb class TestRegularGrid: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" @@ -54,7 +55,6 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - @pytest.mark.fdb def test_regular_grid(self): request = Request( Select("step", [0]), From d141dc15cb8492a42ee02bf53d6b1e1e3e81d3e2 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 18 Dec 2023 11:21:45 +0100 Subject: [PATCH 237/332] add pytest mark before test --- tests/test_regular_grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index f6541fe6d..2005848c6 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -12,7 +12,6 @@ # import matplotlib.pyplot as plt -@pytest.mark.fdb class TestRegularGrid: def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" @@ -54,6 +53,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points + @pytest.mark.fdb @pytest.mark.internet def test_regular_grid(self): request = Request( From 06dc236a70aef062ff8dcb29355708931c1d23bb Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Dec 2023 10:13:54 +0100 Subject: [PATCH 238/332] quick fix to surrounding method --- .../fdb_slice_many_numbers_timeseries.py | 56 +++++++++++++++++++ polytope/datacube/backends/datacube.py | 3 +- polytope/datacube/datacube_axis.py | 15 +++-- polytope/shapes.py | 2 +- 4 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 performance/fdb_slice_many_numbers_timeseries.py diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py new file mode 100644 index 000000000..29a4dd1ed --- /dev/null +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -0,0 +1,56 @@ +import time + +import pandas as pd +import pytest + +from polytope.datacube.backends.fdb import FDBDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import All, Point, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + } + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + "number": {"transformation": {"type_change": "int"}}, + "longitude": {"transformation": {"cyclic": [0, 360]}} + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.skip(reason="can't install fdb branch on CI") + def test_fdb_datacube(self): + request = Request( + # Select("step", [0]), + All("step"), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231205T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["enfo"]), + Select("type", ["pf"]), + # Select("latitude", [0.035149384216], method="surrounding"), + # Select("latitude", [0.04], method="surrounding"), + # Select("longitude", [0], method="surrounding"), + Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"), + All("number") + ) + time1 = time.time() + result = self.API.retrieve(request) + print(time.time() - time1) + print(len(result.leaves)) + # result.pprint() + # assert len(result.leaves) == 9 diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 10b5a6613..9d1ba9f3e 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -40,7 +40,6 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt for unwanted_axis in transformation.unwanted_axes(): self.unwanted_axes.append(unwanted_axis) for axis_name in final_axis_names: - self.complete_axes.append(axis_name) self.fake_axes.append(axis_name) # if axis does not yet exist, create it @@ -84,7 +83,7 @@ def has_index(self, path: DatacubePath, axis, index): def fit_path(self, path): for key in path.keys(): - if key not in self.complete_axes: + if key not in self.complete_axes and key not in self.fake_axes: path.pop(key) return path diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index a84bfd303..5c78c22f6 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -155,8 +155,9 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") else: - start = indexes.index(low) - end = indexes.index(up) + start = bisect.bisect_left(indexes, low) + end = bisect.bisect_right(indexes, up) + if start - 1 < 0: index_val_found = indexes[-1:][0] indexes_between_ranges.append([index_val_found]) @@ -264,8 +265,14 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): if cls.name in transformation._mapped_axes(): for idxs in index_ranges: if method == "surrounding": - start = idxs.index(low) - end = idxs.index(up) + axis_reversed = transform._axis_reversed[cls.name] + if not axis_reversed: + start = bisect.bisect_left(idxs, low) + end = bisect.bisect_right(idxs, up) + else: + # TODO: do the custom bisect + end = bisect_left_cmp(idxs, low, cmp=lambda x, y: x > y) + 1 + start = bisect_right_cmp(idxs, up, cmp=lambda x, y: x > y) start = max(start - 1, 0) end = min(end + 1, len(idxs)) indexes_between = idxs[start:end] diff --git a/polytope/shapes.py b/polytope/shapes.py index 5653f521e..c10b12e4b 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -74,7 +74,7 @@ def __init__(self, axes, values, method=None): self.polytopes = [] for i in range(len(axes)): polytope_points = [v[i] for v in self.values] - self.polytopes.append(ConvexPolytope([axes[i]], [polytope_points], method)) + self.polytopes.append(ConvexPolytope([axes[i]], [polytope_points], self.method)) def axes(self): return self._axes From 0183685f66f6acd8c79978a74fb9453f3ab21cf7 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Dec 2023 10:46:02 +0100 Subject: [PATCH 239/332] black --- performance/fdb_slice_many_numbers_timeseries.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index 29a4dd1ed..c6174f918 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -21,7 +21,7 @@ def setup_method(self, method): "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, "number": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}} + "longitude": {"transformation": {"cyclic": [0, 360]}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -46,7 +46,7 @@ def test_fdb_datacube(self): # Select("latitude", [0.04], method="surrounding"), # Select("longitude", [0], method="surrounding"), Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"), - All("number") + All("number"), ) time1 = time.time() result = self.API.retrieve(request) From 2fb17356693a8d0d6ad8cdecf4b264bf502dca87 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Dec 2023 11:04:30 +0100 Subject: [PATCH 240/332] time components in example --- .../fdb_slice_many_numbers_timeseries.py | 80 ++++++++----------- 1 file changed, 34 insertions(+), 46 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index c6174f918..c65ce2084 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -1,56 +1,44 @@ import time import pandas as pd -import pytest from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import All, Point, Select +time1 = time.time() +# Create a dataarray with 3 labelled axes using different index types +options = { + "values": { + "transformation": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + "number": {"transformation": {"type_change": "int"}}, + "longitude": {"transformation": {"cyclic": [0, 360]}}, +} +config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"} +fdbdatacube = FDBDatacube(config, axis_options=options) +slicer = HullSlicer() +self_API = Polytope(datacube=fdbdatacube, engine=slicer, axis_options=options) -class TestSlicingFDBDatacube: - def setup_method(self, method): - # Create a dataarray with 3 labelled axes using different index types - self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, - "number": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}}, - } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"} - self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) - self.slicer = HullSlicer() - self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) - - # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") - def test_fdb_datacube(self): - request = Request( - # Select("step", [0]), - All("step"), - Select("levtype", ["sfc"]), - Select("date", [pd.Timestamp("20231205T000000")]), - Select("domain", ["g"]), - Select("expver", ["0001"]), - Select("param", ["167"]), - Select("class", ["od"]), - Select("stream", ["enfo"]), - Select("type", ["pf"]), - # Select("latitude", [0.035149384216], method="surrounding"), - # Select("latitude", [0.04], method="surrounding"), - # Select("longitude", [0], method="surrounding"), - Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"), - All("number"), - ) - time1 = time.time() - result = self.API.retrieve(request) - print(time.time() - time1) - print(len(result.leaves)) - # result.pprint() - # assert len(result.leaves) == 9 +time2 = time.time() +request = Request( + All("step"), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231205T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["enfo"]), + Select("type", ["pf"]), + # Select("latitude", [0.035149384216], method="surrounding"), + Point(["latitude", "longitude"], [[0.04, 0]], method="surrounding"), + All("number"), +) +result = self_API.retrieve(request) +print(time.time() - time1) +print(time.time() - time2) +print(len(result.leaves)) From f0d4cbe2e3ee9005281933da176b04dd562844ab Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Dec 2023 13:57:58 +0100 Subject: [PATCH 241/332] remove the transformation keyword in datacube options --- .../fdb_slice_many_numbers_timeseries.py | 12 +++++------- polytope/datacube/backends/datacube.py | 9 +++------ polytope/datacube/backends/fdb.py | 11 ++--------- polytope/datacube/backends/xarray.py | 12 +++++------- tests/test_cyclic_axis_over_negative_vals.py | 4 ++-- tests/test_cyclic_axis_slicer_not_0.py | 4 ++-- tests/test_cyclic_axis_slicing.py | 4 ++-- tests/test_cyclic_simple.py | 2 +- tests/test_cyclic_snapping.py | 2 +- tests/test_datacube_axes_init.py | 6 +----- tests/test_fdb_datacube.py | 15 +++++++-------- tests/test_healpix_mapper.py | 6 ++---- tests/test_incomplete_tree_fdb.py | 16 +++++++--------- tests/test_merge_cyclic_octahedral.py | 10 +++------- tests/test_merge_octahedral_one_axis.py | 8 ++------ tests/test_merge_transformation.py | 2 +- tests/test_octahedral_grid.py | 6 +----- tests/test_regular_grid.py | 14 ++++++-------- tests/test_reverse_transformation.py | 2 +- tests/test_shapes.py | 18 ++++++++---------- tests/test_slice_date_range_fdb.py | 14 ++++++-------- tests/test_slicer_era5.py | 2 +- tests/test_snapping_real_data.py | 4 ++-- tests/test_type_change_transformation.py | 2 +- 24 files changed, 72 insertions(+), 113 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index c65ce2084..45f67699f 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -10,13 +10,11 @@ time1 = time.time() # Create a dataarray with 3 labelled axes using different index types options = { - "values": { - "transformation": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, - "number": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}}, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "longitude": {"cyclic": [0, 360]}, } config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"} fdbdatacube = FDBDatacube(config, axis_options=options) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 9d1ba9f3e..efa20e93d 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -37,8 +37,6 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt ) for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) - for unwanted_axis in transformation.unwanted_axes(): - self.unwanted_axes.append(unwanted_axis) for axis_name in final_axis_names: self.fake_axes.append(axis_name) # if axis does not yet exist, create it @@ -61,12 +59,11 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt self._axes[axis_name].transformations.append(transformation) def _add_all_transformation_axes(self, options, name, values): - transformation_options = options["transformation"] - for transformation_type_key in transformation_options.keys(): - self._create_axes(name, values, transformation_type_key, transformation_options) + for transformation_type_key in options.keys(): + self._create_axes(name, values, transformation_type_key, options) def _check_and_add_axes(self, options, name, values): - if "transformation" in options: + if options is not None: self._add_all_transformation_axes(options, name, values) else: if name not in self.blocked_axes: diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 0679dbdd0..bab3fc2a6 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -11,10 +11,8 @@ def __init__(self, config={}, axis_options={}): self.axis_counter = 0 self._axes = None treated_axes = [] - self.non_complete_axes = [] self.complete_axes = [] self.blocked_axes = [] - self.unwanted_axes = [] self.fake_axes = [] self.unwanted_path = {} @@ -26,7 +24,7 @@ def __init__(self, config={}, axis_options={}): self.fdb_coordinates["values"] = [] for name, values in self.fdb_coordinates.items(): values.sort() - options = axis_options.get(name, {}) + options = axis_options.get(name, None) self._check_and_add_axes(options, name, values) treated_axes.append(name) self.complete_axes.append(name) @@ -34,15 +32,10 @@ def __init__(self, config={}, axis_options={}): # add other options to axis which were just created above like "lat" for the mapper transformations for eg for name in self._axes: if name not in treated_axes: - options = axis_options.get(name, {}) + options = axis_options.get(name, None) val = self._axes[name].type self._check_and_add_axes(options, name, val) - def remove_unwanted_axes(self, leaf_path): - for axis in self.unwanted_axes: - leaf_path.pop(axis) - return leaf_path - def get(self, requests: IndexTree, leaf_path={}): # First when request node is root, go to its children if requests.axis.name == "root": diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 650038e05..f8ca1c2e2 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -14,33 +14,31 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self._axes = None self.dataarray = dataarray treated_axes = [] - self.non_complete_axes = [] self.complete_axes = [] self.blocked_axes = [] self.fake_axes = [] - self.unwanted_axes = [] + for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: - options = axis_options.get(name, {}) + options = axis_options.get(name, None) self._check_and_add_axes(options, name, values) treated_axes.append(name) self.complete_axes.append(name) else: if self.dataarray[name].dims == (): - options = axis_options.get(name, {}) + options = axis_options.get(name, None) self._check_and_add_axes(options, name, values) treated_axes.append(name) - self.non_complete_axes.append(name) for name in dataarray.dims: if name not in treated_axes: - options = axis_options.get(name, {}) + options = axis_options.get(name, None) val = dataarray[name].values[0] self._check_and_add_axes(options, name, val) treated_axes.append(name) # add other options to axis which were just created above like "lat" for the mapper transformations for eg for name in self._axes: if name not in treated_axes: - options = axis_options.get(name, {}) + options = axis_options.get(name, None) val = self._axes[name].type self._check_and_add_axes(options, name, val) diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index 9008f9b82..52cd41c21 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -22,8 +22,8 @@ def setup_method(self, method): }, ) options = { - "long": {"transformation": {"cyclic": [-1.1, -0.1]}}, - "level": {"transformation": {"cyclic": [1, 129]}}, + "long": {"cyclic": [-1.1, -0.1]}, + "level": {"cyclic": [1, 129]}, } self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 12d39b566..8266c0a09 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -22,8 +22,8 @@ def setup_method(self, method): }, ) self.options = { - "long": {"transformation": {"cyclic": [-1.1, -0.1]}}, - "level": {"transformation": {"cyclic": [1, 129]}}, + "long": {"cyclic": [-1.1, -0.1]}, + "level": {"cyclic": [1, 129]}, } self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index 34c2cf6c8..c0ac914f5 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -22,8 +22,8 @@ def setup_method(self, method): }, ) self.options = { - "long": {"transformation": {"cyclic": [0, 1.0]}}, - "level": {"transformation": {"cyclic": [1, 129]}}, + "long": {"cyclic": [0, 1.0]}, + "level": {"cyclic": [1, 129]}, } self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() diff --git a/tests/test_cyclic_simple.py b/tests/test_cyclic_simple.py index bf6660ee7..f3bd1e28b 100644 --- a/tests/test_cyclic_simple.py +++ b/tests/test_cyclic_simple.py @@ -21,7 +21,7 @@ def setup_method(self, method): "long": [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], }, ) - options = {"long": {"transformation": {"cyclic": [0, 1.0]}}, "level": {"transformation": {"cyclic": [1, 129]}}} + options = {"long": {"cyclic": [0, 1.0]}, "level": {"cyclic": [1, 129]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_cyclic_snapping.py b/tests/test_cyclic_snapping.py index ab6189ba3..adb3a7959 100644 --- a/tests/test_cyclic_snapping.py +++ b/tests/test_cyclic_snapping.py @@ -16,7 +16,7 @@ def setup_method(self, method): "long": [0, 0.5, 1.0], }, ) - options = {"long": {"transformation": {"cyclic": [0, 1.0]}}} + options = {"long": {"cyclic": [0, 1.0]}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 6f3d59bbf..f2a7f3eb4 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -19,11 +19,7 @@ def setup_method(self, method): latlon_array = latlon_array.t2m self.xarraydatacube = XArrayDatacube(latlon_array) self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, # "latitude": {"transformation": {"reverse": {True}}}, } self.slicer = HullSlicer() diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index ff56f6709..03bc1084c 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -14,21 +14,19 @@ class TestSlicingFDBDatacube: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + print(self.fdbdatacube.fdb_coordinates) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), @@ -40,6 +38,7 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), + Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index 9014e5e3a..043f1ea12 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -18,10 +18,8 @@ def setup_method(self, method): self.latlon_array = ds.to_xarray().isel(step=0).isel(time=0).isel(isobaricInhPa=0).z self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { - "values": { - "transformation": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}} - }, - "longitude": {"transformation": {"cyclic": [0, 360]}}, + "values": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}}, + "longitude": {"cyclic": [0, 360]}, } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 87d55aef1..6f5f11ecf 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -14,13 +14,11 @@ def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") self.options = { - "values": { - "transformation": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}} - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, - "number": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}}, + "values": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": [" ", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "longitude": {"cyclic": [0, 360]}, } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -51,7 +49,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch(self): request = Request( Select("step", [0]), @@ -74,7 +72,7 @@ def test_incomplete_fdb_branch(self): assert result.is_root() @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch_2(self): request = Request( Select("step", [0]), diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index 6fcf16323..fd50b6b1a 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -22,13 +22,9 @@ def setup_method(self, method): }, ) self.options = { - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, - "step": {"transformation": {"cyclic": [0, 2]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "step": {"cyclic": [0, 2]}, } self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index 5ff5dae91..7c3764904 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -18,12 +18,8 @@ def setup_method(self, method): self.latlon_array = self.latlon_array.t2m self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, - "longitude": {"transformation": {"cyclic": [0, 360.0]}}, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "longitude": {"cyclic": [0, 360.0]}, } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index ac3abf5bb..28f075d56 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -19,7 +19,7 @@ def setup_method(self, method): "time": ["0600"], }, ) - self.options = {"date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}} + self.options = {"date": {"merge": {"with": "time", "linkers": ["T", "00"]}}} self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index 56b21ee53..bf1ace37b 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -19,11 +19,7 @@ def setup_method(self, method): self.latlon_array = self.latlon_array.t2m self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - } + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 1de8a8705..5e5fa3878 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -17,13 +17,11 @@ def setup_method(self, method): nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") self.options = { - "values": { - "transformation": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}} - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, - "number": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}}, + "values": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "longitude": {"cyclic": [0, 360]}, } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -54,7 +52,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_regular_grid(self): request = Request( Select("step", [0]), diff --git a/tests/test_reverse_transformation.py b/tests/test_reverse_transformation.py index 858f2f29f..e501dfdfc 100644 --- a/tests/test_reverse_transformation.py +++ b/tests/test_reverse_transformation.py @@ -17,7 +17,7 @@ def setup_method(self, method): "lat": [4, 3, 2, 1], }, ) - options = {"lat": {"transformation": {"reverse": {True}}}} + options = {"lat": {"reverse": {True}}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_shapes.py b/tests/test_shapes.py index ebc55b527..3d9d8b202 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -24,7 +24,7 @@ def setup_method(self, method): }, ) self.xarraydatacube = XArrayDatacube(array) - self.options = {"longitude": {"transformation": {"cyclic": [0, 360]}}} + self.options = {"longitude": {"cyclic": [0, 360]}} self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) @@ -39,17 +39,14 @@ def test_all_cyclic(self): # result.pprint() assert len(result.leaves) == 360 - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_all_mapper_cyclic(self): self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}}, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "longitude": {"cyclic": [0, 360]}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 11} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -58,6 +55,7 @@ def test_all_mapper_cyclic(self): request = Request( Select("step", [11]), + Select("number", [1]), Select("levtype", ["sfc"]), Select("date", [pd.Timestamp("20230710T120000")]), Select("domain", ["g"]), diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 567230687..fb4255db8 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -11,13 +11,10 @@ class TestSlicingFDBDatacube: def setup_method(self, method): # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": { - "transformation": { - "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} - } - }, - "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, - "step": {"transformation": {"type_change": "int"}}, + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -25,10 +22,11 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), + Select("number", [1]), Select("levtype", ["sfc"]), Span("date", pd.Timestamp("20230625T120000"), pd.Timestamp("20230626T120000")), Select("domain", ["g"]), diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 1cd0fc897..15d355cb1 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -18,7 +18,7 @@ def setup_method(self, method): array = ds.to_xarray().isel(step=0).t self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() - options = {"lat": {"transformation": {"reverse": {True}}}} + options = {"lat": {"reverse": {True}}} self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) @pytest.mark.internet diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index c0d2a7a7d..acad24fbc 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -22,8 +22,8 @@ def setup_method(self, method): self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() options = { - "latitude": {"transformation": {"reverse": {True}}}, - "longitude": {"transformation": {"cyclic": [0, 360.0]}}, + "latitude": {"reverse": {True}}, + "longitude": {"cyclic": [0, 360.0]}, } self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_type_change_transformation.py b/tests/test_type_change_transformation.py index 80f5755aa..d88a03aca 100644 --- a/tests/test_type_change_transformation.py +++ b/tests/test_type_change_transformation.py @@ -18,7 +18,7 @@ def setup_method(self, method): }, ) self.array = array - options = {"step": {"transformation": {"type_change": "int"}}} + options = {"step": {"type_change": "int"}} self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) From 9733d109f26e1c06c1494c74f2e9db6b045bd291 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Dec 2023 14:11:46 +0100 Subject: [PATCH 242/332] show fdb time --- performance/fdb_slice_many_numbers_timeseries.py | 1 + polytope/datacube/backends/fdb.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index 45f67699f..630b48f43 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -39,4 +39,5 @@ result = self_API.retrieve(request) print(time.time() - time1) print(time.time() - time2) +print(fdbdatacube.fdb_time) print(len(result.leaves)) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index bab3fc2a6..fad3ca4a7 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -4,6 +4,8 @@ from .datacube import Datacube, IndexTree +import time + class FDBDatacube(Datacube): def __init__(self, config={}, axis_options={}): @@ -15,6 +17,7 @@ def __init__(self, config={}, axis_options={}): self.blocked_axes = [] self.fake_axes = [] self.unwanted_path = {} + self.fdb_time = 0 partial_request = config # Find values in the level 3 FDB datacube @@ -146,7 +149,9 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) + time1 = time.time() output_values = self.fdb.extract(fdb_requests) + self.fdb_time += time.time() - time1 return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): From 0f5d63cbed210e27d1512af9be175867b4b39d91 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 7 Dec 2023 14:59:09 +0100 Subject: [PATCH 243/332] time different functions --- performance/fdb_slice_many_numbers_timeseries.py | 8 +++++--- polytope/engine/hullslicer.py | 5 +++++ polytope/polytope.py | 8 ++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index 630b48f43..0b592de78 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -18,10 +18,12 @@ } config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "pf"} fdbdatacube = FDBDatacube(config, axis_options=options) -slicer = HullSlicer() -self_API = Polytope(datacube=fdbdatacube, engine=slicer, axis_options=options) +self_API = Polytope(datacube=fdbdatacube, axis_options=options) + +print(time.time() - time1) time2 = time.time() + request = Request( All("step"), Select("levtype", ["sfc"]), @@ -39,5 +41,5 @@ result = self_API.retrieve(request) print(time.time() - time1) print(time.time() - time2) -print(fdbdatacube.fdb_time) +print(time.time()-time2 - fdbdatacube.fdb_time) print(len(result.leaves)) diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 6b0306ed2..51a622535 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -13,6 +13,8 @@ from ..utility.geometry import lerp from .engine import Engine +import time + class HullSlicer(Engine): def __init__(self): @@ -91,6 +93,7 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): request = IndexTree() combinations = tensor_product(groups) + time0 = time.time() for c in combinations: r = IndexTree() r["unsliced_polytopes"] = set(c) @@ -101,6 +104,8 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): self._build_branch(ax, node, datacube, next_nodes) current_nodes = next_nodes request.merge(r) + print("time spent inside extract building tree") + print(time.time() - time0) return request diff --git a/polytope/polytope.py b/polytope/polytope.py index f6d4a723e..5e4fd0914 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -3,6 +3,8 @@ from .shapes import ConvexPolytope from .utility.exceptions import AxisOverdefinedError +import time + class Request: """Encapsulates a request for data""" @@ -50,6 +52,12 @@ def slice(self, polytopes: List[ConvexPolytope]): def retrieve(self, request: Request, method="standard"): """Higher-level API which takes a request and uses it to slice the datacube""" + time1 = time.time() request_tree = self.engine.extract(self.datacube, request.polytopes()) + print("extract time") + print(time.time() - time1) + time0 = time.time() self.datacube.get(request_tree) + print("get time") + print(time.time() - time0 - self.datacube.fdb_time) return request_tree From 7959a1fef149de0bf5a4d259c79afa4fa4fd8d26 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 09:52:23 +0100 Subject: [PATCH 244/332] add shortcut if polytope is flat --- polytope/shapes.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/polytope/shapes.py b/polytope/shapes.py index c10b12e4b..6af698ee2 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -25,14 +25,22 @@ def axes(self) -> List[str]: class ConvexPolytope(Shape): def __init__(self, axes, points, method=None): self._axes = list(axes) + self.is_flat = False + if len(self._axes) == 1: + self.is_flat = True self.points = points self.method = method def extents(self, axis): - slice_axis_idx = self.axes().index(axis) - axis_values = [point[slice_axis_idx] for point in self.points] - lower = min(axis_values) - upper = max(axis_values) + if self.is_flat: + slice_axis_idx = 1 + lower = min(self.points)[0] + upper = max(self.points)[0] + else: + slice_axis_idx = self.axes().index(axis) + axis_values = [point[slice_axis_idx] for point in self.points] + lower = min(axis_values) + upper = max(axis_values) return (lower, upper, slice_axis_idx) def __str__(self): From 902d4ac96a04741a5a519f9ee159b0f7fc5b6ab1 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 09:54:45 +0100 Subject: [PATCH 245/332] add fdb_time to xarray to make tests work --- polytope/datacube/backends/xarray.py | 1 + 1 file changed, 1 insertion(+) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index f8ca1c2e2..9ce40135b 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -17,6 +17,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.complete_axes = [] self.blocked_axes = [] self.fake_axes = [] + self.fdb_time = 0 for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: From 5e12c055b4eb0e130495a89034dfa746bd4502d8 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 14:02:46 +0100 Subject: [PATCH 246/332] small optimisations --- polytope/datacube/backends/datacube.py | 2 ++ polytope/datacube/backends/fdb.py | 1 + polytope/datacube/backends/mock.py | 1 + polytope/datacube/backends/xarray.py | 1 + polytope/engine/hullslicer.py | 48 +++++++++++++++++++++----- 5 files changed, 45 insertions(+), 8 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index efa20e93d..62620eb0b 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -37,6 +37,8 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt ) for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) + if len(final_axis_names) > 1: + self.coupled_axes.append(final_axis_names) for axis_name in final_axis_names: self.fake_axes.append(axis_name) # if axis does not yet exist, create it diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index fad3ca4a7..2182a708b 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -18,6 +18,7 @@ def __init__(self, config={}, axis_options={}): self.fake_axes = [] self.unwanted_path = {} self.fdb_time = 0 + self.coupled_axes = [] partial_request = config # Find values in the level 3 FDB datacube diff --git a/polytope/datacube/backends/mock.py b/polytope/datacube/backends/mock.py index b6bc32d3e..6281c9cf3 100644 --- a/polytope/datacube/backends/mock.py +++ b/polytope/datacube/backends/mock.py @@ -22,6 +22,7 @@ def __init__(self, dimensions): for k, v in reversed(dimensions.items()): self.stride[k] = stride_cumulative stride_cumulative *= self.dimensions[k] + self.coupled_axes = [] def get(self, requests: IndexTree): # Takes in a datacube and verifies the leaves of the tree are complete diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 9ce40135b..26b71dbb9 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -18,6 +18,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.blocked_axes = [] self.fake_axes = [] self.fdb_time = 0 + self.coupled_axes = [] for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 51a622535..a8b1dce87 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -18,21 +18,25 @@ class HullSlicer(Engine): def __init__(self): - pass + self.ax_is_unsliceable = {} + self.axis_values_between = {} def _unique_continuous_points(self, p: ConvexPolytope, datacube: Datacube): for i, ax in enumerate(p._axes): mapper = datacube.get_mapper(ax) - if isinstance(mapper, UnsliceableDatacubeAxis): + if self.ax_is_unsliceable.get(ax, None) is None: + self.ax_is_unsliceable[ax] = isinstance(mapper, UnsliceableDatacubeAxis) + # if isinstance(mapper, UnsliceableDatacubeAxis): + if self.ax_is_unsliceable[ax]: break for j, val in enumerate(p.points): - p.points[j] = list(p.points[j]) + # p.points[j] = list(p.points[j]) p.points[j][i] = mapper.to_float(mapper.parse(p.points[j][i])) # Remove duplicate points unique(p.points) def _build_unsliceable_child(self, polytope, ax, node, datacube, lower, next_nodes, slice_axis_idx): - if polytope._axes != [ax.name]: + if not polytope.is_flat: raise UnsliceableShapeError(ax) path = node.flatten() if datacube.has_index(path, ax, lower): @@ -50,6 +54,25 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex upper = ax.from_float(upper + tol) flattened = node.flatten() method = polytope.method + + # TODO: this hashing doesn't work because we need to know the latitude val for finding longitude values + # TODO: Maybe create a coupled_axes list inside of datacube and add to it during axis formation, then here + # do something like if ax is in second place of coupled_axes, then take the flattened part of the array that corresponds + # to the first place of cooupled_axes in the hashing + # Else, if we do not need the flattened bit in the hash, can just put an empty string instead? + + # if len(datacube.coupled_axes) > 0: + # if flattened.get(datacube.coupled_axes[0][0], None) is not None: + # flattened = {datacube.coupled_axes[0][0] : flattened.get(datacube.coupled_axes[0][0], None)} + # else: + # flattened = {} + + # print(datacube.coupled_axes) + # print(flattened.get("latitude")) + # flattened = interm_flattened + # if self.axis_values_between.get((ax.name, lower, upper, method), None) is None: + # self.axis_values_between[(ax.name, lower, upper, method)] = datacube.get_indices(flattened, ax, lower, upper, method) + # values = self.axis_values_between[(ax.name, lower, upper, method)] values = datacube.get_indices(flattened, ax, lower, upper, method) if len(values) == 0: @@ -77,7 +100,12 @@ def _build_branch(self, ax, node, datacube, next_nodes): if ax.name in polytope._axes: lower, upper, slice_axis_idx = polytope.extents(ax.name) # here, first check if the axis is an unsliceable axis and directly build node if it is - if isinstance(ax, UnsliceableDatacubeAxis): + # if isinstance(ax, UnsliceableDatacubeAxis): + + # NOTE: we should have already created the ax_is_unsliceable cache before + # if self.ax_is_unsliceable.get(ax.name, None) is None: + # self.ax_is_unsliceable[ax.name] = isinstance(ax, UnsliceableDatacubeAxis) + if self.ax_is_unsliceable[ax.name]: self._build_unsliceable_child(polytope, ax, node, datacube, lower, next_nodes, slice_axis_idx) else: self._build_sliceable_child(polytope, ax, node, datacube, lower, upper, next_nodes, slice_axis_idx) @@ -93,6 +121,10 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): request = IndexTree() combinations = tensor_product(groups) + # NOTE: could optimise here if we know combinations will always be for one request. + # Then we do not need to create a new index tree and merge it to request, but can just + # directly work on request and return it... + time0 = time.time() for c in combinations: r = IndexTree() @@ -139,8 +171,7 @@ def _reduce_dimension(intersects, slice_axis_idx): def slice(polytope: ConvexPolytope, axis, value, slice_axis_idx): - if len(polytope.points[0]) == 1: - # Note that in this case, we do not need to do linear interpolation so we can save time + if polytope.is_flat: if value in chain(*polytope.points): intersects = [[value]] else: @@ -154,7 +185,8 @@ def slice(polytope: ConvexPolytope, axis, value, slice_axis_idx): # Reduce dimension of intersection points, removing slice axis intersects = _reduce_dimension(intersects, slice_axis_idx) - axes = [ax for ax in polytope._axes if ax != axis] + axes = copy(polytope._axes) + axes.remove(axis) if len(intersects) < len(intersects[0]) + 1: return ConvexPolytope(axes, intersects) From 72773fe77f9bbb4b0f90d784259c4b2ccd3ee26c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 14:11:24 +0100 Subject: [PATCH 247/332] clean up branch --- .../fdb_slice_many_numbers_timeseries.py | 2 -- polytope/datacube/backends/fdb.py | 6 ----- polytope/datacube/backends/xarray.py | 2 -- polytope/engine/hullslicer.py | 26 ++----------------- polytope/polytope.py | 8 ------ tests/test_datacube_axes_init.py | 1 - tests/test_fdb_datacube.py | 3 +-- tests/test_incomplete_tree_fdb.py | 4 +-- tests/test_regular_grid.py | 2 +- tests/test_shapes.py | 2 +- tests/test_slice_date_range_fdb.py | 2 +- 11 files changed, 8 insertions(+), 50 deletions(-) diff --git a/performance/fdb_slice_many_numbers_timeseries.py b/performance/fdb_slice_many_numbers_timeseries.py index 0b592de78..13dc00f79 100644 --- a/performance/fdb_slice_many_numbers_timeseries.py +++ b/performance/fdb_slice_many_numbers_timeseries.py @@ -3,7 +3,6 @@ import pandas as pd from polytope.datacube.backends.fdb import FDBDatacube -from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import All, Point, Select @@ -41,5 +40,4 @@ result = self_API.retrieve(request) print(time.time() - time1) print(time.time() - time2) -print(time.time()-time2 - fdbdatacube.fdb_time) print(len(result.leaves)) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 2182a708b..bab3fc2a6 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -4,8 +4,6 @@ from .datacube import Datacube, IndexTree -import time - class FDBDatacube(Datacube): def __init__(self, config={}, axis_options={}): @@ -17,8 +15,6 @@ def __init__(self, config={}, axis_options={}): self.blocked_axes = [] self.fake_axes = [] self.unwanted_path = {} - self.fdb_time = 0 - self.coupled_axes = [] partial_request = config # Find values in the level 3 FDB datacube @@ -150,9 +146,7 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) - time1 = time.time() output_values = self.fdb.extract(fdb_requests) - self.fdb_time += time.time() - time1 return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 26b71dbb9..f8ca1c2e2 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -17,8 +17,6 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.complete_axes = [] self.blocked_axes = [] self.fake_axes = [] - self.fdb_time = 0 - self.coupled_axes = [] for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index a8b1dce87..e6c8e3eb0 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -13,8 +13,6 @@ from ..utility.geometry import lerp from .engine import Engine -import time - class HullSlicer(Engine): def __init__(self): @@ -26,11 +24,9 @@ def _unique_continuous_points(self, p: ConvexPolytope, datacube: Datacube): mapper = datacube.get_mapper(ax) if self.ax_is_unsliceable.get(ax, None) is None: self.ax_is_unsliceable[ax] = isinstance(mapper, UnsliceableDatacubeAxis) - # if isinstance(mapper, UnsliceableDatacubeAxis): if self.ax_is_unsliceable[ax]: break for j, val in enumerate(p.points): - # p.points[j] = list(p.points[j]) p.points[j][i] = mapper.to_float(mapper.parse(p.points[j][i])) # Remove duplicate points unique(p.points) @@ -57,22 +53,10 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex # TODO: this hashing doesn't work because we need to know the latitude val for finding longitude values # TODO: Maybe create a coupled_axes list inside of datacube and add to it during axis formation, then here - # do something like if ax is in second place of coupled_axes, then take the flattened part of the array that corresponds - # to the first place of cooupled_axes in the hashing + # do something like if ax is in second place of coupled_axes, then take the flattened part of the array that + # corresponds to the first place of cooupled_axes in the hashing # Else, if we do not need the flattened bit in the hash, can just put an empty string instead? - # if len(datacube.coupled_axes) > 0: - # if flattened.get(datacube.coupled_axes[0][0], None) is not None: - # flattened = {datacube.coupled_axes[0][0] : flattened.get(datacube.coupled_axes[0][0], None)} - # else: - # flattened = {} - - # print(datacube.coupled_axes) - # print(flattened.get("latitude")) - # flattened = interm_flattened - # if self.axis_values_between.get((ax.name, lower, upper, method), None) is None: - # self.axis_values_between[(ax.name, lower, upper, method)] = datacube.get_indices(flattened, ax, lower, upper, method) - # values = self.axis_values_between[(ax.name, lower, upper, method)] values = datacube.get_indices(flattened, ax, lower, upper, method) if len(values) == 0: @@ -100,11 +84,8 @@ def _build_branch(self, ax, node, datacube, next_nodes): if ax.name in polytope._axes: lower, upper, slice_axis_idx = polytope.extents(ax.name) # here, first check if the axis is an unsliceable axis and directly build node if it is - # if isinstance(ax, UnsliceableDatacubeAxis): # NOTE: we should have already created the ax_is_unsliceable cache before - # if self.ax_is_unsliceable.get(ax.name, None) is None: - # self.ax_is_unsliceable[ax.name] = isinstance(ax, UnsliceableDatacubeAxis) if self.ax_is_unsliceable[ax.name]: self._build_unsliceable_child(polytope, ax, node, datacube, lower, next_nodes, slice_axis_idx) else: @@ -125,7 +106,6 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): # Then we do not need to create a new index tree and merge it to request, but can just # directly work on request and return it... - time0 = time.time() for c in combinations: r = IndexTree() r["unsliced_polytopes"] = set(c) @@ -136,8 +116,6 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): self._build_branch(ax, node, datacube, next_nodes) current_nodes = next_nodes request.merge(r) - print("time spent inside extract building tree") - print(time.time() - time0) return request diff --git a/polytope/polytope.py b/polytope/polytope.py index 5e4fd0914..f6d4a723e 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -3,8 +3,6 @@ from .shapes import ConvexPolytope from .utility.exceptions import AxisOverdefinedError -import time - class Request: """Encapsulates a request for data""" @@ -52,12 +50,6 @@ def slice(self, polytopes: List[ConvexPolytope]): def retrieve(self, request: Request, method="standard"): """Higher-level API which takes a request and uses it to slice the datacube""" - time1 = time.time() request_tree = self.engine.extract(self.datacube, request.polytopes()) - print("extract time") - print(time.time() - time1) - time0 = time.time() self.datacube.get(request_tree) - print("get time") - print(time.time() - time0 - self.datacube.fdb_time) return request_tree diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index f2a7f3eb4..f3739e5bf 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -20,7 +20,6 @@ def setup_method(self, method): self.xarraydatacube = XArrayDatacube(latlon_array) self.options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - # "latitude": {"transformation": {"reverse": {True}}}, } self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 03bc1084c..d846b91fb 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -21,12 +21,11 @@ def setup_method(self, method): } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) - print(self.fdbdatacube.fdb_coordinates) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 6f5f11ecf..40fd5933e 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -49,7 +49,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch(self): request = Request( Select("step", [0]), @@ -72,7 +72,7 @@ def test_incomplete_fdb_branch(self): assert result.is_root() @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_incomplete_fdb_branch_2(self): request = Request( Select("step", [0]), diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 5e5fa3878..506cd0cf2 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -52,7 +52,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_regular_grid(self): request = Request( Select("step", [0]), diff --git a/tests/test_shapes.py b/tests/test_shapes.py index 3d9d8b202..29c6b35f6 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -39,7 +39,7 @@ def test_all_cyclic(self): # result.pprint() assert len(result.leaves) == 360 - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_all_mapper_cyclic(self): self.options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index fb4255db8..a233e2c26 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -22,7 +22,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_fdb_datacube(self): request = Request( Select("step", [0]), From a634e1ce4b814149572ea977f3996aa4c645da22 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 14:16:54 +0100 Subject: [PATCH 248/332] clean up --- polytope/datacube/backends/datacube.py | 2 -- polytope/datacube/backends/mock.py | 1 - 2 files changed, 3 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 62620eb0b..efa20e93d 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -37,8 +37,6 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt ) for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) - if len(final_axis_names) > 1: - self.coupled_axes.append(final_axis_names) for axis_name in final_axis_names: self.fake_axes.append(axis_name) # if axis does not yet exist, create it diff --git a/polytope/datacube/backends/mock.py b/polytope/datacube/backends/mock.py index 6281c9cf3..b6bc32d3e 100644 --- a/polytope/datacube/backends/mock.py +++ b/polytope/datacube/backends/mock.py @@ -22,7 +22,6 @@ def __init__(self, dimensions): for k, v in reversed(dimensions.items()): self.stride[k] = stride_cumulative stride_cumulative *= self.dimensions[k] - self.coupled_axes = [] def get(self, requests: IndexTree): # Takes in a datacube and verifies the leaves of the tree are complete From 3c46c8a9b298da55a7d50a11c640ee8862b4cdde Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 15:16:40 +0100 Subject: [PATCH 249/332] special mark for fdb tests --- .github/workflows/ci.yaml | 2 +- pyproject.toml | 3 ++- tests/test_fdb_datacube.py | 2 +- tests/test_incomplete_tree_fdb.py | 4 ++-- tests/test_regular_grid.py | 2 +- tests/test_slice_date_range_fdb.py | 2 +- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 017d4bf0e..56b8e7ee0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -131,7 +131,7 @@ jobs: LD_LIBRARY_PATH: ${{ steps.install-dependencies.outputs.lib_path }} shell: bash -eux {0} run: | - DYLD_LIBRARY_PATH=${{ env.LD_LIBRARY_PATH }} python -m pytest tests --cov=./ --cov-report=xml + DYLD_LIBRARY_PATH=${{ env.LD_LIBRARY_PATH }} python -m pytest -m "not fdb" tests --cov=./ --cov-report=xml python -m coverage report - name: Upload coverage to Codecov diff --git a/pyproject.toml b/pyproject.toml index 28f4ec600..d9a2e1f4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,4 +4,5 @@ line-length = 120 requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -markers = ["internet: downloads test data from the internet (deselect with '-m \"not internet\"')",] \ No newline at end of file +markers = ["internet: downloads test data from the internet (deselect with '-m \"not internet\"')", + "fdb: test which uses fdb (deselect with '-m \"not fdb\"')",] \ No newline at end of file diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index d846b91fb..a018da9b9 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -25,7 +25,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.fdb def test_fdb_datacube(self): request = Request( Select("step", [0]), diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 40fd5933e..a3d4ace62 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -49,7 +49,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.fdb def test_incomplete_fdb_branch(self): request = Request( Select("step", [0]), @@ -72,7 +72,7 @@ def test_incomplete_fdb_branch(self): assert result.is_root() @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.fdb def test_incomplete_fdb_branch_2(self): request = Request( Select("step", [0]), diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 506cd0cf2..536b386ca 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -52,7 +52,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.fdb def test_regular_grid(self): request = Request( Select("step", [0]), diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index a233e2c26..11dba9646 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -22,7 +22,7 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) # Testing different shapes - @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.fdb def test_fdb_datacube(self): request = Request( Select("step", [0]), From 48b3aea700bb324d586f28419f410b237b34d756 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 8 Dec 2023 15:21:54 +0100 Subject: [PATCH 250/332] put fdb marker on fdb test --- tests/test_shapes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_shapes.py b/tests/test_shapes.py index 29c6b35f6..402de26a4 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -39,7 +39,7 @@ def test_all_cyclic(self): # result.pprint() assert len(result.leaves) == 360 - @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.fdb def test_all_mapper_cyclic(self): self.options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, From 58e05b4a4656bb9f15a7573d33848c00fab15ea6 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 3 Jan 2024 17:07:36 +0100 Subject: [PATCH 251/332] add reduced_ll_grid with 1441 latitude values --- .../transformations/datacube_mappers.py | 1502 +++++++++++++++++ 1 file changed, 1502 insertions(+) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index f759a00bf..b2544c4c5 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -129,6 +129,1507 @@ def unmap(self, first_val, second_val): return final_index +class ReducedLatLonMapper(DatacubeMapper): + def __init__(self, base_axis, mapped_axes, resolution): + self._mapped_axes = mapped_axes + self._base_axis = base_axis + self._resolution = resolution + self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} + self._first_axis_vals = self.first_axis_vals() + + def first_axis_vals(self): + resolution = 180 / (self._resolution - 1) + vals = [-90 + i * resolution for i in range(self._resolution)] + return vals + + def map_first_axis(self, lower, upper): + axis_lines = self._first_axis_vals + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def lon_spacing(self): + if self._resolution == 1441: + return [ + 2, + 6, + 14, + 20, + 26, + 32, + 38, + 44, + 50, + 58, + 64, + 70, + 76, + 82, + 88, + 94, + 102, + 108, + 114, + 120, + 126, + 132, + 138, + 144, + 152, + 158, + 164, + 170, + 176, + 182, + 188, + 196, + 202, + 208, + 214, + 220, + 226, + 232, + 238, + 246, + 252, + 258, + 264, + 270, + 276, + 282, + 290, + 296, + 302, + 308, + 314, + 320, + 326, + 332, + 340, + 346, + 352, + 358, + 364, + 370, + 376, + 382, + 388, + 396, + 402, + 408, + 414, + 420, + 426, + 432, + 438, + 444, + 452, + 458, + 464, + 470, + 476, + 482, + 488, + 494, + 500, + 506, + 512, + 520, + 526, + 532, + 538, + 544, + 550, + 556, + 562, + 568, + 574, + 580, + 586, + 594, + 600, + 606, + 612, + 618, + 624, + 630, + 636, + 642, + 648, + 654, + 660, + 666, + 672, + 678, + 686, + 692, + 698, + 704, + 710, + 716, + 722, + 728, + 734, + 740, + 746, + 752, + 758, + 764, + 770, + 776, + 782, + 788, + 794, + 800, + 806, + 812, + 818, + 824, + 830, + 836, + 842, + 848, + 854, + 860, + 866, + 872, + 878, + 884, + 890, + 896, + 902, + 908, + 914, + 920, + 926, + 932, + 938, + 944, + 950, + 956, + 962, + 968, + 974, + 980, + 986, + 992, + 998, + 1004, + 1010, + 1014, + 1020, + 1026, + 1032, + 1038, + 1044, + 1050, + 1056, + 1062, + 1068, + 1074, + 1080, + 1086, + 1092, + 1096, + 1102, + 1108, + 1114, + 1120, + 1126, + 1132, + 1138, + 1144, + 1148, + 1154, + 1160, + 1166, + 1172, + 1178, + 1184, + 1190, + 1194, + 1200, + 1206, + 1212, + 1218, + 1224, + 1230, + 1234, + 1240, + 1246, + 1252, + 1258, + 1264, + 1268, + 1274, + 1280, + 1286, + 1292, + 1296, + 1302, + 1308, + 1314, + 1320, + 1324, + 1330, + 1336, + 1342, + 1348, + 1352, + 1358, + 1364, + 1370, + 1374, + 1380, + 1386, + 1392, + 1396, + 1402, + 1408, + 1414, + 1418, + 1424, + 1430, + 1436, + 1440, + 1446, + 1452, + 1456, + 1462, + 1468, + 1474, + 1478, + 1484, + 1490, + 1494, + 1500, + 1506, + 1510, + 1516, + 1522, + 1526, + 1532, + 1538, + 1542, + 1548, + 1554, + 1558, + 1564, + 1570, + 1574, + 1580, + 1584, + 1590, + 1596, + 1600, + 1606, + 1610, + 1616, + 1622, + 1626, + 1632, + 1636, + 1642, + 1648, + 1652, + 1658, + 1662, + 1668, + 1672, + 1678, + 1684, + 1688, + 1694, + 1698, + 1704, + 1708, + 1714, + 1718, + 1724, + 1728, + 1734, + 1738, + 1744, + 1748, + 1754, + 1758, + 1764, + 1768, + 1774, + 1778, + 1784, + 1788, + 1794, + 1798, + 1804, + 1808, + 1812, + 1818, + 1822, + 1828, + 1832, + 1838, + 1842, + 1846, + 1852, + 1856, + 1862, + 1866, + 1870, + 1876, + 1880, + 1886, + 1890, + 1894, + 1900, + 1904, + 1908, + 1914, + 1918, + 1922, + 1928, + 1932, + 1936, + 1942, + 1946, + 1950, + 1956, + 1960, + 1964, + 1970, + 1974, + 1978, + 1982, + 1988, + 1992, + 1996, + 2002, + 2006, + 2010, + 2014, + 2020, + 2024, + 2028, + 2032, + 2036, + 2042, + 2046, + 2050, + 2054, + 2060, + 2064, + 2068, + 2072, + 2076, + 2080, + 2086, + 2090, + 2094, + 2098, + 2102, + 2106, + 2112, + 2116, + 2120, + 2124, + 2128, + 2132, + 2136, + 2140, + 2144, + 2150, + 2154, + 2158, + 2162, + 2166, + 2170, + 2174, + 2178, + 2182, + 2186, + 2190, + 2194, + 2198, + 2202, + 2206, + 2210, + 2214, + 2218, + 2222, + 2226, + 2230, + 2234, + 2238, + 2242, + 2246, + 2250, + 2254, + 2258, + 2262, + 2266, + 2270, + 2274, + 2278, + 2282, + 2286, + 2290, + 2292, + 2296, + 2300, + 2304, + 2308, + 2312, + 2316, + 2320, + 2324, + 2326, + 2330, + 2334, + 2338, + 2342, + 2346, + 2348, + 2352, + 2356, + 2360, + 2364, + 2366, + 2370, + 2374, + 2378, + 2382, + 2384, + 2388, + 2392, + 2396, + 2398, + 2402, + 2406, + 2410, + 2412, + 2416, + 2420, + 2422, + 2426, + 2430, + 2432, + 2436, + 2440, + 2442, + 2446, + 2450, + 2452, + 2456, + 2460, + 2462, + 2466, + 2470, + 2472, + 2476, + 2478, + 2482, + 2486, + 2488, + 2492, + 2494, + 2498, + 2500, + 2504, + 2508, + 2510, + 2514, + 2516, + 2520, + 2522, + 2526, + 2528, + 2532, + 2534, + 2538, + 2540, + 2544, + 2546, + 2550, + 2552, + 2556, + 2558, + 2560, + 2564, + 2566, + 2570, + 2572, + 2576, + 2578, + 2580, + 2584, + 2586, + 2590, + 2592, + 2594, + 2598, + 2600, + 2602, + 2606, + 2608, + 2610, + 2614, + 2616, + 2618, + 2622, + 2624, + 2626, + 2628, + 2632, + 2634, + 2636, + 2640, + 2642, + 2644, + 2646, + 2650, + 2652, + 2654, + 2656, + 2658, + 2662, + 2664, + 2666, + 2668, + 2670, + 2674, + 2676, + 2678, + 2680, + 2682, + 2684, + 2686, + 2690, + 2692, + 2694, + 2696, + 2698, + 2700, + 2702, + 2704, + 2706, + 2708, + 2712, + 2714, + 2716, + 2718, + 2720, + 2722, + 2724, + 2726, + 2728, + 2730, + 2732, + 2734, + 2736, + 2738, + 2740, + 2742, + 2744, + 2746, + 2748, + 2750, + 2750, + 2752, + 2754, + 2756, + 2758, + 2760, + 2762, + 2764, + 2766, + 2768, + 2768, + 2770, + 2772, + 2774, + 2776, + 2778, + 2780, + 2780, + 2782, + 2784, + 2786, + 2788, + 2788, + 2790, + 2792, + 2794, + 2794, + 2796, + 2798, + 2800, + 2800, + 2802, + 2804, + 2806, + 2806, + 2808, + 2810, + 2810, + 2812, + 2814, + 2814, + 2816, + 2818, + 2818, + 2820, + 2822, + 2822, + 2824, + 2826, + 2826, + 2828, + 2828, + 2830, + 2832, + 2832, + 2834, + 2834, + 2836, + 2836, + 2838, + 2838, + 2840, + 2842, + 2842, + 2844, + 2844, + 2846, + 2846, + 2846, + 2848, + 2848, + 2850, + 2850, + 2852, + 2852, + 2854, + 2854, + 2856, + 2856, + 2856, + 2858, + 2858, + 2860, + 2860, + 2860, + 2862, + 2862, + 2862, + 2864, + 2864, + 2864, + 2866, + 2866, + 2866, + 2868, + 2868, + 2868, + 2868, + 2870, + 2870, + 2870, + 2872, + 2872, + 2872, + 2872, + 2874, + 2874, + 2874, + 2874, + 2874, + 2876, + 2876, + 2876, + 2876, + 2876, + 2876, + 2878, + 2878, + 2878, + 2878, + 2878, + 2878, + 2878, + 2878, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2878, + 2878, + 2878, + 2878, + 2878, + 2878, + 2878, + 2878, + 2876, + 2876, + 2876, + 2876, + 2876, + 2876, + 2874, + 2874, + 2874, + 2874, + 2874, + 2872, + 2872, + 2872, + 2872, + 2870, + 2870, + 2870, + 2868, + 2868, + 2868, + 2868, + 2866, + 2866, + 2866, + 2864, + 2864, + 2864, + 2862, + 2862, + 2862, + 2860, + 2860, + 2860, + 2858, + 2858, + 2856, + 2856, + 2856, + 2854, + 2854, + 2852, + 2852, + 2850, + 2850, + 2848, + 2848, + 2846, + 2846, + 2846, + 2844, + 2844, + 2842, + 2842, + 2840, + 2838, + 2838, + 2836, + 2836, + 2834, + 2834, + 2832, + 2832, + 2830, + 2828, + 2828, + 2826, + 2826, + 2824, + 2822, + 2822, + 2820, + 2818, + 2818, + 2816, + 2814, + 2814, + 2812, + 2810, + 2810, + 2808, + 2806, + 2806, + 2804, + 2802, + 2800, + 2800, + 2798, + 2796, + 2794, + 2794, + 2792, + 2790, + 2788, + 2788, + 2786, + 2784, + 2782, + 2780, + 2780, + 2778, + 2776, + 2774, + 2772, + 2770, + 2768, + 2768, + 2766, + 2764, + 2762, + 2760, + 2758, + 2756, + 2754, + 2752, + 2750, + 2750, + 2748, + 2746, + 2744, + 2742, + 2740, + 2738, + 2736, + 2734, + 2732, + 2730, + 2728, + 2726, + 2724, + 2722, + 2720, + 2718, + 2716, + 2714, + 2712, + 2708, + 2706, + 2704, + 2702, + 2700, + 2698, + 2696, + 2694, + 2692, + 2690, + 2686, + 2684, + 2682, + 2680, + 2678, + 2676, + 2674, + 2670, + 2668, + 2666, + 2664, + 2662, + 2658, + 2656, + 2654, + 2652, + 2650, + 2646, + 2644, + 2642, + 2640, + 2636, + 2634, + 2632, + 2628, + 2626, + 2624, + 2622, + 2618, + 2616, + 2614, + 2610, + 2608, + 2606, + 2602, + 2600, + 2598, + 2594, + 2592, + 2590, + 2586, + 2584, + 2580, + 2578, + 2576, + 2572, + 2570, + 2566, + 2564, + 2560, + 2558, + 2556, + 2552, + 2550, + 2546, + 2544, + 2540, + 2538, + 2534, + 2532, + 2528, + 2526, + 2522, + 2520, + 2516, + 2514, + 2510, + 2508, + 2504, + 2500, + 2498, + 2494, + 2492, + 2488, + 2486, + 2482, + 2478, + 2476, + 2472, + 2470, + 2466, + 2462, + 2460, + 2456, + 2452, + 2450, + 2446, + 2442, + 2440, + 2436, + 2432, + 2430, + 2426, + 2422, + 2420, + 2416, + 2412, + 2410, + 2406, + 2402, + 2398, + 2396, + 2392, + 2388, + 2384, + 2382, + 2378, + 2374, + 2370, + 2366, + 2364, + 2360, + 2356, + 2352, + 2348, + 2346, + 2342, + 2338, + 2334, + 2330, + 2326, + 2324, + 2320, + 2316, + 2312, + 2308, + 2304, + 2300, + 2296, + 2292, + 2290, + 2286, + 2282, + 2278, + 2274, + 2270, + 2266, + 2262, + 2258, + 2254, + 2250, + 2246, + 2242, + 2238, + 2234, + 2230, + 2226, + 2222, + 2218, + 2214, + 2210, + 2206, + 2202, + 2198, + 2194, + 2190, + 2186, + 2182, + 2178, + 2174, + 2170, + 2166, + 2162, + 2158, + 2154, + 2150, + 2144, + 2140, + 2136, + 2132, + 2128, + 2124, + 2120, + 2116, + 2112, + 2106, + 2102, + 2098, + 2094, + 2090, + 2086, + 2080, + 2076, + 2072, + 2068, + 2064, + 2060, + 2054, + 2050, + 2046, + 2042, + 2036, + 2032, + 2028, + 2024, + 2020, + 2014, + 2010, + 2006, + 2002, + 1996, + 1992, + 1988, + 1982, + 1978, + 1974, + 1970, + 1964, + 1960, + 1956, + 1950, + 1946, + 1942, + 1936, + 1932, + 1928, + 1922, + 1918, + 1914, + 1908, + 1904, + 1900, + 1894, + 1890, + 1886, + 1880, + 1876, + 1870, + 1866, + 1862, + 1856, + 1852, + 1846, + 1842, + 1838, + 1832, + 1828, + 1822, + 1818, + 1812, + 1808, + 1804, + 1798, + 1794, + 1788, + 1784, + 1778, + 1774, + 1768, + 1764, + 1758, + 1754, + 1748, + 1744, + 1738, + 1734, + 1728, + 1724, + 1718, + 1714, + 1708, + 1704, + 1698, + 1694, + 1688, + 1684, + 1678, + 1672, + 1668, + 1662, + 1658, + 1652, + 1648, + 1642, + 1636, + 1632, + 1626, + 1622, + 1616, + 1610, + 1606, + 1600, + 1596, + 1590, + 1584, + 1580, + 1574, + 1570, + 1564, + 1558, + 1554, + 1548, + 1542, + 1538, + 1532, + 1526, + 1522, + 1516, + 1510, + 1506, + 1500, + 1494, + 1490, + 1484, + 1478, + 1474, + 1468, + 1462, + 1456, + 1452, + 1446, + 1440, + 1436, + 1430, + 1424, + 1418, + 1414, + 1408, + 1402, + 1396, + 1392, + 1386, + 1380, + 1374, + 1370, + 1364, + 1358, + 1352, + 1348, + 1342, + 1336, + 1330, + 1324, + 1320, + 1314, + 1308, + 1302, + 1296, + 1292, + 1286, + 1280, + 1274, + 1268, + 1264, + 1258, + 1252, + 1246, + 1240, + 1234, + 1230, + 1224, + 1218, + 1212, + 1206, + 1200, + 1194, + 1190, + 1184, + 1178, + 1172, + 1166, + 1160, + 1154, + 1148, + 1144, + 1138, + 1132, + 1126, + 1120, + 1114, + 1108, + 1102, + 1096, + 1092, + 1086, + 1080, + 1074, + 1068, + 1062, + 1056, + 1050, + 1044, + 1038, + 1032, + 1026, + 1020, + 1014, + 1010, + 1004, + 998, + 992, + 986, + 980, + 974, + 968, + 962, + 956, + 950, + 944, + 938, + 932, + 926, + 920, + 914, + 908, + 902, + 896, + 890, + 884, + 878, + 872, + 866, + 860, + 854, + 848, + 842, + 836, + 830, + 824, + 818, + 812, + 806, + 800, + 794, + 788, + 782, + 776, + 770, + 764, + 758, + 752, + 746, + 740, + 734, + 728, + 722, + 716, + 710, + 704, + 698, + 692, + 686, + 678, + 672, + 666, + 660, + 654, + 648, + 642, + 636, + 630, + 624, + 618, + 612, + 606, + 600, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ] + + def second_axis_vals(self, first_val): + first_idx = self._first_axis_vals.index(first_val) + Ny = self.lon_spacing()[first_idx] + second_spacing = 360 / Ny + return [i * second_spacing for i in range(Ny)] + + def map_second_axis(self, first_val, lower, upper): + axis_lines = self.second_axis_vals(first_val) + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def axes_idx_to_reduced_ll_idx(self, first_idx, second_idx): + Ny_array = self.lon_spacing() + idx = 0 + for i in range(self._resolution): + if i != first_idx: + idx += Ny_array[i] + else: + idx += second_idx + return idx + + def find_second_idx(self, first_val, second_val): + tol = 1e-10 + second_axis_vals = self.second_axis_vals(first_val) + second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) + return second_idx + + def unmap(self, first_val, second_val): + tol = 1e-8 + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + first_idx = self._first_axis_vals.index(first_val) + second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] + second_idx = self.second_axis_vals(first_val).index(second_val) + reduced_ll_index = self.axes_idx_to_reduced_ll_idx(first_idx, second_idx) + return reduced_ll_index + + class HealpixGridMapper(DatacubeMapper): def __init__(self, base_axis, mapped_axes, resolution): self._mapped_axes = mapped_axes @@ -3000,4 +4501,5 @@ def unmap(self, first_val, second_val): "octahedral": "OctahedralGridMapper", "healpix": "HealpixGridMapper", "regular": "RegularGridMapper", + "reduced_ll": "ReducedLatLonMapper", } From 5904a4c6eb3375fac05b9b4a1807d46e2162150d Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 3 Jan 2024 17:46:30 +0100 Subject: [PATCH 252/332] test wave grid reduced_ll --- tests/test_reduced_ll_grid.py | 92 +++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tests/test_reduced_ll_grid.py diff --git a/tests/test_reduced_ll_grid.py b/tests/test_reduced_ll_grid.py new file mode 100644 index 000000000..f8dd5c05b --- /dev/null +++ b/tests/test_reduced_ll_grid.py @@ -0,0 +1,92 @@ +import pandas as pd +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file +from helper_functions import download_test_data + +from polytope.datacube.backends.fdb import FDBDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestReducedLatLonGrid: + def setup_method(self, method): + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/wave.grib" + download_test_data(nexus_url, "wave.grib") + self.options = { + "values": { + "transformation": { + "mapper": {"type": "reduced_ll", "resolution": 1441, "axes": ["latitude", "longitude"]} + } + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + "number": {"transformation": {"type_change": "int"}}, + "longitude": {"transformation": {"cyclic": [0, 360]}}, + } + self.config = {"class": "od", "stream": "wave"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + def find_nearest_latlon(self, grib_file, target_lat, target_lon): + # Open the GRIB file + f = open(grib_file) + + # Load the GRIB messages from the file + messages = [] + while True: + message = codes_grib_new_from_file(f) + if message is None: + break + messages.append(message) + + # Find the nearest grid points + nearest_points = [] + for message in messages: + nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) + nearest_points.append(nearest_index) + + # Close the GRIB file + f.close() + + return nearest_points + + def test_reduced_ll_grid(self): + request = Request( + Select("step", [1]), + Select("date", [pd.Timestamp("20231129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["140251"]), + Select("direction", ["1"]), + Select("frequency", ["1"]), + Select("class", ["od"]), + Select("stream", ["wave"]), + Select("levtype", ["sfc"]), + Select("type", ["fc"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.5]), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 10 + + lats = [] + lons = [] + eccodes_lats = [] + tol = 1e-8 + for i in range(len(result.leaves)): + cubepath = result.leaves[i].flatten() + lat = cubepath["latitude"] + lon = cubepath["longitude"] + lats.append(lat) + lons.append(lon) + nearest_points = self.find_nearest_latlon("./tests/data/wave.grib", lat, lon) + eccodes_lat = nearest_points[0][0]["lat"] + eccodes_lon = nearest_points[0][0]["lon"] + eccodes_lats.append(eccodes_lat) + assert eccodes_lat - tol <= lat + assert lat <= eccodes_lat + tol + assert eccodes_lon - tol <= lon + assert lon <= eccodes_lon + tol + + assert len(eccodes_lats) == 10 From fdc67ea94322e9ba8bc2d7b7ac67e36379abb64d Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 3 Jan 2024 17:47:03 +0100 Subject: [PATCH 253/332] fix small bug --- polytope/datacube/transformations/datacube_mappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index b2544c4c5..0a7a3f5e9 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -134,7 +134,7 @@ def __init__(self, base_axis, mapped_axes, resolution): self._mapped_axes = mapped_axes self._base_axis = base_axis self._resolution = resolution - self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} + self._axis_reversed = {mapped_axes[0]: False, mapped_axes[1]: False} self._first_axis_vals = self.first_axis_vals() def first_axis_vals(self): From 4c8d30f7c1057f1637386f3211967827f3a53a46 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 3 Jan 2024 17:48:24 +0100 Subject: [PATCH 254/332] skip fdb test --- tests/test_reduced_ll_grid.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_reduced_ll_grid.py b/tests/test_reduced_ll_grid.py index f8dd5c05b..f6bbffb1a 100644 --- a/tests/test_reduced_ll_grid.py +++ b/tests/test_reduced_ll_grid.py @@ -1,4 +1,5 @@ import pandas as pd +import pytest from eccodes import codes_grib_find_nearest, codes_grib_new_from_file from helper_functions import download_test_data @@ -51,6 +52,8 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points + @pytest.mark.internet + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_reduced_ll_grid(self): request = Request( Select("step", [1]), From 1efa99e0f2e71d3e39280cff46db7504c007885c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 4 Jan 2024 10:21:53 +0100 Subject: [PATCH 255/332] make plot in test reduced ll grid --- tests/test_reduced_ll_grid.py | 42 ++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/tests/test_reduced_ll_grid.py b/tests/test_reduced_ll_grid.py index f6bbffb1a..064bd2909 100644 --- a/tests/test_reduced_ll_grid.py +++ b/tests/test_reduced_ll_grid.py @@ -1,3 +1,5 @@ +# import geopandas as gpd +# import matplotlib.pyplot as plt import pandas as pd import pytest from eccodes import codes_grib_find_nearest, codes_grib_new_from_file @@ -30,30 +32,18 @@ def setup_method(self, method): self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) def find_nearest_latlon(self, grib_file, target_lat, target_lon): - # Open the GRIB file - f = open(grib_file) - - # Load the GRIB messages from the file - messages = [] - while True: - message = codes_grib_new_from_file(f) - if message is None: - break - messages.append(message) + messages = grib_file # Find the nearest grid points nearest_points = [] - for message in messages: + for message in [messages[0]]: nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) nearest_points.append(nearest_index) - # Close the GRIB file - f.close() - return nearest_points @pytest.mark.internet - @pytest.mark.skip(reason="can't install fdb branch on CI") + # @pytest.mark.skip(reason="can't install fdb branch on CI") def test_reduced_ll_grid(self): request = Request( Select("step", [1]), @@ -76,20 +66,40 @@ def test_reduced_ll_grid(self): lats = [] lons = [] eccodes_lats = [] + eccodes_lons = [] tol = 1e-8 + f = open("./tests/data/wave.grib") + messages = [] + while True: + message = codes_grib_new_from_file(f) + if message is None: + break + messages.append(message) + for i in range(len(result.leaves)): cubepath = result.leaves[i].flatten() lat = cubepath["latitude"] lon = cubepath["longitude"] lats.append(lat) lons.append(lon) - nearest_points = self.find_nearest_latlon("./tests/data/wave.grib", lat, lon) + nearest_points = self.find_nearest_latlon(messages, lat, lon) eccodes_lat = nearest_points[0][0]["lat"] eccodes_lon = nearest_points[0][0]["lon"] eccodes_lats.append(eccodes_lat) + eccodes_lons.append(eccodes_lon) assert eccodes_lat - tol <= lat assert lat <= eccodes_lat + tol assert eccodes_lon - tol <= lon assert lon <= eccodes_lon + tol + f.close() + + # worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + # fig, ax = plt.subplots(figsize=(12, 6)) + # worldmap.plot(color="darkgrey", ax=ax) + + # plt.scatter(lons, lats, s=18, c="red", cmap="YlOrRd") + # plt.scatter(eccodes_lons, eccodes_lats, s=6, c="green") + # plt.colorbar(label="Temperature") + # plt.show() assert len(eccodes_lats) == 10 From 8aba8354dd172607ae3e2405d0426d857856d539 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 4 Jan 2024 10:38:29 +0100 Subject: [PATCH 256/332] skip fdb tests --- tests/test_reduced_ll_grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_reduced_ll_grid.py b/tests/test_reduced_ll_grid.py index 064bd2909..74fffa7a3 100644 --- a/tests/test_reduced_ll_grid.py +++ b/tests/test_reduced_ll_grid.py @@ -43,7 +43,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points @pytest.mark.internet - # @pytest.mark.skip(reason="can't install fdb branch on CI") + @pytest.mark.skip(reason="can't install fdb branch on CI") def test_reduced_ll_grid(self): request = Request( Select("step", [1]), From 6b6e6873b89476783074a2165d1379a954224b56 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 4 Jan 2024 15:17:59 +0100 Subject: [PATCH 257/332] make tests more efficient and example of shipping route on wave model --- examples/3D_shipping_route_wave_model.py | 132 +++++++++++++++++++++++ tests/test_reduced_ll_grid.py | 39 +++---- tests/test_regular_grid.py | 5 +- 3 files changed, 148 insertions(+), 28 deletions(-) create mode 100644 examples/3D_shipping_route_wave_model.py diff --git a/examples/3D_shipping_route_wave_model.py b/examples/3D_shipping_route_wave_model.py new file mode 100644 index 000000000..e552afed5 --- /dev/null +++ b/examples/3D_shipping_route_wave_model.py @@ -0,0 +1,132 @@ +import geopandas as gpd +import matplotlib.pyplot as plt +import pandas as pd +import pytest +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file + +from polytope.datacube.backends.fdb import FDBDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Ellipsoid, Path, Select +from tests.helper_functions import download_test_data + + +class TestReducedLatLonGrid: + def setup_method(self, method): + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/wave.grib" + download_test_data(nexus_url, "wave.grib") + self.options = { + "values": { + "transformation": { + "mapper": {"type": "reduced_ll", "resolution": 1441, "axes": ["latitude", "longitude"]} + } + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + "number": {"transformation": {"type_change": "int"}}, + "longitude": {"transformation": {"cyclic": [0, 360]}}, + } + self.config = {"class": "od", "stream": "wave"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + def find_nearest_latlon(self, grib_file, target_lat, target_lon): + messages = grib_file + + # Find the nearest grid points + nearest_points = [] + for message in [messages[0]]: + nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) + nearest_points.append(nearest_index) + + return nearest_points + + @pytest.mark.internet + @pytest.mark.skip(reason="can't install fdb branch on CI") + def test_reduced_ll_grid(self): + + shapefile = gpd.read_file("./examples/data/Shipping-Lanes-v1.shp") + geometry_multiline = shapefile.iloc[2] + geometry_object = geometry_multiline["geometry"] + + lines = [] + i = 0 + + for line in geometry_object[:7]: + for point in line.coords: + point_list = list(point) + if list(point)[0] < 0: + point_list[0] = list(point)[0] + 360 + lines.append(point_list) + + # Append for each point a corresponding step + + new_points = [] + for point in lines[:7]: + new_points.append([point[1], point[0], 1]) + + # Pad the shipping route with an initial shape + + padded_point_upper = [0.24, 0.24, 1] + padded_point_lower = [-0.24, -0.24, 1] + initial_shape = Ellipsoid(["latitude", "longitude", "step"], padded_point_lower, padded_point_upper) + + # Then somehow make this list of points into just a sequence of points + + ship_route_polytope = Path(["latitude", "longitude", "step"], initial_shape, *new_points) + + request = Request( + ship_route_polytope, + Select("date", [pd.Timestamp("20231129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["140251"]), + Select("direction", ["1"]), + Select("frequency", ["1"]), + Select("class", ["od"]), + Select("stream", ["wave"]), + Select("levtype", ["sfc"]), + Select("type", ["fc"]), + ) + result = self.API.retrieve(request) + result.pprint() + + lats = [] + lons = [] + eccodes_lats = [] + eccodes_lons = [] + tol = 1e-8 + f = open("./tests/data/wave.grib", "rb") + messages = [] + message = codes_grib_new_from_file(f) + messages.append(message) + + leaves = result.leaves + for i in range(len(leaves)): + cubepath = leaves[i].flatten() + lat = cubepath["latitude"] + lon = cubepath["longitude"] + del cubepath + lats.append(lat) + lons.append(lon) + nearest_points = codes_grib_find_nearest(message, lat, lon)[0] + eccodes_lat = nearest_points.lat + eccodes_lon = nearest_points.lon + eccodes_lats.append(eccodes_lat) + eccodes_lons.append(eccodes_lon) + assert eccodes_lat - tol <= lat + assert lat <= eccodes_lat + tol + assert eccodes_lon - tol <= lon + assert lon <= eccodes_lon + tol + print(i) + f.close() + + worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + fig, ax = plt.subplots(figsize=(12, 6)) + worldmap.plot(color="darkgrey", ax=ax) + + plt.scatter(lons, lats, s=18, c="red", cmap="YlOrRd") + plt.scatter(eccodes_lons, eccodes_lats, s=6, c="green") + plt.colorbar(label="Temperature") + plt.show() diff --git a/tests/test_reduced_ll_grid.py b/tests/test_reduced_ll_grid.py index 74fffa7a3..1412e2d42 100644 --- a/tests/test_reduced_ll_grid.py +++ b/tests/test_reduced_ll_grid.py @@ -31,20 +31,10 @@ def setup_method(self, method): self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) - def find_nearest_latlon(self, grib_file, target_lat, target_lon): - messages = grib_file - - # Find the nearest grid points - nearest_points = [] - for message in [messages[0]]: - nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) - nearest_points.append(nearest_index) - - return nearest_points - @pytest.mark.internet @pytest.mark.skip(reason="can't install fdb branch on CI") def test_reduced_ll_grid(self): + request = Request( Select("step", [1]), Select("date", [pd.Timestamp("20231129T000000")]), @@ -57,34 +47,33 @@ def test_reduced_ll_grid(self): Select("stream", ["wave"]), Select("levtype", ["sfc"]), Select("type", ["fc"]), - Box(["latitude", "longitude"], [0, 0], [0.2, 0.5]), + Box(["latitude", "longitude"], [0, 0], [1.2, 1.5]), ) result = self.API.retrieve(request) result.pprint() - assert len(result.leaves) == 10 + assert len(result.leaves) == 130 lats = [] lons = [] eccodes_lats = [] eccodes_lons = [] tol = 1e-8 - f = open("./tests/data/wave.grib") + f = open("./tests/data/wave.grib", "rb") messages = [] - while True: - message = codes_grib_new_from_file(f) - if message is None: - break - messages.append(message) + message = codes_grib_new_from_file(f) + messages.append(message) - for i in range(len(result.leaves)): - cubepath = result.leaves[i].flatten() + leaves = result.leaves + for i in range(len(leaves)): + cubepath = leaves[i].flatten() lat = cubepath["latitude"] lon = cubepath["longitude"] + del cubepath lats.append(lat) lons.append(lon) - nearest_points = self.find_nearest_latlon(messages, lat, lon) - eccodes_lat = nearest_points[0][0]["lat"] - eccodes_lon = nearest_points[0][0]["lon"] + nearest_points = codes_grib_find_nearest(message, lat, lon)[0] + eccodes_lat = nearest_points.lat + eccodes_lon = nearest_points.lon eccodes_lats.append(eccodes_lat) eccodes_lons.append(eccodes_lon) assert eccodes_lat - tol <= lat @@ -101,5 +90,3 @@ def test_reduced_ll_grid(self): # plt.scatter(eccodes_lons, eccodes_lats, s=6, c="green") # plt.colorbar(label="Temperature") # plt.show() - - assert len(eccodes_lats) == 10 diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 536b386ca..f5fd5f772 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -76,8 +76,9 @@ def test_regular_grid(self): lons = [] eccodes_lats = [] tol = 1e-8 - for i in range(len(result.leaves)): - cubepath = result.leaves[i].flatten() + leaves = result.leaves + for i in range(len(leaves)): + cubepath = leaves[i].flatten() lat = cubepath["latitude"] lon = cubepath["longitude"] lats.append(lat) From bb65df567c2950bf1681a2fcf4ef801fab52a188 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 4 Jan 2024 15:20:32 +0100 Subject: [PATCH 258/332] black --- examples/3D_shipping_route_wave_model.py | 1 - tests/test_reduced_ll_grid.py | 1 - 2 files changed, 2 deletions(-) diff --git a/examples/3D_shipping_route_wave_model.py b/examples/3D_shipping_route_wave_model.py index e552afed5..e4edabe9f 100644 --- a/examples/3D_shipping_route_wave_model.py +++ b/examples/3D_shipping_route_wave_model.py @@ -45,7 +45,6 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): @pytest.mark.internet @pytest.mark.skip(reason="can't install fdb branch on CI") def test_reduced_ll_grid(self): - shapefile = gpd.read_file("./examples/data/Shipping-Lanes-v1.shp") geometry_multiline = shapefile.iloc[2] geometry_object = geometry_multiline["geometry"] diff --git a/tests/test_reduced_ll_grid.py b/tests/test_reduced_ll_grid.py index 1412e2d42..06c50c2dd 100644 --- a/tests/test_reduced_ll_grid.py +++ b/tests/test_reduced_ll_grid.py @@ -34,7 +34,6 @@ def setup_method(self, method): @pytest.mark.internet @pytest.mark.skip(reason="can't install fdb branch on CI") def test_reduced_ll_grid(self): - request = Request( Select("step", [1]), Select("date", [pd.Timestamp("20231129T000000")]), From 8f4b37970db580d0f39d436e161b7a7db229115c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 5 Dec 2023 11:24:29 +0100 Subject: [PATCH 259/332] add new point examples of date ranges --- tests/test_slice_date_range_fdb_v2.py | 45 +++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/test_slice_date_range_fdb_v2.py diff --git a/tests/test_slice_date_range_fdb_v2.py b/tests/test_slice_date_range_fdb_v2.py new file mode 100644 index 000000000..404e7fc36 --- /dev/null +++ b/tests/test_slice_date_range_fdb_v2.py @@ -0,0 +1,45 @@ +import pandas as pd +import pytest + +from polytope.datacube.backends.fdb import FDBDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Select, Span + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": { + "transformation": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}} + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + } + self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.skip(reason="can't install fdb branch on CI") + def test_fdb_datacube(self): + request = Request( + Select("step", [0]), + Select("levtype", ["pl"]), + Span("date", pd.Timestamp("20170101T120000"), pd.Timestamp("20170102T120000")), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["129"]), + Select("class", ["ea"]), + Select("stream", ["enda"]), + Select("type", ["an"]), + Select("latitude", [0]), + Select("longitude", [0]), + Select("levelist", ["500", "850"]), + Select("number", ["0"]), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 6 From e2a5b5f970c06bb06afa3ca63f34f874c2a0c7af Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 5 Dec 2023 11:41:42 +0100 Subject: [PATCH 260/332] add test on operational ecmwf data --- tests/test_ecmwf_oper_data_fdb.py | 44 +++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/test_ecmwf_oper_data_fdb.py diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py new file mode 100644 index 000000000..8cd7238be --- /dev/null +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -0,0 +1,44 @@ +import pandas as pd +import pytest + +from polytope.datacube.backends.fdb import FDBDatacube +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": { + "transformation": { + "mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]} + } + }, + "date": {"transformation": {"merge": {"with": "time", "linkers": ["T", "00"]}}}, + "step": {"transformation": {"type_change": "int"}}, + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0, "type": "fc"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.skip(reason="can't install fdb branch on CI") + def test_fdb_datacube(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231102T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 9 From 3a5955143068ae8efa22e34396086cd3f9e78d25 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 9 Jan 2024 13:07:04 +0100 Subject: [PATCH 261/332] avoid gribjump import in tests --- tests/test_fdb_datacube.py | 3 ++- tests/test_incomplete_tree_fdb.py | 3 ++- tests/test_regular_grid.py | 3 ++- tests/test_shapes.py | 3 ++- tests/test_slice_date_range_fdb.py | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index bb76ed00e..b18ddc115 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -1,7 +1,6 @@ import pandas as pd import pytest -from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -12,6 +11,8 @@ class TestSlicingFDBDatacube: def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + # Create a dataarray with 3 labelled axes using different index types self.options = { "values": { diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 852cd7bd1..6ed14ece2 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -3,7 +3,6 @@ from eccodes import codes_grib_find_nearest, codes_grib_new_from_file from helper_functions import download_test_data -from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select @@ -11,6 +10,8 @@ class TestRegularGrid: def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") self.options = { diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 2005848c6..536e23a0f 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -3,7 +3,6 @@ from eccodes import codes_grib_find_nearest, codes_grib_new_from_file from helper_functions import download_test_data -from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Disk, Select @@ -14,6 +13,8 @@ class TestRegularGrid: def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") self.options = { diff --git a/tests/test_shapes.py b/tests/test_shapes.py index 2c87f1211..3847d2e68 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -3,7 +3,6 @@ import pytest import xarray as xr -from polytope.datacube.backends.fdb import FDBDatacube from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request @@ -41,6 +40,8 @@ def test_all_cyclic(self): @pytest.mark.fdb def test_all_mapper_cyclic(self): + from polytope.datacube.backends.fdb import FDBDatacube + self.options = { "values": { "transformation": { diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index e700ef551..55e9b002c 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -1,7 +1,6 @@ import pandas as pd import pytest -from polytope.datacube.backends.fdb import FDBDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select, Span @@ -9,6 +8,8 @@ class TestSlicingFDBDatacube: def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + # Create a dataarray with 3 labelled axes using different index types self.options = { "values": { From d6a7f143ad4ac75b877b2b66f3a67f23518d7cac Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 10 Jan 2024 16:45:09 +0100 Subject: [PATCH 262/332] fix tests --- tests/test_ecmwf_oper_data_fdb.py | 2 +- tests/test_fdb_datacube.py | 3 +-- tests/test_slice_date_range_fdb.py | 4 +--- tests/test_slice_date_range_fdb_v2.py | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 7ec4a4659..553407eab 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -16,7 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0, "type": "fc"} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index a7c54816a..bf4452a35 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -20,7 +20,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc"} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) @@ -38,7 +38,6 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), - Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 624a77fe2..319bc2c24 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -17,7 +17,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": "0"} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) @@ -27,7 +27,6 @@ def setup_method(self, method): def test_fdb_datacube(self): request = Request( Select("step", [0]), - Select("number", [1]), Select("levtype", ["sfc"]), Span("date", pd.Timestamp("20230625T120000"), pd.Timestamp("20230626T120000")), Select("domain", ["g"]), @@ -36,7 +35,6 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), - Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) diff --git a/tests/test_slice_date_range_fdb_v2.py b/tests/test_slice_date_range_fdb_v2.py index 24ae1a9a5..ee50e75b6 100644 --- a/tests/test_slice_date_range_fdb_v2.py +++ b/tests/test_slice_date_range_fdb_v2.py @@ -16,7 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, } - self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} + self.config = {"class": "ea", "expver": "0001", "levtype": "pl"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) From 0c6764a2a2aefd204d4eacd81af97ae07f306474 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 11 Jan 2024 11:08:40 +0100 Subject: [PATCH 263/332] rename fdb to gj --- polytope/datacube/backends/fdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index cbc76893f..3231dd2a1 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -19,7 +19,7 @@ def __init__(self, config={}, axis_options={}): partial_request = config # Find values in the level 3 FDB datacube - self.fdb = pygj.GribJump() + self.gj = pygj.GribJump() self.fdb_coordinates = self.fdb.axes(partial_request) self.fdb_coordinates["values"] = [] for name, values in self.fdb_coordinates.items(): From 1dd0cb20dcf6fe9e59c362e1821696534bd30de3 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 11 Jan 2024 11:11:00 +0100 Subject: [PATCH 264/332] rename fdb to gj --- polytope/datacube/backends/fdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 3231dd2a1..a3c230fec 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -20,7 +20,7 @@ def __init__(self, config={}, axis_options={}): # Find values in the level 3 FDB datacube self.gj = pygj.GribJump() - self.fdb_coordinates = self.fdb.axes(partial_request) + self.fdb_coordinates = self.gj.axes(partial_request) self.fdb_coordinates["values"] = [] for name, values in self.fdb_coordinates.items(): values.sort() From 06e634289729240581362c67b849246ece107d67 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 11 Jan 2024 11:11:40 +0100 Subject: [PATCH 265/332] rename fdb to gj --- polytope/datacube/backends/fdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index a3c230fec..19650d14e 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -146,7 +146,7 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) - output_values = self.fdb.extract(fdb_requests) + output_values = self.gj.extract(fdb_requests) return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): From 76e73b942eba22caa570d6a4121c5ae840df47a8 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 11 Jan 2024 14:25:22 +0100 Subject: [PATCH 266/332] replace |= operation with update --- polytope/datacube/backends/fdb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index cbc76893f..6cba64713 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -48,7 +48,7 @@ def get(self, requests: IndexTree, leaf_path={}): (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path ) - leaf_path |= key_value_path + leaf_path.update(key_value_path) if len(requests.children[0].children[0].children) == 0: # remap this last key self.get_2nd_last_values(requests, leaf_path) @@ -79,7 +79,7 @@ def get_2nd_last_values(self, requests, leaf_path={}): (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path ) - leaf_path |= key_value_path + leaf_path.update(key_value_path) (range_lengths[i], current_start_idxs[i], fdb_node_ranges[i]) = self.get_last_layer_before_leaf( lat_child, leaf_path, range_length, current_start_idx, fdb_range_nodes ) @@ -94,7 +94,7 @@ def get_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path ) - leaf_path |= key_value_path + leaf_path.update(key_value_path) last_idx = key_value_path["values"] if current_idx[i] is None: current_idx[i] = last_idx @@ -109,7 +109,7 @@ def get_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path ) - leaf_path |= key_value_path + leaf_path.update(key_value_path) i += 1 current_start_idx = key_value_path["values"] current_idx[i] = current_start_idx From 1d98c8882eefd4ad088e512d5c42e9ef3a92f75c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 11 Jan 2024 16:30:59 +0100 Subject: [PATCH 267/332] test for polytope higher level timeseries shape --- tests/test_ecmwf_oper_data_fdb.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 7ec4a4659..e0d179dd5 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -3,7 +3,7 @@ from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Box, Select +from polytope.shapes import Box, Select, Union class TestSlicingFDBDatacube: @@ -16,7 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0, "type": "fc"} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "fc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) @@ -39,3 +39,28 @@ def test_fdb_datacube(self): result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 9 + + @pytest.mark.fdb + def test_fdb_datacube_point(self): + request = Request( + # Select("step", [0, 1]), + Union( + ["latitude", "longitude", "step"], + *[ + Box(["latitude", "longitude", "step"], lower_corner=[p[0], p[1], 0], upper_corner=[p[0], p[1], 2]) + for p in [[0.035149384216, 0.0]] + ] + ), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240103T0000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + # Point(["latitude", "longitude"], [[0.035149384216, 0.0]]), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 3 From ea909c6e92e72356423ff909b7132a5d7e31cbe6 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 12 Jan 2024 09:18:41 +0100 Subject: [PATCH 268/332] add __init__ to transformations folder --- polytope/datacube/transformations/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 polytope/datacube/transformations/__init__.py diff --git a/polytope/datacube/transformations/__init__.py b/polytope/datacube/transformations/__init__.py new file mode 100644 index 000000000..cf6989be4 --- /dev/null +++ b/polytope/datacube/transformations/__init__.py @@ -0,0 +1 @@ +from ..transformations.datacube_transformations import * From 20f432890f2a13fa462ada0e8863782ea80208bf Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 12 Jan 2024 09:27:31 +0100 Subject: [PATCH 269/332] fix small test issues --- tests/test_fdb_datacube.py | 3 +-- tests/test_slice_date_range_fdb.py | 1 - tests/test_slice_date_range_fdb_v2.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index a7c54816a..bf4452a35 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -20,7 +20,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc"} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) @@ -38,7 +38,6 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), - Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 624a77fe2..fbbdff7f2 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -27,7 +27,6 @@ def setup_method(self, method): def test_fdb_datacube(self): request = Request( Select("step", [0]), - Select("number", [1]), Select("levtype", ["sfc"]), Span("date", pd.Timestamp("20230625T120000"), pd.Timestamp("20230626T120000")), Select("domain", ["g"]), diff --git a/tests/test_slice_date_range_fdb_v2.py b/tests/test_slice_date_range_fdb_v2.py index 24ae1a9a5..ee50e75b6 100644 --- a/tests/test_slice_date_range_fdb_v2.py +++ b/tests/test_slice_date_range_fdb_v2.py @@ -16,7 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, } - self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} + self.config = {"class": "ea", "expver": "0001", "levtype": "pl"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) From c3c6221c49a9c31f882703c70fa87e499dc21398 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 12 Jan 2024 09:29:06 +0100 Subject: [PATCH 270/332] remove {} as default argument in functions --- polytope/datacube/backends/fdb.py | 14 +++++++++++--- polytope/datacube/backends/xarray.py | 4 +++- polytope/polytope.py | 5 ++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 6cba64713..69eb1ed00 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -6,7 +6,11 @@ class FDBDatacube(Datacube): - def __init__(self, config={}, axis_options={}): + def __init__(self, config=None, axis_options=None): + if config is None: + config = {} + if axis_options is None: + axis_options = {} self.axis_options = axis_options self.axis_counter = 0 self._axes = None @@ -36,7 +40,9 @@ def __init__(self, config={}, axis_options={}): val = self._axes[name].type self._check_and_add_axes(options, name, val) - def get(self, requests: IndexTree, leaf_path={}): + def get(self, requests: IndexTree, leaf_path=None): + if leaf_path is None: + leaf_path = {} # First when request node is root, go to its children if requests.axis.name == "root": for c in requests.children: @@ -58,7 +64,9 @@ def get(self, requests: IndexTree, leaf_path={}): for c in requests.children: self.get(c, leaf_path) - def get_2nd_last_values(self, requests, leaf_path={}): + def get_2nd_last_values(self, requests, leaf_path=None): + if leaf_path is None: + leaf_path = {} # In this function, we recursively loop over the last two layers of the tree and store the indices of the # request ranges in those layers lat_length = len(requests.children) diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index f8ca1c2e2..ff0c8ef5c 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -8,7 +8,9 @@ class XArrayDatacube(Datacube): """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0).""" - def __init__(self, dataarray: xr.DataArray, axis_options={}): + def __init__(self, dataarray: xr.DataArray, axis_options=None): + if axis_options is None: + axis_options = {} self.axis_options = axis_options self.axis_counter = 0 self._axes = None diff --git a/polytope/polytope.py b/polytope/polytope.py index f6d4a723e..361bc6f40 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -37,10 +37,13 @@ def __repr__(self): class Polytope: - def __init__(self, datacube, engine=None, axis_options={}): + def __init__(self, datacube, engine=None, axis_options=None): from .datacube import Datacube from .engine import Engine + if axis_options is None: + axis_options = {} + self.datacube = Datacube.create(datacube, axis_options) self.engine = engine if engine is not None else Engine.default() From 3678ffe1c060e6f56700833c7e75048538c39011 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 12 Jan 2024 10:02:59 +0100 Subject: [PATCH 271/332] add point with method=surrounding test --- tests/test_point_shape.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_point_shape.py b/tests/test_point_shape.py index 0bc203d61..56659227e 100644 --- a/tests/test_point_shape.py +++ b/tests/test_point_shape.py @@ -34,3 +34,8 @@ def test_point_surrounding_step(self): request = Request(Point(["step", "level"], [[2, 10]], method="surrounding"), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) assert len(result.leaves) == 6 + + def test_point_surrounding_exact_step(self): + request = Request(Point(["step", "level"], [[3, 10]], method="surrounding"), Select("date", ["2000-01-01"])) + result = self.API.retrieve(request) + assert len(result.leaves) == 9 From 904828e8a8d8955b93627b31091b9575c0cbb720 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 12 Jan 2024 10:14:39 +0100 Subject: [PATCH 272/332] fix point test with method=surrounding on real data --- tests/test_ecmwf_oper_data_fdb.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index e0d179dd5..692924f15 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -3,7 +3,7 @@ from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request -from polytope.shapes import Box, Select, Union +from polytope.shapes import Box, Point, Select class TestSlicingFDBDatacube: @@ -43,14 +43,7 @@ def test_fdb_datacube(self): @pytest.mark.fdb def test_fdb_datacube_point(self): request = Request( - # Select("step", [0, 1]), - Union( - ["latitude", "longitude", "step"], - *[ - Box(["latitude", "longitude", "step"], lower_corner=[p[0], p[1], 0], upper_corner=[p[0], p[1], 2]) - for p in [[0.035149384216, 0.0]] - ] - ), + Select("step", [0, 1]), Select("levtype", ["sfc"]), Select("date", [pd.Timestamp("20240103T0000")]), Select("domain", ["g"]), @@ -59,8 +52,8 @@ def test_fdb_datacube_point(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["fc"]), - # Point(["latitude", "longitude"], [[0.035149384216, 0.0]]), + Point(["latitude", "longitude"], [[0.035149384216, 0.0]], method="surrounding"), ) result = self.API.retrieve(request) result.pprint() - assert len(result.leaves) == 3 + assert len(result.leaves) == 12 From 643c91ac083e1e50ec9cdef5fa709e106b9b413c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 12 Jan 2024 11:25:18 +0100 Subject: [PATCH 273/332] add logging to datacube/fdb backend --- polytope/datacube/backends/datacube.py | 4 ++++ polytope/datacube/backends/fdb.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index efa20e93d..d3f3300aa 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -1,4 +1,5 @@ import importlib +import logging import math from abc import ABC, abstractmethod from typing import Any @@ -105,6 +106,9 @@ def get_indices(self, path: DatacubePath, axis, lower, upper, method=None): if offset is not None: # Note that we can only do unique if not dealing with time values idx_between = unique(idx_between) + + logging.info(f"For axis {axis.name} between {lower} and {upper}, found indices {idx_between}") + return idx_between def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, method): diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 69eb1ed00..0981de031 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -1,3 +1,4 @@ +import logging from copy import deepcopy import pygribjump as pygj @@ -11,6 +12,9 @@ def __init__(self, config=None, axis_options=None): config = {} if axis_options is None: axis_options = {} + + logging.info("Created an FDB datacube with options: " + str(axis_options)) + self.axis_options = axis_options self.axis_counter = 0 self._axes = None @@ -25,6 +29,9 @@ def __init__(self, config=None, axis_options=None): self.fdb = pygj.GribJump() self.fdb_coordinates = self.fdb.axes(partial_request) + + logging.info("Axes returned from GribJump are: " + str(self.fdb_coordinates)) + self.fdb_coordinates["values"] = [] for name, values in self.fdb_coordinates.items(): values.sort() @@ -40,11 +47,18 @@ def __init__(self, config=None, axis_options=None): val = self._axes[name].type self._check_and_add_axes(options, name, val) + logging.info("Polytope created axes for: " + str(self._axes.keys())) + def get(self, requests: IndexTree, leaf_path=None): + if leaf_path is None: leaf_path = {} + # First when request node is root, go to its children if requests.axis.name == "root": + + logging.info("Looking for data for the tree: " + str([leaf.flatten() for leaf in requests.leaves])) + for c in requests.children: self.get(c) # If request node has no children, we have a leaf so need to assign fdb values to it From 3fde0300110b25f0f85d3c69c4f32f5ff80489db Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 12 Jan 2024 13:53:55 +0100 Subject: [PATCH 274/332] add logging for hullslicer engine --- polytope/engine/hullslicer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index e6c8e3eb0..29b166229 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -1,3 +1,4 @@ +import logging import math from copy import copy from itertools import chain @@ -72,6 +73,9 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex remapped_val_interm = ax.remap([value, value])[0] remapped_val = (remapped_val_interm[0] + remapped_val_interm[1]) / 2 remapped_val = round(remapped_val, int(-math.log10(ax.tol))) + + logging.info(f"Added index {remapped_val} on axis {ax.name} to the tree") + child = node.create_child(ax, remapped_val) child["unsliced_polytopes"] = copy(node["unsliced_polytopes"]) child["unsliced_polytopes"].remove(polytope) From 12836ca0593d2b749c7c09b8099eb70493a98d91 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 12 Jan 2024 14:55:42 +0100 Subject: [PATCH 275/332] logging on mapping and merger transformations --- polytope/datacube/backends/fdb.py | 2 -- .../transformations/datacube_mappers.py | 19 +++++++++++++++++++ .../transformations/datacube_merger.py | 10 ++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 0981de031..23bdc1a3a 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -50,13 +50,11 @@ def __init__(self, config=None, axis_options=None): logging.info("Polytope created axes for: " + str(self._axes.keys())) def get(self, requests: IndexTree, leaf_path=None): - if leaf_path is None: leaf_path = {} # First when request node is root, go to its children if requests.axis.name == "root": - logging.info("Looking for data for the tree: " + str([leaf.flatten() for leaf in requests.leaves])) for c in requests.children: diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 0a7a3f5e9..750b8c6fc 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -1,4 +1,5 @@ import bisect +import logging import math from copy import deepcopy from importlib import import_module @@ -126,6 +127,12 @@ def unmap(self, first_val, second_val): second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] second_idx = self.second_axis_vals(first_val).index(second_val) final_index = self.axes_idx_to_regular_idx(first_idx, second_idx) + + logging.info( + f"Mapped the values {first_val} on axis {self._mapped_axes[0]} \ + and {second_val} on axis {self._mapped_axes[1]} to value {final_index} on axis {self._base_axis}" + ) + return final_index @@ -1627,6 +1634,12 @@ def unmap(self, first_val, second_val): second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] second_idx = self.second_axis_vals(first_val).index(second_val) reduced_ll_index = self.axes_idx_to_reduced_ll_idx(first_idx, second_idx) + + logging.info( + f"Mapped the values {first_val} on axis {self._mapped_axes[0]} \ + and {second_val} on axis {self._mapped_axes[1]} to value {reduced_ll_index} on axis {self._base_axis}" + ) + return reduced_ll_index @@ -4494,6 +4507,12 @@ def find_second_axis_idx(self, first_val, second_val): def unmap(self, first_val, second_val): (first_idx, second_idx) = self.find_second_axis_idx(first_val, second_val) octahedral_index = self.axes_idx_to_octahedral_idx(first_idx, second_idx) + + logging.info( + f"Mapped the values {first_val} on axis {self._mapped_axes[0]} \ + and {second_val} on axis {self._mapped_axes[1]} to value {octahedral_index} on axis {self._base_axis}" + ) + return octahedral_index diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index d60278671..8864cabf5 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -1,3 +1,5 @@ +import logging + import numpy as np import pandas as pd @@ -37,6 +39,10 @@ def merged_values(self, datacube): val_to_add = val_to_add.astype("datetime64[s]") merged_values.append(val_to_add) merged_values = np.array(merged_values) + logging.info( + f"Merged values {first_ax_vals} on axis {self.name} and \ + values {second_ax_vals} on axis {second_ax_name} to values {merged_values}" + ) return merged_values def transformation_axes_final(self): @@ -56,6 +62,10 @@ def unmerge(self, merged_val): # TODO: maybe replacing like this is too specific to time/dates? first_val = str(first_val).replace("-", "") second_val = second_val.replace(":", "") + logging.info( + f"Unmerged value {merged_val} to values {first_val} on axis {self.name} \ + and {second_val} on axis {self._second_axis}" + ) return (first_val, second_val) def change_val_type(self, axis_name, values): From 3c1d417ee469185ef8de4059a6e65bc3627857a3 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 16 Jan 2024 14:57:18 +0100 Subject: [PATCH 276/332] update requirements --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 50594c4fc..66101f546 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,8 @@ decorator==5.1.1 numpy==1.23.5 pandas==1.5.2 pypi==2.1 -requests==2.28.1 -scipy==1.9.3 +requests==2.31.0 +scipy==1.11.4 sortedcontainers==2.4.0 tripy==1.0.0 typing==3.7.4.3 From bf3e4965021f6dd541173318bbf463470172fbee Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 17 Jan 2024 13:10:19 +0100 Subject: [PATCH 277/332] put the mappers in separate files for clarity --- .../transformations/datacube_mappers.py | 4426 +---------------- .../transformations/mappers/healpix.py | 123 + .../transformations/mappers/octahedral.py | 2752 ++++++++++ .../transformations/mappers/reduced_ll.py | 1504 ++++++ .../transformations/mappers/regular.py | 56 + tests/test_ecmwf_oper_data_fdb.py | 2 +- tests/test_fdb_datacube.py | 3 +- tests/test_mappers.py | 2 +- tests/test_slice_date_range_fdb.py | 4 +- tests/test_slice_date_range_fdb_v2.py | 2 +- 10 files changed, 4441 insertions(+), 4433 deletions(-) create mode 100644 polytope/datacube/transformations/mappers/healpix.py create mode 100644 polytope/datacube/transformations/mappers/octahedral.py create mode 100644 polytope/datacube/transformations/mappers/reduced_ll.py create mode 100644 polytope/datacube/transformations/mappers/regular.py diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 0a7a3f5e9..2e94c6494 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -1,9 +1,6 @@ -import bisect -import math from copy import deepcopy from importlib import import_module -from ...utility.list_tools import bisect_left_cmp, bisect_right_cmp from .datacube_transformations import DatacubeAxisTransformation @@ -22,7 +19,7 @@ def __init__(self, name, mapper_options): def generate_final_transformation(self): map_type = _type_to_datacube_mapper_lookup[self.grid_type] - module = import_module("polytope.datacube.transformations.datacube_mappers") + module = import_module("polytope.datacube.transformations.mappers." + self.grid_type) constructor = getattr(module, map_type) transformation = deepcopy(constructor(self.old_axis, self.grid_axes, self.grid_resolution)) return transformation @@ -76,4427 +73,6 @@ def unmap(self, first_val, second_val): return self._final_transformation.unmap(first_val, second_val) -class RegularGridMapper(DatacubeMapper): - def __init__(self, base_axis, mapped_axes, resolution): - self._mapped_axes = mapped_axes - self._base_axis = base_axis - self._resolution = resolution - self.deg_increment = 90 / self._resolution - self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} - self._first_axis_vals = self.first_axis_vals() - - def first_axis_vals(self): - first_ax_vals = [90 - i * self.deg_increment for i in range(2 * self._resolution)] - return first_ax_vals - - def map_first_axis(self, lower, upper): - axis_lines = self._first_axis_vals - return_vals = [val for val in axis_lines if lower <= val <= upper] - return return_vals - - def second_axis_vals(self, first_val): - second_ax_vals = [i * self.deg_increment for i in range(4 * self._resolution)] - return second_ax_vals - - def map_second_axis(self, first_val, lower, upper): - axis_lines = self.second_axis_vals(first_val) - return_vals = [val for val in axis_lines if lower <= val <= upper] - return return_vals - - def axes_idx_to_regular_idx(self, first_idx, second_idx): - final_idx = first_idx * 4 * self._resolution + second_idx - return final_idx - - def find_second_idx(self, first_val, second_val): - tol = 1e-10 - second_axis_vals = self.second_axis_vals(first_val) - second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) - return second_idx - - def unmap_first_val_to_start_line_idx(self, first_val): - tol = 1e-8 - first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] - first_idx = self._first_axis_vals.index(first_val) - return first_idx * 4 * self._resolution - - def unmap(self, first_val, second_val): - tol = 1e-8 - first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] - first_idx = self._first_axis_vals.index(first_val) - second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] - second_idx = self.second_axis_vals(first_val).index(second_val) - final_index = self.axes_idx_to_regular_idx(first_idx, second_idx) - return final_index - - -class ReducedLatLonMapper(DatacubeMapper): - def __init__(self, base_axis, mapped_axes, resolution): - self._mapped_axes = mapped_axes - self._base_axis = base_axis - self._resolution = resolution - self._axis_reversed = {mapped_axes[0]: False, mapped_axes[1]: False} - self._first_axis_vals = self.first_axis_vals() - - def first_axis_vals(self): - resolution = 180 / (self._resolution - 1) - vals = [-90 + i * resolution for i in range(self._resolution)] - return vals - - def map_first_axis(self, lower, upper): - axis_lines = self._first_axis_vals - return_vals = [val for val in axis_lines if lower <= val <= upper] - return return_vals - - def lon_spacing(self): - if self._resolution == 1441: - return [ - 2, - 6, - 14, - 20, - 26, - 32, - 38, - 44, - 50, - 58, - 64, - 70, - 76, - 82, - 88, - 94, - 102, - 108, - 114, - 120, - 126, - 132, - 138, - 144, - 152, - 158, - 164, - 170, - 176, - 182, - 188, - 196, - 202, - 208, - 214, - 220, - 226, - 232, - 238, - 246, - 252, - 258, - 264, - 270, - 276, - 282, - 290, - 296, - 302, - 308, - 314, - 320, - 326, - 332, - 340, - 346, - 352, - 358, - 364, - 370, - 376, - 382, - 388, - 396, - 402, - 408, - 414, - 420, - 426, - 432, - 438, - 444, - 452, - 458, - 464, - 470, - 476, - 482, - 488, - 494, - 500, - 506, - 512, - 520, - 526, - 532, - 538, - 544, - 550, - 556, - 562, - 568, - 574, - 580, - 586, - 594, - 600, - 606, - 612, - 618, - 624, - 630, - 636, - 642, - 648, - 654, - 660, - 666, - 672, - 678, - 686, - 692, - 698, - 704, - 710, - 716, - 722, - 728, - 734, - 740, - 746, - 752, - 758, - 764, - 770, - 776, - 782, - 788, - 794, - 800, - 806, - 812, - 818, - 824, - 830, - 836, - 842, - 848, - 854, - 860, - 866, - 872, - 878, - 884, - 890, - 896, - 902, - 908, - 914, - 920, - 926, - 932, - 938, - 944, - 950, - 956, - 962, - 968, - 974, - 980, - 986, - 992, - 998, - 1004, - 1010, - 1014, - 1020, - 1026, - 1032, - 1038, - 1044, - 1050, - 1056, - 1062, - 1068, - 1074, - 1080, - 1086, - 1092, - 1096, - 1102, - 1108, - 1114, - 1120, - 1126, - 1132, - 1138, - 1144, - 1148, - 1154, - 1160, - 1166, - 1172, - 1178, - 1184, - 1190, - 1194, - 1200, - 1206, - 1212, - 1218, - 1224, - 1230, - 1234, - 1240, - 1246, - 1252, - 1258, - 1264, - 1268, - 1274, - 1280, - 1286, - 1292, - 1296, - 1302, - 1308, - 1314, - 1320, - 1324, - 1330, - 1336, - 1342, - 1348, - 1352, - 1358, - 1364, - 1370, - 1374, - 1380, - 1386, - 1392, - 1396, - 1402, - 1408, - 1414, - 1418, - 1424, - 1430, - 1436, - 1440, - 1446, - 1452, - 1456, - 1462, - 1468, - 1474, - 1478, - 1484, - 1490, - 1494, - 1500, - 1506, - 1510, - 1516, - 1522, - 1526, - 1532, - 1538, - 1542, - 1548, - 1554, - 1558, - 1564, - 1570, - 1574, - 1580, - 1584, - 1590, - 1596, - 1600, - 1606, - 1610, - 1616, - 1622, - 1626, - 1632, - 1636, - 1642, - 1648, - 1652, - 1658, - 1662, - 1668, - 1672, - 1678, - 1684, - 1688, - 1694, - 1698, - 1704, - 1708, - 1714, - 1718, - 1724, - 1728, - 1734, - 1738, - 1744, - 1748, - 1754, - 1758, - 1764, - 1768, - 1774, - 1778, - 1784, - 1788, - 1794, - 1798, - 1804, - 1808, - 1812, - 1818, - 1822, - 1828, - 1832, - 1838, - 1842, - 1846, - 1852, - 1856, - 1862, - 1866, - 1870, - 1876, - 1880, - 1886, - 1890, - 1894, - 1900, - 1904, - 1908, - 1914, - 1918, - 1922, - 1928, - 1932, - 1936, - 1942, - 1946, - 1950, - 1956, - 1960, - 1964, - 1970, - 1974, - 1978, - 1982, - 1988, - 1992, - 1996, - 2002, - 2006, - 2010, - 2014, - 2020, - 2024, - 2028, - 2032, - 2036, - 2042, - 2046, - 2050, - 2054, - 2060, - 2064, - 2068, - 2072, - 2076, - 2080, - 2086, - 2090, - 2094, - 2098, - 2102, - 2106, - 2112, - 2116, - 2120, - 2124, - 2128, - 2132, - 2136, - 2140, - 2144, - 2150, - 2154, - 2158, - 2162, - 2166, - 2170, - 2174, - 2178, - 2182, - 2186, - 2190, - 2194, - 2198, - 2202, - 2206, - 2210, - 2214, - 2218, - 2222, - 2226, - 2230, - 2234, - 2238, - 2242, - 2246, - 2250, - 2254, - 2258, - 2262, - 2266, - 2270, - 2274, - 2278, - 2282, - 2286, - 2290, - 2292, - 2296, - 2300, - 2304, - 2308, - 2312, - 2316, - 2320, - 2324, - 2326, - 2330, - 2334, - 2338, - 2342, - 2346, - 2348, - 2352, - 2356, - 2360, - 2364, - 2366, - 2370, - 2374, - 2378, - 2382, - 2384, - 2388, - 2392, - 2396, - 2398, - 2402, - 2406, - 2410, - 2412, - 2416, - 2420, - 2422, - 2426, - 2430, - 2432, - 2436, - 2440, - 2442, - 2446, - 2450, - 2452, - 2456, - 2460, - 2462, - 2466, - 2470, - 2472, - 2476, - 2478, - 2482, - 2486, - 2488, - 2492, - 2494, - 2498, - 2500, - 2504, - 2508, - 2510, - 2514, - 2516, - 2520, - 2522, - 2526, - 2528, - 2532, - 2534, - 2538, - 2540, - 2544, - 2546, - 2550, - 2552, - 2556, - 2558, - 2560, - 2564, - 2566, - 2570, - 2572, - 2576, - 2578, - 2580, - 2584, - 2586, - 2590, - 2592, - 2594, - 2598, - 2600, - 2602, - 2606, - 2608, - 2610, - 2614, - 2616, - 2618, - 2622, - 2624, - 2626, - 2628, - 2632, - 2634, - 2636, - 2640, - 2642, - 2644, - 2646, - 2650, - 2652, - 2654, - 2656, - 2658, - 2662, - 2664, - 2666, - 2668, - 2670, - 2674, - 2676, - 2678, - 2680, - 2682, - 2684, - 2686, - 2690, - 2692, - 2694, - 2696, - 2698, - 2700, - 2702, - 2704, - 2706, - 2708, - 2712, - 2714, - 2716, - 2718, - 2720, - 2722, - 2724, - 2726, - 2728, - 2730, - 2732, - 2734, - 2736, - 2738, - 2740, - 2742, - 2744, - 2746, - 2748, - 2750, - 2750, - 2752, - 2754, - 2756, - 2758, - 2760, - 2762, - 2764, - 2766, - 2768, - 2768, - 2770, - 2772, - 2774, - 2776, - 2778, - 2780, - 2780, - 2782, - 2784, - 2786, - 2788, - 2788, - 2790, - 2792, - 2794, - 2794, - 2796, - 2798, - 2800, - 2800, - 2802, - 2804, - 2806, - 2806, - 2808, - 2810, - 2810, - 2812, - 2814, - 2814, - 2816, - 2818, - 2818, - 2820, - 2822, - 2822, - 2824, - 2826, - 2826, - 2828, - 2828, - 2830, - 2832, - 2832, - 2834, - 2834, - 2836, - 2836, - 2838, - 2838, - 2840, - 2842, - 2842, - 2844, - 2844, - 2846, - 2846, - 2846, - 2848, - 2848, - 2850, - 2850, - 2852, - 2852, - 2854, - 2854, - 2856, - 2856, - 2856, - 2858, - 2858, - 2860, - 2860, - 2860, - 2862, - 2862, - 2862, - 2864, - 2864, - 2864, - 2866, - 2866, - 2866, - 2868, - 2868, - 2868, - 2868, - 2870, - 2870, - 2870, - 2872, - 2872, - 2872, - 2872, - 2874, - 2874, - 2874, - 2874, - 2874, - 2876, - 2876, - 2876, - 2876, - 2876, - 2876, - 2878, - 2878, - 2878, - 2878, - 2878, - 2878, - 2878, - 2878, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2880, - 2878, - 2878, - 2878, - 2878, - 2878, - 2878, - 2878, - 2878, - 2876, - 2876, - 2876, - 2876, - 2876, - 2876, - 2874, - 2874, - 2874, - 2874, - 2874, - 2872, - 2872, - 2872, - 2872, - 2870, - 2870, - 2870, - 2868, - 2868, - 2868, - 2868, - 2866, - 2866, - 2866, - 2864, - 2864, - 2864, - 2862, - 2862, - 2862, - 2860, - 2860, - 2860, - 2858, - 2858, - 2856, - 2856, - 2856, - 2854, - 2854, - 2852, - 2852, - 2850, - 2850, - 2848, - 2848, - 2846, - 2846, - 2846, - 2844, - 2844, - 2842, - 2842, - 2840, - 2838, - 2838, - 2836, - 2836, - 2834, - 2834, - 2832, - 2832, - 2830, - 2828, - 2828, - 2826, - 2826, - 2824, - 2822, - 2822, - 2820, - 2818, - 2818, - 2816, - 2814, - 2814, - 2812, - 2810, - 2810, - 2808, - 2806, - 2806, - 2804, - 2802, - 2800, - 2800, - 2798, - 2796, - 2794, - 2794, - 2792, - 2790, - 2788, - 2788, - 2786, - 2784, - 2782, - 2780, - 2780, - 2778, - 2776, - 2774, - 2772, - 2770, - 2768, - 2768, - 2766, - 2764, - 2762, - 2760, - 2758, - 2756, - 2754, - 2752, - 2750, - 2750, - 2748, - 2746, - 2744, - 2742, - 2740, - 2738, - 2736, - 2734, - 2732, - 2730, - 2728, - 2726, - 2724, - 2722, - 2720, - 2718, - 2716, - 2714, - 2712, - 2708, - 2706, - 2704, - 2702, - 2700, - 2698, - 2696, - 2694, - 2692, - 2690, - 2686, - 2684, - 2682, - 2680, - 2678, - 2676, - 2674, - 2670, - 2668, - 2666, - 2664, - 2662, - 2658, - 2656, - 2654, - 2652, - 2650, - 2646, - 2644, - 2642, - 2640, - 2636, - 2634, - 2632, - 2628, - 2626, - 2624, - 2622, - 2618, - 2616, - 2614, - 2610, - 2608, - 2606, - 2602, - 2600, - 2598, - 2594, - 2592, - 2590, - 2586, - 2584, - 2580, - 2578, - 2576, - 2572, - 2570, - 2566, - 2564, - 2560, - 2558, - 2556, - 2552, - 2550, - 2546, - 2544, - 2540, - 2538, - 2534, - 2532, - 2528, - 2526, - 2522, - 2520, - 2516, - 2514, - 2510, - 2508, - 2504, - 2500, - 2498, - 2494, - 2492, - 2488, - 2486, - 2482, - 2478, - 2476, - 2472, - 2470, - 2466, - 2462, - 2460, - 2456, - 2452, - 2450, - 2446, - 2442, - 2440, - 2436, - 2432, - 2430, - 2426, - 2422, - 2420, - 2416, - 2412, - 2410, - 2406, - 2402, - 2398, - 2396, - 2392, - 2388, - 2384, - 2382, - 2378, - 2374, - 2370, - 2366, - 2364, - 2360, - 2356, - 2352, - 2348, - 2346, - 2342, - 2338, - 2334, - 2330, - 2326, - 2324, - 2320, - 2316, - 2312, - 2308, - 2304, - 2300, - 2296, - 2292, - 2290, - 2286, - 2282, - 2278, - 2274, - 2270, - 2266, - 2262, - 2258, - 2254, - 2250, - 2246, - 2242, - 2238, - 2234, - 2230, - 2226, - 2222, - 2218, - 2214, - 2210, - 2206, - 2202, - 2198, - 2194, - 2190, - 2186, - 2182, - 2178, - 2174, - 2170, - 2166, - 2162, - 2158, - 2154, - 2150, - 2144, - 2140, - 2136, - 2132, - 2128, - 2124, - 2120, - 2116, - 2112, - 2106, - 2102, - 2098, - 2094, - 2090, - 2086, - 2080, - 2076, - 2072, - 2068, - 2064, - 2060, - 2054, - 2050, - 2046, - 2042, - 2036, - 2032, - 2028, - 2024, - 2020, - 2014, - 2010, - 2006, - 2002, - 1996, - 1992, - 1988, - 1982, - 1978, - 1974, - 1970, - 1964, - 1960, - 1956, - 1950, - 1946, - 1942, - 1936, - 1932, - 1928, - 1922, - 1918, - 1914, - 1908, - 1904, - 1900, - 1894, - 1890, - 1886, - 1880, - 1876, - 1870, - 1866, - 1862, - 1856, - 1852, - 1846, - 1842, - 1838, - 1832, - 1828, - 1822, - 1818, - 1812, - 1808, - 1804, - 1798, - 1794, - 1788, - 1784, - 1778, - 1774, - 1768, - 1764, - 1758, - 1754, - 1748, - 1744, - 1738, - 1734, - 1728, - 1724, - 1718, - 1714, - 1708, - 1704, - 1698, - 1694, - 1688, - 1684, - 1678, - 1672, - 1668, - 1662, - 1658, - 1652, - 1648, - 1642, - 1636, - 1632, - 1626, - 1622, - 1616, - 1610, - 1606, - 1600, - 1596, - 1590, - 1584, - 1580, - 1574, - 1570, - 1564, - 1558, - 1554, - 1548, - 1542, - 1538, - 1532, - 1526, - 1522, - 1516, - 1510, - 1506, - 1500, - 1494, - 1490, - 1484, - 1478, - 1474, - 1468, - 1462, - 1456, - 1452, - 1446, - 1440, - 1436, - 1430, - 1424, - 1418, - 1414, - 1408, - 1402, - 1396, - 1392, - 1386, - 1380, - 1374, - 1370, - 1364, - 1358, - 1352, - 1348, - 1342, - 1336, - 1330, - 1324, - 1320, - 1314, - 1308, - 1302, - 1296, - 1292, - 1286, - 1280, - 1274, - 1268, - 1264, - 1258, - 1252, - 1246, - 1240, - 1234, - 1230, - 1224, - 1218, - 1212, - 1206, - 1200, - 1194, - 1190, - 1184, - 1178, - 1172, - 1166, - 1160, - 1154, - 1148, - 1144, - 1138, - 1132, - 1126, - 1120, - 1114, - 1108, - 1102, - 1096, - 1092, - 1086, - 1080, - 1074, - 1068, - 1062, - 1056, - 1050, - 1044, - 1038, - 1032, - 1026, - 1020, - 1014, - 1010, - 1004, - 998, - 992, - 986, - 980, - 974, - 968, - 962, - 956, - 950, - 944, - 938, - 932, - 926, - 920, - 914, - 908, - 902, - 896, - 890, - 884, - 878, - 872, - 866, - 860, - 854, - 848, - 842, - 836, - 830, - 824, - 818, - 812, - 806, - 800, - 794, - 788, - 782, - 776, - 770, - 764, - 758, - 752, - 746, - 740, - 734, - 728, - 722, - 716, - 710, - 704, - 698, - 692, - 686, - 678, - 672, - 666, - 660, - 654, - 648, - 642, - 636, - 630, - 624, - 618, - 612, - 606, - 600, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - ] - - def second_axis_vals(self, first_val): - first_idx = self._first_axis_vals.index(first_val) - Ny = self.lon_spacing()[first_idx] - second_spacing = 360 / Ny - return [i * second_spacing for i in range(Ny)] - - def map_second_axis(self, first_val, lower, upper): - axis_lines = self.second_axis_vals(first_val) - return_vals = [val for val in axis_lines if lower <= val <= upper] - return return_vals - - def axes_idx_to_reduced_ll_idx(self, first_idx, second_idx): - Ny_array = self.lon_spacing() - idx = 0 - for i in range(self._resolution): - if i != first_idx: - idx += Ny_array[i] - else: - idx += second_idx - return idx - - def find_second_idx(self, first_val, second_val): - tol = 1e-10 - second_axis_vals = self.second_axis_vals(first_val) - second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) - return second_idx - - def unmap(self, first_val, second_val): - tol = 1e-8 - first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] - first_idx = self._first_axis_vals.index(first_val) - second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] - second_idx = self.second_axis_vals(first_val).index(second_val) - reduced_ll_index = self.axes_idx_to_reduced_ll_idx(first_idx, second_idx) - return reduced_ll_index - - -class HealpixGridMapper(DatacubeMapper): - def __init__(self, base_axis, mapped_axes, resolution): - self._mapped_axes = mapped_axes - self._base_axis = base_axis - self._resolution = resolution - self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} - self._first_axis_vals = self.first_axis_vals() - - def first_axis_vals(self): - rad2deg = 180 / math.pi - vals = [0] * (4 * self._resolution - 1) - - # Polar caps - for i in range(1, self._resolution): - val = 90 - (rad2deg * math.acos(1 - (i * i / (3 * self._resolution * self._resolution)))) - vals[i - 1] = val - vals[4 * self._resolution - 1 - i] = -val - # Equatorial belts - for i in range(self._resolution, 2 * self._resolution): - val = 90 - (rad2deg * math.acos((4 * self._resolution - 2 * i) / (3 * self._resolution))) - vals[i - 1] = val - vals[4 * self._resolution - 1 - i] = -val - # Equator - vals[2 * self._resolution - 1] = 0 - return vals - - def map_first_axis(self, lower, upper): - axis_lines = self._first_axis_vals - return_vals = [val for val in axis_lines if lower <= val <= upper] - return return_vals - - def second_axis_vals(self, first_val): - tol = 1e-8 - first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] - idx = self._first_axis_vals.index(first_val) - - # Polar caps - if idx < self._resolution - 1 or 3 * self._resolution - 1 < idx <= 4 * self._resolution - 2: - start = 45 / (idx + 1) - vals = [start + i * (360 / (4 * (idx + 1))) for i in range(4 * (idx + 1))] - return vals - # Equatorial belts - start = 45 / self._resolution - if self._resolution - 1 <= idx < 2 * self._resolution - 1 or 2 * self._resolution <= idx < 3 * self._resolution: - r_start = start * (2 - (((idx + 1) - self._resolution + 1) % 2)) - vals = [r_start + i * (360 / (4 * self._resolution)) for i in range(4 * self._resolution)] - if vals[-1] == 360: - vals[-1] = 0 - return vals - # Equator - temp_val = 1 if self._resolution % 2 else 0 - r_start = start * (1 - temp_val) - if idx == 2 * self._resolution - 1: - vals = [r_start + i * (360 / (4 * self._resolution)) for i in range(4 * self._resolution)] - return vals - - def map_second_axis(self, first_val, lower, upper): - axis_lines = self.second_axis_vals(first_val) - return_vals = [val for val in axis_lines if lower <= val <= upper] - return return_vals - - def axes_idx_to_healpix_idx(self, first_idx, second_idx): - idx = 0 - for i in range(self._resolution - 1): - if i != first_idx: - idx += 4 * (i + 1) - else: - idx += second_idx - return idx - for i in range(self._resolution - 1, 3 * self._resolution): - if i != first_idx: - idx += 4 * self._resolution - else: - idx += second_idx - return idx - for i in range(3 * self._resolution, 4 * self._resolution - 1): - if i != first_idx: - idx += 4 * (4 * self._resolution - 1 - i + 1) - else: - idx += second_idx - return idx - - def find_second_idx(self, first_val, second_val): - tol = 1e-10 - second_axis_vals = self.second_axis_vals(first_val) - second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) - return second_idx - - def unmap_first_val_to_start_line_idx(self, first_val): - tol = 1e-8 - first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] - first_idx = self._first_axis_vals.index(first_val) - idx = 0 - for i in range(self._resolution - 1): - if i != first_idx: - idx += 4 * (i + 1) - else: - return idx - for i in range(self._resolution - 1, 3 * self._resolution): - if i != first_idx: - idx += 4 * self._resolution - else: - return idx - for i in range(3 * self._resolution, 4 * self._resolution - 1): - if i != first_idx: - idx += 4 * (4 * self._resolution - 1 - i + 1) - else: - return idx - - def unmap(self, first_val, second_val): - tol = 1e-8 - first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] - first_idx = self._first_axis_vals.index(first_val) - second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] - second_idx = self.second_axis_vals(first_val).index(second_val) - healpix_index = self.axes_idx_to_healpix_idx(first_idx, second_idx) - return healpix_index - - -class OctahedralGridMapper(DatacubeMapper): - def __init__(self, base_axis, mapped_axes, resolution): - self._mapped_axes = mapped_axes - self._base_axis = base_axis - self._resolution = resolution - self._first_axis_vals = self.first_axis_vals() - self._first_idx_map = self.create_first_idx_map() - self._second_axis_spacing = {} - self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} - - def gauss_first_guess(self): - i = 0 - gvals = [ - 2.4048255577e0, - 5.5200781103e0, - 8.6537279129e0, - 11.7915344391e0, - 14.9309177086e0, - 18.0710639679e0, - 21.2116366299e0, - 24.3524715308e0, - 27.4934791320e0, - 30.6346064684e0, - 33.7758202136e0, - 36.9170983537e0, - 40.0584257646e0, - 43.1997917132e0, - 46.3411883717e0, - 49.4826098974e0, - 52.6240518411e0, - 55.7655107550e0, - 58.9069839261e0, - 62.0484691902e0, - 65.1899648002e0, - 68.3314693299e0, - 71.4729816036e0, - 74.6145006437e0, - 77.7560256304e0, - 80.8975558711e0, - 84.0390907769e0, - 87.1806298436e0, - 90.3221726372e0, - 93.4637187819e0, - 96.6052679510e0, - 99.7468198587e0, - 102.8883742542e0, - 106.0299309165e0, - 109.1714896498e0, - 112.3130502805e0, - 115.4546126537e0, - 118.5961766309e0, - 121.7377420880e0, - 124.8793089132e0, - 128.0208770059e0, - 131.1624462752e0, - 134.3040166383e0, - 137.4455880203e0, - 140.5871603528e0, - 143.7287335737e0, - 146.8703076258e0, - 150.0118824570e0, - 153.1534580192e0, - 156.2950342685e0, - ] - - numVals = len(gvals) - vals = [] - for i in range(self._resolution): - if i < numVals: - vals.append(gvals[i]) - else: - vals.append(vals[i - 1] + math.pi) - return vals - - def get_precomputed_values_N1280(self): - lats = [0] * 2560 - # lats = SortedList() - # lats = {} - lats[0] = 89.946187715665616 - lats[1] = 89.876478353332288 - lats[2] = 89.806357319542244 - lats[3] = 89.736143271609578 - lats[4] = 89.6658939412157 - lats[5] = 89.595627537554492 - lats[6] = 89.525351592371393 - lats[7] = 89.45506977912261 - lats[8] = 89.3847841013921 - lats[9] = 89.314495744374256 - lats[10] = 89.24420545380525 - lats[11] = 89.173913722284126 - lats[12] = 89.103620888238879 - lats[13] = 89.033327191845927 - lats[14] = 88.96303280826325 - lats[15] = 88.892737868230952 - lats[16] = 88.822442471310097 - lats[17] = 88.752146694650691 - lats[18] = 88.681850598961759 - lats[19] = 88.611554232668382 - lats[20] = 88.541257634868515 - lats[21] = 88.470960837474877 - lats[22] = 88.40066386679355 - lats[23] = 88.330366744702559 - lats[24] = 88.26006948954614 - lats[25] = 88.189772116820762 - lats[26] = 88.119474639706425 - lats[27] = 88.049177069484486 - lats[28] = 87.978879415867283 - lats[29] = 87.908581687261687 - lats[30] = 87.838283890981543 - lats[31] = 87.767986033419561 - lats[32] = 87.697688120188062 - lats[33] = 87.627390156234085 - lats[34] = 87.557092145935584 - lats[35] = 87.486794093180748 - lats[36] = 87.416496001434894 - lats[37] = 87.346197873795816 - lats[38] = 87.275899713041966 - lats[39] = 87.205601521672108 - lats[40] = 87.135303301939786 - lats[41] = 87.065005055882821 - lats[42] = 86.994706785348129 - lats[43] = 86.924408492014166 - lats[44] = 86.854110177408927 - lats[45] = 86.783811842927179 - lats[46] = 86.713513489844246 - lats[47] = 86.643215119328573 - lats[48] = 86.572916732453024 - lats[49] = 86.502618330203831 - lats[50] = 86.432319913489792 - lats[51] = 86.362021483149363 - lats[52] = 86.291723039957418 - lats[53] = 86.221424584631109 - lats[54] = 86.151126117835304 - lats[55] = 86.080827640187209 - lats[56] = 86.010529152260403 - lats[57] = 85.940230654588888 - lats[58] = 85.869932147670127 - lats[59] = 85.799633631968391 - lats[60] = 85.729335107917464 - lats[61] = 85.659036575922883 - lats[62] = 85.588738036364362 - lats[63] = 85.518439489597966 - lats[64] = 85.448140935957483 - lats[65] = 85.377842375756586 - lats[66] = 85.307543809290152 - lats[67] = 85.237245236835548 - lats[68] = 85.16694665865414 - lats[69] = 85.09664807499216 - lats[70] = 85.026349486081983 - lats[71] = 84.95605089214304 - lats[72] = 84.885752293382765 - lats[73] = 84.81545368999717 - lats[74] = 84.745155082171991 - lats[75] = 84.674856470082915 - lats[76] = 84.604557853896708 - lats[77] = 84.534259233771479 - lats[78] = 84.463960609857125 - lats[79] = 84.393661982296322 - lats[80] = 84.323363351224444 - lats[81] = 84.253064716770425 - lats[82] = 84.18276607905679 - lats[83] = 84.112467438200326 - lats[84] = 84.042168794312317 - lats[85] = 83.971870147498763 - lats[86] = 83.901571497860914 - lats[87] = 83.831272845495249 - lats[88] = 83.760974190494011 - lats[89] = 83.690675532945292 - lats[90] = 83.620376872933264 - lats[91] = 83.550078210538487 - lats[92] = 83.479779545838113 - lats[93] = 83.409480878905782 - lats[94] = 83.339182209812321 - lats[95] = 83.268883538625232 - lats[96] = 83.198584865409657 - lats[97] = 83.128286190227698 - lats[98] = 83.057987513139125 - lats[99] = 82.987688834201322 - lats[100] = 82.917390153469313 - lats[101] = 82.84709147099602 - lats[102] = 82.77679278683226 - lats[103] = 82.706494101026948 - lats[104] = 82.63619541362705 - lats[105] = 82.56589672467787 - lats[106] = 82.495598034222837 - lats[107] = 82.425299342304029 - lats[108] = 82.355000648961692 - lats[109] = 82.284701954234833 - lats[110] = 82.214403258160871 - lats[111] = 82.144104560776 - lats[112] = 82.073805862115165 - lats[113] = 82.003507162211946 - lats[114] = 81.933208461098829 - lats[115] = 81.862909758807191 - lats[116] = 81.792611055367345 - lats[117] = 81.722312350808508 - lats[118] = 81.652013645158945 - lats[119] = 81.581714938445955 - lats[120] = 81.511416230696042 - lats[121] = 81.441117521934686 - lats[122] = 81.370818812186627 - lats[123] = 81.300520101475826 - lats[124] = 81.230221389825374 - lats[125] = 81.159922677257711 - lats[126] = 81.089623963794551 - lats[127] = 81.019325249456955 - lats[128] = 80.949026534265244 - lats[129] = 80.878727818239184 - lats[130] = 80.808429101397948 - lats[131] = 80.73813038376008 - lats[132] = 80.667831665343556 - lats[133] = 80.59753294616587 - lats[134] = 80.527234226243991 - lats[135] = 80.456935505594302 - lats[136] = 80.386636784232863 - lats[137] = 80.316338062175078 - lats[138] = 80.246039339436052 - lats[139] = 80.175740616030438 - lats[140] = 80.105441891972376 - lats[141] = 80.035143167275749 - lats[142] = 79.9648444419539 - lats[143] = 79.894545716019948 - lats[144] = 79.824246989486554 - lats[145] = 79.753948262366038 - lats[146] = 79.683649534670437 - lats[147] = 79.61335080641139 - lats[148] = 79.543052077600308 - lats[149] = 79.472753348248219 - lats[150] = 79.402454618365894 - lats[151] = 79.332155887963822 - lats[152] = 79.261857157052191 - lats[153] = 79.191558425640977 - lats[154] = 79.121259693739859 - lats[155] = 79.050960961358285 - lats[156] = 78.980662228505423 - lats[157] = 78.910363495190211 - lats[158] = 78.840064761421445 - lats[159] = 78.769766027207638 - lats[160] = 78.699467292557102 - lats[161] = 78.629168557477882 - lats[162] = 78.558869821977908 - lats[163] = 78.488571086064923 - lats[164] = 78.418272349746417 - lats[165] = 78.347973613029708 - lats[166] = 78.277674875922045 - lats[167] = 78.207376138430348 - lats[168] = 78.137077400561424 - lats[169] = 78.066778662322022 - lats[170] = 77.996479923718596 - lats[171] = 77.926181184757539 - lats[172] = 77.855882445445019 - lats[173] = 77.785583705787161 - lats[174] = 77.71528496578982 - lats[175] = 77.644986225458879 - lats[176] = 77.574687484799924 - lats[177] = 77.504388743818524 - lats[178] = 77.434090002520122 - lats[179] = 77.363791260909963 - lats[180] = 77.293492518993247 - lats[181] = 77.22319377677502 - lats[182] = 77.15289503426024 - lats[183] = 77.082596291453768 - lats[184] = 77.012297548360323 - lats[185] = 76.941998804984564 - lats[186] = 76.871700061330955 - lats[187] = 76.801401317404 - lats[188] = 76.731102573208048 - lats[189] = 76.660803828747362 - lats[190] = 76.59050508402602 - lats[191] = 76.520206339048215 - lats[192] = 76.449907593817869 - lats[193] = 76.379608848338933 - lats[194] = 76.3093101026152 - lats[195] = 76.239011356650423 - lats[196] = 76.16871261044831 - lats[197] = 76.098413864012443 - lats[198] = 76.028115117346374 - lats[199] = 75.957816370453543 - lats[200] = 75.887517623337317 - lats[201] = 75.81721887600105 - lats[202] = 75.746920128447996 - lats[203] = 75.67662138068134 - lats[204] = 75.60632263270422 - lats[205] = 75.536023884519707 - lats[206] = 75.465725136130786 - lats[207] = 75.395426387540439 - lats[208] = 75.325127638751567 - lats[209] = 75.254828889766983 - lats[210] = 75.184530140589501 - lats[211] = 75.114231391221821 - lats[212] = 75.043932641666672 - lats[213] = 74.973633891926625 - lats[214] = 74.903335142004323 - lats[215] = 74.833036391902269 - lats[216] = 74.762737641622991 - lats[217] = 74.692438891168877 - lats[218] = 74.622140140542356 - lats[219] = 74.551841389745761 - lats[220] = 74.481542638781434 - lats[221] = 74.411243887651622 - lats[222] = 74.340945136358584 - lats[223] = 74.270646384904481 - lats[224] = 74.200347633291472 - lats[225] = 74.13004888152166 - lats[226] = 74.059750129597163 - lats[227] = 73.98945137751997 - lats[228] = 73.919152625292114 - lats[229] = 73.848853872915541 - lats[230] = 73.778555120392184 - lats[231] = 73.70825636772399 - lats[232] = 73.637957614912779 - lats[233] = 73.567658861960396 - lats[234] = 73.497360108868662 - lats[235] = 73.427061355639339 - lats[236] = 73.356762602274188 - lats[237] = 73.2864638487749 - lats[238] = 73.216165095143182 - lats[239] = 73.145866341380668 - lats[240] = 73.075567587489019 - lats[241] = 73.005268833469799 - lats[242] = 72.934970079324657 - lats[243] = 72.864671325055056 - lats[244] = 72.794372570662574 - lats[245] = 72.724073816148703 - lats[246] = 72.653775061514935 - lats[247] = 72.583476306762691 - lats[248] = 72.513177551893421 - lats[249] = 72.442878796908545 - lats[250] = 72.3725800418094 - lats[251] = 72.302281286597392 - lats[252] = 72.231982531273843 - lats[253] = 72.161683775840089 - lats[254] = 72.091385020297409 - lats[255] = 72.02108626464711 - lats[256] = 71.950787508890414 - lats[257] = 71.880488753028587 - lats[258] = 71.810189997062835 - lats[259] = 71.739891240994368 - lats[260] = 71.669592484824364 - lats[261] = 71.599293728553988 - lats[262] = 71.528994972184378 - lats[263] = 71.458696215716685 - lats[264] = 71.388397459152031 - lats[265] = 71.318098702491469 - lats[266] = 71.247799945736105 - lats[267] = 71.177501188887007 - lats[268] = 71.107202431945211 - lats[269] = 71.036903674911756 - lats[270] = 70.966604917787635 - lats[271] = 70.896306160573886 - lats[272] = 70.826007403271475 - lats[273] = 70.755708645881384 - lats[274] = 70.685409888404578 - lats[275] = 70.615111130841967 - lats[276] = 70.544812373194532 - lats[277] = 70.474513615463138 - lats[278] = 70.404214857648739 - lats[279] = 70.333916099752187 - lats[280] = 70.263617341774406 - lats[281] = 70.193318583716191 - lats[282] = 70.123019825578467 - lats[283] = 70.052721067362043 - lats[284] = 69.982422309067744 - lats[285] = 69.912123550696421 - lats[286] = 69.841824792248843 - lats[287] = 69.771526033725834 - lats[288] = 69.701227275128161 - lats[289] = 69.630928516456592 - lats[290] = 69.560629757711908 - lats[291] = 69.490330998894862 - lats[292] = 69.420032240006194 - lats[293] = 69.349733481046613 - lats[294] = 69.279434722016902 - lats[295] = 69.209135962917699 - lats[296] = 69.138837203749759 - lats[297] = 69.068538444513763 - lats[298] = 68.998239685210365 - lats[299] = 68.927940925840304 - lats[300] = 68.85764216640419 - lats[301] = 68.787343406902693 - lats[302] = 68.717044647336493 - lats[303] = 68.646745887706189 - lats[304] = 68.576447128012447 - lats[305] = 68.506148368255865 - lats[306] = 68.435849608437067 - lats[307] = 68.365550848556666 - lats[308] = 68.295252088615257 - lats[309] = 68.224953328613438 - lats[310] = 68.154654568551791 - lats[311] = 68.084355808430871 - lats[312] = 68.014057048251274 - lats[313] = 67.943758288013555 - lats[314] = 67.873459527718282 - lats[315] = 67.803160767365966 - lats[316] = 67.732862006957205 - lats[317] = 67.662563246492482 - lats[318] = 67.592264485972336 - lats[319] = 67.521965725397308 - lats[320] = 67.451666964767895 - lats[321] = 67.381368204084609 - lats[322] = 67.311069443347961 - lats[323] = 67.240770682558434 - lats[324] = 67.170471921716526 - lats[325] = 67.100173160822706 - lats[326] = 67.029874399877471 - lats[327] = 66.95957563888129 - lats[328] = 66.889276877834618 - lats[329] = 66.818978116737924 - lats[330] = 66.748679355591662 - lats[331] = 66.678380594396273 - lats[332] = 66.608081833152212 - lats[333] = 66.537783071859891 - lats[334] = 66.467484310519808 - lats[335] = 66.397185549132331 - lats[336] = 66.326886787697887 - lats[337] = 66.256588026216932 - lats[338] = 66.186289264689833 - lats[339] = 66.115990503117033 - lats[340] = 66.045691741498899 - lats[341] = 65.975392979835888 - lats[342] = 65.905094218128355 - lats[343] = 65.834795456376696 - lats[344] = 65.764496694581283 - lats[345] = 65.694197932742526 - lats[346] = 65.623899170860767 - lats[347] = 65.553600408936404 - lats[348] = 65.483301646969792 - lats[349] = 65.413002884961315 - lats[350] = 65.342704122911286 - lats[351] = 65.272405360820116 - lats[352] = 65.202106598688133 - lats[353] = 65.131807836515677 - lats[354] = 65.061509074303089 - lats[355] = 64.991210312050711 - lats[356] = 64.920911549758912 - lats[357] = 64.850612787427963 - lats[358] = 64.780314025058246 - lats[359] = 64.710015262650074 - lats[360] = 64.639716500203733 - lats[361] = 64.569417737719576 - lats[362] = 64.499118975197902 - lats[363] = 64.428820212639039 - lats[364] = 64.358521450043284 - lats[365] = 64.288222687410922 - lats[366] = 64.21792392474228 - lats[367] = 64.147625162037642 - lats[368] = 64.07732639929732 - lats[369] = 64.00702763652157 - lats[370] = 63.93672887371072 - lats[371] = 63.866430110865004 - lats[372] = 63.796131347984762 - lats[373] = 63.725832585070251 - lats[374] = 63.655533822121711 - lats[375] = 63.585235059139464 - lats[376] = 63.514936296123757 - lats[377] = 63.444637533074854 - lats[378] = 63.374338769993031 - lats[379] = 63.304040006878537 - lats[380] = 63.23374124373165 - lats[381] = 63.163442480552604 - lats[382] = 63.093143717341647 - lats[383] = 63.022844954099064 - lats[384] = 62.952546190825068 - lats[385] = 62.882247427519928 - lats[386] = 62.811948664183866 - lats[387] = 62.741649900817137 - lats[388] = 62.67135113741999 - lats[389] = 62.60105237399263 - lats[390] = 62.530753610535321 - lats[391] = 62.460454847048261 - lats[392] = 62.3901560835317 - lats[393] = 62.319857319985871 - lats[394] = 62.249558556410982 - lats[395] = 62.179259792807258 - lats[396] = 62.108961029174914 - lats[397] = 62.038662265514176 - lats[398] = 61.968363501825259 - lats[399] = 61.898064738108381 - lats[400] = 61.827765974363729 - lats[401] = 61.757467210591535 - lats[402] = 61.687168446791986 - lats[403] = 61.616869682965287 - lats[404] = 61.546570919111666 - lats[405] = 61.476272155231321 - lats[406] = 61.405973391324409 - lats[407] = 61.335674627391185 - lats[408] = 61.265375863431785 - lats[409] = 61.195077099446451 - lats[410] = 61.124778335435344 - lats[411] = 61.054479571398652 - lats[412] = 60.984180807336578 - lats[413] = 60.913882043249295 - lats[414] = 60.843583279137007 - lats[415] = 60.773284514999872 - lats[416] = 60.702985750838074 - lats[417] = 60.632686986651805 - lats[418] = 60.562388222441243 - lats[419] = 60.492089458206543 - lats[420] = 60.421790693947884 - lats[421] = 60.35149192966545 - lats[422] = 60.28119316535939 - lats[423] = 60.21089440102989 - lats[424] = 60.140595636677112 - lats[425] = 60.070296872301235 - lats[426] = 59.999998107902378 - lats[427] = 59.929699343480763 - lats[428] = 59.859400579036503 - lats[429] = 59.78910181456979 - lats[430] = 59.718803050080759 - lats[431] = 59.64850428556958 - lats[432] = 59.578205521036402 - lats[433] = 59.507906756481383 - lats[434] = 59.43760799190467 - lats[435] = 59.3673092273064 - lats[436] = 59.29701046268675 - lats[437] = 59.226711698045854 - lats[438] = 59.156412933383855 - lats[439] = 59.086114168700909 - lats[440] = 59.015815403997145 - lats[441] = 58.945516639272725 - lats[442] = 58.875217874527763 - lats[443] = 58.804919109762423 - lats[444] = 58.73462034497684 - lats[445] = 58.664321580171141 - lats[446] = 58.594022815345468 - lats[447] = 58.523724050499972 - lats[448] = 58.453425285634758 - lats[449] = 58.383126520749968 - lats[450] = 58.312827755845746 - lats[451] = 58.242528990922203 - lats[452] = 58.172230225979497 - lats[453] = 58.101931461017728 - lats[454] = 58.031632696037022 - lats[455] = 57.961333931037537 - lats[456] = 57.891035166019364 - lats[457] = 57.820736400982646 - lats[458] = 57.75043763592749 - lats[459] = 57.680138870854037 - lats[460] = 57.60984010576238 - lats[461] = 57.539541340652676 - lats[462] = 57.469242575525016 - lats[463] = 57.398943810379521 - lats[464] = 57.328645045216312 - lats[465] = 57.258346280035504 - lats[466] = 57.188047514837208 - lats[467] = 57.117748749621541 - lats[468] = 57.047449984388614 - lats[469] = 56.977151219138541 - lats[470] = 56.90685245387143 - lats[471] = 56.836553688587379 - lats[472] = 56.766254923286517 - lats[473] = 56.695956157968951 - lats[474] = 56.625657392634771 - lats[475] = 56.555358627284086 - lats[476] = 56.485059861917016 - lats[477] = 56.41476109653366 - lats[478] = 56.34446233113411 - lats[479] = 56.274163565718467 - lats[480] = 56.203864800286865 - lats[481] = 56.133566034839362 - lats[482] = 56.063267269376091 - lats[483] = 55.992968503897131 - lats[484] = 55.922669738402583 - lats[485] = 55.852370972892551 - lats[486] = 55.782072207367136 - lats[487] = 55.711773441826416 - lats[488] = 55.641474676270505 - lats[489] = 55.571175910699488 - lats[490] = 55.500877145113449 - lats[491] = 55.430578379512511 - lats[492] = 55.360279613896743 - lats[493] = 55.289980848266232 - lats[494] = 55.219682082621084 - lats[495] = 55.149383316961377 - lats[496] = 55.07908455128721 - lats[497] = 55.008785785598668 - lats[498] = 54.938487019895831 - lats[499] = 54.868188254178797 - lats[500] = 54.797889488447652 - lats[501] = 54.727590722702473 - lats[502] = 54.657291956943347 - lats[503] = 54.586993191170357 - lats[504] = 54.516694425383605 - lats[505] = 54.446395659583146 - lats[506] = 54.376096893769081 - lats[507] = 54.305798127941479 - lats[508] = 54.235499362100448 - lats[509] = 54.165200596246031 - lats[510] = 54.094901830378333 - lats[511] = 54.024603064497434 - lats[512] = 53.954304298603383 - lats[513] = 53.884005532696307 - lats[514] = 53.813706766776235 - lats[515] = 53.743408000843282 - lats[516] = 53.673109234897495 - lats[517] = 53.602810468938962 - lats[518] = 53.53251170296776 - lats[519] = 53.462212936983953 - lats[520] = 53.391914170987633 - lats[521] = 53.321615404978871 - lats[522] = 53.251316638957725 - lats[523] = 53.181017872924265 - lats[524] = 53.110719106878584 - lats[525] = 53.040420340820731 - lats[526] = 52.970121574750792 - lats[527] = 52.899822808668837 - lats[528] = 52.829524042574917 - lats[529] = 52.759225276469131 - lats[530] = 52.688926510351514 - lats[531] = 52.618627744222159 - lats[532] = 52.548328978081123 - lats[533] = 52.478030211928477 - lats[534] = 52.407731445764284 - lats[535] = 52.337432679588609 - lats[536] = 52.26713391340153 - lats[537] = 52.196835147203096 - lats[538] = 52.126536380993372 - lats[539] = 52.056237614772435 - lats[540] = 51.985938848540336 - lats[541] = 51.915640082297152 - lats[542] = 51.845341316042933 - lats[543] = 51.775042549777737 - lats[544] = 51.704743783501634 - lats[545] = 51.634445017214695 - lats[546] = 51.56414625091697 - lats[547] = 51.493847484608516 - lats[548] = 51.423548718289396 - lats[549] = 51.353249951959683 - lats[550] = 51.282951185619417 - lats[551] = 51.21265241926865 - lats[552] = 51.14235365290746 - lats[553] = 51.072054886535909 - lats[554] = 51.001756120154049 - lats[555] = 50.931457353761914 - lats[556] = 50.86115858735959 - lats[557] = 50.790859820947119 - lats[558] = 50.720561054524559 - lats[559] = 50.650262288091959 - lats[560] = 50.579963521649397 - lats[561] = 50.509664755196901 - lats[562] = 50.439365988734544 - lats[563] = 50.369067222262359 - lats[564] = 50.298768455780426 - lats[565] = 50.228469689288779 - lats[566] = 50.158170922787484 - lats[567] = 50.087872156276575 - lats[568] = 50.017573389756123 - lats[569] = 49.947274623226157 - lats[570] = 49.876975856686762 - lats[571] = 49.80667709013796 - lats[572] = 49.736378323579807 - lats[573] = 49.66607955701236 - lats[574] = 49.595780790435676 - lats[575] = 49.525482023849783 - lats[576] = 49.455183257254745 - lats[577] = 49.384884490650613 - lats[578] = 49.314585724037435 - lats[579] = 49.244286957415234 - lats[580] = 49.173988190784094 - lats[581] = 49.103689424144044 - lats[582] = 49.03339065749514 - lats[583] = 48.963091890837418 - lats[584] = 48.892793124170929 - lats[585] = 48.822494357495721 - lats[586] = 48.752195590811837 - lats[587] = 48.681896824119335 - lats[588] = 48.611598057418242 - lats[589] = 48.541299290708608 - lats[590] = 48.47100052399049 - lats[591] = 48.400701757263917 - lats[592] = 48.330402990528938 - lats[593] = 48.260104223785596 - lats[594] = 48.189805457033941 - lats[595] = 48.119506690274015 - lats[596] = 48.049207923505868 - lats[597] = 47.978909156729507 - lats[598] = 47.908610389945018 - lats[599] = 47.838311623152421 - lats[600] = 47.76801285635176 - lats[601] = 47.697714089543084 - lats[602] = 47.627415322726435 - lats[603] = 47.557116555901828 - lats[604] = 47.486817789069342 - lats[605] = 47.416519022228997 - lats[606] = 47.346220255380835 - lats[607] = 47.275921488524894 - lats[608] = 47.205622721661214 - lats[609] = 47.13532395478984 - lats[610] = 47.065025187910805 - lats[611] = 46.994726421024154 - lats[612] = 46.924427654129929 - lats[613] = 46.85412888722815 - lats[614] = 46.783830120318882 - lats[615] = 46.713531353402139 - lats[616] = 46.643232586477971 - lats[617] = 46.572933819546414 - lats[618] = 46.502635052607502 - lats[619] = 46.432336285661272 - lats[620] = 46.362037518707766 - lats[621] = 46.291738751747012 - lats[622] = 46.221439984779053 - lats[623] = 46.151141217803925 - lats[624] = 46.080842450821663 - lats[625] = 46.01054368383231 - lats[626] = 45.94024491683588 - lats[627] = 45.869946149832437 - lats[628] = 45.799647382821995 - lats[629] = 45.729348615804589 - lats[630] = 45.659049848780263 - lats[631] = 45.588751081749038 - lats[632] = 45.51845231471097 - lats[633] = 45.448153547666081 - lats[634] = 45.377854780614399 - lats[635] = 45.30755601355596 - lats[636] = 45.237257246490813 - lats[637] = 45.166958479418959 - lats[638] = 45.096659712340461 - lats[639] = 45.026360945255341 - lats[640] = 44.956062178163634 - lats[641] = 44.885763411065362 - lats[642] = 44.81546464396056 - lats[643] = 44.745165876849271 - lats[644] = 44.674867109731515 - lats[645] = 44.604568342607337 - lats[646] = 44.534269575476756 - lats[647] = 44.463970808339802 - lats[648] = 44.39367204119651 - lats[649] = 44.323373274046915 - lats[650] = 44.253074506891046 - lats[651] = 44.182775739728925 - lats[652] = 44.112476972560586 - lats[653] = 44.042178205386072 - lats[654] = 43.971879438205391 - lats[655] = 43.9015806710186 - lats[656] = 43.831281903825705 - lats[657] = 43.760983136626741 - lats[658] = 43.690684369421732 - lats[659] = 43.620385602210717 - lats[660] = 43.550086834993728 - lats[661] = 43.479788067770777 - lats[662] = 43.409489300541907 - lats[663] = 43.339190533307139 - lats[664] = 43.26889176606651 - lats[665] = 43.19859299882004 - lats[666] = 43.128294231567757 - lats[667] = 43.057995464309691 - lats[668] = 42.987696697045862 - lats[669] = 42.917397929776307 - lats[670] = 42.847099162501053 - lats[671] = 42.776800395220121 - lats[672] = 42.706501627933541 - lats[673] = 42.63620286064134 - lats[674] = 42.565904093343548 - lats[675] = 42.495605326040177 - lats[676] = 42.425306558731272 - lats[677] = 42.355007791416853 - lats[678] = 42.284709024096927 - lats[679] = 42.214410256771551 - lats[680] = 42.144111489440725 - lats[681] = 42.073812722104492 - lats[682] = 42.003513954762873 - lats[683] = 41.933215187415882 - lats[684] = 41.862916420063563 - lats[685] = 41.792617652705921 - lats[686] = 41.722318885343 - lats[687] = 41.6520201179748 - lats[688] = 41.581721350601363 - lats[689] = 41.511422583222718 - lats[690] = 41.441123815838885 - lats[691] = 41.370825048449873 - lats[692] = 41.300526281055724 - lats[693] = 41.230227513656445 - lats[694] = 41.159928746252085 - lats[695] = 41.089629978842645 - lats[696] = 41.01933121142816 - lats[697] = 40.949032444008644 - lats[698] = 40.878733676584126 - lats[699] = 40.808434909154634 - lats[700] = 40.738136141720176 - lats[701] = 40.667837374280786 - lats[702] = 40.597538606836487 - lats[703] = 40.527239839387299 - lats[704] = 40.456941071933244 - lats[705] = 40.386642304474343 - lats[706] = 40.316343537010617 - lats[707] = 40.246044769542102 - lats[708] = 40.175746002068806 - lats[709] = 40.105447234590748 - lats[710] = 40.035148467107952 - lats[711] = 39.964849699620437 - lats[712] = 39.894550932128247 - lats[713] = 39.824252164631375 - lats[714] = 39.753953397129855 - lats[715] = 39.683654629623703 - lats[716] = 39.613355862112947 - lats[717] = 39.543057094597607 - lats[718] = 39.472758327077692 - lats[719] = 39.402459559553229 - lats[720] = 39.332160792024254 - lats[721] = 39.261862024490775 - lats[722] = 39.191563256952804 - lats[723] = 39.121264489410365 - lats[724] = 39.050965721863491 - lats[725] = 38.980666954312184 - lats[726] = 38.910368186756479 - lats[727] = 38.840069419196389 - lats[728] = 38.769770651631937 - lats[729] = 38.699471884063136 - lats[730] = 38.629173116490001 - lats[731] = 38.558874348912568 - lats[732] = 38.488575581330842 - lats[733] = 38.418276813744846 - lats[734] = 38.347978046154608 - lats[735] = 38.277679278560143 - lats[736] = 38.20738051096145 - lats[737] = 38.137081743358586 - lats[738] = 38.066782975751536 - lats[739] = 37.99648420814033 - lats[740] = 37.926185440524989 - lats[741] = 37.855886672905527 - lats[742] = 37.785587905281965 - lats[743] = 37.715289137654317 - lats[744] = 37.644990370022605 - lats[745] = 37.574691602386856 - lats[746] = 37.504392834747065 - lats[747] = 37.434094067103274 - lats[748] = 37.363795299455489 - lats[749] = 37.293496531803719 - lats[750] = 37.223197764147997 - lats[751] = 37.152898996488332 - lats[752] = 37.082600228824752 - lats[753] = 37.012301461157264 - lats[754] = 36.942002693485883 - lats[755] = 36.871703925810628 - lats[756] = 36.801405158131523 - lats[757] = 36.731106390448581 - lats[758] = 36.660807622761808 - lats[759] = 36.590508855071242 - lats[760] = 36.520210087376888 - lats[761] = 36.449911319678755 - lats[762] = 36.379612551976876 - lats[763] = 36.309313784271254 - lats[764] = 36.239015016561908 - lats[765] = 36.16871624884886 - lats[766] = 36.098417481132117 - lats[767] = 36.028118713411708 - lats[768] = 35.957819945687639 - lats[769] = 35.887521177959933 - lats[770] = 35.817222410228595 - lats[771] = 35.746923642493655 - lats[772] = 35.676624874755113 - lats[773] = 35.606326107012997 - lats[774] = 35.536027339267314 - lats[775] = 35.465728571518085 - lats[776] = 35.395429803765317 - lats[777] = 35.325131036009047 - lats[778] = 35.254832268249267 - lats[779] = 35.184533500486005 - lats[780] = 35.114234732719261 - lats[781] = 35.043935964949064 - lats[782] = 34.973637197175435 - lats[783] = 34.903338429398374 - lats[784] = 34.833039661617903 - lats[785] = 34.762740893834028 - lats[786] = 34.692442126046771 - lats[787] = 34.622143358256153 - lats[788] = 34.551844590462188 - lats[789] = 34.481545822664863 - lats[790] = 34.411247054864234 - lats[791] = 34.340948287060286 - lats[792] = 34.270649519253041 - lats[793] = 34.200350751442521 - lats[794] = 34.130051983628725 - lats[795] = 34.059753215811682 - lats[796] = 33.989454447991392 - lats[797] = 33.919155680167876 - lats[798] = 33.848856912341155 - lats[799] = 33.778558144511237 - lats[800] = 33.708259376678136 - lats[801] = 33.637960608841851 - lats[802] = 33.567661841002426 - lats[803] = 33.497363073159853 - lats[804] = 33.42706430531414 - lats[805] = 33.356765537465314 - lats[806] = 33.286466769613391 - lats[807] = 33.216168001758369 - lats[808] = 33.145869233900278 - lats[809] = 33.075570466039117 - lats[810] = 33.005271698174909 - lats[811] = 32.934972930307666 - lats[812] = 32.864674162437396 - lats[813] = 32.794375394564113 - lats[814] = 32.724076626687825 - lats[815] = 32.653777858808567 - lats[816] = 32.583479090926325 - lats[817] = 32.513180323041112 - lats[818] = 32.442881555152965 - lats[819] = 32.372582787261891 - lats[820] = 32.302284019367875 - lats[821] = 32.231985251470959 - lats[822] = 32.161686483571145 - lats[823] = 32.091387715668439 - lats[824] = 32.021088947762863 - lats[825] = 31.950790179854422 - lats[826] = 31.880491411943137 - lats[827] = 31.810192644029012 - lats[828] = 31.739893876112063 - lats[829] = 31.669595108192297 - lats[830] = 31.599296340269738 - lats[831] = 31.528997572344384 - lats[832] = 31.458698804416255 - lats[833] = 31.388400036485361 - lats[834] = 31.318101268551715 - lats[835] = 31.247802500615318 - lats[836] = 31.177503732676204 - lats[837] = 31.107204964734358 - lats[838] = 31.036906196789811 - lats[839] = 30.966607428842572 - lats[840] = 30.896308660892647 - lats[841] = 30.826009892940046 - lats[842] = 30.755711124984781 - lats[843] = 30.685412357026873 - lats[844] = 30.615113589066322 - lats[845] = 30.544814821103138 - lats[846] = 30.47451605313735 - lats[847] = 30.404217285168947 - lats[848] = 30.333918517197947 - lats[849] = 30.263619749224372 - lats[850] = 30.19332098124822 - lats[851] = 30.123022213269511 - lats[852] = 30.052723445288244 - lats[853] = 29.98242467730444 - lats[854] = 29.91212590931811 - lats[855] = 29.841827141329258 - lats[856] = 29.771528373337894 - lats[857] = 29.701229605344039 - lats[858] = 29.630930837347698 - lats[859] = 29.560632069348884 - lats[860] = 29.490333301347597 - lats[861] = 29.420034533343859 - lats[862] = 29.349735765337677 - lats[863] = 29.279436997329057 - lats[864] = 29.209138229318015 - lats[865] = 29.138839461304556 - lats[866] = 29.068540693288696 - lats[867] = 28.998241925270449 - lats[868] = 28.927943157249814 - lats[869] = 28.857644389226806 - lats[870] = 28.787345621201432 - lats[871] = 28.717046853173709 - lats[872] = 28.646748085143642 - lats[873] = 28.576449317111244 - lats[874] = 28.506150549076519 - lats[875] = 28.435851781039485 - lats[876] = 28.365553013000145 - lats[877] = 28.29525424495851 - lats[878] = 28.224955476914594 - lats[879] = 28.154656708868405 - lats[880] = 28.084357940819952 - lats[881] = 28.014059172769244 - lats[882] = 27.94376040471629 - lats[883] = 27.873461636661098 - lats[884] = 27.803162868603682 - lats[885] = 27.732864100544052 - lats[886] = 27.662565332482213 - lats[887] = 27.592266564418171 - lats[888] = 27.521967796351948 - lats[889] = 27.451669028283543 - lats[890] = 27.381370260212968 - lats[891] = 27.311071492140236 - lats[892] = 27.240772724065348 - lats[893] = 27.170473955988321 - lats[894] = 27.100175187909159 - lats[895] = 27.029876419827872 - lats[896] = 26.959577651744471 - lats[897] = 26.889278883658971 - lats[898] = 26.818980115571364 - lats[899] = 26.748681347481678 - lats[900] = 26.678382579389908 - lats[901] = 26.608083811296069 - lats[902] = 26.53778504320017 - lats[903] = 26.467486275102218 - lats[904] = 26.397187507002222 - lats[905] = 26.326888738900195 - lats[906] = 26.256589970796135 - lats[907] = 26.186291202690064 - lats[908] = 26.115992434581983 - lats[909] = 26.045693666471902 - lats[910] = 25.975394898359827 - lats[911] = 25.90509613024577 - lats[912] = 25.834797362129745 - lats[913] = 25.764498594011751 - lats[914] = 25.694199825891793 - lats[915] = 25.623901057769892 - lats[916] = 25.553602289646051 - lats[917] = 25.483303521520277 - lats[918] = 25.413004753392578 - lats[919] = 25.342705985262967 - lats[920] = 25.272407217131445 - lats[921] = 25.202108448998025 - lats[922] = 25.13180968086272 - lats[923] = 25.061510912725527 - lats[924] = 24.991212144586456 - lats[925] = 24.920913376445526 - lats[926] = 24.850614608302738 - lats[927] = 24.780315840158096 - lats[928] = 24.710017072011613 - lats[929] = 24.639718303863294 - lats[930] = 24.569419535713152 - lats[931] = 24.499120767561195 - lats[932] = 24.428821999407425 - lats[933] = 24.358523231251851 - lats[934] = 24.288224463094483 - lats[935] = 24.217925694935328 - lats[936] = 24.1476269267744 - lats[937] = 24.077328158611696 - lats[938] = 24.007029390447226 - lats[939] = 23.936730622281004 - lats[940] = 23.866431854113038 - lats[941] = 23.796133085943328 - lats[942] = 23.725834317771888 - lats[943] = 23.655535549598721 - lats[944] = 23.585236781423838 - lats[945] = 23.514938013247242 - lats[946] = 23.444639245068949 - lats[947] = 23.374340476888957 - lats[948] = 23.304041708707278 - lats[949] = 23.233742940523921 - lats[950] = 23.163444172338895 - lats[951] = 23.0931454041522 - lats[952] = 23.022846635963852 - lats[953] = 22.952547867773848 - lats[954] = 22.882249099582204 - lats[955] = 22.811950331388925 - lats[956] = 22.741651563194019 - lats[957] = 22.671352794997489 - lats[958] = 22.60105402679935 - lats[959] = 22.530755258599601 - lats[960] = 22.460456490398254 - lats[961] = 22.390157722195315 - lats[962] = 22.319858953990789 - lats[963] = 22.249560185784691 - lats[964] = 22.179261417577013 - lats[965] = 22.108962649367779 - lats[966] = 22.038663881156989 - lats[967] = 21.968365112944642 - lats[968] = 21.898066344730758 - lats[969] = 21.827767576515338 - lats[970] = 21.757468808298391 - lats[971] = 21.687170040079913 - lats[972] = 21.616871271859928 - lats[973] = 21.546572503638437 - lats[974] = 21.47627373541544 - lats[975] = 21.40597496719095 - lats[976] = 21.335676198964972 - lats[977] = 21.265377430737512 - lats[978] = 21.195078662508585 - lats[979] = 21.124779894278181 - lats[980] = 21.054481126046323 - lats[981] = 20.984182357813012 - lats[982] = 20.913883589578251 - lats[983] = 20.843584821342048 - lats[984] = 20.773286053104417 - lats[985] = 20.702987284865355 - lats[986] = 20.632688516624874 - lats[987] = 20.562389748382977 - lats[988] = 20.492090980139672 - lats[989] = 20.421792211894967 - lats[990] = 20.35149344364887 - lats[991] = 20.28119467540138 - lats[992] = 20.210895907152516 - lats[993] = 20.140597138902272 - lats[994] = 20.070298370650661 - lats[995] = 19.999999602397686 - lats[996] = 19.929700834143357 - lats[997] = 19.859402065887682 - lats[998] = 19.789103297630657 - lats[999] = 19.718804529372303 - lats[1000] = 19.648505761112613 - lats[1001] = 19.578206992851602 - lats[1002] = 19.507908224589269 - lats[1003] = 19.437609456325632 - lats[1004] = 19.367310688060684 - lats[1005] = 19.297011919794439 - lats[1006] = 19.226713151526898 - lats[1007] = 19.15641438325807 - lats[1008] = 19.086115614987968 - lats[1009] = 19.015816846716586 - lats[1010] = 18.945518078443939 - lats[1011] = 18.875219310170031 - lats[1012] = 18.804920541894862 - lats[1013] = 18.734621773618446 - lats[1014] = 18.664323005340787 - lats[1015] = 18.594024237061891 - lats[1016] = 18.523725468781763 - lats[1017] = 18.453426700500408 - lats[1018] = 18.383127932217832 - lats[1019] = 18.312829163934047 - lats[1020] = 18.242530395649048 - lats[1021] = 18.172231627362851 - lats[1022] = 18.101932859075458 - lats[1023] = 18.031634090786874 - lats[1024] = 17.96133532249711 - lats[1025] = 17.89103655420616 - lats[1026] = 17.820737785914044 - lats[1027] = 17.75043901762076 - lats[1028] = 17.680140249326314 - lats[1029] = 17.60984148103071 - lats[1030] = 17.539542712733962 - lats[1031] = 17.469243944436066 - lats[1032] = 17.39894517613704 - lats[1033] = 17.328646407836878 - lats[1034] = 17.258347639535586 - lats[1035] = 17.188048871233182 - lats[1036] = 17.117750102929655 - lats[1037] = 17.04745133462502 - lats[1038] = 16.977152566319283 - lats[1039] = 16.906853798012452 - lats[1040] = 16.836555029704527 - lats[1041] = 16.766256261395515 - lats[1042] = 16.69595749308542 - lats[1043] = 16.625658724774254 - lats[1044] = 16.555359956462013 - lats[1045] = 16.485061188148713 - lats[1046] = 16.41476241983435 - lats[1047] = 16.344463651518936 - lats[1048] = 16.274164883202477 - lats[1049] = 16.203866114884974 - lats[1050] = 16.133567346566434 - lats[1051] = 16.063268578246863 - lats[1052] = 15.992969809926265 - lats[1053] = 15.922671041604652 - lats[1054] = 15.852372273282016 - lats[1055] = 15.78207350495838 - lats[1056] = 15.711774736633735 - lats[1057] = 15.641475968308091 - lats[1058] = 15.571177199981456 - lats[1059] = 15.500878431653829 - lats[1060] = 15.430579663325226 - lats[1061] = 15.360280894995643 - lats[1062] = 15.289982126665089 - lats[1063] = 15.219683358333569 - lats[1064] = 15.149384590001089 - lats[1065] = 15.07908582166765 - lats[1066] = 15.008787053333259 - lats[1067] = 14.938488284997929 - lats[1068] = 14.868189516661655 - lats[1069] = 14.797890748324447 - lats[1070] = 14.727591979986309 - lats[1071] = 14.657293211647247 - lats[1072] = 14.586994443307265 - lats[1073] = 14.516695674966371 - lats[1074] = 14.446396906624567 - lats[1075] = 14.376098138281863 - lats[1076] = 14.305799369938256 - lats[1077] = 14.23550060159376 - lats[1078] = 14.165201833248371 - lats[1079] = 14.0949030649021 - lats[1080] = 14.024604296554955 - lats[1081] = 13.954305528206934 - lats[1082] = 13.884006759858046 - lats[1083] = 13.813707991508297 - lats[1084] = 13.743409223157688 - lats[1085] = 13.673110454806226 - lats[1086] = 13.602811686453919 - lats[1087] = 13.532512918100766 - lats[1088] = 13.46221414974678 - lats[1089] = 13.391915381391959 - lats[1090] = 13.32161661303631 - lats[1091] = 13.251317844679837 - lats[1092] = 13.181019076322551 - lats[1093] = 13.110720307964451 - lats[1094] = 13.040421539605545 - lats[1095] = 12.970122771245832 - lats[1096] = 12.899824002885323 - lats[1097] = 12.829525234524022 - lats[1098] = 12.759226466161934 - lats[1099] = 12.688927697799061 - lats[1100] = 12.618628929435411 - lats[1101] = 12.548330161070988 - lats[1102] = 12.478031392705796 - lats[1103] = 12.407732624339841 - lats[1104] = 12.337433855973126 - lats[1105] = 12.267135087605659 - lats[1106] = 12.196836319237443 - lats[1107] = 12.126537550868482 - lats[1108] = 12.056238782498781 - lats[1109] = 11.985940014128348 - lats[1110] = 11.915641245757183 - lats[1111] = 11.845342477385294 - lats[1112] = 11.775043709012685 - lats[1113] = 11.704744940639358 - lats[1114] = 11.634446172265324 - lats[1115] = 11.564147403890583 - lats[1116] = 11.493848635515141 - lats[1117] = 11.423549867139002 - lats[1118] = 11.35325109876217 - lats[1119] = 11.282952330384653 - lats[1120] = 11.212653562006453 - lats[1121] = 11.142354793627575 - lats[1122] = 11.072056025248026 - lats[1123] = 11.001757256867807 - lats[1124] = 10.931458488486923 - lats[1125] = 10.861159720105382 - lats[1126] = 10.790860951723188 - lats[1127] = 10.720562183340341 - lats[1128] = 10.65026341495685 - lats[1129] = 10.579964646572719 - lats[1130] = 10.509665878187954 - lats[1131] = 10.439367109802557 - lats[1132] = 10.369068341416533 - lats[1133] = 10.298769573029887 - lats[1134] = 10.228470804642624 - lats[1135] = 10.158172036254747 - lats[1136] = 10.087873267866264 - lats[1137] = 10.017574499477174 - lats[1138] = 9.9472757310874869 - lats[1139] = 9.8769769626972046 - lats[1140] = 9.8066781943063344 - lats[1141] = 9.7363794259148779 - lats[1142] = 9.6660806575228388 - lats[1143] = 9.5957818891302242 - lats[1144] = 9.5254831207370376 - lats[1145] = 9.4551843523432826 - lats[1146] = 9.3848855839489662 - lats[1147] = 9.3145868155540921 - lats[1148] = 9.2442880471586619 - lats[1149] = 9.1739892787626829 - lats[1150] = 9.1036905103661585 - lats[1151] = 9.0333917419690941 - lats[1152] = 8.963092973571495 - lats[1153] = 8.8927942051733631 - lats[1154] = 8.8224954367747017 - lats[1155] = 8.7521966683755217 - lats[1156] = 8.6818978999758194 - lats[1157] = 8.6115991315756055 - lats[1158] = 8.5413003631748801 - lats[1159] = 8.4710015947736537 - lats[1160] = 8.4007028263719228 - lats[1161] = 8.3304040579696963 - lats[1162] = 8.2601052895669778 - lats[1163] = 8.1898065211637725 - lats[1164] = 8.1195077527600841 - lats[1165] = 8.049208984355916 - lats[1166] = 7.9789102159512737 - lats[1167] = 7.9086114475461606 - lats[1168] = 7.8383126791405831 - lats[1169] = 7.7680139107345463 - lats[1170] = 7.6977151423280494 - lats[1171] = 7.6274163739210996 - lats[1172] = 7.557117605513703 - lats[1173] = 7.4868188371058624 - lats[1174] = 7.4165200686975803 - lats[1175] = 7.3462213002888648 - lats[1176] = 7.2759225318797176 - lats[1177] = 7.2056237634701441 - lats[1178] = 7.1353249950601469 - lats[1179] = 7.0650262266497315 - lats[1180] = 6.994727458238903 - lats[1181] = 6.924428689827665 - lats[1182] = 6.8541299214160212 - lats[1183] = 6.7838311530039768 - lats[1184] = 6.7135323845915353 - lats[1185] = 6.6432336161787013 - lats[1186] = 6.5729348477654792 - lats[1187] = 6.5026360793518734 - lats[1188] = 6.4323373109378874 - lats[1189] = 6.3620385425235257 - lats[1190] = 6.2917397741087928 - lats[1191] = 6.2214410056936931 - lats[1192] = 6.151142237278231 - lats[1193] = 6.0808434688624091 - lats[1194] = 6.0105447004462347 - lats[1195] = 5.9402459320297085 - lats[1196] = 5.869947163612836 - lats[1197] = 5.7996483951956233 - lats[1198] = 5.729349626778073 - lats[1199] = 5.6590508583601888 - lats[1200] = 5.5887520899419751 - lats[1201] = 5.5184533215234373 - lats[1202] = 5.4481545531045787 - lats[1203] = 5.3778557846854023 - lats[1204] = 5.3075570162659149 - lats[1205] = 5.2372582478461194 - lats[1206] = 5.1669594794260192 - lats[1207] = 5.0966607110056197 - lats[1208] = 5.0263619425849244 - lats[1209] = 4.9560631741639369 - lats[1210] = 4.8857644057426626 - lats[1211] = 4.8154656373211049 - lats[1212] = 4.7451668688992683 - lats[1213] = 4.6748681004771564 - lats[1214] = 4.6045693320547736 - lats[1215] = 4.5342705636321252 - lats[1216] = 4.4639717952092139 - lats[1217] = 4.3936730267860451 - lats[1218] = 4.3233742583626205 - lats[1219] = 4.2530754899389471 - lats[1220] = 4.1827767215150269 - lats[1221] = 4.1124779530908659 - lats[1222] = 4.0421791846664661 - lats[1223] = 3.9718804162418326 - lats[1224] = 3.90158164781697 - lats[1225] = 3.8312828793918823 - lats[1226] = 3.7609841109665734 - lats[1227] = 3.6906853425410477 - lats[1228] = 3.6203865741153085 - lats[1229] = 3.5500878056893601 - lats[1230] = 3.4797890372632065 - lats[1231] = 3.4094902688368531 - lats[1232] = 3.339191500410303 - lats[1233] = 3.2688927319835597 - lats[1234] = 3.1985939635566285 - lats[1235] = 3.1282951951295126 - lats[1236] = 3.0579964267022164 - lats[1237] = 2.9876976582747439 - lats[1238] = 2.9173988898470999 - lats[1239] = 2.8471001214192873 - lats[1240] = 2.7768013529913107 - lats[1241] = 2.7065025845631743 - lats[1242] = 2.6362038161348824 - lats[1243] = 2.5659050477064382 - lats[1244] = 2.4956062792778466 - lats[1245] = 2.4253075108491116 - lats[1246] = 2.3550087424202366 - lats[1247] = 2.2847099739912267 - lats[1248] = 2.2144112055620848 - lats[1249] = 2.1441124371328155 - lats[1250] = 2.0738136687034232 - lats[1251] = 2.0035149002739114 - lats[1252] = 1.9332161318442849 - lats[1253] = 1.8629173634145471 - lats[1254] = 1.792618594984702 - lats[1255] = 1.7223198265547539 - lats[1256] = 1.6520210581247066 - lats[1257] = 1.5817222896945646 - lats[1258] = 1.5114235212643317 - lats[1259] = 1.4411247528340119 - lats[1260] = 1.3708259844036093 - lats[1261] = 1.300527215973128 - lats[1262] = 1.2302284475425722 - lats[1263] = 1.1599296791119456 - lats[1264] = 1.0896309106812523 - lats[1265] = 1.0193321422504964 - lats[1266] = 0.949033373819682 - lats[1267] = 0.87873460538881287 - lats[1268] = 0.80843583695789356 - lats[1269] = 0.73813706852692773 - lats[1270] = 0.66783830009591949 - lats[1271] = 0.59753953166487306 - lats[1272] = 0.52724076323379232 - lats[1273] = 0.45694199480268116 - lats[1274] = 0.3866432263715438 - lats[1275] = 0.31634445794038429 - lats[1276] = 0.24604568950920663 - lats[1277] = 0.17574692107801482 - lats[1278] = 0.10544815264681295 - lats[1279] = 0.035149384215604956 - lats[1280] = -0.035149384215604956 - lats[1281] = -0.10544815264681295 - lats[1282] = -0.17574692107801482 - lats[1283] = -0.24604568950920663 - lats[1284] = -0.31634445794038429 - lats[1285] = -0.3866432263715438 - lats[1286] = -0.45694199480268116 - lats[1287] = -0.52724076323379232 - lats[1288] = -0.59753953166487306 - lats[1289] = -0.66783830009591949 - lats[1290] = -0.73813706852692773 - lats[1291] = -0.80843583695789356 - lats[1292] = -0.87873460538881287 - lats[1293] = -0.949033373819682 - lats[1294] = -1.0193321422504964 - lats[1295] = -1.0896309106812523 - lats[1296] = -1.1599296791119456 - lats[1297] = -1.2302284475425722 - lats[1298] = -1.300527215973128 - lats[1299] = -1.3708259844036093 - lats[1300] = -1.4411247528340119 - lats[1301] = -1.5114235212643317 - lats[1302] = -1.5817222896945646 - lats[1303] = -1.6520210581247066 - lats[1304] = -1.7223198265547539 - lats[1305] = -1.792618594984702 - lats[1306] = -1.8629173634145471 - lats[1307] = -1.9332161318442849 - lats[1308] = -2.0035149002739114 - lats[1309] = -2.0738136687034232 - lats[1310] = -2.1441124371328155 - lats[1311] = -2.2144112055620848 - lats[1312] = -2.2847099739912267 - lats[1313] = -2.3550087424202366 - lats[1314] = -2.4253075108491116 - lats[1315] = -2.4956062792778466 - lats[1316] = -2.5659050477064382 - lats[1317] = -2.6362038161348824 - lats[1318] = -2.7065025845631743 - lats[1319] = -2.7768013529913107 - lats[1320] = -2.8471001214192873 - lats[1321] = -2.9173988898470999 - lats[1322] = -2.9876976582747439 - lats[1323] = -3.0579964267022164 - lats[1324] = -3.1282951951295126 - lats[1325] = -3.1985939635566285 - lats[1326] = -3.2688927319835597 - lats[1327] = -3.339191500410303 - lats[1328] = -3.4094902688368531 - lats[1329] = -3.4797890372632065 - lats[1330] = -3.5500878056893601 - lats[1331] = -3.6203865741153085 - lats[1332] = -3.6906853425410477 - lats[1333] = -3.7609841109665734 - lats[1334] = -3.8312828793918823 - lats[1335] = -3.90158164781697 - lats[1336] = -3.9718804162418326 - lats[1337] = -4.0421791846664661 - lats[1338] = -4.1124779530908659 - lats[1339] = -4.1827767215150269 - lats[1340] = -4.2530754899389471 - lats[1341] = -4.3233742583626205 - lats[1342] = -4.3936730267860451 - lats[1343] = -4.4639717952092139 - lats[1344] = -4.5342705636321252 - lats[1345] = -4.6045693320547736 - lats[1346] = -4.6748681004771564 - lats[1347] = -4.7451668688992683 - lats[1348] = -4.8154656373211049 - lats[1349] = -4.8857644057426626 - lats[1350] = -4.9560631741639369 - lats[1351] = -5.0263619425849244 - lats[1352] = -5.0966607110056197 - lats[1353] = -5.1669594794260192 - lats[1354] = -5.2372582478461194 - lats[1355] = -5.3075570162659149 - lats[1356] = -5.3778557846854023 - lats[1357] = -5.4481545531045787 - lats[1358] = -5.5184533215234373 - lats[1359] = -5.5887520899419751 - lats[1360] = -5.6590508583601888 - lats[1361] = -5.729349626778073 - lats[1362] = -5.7996483951956233 - lats[1363] = -5.869947163612836 - lats[1364] = -5.9402459320297085 - lats[1365] = -6.0105447004462347 - lats[1366] = -6.0808434688624091 - lats[1367] = -6.151142237278231 - lats[1368] = -6.2214410056936931 - lats[1369] = -6.2917397741087928 - lats[1370] = -6.3620385425235257 - lats[1371] = -6.4323373109378874 - lats[1372] = -6.5026360793518734 - lats[1373] = -6.5729348477654792 - lats[1374] = -6.6432336161787013 - lats[1375] = -6.7135323845915353 - lats[1376] = -6.7838311530039768 - lats[1377] = -6.8541299214160212 - lats[1378] = -6.924428689827665 - lats[1379] = -6.994727458238903 - lats[1380] = -7.0650262266497315 - lats[1381] = -7.1353249950601469 - lats[1382] = -7.2056237634701441 - lats[1383] = -7.2759225318797176 - lats[1384] = -7.3462213002888648 - lats[1385] = -7.4165200686975803 - lats[1386] = -7.4868188371058624 - lats[1387] = -7.557117605513703 - lats[1388] = -7.6274163739210996 - lats[1389] = -7.6977151423280494 - lats[1390] = -7.7680139107345463 - lats[1391] = -7.8383126791405831 - lats[1392] = -7.9086114475461606 - lats[1393] = -7.9789102159512737 - lats[1394] = -8.049208984355916 - lats[1395] = -8.1195077527600841 - lats[1396] = -8.1898065211637725 - lats[1397] = -8.2601052895669778 - lats[1398] = -8.3304040579696963 - lats[1399] = -8.4007028263719228 - lats[1400] = -8.4710015947736537 - lats[1401] = -8.5413003631748801 - lats[1402] = -8.6115991315756055 - lats[1403] = -8.6818978999758194 - lats[1404] = -8.7521966683755217 - lats[1405] = -8.8224954367747017 - lats[1406] = -8.8927942051733631 - lats[1407] = -8.963092973571495 - lats[1408] = -9.0333917419690941 - lats[1409] = -9.1036905103661585 - lats[1410] = -9.1739892787626829 - lats[1411] = -9.2442880471586619 - lats[1412] = -9.3145868155540921 - lats[1413] = -9.3848855839489662 - lats[1414] = -9.4551843523432826 - lats[1415] = -9.5254831207370376 - lats[1416] = -9.5957818891302242 - lats[1417] = -9.6660806575228388 - lats[1418] = -9.7363794259148779 - lats[1419] = -9.8066781943063344 - lats[1420] = -9.8769769626972046 - lats[1421] = -9.9472757310874869 - lats[1422] = -10.017574499477174 - lats[1423] = -10.087873267866264 - lats[1424] = -10.158172036254747 - lats[1425] = -10.228470804642624 - lats[1426] = -10.298769573029887 - lats[1427] = -10.369068341416533 - lats[1428] = -10.439367109802557 - lats[1429] = -10.509665878187954 - lats[1430] = -10.579964646572719 - lats[1431] = -10.65026341495685 - lats[1432] = -10.720562183340341 - lats[1433] = -10.790860951723188 - lats[1434] = -10.861159720105382 - lats[1435] = -10.931458488486923 - lats[1436] = -11.001757256867807 - lats[1437] = -11.072056025248026 - lats[1438] = -11.142354793627575 - lats[1439] = -11.212653562006453 - lats[1440] = -11.282952330384653 - lats[1441] = -11.35325109876217 - lats[1442] = -11.423549867139002 - lats[1443] = -11.493848635515141 - lats[1444] = -11.564147403890583 - lats[1445] = -11.634446172265324 - lats[1446] = -11.704744940639358 - lats[1447] = -11.775043709012685 - lats[1448] = -11.845342477385294 - lats[1449] = -11.915641245757183 - lats[1450] = -11.985940014128348 - lats[1451] = -12.056238782498781 - lats[1452] = -12.126537550868482 - lats[1453] = -12.196836319237443 - lats[1454] = -12.267135087605659 - lats[1455] = -12.337433855973126 - lats[1456] = -12.407732624339841 - lats[1457] = -12.478031392705796 - lats[1458] = -12.548330161070988 - lats[1459] = -12.618628929435411 - lats[1460] = -12.688927697799061 - lats[1461] = -12.759226466161934 - lats[1462] = -12.829525234524022 - lats[1463] = -12.899824002885323 - lats[1464] = -12.970122771245832 - lats[1465] = -13.040421539605545 - lats[1466] = -13.110720307964451 - lats[1467] = -13.181019076322551 - lats[1468] = -13.251317844679837 - lats[1469] = -13.32161661303631 - lats[1470] = -13.391915381391959 - lats[1471] = -13.46221414974678 - lats[1472] = -13.532512918100766 - lats[1473] = -13.602811686453919 - lats[1474] = -13.673110454806226 - lats[1475] = -13.743409223157688 - lats[1476] = -13.813707991508297 - lats[1477] = -13.884006759858046 - lats[1478] = -13.954305528206934 - lats[1479] = -14.024604296554955 - lats[1480] = -14.0949030649021 - lats[1481] = -14.165201833248371 - lats[1482] = -14.23550060159376 - lats[1483] = -14.305799369938256 - lats[1484] = -14.376098138281863 - lats[1485] = -14.446396906624567 - lats[1486] = -14.516695674966371 - lats[1487] = -14.586994443307265 - lats[1488] = -14.657293211647247 - lats[1489] = -14.727591979986309 - lats[1490] = -14.797890748324447 - lats[1491] = -14.868189516661655 - lats[1492] = -14.938488284997929 - lats[1493] = -15.008787053333259 - lats[1494] = -15.07908582166765 - lats[1495] = -15.149384590001089 - lats[1496] = -15.219683358333569 - lats[1497] = -15.289982126665089 - lats[1498] = -15.360280894995643 - lats[1499] = -15.430579663325226 - lats[1500] = -15.500878431653829 - lats[1501] = -15.571177199981456 - lats[1502] = -15.641475968308091 - lats[1503] = -15.711774736633735 - lats[1504] = -15.78207350495838 - lats[1505] = -15.852372273282016 - lats[1506] = -15.922671041604652 - lats[1507] = -15.992969809926265 - lats[1508] = -16.063268578246863 - lats[1509] = -16.133567346566434 - lats[1510] = -16.203866114884974 - lats[1511] = -16.274164883202477 - lats[1512] = -16.344463651518936 - lats[1513] = -16.41476241983435 - lats[1514] = -16.485061188148713 - lats[1515] = -16.555359956462013 - lats[1516] = -16.625658724774254 - lats[1517] = -16.69595749308542 - lats[1518] = -16.766256261395515 - lats[1519] = -16.836555029704527 - lats[1520] = -16.906853798012452 - lats[1521] = -16.977152566319283 - lats[1522] = -17.04745133462502 - lats[1523] = -17.117750102929655 - lats[1524] = -17.188048871233182 - lats[1525] = -17.258347639535586 - lats[1526] = -17.328646407836878 - lats[1527] = -17.39894517613704 - lats[1528] = -17.469243944436066 - lats[1529] = -17.539542712733962 - lats[1530] = -17.60984148103071 - lats[1531] = -17.680140249326314 - lats[1532] = -17.75043901762076 - lats[1533] = -17.820737785914044 - lats[1534] = -17.89103655420616 - lats[1535] = -17.96133532249711 - lats[1536] = -18.031634090786874 - lats[1537] = -18.101932859075458 - lats[1538] = -18.172231627362851 - lats[1539] = -18.242530395649048 - lats[1540] = -18.312829163934047 - lats[1541] = -18.383127932217832 - lats[1542] = -18.453426700500408 - lats[1543] = -18.523725468781763 - lats[1544] = -18.594024237061891 - lats[1545] = -18.664323005340787 - lats[1546] = -18.734621773618446 - lats[1547] = -18.804920541894862 - lats[1548] = -18.875219310170031 - lats[1549] = -18.945518078443939 - lats[1550] = -19.015816846716586 - lats[1551] = -19.086115614987968 - lats[1552] = -19.15641438325807 - lats[1553] = -19.226713151526898 - lats[1554] = -19.297011919794439 - lats[1555] = -19.367310688060684 - lats[1556] = -19.437609456325632 - lats[1557] = -19.507908224589269 - lats[1558] = -19.578206992851602 - lats[1559] = -19.648505761112613 - lats[1560] = -19.718804529372303 - lats[1561] = -19.789103297630657 - lats[1562] = -19.859402065887682 - lats[1563] = -19.929700834143357 - lats[1564] = -19.999999602397686 - lats[1565] = -20.070298370650661 - lats[1566] = -20.140597138902272 - lats[1567] = -20.210895907152516 - lats[1568] = -20.28119467540138 - lats[1569] = -20.35149344364887 - lats[1570] = -20.421792211894967 - lats[1571] = -20.492090980139672 - lats[1572] = -20.562389748382977 - lats[1573] = -20.632688516624874 - lats[1574] = -20.702987284865355 - lats[1575] = -20.773286053104417 - lats[1576] = -20.843584821342048 - lats[1577] = -20.913883589578251 - lats[1578] = -20.984182357813012 - lats[1579] = -21.054481126046323 - lats[1580] = -21.124779894278181 - lats[1581] = -21.195078662508585 - lats[1582] = -21.265377430737512 - lats[1583] = -21.335676198964972 - lats[1584] = -21.40597496719095 - lats[1585] = -21.47627373541544 - lats[1586] = -21.546572503638437 - lats[1587] = -21.616871271859928 - lats[1588] = -21.687170040079913 - lats[1589] = -21.757468808298391 - lats[1590] = -21.827767576515338 - lats[1591] = -21.898066344730758 - lats[1592] = -21.968365112944642 - lats[1593] = -22.038663881156989 - lats[1594] = -22.108962649367779 - lats[1595] = -22.179261417577013 - lats[1596] = -22.249560185784691 - lats[1597] = -22.319858953990789 - lats[1598] = -22.390157722195315 - lats[1599] = -22.460456490398254 - lats[1600] = -22.530755258599601 - lats[1601] = -22.60105402679935 - lats[1602] = -22.671352794997489 - lats[1603] = -22.741651563194019 - lats[1604] = -22.811950331388925 - lats[1605] = -22.882249099582204 - lats[1606] = -22.952547867773848 - lats[1607] = -23.022846635963852 - lats[1608] = -23.0931454041522 - lats[1609] = -23.163444172338895 - lats[1610] = -23.233742940523921 - lats[1611] = -23.304041708707278 - lats[1612] = -23.374340476888957 - lats[1613] = -23.444639245068949 - lats[1614] = -23.514938013247242 - lats[1615] = -23.585236781423838 - lats[1616] = -23.655535549598721 - lats[1617] = -23.725834317771888 - lats[1618] = -23.796133085943328 - lats[1619] = -23.866431854113038 - lats[1620] = -23.936730622281004 - lats[1621] = -24.007029390447226 - lats[1622] = -24.077328158611696 - lats[1623] = -24.1476269267744 - lats[1624] = -24.217925694935328 - lats[1625] = -24.288224463094483 - lats[1626] = -24.358523231251851 - lats[1627] = -24.428821999407425 - lats[1628] = -24.499120767561195 - lats[1629] = -24.569419535713152 - lats[1630] = -24.639718303863294 - lats[1631] = -24.710017072011613 - lats[1632] = -24.780315840158096 - lats[1633] = -24.850614608302738 - lats[1634] = -24.920913376445526 - lats[1635] = -24.991212144586456 - lats[1636] = -25.061510912725527 - lats[1637] = -25.13180968086272 - lats[1638] = -25.202108448998025 - lats[1639] = -25.272407217131445 - lats[1640] = -25.342705985262967 - lats[1641] = -25.413004753392578 - lats[1642] = -25.483303521520277 - lats[1643] = -25.553602289646051 - lats[1644] = -25.623901057769892 - lats[1645] = -25.694199825891793 - lats[1646] = -25.764498594011751 - lats[1647] = -25.834797362129745 - lats[1648] = -25.90509613024577 - lats[1649] = -25.975394898359827 - lats[1650] = -26.045693666471902 - lats[1651] = -26.115992434581983 - lats[1652] = -26.186291202690064 - lats[1653] = -26.256589970796135 - lats[1654] = -26.326888738900195 - lats[1655] = -26.397187507002222 - lats[1656] = -26.467486275102218 - lats[1657] = -26.53778504320017 - lats[1658] = -26.608083811296069 - lats[1659] = -26.678382579389908 - lats[1660] = -26.748681347481678 - lats[1661] = -26.818980115571364 - lats[1662] = -26.889278883658971 - lats[1663] = -26.959577651744471 - lats[1664] = -27.029876419827872 - lats[1665] = -27.100175187909159 - lats[1666] = -27.170473955988321 - lats[1667] = -27.240772724065348 - lats[1668] = -27.311071492140236 - lats[1669] = -27.381370260212968 - lats[1670] = -27.451669028283543 - lats[1671] = -27.521967796351948 - lats[1672] = -27.592266564418171 - lats[1673] = -27.662565332482213 - lats[1674] = -27.732864100544052 - lats[1675] = -27.803162868603682 - lats[1676] = -27.873461636661098 - lats[1677] = -27.94376040471629 - lats[1678] = -28.014059172769244 - lats[1679] = -28.084357940819952 - lats[1680] = -28.154656708868405 - lats[1681] = -28.224955476914594 - lats[1682] = -28.29525424495851 - lats[1683] = -28.365553013000145 - lats[1684] = -28.435851781039485 - lats[1685] = -28.506150549076519 - lats[1686] = -28.576449317111244 - lats[1687] = -28.646748085143642 - lats[1688] = -28.717046853173709 - lats[1689] = -28.787345621201432 - lats[1690] = -28.857644389226806 - lats[1691] = -28.927943157249814 - lats[1692] = -28.998241925270449 - lats[1693] = -29.068540693288696 - lats[1694] = -29.138839461304556 - lats[1695] = -29.209138229318015 - lats[1696] = -29.279436997329057 - lats[1697] = -29.349735765337677 - lats[1698] = -29.420034533343859 - lats[1699] = -29.490333301347597 - lats[1700] = -29.560632069348884 - lats[1701] = -29.630930837347698 - lats[1702] = -29.701229605344039 - lats[1703] = -29.771528373337894 - lats[1704] = -29.841827141329258 - lats[1705] = -29.91212590931811 - lats[1706] = -29.98242467730444 - lats[1707] = -30.052723445288244 - lats[1708] = -30.123022213269511 - lats[1709] = -30.19332098124822 - lats[1710] = -30.263619749224372 - lats[1711] = -30.333918517197947 - lats[1712] = -30.404217285168947 - lats[1713] = -30.47451605313735 - lats[1714] = -30.544814821103138 - lats[1715] = -30.615113589066322 - lats[1716] = -30.685412357026873 - lats[1717] = -30.755711124984781 - lats[1718] = -30.826009892940046 - lats[1719] = -30.896308660892647 - lats[1720] = -30.966607428842572 - lats[1721] = -31.036906196789811 - lats[1722] = -31.107204964734358 - lats[1723] = -31.177503732676204 - lats[1724] = -31.247802500615318 - lats[1725] = -31.318101268551715 - lats[1726] = -31.388400036485361 - lats[1727] = -31.458698804416255 - lats[1728] = -31.528997572344384 - lats[1729] = -31.599296340269738 - lats[1730] = -31.669595108192297 - lats[1731] = -31.739893876112063 - lats[1732] = -31.810192644029012 - lats[1733] = -31.880491411943137 - lats[1734] = -31.950790179854422 - lats[1735] = -32.021088947762863 - lats[1736] = -32.091387715668439 - lats[1737] = -32.161686483571145 - lats[1738] = -32.231985251470959 - lats[1739] = -32.302284019367875 - lats[1740] = -32.372582787261891 - lats[1741] = -32.442881555152965 - lats[1742] = -32.513180323041112 - lats[1743] = -32.583479090926325 - lats[1744] = -32.653777858808567 - lats[1745] = -32.724076626687825 - lats[1746] = -32.794375394564113 - lats[1747] = -32.864674162437396 - lats[1748] = -32.934972930307666 - lats[1749] = -33.005271698174909 - lats[1750] = -33.075570466039117 - lats[1751] = -33.145869233900278 - lats[1752] = -33.216168001758369 - lats[1753] = -33.286466769613391 - lats[1754] = -33.356765537465314 - lats[1755] = -33.42706430531414 - lats[1756] = -33.497363073159853 - lats[1757] = -33.567661841002426 - lats[1758] = -33.637960608841851 - lats[1759] = -33.708259376678136 - lats[1760] = -33.778558144511237 - lats[1761] = -33.848856912341155 - lats[1762] = -33.919155680167876 - lats[1763] = -33.989454447991392 - lats[1764] = -34.059753215811682 - lats[1765] = -34.130051983628725 - lats[1766] = -34.200350751442521 - lats[1767] = -34.270649519253041 - lats[1768] = -34.340948287060286 - lats[1769] = -34.411247054864234 - lats[1770] = -34.481545822664863 - lats[1771] = -34.551844590462188 - lats[1772] = -34.622143358256153 - lats[1773] = -34.692442126046771 - lats[1774] = -34.762740893834028 - lats[1775] = -34.833039661617903 - lats[1776] = -34.903338429398374 - lats[1777] = -34.973637197175435 - lats[1778] = -35.043935964949064 - lats[1779] = -35.114234732719261 - lats[1780] = -35.184533500486005 - lats[1781] = -35.254832268249267 - lats[1782] = -35.325131036009047 - lats[1783] = -35.395429803765317 - lats[1784] = -35.465728571518085 - lats[1785] = -35.536027339267314 - lats[1786] = -35.606326107012997 - lats[1787] = -35.676624874755113 - lats[1788] = -35.746923642493655 - lats[1789] = -35.817222410228595 - lats[1790] = -35.887521177959933 - lats[1791] = -35.957819945687639 - lats[1792] = -36.028118713411708 - lats[1793] = -36.098417481132117 - lats[1794] = -36.16871624884886 - lats[1795] = -36.239015016561908 - lats[1796] = -36.309313784271254 - lats[1797] = -36.379612551976876 - lats[1798] = -36.449911319678755 - lats[1799] = -36.520210087376888 - lats[1800] = -36.590508855071242 - lats[1801] = -36.660807622761808 - lats[1802] = -36.731106390448581 - lats[1803] = -36.801405158131523 - lats[1804] = -36.871703925810628 - lats[1805] = -36.942002693485883 - lats[1806] = -37.012301461157264 - lats[1807] = -37.082600228824752 - lats[1808] = -37.152898996488332 - lats[1809] = -37.223197764147997 - lats[1810] = -37.293496531803719 - lats[1811] = -37.363795299455489 - lats[1812] = -37.434094067103274 - lats[1813] = -37.504392834747065 - lats[1814] = -37.574691602386856 - lats[1815] = -37.644990370022605 - lats[1816] = -37.715289137654317 - lats[1817] = -37.785587905281965 - lats[1818] = -37.855886672905527 - lats[1819] = -37.926185440524989 - lats[1820] = -37.99648420814033 - lats[1821] = -38.066782975751536 - lats[1822] = -38.137081743358586 - lats[1823] = -38.20738051096145 - lats[1824] = -38.277679278560143 - lats[1825] = -38.347978046154608 - lats[1826] = -38.418276813744846 - lats[1827] = -38.488575581330842 - lats[1828] = -38.558874348912568 - lats[1829] = -38.629173116490001 - lats[1830] = -38.699471884063136 - lats[1831] = -38.769770651631937 - lats[1832] = -38.840069419196389 - lats[1833] = -38.910368186756479 - lats[1834] = -38.980666954312184 - lats[1835] = -39.050965721863491 - lats[1836] = -39.121264489410365 - lats[1837] = -39.191563256952804 - lats[1838] = -39.261862024490775 - lats[1839] = -39.332160792024254 - lats[1840] = -39.402459559553229 - lats[1841] = -39.472758327077692 - lats[1842] = -39.543057094597607 - lats[1843] = -39.613355862112947 - lats[1844] = -39.683654629623703 - lats[1845] = -39.753953397129855 - lats[1846] = -39.824252164631375 - lats[1847] = -39.894550932128247 - lats[1848] = -39.964849699620437 - lats[1849] = -40.035148467107952 - lats[1850] = -40.105447234590748 - lats[1851] = -40.175746002068806 - lats[1852] = -40.246044769542102 - lats[1853] = -40.316343537010617 - lats[1854] = -40.386642304474343 - lats[1855] = -40.456941071933244 - lats[1856] = -40.527239839387299 - lats[1857] = -40.597538606836487 - lats[1858] = -40.667837374280786 - lats[1859] = -40.738136141720176 - lats[1860] = -40.808434909154634 - lats[1861] = -40.878733676584126 - lats[1862] = -40.949032444008644 - lats[1863] = -41.01933121142816 - lats[1864] = -41.089629978842645 - lats[1865] = -41.159928746252085 - lats[1866] = -41.230227513656445 - lats[1867] = -41.300526281055724 - lats[1868] = -41.370825048449873 - lats[1869] = -41.441123815838885 - lats[1870] = -41.511422583222718 - lats[1871] = -41.581721350601363 - lats[1872] = -41.6520201179748 - lats[1873] = -41.722318885343 - lats[1874] = -41.792617652705921 - lats[1875] = -41.862916420063563 - lats[1876] = -41.933215187415882 - lats[1877] = -42.003513954762873 - lats[1878] = -42.073812722104492 - lats[1879] = -42.144111489440725 - lats[1880] = -42.214410256771551 - lats[1881] = -42.284709024096927 - lats[1882] = -42.355007791416853 - lats[1883] = -42.425306558731272 - lats[1884] = -42.495605326040177 - lats[1885] = -42.565904093343548 - lats[1886] = -42.63620286064134 - lats[1887] = -42.706501627933541 - lats[1888] = -42.776800395220121 - lats[1889] = -42.847099162501053 - lats[1890] = -42.917397929776307 - lats[1891] = -42.987696697045862 - lats[1892] = -43.057995464309691 - lats[1893] = -43.128294231567757 - lats[1894] = -43.19859299882004 - lats[1895] = -43.26889176606651 - lats[1896] = -43.339190533307139 - lats[1897] = -43.409489300541907 - lats[1898] = -43.479788067770777 - lats[1899] = -43.550086834993728 - lats[1900] = -43.620385602210717 - lats[1901] = -43.690684369421732 - lats[1902] = -43.760983136626741 - lats[1903] = -43.831281903825705 - lats[1904] = -43.9015806710186 - lats[1905] = -43.971879438205391 - lats[1906] = -44.042178205386072 - lats[1907] = -44.112476972560586 - lats[1908] = -44.182775739728925 - lats[1909] = -44.253074506891046 - lats[1910] = -44.323373274046915 - lats[1911] = -44.39367204119651 - lats[1912] = -44.463970808339802 - lats[1913] = -44.534269575476756 - lats[1914] = -44.604568342607337 - lats[1915] = -44.674867109731515 - lats[1916] = -44.745165876849271 - lats[1917] = -44.81546464396056 - lats[1918] = -44.885763411065362 - lats[1919] = -44.956062178163634 - lats[1920] = -45.026360945255341 - lats[1921] = -45.096659712340461 - lats[1922] = -45.166958479418959 - lats[1923] = -45.237257246490813 - lats[1924] = -45.30755601355596 - lats[1925] = -45.377854780614399 - lats[1926] = -45.448153547666081 - lats[1927] = -45.51845231471097 - lats[1928] = -45.588751081749038 - lats[1929] = -45.659049848780263 - lats[1930] = -45.729348615804589 - lats[1931] = -45.799647382821995 - lats[1932] = -45.869946149832437 - lats[1933] = -45.94024491683588 - lats[1934] = -46.01054368383231 - lats[1935] = -46.080842450821663 - lats[1936] = -46.151141217803925 - lats[1937] = -46.221439984779053 - lats[1938] = -46.291738751747012 - lats[1939] = -46.362037518707766 - lats[1940] = -46.432336285661272 - lats[1941] = -46.502635052607502 - lats[1942] = -46.572933819546414 - lats[1943] = -46.643232586477971 - lats[1944] = -46.713531353402139 - lats[1945] = -46.783830120318882 - lats[1946] = -46.85412888722815 - lats[1947] = -46.924427654129929 - lats[1948] = -46.994726421024154 - lats[1949] = -47.065025187910805 - lats[1950] = -47.13532395478984 - lats[1951] = -47.205622721661214 - lats[1952] = -47.275921488524894 - lats[1953] = -47.346220255380835 - lats[1954] = -47.416519022228997 - lats[1955] = -47.486817789069342 - lats[1956] = -47.557116555901828 - lats[1957] = -47.627415322726435 - lats[1958] = -47.697714089543084 - lats[1959] = -47.76801285635176 - lats[1960] = -47.838311623152421 - lats[1961] = -47.908610389945018 - lats[1962] = -47.978909156729507 - lats[1963] = -48.049207923505868 - lats[1964] = -48.119506690274015 - lats[1965] = -48.189805457033941 - lats[1966] = -48.260104223785596 - lats[1967] = -48.330402990528938 - lats[1968] = -48.400701757263917 - lats[1969] = -48.47100052399049 - lats[1970] = -48.541299290708608 - lats[1971] = -48.611598057418242 - lats[1972] = -48.681896824119335 - lats[1973] = -48.752195590811837 - lats[1974] = -48.822494357495721 - lats[1975] = -48.892793124170929 - lats[1976] = -48.963091890837418 - lats[1977] = -49.03339065749514 - lats[1978] = -49.103689424144044 - lats[1979] = -49.173988190784094 - lats[1980] = -49.244286957415234 - lats[1981] = -49.314585724037435 - lats[1982] = -49.384884490650613 - lats[1983] = -49.455183257254745 - lats[1984] = -49.525482023849783 - lats[1985] = -49.595780790435676 - lats[1986] = -49.66607955701236 - lats[1987] = -49.736378323579807 - lats[1988] = -49.80667709013796 - lats[1989] = -49.876975856686762 - lats[1990] = -49.947274623226157 - lats[1991] = -50.017573389756123 - lats[1992] = -50.087872156276575 - lats[1993] = -50.158170922787484 - lats[1994] = -50.228469689288779 - lats[1995] = -50.298768455780426 - lats[1996] = -50.369067222262359 - lats[1997] = -50.439365988734544 - lats[1998] = -50.509664755196901 - lats[1999] = -50.579963521649397 - lats[2000] = -50.650262288091959 - lats[2001] = -50.720561054524559 - lats[2002] = -50.790859820947119 - lats[2003] = -50.86115858735959 - lats[2004] = -50.931457353761914 - lats[2005] = -51.001756120154049 - lats[2006] = -51.072054886535909 - lats[2007] = -51.14235365290746 - lats[2008] = -51.21265241926865 - lats[2009] = -51.282951185619417 - lats[2010] = -51.353249951959683 - lats[2011] = -51.423548718289396 - lats[2012] = -51.493847484608516 - lats[2013] = -51.56414625091697 - lats[2014] = -51.634445017214695 - lats[2015] = -51.704743783501634 - lats[2016] = -51.775042549777737 - lats[2017] = -51.845341316042933 - lats[2018] = -51.915640082297152 - lats[2019] = -51.985938848540336 - lats[2020] = -52.056237614772435 - lats[2021] = -52.126536380993372 - lats[2022] = -52.196835147203096 - lats[2023] = -52.26713391340153 - lats[2024] = -52.337432679588609 - lats[2025] = -52.407731445764284 - lats[2026] = -52.478030211928477 - lats[2027] = -52.548328978081123 - lats[2028] = -52.618627744222159 - lats[2029] = -52.688926510351514 - lats[2030] = -52.759225276469131 - lats[2031] = -52.829524042574917 - lats[2032] = -52.899822808668837 - lats[2033] = -52.970121574750792 - lats[2034] = -53.040420340820731 - lats[2035] = -53.110719106878584 - lats[2036] = -53.181017872924265 - lats[2037] = -53.251316638957725 - lats[2038] = -53.321615404978871 - lats[2039] = -53.391914170987633 - lats[2040] = -53.462212936983953 - lats[2041] = -53.53251170296776 - lats[2042] = -53.602810468938962 - lats[2043] = -53.673109234897495 - lats[2044] = -53.743408000843282 - lats[2045] = -53.813706766776235 - lats[2046] = -53.884005532696307 - lats[2047] = -53.954304298603383 - lats[2048] = -54.024603064497434 - lats[2049] = -54.094901830378333 - lats[2050] = -54.165200596246031 - lats[2051] = -54.235499362100448 - lats[2052] = -54.305798127941479 - lats[2053] = -54.376096893769081 - lats[2054] = -54.446395659583146 - lats[2055] = -54.516694425383605 - lats[2056] = -54.586993191170357 - lats[2057] = -54.657291956943347 - lats[2058] = -54.727590722702473 - lats[2059] = -54.797889488447652 - lats[2060] = -54.868188254178797 - lats[2061] = -54.938487019895831 - lats[2062] = -55.008785785598668 - lats[2063] = -55.07908455128721 - lats[2064] = -55.149383316961377 - lats[2065] = -55.219682082621084 - lats[2066] = -55.289980848266232 - lats[2067] = -55.360279613896743 - lats[2068] = -55.430578379512511 - lats[2069] = -55.500877145113449 - lats[2070] = -55.571175910699488 - lats[2071] = -55.641474676270505 - lats[2072] = -55.711773441826416 - lats[2073] = -55.782072207367136 - lats[2074] = -55.852370972892551 - lats[2075] = -55.922669738402583 - lats[2076] = -55.992968503897131 - lats[2077] = -56.063267269376091 - lats[2078] = -56.133566034839362 - lats[2079] = -56.203864800286865 - lats[2080] = -56.274163565718467 - lats[2081] = -56.34446233113411 - lats[2082] = -56.41476109653366 - lats[2083] = -56.485059861917016 - lats[2084] = -56.555358627284086 - lats[2085] = -56.625657392634771 - lats[2086] = -56.695956157968951 - lats[2087] = -56.766254923286517 - lats[2088] = -56.836553688587379 - lats[2089] = -56.90685245387143 - lats[2090] = -56.977151219138541 - lats[2091] = -57.047449984388614 - lats[2092] = -57.117748749621541 - lats[2093] = -57.188047514837208 - lats[2094] = -57.258346280035504 - lats[2095] = -57.328645045216312 - lats[2096] = -57.398943810379521 - lats[2097] = -57.469242575525016 - lats[2098] = -57.539541340652676 - lats[2099] = -57.60984010576238 - lats[2100] = -57.680138870854037 - lats[2101] = -57.75043763592749 - lats[2102] = -57.820736400982646 - lats[2103] = -57.891035166019364 - lats[2104] = -57.961333931037537 - lats[2105] = -58.031632696037022 - lats[2106] = -58.101931461017728 - lats[2107] = -58.172230225979497 - lats[2108] = -58.242528990922203 - lats[2109] = -58.312827755845746 - lats[2110] = -58.383126520749968 - lats[2111] = -58.453425285634758 - lats[2112] = -58.523724050499972 - lats[2113] = -58.594022815345468 - lats[2114] = -58.664321580171141 - lats[2115] = -58.73462034497684 - lats[2116] = -58.804919109762423 - lats[2117] = -58.875217874527763 - lats[2118] = -58.945516639272725 - lats[2119] = -59.015815403997145 - lats[2120] = -59.086114168700909 - lats[2121] = -59.156412933383855 - lats[2122] = -59.226711698045854 - lats[2123] = -59.29701046268675 - lats[2124] = -59.3673092273064 - lats[2125] = -59.43760799190467 - lats[2126] = -59.507906756481383 - lats[2127] = -59.578205521036402 - lats[2128] = -59.64850428556958 - lats[2129] = -59.718803050080759 - lats[2130] = -59.78910181456979 - lats[2131] = -59.859400579036503 - lats[2132] = -59.929699343480763 - lats[2133] = -59.999998107902378 - lats[2134] = -60.070296872301235 - lats[2135] = -60.140595636677112 - lats[2136] = -60.21089440102989 - lats[2137] = -60.28119316535939 - lats[2138] = -60.35149192966545 - lats[2139] = -60.421790693947884 - lats[2140] = -60.492089458206543 - lats[2141] = -60.562388222441243 - lats[2142] = -60.632686986651805 - lats[2143] = -60.702985750838074 - lats[2144] = -60.773284514999872 - lats[2145] = -60.843583279137007 - lats[2146] = -60.913882043249295 - lats[2147] = -60.984180807336578 - lats[2148] = -61.054479571398652 - lats[2149] = -61.124778335435344 - lats[2150] = -61.195077099446451 - lats[2151] = -61.265375863431785 - lats[2152] = -61.335674627391185 - lats[2153] = -61.405973391324409 - lats[2154] = -61.476272155231321 - lats[2155] = -61.546570919111666 - lats[2156] = -61.616869682965287 - lats[2157] = -61.687168446791986 - lats[2158] = -61.757467210591535 - lats[2159] = -61.827765974363729 - lats[2160] = -61.898064738108381 - lats[2161] = -61.968363501825259 - lats[2162] = -62.038662265514176 - lats[2163] = -62.108961029174914 - lats[2164] = -62.179259792807258 - lats[2165] = -62.249558556410982 - lats[2166] = -62.319857319985871 - lats[2167] = -62.3901560835317 - lats[2168] = -62.460454847048261 - lats[2169] = -62.530753610535321 - lats[2170] = -62.60105237399263 - lats[2171] = -62.67135113741999 - lats[2172] = -62.741649900817137 - lats[2173] = -62.811948664183866 - lats[2174] = -62.882247427519928 - lats[2175] = -62.952546190825068 - lats[2176] = -63.022844954099064 - lats[2177] = -63.093143717341647 - lats[2178] = -63.163442480552604 - lats[2179] = -63.23374124373165 - lats[2180] = -63.304040006878537 - lats[2181] = -63.374338769993031 - lats[2182] = -63.444637533074854 - lats[2183] = -63.514936296123757 - lats[2184] = -63.585235059139464 - lats[2185] = -63.655533822121711 - lats[2186] = -63.725832585070251 - lats[2187] = -63.796131347984762 - lats[2188] = -63.866430110865004 - lats[2189] = -63.93672887371072 - lats[2190] = -64.00702763652157 - lats[2191] = -64.07732639929732 - lats[2192] = -64.147625162037642 - lats[2193] = -64.21792392474228 - lats[2194] = -64.288222687410922 - lats[2195] = -64.358521450043284 - lats[2196] = -64.428820212639039 - lats[2197] = -64.499118975197902 - lats[2198] = -64.569417737719576 - lats[2199] = -64.639716500203733 - lats[2200] = -64.710015262650074 - lats[2201] = -64.780314025058246 - lats[2202] = -64.850612787427963 - lats[2203] = -64.920911549758912 - lats[2204] = -64.991210312050711 - lats[2205] = -65.061509074303089 - lats[2206] = -65.131807836515677 - lats[2207] = -65.202106598688133 - lats[2208] = -65.272405360820116 - lats[2209] = -65.342704122911286 - lats[2210] = -65.413002884961315 - lats[2211] = -65.483301646969792 - lats[2212] = -65.553600408936404 - lats[2213] = -65.623899170860767 - lats[2214] = -65.694197932742526 - lats[2215] = -65.764496694581283 - lats[2216] = -65.834795456376696 - lats[2217] = -65.905094218128355 - lats[2218] = -65.975392979835888 - lats[2219] = -66.045691741498899 - lats[2220] = -66.115990503117033 - lats[2221] = -66.186289264689833 - lats[2222] = -66.256588026216932 - lats[2223] = -66.326886787697887 - lats[2224] = -66.397185549132331 - lats[2225] = -66.467484310519808 - lats[2226] = -66.537783071859891 - lats[2227] = -66.608081833152212 - lats[2228] = -66.678380594396273 - lats[2229] = -66.748679355591662 - lats[2230] = -66.818978116737924 - lats[2231] = -66.889276877834618 - lats[2232] = -66.95957563888129 - lats[2233] = -67.029874399877471 - lats[2234] = -67.100173160822706 - lats[2235] = -67.170471921716526 - lats[2236] = -67.240770682558434 - lats[2237] = -67.311069443347961 - lats[2238] = -67.381368204084609 - lats[2239] = -67.451666964767895 - lats[2240] = -67.521965725397308 - lats[2241] = -67.592264485972336 - lats[2242] = -67.662563246492482 - lats[2243] = -67.732862006957205 - lats[2244] = -67.803160767365966 - lats[2245] = -67.873459527718282 - lats[2246] = -67.943758288013555 - lats[2247] = -68.014057048251274 - lats[2248] = -68.084355808430871 - lats[2249] = -68.154654568551791 - lats[2250] = -68.224953328613438 - lats[2251] = -68.295252088615257 - lats[2252] = -68.365550848556666 - lats[2253] = -68.435849608437067 - lats[2254] = -68.506148368255865 - lats[2255] = -68.576447128012447 - lats[2256] = -68.646745887706189 - lats[2257] = -68.717044647336493 - lats[2258] = -68.787343406902693 - lats[2259] = -68.85764216640419 - lats[2260] = -68.927940925840304 - lats[2261] = -68.998239685210365 - lats[2262] = -69.068538444513763 - lats[2263] = -69.138837203749759 - lats[2264] = -69.209135962917699 - lats[2265] = -69.279434722016902 - lats[2266] = -69.349733481046613 - lats[2267] = -69.420032240006194 - lats[2268] = -69.490330998894862 - lats[2269] = -69.560629757711908 - lats[2270] = -69.630928516456592 - lats[2271] = -69.701227275128161 - lats[2272] = -69.771526033725834 - lats[2273] = -69.841824792248843 - lats[2274] = -69.912123550696421 - lats[2275] = -69.982422309067744 - lats[2276] = -70.052721067362043 - lats[2277] = -70.123019825578467 - lats[2278] = -70.193318583716191 - lats[2279] = -70.263617341774406 - lats[2280] = -70.333916099752187 - lats[2281] = -70.404214857648739 - lats[2282] = -70.474513615463138 - lats[2283] = -70.544812373194532 - lats[2284] = -70.615111130841967 - lats[2285] = -70.685409888404578 - lats[2286] = -70.755708645881384 - lats[2287] = -70.826007403271475 - lats[2288] = -70.896306160573886 - lats[2289] = -70.966604917787635 - lats[2290] = -71.036903674911756 - lats[2291] = -71.107202431945211 - lats[2292] = -71.177501188887007 - lats[2293] = -71.247799945736105 - lats[2294] = -71.318098702491469 - lats[2295] = -71.388397459152031 - lats[2296] = -71.458696215716685 - lats[2297] = -71.528994972184378 - lats[2298] = -71.599293728553988 - lats[2299] = -71.669592484824364 - lats[2300] = -71.739891240994368 - lats[2301] = -71.810189997062835 - lats[2302] = -71.880488753028587 - lats[2303] = -71.950787508890414 - lats[2304] = -72.02108626464711 - lats[2305] = -72.091385020297409 - lats[2306] = -72.161683775840089 - lats[2307] = -72.231982531273843 - lats[2308] = -72.302281286597392 - lats[2309] = -72.3725800418094 - lats[2310] = -72.442878796908545 - lats[2311] = -72.513177551893421 - lats[2312] = -72.583476306762691 - lats[2313] = -72.653775061514935 - lats[2314] = -72.724073816148703 - lats[2315] = -72.794372570662574 - lats[2316] = -72.864671325055056 - lats[2317] = -72.934970079324657 - lats[2318] = -73.005268833469799 - lats[2319] = -73.075567587489019 - lats[2320] = -73.145866341380668 - lats[2321] = -73.216165095143182 - lats[2322] = -73.2864638487749 - lats[2323] = -73.356762602274188 - lats[2324] = -73.427061355639339 - lats[2325] = -73.497360108868662 - lats[2326] = -73.567658861960396 - lats[2327] = -73.637957614912779 - lats[2328] = -73.70825636772399 - lats[2329] = -73.778555120392184 - lats[2330] = -73.848853872915541 - lats[2331] = -73.919152625292114 - lats[2332] = -73.98945137751997 - lats[2333] = -74.059750129597163 - lats[2334] = -74.13004888152166 - lats[2335] = -74.200347633291472 - lats[2336] = -74.270646384904481 - lats[2337] = -74.340945136358584 - lats[2338] = -74.411243887651622 - lats[2339] = -74.481542638781434 - lats[2340] = -74.551841389745761 - lats[2341] = -74.622140140542356 - lats[2342] = -74.692438891168877 - lats[2343] = -74.762737641622991 - lats[2344] = -74.833036391902269 - lats[2345] = -74.903335142004323 - lats[2346] = -74.973633891926625 - lats[2347] = -75.043932641666672 - lats[2348] = -75.114231391221821 - lats[2349] = -75.184530140589501 - lats[2350] = -75.254828889766983 - lats[2351] = -75.325127638751567 - lats[2352] = -75.395426387540439 - lats[2353] = -75.465725136130786 - lats[2354] = -75.536023884519707 - lats[2355] = -75.60632263270422 - lats[2356] = -75.67662138068134 - lats[2357] = -75.746920128447996 - lats[2358] = -75.81721887600105 - lats[2359] = -75.887517623337317 - lats[2360] = -75.957816370453543 - lats[2361] = -76.028115117346374 - lats[2362] = -76.098413864012443 - lats[2363] = -76.16871261044831 - lats[2364] = -76.239011356650423 - lats[2365] = -76.3093101026152 - lats[2366] = -76.379608848338933 - lats[2367] = -76.449907593817869 - lats[2368] = -76.520206339048215 - lats[2369] = -76.59050508402602 - lats[2370] = -76.660803828747362 - lats[2371] = -76.731102573208048 - lats[2372] = -76.801401317404 - lats[2373] = -76.871700061330955 - lats[2374] = -76.941998804984564 - lats[2375] = -77.012297548360323 - lats[2376] = -77.082596291453768 - lats[2377] = -77.15289503426024 - lats[2378] = -77.22319377677502 - lats[2379] = -77.293492518993247 - lats[2380] = -77.363791260909963 - lats[2381] = -77.434090002520122 - lats[2382] = -77.504388743818524 - lats[2383] = -77.574687484799924 - lats[2384] = -77.644986225458879 - lats[2385] = -77.71528496578982 - lats[2386] = -77.785583705787161 - lats[2387] = -77.855882445445019 - lats[2388] = -77.926181184757539 - lats[2389] = -77.996479923718596 - lats[2390] = -78.066778662322022 - lats[2391] = -78.137077400561424 - lats[2392] = -78.207376138430348 - lats[2393] = -78.277674875922045 - lats[2394] = -78.347973613029708 - lats[2395] = -78.418272349746417 - lats[2396] = -78.488571086064923 - lats[2397] = -78.558869821977908 - lats[2398] = -78.629168557477882 - lats[2399] = -78.699467292557102 - lats[2400] = -78.769766027207638 - lats[2401] = -78.840064761421445 - lats[2402] = -78.910363495190211 - lats[2403] = -78.980662228505423 - lats[2404] = -79.050960961358285 - lats[2405] = -79.121259693739859 - lats[2406] = -79.191558425640977 - lats[2407] = -79.261857157052191 - lats[2408] = -79.332155887963822 - lats[2409] = -79.402454618365894 - lats[2410] = -79.472753348248219 - lats[2411] = -79.543052077600308 - lats[2412] = -79.61335080641139 - lats[2413] = -79.683649534670437 - lats[2414] = -79.753948262366038 - lats[2415] = -79.824246989486554 - lats[2416] = -79.894545716019948 - lats[2417] = -79.9648444419539 - lats[2418] = -80.035143167275749 - lats[2419] = -80.105441891972376 - lats[2420] = -80.175740616030438 - lats[2421] = -80.246039339436052 - lats[2422] = -80.316338062175078 - lats[2423] = -80.386636784232863 - lats[2424] = -80.456935505594302 - lats[2425] = -80.527234226243991 - lats[2426] = -80.59753294616587 - lats[2427] = -80.667831665343556 - lats[2428] = -80.73813038376008 - lats[2429] = -80.808429101397948 - lats[2430] = -80.878727818239184 - lats[2431] = -80.949026534265244 - lats[2432] = -81.019325249456955 - lats[2433] = -81.089623963794551 - lats[2434] = -81.159922677257711 - lats[2435] = -81.230221389825374 - lats[2436] = -81.300520101475826 - lats[2437] = -81.370818812186627 - lats[2438] = -81.441117521934686 - lats[2439] = -81.511416230696042 - lats[2440] = -81.581714938445955 - lats[2441] = -81.652013645158945 - lats[2442] = -81.722312350808508 - lats[2443] = -81.792611055367345 - lats[2444] = -81.862909758807191 - lats[2445] = -81.933208461098829 - lats[2446] = -82.003507162211946 - lats[2447] = -82.073805862115165 - lats[2448] = -82.144104560776 - lats[2449] = -82.214403258160871 - lats[2450] = -82.284701954234833 - lats[2451] = -82.355000648961692 - lats[2452] = -82.425299342304029 - lats[2453] = -82.495598034222837 - lats[2454] = -82.56589672467787 - lats[2455] = -82.63619541362705 - lats[2456] = -82.706494101026948 - lats[2457] = -82.77679278683226 - lats[2458] = -82.84709147099602 - lats[2459] = -82.917390153469313 - lats[2460] = -82.987688834201322 - lats[2461] = -83.057987513139125 - lats[2462] = -83.128286190227698 - lats[2463] = -83.198584865409657 - lats[2464] = -83.268883538625232 - lats[2465] = -83.339182209812321 - lats[2466] = -83.409480878905782 - lats[2467] = -83.479779545838113 - lats[2468] = -83.550078210538487 - lats[2469] = -83.620376872933264 - lats[2470] = -83.690675532945292 - lats[2471] = -83.760974190494011 - lats[2472] = -83.831272845495249 - lats[2473] = -83.901571497860914 - lats[2474] = -83.971870147498763 - lats[2475] = -84.042168794312317 - lats[2476] = -84.112467438200326 - lats[2477] = -84.18276607905679 - lats[2478] = -84.253064716770425 - lats[2479] = -84.323363351224444 - lats[2480] = -84.393661982296322 - lats[2481] = -84.463960609857125 - lats[2482] = -84.534259233771479 - lats[2483] = -84.604557853896708 - lats[2484] = -84.674856470082915 - lats[2485] = -84.745155082171991 - lats[2486] = -84.81545368999717 - lats[2487] = -84.885752293382765 - lats[2488] = -84.95605089214304 - lats[2489] = -85.026349486081983 - lats[2490] = -85.09664807499216 - lats[2491] = -85.16694665865414 - lats[2492] = -85.237245236835548 - lats[2493] = -85.307543809290152 - lats[2494] = -85.377842375756586 - lats[2495] = -85.448140935957483 - lats[2496] = -85.518439489597966 - lats[2497] = -85.588738036364362 - lats[2498] = -85.659036575922883 - lats[2499] = -85.729335107917464 - lats[2500] = -85.799633631968391 - lats[2501] = -85.869932147670127 - lats[2502] = -85.940230654588888 - lats[2503] = -86.010529152260403 - lats[2504] = -86.080827640187209 - lats[2505] = -86.151126117835304 - lats[2506] = -86.221424584631109 - lats[2507] = -86.291723039957418 - lats[2508] = -86.362021483149363 - lats[2509] = -86.432319913489792 - lats[2510] = -86.502618330203831 - lats[2511] = -86.572916732453024 - lats[2512] = -86.643215119328573 - lats[2513] = -86.713513489844246 - lats[2514] = -86.783811842927179 - lats[2515] = -86.854110177408927 - lats[2516] = -86.924408492014166 - lats[2517] = -86.994706785348129 - lats[2518] = -87.065005055882821 - lats[2519] = -87.135303301939786 - lats[2520] = -87.205601521672108 - lats[2521] = -87.275899713041966 - lats[2522] = -87.346197873795816 - lats[2523] = -87.416496001434894 - lats[2524] = -87.486794093180748 - lats[2525] = -87.557092145935584 - lats[2526] = -87.627390156234085 - lats[2527] = -87.697688120188062 - lats[2528] = -87.767986033419561 - lats[2529] = -87.838283890981543 - lats[2530] = -87.908581687261687 - lats[2531] = -87.978879415867283 - lats[2532] = -88.049177069484486 - lats[2533] = -88.119474639706425 - lats[2534] = -88.189772116820762 - lats[2535] = -88.26006948954614 - lats[2536] = -88.330366744702559 - lats[2537] = -88.40066386679355 - lats[2538] = -88.470960837474877 - lats[2539] = -88.541257634868515 - lats[2540] = -88.611554232668382 - lats[2541] = -88.681850598961759 - lats[2542] = -88.752146694650691 - lats[2543] = -88.822442471310097 - lats[2544] = -88.892737868230952 - lats[2545] = -88.96303280826325 - lats[2546] = -89.033327191845927 - lats[2547] = -89.103620888238879 - lats[2548] = -89.173913722284126 - lats[2549] = -89.24420545380525 - lats[2550] = -89.314495744374256 - lats[2551] = -89.3847841013921 - lats[2552] = -89.45506977912261 - lats[2553] = -89.525351592371393 - lats[2554] = -89.595627537554492 - lats[2555] = -89.6658939412157 - lats[2556] = -89.736143271609578 - lats[2557] = -89.806357319542244 - lats[2558] = -89.876478353332288 - lats[2559] = -89.946187715665616 - return lats - - def first_axis_vals(self): - if self._resolution == 1280: - return self.get_precomputed_values_N1280() - else: - precision = 1.0e-14 - nval = self._resolution * 2 - rad2deg = 180 / math.pi - convval = 1 - ((2 / math.pi) * (2 / math.pi)) * 0.25 - vals = self.gauss_first_guess() - new_vals = [0] * nval - denom = math.sqrt(((nval + 0.5) * (nval + 0.5)) + convval) - for jval in range(self._resolution): - root = math.cos(vals[jval] / denom) - conv = 1 - while abs(conv) >= precision: - mem2 = 1 - mem1 = root - for legi in range(nval): - legfonc = ((2.0 * (legi + 1) - 1.0) * root * mem1 - legi * mem2) / (legi + 1) - mem2 = mem1 - mem1 = legfonc - conv = legfonc / ((nval * (mem2 - root * legfonc)) / (1.0 - (root * root))) - root = root - conv - # add maybe a max iter here to make sure we converge at some point - new_vals[jval] = math.asin(root) * rad2deg - new_vals[nval - 1 - jval] = -new_vals[jval] - return new_vals - - def map_first_axis(self, lower, upper): - axis_lines = self._first_axis_vals - end_idx = bisect_left_cmp(axis_lines, lower, cmp=lambda x, y: x > y) + 1 - start_idx = bisect_right_cmp(axis_lines, upper, cmp=lambda x, y: x > y) - return_vals = axis_lines[start_idx:end_idx] - return return_vals - - def second_axis_vals(self, first_val): - first_axis_vals = self._first_axis_vals - tol = 1e-10 - first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) - if first_idx >= self._resolution: - first_idx = (2 * self._resolution) - 1 - first_idx - first_idx = first_idx + 1 - npoints = 4 * first_idx + 16 - second_axis_spacing = 360 / npoints - second_axis_vals = [i * second_axis_spacing for i in range(npoints)] - return second_axis_vals - - def second_axis_spacing(self, first_val): - first_axis_vals = self._first_axis_vals - tol = 1e-10 - _first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) - first_idx = _first_idx - if first_idx >= self._resolution: - first_idx = (2 * self._resolution) - 1 - first_idx - first_idx = first_idx + 1 - npoints = 4 * first_idx + 16 - second_axis_spacing = 360 / npoints - return (second_axis_spacing, _first_idx + 1) - - def map_second_axis(self, first_val, lower, upper): - second_axis_spacing, first_idx = self.second_axis_spacing(first_val) - start_idx = int(lower / second_axis_spacing) - end_idx = int(upper / second_axis_spacing) + 1 - return_vals = [i * second_axis_spacing for i in range(start_idx, end_idx)] - return return_vals - - def axes_idx_to_octahedral_idx(self, first_idx, second_idx): - # NOTE: for now this takes ~2e-4s per point, so taking significant time -> for 20k points, takes 4s - # Would it be better to store a dictionary of first_idx with cumulative number of points on that idx? - # Because this is what we are doing here, but we are calculating for each point... - # But then this would only work for special grid resolutions, so need to do like a O1280 version of this - - # NOTE: OR somehow cache this for a given first_idx and then only modify the axis idx for second_idx when the - # first_idx changes - octa_idx = self._first_idx_map[first_idx - 1] + second_idx - return octa_idx - - def create_first_idx_map(self): - first_idx_list = {} - idx = 0 - for i in range(2 * self._resolution): - first_idx_list[i] = idx - if i <= self._resolution - 1: - idx += 20 + 4 * i - else: - i = i - self._resolution + 1 - if i == 1: - idx += 16 + 4 * self._resolution - else: - i = i - 1 - idx += 16 + 4 * (self._resolution - i) - return first_idx_list - - def find_second_axis_idx(self, first_val, second_val): - (second_axis_spacing, first_idx) = self.second_axis_spacing(first_val) - tol = 1e-8 - if second_val / second_axis_spacing > int(second_val / second_axis_spacing) + 1 - tol: - second_idx = int(second_val / second_axis_spacing) + 1 - else: - second_idx = int(second_val / second_axis_spacing) - return (first_idx, second_idx) - - def unmap(self, first_val, second_val): - (first_idx, second_idx) = self.find_second_axis_idx(first_val, second_val) - octahedral_index = self.axes_idx_to_octahedral_idx(first_idx, second_idx) - return octahedral_index - - _type_to_datacube_mapper_lookup = { "octahedral": "OctahedralGridMapper", "healpix": "HealpixGridMapper", diff --git a/polytope/datacube/transformations/mappers/healpix.py b/polytope/datacube/transformations/mappers/healpix.py new file mode 100644 index 000000000..627a1be94 --- /dev/null +++ b/polytope/datacube/transformations/mappers/healpix.py @@ -0,0 +1,123 @@ +import bisect +import math + +from ..datacube_mappers import DatacubeMapper + + +class HealpixGridMapper(DatacubeMapper): + def __init__(self, base_axis, mapped_axes, resolution): + self._mapped_axes = mapped_axes + self._base_axis = base_axis + self._resolution = resolution + self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} + self._first_axis_vals = self.first_axis_vals() + + def first_axis_vals(self): + rad2deg = 180 / math.pi + vals = [0] * (4 * self._resolution - 1) + + # Polar caps + for i in range(1, self._resolution): + val = 90 - (rad2deg * math.acos(1 - (i * i / (3 * self._resolution * self._resolution)))) + vals[i - 1] = val + vals[4 * self._resolution - 1 - i] = -val + # Equatorial belts + for i in range(self._resolution, 2 * self._resolution): + val = 90 - (rad2deg * math.acos((4 * self._resolution - 2 * i) / (3 * self._resolution))) + vals[i - 1] = val + vals[4 * self._resolution - 1 - i] = -val + # Equator + vals[2 * self._resolution - 1] = 0 + return vals + + def map_first_axis(self, lower, upper): + axis_lines = self._first_axis_vals + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def second_axis_vals(self, first_val): + tol = 1e-8 + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + idx = self._first_axis_vals.index(first_val) + + # Polar caps + if idx < self._resolution - 1 or 3 * self._resolution - 1 < idx <= 4 * self._resolution - 2: + start = 45 / (idx + 1) + vals = [start + i * (360 / (4 * (idx + 1))) for i in range(4 * (idx + 1))] + return vals + # Equatorial belts + start = 45 / self._resolution + if self._resolution - 1 <= idx < 2 * self._resolution - 1 or 2 * self._resolution <= idx < 3 * self._resolution: + r_start = start * (2 - (((idx + 1) - self._resolution + 1) % 2)) + vals = [r_start + i * (360 / (4 * self._resolution)) for i in range(4 * self._resolution)] + if vals[-1] == 360: + vals[-1] = 0 + return vals + # Equator + temp_val = 1 if self._resolution % 2 else 0 + r_start = start * (1 - temp_val) + if idx == 2 * self._resolution - 1: + vals = [r_start + i * (360 / (4 * self._resolution)) for i in range(4 * self._resolution)] + return vals + + def map_second_axis(self, first_val, lower, upper): + axis_lines = self.second_axis_vals(first_val) + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def axes_idx_to_healpix_idx(self, first_idx, second_idx): + idx = 0 + for i in range(self._resolution - 1): + if i != first_idx: + idx += 4 * (i + 1) + else: + idx += second_idx + return idx + for i in range(self._resolution - 1, 3 * self._resolution): + if i != first_idx: + idx += 4 * self._resolution + else: + idx += second_idx + return idx + for i in range(3 * self._resolution, 4 * self._resolution - 1): + if i != first_idx: + idx += 4 * (4 * self._resolution - 1 - i + 1) + else: + idx += second_idx + return idx + + def find_second_idx(self, first_val, second_val): + tol = 1e-10 + second_axis_vals = self.second_axis_vals(first_val) + second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) + return second_idx + + def unmap_first_val_to_start_line_idx(self, first_val): + tol = 1e-8 + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + first_idx = self._first_axis_vals.index(first_val) + idx = 0 + for i in range(self._resolution - 1): + if i != first_idx: + idx += 4 * (i + 1) + else: + return idx + for i in range(self._resolution - 1, 3 * self._resolution): + if i != first_idx: + idx += 4 * self._resolution + else: + return idx + for i in range(3 * self._resolution, 4 * self._resolution - 1): + if i != first_idx: + idx += 4 * (4 * self._resolution - 1 - i + 1) + else: + return idx + + def unmap(self, first_val, second_val): + tol = 1e-8 + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + first_idx = self._first_axis_vals.index(first_val) + second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] + second_idx = self.second_axis_vals(first_val).index(second_val) + healpix_index = self.axes_idx_to_healpix_idx(first_idx, second_idx) + return healpix_index diff --git a/polytope/datacube/transformations/mappers/octahedral.py b/polytope/datacube/transformations/mappers/octahedral.py new file mode 100644 index 000000000..1c24703df --- /dev/null +++ b/polytope/datacube/transformations/mappers/octahedral.py @@ -0,0 +1,2752 @@ +import math + +from ....utility.list_tools import bisect_left_cmp, bisect_right_cmp +from ..datacube_mappers import DatacubeMapper + + +class OctahedralGridMapper(DatacubeMapper): + def __init__(self, base_axis, mapped_axes, resolution): + self._mapped_axes = mapped_axes + self._base_axis = base_axis + self._resolution = resolution + self._first_axis_vals = self.first_axis_vals() + self._first_idx_map = self.create_first_idx_map() + self._second_axis_spacing = {} + self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} + + def gauss_first_guess(self): + i = 0 + gvals = [ + 2.4048255577e0, + 5.5200781103e0, + 8.6537279129e0, + 11.7915344391e0, + 14.9309177086e0, + 18.0710639679e0, + 21.2116366299e0, + 24.3524715308e0, + 27.4934791320e0, + 30.6346064684e0, + 33.7758202136e0, + 36.9170983537e0, + 40.0584257646e0, + 43.1997917132e0, + 46.3411883717e0, + 49.4826098974e0, + 52.6240518411e0, + 55.7655107550e0, + 58.9069839261e0, + 62.0484691902e0, + 65.1899648002e0, + 68.3314693299e0, + 71.4729816036e0, + 74.6145006437e0, + 77.7560256304e0, + 80.8975558711e0, + 84.0390907769e0, + 87.1806298436e0, + 90.3221726372e0, + 93.4637187819e0, + 96.6052679510e0, + 99.7468198587e0, + 102.8883742542e0, + 106.0299309165e0, + 109.1714896498e0, + 112.3130502805e0, + 115.4546126537e0, + 118.5961766309e0, + 121.7377420880e0, + 124.8793089132e0, + 128.0208770059e0, + 131.1624462752e0, + 134.3040166383e0, + 137.4455880203e0, + 140.5871603528e0, + 143.7287335737e0, + 146.8703076258e0, + 150.0118824570e0, + 153.1534580192e0, + 156.2950342685e0, + ] + + numVals = len(gvals) + vals = [] + for i in range(self._resolution): + if i < numVals: + vals.append(gvals[i]) + else: + vals.append(vals[i - 1] + math.pi) + return vals + + def get_precomputed_values_N1280(self): + lats = [0] * 2560 + # lats = SortedList() + # lats = {} + lats[0] = 89.946187715665616 + lats[1] = 89.876478353332288 + lats[2] = 89.806357319542244 + lats[3] = 89.736143271609578 + lats[4] = 89.6658939412157 + lats[5] = 89.595627537554492 + lats[6] = 89.525351592371393 + lats[7] = 89.45506977912261 + lats[8] = 89.3847841013921 + lats[9] = 89.314495744374256 + lats[10] = 89.24420545380525 + lats[11] = 89.173913722284126 + lats[12] = 89.103620888238879 + lats[13] = 89.033327191845927 + lats[14] = 88.96303280826325 + lats[15] = 88.892737868230952 + lats[16] = 88.822442471310097 + lats[17] = 88.752146694650691 + lats[18] = 88.681850598961759 + lats[19] = 88.611554232668382 + lats[20] = 88.541257634868515 + lats[21] = 88.470960837474877 + lats[22] = 88.40066386679355 + lats[23] = 88.330366744702559 + lats[24] = 88.26006948954614 + lats[25] = 88.189772116820762 + lats[26] = 88.119474639706425 + lats[27] = 88.049177069484486 + lats[28] = 87.978879415867283 + lats[29] = 87.908581687261687 + lats[30] = 87.838283890981543 + lats[31] = 87.767986033419561 + lats[32] = 87.697688120188062 + lats[33] = 87.627390156234085 + lats[34] = 87.557092145935584 + lats[35] = 87.486794093180748 + lats[36] = 87.416496001434894 + lats[37] = 87.346197873795816 + lats[38] = 87.275899713041966 + lats[39] = 87.205601521672108 + lats[40] = 87.135303301939786 + lats[41] = 87.065005055882821 + lats[42] = 86.994706785348129 + lats[43] = 86.924408492014166 + lats[44] = 86.854110177408927 + lats[45] = 86.783811842927179 + lats[46] = 86.713513489844246 + lats[47] = 86.643215119328573 + lats[48] = 86.572916732453024 + lats[49] = 86.502618330203831 + lats[50] = 86.432319913489792 + lats[51] = 86.362021483149363 + lats[52] = 86.291723039957418 + lats[53] = 86.221424584631109 + lats[54] = 86.151126117835304 + lats[55] = 86.080827640187209 + lats[56] = 86.010529152260403 + lats[57] = 85.940230654588888 + lats[58] = 85.869932147670127 + lats[59] = 85.799633631968391 + lats[60] = 85.729335107917464 + lats[61] = 85.659036575922883 + lats[62] = 85.588738036364362 + lats[63] = 85.518439489597966 + lats[64] = 85.448140935957483 + lats[65] = 85.377842375756586 + lats[66] = 85.307543809290152 + lats[67] = 85.237245236835548 + lats[68] = 85.16694665865414 + lats[69] = 85.09664807499216 + lats[70] = 85.026349486081983 + lats[71] = 84.95605089214304 + lats[72] = 84.885752293382765 + lats[73] = 84.81545368999717 + lats[74] = 84.745155082171991 + lats[75] = 84.674856470082915 + lats[76] = 84.604557853896708 + lats[77] = 84.534259233771479 + lats[78] = 84.463960609857125 + lats[79] = 84.393661982296322 + lats[80] = 84.323363351224444 + lats[81] = 84.253064716770425 + lats[82] = 84.18276607905679 + lats[83] = 84.112467438200326 + lats[84] = 84.042168794312317 + lats[85] = 83.971870147498763 + lats[86] = 83.901571497860914 + lats[87] = 83.831272845495249 + lats[88] = 83.760974190494011 + lats[89] = 83.690675532945292 + lats[90] = 83.620376872933264 + lats[91] = 83.550078210538487 + lats[92] = 83.479779545838113 + lats[93] = 83.409480878905782 + lats[94] = 83.339182209812321 + lats[95] = 83.268883538625232 + lats[96] = 83.198584865409657 + lats[97] = 83.128286190227698 + lats[98] = 83.057987513139125 + lats[99] = 82.987688834201322 + lats[100] = 82.917390153469313 + lats[101] = 82.84709147099602 + lats[102] = 82.77679278683226 + lats[103] = 82.706494101026948 + lats[104] = 82.63619541362705 + lats[105] = 82.56589672467787 + lats[106] = 82.495598034222837 + lats[107] = 82.425299342304029 + lats[108] = 82.355000648961692 + lats[109] = 82.284701954234833 + lats[110] = 82.214403258160871 + lats[111] = 82.144104560776 + lats[112] = 82.073805862115165 + lats[113] = 82.003507162211946 + lats[114] = 81.933208461098829 + lats[115] = 81.862909758807191 + lats[116] = 81.792611055367345 + lats[117] = 81.722312350808508 + lats[118] = 81.652013645158945 + lats[119] = 81.581714938445955 + lats[120] = 81.511416230696042 + lats[121] = 81.441117521934686 + lats[122] = 81.370818812186627 + lats[123] = 81.300520101475826 + lats[124] = 81.230221389825374 + lats[125] = 81.159922677257711 + lats[126] = 81.089623963794551 + lats[127] = 81.019325249456955 + lats[128] = 80.949026534265244 + lats[129] = 80.878727818239184 + lats[130] = 80.808429101397948 + lats[131] = 80.73813038376008 + lats[132] = 80.667831665343556 + lats[133] = 80.59753294616587 + lats[134] = 80.527234226243991 + lats[135] = 80.456935505594302 + lats[136] = 80.386636784232863 + lats[137] = 80.316338062175078 + lats[138] = 80.246039339436052 + lats[139] = 80.175740616030438 + lats[140] = 80.105441891972376 + lats[141] = 80.035143167275749 + lats[142] = 79.9648444419539 + lats[143] = 79.894545716019948 + lats[144] = 79.824246989486554 + lats[145] = 79.753948262366038 + lats[146] = 79.683649534670437 + lats[147] = 79.61335080641139 + lats[148] = 79.543052077600308 + lats[149] = 79.472753348248219 + lats[150] = 79.402454618365894 + lats[151] = 79.332155887963822 + lats[152] = 79.261857157052191 + lats[153] = 79.191558425640977 + lats[154] = 79.121259693739859 + lats[155] = 79.050960961358285 + lats[156] = 78.980662228505423 + lats[157] = 78.910363495190211 + lats[158] = 78.840064761421445 + lats[159] = 78.769766027207638 + lats[160] = 78.699467292557102 + lats[161] = 78.629168557477882 + lats[162] = 78.558869821977908 + lats[163] = 78.488571086064923 + lats[164] = 78.418272349746417 + lats[165] = 78.347973613029708 + lats[166] = 78.277674875922045 + lats[167] = 78.207376138430348 + lats[168] = 78.137077400561424 + lats[169] = 78.066778662322022 + lats[170] = 77.996479923718596 + lats[171] = 77.926181184757539 + lats[172] = 77.855882445445019 + lats[173] = 77.785583705787161 + lats[174] = 77.71528496578982 + lats[175] = 77.644986225458879 + lats[176] = 77.574687484799924 + lats[177] = 77.504388743818524 + lats[178] = 77.434090002520122 + lats[179] = 77.363791260909963 + lats[180] = 77.293492518993247 + lats[181] = 77.22319377677502 + lats[182] = 77.15289503426024 + lats[183] = 77.082596291453768 + lats[184] = 77.012297548360323 + lats[185] = 76.941998804984564 + lats[186] = 76.871700061330955 + lats[187] = 76.801401317404 + lats[188] = 76.731102573208048 + lats[189] = 76.660803828747362 + lats[190] = 76.59050508402602 + lats[191] = 76.520206339048215 + lats[192] = 76.449907593817869 + lats[193] = 76.379608848338933 + lats[194] = 76.3093101026152 + lats[195] = 76.239011356650423 + lats[196] = 76.16871261044831 + lats[197] = 76.098413864012443 + lats[198] = 76.028115117346374 + lats[199] = 75.957816370453543 + lats[200] = 75.887517623337317 + lats[201] = 75.81721887600105 + lats[202] = 75.746920128447996 + lats[203] = 75.67662138068134 + lats[204] = 75.60632263270422 + lats[205] = 75.536023884519707 + lats[206] = 75.465725136130786 + lats[207] = 75.395426387540439 + lats[208] = 75.325127638751567 + lats[209] = 75.254828889766983 + lats[210] = 75.184530140589501 + lats[211] = 75.114231391221821 + lats[212] = 75.043932641666672 + lats[213] = 74.973633891926625 + lats[214] = 74.903335142004323 + lats[215] = 74.833036391902269 + lats[216] = 74.762737641622991 + lats[217] = 74.692438891168877 + lats[218] = 74.622140140542356 + lats[219] = 74.551841389745761 + lats[220] = 74.481542638781434 + lats[221] = 74.411243887651622 + lats[222] = 74.340945136358584 + lats[223] = 74.270646384904481 + lats[224] = 74.200347633291472 + lats[225] = 74.13004888152166 + lats[226] = 74.059750129597163 + lats[227] = 73.98945137751997 + lats[228] = 73.919152625292114 + lats[229] = 73.848853872915541 + lats[230] = 73.778555120392184 + lats[231] = 73.70825636772399 + lats[232] = 73.637957614912779 + lats[233] = 73.567658861960396 + lats[234] = 73.497360108868662 + lats[235] = 73.427061355639339 + lats[236] = 73.356762602274188 + lats[237] = 73.2864638487749 + lats[238] = 73.216165095143182 + lats[239] = 73.145866341380668 + lats[240] = 73.075567587489019 + lats[241] = 73.005268833469799 + lats[242] = 72.934970079324657 + lats[243] = 72.864671325055056 + lats[244] = 72.794372570662574 + lats[245] = 72.724073816148703 + lats[246] = 72.653775061514935 + lats[247] = 72.583476306762691 + lats[248] = 72.513177551893421 + lats[249] = 72.442878796908545 + lats[250] = 72.3725800418094 + lats[251] = 72.302281286597392 + lats[252] = 72.231982531273843 + lats[253] = 72.161683775840089 + lats[254] = 72.091385020297409 + lats[255] = 72.02108626464711 + lats[256] = 71.950787508890414 + lats[257] = 71.880488753028587 + lats[258] = 71.810189997062835 + lats[259] = 71.739891240994368 + lats[260] = 71.669592484824364 + lats[261] = 71.599293728553988 + lats[262] = 71.528994972184378 + lats[263] = 71.458696215716685 + lats[264] = 71.388397459152031 + lats[265] = 71.318098702491469 + lats[266] = 71.247799945736105 + lats[267] = 71.177501188887007 + lats[268] = 71.107202431945211 + lats[269] = 71.036903674911756 + lats[270] = 70.966604917787635 + lats[271] = 70.896306160573886 + lats[272] = 70.826007403271475 + lats[273] = 70.755708645881384 + lats[274] = 70.685409888404578 + lats[275] = 70.615111130841967 + lats[276] = 70.544812373194532 + lats[277] = 70.474513615463138 + lats[278] = 70.404214857648739 + lats[279] = 70.333916099752187 + lats[280] = 70.263617341774406 + lats[281] = 70.193318583716191 + lats[282] = 70.123019825578467 + lats[283] = 70.052721067362043 + lats[284] = 69.982422309067744 + lats[285] = 69.912123550696421 + lats[286] = 69.841824792248843 + lats[287] = 69.771526033725834 + lats[288] = 69.701227275128161 + lats[289] = 69.630928516456592 + lats[290] = 69.560629757711908 + lats[291] = 69.490330998894862 + lats[292] = 69.420032240006194 + lats[293] = 69.349733481046613 + lats[294] = 69.279434722016902 + lats[295] = 69.209135962917699 + lats[296] = 69.138837203749759 + lats[297] = 69.068538444513763 + lats[298] = 68.998239685210365 + lats[299] = 68.927940925840304 + lats[300] = 68.85764216640419 + lats[301] = 68.787343406902693 + lats[302] = 68.717044647336493 + lats[303] = 68.646745887706189 + lats[304] = 68.576447128012447 + lats[305] = 68.506148368255865 + lats[306] = 68.435849608437067 + lats[307] = 68.365550848556666 + lats[308] = 68.295252088615257 + lats[309] = 68.224953328613438 + lats[310] = 68.154654568551791 + lats[311] = 68.084355808430871 + lats[312] = 68.014057048251274 + lats[313] = 67.943758288013555 + lats[314] = 67.873459527718282 + lats[315] = 67.803160767365966 + lats[316] = 67.732862006957205 + lats[317] = 67.662563246492482 + lats[318] = 67.592264485972336 + lats[319] = 67.521965725397308 + lats[320] = 67.451666964767895 + lats[321] = 67.381368204084609 + lats[322] = 67.311069443347961 + lats[323] = 67.240770682558434 + lats[324] = 67.170471921716526 + lats[325] = 67.100173160822706 + lats[326] = 67.029874399877471 + lats[327] = 66.95957563888129 + lats[328] = 66.889276877834618 + lats[329] = 66.818978116737924 + lats[330] = 66.748679355591662 + lats[331] = 66.678380594396273 + lats[332] = 66.608081833152212 + lats[333] = 66.537783071859891 + lats[334] = 66.467484310519808 + lats[335] = 66.397185549132331 + lats[336] = 66.326886787697887 + lats[337] = 66.256588026216932 + lats[338] = 66.186289264689833 + lats[339] = 66.115990503117033 + lats[340] = 66.045691741498899 + lats[341] = 65.975392979835888 + lats[342] = 65.905094218128355 + lats[343] = 65.834795456376696 + lats[344] = 65.764496694581283 + lats[345] = 65.694197932742526 + lats[346] = 65.623899170860767 + lats[347] = 65.553600408936404 + lats[348] = 65.483301646969792 + lats[349] = 65.413002884961315 + lats[350] = 65.342704122911286 + lats[351] = 65.272405360820116 + lats[352] = 65.202106598688133 + lats[353] = 65.131807836515677 + lats[354] = 65.061509074303089 + lats[355] = 64.991210312050711 + lats[356] = 64.920911549758912 + lats[357] = 64.850612787427963 + lats[358] = 64.780314025058246 + lats[359] = 64.710015262650074 + lats[360] = 64.639716500203733 + lats[361] = 64.569417737719576 + lats[362] = 64.499118975197902 + lats[363] = 64.428820212639039 + lats[364] = 64.358521450043284 + lats[365] = 64.288222687410922 + lats[366] = 64.21792392474228 + lats[367] = 64.147625162037642 + lats[368] = 64.07732639929732 + lats[369] = 64.00702763652157 + lats[370] = 63.93672887371072 + lats[371] = 63.866430110865004 + lats[372] = 63.796131347984762 + lats[373] = 63.725832585070251 + lats[374] = 63.655533822121711 + lats[375] = 63.585235059139464 + lats[376] = 63.514936296123757 + lats[377] = 63.444637533074854 + lats[378] = 63.374338769993031 + lats[379] = 63.304040006878537 + lats[380] = 63.23374124373165 + lats[381] = 63.163442480552604 + lats[382] = 63.093143717341647 + lats[383] = 63.022844954099064 + lats[384] = 62.952546190825068 + lats[385] = 62.882247427519928 + lats[386] = 62.811948664183866 + lats[387] = 62.741649900817137 + lats[388] = 62.67135113741999 + lats[389] = 62.60105237399263 + lats[390] = 62.530753610535321 + lats[391] = 62.460454847048261 + lats[392] = 62.3901560835317 + lats[393] = 62.319857319985871 + lats[394] = 62.249558556410982 + lats[395] = 62.179259792807258 + lats[396] = 62.108961029174914 + lats[397] = 62.038662265514176 + lats[398] = 61.968363501825259 + lats[399] = 61.898064738108381 + lats[400] = 61.827765974363729 + lats[401] = 61.757467210591535 + lats[402] = 61.687168446791986 + lats[403] = 61.616869682965287 + lats[404] = 61.546570919111666 + lats[405] = 61.476272155231321 + lats[406] = 61.405973391324409 + lats[407] = 61.335674627391185 + lats[408] = 61.265375863431785 + lats[409] = 61.195077099446451 + lats[410] = 61.124778335435344 + lats[411] = 61.054479571398652 + lats[412] = 60.984180807336578 + lats[413] = 60.913882043249295 + lats[414] = 60.843583279137007 + lats[415] = 60.773284514999872 + lats[416] = 60.702985750838074 + lats[417] = 60.632686986651805 + lats[418] = 60.562388222441243 + lats[419] = 60.492089458206543 + lats[420] = 60.421790693947884 + lats[421] = 60.35149192966545 + lats[422] = 60.28119316535939 + lats[423] = 60.21089440102989 + lats[424] = 60.140595636677112 + lats[425] = 60.070296872301235 + lats[426] = 59.999998107902378 + lats[427] = 59.929699343480763 + lats[428] = 59.859400579036503 + lats[429] = 59.78910181456979 + lats[430] = 59.718803050080759 + lats[431] = 59.64850428556958 + lats[432] = 59.578205521036402 + lats[433] = 59.507906756481383 + lats[434] = 59.43760799190467 + lats[435] = 59.3673092273064 + lats[436] = 59.29701046268675 + lats[437] = 59.226711698045854 + lats[438] = 59.156412933383855 + lats[439] = 59.086114168700909 + lats[440] = 59.015815403997145 + lats[441] = 58.945516639272725 + lats[442] = 58.875217874527763 + lats[443] = 58.804919109762423 + lats[444] = 58.73462034497684 + lats[445] = 58.664321580171141 + lats[446] = 58.594022815345468 + lats[447] = 58.523724050499972 + lats[448] = 58.453425285634758 + lats[449] = 58.383126520749968 + lats[450] = 58.312827755845746 + lats[451] = 58.242528990922203 + lats[452] = 58.172230225979497 + lats[453] = 58.101931461017728 + lats[454] = 58.031632696037022 + lats[455] = 57.961333931037537 + lats[456] = 57.891035166019364 + lats[457] = 57.820736400982646 + lats[458] = 57.75043763592749 + lats[459] = 57.680138870854037 + lats[460] = 57.60984010576238 + lats[461] = 57.539541340652676 + lats[462] = 57.469242575525016 + lats[463] = 57.398943810379521 + lats[464] = 57.328645045216312 + lats[465] = 57.258346280035504 + lats[466] = 57.188047514837208 + lats[467] = 57.117748749621541 + lats[468] = 57.047449984388614 + lats[469] = 56.977151219138541 + lats[470] = 56.90685245387143 + lats[471] = 56.836553688587379 + lats[472] = 56.766254923286517 + lats[473] = 56.695956157968951 + lats[474] = 56.625657392634771 + lats[475] = 56.555358627284086 + lats[476] = 56.485059861917016 + lats[477] = 56.41476109653366 + lats[478] = 56.34446233113411 + lats[479] = 56.274163565718467 + lats[480] = 56.203864800286865 + lats[481] = 56.133566034839362 + lats[482] = 56.063267269376091 + lats[483] = 55.992968503897131 + lats[484] = 55.922669738402583 + lats[485] = 55.852370972892551 + lats[486] = 55.782072207367136 + lats[487] = 55.711773441826416 + lats[488] = 55.641474676270505 + lats[489] = 55.571175910699488 + lats[490] = 55.500877145113449 + lats[491] = 55.430578379512511 + lats[492] = 55.360279613896743 + lats[493] = 55.289980848266232 + lats[494] = 55.219682082621084 + lats[495] = 55.149383316961377 + lats[496] = 55.07908455128721 + lats[497] = 55.008785785598668 + lats[498] = 54.938487019895831 + lats[499] = 54.868188254178797 + lats[500] = 54.797889488447652 + lats[501] = 54.727590722702473 + lats[502] = 54.657291956943347 + lats[503] = 54.586993191170357 + lats[504] = 54.516694425383605 + lats[505] = 54.446395659583146 + lats[506] = 54.376096893769081 + lats[507] = 54.305798127941479 + lats[508] = 54.235499362100448 + lats[509] = 54.165200596246031 + lats[510] = 54.094901830378333 + lats[511] = 54.024603064497434 + lats[512] = 53.954304298603383 + lats[513] = 53.884005532696307 + lats[514] = 53.813706766776235 + lats[515] = 53.743408000843282 + lats[516] = 53.673109234897495 + lats[517] = 53.602810468938962 + lats[518] = 53.53251170296776 + lats[519] = 53.462212936983953 + lats[520] = 53.391914170987633 + lats[521] = 53.321615404978871 + lats[522] = 53.251316638957725 + lats[523] = 53.181017872924265 + lats[524] = 53.110719106878584 + lats[525] = 53.040420340820731 + lats[526] = 52.970121574750792 + lats[527] = 52.899822808668837 + lats[528] = 52.829524042574917 + lats[529] = 52.759225276469131 + lats[530] = 52.688926510351514 + lats[531] = 52.618627744222159 + lats[532] = 52.548328978081123 + lats[533] = 52.478030211928477 + lats[534] = 52.407731445764284 + lats[535] = 52.337432679588609 + lats[536] = 52.26713391340153 + lats[537] = 52.196835147203096 + lats[538] = 52.126536380993372 + lats[539] = 52.056237614772435 + lats[540] = 51.985938848540336 + lats[541] = 51.915640082297152 + lats[542] = 51.845341316042933 + lats[543] = 51.775042549777737 + lats[544] = 51.704743783501634 + lats[545] = 51.634445017214695 + lats[546] = 51.56414625091697 + lats[547] = 51.493847484608516 + lats[548] = 51.423548718289396 + lats[549] = 51.353249951959683 + lats[550] = 51.282951185619417 + lats[551] = 51.21265241926865 + lats[552] = 51.14235365290746 + lats[553] = 51.072054886535909 + lats[554] = 51.001756120154049 + lats[555] = 50.931457353761914 + lats[556] = 50.86115858735959 + lats[557] = 50.790859820947119 + lats[558] = 50.720561054524559 + lats[559] = 50.650262288091959 + lats[560] = 50.579963521649397 + lats[561] = 50.509664755196901 + lats[562] = 50.439365988734544 + lats[563] = 50.369067222262359 + lats[564] = 50.298768455780426 + lats[565] = 50.228469689288779 + lats[566] = 50.158170922787484 + lats[567] = 50.087872156276575 + lats[568] = 50.017573389756123 + lats[569] = 49.947274623226157 + lats[570] = 49.876975856686762 + lats[571] = 49.80667709013796 + lats[572] = 49.736378323579807 + lats[573] = 49.66607955701236 + lats[574] = 49.595780790435676 + lats[575] = 49.525482023849783 + lats[576] = 49.455183257254745 + lats[577] = 49.384884490650613 + lats[578] = 49.314585724037435 + lats[579] = 49.244286957415234 + lats[580] = 49.173988190784094 + lats[581] = 49.103689424144044 + lats[582] = 49.03339065749514 + lats[583] = 48.963091890837418 + lats[584] = 48.892793124170929 + lats[585] = 48.822494357495721 + lats[586] = 48.752195590811837 + lats[587] = 48.681896824119335 + lats[588] = 48.611598057418242 + lats[589] = 48.541299290708608 + lats[590] = 48.47100052399049 + lats[591] = 48.400701757263917 + lats[592] = 48.330402990528938 + lats[593] = 48.260104223785596 + lats[594] = 48.189805457033941 + lats[595] = 48.119506690274015 + lats[596] = 48.049207923505868 + lats[597] = 47.978909156729507 + lats[598] = 47.908610389945018 + lats[599] = 47.838311623152421 + lats[600] = 47.76801285635176 + lats[601] = 47.697714089543084 + lats[602] = 47.627415322726435 + lats[603] = 47.557116555901828 + lats[604] = 47.486817789069342 + lats[605] = 47.416519022228997 + lats[606] = 47.346220255380835 + lats[607] = 47.275921488524894 + lats[608] = 47.205622721661214 + lats[609] = 47.13532395478984 + lats[610] = 47.065025187910805 + lats[611] = 46.994726421024154 + lats[612] = 46.924427654129929 + lats[613] = 46.85412888722815 + lats[614] = 46.783830120318882 + lats[615] = 46.713531353402139 + lats[616] = 46.643232586477971 + lats[617] = 46.572933819546414 + lats[618] = 46.502635052607502 + lats[619] = 46.432336285661272 + lats[620] = 46.362037518707766 + lats[621] = 46.291738751747012 + lats[622] = 46.221439984779053 + lats[623] = 46.151141217803925 + lats[624] = 46.080842450821663 + lats[625] = 46.01054368383231 + lats[626] = 45.94024491683588 + lats[627] = 45.869946149832437 + lats[628] = 45.799647382821995 + lats[629] = 45.729348615804589 + lats[630] = 45.659049848780263 + lats[631] = 45.588751081749038 + lats[632] = 45.51845231471097 + lats[633] = 45.448153547666081 + lats[634] = 45.377854780614399 + lats[635] = 45.30755601355596 + lats[636] = 45.237257246490813 + lats[637] = 45.166958479418959 + lats[638] = 45.096659712340461 + lats[639] = 45.026360945255341 + lats[640] = 44.956062178163634 + lats[641] = 44.885763411065362 + lats[642] = 44.81546464396056 + lats[643] = 44.745165876849271 + lats[644] = 44.674867109731515 + lats[645] = 44.604568342607337 + lats[646] = 44.534269575476756 + lats[647] = 44.463970808339802 + lats[648] = 44.39367204119651 + lats[649] = 44.323373274046915 + lats[650] = 44.253074506891046 + lats[651] = 44.182775739728925 + lats[652] = 44.112476972560586 + lats[653] = 44.042178205386072 + lats[654] = 43.971879438205391 + lats[655] = 43.9015806710186 + lats[656] = 43.831281903825705 + lats[657] = 43.760983136626741 + lats[658] = 43.690684369421732 + lats[659] = 43.620385602210717 + lats[660] = 43.550086834993728 + lats[661] = 43.479788067770777 + lats[662] = 43.409489300541907 + lats[663] = 43.339190533307139 + lats[664] = 43.26889176606651 + lats[665] = 43.19859299882004 + lats[666] = 43.128294231567757 + lats[667] = 43.057995464309691 + lats[668] = 42.987696697045862 + lats[669] = 42.917397929776307 + lats[670] = 42.847099162501053 + lats[671] = 42.776800395220121 + lats[672] = 42.706501627933541 + lats[673] = 42.63620286064134 + lats[674] = 42.565904093343548 + lats[675] = 42.495605326040177 + lats[676] = 42.425306558731272 + lats[677] = 42.355007791416853 + lats[678] = 42.284709024096927 + lats[679] = 42.214410256771551 + lats[680] = 42.144111489440725 + lats[681] = 42.073812722104492 + lats[682] = 42.003513954762873 + lats[683] = 41.933215187415882 + lats[684] = 41.862916420063563 + lats[685] = 41.792617652705921 + lats[686] = 41.722318885343 + lats[687] = 41.6520201179748 + lats[688] = 41.581721350601363 + lats[689] = 41.511422583222718 + lats[690] = 41.441123815838885 + lats[691] = 41.370825048449873 + lats[692] = 41.300526281055724 + lats[693] = 41.230227513656445 + lats[694] = 41.159928746252085 + lats[695] = 41.089629978842645 + lats[696] = 41.01933121142816 + lats[697] = 40.949032444008644 + lats[698] = 40.878733676584126 + lats[699] = 40.808434909154634 + lats[700] = 40.738136141720176 + lats[701] = 40.667837374280786 + lats[702] = 40.597538606836487 + lats[703] = 40.527239839387299 + lats[704] = 40.456941071933244 + lats[705] = 40.386642304474343 + lats[706] = 40.316343537010617 + lats[707] = 40.246044769542102 + lats[708] = 40.175746002068806 + lats[709] = 40.105447234590748 + lats[710] = 40.035148467107952 + lats[711] = 39.964849699620437 + lats[712] = 39.894550932128247 + lats[713] = 39.824252164631375 + lats[714] = 39.753953397129855 + lats[715] = 39.683654629623703 + lats[716] = 39.613355862112947 + lats[717] = 39.543057094597607 + lats[718] = 39.472758327077692 + lats[719] = 39.402459559553229 + lats[720] = 39.332160792024254 + lats[721] = 39.261862024490775 + lats[722] = 39.191563256952804 + lats[723] = 39.121264489410365 + lats[724] = 39.050965721863491 + lats[725] = 38.980666954312184 + lats[726] = 38.910368186756479 + lats[727] = 38.840069419196389 + lats[728] = 38.769770651631937 + lats[729] = 38.699471884063136 + lats[730] = 38.629173116490001 + lats[731] = 38.558874348912568 + lats[732] = 38.488575581330842 + lats[733] = 38.418276813744846 + lats[734] = 38.347978046154608 + lats[735] = 38.277679278560143 + lats[736] = 38.20738051096145 + lats[737] = 38.137081743358586 + lats[738] = 38.066782975751536 + lats[739] = 37.99648420814033 + lats[740] = 37.926185440524989 + lats[741] = 37.855886672905527 + lats[742] = 37.785587905281965 + lats[743] = 37.715289137654317 + lats[744] = 37.644990370022605 + lats[745] = 37.574691602386856 + lats[746] = 37.504392834747065 + lats[747] = 37.434094067103274 + lats[748] = 37.363795299455489 + lats[749] = 37.293496531803719 + lats[750] = 37.223197764147997 + lats[751] = 37.152898996488332 + lats[752] = 37.082600228824752 + lats[753] = 37.012301461157264 + lats[754] = 36.942002693485883 + lats[755] = 36.871703925810628 + lats[756] = 36.801405158131523 + lats[757] = 36.731106390448581 + lats[758] = 36.660807622761808 + lats[759] = 36.590508855071242 + lats[760] = 36.520210087376888 + lats[761] = 36.449911319678755 + lats[762] = 36.379612551976876 + lats[763] = 36.309313784271254 + lats[764] = 36.239015016561908 + lats[765] = 36.16871624884886 + lats[766] = 36.098417481132117 + lats[767] = 36.028118713411708 + lats[768] = 35.957819945687639 + lats[769] = 35.887521177959933 + lats[770] = 35.817222410228595 + lats[771] = 35.746923642493655 + lats[772] = 35.676624874755113 + lats[773] = 35.606326107012997 + lats[774] = 35.536027339267314 + lats[775] = 35.465728571518085 + lats[776] = 35.395429803765317 + lats[777] = 35.325131036009047 + lats[778] = 35.254832268249267 + lats[779] = 35.184533500486005 + lats[780] = 35.114234732719261 + lats[781] = 35.043935964949064 + lats[782] = 34.973637197175435 + lats[783] = 34.903338429398374 + lats[784] = 34.833039661617903 + lats[785] = 34.762740893834028 + lats[786] = 34.692442126046771 + lats[787] = 34.622143358256153 + lats[788] = 34.551844590462188 + lats[789] = 34.481545822664863 + lats[790] = 34.411247054864234 + lats[791] = 34.340948287060286 + lats[792] = 34.270649519253041 + lats[793] = 34.200350751442521 + lats[794] = 34.130051983628725 + lats[795] = 34.059753215811682 + lats[796] = 33.989454447991392 + lats[797] = 33.919155680167876 + lats[798] = 33.848856912341155 + lats[799] = 33.778558144511237 + lats[800] = 33.708259376678136 + lats[801] = 33.637960608841851 + lats[802] = 33.567661841002426 + lats[803] = 33.497363073159853 + lats[804] = 33.42706430531414 + lats[805] = 33.356765537465314 + lats[806] = 33.286466769613391 + lats[807] = 33.216168001758369 + lats[808] = 33.145869233900278 + lats[809] = 33.075570466039117 + lats[810] = 33.005271698174909 + lats[811] = 32.934972930307666 + lats[812] = 32.864674162437396 + lats[813] = 32.794375394564113 + lats[814] = 32.724076626687825 + lats[815] = 32.653777858808567 + lats[816] = 32.583479090926325 + lats[817] = 32.513180323041112 + lats[818] = 32.442881555152965 + lats[819] = 32.372582787261891 + lats[820] = 32.302284019367875 + lats[821] = 32.231985251470959 + lats[822] = 32.161686483571145 + lats[823] = 32.091387715668439 + lats[824] = 32.021088947762863 + lats[825] = 31.950790179854422 + lats[826] = 31.880491411943137 + lats[827] = 31.810192644029012 + lats[828] = 31.739893876112063 + lats[829] = 31.669595108192297 + lats[830] = 31.599296340269738 + lats[831] = 31.528997572344384 + lats[832] = 31.458698804416255 + lats[833] = 31.388400036485361 + lats[834] = 31.318101268551715 + lats[835] = 31.247802500615318 + lats[836] = 31.177503732676204 + lats[837] = 31.107204964734358 + lats[838] = 31.036906196789811 + lats[839] = 30.966607428842572 + lats[840] = 30.896308660892647 + lats[841] = 30.826009892940046 + lats[842] = 30.755711124984781 + lats[843] = 30.685412357026873 + lats[844] = 30.615113589066322 + lats[845] = 30.544814821103138 + lats[846] = 30.47451605313735 + lats[847] = 30.404217285168947 + lats[848] = 30.333918517197947 + lats[849] = 30.263619749224372 + lats[850] = 30.19332098124822 + lats[851] = 30.123022213269511 + lats[852] = 30.052723445288244 + lats[853] = 29.98242467730444 + lats[854] = 29.91212590931811 + lats[855] = 29.841827141329258 + lats[856] = 29.771528373337894 + lats[857] = 29.701229605344039 + lats[858] = 29.630930837347698 + lats[859] = 29.560632069348884 + lats[860] = 29.490333301347597 + lats[861] = 29.420034533343859 + lats[862] = 29.349735765337677 + lats[863] = 29.279436997329057 + lats[864] = 29.209138229318015 + lats[865] = 29.138839461304556 + lats[866] = 29.068540693288696 + lats[867] = 28.998241925270449 + lats[868] = 28.927943157249814 + lats[869] = 28.857644389226806 + lats[870] = 28.787345621201432 + lats[871] = 28.717046853173709 + lats[872] = 28.646748085143642 + lats[873] = 28.576449317111244 + lats[874] = 28.506150549076519 + lats[875] = 28.435851781039485 + lats[876] = 28.365553013000145 + lats[877] = 28.29525424495851 + lats[878] = 28.224955476914594 + lats[879] = 28.154656708868405 + lats[880] = 28.084357940819952 + lats[881] = 28.014059172769244 + lats[882] = 27.94376040471629 + lats[883] = 27.873461636661098 + lats[884] = 27.803162868603682 + lats[885] = 27.732864100544052 + lats[886] = 27.662565332482213 + lats[887] = 27.592266564418171 + lats[888] = 27.521967796351948 + lats[889] = 27.451669028283543 + lats[890] = 27.381370260212968 + lats[891] = 27.311071492140236 + lats[892] = 27.240772724065348 + lats[893] = 27.170473955988321 + lats[894] = 27.100175187909159 + lats[895] = 27.029876419827872 + lats[896] = 26.959577651744471 + lats[897] = 26.889278883658971 + lats[898] = 26.818980115571364 + lats[899] = 26.748681347481678 + lats[900] = 26.678382579389908 + lats[901] = 26.608083811296069 + lats[902] = 26.53778504320017 + lats[903] = 26.467486275102218 + lats[904] = 26.397187507002222 + lats[905] = 26.326888738900195 + lats[906] = 26.256589970796135 + lats[907] = 26.186291202690064 + lats[908] = 26.115992434581983 + lats[909] = 26.045693666471902 + lats[910] = 25.975394898359827 + lats[911] = 25.90509613024577 + lats[912] = 25.834797362129745 + lats[913] = 25.764498594011751 + lats[914] = 25.694199825891793 + lats[915] = 25.623901057769892 + lats[916] = 25.553602289646051 + lats[917] = 25.483303521520277 + lats[918] = 25.413004753392578 + lats[919] = 25.342705985262967 + lats[920] = 25.272407217131445 + lats[921] = 25.202108448998025 + lats[922] = 25.13180968086272 + lats[923] = 25.061510912725527 + lats[924] = 24.991212144586456 + lats[925] = 24.920913376445526 + lats[926] = 24.850614608302738 + lats[927] = 24.780315840158096 + lats[928] = 24.710017072011613 + lats[929] = 24.639718303863294 + lats[930] = 24.569419535713152 + lats[931] = 24.499120767561195 + lats[932] = 24.428821999407425 + lats[933] = 24.358523231251851 + lats[934] = 24.288224463094483 + lats[935] = 24.217925694935328 + lats[936] = 24.1476269267744 + lats[937] = 24.077328158611696 + lats[938] = 24.007029390447226 + lats[939] = 23.936730622281004 + lats[940] = 23.866431854113038 + lats[941] = 23.796133085943328 + lats[942] = 23.725834317771888 + lats[943] = 23.655535549598721 + lats[944] = 23.585236781423838 + lats[945] = 23.514938013247242 + lats[946] = 23.444639245068949 + lats[947] = 23.374340476888957 + lats[948] = 23.304041708707278 + lats[949] = 23.233742940523921 + lats[950] = 23.163444172338895 + lats[951] = 23.0931454041522 + lats[952] = 23.022846635963852 + lats[953] = 22.952547867773848 + lats[954] = 22.882249099582204 + lats[955] = 22.811950331388925 + lats[956] = 22.741651563194019 + lats[957] = 22.671352794997489 + lats[958] = 22.60105402679935 + lats[959] = 22.530755258599601 + lats[960] = 22.460456490398254 + lats[961] = 22.390157722195315 + lats[962] = 22.319858953990789 + lats[963] = 22.249560185784691 + lats[964] = 22.179261417577013 + lats[965] = 22.108962649367779 + lats[966] = 22.038663881156989 + lats[967] = 21.968365112944642 + lats[968] = 21.898066344730758 + lats[969] = 21.827767576515338 + lats[970] = 21.757468808298391 + lats[971] = 21.687170040079913 + lats[972] = 21.616871271859928 + lats[973] = 21.546572503638437 + lats[974] = 21.47627373541544 + lats[975] = 21.40597496719095 + lats[976] = 21.335676198964972 + lats[977] = 21.265377430737512 + lats[978] = 21.195078662508585 + lats[979] = 21.124779894278181 + lats[980] = 21.054481126046323 + lats[981] = 20.984182357813012 + lats[982] = 20.913883589578251 + lats[983] = 20.843584821342048 + lats[984] = 20.773286053104417 + lats[985] = 20.702987284865355 + lats[986] = 20.632688516624874 + lats[987] = 20.562389748382977 + lats[988] = 20.492090980139672 + lats[989] = 20.421792211894967 + lats[990] = 20.35149344364887 + lats[991] = 20.28119467540138 + lats[992] = 20.210895907152516 + lats[993] = 20.140597138902272 + lats[994] = 20.070298370650661 + lats[995] = 19.999999602397686 + lats[996] = 19.929700834143357 + lats[997] = 19.859402065887682 + lats[998] = 19.789103297630657 + lats[999] = 19.718804529372303 + lats[1000] = 19.648505761112613 + lats[1001] = 19.578206992851602 + lats[1002] = 19.507908224589269 + lats[1003] = 19.437609456325632 + lats[1004] = 19.367310688060684 + lats[1005] = 19.297011919794439 + lats[1006] = 19.226713151526898 + lats[1007] = 19.15641438325807 + lats[1008] = 19.086115614987968 + lats[1009] = 19.015816846716586 + lats[1010] = 18.945518078443939 + lats[1011] = 18.875219310170031 + lats[1012] = 18.804920541894862 + lats[1013] = 18.734621773618446 + lats[1014] = 18.664323005340787 + lats[1015] = 18.594024237061891 + lats[1016] = 18.523725468781763 + lats[1017] = 18.453426700500408 + lats[1018] = 18.383127932217832 + lats[1019] = 18.312829163934047 + lats[1020] = 18.242530395649048 + lats[1021] = 18.172231627362851 + lats[1022] = 18.101932859075458 + lats[1023] = 18.031634090786874 + lats[1024] = 17.96133532249711 + lats[1025] = 17.89103655420616 + lats[1026] = 17.820737785914044 + lats[1027] = 17.75043901762076 + lats[1028] = 17.680140249326314 + lats[1029] = 17.60984148103071 + lats[1030] = 17.539542712733962 + lats[1031] = 17.469243944436066 + lats[1032] = 17.39894517613704 + lats[1033] = 17.328646407836878 + lats[1034] = 17.258347639535586 + lats[1035] = 17.188048871233182 + lats[1036] = 17.117750102929655 + lats[1037] = 17.04745133462502 + lats[1038] = 16.977152566319283 + lats[1039] = 16.906853798012452 + lats[1040] = 16.836555029704527 + lats[1041] = 16.766256261395515 + lats[1042] = 16.69595749308542 + lats[1043] = 16.625658724774254 + lats[1044] = 16.555359956462013 + lats[1045] = 16.485061188148713 + lats[1046] = 16.41476241983435 + lats[1047] = 16.344463651518936 + lats[1048] = 16.274164883202477 + lats[1049] = 16.203866114884974 + lats[1050] = 16.133567346566434 + lats[1051] = 16.063268578246863 + lats[1052] = 15.992969809926265 + lats[1053] = 15.922671041604652 + lats[1054] = 15.852372273282016 + lats[1055] = 15.78207350495838 + lats[1056] = 15.711774736633735 + lats[1057] = 15.641475968308091 + lats[1058] = 15.571177199981456 + lats[1059] = 15.500878431653829 + lats[1060] = 15.430579663325226 + lats[1061] = 15.360280894995643 + lats[1062] = 15.289982126665089 + lats[1063] = 15.219683358333569 + lats[1064] = 15.149384590001089 + lats[1065] = 15.07908582166765 + lats[1066] = 15.008787053333259 + lats[1067] = 14.938488284997929 + lats[1068] = 14.868189516661655 + lats[1069] = 14.797890748324447 + lats[1070] = 14.727591979986309 + lats[1071] = 14.657293211647247 + lats[1072] = 14.586994443307265 + lats[1073] = 14.516695674966371 + lats[1074] = 14.446396906624567 + lats[1075] = 14.376098138281863 + lats[1076] = 14.305799369938256 + lats[1077] = 14.23550060159376 + lats[1078] = 14.165201833248371 + lats[1079] = 14.0949030649021 + lats[1080] = 14.024604296554955 + lats[1081] = 13.954305528206934 + lats[1082] = 13.884006759858046 + lats[1083] = 13.813707991508297 + lats[1084] = 13.743409223157688 + lats[1085] = 13.673110454806226 + lats[1086] = 13.602811686453919 + lats[1087] = 13.532512918100766 + lats[1088] = 13.46221414974678 + lats[1089] = 13.391915381391959 + lats[1090] = 13.32161661303631 + lats[1091] = 13.251317844679837 + lats[1092] = 13.181019076322551 + lats[1093] = 13.110720307964451 + lats[1094] = 13.040421539605545 + lats[1095] = 12.970122771245832 + lats[1096] = 12.899824002885323 + lats[1097] = 12.829525234524022 + lats[1098] = 12.759226466161934 + lats[1099] = 12.688927697799061 + lats[1100] = 12.618628929435411 + lats[1101] = 12.548330161070988 + lats[1102] = 12.478031392705796 + lats[1103] = 12.407732624339841 + lats[1104] = 12.337433855973126 + lats[1105] = 12.267135087605659 + lats[1106] = 12.196836319237443 + lats[1107] = 12.126537550868482 + lats[1108] = 12.056238782498781 + lats[1109] = 11.985940014128348 + lats[1110] = 11.915641245757183 + lats[1111] = 11.845342477385294 + lats[1112] = 11.775043709012685 + lats[1113] = 11.704744940639358 + lats[1114] = 11.634446172265324 + lats[1115] = 11.564147403890583 + lats[1116] = 11.493848635515141 + lats[1117] = 11.423549867139002 + lats[1118] = 11.35325109876217 + lats[1119] = 11.282952330384653 + lats[1120] = 11.212653562006453 + lats[1121] = 11.142354793627575 + lats[1122] = 11.072056025248026 + lats[1123] = 11.001757256867807 + lats[1124] = 10.931458488486923 + lats[1125] = 10.861159720105382 + lats[1126] = 10.790860951723188 + lats[1127] = 10.720562183340341 + lats[1128] = 10.65026341495685 + lats[1129] = 10.579964646572719 + lats[1130] = 10.509665878187954 + lats[1131] = 10.439367109802557 + lats[1132] = 10.369068341416533 + lats[1133] = 10.298769573029887 + lats[1134] = 10.228470804642624 + lats[1135] = 10.158172036254747 + lats[1136] = 10.087873267866264 + lats[1137] = 10.017574499477174 + lats[1138] = 9.9472757310874869 + lats[1139] = 9.8769769626972046 + lats[1140] = 9.8066781943063344 + lats[1141] = 9.7363794259148779 + lats[1142] = 9.6660806575228388 + lats[1143] = 9.5957818891302242 + lats[1144] = 9.5254831207370376 + lats[1145] = 9.4551843523432826 + lats[1146] = 9.3848855839489662 + lats[1147] = 9.3145868155540921 + lats[1148] = 9.2442880471586619 + lats[1149] = 9.1739892787626829 + lats[1150] = 9.1036905103661585 + lats[1151] = 9.0333917419690941 + lats[1152] = 8.963092973571495 + lats[1153] = 8.8927942051733631 + lats[1154] = 8.8224954367747017 + lats[1155] = 8.7521966683755217 + lats[1156] = 8.6818978999758194 + lats[1157] = 8.6115991315756055 + lats[1158] = 8.5413003631748801 + lats[1159] = 8.4710015947736537 + lats[1160] = 8.4007028263719228 + lats[1161] = 8.3304040579696963 + lats[1162] = 8.2601052895669778 + lats[1163] = 8.1898065211637725 + lats[1164] = 8.1195077527600841 + lats[1165] = 8.049208984355916 + lats[1166] = 7.9789102159512737 + lats[1167] = 7.9086114475461606 + lats[1168] = 7.8383126791405831 + lats[1169] = 7.7680139107345463 + lats[1170] = 7.6977151423280494 + lats[1171] = 7.6274163739210996 + lats[1172] = 7.557117605513703 + lats[1173] = 7.4868188371058624 + lats[1174] = 7.4165200686975803 + lats[1175] = 7.3462213002888648 + lats[1176] = 7.2759225318797176 + lats[1177] = 7.2056237634701441 + lats[1178] = 7.1353249950601469 + lats[1179] = 7.0650262266497315 + lats[1180] = 6.994727458238903 + lats[1181] = 6.924428689827665 + lats[1182] = 6.8541299214160212 + lats[1183] = 6.7838311530039768 + lats[1184] = 6.7135323845915353 + lats[1185] = 6.6432336161787013 + lats[1186] = 6.5729348477654792 + lats[1187] = 6.5026360793518734 + lats[1188] = 6.4323373109378874 + lats[1189] = 6.3620385425235257 + lats[1190] = 6.2917397741087928 + lats[1191] = 6.2214410056936931 + lats[1192] = 6.151142237278231 + lats[1193] = 6.0808434688624091 + lats[1194] = 6.0105447004462347 + lats[1195] = 5.9402459320297085 + lats[1196] = 5.869947163612836 + lats[1197] = 5.7996483951956233 + lats[1198] = 5.729349626778073 + lats[1199] = 5.6590508583601888 + lats[1200] = 5.5887520899419751 + lats[1201] = 5.5184533215234373 + lats[1202] = 5.4481545531045787 + lats[1203] = 5.3778557846854023 + lats[1204] = 5.3075570162659149 + lats[1205] = 5.2372582478461194 + lats[1206] = 5.1669594794260192 + lats[1207] = 5.0966607110056197 + lats[1208] = 5.0263619425849244 + lats[1209] = 4.9560631741639369 + lats[1210] = 4.8857644057426626 + lats[1211] = 4.8154656373211049 + lats[1212] = 4.7451668688992683 + lats[1213] = 4.6748681004771564 + lats[1214] = 4.6045693320547736 + lats[1215] = 4.5342705636321252 + lats[1216] = 4.4639717952092139 + lats[1217] = 4.3936730267860451 + lats[1218] = 4.3233742583626205 + lats[1219] = 4.2530754899389471 + lats[1220] = 4.1827767215150269 + lats[1221] = 4.1124779530908659 + lats[1222] = 4.0421791846664661 + lats[1223] = 3.9718804162418326 + lats[1224] = 3.90158164781697 + lats[1225] = 3.8312828793918823 + lats[1226] = 3.7609841109665734 + lats[1227] = 3.6906853425410477 + lats[1228] = 3.6203865741153085 + lats[1229] = 3.5500878056893601 + lats[1230] = 3.4797890372632065 + lats[1231] = 3.4094902688368531 + lats[1232] = 3.339191500410303 + lats[1233] = 3.2688927319835597 + lats[1234] = 3.1985939635566285 + lats[1235] = 3.1282951951295126 + lats[1236] = 3.0579964267022164 + lats[1237] = 2.9876976582747439 + lats[1238] = 2.9173988898470999 + lats[1239] = 2.8471001214192873 + lats[1240] = 2.7768013529913107 + lats[1241] = 2.7065025845631743 + lats[1242] = 2.6362038161348824 + lats[1243] = 2.5659050477064382 + lats[1244] = 2.4956062792778466 + lats[1245] = 2.4253075108491116 + lats[1246] = 2.3550087424202366 + lats[1247] = 2.2847099739912267 + lats[1248] = 2.2144112055620848 + lats[1249] = 2.1441124371328155 + lats[1250] = 2.0738136687034232 + lats[1251] = 2.0035149002739114 + lats[1252] = 1.9332161318442849 + lats[1253] = 1.8629173634145471 + lats[1254] = 1.792618594984702 + lats[1255] = 1.7223198265547539 + lats[1256] = 1.6520210581247066 + lats[1257] = 1.5817222896945646 + lats[1258] = 1.5114235212643317 + lats[1259] = 1.4411247528340119 + lats[1260] = 1.3708259844036093 + lats[1261] = 1.300527215973128 + lats[1262] = 1.2302284475425722 + lats[1263] = 1.1599296791119456 + lats[1264] = 1.0896309106812523 + lats[1265] = 1.0193321422504964 + lats[1266] = 0.949033373819682 + lats[1267] = 0.87873460538881287 + lats[1268] = 0.80843583695789356 + lats[1269] = 0.73813706852692773 + lats[1270] = 0.66783830009591949 + lats[1271] = 0.59753953166487306 + lats[1272] = 0.52724076323379232 + lats[1273] = 0.45694199480268116 + lats[1274] = 0.3866432263715438 + lats[1275] = 0.31634445794038429 + lats[1276] = 0.24604568950920663 + lats[1277] = 0.17574692107801482 + lats[1278] = 0.10544815264681295 + lats[1279] = 0.035149384215604956 + lats[1280] = -0.035149384215604956 + lats[1281] = -0.10544815264681295 + lats[1282] = -0.17574692107801482 + lats[1283] = -0.24604568950920663 + lats[1284] = -0.31634445794038429 + lats[1285] = -0.3866432263715438 + lats[1286] = -0.45694199480268116 + lats[1287] = -0.52724076323379232 + lats[1288] = -0.59753953166487306 + lats[1289] = -0.66783830009591949 + lats[1290] = -0.73813706852692773 + lats[1291] = -0.80843583695789356 + lats[1292] = -0.87873460538881287 + lats[1293] = -0.949033373819682 + lats[1294] = -1.0193321422504964 + lats[1295] = -1.0896309106812523 + lats[1296] = -1.1599296791119456 + lats[1297] = -1.2302284475425722 + lats[1298] = -1.300527215973128 + lats[1299] = -1.3708259844036093 + lats[1300] = -1.4411247528340119 + lats[1301] = -1.5114235212643317 + lats[1302] = -1.5817222896945646 + lats[1303] = -1.6520210581247066 + lats[1304] = -1.7223198265547539 + lats[1305] = -1.792618594984702 + lats[1306] = -1.8629173634145471 + lats[1307] = -1.9332161318442849 + lats[1308] = -2.0035149002739114 + lats[1309] = -2.0738136687034232 + lats[1310] = -2.1441124371328155 + lats[1311] = -2.2144112055620848 + lats[1312] = -2.2847099739912267 + lats[1313] = -2.3550087424202366 + lats[1314] = -2.4253075108491116 + lats[1315] = -2.4956062792778466 + lats[1316] = -2.5659050477064382 + lats[1317] = -2.6362038161348824 + lats[1318] = -2.7065025845631743 + lats[1319] = -2.7768013529913107 + lats[1320] = -2.8471001214192873 + lats[1321] = -2.9173988898470999 + lats[1322] = -2.9876976582747439 + lats[1323] = -3.0579964267022164 + lats[1324] = -3.1282951951295126 + lats[1325] = -3.1985939635566285 + lats[1326] = -3.2688927319835597 + lats[1327] = -3.339191500410303 + lats[1328] = -3.4094902688368531 + lats[1329] = -3.4797890372632065 + lats[1330] = -3.5500878056893601 + lats[1331] = -3.6203865741153085 + lats[1332] = -3.6906853425410477 + lats[1333] = -3.7609841109665734 + lats[1334] = -3.8312828793918823 + lats[1335] = -3.90158164781697 + lats[1336] = -3.9718804162418326 + lats[1337] = -4.0421791846664661 + lats[1338] = -4.1124779530908659 + lats[1339] = -4.1827767215150269 + lats[1340] = -4.2530754899389471 + lats[1341] = -4.3233742583626205 + lats[1342] = -4.3936730267860451 + lats[1343] = -4.4639717952092139 + lats[1344] = -4.5342705636321252 + lats[1345] = -4.6045693320547736 + lats[1346] = -4.6748681004771564 + lats[1347] = -4.7451668688992683 + lats[1348] = -4.8154656373211049 + lats[1349] = -4.8857644057426626 + lats[1350] = -4.9560631741639369 + lats[1351] = -5.0263619425849244 + lats[1352] = -5.0966607110056197 + lats[1353] = -5.1669594794260192 + lats[1354] = -5.2372582478461194 + lats[1355] = -5.3075570162659149 + lats[1356] = -5.3778557846854023 + lats[1357] = -5.4481545531045787 + lats[1358] = -5.5184533215234373 + lats[1359] = -5.5887520899419751 + lats[1360] = -5.6590508583601888 + lats[1361] = -5.729349626778073 + lats[1362] = -5.7996483951956233 + lats[1363] = -5.869947163612836 + lats[1364] = -5.9402459320297085 + lats[1365] = -6.0105447004462347 + lats[1366] = -6.0808434688624091 + lats[1367] = -6.151142237278231 + lats[1368] = -6.2214410056936931 + lats[1369] = -6.2917397741087928 + lats[1370] = -6.3620385425235257 + lats[1371] = -6.4323373109378874 + lats[1372] = -6.5026360793518734 + lats[1373] = -6.5729348477654792 + lats[1374] = -6.6432336161787013 + lats[1375] = -6.7135323845915353 + lats[1376] = -6.7838311530039768 + lats[1377] = -6.8541299214160212 + lats[1378] = -6.924428689827665 + lats[1379] = -6.994727458238903 + lats[1380] = -7.0650262266497315 + lats[1381] = -7.1353249950601469 + lats[1382] = -7.2056237634701441 + lats[1383] = -7.2759225318797176 + lats[1384] = -7.3462213002888648 + lats[1385] = -7.4165200686975803 + lats[1386] = -7.4868188371058624 + lats[1387] = -7.557117605513703 + lats[1388] = -7.6274163739210996 + lats[1389] = -7.6977151423280494 + lats[1390] = -7.7680139107345463 + lats[1391] = -7.8383126791405831 + lats[1392] = -7.9086114475461606 + lats[1393] = -7.9789102159512737 + lats[1394] = -8.049208984355916 + lats[1395] = -8.1195077527600841 + lats[1396] = -8.1898065211637725 + lats[1397] = -8.2601052895669778 + lats[1398] = -8.3304040579696963 + lats[1399] = -8.4007028263719228 + lats[1400] = -8.4710015947736537 + lats[1401] = -8.5413003631748801 + lats[1402] = -8.6115991315756055 + lats[1403] = -8.6818978999758194 + lats[1404] = -8.7521966683755217 + lats[1405] = -8.8224954367747017 + lats[1406] = -8.8927942051733631 + lats[1407] = -8.963092973571495 + lats[1408] = -9.0333917419690941 + lats[1409] = -9.1036905103661585 + lats[1410] = -9.1739892787626829 + lats[1411] = -9.2442880471586619 + lats[1412] = -9.3145868155540921 + lats[1413] = -9.3848855839489662 + lats[1414] = -9.4551843523432826 + lats[1415] = -9.5254831207370376 + lats[1416] = -9.5957818891302242 + lats[1417] = -9.6660806575228388 + lats[1418] = -9.7363794259148779 + lats[1419] = -9.8066781943063344 + lats[1420] = -9.8769769626972046 + lats[1421] = -9.9472757310874869 + lats[1422] = -10.017574499477174 + lats[1423] = -10.087873267866264 + lats[1424] = -10.158172036254747 + lats[1425] = -10.228470804642624 + lats[1426] = -10.298769573029887 + lats[1427] = -10.369068341416533 + lats[1428] = -10.439367109802557 + lats[1429] = -10.509665878187954 + lats[1430] = -10.579964646572719 + lats[1431] = -10.65026341495685 + lats[1432] = -10.720562183340341 + lats[1433] = -10.790860951723188 + lats[1434] = -10.861159720105382 + lats[1435] = -10.931458488486923 + lats[1436] = -11.001757256867807 + lats[1437] = -11.072056025248026 + lats[1438] = -11.142354793627575 + lats[1439] = -11.212653562006453 + lats[1440] = -11.282952330384653 + lats[1441] = -11.35325109876217 + lats[1442] = -11.423549867139002 + lats[1443] = -11.493848635515141 + lats[1444] = -11.564147403890583 + lats[1445] = -11.634446172265324 + lats[1446] = -11.704744940639358 + lats[1447] = -11.775043709012685 + lats[1448] = -11.845342477385294 + lats[1449] = -11.915641245757183 + lats[1450] = -11.985940014128348 + lats[1451] = -12.056238782498781 + lats[1452] = -12.126537550868482 + lats[1453] = -12.196836319237443 + lats[1454] = -12.267135087605659 + lats[1455] = -12.337433855973126 + lats[1456] = -12.407732624339841 + lats[1457] = -12.478031392705796 + lats[1458] = -12.548330161070988 + lats[1459] = -12.618628929435411 + lats[1460] = -12.688927697799061 + lats[1461] = -12.759226466161934 + lats[1462] = -12.829525234524022 + lats[1463] = -12.899824002885323 + lats[1464] = -12.970122771245832 + lats[1465] = -13.040421539605545 + lats[1466] = -13.110720307964451 + lats[1467] = -13.181019076322551 + lats[1468] = -13.251317844679837 + lats[1469] = -13.32161661303631 + lats[1470] = -13.391915381391959 + lats[1471] = -13.46221414974678 + lats[1472] = -13.532512918100766 + lats[1473] = -13.602811686453919 + lats[1474] = -13.673110454806226 + lats[1475] = -13.743409223157688 + lats[1476] = -13.813707991508297 + lats[1477] = -13.884006759858046 + lats[1478] = -13.954305528206934 + lats[1479] = -14.024604296554955 + lats[1480] = -14.0949030649021 + lats[1481] = -14.165201833248371 + lats[1482] = -14.23550060159376 + lats[1483] = -14.305799369938256 + lats[1484] = -14.376098138281863 + lats[1485] = -14.446396906624567 + lats[1486] = -14.516695674966371 + lats[1487] = -14.586994443307265 + lats[1488] = -14.657293211647247 + lats[1489] = -14.727591979986309 + lats[1490] = -14.797890748324447 + lats[1491] = -14.868189516661655 + lats[1492] = -14.938488284997929 + lats[1493] = -15.008787053333259 + lats[1494] = -15.07908582166765 + lats[1495] = -15.149384590001089 + lats[1496] = -15.219683358333569 + lats[1497] = -15.289982126665089 + lats[1498] = -15.360280894995643 + lats[1499] = -15.430579663325226 + lats[1500] = -15.500878431653829 + lats[1501] = -15.571177199981456 + lats[1502] = -15.641475968308091 + lats[1503] = -15.711774736633735 + lats[1504] = -15.78207350495838 + lats[1505] = -15.852372273282016 + lats[1506] = -15.922671041604652 + lats[1507] = -15.992969809926265 + lats[1508] = -16.063268578246863 + lats[1509] = -16.133567346566434 + lats[1510] = -16.203866114884974 + lats[1511] = -16.274164883202477 + lats[1512] = -16.344463651518936 + lats[1513] = -16.41476241983435 + lats[1514] = -16.485061188148713 + lats[1515] = -16.555359956462013 + lats[1516] = -16.625658724774254 + lats[1517] = -16.69595749308542 + lats[1518] = -16.766256261395515 + lats[1519] = -16.836555029704527 + lats[1520] = -16.906853798012452 + lats[1521] = -16.977152566319283 + lats[1522] = -17.04745133462502 + lats[1523] = -17.117750102929655 + lats[1524] = -17.188048871233182 + lats[1525] = -17.258347639535586 + lats[1526] = -17.328646407836878 + lats[1527] = -17.39894517613704 + lats[1528] = -17.469243944436066 + lats[1529] = -17.539542712733962 + lats[1530] = -17.60984148103071 + lats[1531] = -17.680140249326314 + lats[1532] = -17.75043901762076 + lats[1533] = -17.820737785914044 + lats[1534] = -17.89103655420616 + lats[1535] = -17.96133532249711 + lats[1536] = -18.031634090786874 + lats[1537] = -18.101932859075458 + lats[1538] = -18.172231627362851 + lats[1539] = -18.242530395649048 + lats[1540] = -18.312829163934047 + lats[1541] = -18.383127932217832 + lats[1542] = -18.453426700500408 + lats[1543] = -18.523725468781763 + lats[1544] = -18.594024237061891 + lats[1545] = -18.664323005340787 + lats[1546] = -18.734621773618446 + lats[1547] = -18.804920541894862 + lats[1548] = -18.875219310170031 + lats[1549] = -18.945518078443939 + lats[1550] = -19.015816846716586 + lats[1551] = -19.086115614987968 + lats[1552] = -19.15641438325807 + lats[1553] = -19.226713151526898 + lats[1554] = -19.297011919794439 + lats[1555] = -19.367310688060684 + lats[1556] = -19.437609456325632 + lats[1557] = -19.507908224589269 + lats[1558] = -19.578206992851602 + lats[1559] = -19.648505761112613 + lats[1560] = -19.718804529372303 + lats[1561] = -19.789103297630657 + lats[1562] = -19.859402065887682 + lats[1563] = -19.929700834143357 + lats[1564] = -19.999999602397686 + lats[1565] = -20.070298370650661 + lats[1566] = -20.140597138902272 + lats[1567] = -20.210895907152516 + lats[1568] = -20.28119467540138 + lats[1569] = -20.35149344364887 + lats[1570] = -20.421792211894967 + lats[1571] = -20.492090980139672 + lats[1572] = -20.562389748382977 + lats[1573] = -20.632688516624874 + lats[1574] = -20.702987284865355 + lats[1575] = -20.773286053104417 + lats[1576] = -20.843584821342048 + lats[1577] = -20.913883589578251 + lats[1578] = -20.984182357813012 + lats[1579] = -21.054481126046323 + lats[1580] = -21.124779894278181 + lats[1581] = -21.195078662508585 + lats[1582] = -21.265377430737512 + lats[1583] = -21.335676198964972 + lats[1584] = -21.40597496719095 + lats[1585] = -21.47627373541544 + lats[1586] = -21.546572503638437 + lats[1587] = -21.616871271859928 + lats[1588] = -21.687170040079913 + lats[1589] = -21.757468808298391 + lats[1590] = -21.827767576515338 + lats[1591] = -21.898066344730758 + lats[1592] = -21.968365112944642 + lats[1593] = -22.038663881156989 + lats[1594] = -22.108962649367779 + lats[1595] = -22.179261417577013 + lats[1596] = -22.249560185784691 + lats[1597] = -22.319858953990789 + lats[1598] = -22.390157722195315 + lats[1599] = -22.460456490398254 + lats[1600] = -22.530755258599601 + lats[1601] = -22.60105402679935 + lats[1602] = -22.671352794997489 + lats[1603] = -22.741651563194019 + lats[1604] = -22.811950331388925 + lats[1605] = -22.882249099582204 + lats[1606] = -22.952547867773848 + lats[1607] = -23.022846635963852 + lats[1608] = -23.0931454041522 + lats[1609] = -23.163444172338895 + lats[1610] = -23.233742940523921 + lats[1611] = -23.304041708707278 + lats[1612] = -23.374340476888957 + lats[1613] = -23.444639245068949 + lats[1614] = -23.514938013247242 + lats[1615] = -23.585236781423838 + lats[1616] = -23.655535549598721 + lats[1617] = -23.725834317771888 + lats[1618] = -23.796133085943328 + lats[1619] = -23.866431854113038 + lats[1620] = -23.936730622281004 + lats[1621] = -24.007029390447226 + lats[1622] = -24.077328158611696 + lats[1623] = -24.1476269267744 + lats[1624] = -24.217925694935328 + lats[1625] = -24.288224463094483 + lats[1626] = -24.358523231251851 + lats[1627] = -24.428821999407425 + lats[1628] = -24.499120767561195 + lats[1629] = -24.569419535713152 + lats[1630] = -24.639718303863294 + lats[1631] = -24.710017072011613 + lats[1632] = -24.780315840158096 + lats[1633] = -24.850614608302738 + lats[1634] = -24.920913376445526 + lats[1635] = -24.991212144586456 + lats[1636] = -25.061510912725527 + lats[1637] = -25.13180968086272 + lats[1638] = -25.202108448998025 + lats[1639] = -25.272407217131445 + lats[1640] = -25.342705985262967 + lats[1641] = -25.413004753392578 + lats[1642] = -25.483303521520277 + lats[1643] = -25.553602289646051 + lats[1644] = -25.623901057769892 + lats[1645] = -25.694199825891793 + lats[1646] = -25.764498594011751 + lats[1647] = -25.834797362129745 + lats[1648] = -25.90509613024577 + lats[1649] = -25.975394898359827 + lats[1650] = -26.045693666471902 + lats[1651] = -26.115992434581983 + lats[1652] = -26.186291202690064 + lats[1653] = -26.256589970796135 + lats[1654] = -26.326888738900195 + lats[1655] = -26.397187507002222 + lats[1656] = -26.467486275102218 + lats[1657] = -26.53778504320017 + lats[1658] = -26.608083811296069 + lats[1659] = -26.678382579389908 + lats[1660] = -26.748681347481678 + lats[1661] = -26.818980115571364 + lats[1662] = -26.889278883658971 + lats[1663] = -26.959577651744471 + lats[1664] = -27.029876419827872 + lats[1665] = -27.100175187909159 + lats[1666] = -27.170473955988321 + lats[1667] = -27.240772724065348 + lats[1668] = -27.311071492140236 + lats[1669] = -27.381370260212968 + lats[1670] = -27.451669028283543 + lats[1671] = -27.521967796351948 + lats[1672] = -27.592266564418171 + lats[1673] = -27.662565332482213 + lats[1674] = -27.732864100544052 + lats[1675] = -27.803162868603682 + lats[1676] = -27.873461636661098 + lats[1677] = -27.94376040471629 + lats[1678] = -28.014059172769244 + lats[1679] = -28.084357940819952 + lats[1680] = -28.154656708868405 + lats[1681] = -28.224955476914594 + lats[1682] = -28.29525424495851 + lats[1683] = -28.365553013000145 + lats[1684] = -28.435851781039485 + lats[1685] = -28.506150549076519 + lats[1686] = -28.576449317111244 + lats[1687] = -28.646748085143642 + lats[1688] = -28.717046853173709 + lats[1689] = -28.787345621201432 + lats[1690] = -28.857644389226806 + lats[1691] = -28.927943157249814 + lats[1692] = -28.998241925270449 + lats[1693] = -29.068540693288696 + lats[1694] = -29.138839461304556 + lats[1695] = -29.209138229318015 + lats[1696] = -29.279436997329057 + lats[1697] = -29.349735765337677 + lats[1698] = -29.420034533343859 + lats[1699] = -29.490333301347597 + lats[1700] = -29.560632069348884 + lats[1701] = -29.630930837347698 + lats[1702] = -29.701229605344039 + lats[1703] = -29.771528373337894 + lats[1704] = -29.841827141329258 + lats[1705] = -29.91212590931811 + lats[1706] = -29.98242467730444 + lats[1707] = -30.052723445288244 + lats[1708] = -30.123022213269511 + lats[1709] = -30.19332098124822 + lats[1710] = -30.263619749224372 + lats[1711] = -30.333918517197947 + lats[1712] = -30.404217285168947 + lats[1713] = -30.47451605313735 + lats[1714] = -30.544814821103138 + lats[1715] = -30.615113589066322 + lats[1716] = -30.685412357026873 + lats[1717] = -30.755711124984781 + lats[1718] = -30.826009892940046 + lats[1719] = -30.896308660892647 + lats[1720] = -30.966607428842572 + lats[1721] = -31.036906196789811 + lats[1722] = -31.107204964734358 + lats[1723] = -31.177503732676204 + lats[1724] = -31.247802500615318 + lats[1725] = -31.318101268551715 + lats[1726] = -31.388400036485361 + lats[1727] = -31.458698804416255 + lats[1728] = -31.528997572344384 + lats[1729] = -31.599296340269738 + lats[1730] = -31.669595108192297 + lats[1731] = -31.739893876112063 + lats[1732] = -31.810192644029012 + lats[1733] = -31.880491411943137 + lats[1734] = -31.950790179854422 + lats[1735] = -32.021088947762863 + lats[1736] = -32.091387715668439 + lats[1737] = -32.161686483571145 + lats[1738] = -32.231985251470959 + lats[1739] = -32.302284019367875 + lats[1740] = -32.372582787261891 + lats[1741] = -32.442881555152965 + lats[1742] = -32.513180323041112 + lats[1743] = -32.583479090926325 + lats[1744] = -32.653777858808567 + lats[1745] = -32.724076626687825 + lats[1746] = -32.794375394564113 + lats[1747] = -32.864674162437396 + lats[1748] = -32.934972930307666 + lats[1749] = -33.005271698174909 + lats[1750] = -33.075570466039117 + lats[1751] = -33.145869233900278 + lats[1752] = -33.216168001758369 + lats[1753] = -33.286466769613391 + lats[1754] = -33.356765537465314 + lats[1755] = -33.42706430531414 + lats[1756] = -33.497363073159853 + lats[1757] = -33.567661841002426 + lats[1758] = -33.637960608841851 + lats[1759] = -33.708259376678136 + lats[1760] = -33.778558144511237 + lats[1761] = -33.848856912341155 + lats[1762] = -33.919155680167876 + lats[1763] = -33.989454447991392 + lats[1764] = -34.059753215811682 + lats[1765] = -34.130051983628725 + lats[1766] = -34.200350751442521 + lats[1767] = -34.270649519253041 + lats[1768] = -34.340948287060286 + lats[1769] = -34.411247054864234 + lats[1770] = -34.481545822664863 + lats[1771] = -34.551844590462188 + lats[1772] = -34.622143358256153 + lats[1773] = -34.692442126046771 + lats[1774] = -34.762740893834028 + lats[1775] = -34.833039661617903 + lats[1776] = -34.903338429398374 + lats[1777] = -34.973637197175435 + lats[1778] = -35.043935964949064 + lats[1779] = -35.114234732719261 + lats[1780] = -35.184533500486005 + lats[1781] = -35.254832268249267 + lats[1782] = -35.325131036009047 + lats[1783] = -35.395429803765317 + lats[1784] = -35.465728571518085 + lats[1785] = -35.536027339267314 + lats[1786] = -35.606326107012997 + lats[1787] = -35.676624874755113 + lats[1788] = -35.746923642493655 + lats[1789] = -35.817222410228595 + lats[1790] = -35.887521177959933 + lats[1791] = -35.957819945687639 + lats[1792] = -36.028118713411708 + lats[1793] = -36.098417481132117 + lats[1794] = -36.16871624884886 + lats[1795] = -36.239015016561908 + lats[1796] = -36.309313784271254 + lats[1797] = -36.379612551976876 + lats[1798] = -36.449911319678755 + lats[1799] = -36.520210087376888 + lats[1800] = -36.590508855071242 + lats[1801] = -36.660807622761808 + lats[1802] = -36.731106390448581 + lats[1803] = -36.801405158131523 + lats[1804] = -36.871703925810628 + lats[1805] = -36.942002693485883 + lats[1806] = -37.012301461157264 + lats[1807] = -37.082600228824752 + lats[1808] = -37.152898996488332 + lats[1809] = -37.223197764147997 + lats[1810] = -37.293496531803719 + lats[1811] = -37.363795299455489 + lats[1812] = -37.434094067103274 + lats[1813] = -37.504392834747065 + lats[1814] = -37.574691602386856 + lats[1815] = -37.644990370022605 + lats[1816] = -37.715289137654317 + lats[1817] = -37.785587905281965 + lats[1818] = -37.855886672905527 + lats[1819] = -37.926185440524989 + lats[1820] = -37.99648420814033 + lats[1821] = -38.066782975751536 + lats[1822] = -38.137081743358586 + lats[1823] = -38.20738051096145 + lats[1824] = -38.277679278560143 + lats[1825] = -38.347978046154608 + lats[1826] = -38.418276813744846 + lats[1827] = -38.488575581330842 + lats[1828] = -38.558874348912568 + lats[1829] = -38.629173116490001 + lats[1830] = -38.699471884063136 + lats[1831] = -38.769770651631937 + lats[1832] = -38.840069419196389 + lats[1833] = -38.910368186756479 + lats[1834] = -38.980666954312184 + lats[1835] = -39.050965721863491 + lats[1836] = -39.121264489410365 + lats[1837] = -39.191563256952804 + lats[1838] = -39.261862024490775 + lats[1839] = -39.332160792024254 + lats[1840] = -39.402459559553229 + lats[1841] = -39.472758327077692 + lats[1842] = -39.543057094597607 + lats[1843] = -39.613355862112947 + lats[1844] = -39.683654629623703 + lats[1845] = -39.753953397129855 + lats[1846] = -39.824252164631375 + lats[1847] = -39.894550932128247 + lats[1848] = -39.964849699620437 + lats[1849] = -40.035148467107952 + lats[1850] = -40.105447234590748 + lats[1851] = -40.175746002068806 + lats[1852] = -40.246044769542102 + lats[1853] = -40.316343537010617 + lats[1854] = -40.386642304474343 + lats[1855] = -40.456941071933244 + lats[1856] = -40.527239839387299 + lats[1857] = -40.597538606836487 + lats[1858] = -40.667837374280786 + lats[1859] = -40.738136141720176 + lats[1860] = -40.808434909154634 + lats[1861] = -40.878733676584126 + lats[1862] = -40.949032444008644 + lats[1863] = -41.01933121142816 + lats[1864] = -41.089629978842645 + lats[1865] = -41.159928746252085 + lats[1866] = -41.230227513656445 + lats[1867] = -41.300526281055724 + lats[1868] = -41.370825048449873 + lats[1869] = -41.441123815838885 + lats[1870] = -41.511422583222718 + lats[1871] = -41.581721350601363 + lats[1872] = -41.6520201179748 + lats[1873] = -41.722318885343 + lats[1874] = -41.792617652705921 + lats[1875] = -41.862916420063563 + lats[1876] = -41.933215187415882 + lats[1877] = -42.003513954762873 + lats[1878] = -42.073812722104492 + lats[1879] = -42.144111489440725 + lats[1880] = -42.214410256771551 + lats[1881] = -42.284709024096927 + lats[1882] = -42.355007791416853 + lats[1883] = -42.425306558731272 + lats[1884] = -42.495605326040177 + lats[1885] = -42.565904093343548 + lats[1886] = -42.63620286064134 + lats[1887] = -42.706501627933541 + lats[1888] = -42.776800395220121 + lats[1889] = -42.847099162501053 + lats[1890] = -42.917397929776307 + lats[1891] = -42.987696697045862 + lats[1892] = -43.057995464309691 + lats[1893] = -43.128294231567757 + lats[1894] = -43.19859299882004 + lats[1895] = -43.26889176606651 + lats[1896] = -43.339190533307139 + lats[1897] = -43.409489300541907 + lats[1898] = -43.479788067770777 + lats[1899] = -43.550086834993728 + lats[1900] = -43.620385602210717 + lats[1901] = -43.690684369421732 + lats[1902] = -43.760983136626741 + lats[1903] = -43.831281903825705 + lats[1904] = -43.9015806710186 + lats[1905] = -43.971879438205391 + lats[1906] = -44.042178205386072 + lats[1907] = -44.112476972560586 + lats[1908] = -44.182775739728925 + lats[1909] = -44.253074506891046 + lats[1910] = -44.323373274046915 + lats[1911] = -44.39367204119651 + lats[1912] = -44.463970808339802 + lats[1913] = -44.534269575476756 + lats[1914] = -44.604568342607337 + lats[1915] = -44.674867109731515 + lats[1916] = -44.745165876849271 + lats[1917] = -44.81546464396056 + lats[1918] = -44.885763411065362 + lats[1919] = -44.956062178163634 + lats[1920] = -45.026360945255341 + lats[1921] = -45.096659712340461 + lats[1922] = -45.166958479418959 + lats[1923] = -45.237257246490813 + lats[1924] = -45.30755601355596 + lats[1925] = -45.377854780614399 + lats[1926] = -45.448153547666081 + lats[1927] = -45.51845231471097 + lats[1928] = -45.588751081749038 + lats[1929] = -45.659049848780263 + lats[1930] = -45.729348615804589 + lats[1931] = -45.799647382821995 + lats[1932] = -45.869946149832437 + lats[1933] = -45.94024491683588 + lats[1934] = -46.01054368383231 + lats[1935] = -46.080842450821663 + lats[1936] = -46.151141217803925 + lats[1937] = -46.221439984779053 + lats[1938] = -46.291738751747012 + lats[1939] = -46.362037518707766 + lats[1940] = -46.432336285661272 + lats[1941] = -46.502635052607502 + lats[1942] = -46.572933819546414 + lats[1943] = -46.643232586477971 + lats[1944] = -46.713531353402139 + lats[1945] = -46.783830120318882 + lats[1946] = -46.85412888722815 + lats[1947] = -46.924427654129929 + lats[1948] = -46.994726421024154 + lats[1949] = -47.065025187910805 + lats[1950] = -47.13532395478984 + lats[1951] = -47.205622721661214 + lats[1952] = -47.275921488524894 + lats[1953] = -47.346220255380835 + lats[1954] = -47.416519022228997 + lats[1955] = -47.486817789069342 + lats[1956] = -47.557116555901828 + lats[1957] = -47.627415322726435 + lats[1958] = -47.697714089543084 + lats[1959] = -47.76801285635176 + lats[1960] = -47.838311623152421 + lats[1961] = -47.908610389945018 + lats[1962] = -47.978909156729507 + lats[1963] = -48.049207923505868 + lats[1964] = -48.119506690274015 + lats[1965] = -48.189805457033941 + lats[1966] = -48.260104223785596 + lats[1967] = -48.330402990528938 + lats[1968] = -48.400701757263917 + lats[1969] = -48.47100052399049 + lats[1970] = -48.541299290708608 + lats[1971] = -48.611598057418242 + lats[1972] = -48.681896824119335 + lats[1973] = -48.752195590811837 + lats[1974] = -48.822494357495721 + lats[1975] = -48.892793124170929 + lats[1976] = -48.963091890837418 + lats[1977] = -49.03339065749514 + lats[1978] = -49.103689424144044 + lats[1979] = -49.173988190784094 + lats[1980] = -49.244286957415234 + lats[1981] = -49.314585724037435 + lats[1982] = -49.384884490650613 + lats[1983] = -49.455183257254745 + lats[1984] = -49.525482023849783 + lats[1985] = -49.595780790435676 + lats[1986] = -49.66607955701236 + lats[1987] = -49.736378323579807 + lats[1988] = -49.80667709013796 + lats[1989] = -49.876975856686762 + lats[1990] = -49.947274623226157 + lats[1991] = -50.017573389756123 + lats[1992] = -50.087872156276575 + lats[1993] = -50.158170922787484 + lats[1994] = -50.228469689288779 + lats[1995] = -50.298768455780426 + lats[1996] = -50.369067222262359 + lats[1997] = -50.439365988734544 + lats[1998] = -50.509664755196901 + lats[1999] = -50.579963521649397 + lats[2000] = -50.650262288091959 + lats[2001] = -50.720561054524559 + lats[2002] = -50.790859820947119 + lats[2003] = -50.86115858735959 + lats[2004] = -50.931457353761914 + lats[2005] = -51.001756120154049 + lats[2006] = -51.072054886535909 + lats[2007] = -51.14235365290746 + lats[2008] = -51.21265241926865 + lats[2009] = -51.282951185619417 + lats[2010] = -51.353249951959683 + lats[2011] = -51.423548718289396 + lats[2012] = -51.493847484608516 + lats[2013] = -51.56414625091697 + lats[2014] = -51.634445017214695 + lats[2015] = -51.704743783501634 + lats[2016] = -51.775042549777737 + lats[2017] = -51.845341316042933 + lats[2018] = -51.915640082297152 + lats[2019] = -51.985938848540336 + lats[2020] = -52.056237614772435 + lats[2021] = -52.126536380993372 + lats[2022] = -52.196835147203096 + lats[2023] = -52.26713391340153 + lats[2024] = -52.337432679588609 + lats[2025] = -52.407731445764284 + lats[2026] = -52.478030211928477 + lats[2027] = -52.548328978081123 + lats[2028] = -52.618627744222159 + lats[2029] = -52.688926510351514 + lats[2030] = -52.759225276469131 + lats[2031] = -52.829524042574917 + lats[2032] = -52.899822808668837 + lats[2033] = -52.970121574750792 + lats[2034] = -53.040420340820731 + lats[2035] = -53.110719106878584 + lats[2036] = -53.181017872924265 + lats[2037] = -53.251316638957725 + lats[2038] = -53.321615404978871 + lats[2039] = -53.391914170987633 + lats[2040] = -53.462212936983953 + lats[2041] = -53.53251170296776 + lats[2042] = -53.602810468938962 + lats[2043] = -53.673109234897495 + lats[2044] = -53.743408000843282 + lats[2045] = -53.813706766776235 + lats[2046] = -53.884005532696307 + lats[2047] = -53.954304298603383 + lats[2048] = -54.024603064497434 + lats[2049] = -54.094901830378333 + lats[2050] = -54.165200596246031 + lats[2051] = -54.235499362100448 + lats[2052] = -54.305798127941479 + lats[2053] = -54.376096893769081 + lats[2054] = -54.446395659583146 + lats[2055] = -54.516694425383605 + lats[2056] = -54.586993191170357 + lats[2057] = -54.657291956943347 + lats[2058] = -54.727590722702473 + lats[2059] = -54.797889488447652 + lats[2060] = -54.868188254178797 + lats[2061] = -54.938487019895831 + lats[2062] = -55.008785785598668 + lats[2063] = -55.07908455128721 + lats[2064] = -55.149383316961377 + lats[2065] = -55.219682082621084 + lats[2066] = -55.289980848266232 + lats[2067] = -55.360279613896743 + lats[2068] = -55.430578379512511 + lats[2069] = -55.500877145113449 + lats[2070] = -55.571175910699488 + lats[2071] = -55.641474676270505 + lats[2072] = -55.711773441826416 + lats[2073] = -55.782072207367136 + lats[2074] = -55.852370972892551 + lats[2075] = -55.922669738402583 + lats[2076] = -55.992968503897131 + lats[2077] = -56.063267269376091 + lats[2078] = -56.133566034839362 + lats[2079] = -56.203864800286865 + lats[2080] = -56.274163565718467 + lats[2081] = -56.34446233113411 + lats[2082] = -56.41476109653366 + lats[2083] = -56.485059861917016 + lats[2084] = -56.555358627284086 + lats[2085] = -56.625657392634771 + lats[2086] = -56.695956157968951 + lats[2087] = -56.766254923286517 + lats[2088] = -56.836553688587379 + lats[2089] = -56.90685245387143 + lats[2090] = -56.977151219138541 + lats[2091] = -57.047449984388614 + lats[2092] = -57.117748749621541 + lats[2093] = -57.188047514837208 + lats[2094] = -57.258346280035504 + lats[2095] = -57.328645045216312 + lats[2096] = -57.398943810379521 + lats[2097] = -57.469242575525016 + lats[2098] = -57.539541340652676 + lats[2099] = -57.60984010576238 + lats[2100] = -57.680138870854037 + lats[2101] = -57.75043763592749 + lats[2102] = -57.820736400982646 + lats[2103] = -57.891035166019364 + lats[2104] = -57.961333931037537 + lats[2105] = -58.031632696037022 + lats[2106] = -58.101931461017728 + lats[2107] = -58.172230225979497 + lats[2108] = -58.242528990922203 + lats[2109] = -58.312827755845746 + lats[2110] = -58.383126520749968 + lats[2111] = -58.453425285634758 + lats[2112] = -58.523724050499972 + lats[2113] = -58.594022815345468 + lats[2114] = -58.664321580171141 + lats[2115] = -58.73462034497684 + lats[2116] = -58.804919109762423 + lats[2117] = -58.875217874527763 + lats[2118] = -58.945516639272725 + lats[2119] = -59.015815403997145 + lats[2120] = -59.086114168700909 + lats[2121] = -59.156412933383855 + lats[2122] = -59.226711698045854 + lats[2123] = -59.29701046268675 + lats[2124] = -59.3673092273064 + lats[2125] = -59.43760799190467 + lats[2126] = -59.507906756481383 + lats[2127] = -59.578205521036402 + lats[2128] = -59.64850428556958 + lats[2129] = -59.718803050080759 + lats[2130] = -59.78910181456979 + lats[2131] = -59.859400579036503 + lats[2132] = -59.929699343480763 + lats[2133] = -59.999998107902378 + lats[2134] = -60.070296872301235 + lats[2135] = -60.140595636677112 + lats[2136] = -60.21089440102989 + lats[2137] = -60.28119316535939 + lats[2138] = -60.35149192966545 + lats[2139] = -60.421790693947884 + lats[2140] = -60.492089458206543 + lats[2141] = -60.562388222441243 + lats[2142] = -60.632686986651805 + lats[2143] = -60.702985750838074 + lats[2144] = -60.773284514999872 + lats[2145] = -60.843583279137007 + lats[2146] = -60.913882043249295 + lats[2147] = -60.984180807336578 + lats[2148] = -61.054479571398652 + lats[2149] = -61.124778335435344 + lats[2150] = -61.195077099446451 + lats[2151] = -61.265375863431785 + lats[2152] = -61.335674627391185 + lats[2153] = -61.405973391324409 + lats[2154] = -61.476272155231321 + lats[2155] = -61.546570919111666 + lats[2156] = -61.616869682965287 + lats[2157] = -61.687168446791986 + lats[2158] = -61.757467210591535 + lats[2159] = -61.827765974363729 + lats[2160] = -61.898064738108381 + lats[2161] = -61.968363501825259 + lats[2162] = -62.038662265514176 + lats[2163] = -62.108961029174914 + lats[2164] = -62.179259792807258 + lats[2165] = -62.249558556410982 + lats[2166] = -62.319857319985871 + lats[2167] = -62.3901560835317 + lats[2168] = -62.460454847048261 + lats[2169] = -62.530753610535321 + lats[2170] = -62.60105237399263 + lats[2171] = -62.67135113741999 + lats[2172] = -62.741649900817137 + lats[2173] = -62.811948664183866 + lats[2174] = -62.882247427519928 + lats[2175] = -62.952546190825068 + lats[2176] = -63.022844954099064 + lats[2177] = -63.093143717341647 + lats[2178] = -63.163442480552604 + lats[2179] = -63.23374124373165 + lats[2180] = -63.304040006878537 + lats[2181] = -63.374338769993031 + lats[2182] = -63.444637533074854 + lats[2183] = -63.514936296123757 + lats[2184] = -63.585235059139464 + lats[2185] = -63.655533822121711 + lats[2186] = -63.725832585070251 + lats[2187] = -63.796131347984762 + lats[2188] = -63.866430110865004 + lats[2189] = -63.93672887371072 + lats[2190] = -64.00702763652157 + lats[2191] = -64.07732639929732 + lats[2192] = -64.147625162037642 + lats[2193] = -64.21792392474228 + lats[2194] = -64.288222687410922 + lats[2195] = -64.358521450043284 + lats[2196] = -64.428820212639039 + lats[2197] = -64.499118975197902 + lats[2198] = -64.569417737719576 + lats[2199] = -64.639716500203733 + lats[2200] = -64.710015262650074 + lats[2201] = -64.780314025058246 + lats[2202] = -64.850612787427963 + lats[2203] = -64.920911549758912 + lats[2204] = -64.991210312050711 + lats[2205] = -65.061509074303089 + lats[2206] = -65.131807836515677 + lats[2207] = -65.202106598688133 + lats[2208] = -65.272405360820116 + lats[2209] = -65.342704122911286 + lats[2210] = -65.413002884961315 + lats[2211] = -65.483301646969792 + lats[2212] = -65.553600408936404 + lats[2213] = -65.623899170860767 + lats[2214] = -65.694197932742526 + lats[2215] = -65.764496694581283 + lats[2216] = -65.834795456376696 + lats[2217] = -65.905094218128355 + lats[2218] = -65.975392979835888 + lats[2219] = -66.045691741498899 + lats[2220] = -66.115990503117033 + lats[2221] = -66.186289264689833 + lats[2222] = -66.256588026216932 + lats[2223] = -66.326886787697887 + lats[2224] = -66.397185549132331 + lats[2225] = -66.467484310519808 + lats[2226] = -66.537783071859891 + lats[2227] = -66.608081833152212 + lats[2228] = -66.678380594396273 + lats[2229] = -66.748679355591662 + lats[2230] = -66.818978116737924 + lats[2231] = -66.889276877834618 + lats[2232] = -66.95957563888129 + lats[2233] = -67.029874399877471 + lats[2234] = -67.100173160822706 + lats[2235] = -67.170471921716526 + lats[2236] = -67.240770682558434 + lats[2237] = -67.311069443347961 + lats[2238] = -67.381368204084609 + lats[2239] = -67.451666964767895 + lats[2240] = -67.521965725397308 + lats[2241] = -67.592264485972336 + lats[2242] = -67.662563246492482 + lats[2243] = -67.732862006957205 + lats[2244] = -67.803160767365966 + lats[2245] = -67.873459527718282 + lats[2246] = -67.943758288013555 + lats[2247] = -68.014057048251274 + lats[2248] = -68.084355808430871 + lats[2249] = -68.154654568551791 + lats[2250] = -68.224953328613438 + lats[2251] = -68.295252088615257 + lats[2252] = -68.365550848556666 + lats[2253] = -68.435849608437067 + lats[2254] = -68.506148368255865 + lats[2255] = -68.576447128012447 + lats[2256] = -68.646745887706189 + lats[2257] = -68.717044647336493 + lats[2258] = -68.787343406902693 + lats[2259] = -68.85764216640419 + lats[2260] = -68.927940925840304 + lats[2261] = -68.998239685210365 + lats[2262] = -69.068538444513763 + lats[2263] = -69.138837203749759 + lats[2264] = -69.209135962917699 + lats[2265] = -69.279434722016902 + lats[2266] = -69.349733481046613 + lats[2267] = -69.420032240006194 + lats[2268] = -69.490330998894862 + lats[2269] = -69.560629757711908 + lats[2270] = -69.630928516456592 + lats[2271] = -69.701227275128161 + lats[2272] = -69.771526033725834 + lats[2273] = -69.841824792248843 + lats[2274] = -69.912123550696421 + lats[2275] = -69.982422309067744 + lats[2276] = -70.052721067362043 + lats[2277] = -70.123019825578467 + lats[2278] = -70.193318583716191 + lats[2279] = -70.263617341774406 + lats[2280] = -70.333916099752187 + lats[2281] = -70.404214857648739 + lats[2282] = -70.474513615463138 + lats[2283] = -70.544812373194532 + lats[2284] = -70.615111130841967 + lats[2285] = -70.685409888404578 + lats[2286] = -70.755708645881384 + lats[2287] = -70.826007403271475 + lats[2288] = -70.896306160573886 + lats[2289] = -70.966604917787635 + lats[2290] = -71.036903674911756 + lats[2291] = -71.107202431945211 + lats[2292] = -71.177501188887007 + lats[2293] = -71.247799945736105 + lats[2294] = -71.318098702491469 + lats[2295] = -71.388397459152031 + lats[2296] = -71.458696215716685 + lats[2297] = -71.528994972184378 + lats[2298] = -71.599293728553988 + lats[2299] = -71.669592484824364 + lats[2300] = -71.739891240994368 + lats[2301] = -71.810189997062835 + lats[2302] = -71.880488753028587 + lats[2303] = -71.950787508890414 + lats[2304] = -72.02108626464711 + lats[2305] = -72.091385020297409 + lats[2306] = -72.161683775840089 + lats[2307] = -72.231982531273843 + lats[2308] = -72.302281286597392 + lats[2309] = -72.3725800418094 + lats[2310] = -72.442878796908545 + lats[2311] = -72.513177551893421 + lats[2312] = -72.583476306762691 + lats[2313] = -72.653775061514935 + lats[2314] = -72.724073816148703 + lats[2315] = -72.794372570662574 + lats[2316] = -72.864671325055056 + lats[2317] = -72.934970079324657 + lats[2318] = -73.005268833469799 + lats[2319] = -73.075567587489019 + lats[2320] = -73.145866341380668 + lats[2321] = -73.216165095143182 + lats[2322] = -73.2864638487749 + lats[2323] = -73.356762602274188 + lats[2324] = -73.427061355639339 + lats[2325] = -73.497360108868662 + lats[2326] = -73.567658861960396 + lats[2327] = -73.637957614912779 + lats[2328] = -73.70825636772399 + lats[2329] = -73.778555120392184 + lats[2330] = -73.848853872915541 + lats[2331] = -73.919152625292114 + lats[2332] = -73.98945137751997 + lats[2333] = -74.059750129597163 + lats[2334] = -74.13004888152166 + lats[2335] = -74.200347633291472 + lats[2336] = -74.270646384904481 + lats[2337] = -74.340945136358584 + lats[2338] = -74.411243887651622 + lats[2339] = -74.481542638781434 + lats[2340] = -74.551841389745761 + lats[2341] = -74.622140140542356 + lats[2342] = -74.692438891168877 + lats[2343] = -74.762737641622991 + lats[2344] = -74.833036391902269 + lats[2345] = -74.903335142004323 + lats[2346] = -74.973633891926625 + lats[2347] = -75.043932641666672 + lats[2348] = -75.114231391221821 + lats[2349] = -75.184530140589501 + lats[2350] = -75.254828889766983 + lats[2351] = -75.325127638751567 + lats[2352] = -75.395426387540439 + lats[2353] = -75.465725136130786 + lats[2354] = -75.536023884519707 + lats[2355] = -75.60632263270422 + lats[2356] = -75.67662138068134 + lats[2357] = -75.746920128447996 + lats[2358] = -75.81721887600105 + lats[2359] = -75.887517623337317 + lats[2360] = -75.957816370453543 + lats[2361] = -76.028115117346374 + lats[2362] = -76.098413864012443 + lats[2363] = -76.16871261044831 + lats[2364] = -76.239011356650423 + lats[2365] = -76.3093101026152 + lats[2366] = -76.379608848338933 + lats[2367] = -76.449907593817869 + lats[2368] = -76.520206339048215 + lats[2369] = -76.59050508402602 + lats[2370] = -76.660803828747362 + lats[2371] = -76.731102573208048 + lats[2372] = -76.801401317404 + lats[2373] = -76.871700061330955 + lats[2374] = -76.941998804984564 + lats[2375] = -77.012297548360323 + lats[2376] = -77.082596291453768 + lats[2377] = -77.15289503426024 + lats[2378] = -77.22319377677502 + lats[2379] = -77.293492518993247 + lats[2380] = -77.363791260909963 + lats[2381] = -77.434090002520122 + lats[2382] = -77.504388743818524 + lats[2383] = -77.574687484799924 + lats[2384] = -77.644986225458879 + lats[2385] = -77.71528496578982 + lats[2386] = -77.785583705787161 + lats[2387] = -77.855882445445019 + lats[2388] = -77.926181184757539 + lats[2389] = -77.996479923718596 + lats[2390] = -78.066778662322022 + lats[2391] = -78.137077400561424 + lats[2392] = -78.207376138430348 + lats[2393] = -78.277674875922045 + lats[2394] = -78.347973613029708 + lats[2395] = -78.418272349746417 + lats[2396] = -78.488571086064923 + lats[2397] = -78.558869821977908 + lats[2398] = -78.629168557477882 + lats[2399] = -78.699467292557102 + lats[2400] = -78.769766027207638 + lats[2401] = -78.840064761421445 + lats[2402] = -78.910363495190211 + lats[2403] = -78.980662228505423 + lats[2404] = -79.050960961358285 + lats[2405] = -79.121259693739859 + lats[2406] = -79.191558425640977 + lats[2407] = -79.261857157052191 + lats[2408] = -79.332155887963822 + lats[2409] = -79.402454618365894 + lats[2410] = -79.472753348248219 + lats[2411] = -79.543052077600308 + lats[2412] = -79.61335080641139 + lats[2413] = -79.683649534670437 + lats[2414] = -79.753948262366038 + lats[2415] = -79.824246989486554 + lats[2416] = -79.894545716019948 + lats[2417] = -79.9648444419539 + lats[2418] = -80.035143167275749 + lats[2419] = -80.105441891972376 + lats[2420] = -80.175740616030438 + lats[2421] = -80.246039339436052 + lats[2422] = -80.316338062175078 + lats[2423] = -80.386636784232863 + lats[2424] = -80.456935505594302 + lats[2425] = -80.527234226243991 + lats[2426] = -80.59753294616587 + lats[2427] = -80.667831665343556 + lats[2428] = -80.73813038376008 + lats[2429] = -80.808429101397948 + lats[2430] = -80.878727818239184 + lats[2431] = -80.949026534265244 + lats[2432] = -81.019325249456955 + lats[2433] = -81.089623963794551 + lats[2434] = -81.159922677257711 + lats[2435] = -81.230221389825374 + lats[2436] = -81.300520101475826 + lats[2437] = -81.370818812186627 + lats[2438] = -81.441117521934686 + lats[2439] = -81.511416230696042 + lats[2440] = -81.581714938445955 + lats[2441] = -81.652013645158945 + lats[2442] = -81.722312350808508 + lats[2443] = -81.792611055367345 + lats[2444] = -81.862909758807191 + lats[2445] = -81.933208461098829 + lats[2446] = -82.003507162211946 + lats[2447] = -82.073805862115165 + lats[2448] = -82.144104560776 + lats[2449] = -82.214403258160871 + lats[2450] = -82.284701954234833 + lats[2451] = -82.355000648961692 + lats[2452] = -82.425299342304029 + lats[2453] = -82.495598034222837 + lats[2454] = -82.56589672467787 + lats[2455] = -82.63619541362705 + lats[2456] = -82.706494101026948 + lats[2457] = -82.77679278683226 + lats[2458] = -82.84709147099602 + lats[2459] = -82.917390153469313 + lats[2460] = -82.987688834201322 + lats[2461] = -83.057987513139125 + lats[2462] = -83.128286190227698 + lats[2463] = -83.198584865409657 + lats[2464] = -83.268883538625232 + lats[2465] = -83.339182209812321 + lats[2466] = -83.409480878905782 + lats[2467] = -83.479779545838113 + lats[2468] = -83.550078210538487 + lats[2469] = -83.620376872933264 + lats[2470] = -83.690675532945292 + lats[2471] = -83.760974190494011 + lats[2472] = -83.831272845495249 + lats[2473] = -83.901571497860914 + lats[2474] = -83.971870147498763 + lats[2475] = -84.042168794312317 + lats[2476] = -84.112467438200326 + lats[2477] = -84.18276607905679 + lats[2478] = -84.253064716770425 + lats[2479] = -84.323363351224444 + lats[2480] = -84.393661982296322 + lats[2481] = -84.463960609857125 + lats[2482] = -84.534259233771479 + lats[2483] = -84.604557853896708 + lats[2484] = -84.674856470082915 + lats[2485] = -84.745155082171991 + lats[2486] = -84.81545368999717 + lats[2487] = -84.885752293382765 + lats[2488] = -84.95605089214304 + lats[2489] = -85.026349486081983 + lats[2490] = -85.09664807499216 + lats[2491] = -85.16694665865414 + lats[2492] = -85.237245236835548 + lats[2493] = -85.307543809290152 + lats[2494] = -85.377842375756586 + lats[2495] = -85.448140935957483 + lats[2496] = -85.518439489597966 + lats[2497] = -85.588738036364362 + lats[2498] = -85.659036575922883 + lats[2499] = -85.729335107917464 + lats[2500] = -85.799633631968391 + lats[2501] = -85.869932147670127 + lats[2502] = -85.940230654588888 + lats[2503] = -86.010529152260403 + lats[2504] = -86.080827640187209 + lats[2505] = -86.151126117835304 + lats[2506] = -86.221424584631109 + lats[2507] = -86.291723039957418 + lats[2508] = -86.362021483149363 + lats[2509] = -86.432319913489792 + lats[2510] = -86.502618330203831 + lats[2511] = -86.572916732453024 + lats[2512] = -86.643215119328573 + lats[2513] = -86.713513489844246 + lats[2514] = -86.783811842927179 + lats[2515] = -86.854110177408927 + lats[2516] = -86.924408492014166 + lats[2517] = -86.994706785348129 + lats[2518] = -87.065005055882821 + lats[2519] = -87.135303301939786 + lats[2520] = -87.205601521672108 + lats[2521] = -87.275899713041966 + lats[2522] = -87.346197873795816 + lats[2523] = -87.416496001434894 + lats[2524] = -87.486794093180748 + lats[2525] = -87.557092145935584 + lats[2526] = -87.627390156234085 + lats[2527] = -87.697688120188062 + lats[2528] = -87.767986033419561 + lats[2529] = -87.838283890981543 + lats[2530] = -87.908581687261687 + lats[2531] = -87.978879415867283 + lats[2532] = -88.049177069484486 + lats[2533] = -88.119474639706425 + lats[2534] = -88.189772116820762 + lats[2535] = -88.26006948954614 + lats[2536] = -88.330366744702559 + lats[2537] = -88.40066386679355 + lats[2538] = -88.470960837474877 + lats[2539] = -88.541257634868515 + lats[2540] = -88.611554232668382 + lats[2541] = -88.681850598961759 + lats[2542] = -88.752146694650691 + lats[2543] = -88.822442471310097 + lats[2544] = -88.892737868230952 + lats[2545] = -88.96303280826325 + lats[2546] = -89.033327191845927 + lats[2547] = -89.103620888238879 + lats[2548] = -89.173913722284126 + lats[2549] = -89.24420545380525 + lats[2550] = -89.314495744374256 + lats[2551] = -89.3847841013921 + lats[2552] = -89.45506977912261 + lats[2553] = -89.525351592371393 + lats[2554] = -89.595627537554492 + lats[2555] = -89.6658939412157 + lats[2556] = -89.736143271609578 + lats[2557] = -89.806357319542244 + lats[2558] = -89.876478353332288 + lats[2559] = -89.946187715665616 + return lats + + def first_axis_vals(self): + if self._resolution == 1280: + return self.get_precomputed_values_N1280() + else: + precision = 1.0e-14 + nval = self._resolution * 2 + rad2deg = 180 / math.pi + convval = 1 - ((2 / math.pi) * (2 / math.pi)) * 0.25 + vals = self.gauss_first_guess() + new_vals = [0] * nval + denom = math.sqrt(((nval + 0.5) * (nval + 0.5)) + convval) + for jval in range(self._resolution): + root = math.cos(vals[jval] / denom) + conv = 1 + while abs(conv) >= precision: + mem2 = 1 + mem1 = root + for legi in range(nval): + legfonc = ((2.0 * (legi + 1) - 1.0) * root * mem1 - legi * mem2) / (legi + 1) + mem2 = mem1 + mem1 = legfonc + conv = legfonc / ((nval * (mem2 - root * legfonc)) / (1.0 - (root * root))) + root = root - conv + # add maybe a max iter here to make sure we converge at some point + new_vals[jval] = math.asin(root) * rad2deg + new_vals[nval - 1 - jval] = -new_vals[jval] + return new_vals + + def map_first_axis(self, lower, upper): + axis_lines = self._first_axis_vals + end_idx = bisect_left_cmp(axis_lines, lower, cmp=lambda x, y: x > y) + 1 + start_idx = bisect_right_cmp(axis_lines, upper, cmp=lambda x, y: x > y) + return_vals = axis_lines[start_idx:end_idx] + return return_vals + + def second_axis_vals(self, first_val): + first_axis_vals = self._first_axis_vals + tol = 1e-10 + first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + if first_idx >= self._resolution: + first_idx = (2 * self._resolution) - 1 - first_idx + first_idx = first_idx + 1 + npoints = 4 * first_idx + 16 + second_axis_spacing = 360 / npoints + second_axis_vals = [i * second_axis_spacing for i in range(npoints)] + return second_axis_vals + + def second_axis_spacing(self, first_val): + first_axis_vals = self._first_axis_vals + tol = 1e-10 + _first_idx = bisect_left_cmp(first_axis_vals, first_val - tol, cmp=lambda x, y: x > y) + first_idx = _first_idx + if first_idx >= self._resolution: + first_idx = (2 * self._resolution) - 1 - first_idx + first_idx = first_idx + 1 + npoints = 4 * first_idx + 16 + second_axis_spacing = 360 / npoints + return (second_axis_spacing, _first_idx + 1) + + def map_second_axis(self, first_val, lower, upper): + second_axis_spacing, first_idx = self.second_axis_spacing(first_val) + start_idx = int(lower / second_axis_spacing) + end_idx = int(upper / second_axis_spacing) + 1 + return_vals = [i * second_axis_spacing for i in range(start_idx, end_idx)] + return return_vals + + def axes_idx_to_octahedral_idx(self, first_idx, second_idx): + # NOTE: for now this takes ~2e-4s per point, so taking significant time -> for 20k points, takes 4s + # Would it be better to store a dictionary of first_idx with cumulative number of points on that idx? + # Because this is what we are doing here, but we are calculating for each point... + # But then this would only work for special grid resolutions, so need to do like a O1280 version of this + + # NOTE: OR somehow cache this for a given first_idx and then only modify the axis idx for second_idx when the + # first_idx changes + octa_idx = self._first_idx_map[first_idx - 1] + second_idx + return octa_idx + + def create_first_idx_map(self): + first_idx_list = {} + idx = 0 + for i in range(2 * self._resolution): + first_idx_list[i] = idx + if i <= self._resolution - 1: + idx += 20 + 4 * i + else: + i = i - self._resolution + 1 + if i == 1: + idx += 16 + 4 * self._resolution + else: + i = i - 1 + idx += 16 + 4 * (self._resolution - i) + return first_idx_list + + def find_second_axis_idx(self, first_val, second_val): + (second_axis_spacing, first_idx) = self.second_axis_spacing(first_val) + tol = 1e-8 + if second_val / second_axis_spacing > int(second_val / second_axis_spacing) + 1 - tol: + second_idx = int(second_val / second_axis_spacing) + 1 + else: + second_idx = int(second_val / second_axis_spacing) + return (first_idx, second_idx) + + def unmap(self, first_val, second_val): + (first_idx, second_idx) = self.find_second_axis_idx(first_val, second_val) + octahedral_index = self.axes_idx_to_octahedral_idx(first_idx, second_idx) + return octahedral_index diff --git a/polytope/datacube/transformations/mappers/reduced_ll.py b/polytope/datacube/transformations/mappers/reduced_ll.py new file mode 100644 index 000000000..f056b2329 --- /dev/null +++ b/polytope/datacube/transformations/mappers/reduced_ll.py @@ -0,0 +1,1504 @@ +import bisect + +from ..datacube_mappers import DatacubeMapper + + +class ReducedLatLonMapper(DatacubeMapper): + def __init__(self, base_axis, mapped_axes, resolution): + self._mapped_axes = mapped_axes + self._base_axis = base_axis + self._resolution = resolution + self._axis_reversed = {mapped_axes[0]: False, mapped_axes[1]: False} + self._first_axis_vals = self.first_axis_vals() + + def first_axis_vals(self): + resolution = 180 / (self._resolution - 1) + vals = [-90 + i * resolution for i in range(self._resolution)] + return vals + + def map_first_axis(self, lower, upper): + axis_lines = self._first_axis_vals + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def lon_spacing(self): + if self._resolution == 1441: + return [ + 2, + 6, + 14, + 20, + 26, + 32, + 38, + 44, + 50, + 58, + 64, + 70, + 76, + 82, + 88, + 94, + 102, + 108, + 114, + 120, + 126, + 132, + 138, + 144, + 152, + 158, + 164, + 170, + 176, + 182, + 188, + 196, + 202, + 208, + 214, + 220, + 226, + 232, + 238, + 246, + 252, + 258, + 264, + 270, + 276, + 282, + 290, + 296, + 302, + 308, + 314, + 320, + 326, + 332, + 340, + 346, + 352, + 358, + 364, + 370, + 376, + 382, + 388, + 396, + 402, + 408, + 414, + 420, + 426, + 432, + 438, + 444, + 452, + 458, + 464, + 470, + 476, + 482, + 488, + 494, + 500, + 506, + 512, + 520, + 526, + 532, + 538, + 544, + 550, + 556, + 562, + 568, + 574, + 580, + 586, + 594, + 600, + 606, + 612, + 618, + 624, + 630, + 636, + 642, + 648, + 654, + 660, + 666, + 672, + 678, + 686, + 692, + 698, + 704, + 710, + 716, + 722, + 728, + 734, + 740, + 746, + 752, + 758, + 764, + 770, + 776, + 782, + 788, + 794, + 800, + 806, + 812, + 818, + 824, + 830, + 836, + 842, + 848, + 854, + 860, + 866, + 872, + 878, + 884, + 890, + 896, + 902, + 908, + 914, + 920, + 926, + 932, + 938, + 944, + 950, + 956, + 962, + 968, + 974, + 980, + 986, + 992, + 998, + 1004, + 1010, + 1014, + 1020, + 1026, + 1032, + 1038, + 1044, + 1050, + 1056, + 1062, + 1068, + 1074, + 1080, + 1086, + 1092, + 1096, + 1102, + 1108, + 1114, + 1120, + 1126, + 1132, + 1138, + 1144, + 1148, + 1154, + 1160, + 1166, + 1172, + 1178, + 1184, + 1190, + 1194, + 1200, + 1206, + 1212, + 1218, + 1224, + 1230, + 1234, + 1240, + 1246, + 1252, + 1258, + 1264, + 1268, + 1274, + 1280, + 1286, + 1292, + 1296, + 1302, + 1308, + 1314, + 1320, + 1324, + 1330, + 1336, + 1342, + 1348, + 1352, + 1358, + 1364, + 1370, + 1374, + 1380, + 1386, + 1392, + 1396, + 1402, + 1408, + 1414, + 1418, + 1424, + 1430, + 1436, + 1440, + 1446, + 1452, + 1456, + 1462, + 1468, + 1474, + 1478, + 1484, + 1490, + 1494, + 1500, + 1506, + 1510, + 1516, + 1522, + 1526, + 1532, + 1538, + 1542, + 1548, + 1554, + 1558, + 1564, + 1570, + 1574, + 1580, + 1584, + 1590, + 1596, + 1600, + 1606, + 1610, + 1616, + 1622, + 1626, + 1632, + 1636, + 1642, + 1648, + 1652, + 1658, + 1662, + 1668, + 1672, + 1678, + 1684, + 1688, + 1694, + 1698, + 1704, + 1708, + 1714, + 1718, + 1724, + 1728, + 1734, + 1738, + 1744, + 1748, + 1754, + 1758, + 1764, + 1768, + 1774, + 1778, + 1784, + 1788, + 1794, + 1798, + 1804, + 1808, + 1812, + 1818, + 1822, + 1828, + 1832, + 1838, + 1842, + 1846, + 1852, + 1856, + 1862, + 1866, + 1870, + 1876, + 1880, + 1886, + 1890, + 1894, + 1900, + 1904, + 1908, + 1914, + 1918, + 1922, + 1928, + 1932, + 1936, + 1942, + 1946, + 1950, + 1956, + 1960, + 1964, + 1970, + 1974, + 1978, + 1982, + 1988, + 1992, + 1996, + 2002, + 2006, + 2010, + 2014, + 2020, + 2024, + 2028, + 2032, + 2036, + 2042, + 2046, + 2050, + 2054, + 2060, + 2064, + 2068, + 2072, + 2076, + 2080, + 2086, + 2090, + 2094, + 2098, + 2102, + 2106, + 2112, + 2116, + 2120, + 2124, + 2128, + 2132, + 2136, + 2140, + 2144, + 2150, + 2154, + 2158, + 2162, + 2166, + 2170, + 2174, + 2178, + 2182, + 2186, + 2190, + 2194, + 2198, + 2202, + 2206, + 2210, + 2214, + 2218, + 2222, + 2226, + 2230, + 2234, + 2238, + 2242, + 2246, + 2250, + 2254, + 2258, + 2262, + 2266, + 2270, + 2274, + 2278, + 2282, + 2286, + 2290, + 2292, + 2296, + 2300, + 2304, + 2308, + 2312, + 2316, + 2320, + 2324, + 2326, + 2330, + 2334, + 2338, + 2342, + 2346, + 2348, + 2352, + 2356, + 2360, + 2364, + 2366, + 2370, + 2374, + 2378, + 2382, + 2384, + 2388, + 2392, + 2396, + 2398, + 2402, + 2406, + 2410, + 2412, + 2416, + 2420, + 2422, + 2426, + 2430, + 2432, + 2436, + 2440, + 2442, + 2446, + 2450, + 2452, + 2456, + 2460, + 2462, + 2466, + 2470, + 2472, + 2476, + 2478, + 2482, + 2486, + 2488, + 2492, + 2494, + 2498, + 2500, + 2504, + 2508, + 2510, + 2514, + 2516, + 2520, + 2522, + 2526, + 2528, + 2532, + 2534, + 2538, + 2540, + 2544, + 2546, + 2550, + 2552, + 2556, + 2558, + 2560, + 2564, + 2566, + 2570, + 2572, + 2576, + 2578, + 2580, + 2584, + 2586, + 2590, + 2592, + 2594, + 2598, + 2600, + 2602, + 2606, + 2608, + 2610, + 2614, + 2616, + 2618, + 2622, + 2624, + 2626, + 2628, + 2632, + 2634, + 2636, + 2640, + 2642, + 2644, + 2646, + 2650, + 2652, + 2654, + 2656, + 2658, + 2662, + 2664, + 2666, + 2668, + 2670, + 2674, + 2676, + 2678, + 2680, + 2682, + 2684, + 2686, + 2690, + 2692, + 2694, + 2696, + 2698, + 2700, + 2702, + 2704, + 2706, + 2708, + 2712, + 2714, + 2716, + 2718, + 2720, + 2722, + 2724, + 2726, + 2728, + 2730, + 2732, + 2734, + 2736, + 2738, + 2740, + 2742, + 2744, + 2746, + 2748, + 2750, + 2750, + 2752, + 2754, + 2756, + 2758, + 2760, + 2762, + 2764, + 2766, + 2768, + 2768, + 2770, + 2772, + 2774, + 2776, + 2778, + 2780, + 2780, + 2782, + 2784, + 2786, + 2788, + 2788, + 2790, + 2792, + 2794, + 2794, + 2796, + 2798, + 2800, + 2800, + 2802, + 2804, + 2806, + 2806, + 2808, + 2810, + 2810, + 2812, + 2814, + 2814, + 2816, + 2818, + 2818, + 2820, + 2822, + 2822, + 2824, + 2826, + 2826, + 2828, + 2828, + 2830, + 2832, + 2832, + 2834, + 2834, + 2836, + 2836, + 2838, + 2838, + 2840, + 2842, + 2842, + 2844, + 2844, + 2846, + 2846, + 2846, + 2848, + 2848, + 2850, + 2850, + 2852, + 2852, + 2854, + 2854, + 2856, + 2856, + 2856, + 2858, + 2858, + 2860, + 2860, + 2860, + 2862, + 2862, + 2862, + 2864, + 2864, + 2864, + 2866, + 2866, + 2866, + 2868, + 2868, + 2868, + 2868, + 2870, + 2870, + 2870, + 2872, + 2872, + 2872, + 2872, + 2874, + 2874, + 2874, + 2874, + 2874, + 2876, + 2876, + 2876, + 2876, + 2876, + 2876, + 2878, + 2878, + 2878, + 2878, + 2878, + 2878, + 2878, + 2878, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2880, + 2878, + 2878, + 2878, + 2878, + 2878, + 2878, + 2878, + 2878, + 2876, + 2876, + 2876, + 2876, + 2876, + 2876, + 2874, + 2874, + 2874, + 2874, + 2874, + 2872, + 2872, + 2872, + 2872, + 2870, + 2870, + 2870, + 2868, + 2868, + 2868, + 2868, + 2866, + 2866, + 2866, + 2864, + 2864, + 2864, + 2862, + 2862, + 2862, + 2860, + 2860, + 2860, + 2858, + 2858, + 2856, + 2856, + 2856, + 2854, + 2854, + 2852, + 2852, + 2850, + 2850, + 2848, + 2848, + 2846, + 2846, + 2846, + 2844, + 2844, + 2842, + 2842, + 2840, + 2838, + 2838, + 2836, + 2836, + 2834, + 2834, + 2832, + 2832, + 2830, + 2828, + 2828, + 2826, + 2826, + 2824, + 2822, + 2822, + 2820, + 2818, + 2818, + 2816, + 2814, + 2814, + 2812, + 2810, + 2810, + 2808, + 2806, + 2806, + 2804, + 2802, + 2800, + 2800, + 2798, + 2796, + 2794, + 2794, + 2792, + 2790, + 2788, + 2788, + 2786, + 2784, + 2782, + 2780, + 2780, + 2778, + 2776, + 2774, + 2772, + 2770, + 2768, + 2768, + 2766, + 2764, + 2762, + 2760, + 2758, + 2756, + 2754, + 2752, + 2750, + 2750, + 2748, + 2746, + 2744, + 2742, + 2740, + 2738, + 2736, + 2734, + 2732, + 2730, + 2728, + 2726, + 2724, + 2722, + 2720, + 2718, + 2716, + 2714, + 2712, + 2708, + 2706, + 2704, + 2702, + 2700, + 2698, + 2696, + 2694, + 2692, + 2690, + 2686, + 2684, + 2682, + 2680, + 2678, + 2676, + 2674, + 2670, + 2668, + 2666, + 2664, + 2662, + 2658, + 2656, + 2654, + 2652, + 2650, + 2646, + 2644, + 2642, + 2640, + 2636, + 2634, + 2632, + 2628, + 2626, + 2624, + 2622, + 2618, + 2616, + 2614, + 2610, + 2608, + 2606, + 2602, + 2600, + 2598, + 2594, + 2592, + 2590, + 2586, + 2584, + 2580, + 2578, + 2576, + 2572, + 2570, + 2566, + 2564, + 2560, + 2558, + 2556, + 2552, + 2550, + 2546, + 2544, + 2540, + 2538, + 2534, + 2532, + 2528, + 2526, + 2522, + 2520, + 2516, + 2514, + 2510, + 2508, + 2504, + 2500, + 2498, + 2494, + 2492, + 2488, + 2486, + 2482, + 2478, + 2476, + 2472, + 2470, + 2466, + 2462, + 2460, + 2456, + 2452, + 2450, + 2446, + 2442, + 2440, + 2436, + 2432, + 2430, + 2426, + 2422, + 2420, + 2416, + 2412, + 2410, + 2406, + 2402, + 2398, + 2396, + 2392, + 2388, + 2384, + 2382, + 2378, + 2374, + 2370, + 2366, + 2364, + 2360, + 2356, + 2352, + 2348, + 2346, + 2342, + 2338, + 2334, + 2330, + 2326, + 2324, + 2320, + 2316, + 2312, + 2308, + 2304, + 2300, + 2296, + 2292, + 2290, + 2286, + 2282, + 2278, + 2274, + 2270, + 2266, + 2262, + 2258, + 2254, + 2250, + 2246, + 2242, + 2238, + 2234, + 2230, + 2226, + 2222, + 2218, + 2214, + 2210, + 2206, + 2202, + 2198, + 2194, + 2190, + 2186, + 2182, + 2178, + 2174, + 2170, + 2166, + 2162, + 2158, + 2154, + 2150, + 2144, + 2140, + 2136, + 2132, + 2128, + 2124, + 2120, + 2116, + 2112, + 2106, + 2102, + 2098, + 2094, + 2090, + 2086, + 2080, + 2076, + 2072, + 2068, + 2064, + 2060, + 2054, + 2050, + 2046, + 2042, + 2036, + 2032, + 2028, + 2024, + 2020, + 2014, + 2010, + 2006, + 2002, + 1996, + 1992, + 1988, + 1982, + 1978, + 1974, + 1970, + 1964, + 1960, + 1956, + 1950, + 1946, + 1942, + 1936, + 1932, + 1928, + 1922, + 1918, + 1914, + 1908, + 1904, + 1900, + 1894, + 1890, + 1886, + 1880, + 1876, + 1870, + 1866, + 1862, + 1856, + 1852, + 1846, + 1842, + 1838, + 1832, + 1828, + 1822, + 1818, + 1812, + 1808, + 1804, + 1798, + 1794, + 1788, + 1784, + 1778, + 1774, + 1768, + 1764, + 1758, + 1754, + 1748, + 1744, + 1738, + 1734, + 1728, + 1724, + 1718, + 1714, + 1708, + 1704, + 1698, + 1694, + 1688, + 1684, + 1678, + 1672, + 1668, + 1662, + 1658, + 1652, + 1648, + 1642, + 1636, + 1632, + 1626, + 1622, + 1616, + 1610, + 1606, + 1600, + 1596, + 1590, + 1584, + 1580, + 1574, + 1570, + 1564, + 1558, + 1554, + 1548, + 1542, + 1538, + 1532, + 1526, + 1522, + 1516, + 1510, + 1506, + 1500, + 1494, + 1490, + 1484, + 1478, + 1474, + 1468, + 1462, + 1456, + 1452, + 1446, + 1440, + 1436, + 1430, + 1424, + 1418, + 1414, + 1408, + 1402, + 1396, + 1392, + 1386, + 1380, + 1374, + 1370, + 1364, + 1358, + 1352, + 1348, + 1342, + 1336, + 1330, + 1324, + 1320, + 1314, + 1308, + 1302, + 1296, + 1292, + 1286, + 1280, + 1274, + 1268, + 1264, + 1258, + 1252, + 1246, + 1240, + 1234, + 1230, + 1224, + 1218, + 1212, + 1206, + 1200, + 1194, + 1190, + 1184, + 1178, + 1172, + 1166, + 1160, + 1154, + 1148, + 1144, + 1138, + 1132, + 1126, + 1120, + 1114, + 1108, + 1102, + 1096, + 1092, + 1086, + 1080, + 1074, + 1068, + 1062, + 1056, + 1050, + 1044, + 1038, + 1032, + 1026, + 1020, + 1014, + 1010, + 1004, + 998, + 992, + 986, + 980, + 974, + 968, + 962, + 956, + 950, + 944, + 938, + 932, + 926, + 920, + 914, + 908, + 902, + 896, + 890, + 884, + 878, + 872, + 866, + 860, + 854, + 848, + 842, + 836, + 830, + 824, + 818, + 812, + 806, + 800, + 794, + 788, + 782, + 776, + 770, + 764, + 758, + 752, + 746, + 740, + 734, + 728, + 722, + 716, + 710, + 704, + 698, + 692, + 686, + 678, + 672, + 666, + 660, + 654, + 648, + 642, + 636, + 630, + 624, + 618, + 612, + 606, + 600, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ] + + def second_axis_vals(self, first_val): + first_idx = self._first_axis_vals.index(first_val) + Ny = self.lon_spacing()[first_idx] + second_spacing = 360 / Ny + return [i * second_spacing for i in range(Ny)] + + def map_second_axis(self, first_val, lower, upper): + axis_lines = self.second_axis_vals(first_val) + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def axes_idx_to_reduced_ll_idx(self, first_idx, second_idx): + Ny_array = self.lon_spacing() + idx = 0 + for i in range(self._resolution): + if i != first_idx: + idx += Ny_array[i] + else: + idx += second_idx + return idx + + def find_second_idx(self, first_val, second_val): + tol = 1e-10 + second_axis_vals = self.second_axis_vals(first_val) + second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) + return second_idx + + def unmap(self, first_val, second_val): + tol = 1e-8 + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + first_idx = self._first_axis_vals.index(first_val) + second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] + second_idx = self.second_axis_vals(first_val).index(second_val) + reduced_ll_index = self.axes_idx_to_reduced_ll_idx(first_idx, second_idx) + return reduced_ll_index diff --git a/polytope/datacube/transformations/mappers/regular.py b/polytope/datacube/transformations/mappers/regular.py new file mode 100644 index 000000000..3b0598a0b --- /dev/null +++ b/polytope/datacube/transformations/mappers/regular.py @@ -0,0 +1,56 @@ +import bisect + +from ..datacube_mappers import DatacubeMapper + + +class RegularGridMapper(DatacubeMapper): + def __init__(self, base_axis, mapped_axes, resolution): + self._mapped_axes = mapped_axes + self._base_axis = base_axis + self._resolution = resolution + self.deg_increment = 90 / self._resolution + self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} + self._first_axis_vals = self.first_axis_vals() + + def first_axis_vals(self): + first_ax_vals = [90 - i * self.deg_increment for i in range(2 * self._resolution)] + return first_ax_vals + + def map_first_axis(self, lower, upper): + axis_lines = self._first_axis_vals + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def second_axis_vals(self, first_val): + second_ax_vals = [i * self.deg_increment for i in range(4 * self._resolution)] + return second_ax_vals + + def map_second_axis(self, first_val, lower, upper): + axis_lines = self.second_axis_vals(first_val) + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def axes_idx_to_regular_idx(self, first_idx, second_idx): + final_idx = first_idx * 4 * self._resolution + second_idx + return final_idx + + def find_second_idx(self, first_val, second_val): + tol = 1e-10 + second_axis_vals = self.second_axis_vals(first_val) + second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) + return second_idx + + def unmap_first_val_to_start_line_idx(self, first_val): + tol = 1e-8 + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + first_idx = self._first_axis_vals.index(first_val) + return first_idx * 4 * self._resolution + + def unmap(self, first_val, second_val): + tol = 1e-8 + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + first_idx = self._first_axis_vals.index(first_val) + second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] + second_idx = self.second_axis_vals(first_val).index(second_val) + final_index = self.axes_idx_to_regular_idx(first_idx, second_idx) + return final_index diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 7ec4a4659..ebea9d996 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -16,7 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0, "type": "fc"} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "fc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index a7c54816a..bf4452a35 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -20,7 +20,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc"} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) @@ -38,7 +38,6 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), - Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) diff --git a/tests/test_mappers.py b/tests/test_mappers.py index b9a39477f..b3661822b 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -1,4 +1,4 @@ -from polytope.datacube.transformations.datacube_mappers import OctahedralGridMapper +from polytope.datacube.transformations.mappers.octahedral import OctahedralGridMapper class TestMapper: diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 624a77fe2..319bc2c24 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -17,7 +17,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": "0"} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) @@ -27,7 +27,6 @@ def setup_method(self, method): def test_fdb_datacube(self): request = Request( Select("step", [0]), - Select("number", [1]), Select("levtype", ["sfc"]), Span("date", pd.Timestamp("20230625T120000"), pd.Timestamp("20230626T120000")), Select("domain", ["g"]), @@ -36,7 +35,6 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), - Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) diff --git a/tests/test_slice_date_range_fdb_v2.py b/tests/test_slice_date_range_fdb_v2.py index 24ae1a9a5..63ce5e678 100644 --- a/tests/test_slice_date_range_fdb_v2.py +++ b/tests/test_slice_date_range_fdb_v2.py @@ -16,7 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, } - self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} + self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "stream": "enda"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) From f02c03edd5f5de1dd24bfbb12baf699ff9c37365 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 17 Jan 2024 17:31:42 +0100 Subject: [PATCH 278/332] start nearest method by finding indices between like for surrounding method --- polytope/datacube/datacube_axis.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 5c78c22f6..1119be943 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -147,7 +147,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): update_range() indexes_between_ranges = [] - if method != "surrounding": + if method != "surrounding" and method != "nearest": return old_find_indices_between(index_ranges, low, up, datacube, method) else: for indexes in index_ranges: @@ -264,7 +264,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): transformation = transform if cls.name in transformation._mapped_axes(): for idxs in index_ranges: - if method == "surrounding": + if method == "surrounding" or method == "nearest": axis_reversed = transform._axis_reversed[cls.name] if not axis_reversed: start = bisect.bisect_left(idxs, low) @@ -349,7 +349,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): transformation = transform if cls.name in transformation._mapped_axes(): for indexes in index_ranges: - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.index(low) end = indexes.index(up) start = max(start - 1, 0) @@ -403,7 +403,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically # increasing - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") start = max(start - 1, 0) @@ -416,7 +416,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between = indexes[start:end].to_list() indexes_between_ranges.append(indexes_between) else: - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.index(low) end = indexes.index(up) start = max(start - 1, 0) @@ -486,7 +486,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): transformation = transform if cls.name == transformation.name: for indexes in index_ranges: - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.index(low) end = indexes.index(up) start = max(start - 1, 0) @@ -598,7 +598,7 @@ def find_indices_between(self, index_ranges, low, up, datacube, method=None): # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") start = max(start - 1, 0) @@ -611,7 +611,7 @@ def find_indices_between(self, index_ranges, low, up, datacube, method=None): indexes_between = indexes[start:end].to_list() indexes_between_ranges.append(indexes_between) else: - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.index(low) end = indexes.index(up) start = max(start - 1, 0) From cf4a1997d2ea1d1d5e323263d41f836d80fb4db5 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 18 Jan 2024 11:19:26 +0100 Subject: [PATCH 279/332] make nearest method return only nearest point for end lat/lon grid --- polytope/datacube/backends/fdb.py | 34 ++++++++++++++++ polytope/datacube/backends/xarray.py | 2 + polytope/engine/hullslicer.py | 2 + polytope/shapes.py | 12 ++++-- polytope/utility/geometry.py | 18 +++++++++ tests/test_fdb_datacube.py | 3 +- tests/test_point_nearest.py | 60 ++++++++++++++++++++++++++++ tests/test_point_shape.py | 7 ++++ 8 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 tests/test_point_nearest.py diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index cbc76893f..9e9bc1a4a 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -2,6 +2,7 @@ import pygribjump as pygj +from ...utility.geometry import nearest_pt from .datacube import Datacube, IndexTree @@ -15,6 +16,7 @@ def __init__(self, config={}, axis_options={}): self.blocked_axes = [] self.fake_axes = [] self.unwanted_path = {} + self.nearest_search = {} partial_request = config # Find values in the level 3 FDB datacube @@ -61,6 +63,38 @@ def get(self, requests: IndexTree, leaf_path={}): def get_2nd_last_values(self, requests, leaf_path={}): # In this function, we recursively loop over the last two layers of the tree and store the indices of the # request ranges in those layers + first_ax_name = requests.children[0].axis.name + second_ax_name = requests.children[0].children[0].axis.name + nearest_pts = [ + [lat_val, lon_val] + for (lat_val, lon_val) in zip(self.nearest_search[first_ax_name][0], self.nearest_search[second_ax_name][0]) + ] + # TODO: here find nearest point first before retrieving etc + if len(self.nearest_search) != 0: + # first collect the lat lon points found + found_latlon_pts = [] + for lat_child in requests.children: + for lon_child in lat_child.children: + found_latlon_pts.append([lat_child.value, lon_child.value]) + # now find the nearest lat lon to the points requested + nearest_latlons = [] + for pt in nearest_pts: + nearest_latlon = nearest_pt(found_latlon_pts, pt) + nearest_latlons.append(nearest_latlon) + # TODO: now combine with the rest of the function.... + # TODO: need to remove the branches that do not fit + copy_requests = deepcopy(requests) + for i in range(len(copy_requests.children)): + lat_child = copy_requests.children[i] + lat_child = [child for child in requests.children if child.value == lat_child.value][0] + if lat_child.value not in [latlon[0] for latlon in nearest_latlons]: + lat_child.remove_branch() + else: + possible_lons = [latlon[1] for latlon in nearest_latlons if latlon[0] == lat_child.value] + for lon_child in lat_child.children: + if lon_child.value not in possible_lons: + lon_child.remove_branch() + lat_length = len(requests.children) range_lengths = [False] * lat_length current_start_idxs = [False] * lat_length diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index f8ca1c2e2..34cd8066f 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -17,6 +17,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.complete_axes = [] self.blocked_axes = [] self.fake_axes = [] + self.nearest_search = None for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: @@ -52,6 +53,7 @@ def get(self, requests: IndexTree): for key in path_copy: axis = self._axes[key] (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) + # TODO: here do nearest point search path = self.fit_path(path) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmapped_path) diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index e6c8e3eb0..f95f71e2e 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -50,6 +50,8 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex upper = ax.from_float(upper + tol) flattened = node.flatten() method = polytope.method + if method == "nearest": + datacube.nearest_search[ax.name] = polytope.points # TODO: this hashing doesn't work because we need to know the latitude val for finding longitude values # TODO: Maybe create a coupled_axes list inside of datacube and add to it during axis formation, then here diff --git a/polytope/shapes.py b/polytope/shapes.py index 6af698ee2..c0378af06 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -80,9 +80,15 @@ def __init__(self, axes, values, method=None): self.values = values self.method = method self.polytopes = [] - for i in range(len(axes)): - polytope_points = [v[i] for v in self.values] - self.polytopes.append(ConvexPolytope([axes[i]], [polytope_points], self.method)) + # if method != "nearest": + if True: + # if the method is surrounding, need to treat as 1D polytopes + for i in range(len(axes)): + polytope_points = [v[i] for v in self.values] + # TODO: IS THIS RIGHT? FOR MULTIPLE POINTS, DOES IT CREATE A LINE SEGMENT INSTEAD? + self.polytopes.extend([ConvexPolytope([axes[i]], [[point]], self.method) for point in polytope_points]) + # if method == "nearest": + # self.polytopes.extend([ConvexPolytope(axes, [v], self.method) for v in self.values]) def axes(self): return self._axes diff --git a/polytope/utility/geometry.py b/polytope/utility/geometry.py index bbbb75152..2c88d9655 100644 --- a/polytope/utility/geometry.py +++ b/polytope/utility/geometry.py @@ -1,4 +1,22 @@ +import math + + def lerp(a, b, value): direction = [a - b for a, b in zip(a, b)] intersect = [b + value * d for b, d in zip(b, direction)] return intersect + + +def nearest_pt(pts_list, pt): + nearest_pt = pts_list[0] + distance = l2_norm(pts_list[0], pt) + for new_pt in pts_list[1:]: + new_distance = l2_norm(new_pt, pt) + if new_distance < distance: + distance = new_distance + nearest_pt = new_pt + return nearest_pt + + +def l2_norm(pt1, pt2): + return math.sqrt((pt1[0] - pt2[0]) * (pt1[0] - pt2[0]) + (pt1[1] - pt2[1]) * (pt1[1] - pt2[1])) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index a7c54816a..bf4452a35 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -20,7 +20,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc"} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) @@ -38,7 +38,6 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), - Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py new file mode 100644 index 000000000..9166fd794 --- /dev/null +++ b/tests/test_point_nearest.py @@ -0,0 +1,60 @@ +import pandas as pd +import pytest + +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Point, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.fdb + def test_fdb_datacube(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20230625T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Point(["latitude", "longitude"], [[0.16, 0.176]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + + @pytest.mark.fdb + def test_fdb_datacube_true_point(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20230625T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Point(["latitude", "longitude"], [[0.175746921078, 0.210608424337]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 diff --git a/tests/test_point_shape.py b/tests/test_point_shape.py index 0bc203d61..7dbbd9b52 100644 --- a/tests/test_point_shape.py +++ b/tests/test_point_shape.py @@ -30,6 +30,13 @@ def test_point(self): assert len(result.leaves) == 1 assert result.leaves[0].axis.name == "level" + def test_multiple_points(self): + request = Request(Point(["step", "level"], [[3, 10], [3, 12]]), Select("date", ["2000-01-01"])) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 2 + assert result.leaves[0].axis.name == "level" + def test_point_surrounding_step(self): request = Request(Point(["step", "level"], [[2, 10]], method="surrounding"), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) From 0a522dbe67f52adbd2ad4a1b6260e716e80affb5 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 18 Jan 2024 13:45:14 +0100 Subject: [PATCH 280/332] fix small bug --- polytope/datacube/backends/fdb.py | 14 ++++++++------ tests/test_ecmwf_oper_data_fdb.py | 2 +- tests/test_slice_date_range_fdb.py | 1 - tests/test_slice_date_range_fdb_v2.py | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 9e9bc1a4a..f682b671f 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -63,14 +63,16 @@ def get(self, requests: IndexTree, leaf_path={}): def get_2nd_last_values(self, requests, leaf_path={}): # In this function, we recursively loop over the last two layers of the tree and store the indices of the # request ranges in those layers - first_ax_name = requests.children[0].axis.name - second_ax_name = requests.children[0].children[0].axis.name - nearest_pts = [ - [lat_val, lon_val] - for (lat_val, lon_val) in zip(self.nearest_search[first_ax_name][0], self.nearest_search[second_ax_name][0]) - ] # TODO: here find nearest point first before retrieving etc if len(self.nearest_search) != 0: + first_ax_name = requests.children[0].axis.name + second_ax_name = requests.children[0].children[0].axis.name + nearest_pts = [ + [lat_val, lon_val] + for (lat_val, lon_val) in zip( + self.nearest_search[first_ax_name][0], self.nearest_search[second_ax_name][0] + ) + ] # first collect the lat lon points found found_latlon_pts = [] for lat_child in requests.children: diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 7ec4a4659..553407eab 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -16,7 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0, "type": "fc"} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 624a77fe2..ec14ba48c 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -36,7 +36,6 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), - Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) diff --git a/tests/test_slice_date_range_fdb_v2.py b/tests/test_slice_date_range_fdb_v2.py index 24ae1a9a5..63ce5e678 100644 --- a/tests/test_slice_date_range_fdb_v2.py +++ b/tests/test_slice_date_range_fdb_v2.py @@ -16,7 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, } - self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} + self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "stream": "enda"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) From ee500212c1803a6eceab81b676be0c6515bc5a9a Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 18 Jan 2024 13:49:07 +0100 Subject: [PATCH 281/332] make sure nearest is only called on one point --- polytope/shapes.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/polytope/shapes.py b/polytope/shapes.py index c0378af06..0c170b3fc 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -80,15 +80,11 @@ def __init__(self, axes, values, method=None): self.values = values self.method = method self.polytopes = [] - # if method != "nearest": - if True: - # if the method is surrounding, need to treat as 1D polytopes - for i in range(len(axes)): - polytope_points = [v[i] for v in self.values] - # TODO: IS THIS RIGHT? FOR MULTIPLE POINTS, DOES IT CREATE A LINE SEGMENT INSTEAD? - self.polytopes.extend([ConvexPolytope([axes[i]], [[point]], self.method) for point in polytope_points]) - # if method == "nearest": - # self.polytopes.extend([ConvexPolytope(axes, [v], self.method) for v in self.values]) + if method == "nearest": + assert len(self.values) == 1 + for i in range(len(axes)): + polytope_points = [v[i] for v in self.values] + self.polytopes.extend([ConvexPolytope([axes[i]], [[point]], self.method) for point in polytope_points]) def axes(self): return self._axes From ad5c029a327557090c9d14e0e9a65c62352ad324 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 18 Jan 2024 14:55:34 +0100 Subject: [PATCH 282/332] fix bug --- polytope/datacube/backends/fdb.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index f682b671f..4d7b7aa57 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -85,10 +85,10 @@ def get_2nd_last_values(self, requests, leaf_path={}): nearest_latlons.append(nearest_latlon) # TODO: now combine with the rest of the function.... # TODO: need to remove the branches that do not fit - copy_requests = deepcopy(requests) - for i in range(len(copy_requests.children)): - lat_child = copy_requests.children[i] - lat_child = [child for child in requests.children if child.value == lat_child.value][0] + lat_children_values = [child.value for child in requests.children] + for i in range(len(lat_children_values)): + lat_child_val = lat_children_values[i] + lat_child = [child for child in requests.children if child.value == lat_child_val][0] if lat_child.value not in [latlon[0] for latlon in nearest_latlons]: lat_child.remove_branch() else: @@ -182,7 +182,10 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) + print("REQUEST TO FDB") + print(fdb_requests) output_values = self.fdb.extract(fdb_requests) + print(output_values) return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): From f42f37d89d3780aaaa24c8a7c696991620f3263f Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 19 Jan 2024 10:07:13 +0100 Subject: [PATCH 283/332] make tests work --- examples/healpix_grid_box_example.py | 6 +----- tests/test_point_nearest.py | 22 ++++++++++++++++++++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/examples/healpix_grid_box_example.py b/examples/healpix_grid_box_example.py index 8e676fa3b..6cfbb5e78 100644 --- a/examples/healpix_grid_box_example.py +++ b/examples/healpix_grid_box_example.py @@ -14,11 +14,7 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/healpix.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(time=0).isel(isobaricInhPa=0).z self.xarraydatacube = XArrayDatacube(self.latlon_array) - self.options = { - "values": { - "transformation": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}} - } - } + self.options = {"values": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}}} self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index 9166fd794..c1ae7dbd8 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -38,7 +38,7 @@ def test_fdb_datacube(self): Point(["latitude", "longitude"], [[0.16, 0.176]], method="nearest"), ) result = self.API.retrieve(request) - result.pprint() + # result.pprint() assert len(result.leaves) == 1 @pytest.mark.fdb @@ -56,5 +56,23 @@ def test_fdb_datacube_true_point(self): Point(["latitude", "longitude"], [[0.175746921078, 0.210608424337]], method="nearest"), ) result = self.API.retrieve(request) - result.pprint() + # result.pprint() assert len(result.leaves) == 1 + + @pytest.mark.fdb + def test_fdb_datacube_true_point_2(self): + request = Request( + Select("step", [21, 22, 23]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231102T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[0.035149384216, 0.0]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 3 From e2161ca801751eac5baf48003b4f57b458f55e38 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 19 Jan 2024 10:37:01 +0100 Subject: [PATCH 284/332] fix tree merging for multiple params --- polytope/datacube/index_tree.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index 9054cd0c5..f5a3106d1 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -84,13 +84,16 @@ def __eq__(self, other): else: if other.value == self.value: return True - if other.value - 2 * other.axis.tol <= self.value <= other.value + 2 * other.axis.tol: - return True - elif self.value - 2 * self.axis.tol <= other.value <= self.value + 2 * self.axis.tol: - return True else: - return False - # return (self.axis.name, self.value) == (other.axis.name, other.value) + if isinstance(self.value, str): + return False + else: + if other.value - 2 * other.axis.tol <= self.value <= other.value + 2 * other.axis.tol: + return True + elif self.value - 2 * self.axis.tol <= other.value <= self.value + 2 * self.axis.tol: + return True + else: + return False def __lt__(self, other): return (self.axis.name, self.value) < (other.axis.name, other.value) From 721c53adbefd5061e7769141cee17cf8990a0fa5 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 19 Jan 2024 10:44:23 +0100 Subject: [PATCH 285/332] nicer fix --- polytope/datacube/index_tree.py | 4 ++-- tests/test_multiple_param_fdb.py | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 tests/test_multiple_param_fdb.py diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index f5a3106d1..303bd4d84 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -3,7 +3,7 @@ from sortedcontainers import SortedList -from .datacube_axis import IntDatacubeAxis +from .datacube_axis import IntDatacubeAxis, UnsliceableDatacubeAxis class DatacubePath(OrderedDict): @@ -85,7 +85,7 @@ def __eq__(self, other): if other.value == self.value: return True else: - if isinstance(self.value, str): + if isinstance(self.axis, UnsliceableDatacubeAxis): return False else: if other.value - 2 * other.axis.tol <= self.value <= other.value + 2 * other.axis.tol: diff --git a/tests/test_multiple_param_fdb.py b/tests/test_multiple_param_fdb.py new file mode 100644 index 000000000..04b8e7127 --- /dev/null +++ b/tests/test_multiple_param_fdb.py @@ -0,0 +1,41 @@ +import pandas as pd +import pytest + +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.fdb + def test_fdb_datacube(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240118T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["49", "167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 18 From 8566ff6c5f3f5fafb1c640b226c05c8d2c865f32 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 22 Jan 2024 13:21:12 +0100 Subject: [PATCH 286/332] add decorators in individual transformation files --- polytope/datacube/datacube_axis.py | 534 +----------------- .../transformations/datacube_cyclic.py | 187 ++++++ .../transformations/datacube_mappers.py | 106 ++++ .../transformations/datacube_merger.py | 76 +++ .../datacube_null_transformation.py | 24 + .../transformations/datacube_reverse.py | 65 +++ .../transformations/datacube_type_change.py | 71 +++ 7 files changed, 534 insertions(+), 529 deletions(-) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 5c78c22f6..129581d00 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -1,5 +1,3 @@ -import bisect -import math from abc import ABC, abstractmethod from copy import deepcopy from typing import Any, List @@ -7,533 +5,11 @@ import numpy as np import pandas as pd -from ..utility.list_tools import bisect_left_cmp, bisect_right_cmp - - -def cyclic(cls): - if cls.is_cyclic: - from .transformations.datacube_cyclic import DatacubeAxisCyclic - - def update_range(): - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisCyclic): - transformation = transform - cls.range = transformation.range - - def to_intervals(range): - update_range() - if range[0] == -math.inf: - range[0] = cls.range[0] - if range[1] == math.inf: - range[1] = cls.range[1] - axis_lower = cls.range[0] - axis_upper = cls.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - intervals = [] - if lower < axis_upper: - # In this case, we want to go from lower to the first remapped cyclic axis upper - # or the asked upper range value. - # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, - # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper - # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], - # where -180 is the asked upper range value. - loops = int((axis_upper - lower) / axis_range) - remapped_up = axis_upper - (loops) * axis_range - new_upper = min(upper, remapped_up) - else: - # In this case, since lower >= axis_upper, we need to either go to the asked upper range - # or we need to go to the first remapped cyclic axis upper which is higher than lower - new_upper = min(axis_upper + axis_range, upper) - while new_upper < lower: - new_upper = min(new_upper + axis_range, upper) - intervals.append([lower, new_upper]) - # Now that we have established what the first interval should be, we should just jump from cyclic range - # to cyclic range until we hit the asked upper range value. - new_up = deepcopy(new_upper) - while new_up < upper: - new_upper = new_up - new_up = min(upper, new_upper + axis_range) - intervals.append([new_upper, new_up]) - # Once we have added all the in-between ranges, we need to add the last interval - intervals.append([new_up, upper]) - return intervals - - def _remap_range_to_axis_range(range): - update_range() - axis_lower = cls.range[0] - axis_upper = cls.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - if lower < axis_lower: - # In this case we need to calculate the number of loops between the axis lower - # and the lower to recenter the lower - loops = int((axis_lower - lower - cls.tol) / axis_range) - return_lower = lower + (loops + 1) * axis_range - return_upper = upper + (loops + 1) * axis_range - elif lower >= axis_upper: - # In this case we need to calculate the number of loops between the axis upper - # and the lower to recenter the lower - loops = int((lower - axis_upper) / axis_range) - return_lower = lower - (loops + 1) * axis_range - return_upper = upper - (loops + 1) * axis_range - else: - # In this case, the lower value is already in the right range - return_lower = lower - return_upper = upper - return [return_lower, return_upper] - - def _remap_val_to_axis_range(value): - return_range = _remap_range_to_axis_range([value, value]) - return return_range[0] - - def remap(range: List): - update_range() - if cls.range[0] - cls.tol <= range[0] <= cls.range[1] + cls.tol: - if cls.range[0] - cls.tol <= range[1] <= cls.range[1] + cls.tol: - # If we are already in the cyclic range, return it - return [range] - elif abs(range[0] - range[1]) <= 2 * cls.tol: - # If we have a range that is just one point, then it should still be counted - # and so we should take a small interval around it to find values inbetween - range = [ - _remap_val_to_axis_range(range[0]) - cls.tol, - _remap_val_to_axis_range(range[0]) + cls.tol, - ] - return [range] - range_intervals = cls.to_intervals(range) - ranges = [] - for interval in range_intervals: - if abs(interval[0] - interval[1]) > 0: - # If the interval is not just a single point, we remap it to the axis range - range = _remap_range_to_axis_range([interval[0], interval[1]]) - up = range[1] - low = range[0] - if up < low: - # Make sure we remap in the right order - ranges.append([up - cls.tol, low + cls.tol]) - else: - ranges.append([low - cls.tol, up + cls.tol]) - return ranges - - old_find_indexes = cls.find_indexes - - def find_indexes(path, datacube): - return old_find_indexes(path, datacube) - - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path, unwanted_path): - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisCyclic): - if cls.name == transform.name: - new_val = _remap_val_to_axis_range(value) - key_value_path[cls.name] = new_val - key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) - return (key_value_path, leaf_path, unwanted_path) - - old_unmap_to_datacube = cls.unmap_to_datacube - - def unmap_to_datacube(path, unmapped_path): - (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) - return (path, unmapped_path) - - old_find_indices_between = cls.find_indices_between - - def find_indices_between(index_ranges, low, up, datacube, method=None): - update_range() - indexes_between_ranges = [] - - if method != "surrounding": - return old_find_indices_between(index_ranges, low, up, datacube, method) - else: - for indexes in index_ranges: - if cls.name in datacube.complete_axes: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - else: - start = bisect.bisect_left(indexes, low) - end = bisect.bisect_right(indexes, up) - - if start - 1 < 0: - index_val_found = indexes[-1:][0] - indexes_between_ranges.append([index_val_found]) - if end + 1 > len(indexes): - index_val_found = indexes[:2][0] - indexes_between_ranges.append([index_val_found]) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - if cls.name in datacube.complete_axes: - indexes_between = indexes[start:end].to_list() - else: - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def offset(range): - # We first unpad the range by the axis tolerance to make sure that - # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. - # Also, it's safer that we find the offset of a value inside the range instead of on the border - unpadded_range = [range[0] + 1.5 * cls.tol, range[1] - 1.5 * cls.tol] - cyclic_range = _remap_range_to_axis_range(unpadded_range) - offset = unpadded_range[0] - cyclic_range[0] - return offset - - cls.to_intervals = to_intervals - cls.remap = remap - cls.offset = offset - cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube - cls.find_indices_between = find_indices_between - cls.unmap_path_key = unmap_path_key - - return cls - - -def mapper(cls): - from .transformations.datacube_mappers import DatacubeMapper - - if cls.has_mapper: - - def find_indexes(path, datacube): - # first, find the relevant transformation object that is a mapping in the cls.transformation dico - for transform in cls.transformations: - if isinstance(transform, DatacubeMapper): - transformation = transform - if cls.name == transformation._mapped_axes()[0]: - return transformation.first_axis_vals() - if cls.name == transformation._mapped_axes()[1]: - first_val = path[transformation._mapped_axes()[0]] - return transformation.second_axis_vals(first_val) - - old_unmap_to_datacube = cls.unmap_to_datacube - - def unmap_to_datacube(path, unmapped_path): - (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) - for transform in cls.transformations: - if isinstance(transform, DatacubeMapper): - if cls.name == transform._mapped_axes()[0]: - # if we are on the first axis, then need to add the first val to unmapped_path - first_val = path.get(cls.name, None) - path.pop(cls.name, None) - if cls.name not in unmapped_path: - # if for some reason, the unmapped_path already has the first axis val, then don't update - unmapped_path[cls.name] = first_val - if cls.name == transform._mapped_axes()[1]: - # if we are on the second axis, then the val of the first axis is stored - # inside unmapped_path so can get it from there - second_val = path.get(cls.name, None) - path.pop(cls.name, None) - first_val = unmapped_path.get(transform._mapped_axes()[0], None) - unmapped_path.pop(transform._mapped_axes()[0], None) - # if the first_val was not in the unmapped_path, then it's still in path - if first_val is None: - first_val = path.get(transform._mapped_axes()[0], None) - path.pop(transform._mapped_axes()[0], None) - if first_val is not None and second_val is not None: - unmapped_idx = transform.unmap(first_val, second_val) - unmapped_path[transform.old_axis] = unmapped_idx - return (path, unmapped_path) - - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path, unwanted_path): - key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeMapper): - if cls.name == transform._mapped_axes()[0]: - unwanted_val = key_value_path[transform._mapped_axes()[0]] - unwanted_path[cls.name] = unwanted_val - if cls.name == transform._mapped_axes()[1]: - first_val = unwanted_path[transform._mapped_axes()[0]] - unmapped_idx = transform.unmap(first_val, value) - leaf_path.pop(transform._mapped_axes()[0], None) - key_value_path.pop(cls.name) - key_value_path[transform.old_axis] = unmapped_idx - return (key_value_path, leaf_path, unwanted_path) - - def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping - indexes_between_ranges = [] - for transform in cls.transformations: - if isinstance(transform, DatacubeMapper): - transformation = transform - if cls.name in transformation._mapped_axes(): - for idxs in index_ranges: - if method == "surrounding": - axis_reversed = transform._axis_reversed[cls.name] - if not axis_reversed: - start = bisect.bisect_left(idxs, low) - end = bisect.bisect_right(idxs, up) - else: - # TODO: do the custom bisect - end = bisect_left_cmp(idxs, low, cmp=lambda x, y: x > y) + 1 - start = bisect_right_cmp(idxs, up, cmp=lambda x, y: x > y) - start = max(start - 1, 0) - end = min(end + 1, len(idxs)) - indexes_between = idxs[start:end] - indexes_between_ranges.append(indexes_between) - else: - axis_reversed = transform._axis_reversed[cls.name] - if not axis_reversed: - lower_idx = bisect.bisect_left(idxs, low) - upper_idx = bisect.bisect_right(idxs, up) - indexes_between = idxs[lower_idx:upper_idx] - else: - # TODO: do the custom bisect - end_idx = bisect_left_cmp(idxs, low, cmp=lambda x, y: x > y) + 1 - start_idx = bisect_right_cmp(idxs, up, cmp=lambda x, y: x > y) - indexes_between = idxs[start_idx:end_idx] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube - cls.find_indices_between = find_indices_between - cls.unmap_path_key = unmap_path_key - - return cls - - -def merge(cls): - from .transformations.datacube_merger import DatacubeAxisMerger - - if cls.has_merger: - - def find_indexes(path, datacube): - # first, find the relevant transformation object that is a mapping in the cls.transformation dico - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name == transformation._first_axis: - return transformation.merged_values(datacube) - - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path, unwanted_path): - key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) - new_key_value_path = {} - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - if cls.name == transform._first_axis: - (first_val, second_val) = transform.unmerge(value) - new_key_value_path[transform._first_axis] = first_val - new_key_value_path[transform._second_axis] = second_val - return (new_key_value_path, leaf_path, unwanted_path) - - old_unmap_to_datacube = cls.unmap_to_datacube - - def unmap_to_datacube(path, unmapped_path): - (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name == transformation._first_axis: - old_val = path.get(cls.name, None) - (first_val, second_val) = transformation.unmerge(old_val) - path.pop(cls.name, None) - path[transformation._first_axis] = first_val - path[transformation._second_axis] = second_val - return (path, unmapped_path) - - def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping - indexes_between_ranges = [] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name in transformation._mapped_axes(): - for indexes in index_ranges: - if method == "surrounding": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def remap(range): - return [range] - - cls.remap = remap - cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube - cls.find_indices_between = find_indices_between - cls.unmap_path_key = unmap_path_key - - return cls - - -def reverse(cls): - from .transformations.datacube_reverse import DatacubeAxisReverse - - if cls.reorder: - - def find_indexes(path, datacube): - # first, find the relevant transformation object that is a mapping in the cls.transformation dico - subarray = datacube.dataarray.sel(path, method="nearest") - unordered_indices = datacube.datacube_natural_indexes(cls, subarray) - if cls.name in datacube.complete_axes: - ordered_indices = unordered_indices.sort_values() - else: - ordered_indices = unordered_indices - return ordered_indices - - def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping - indexes_between_ranges = [] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisReverse): - transformation = transform - if cls.name == transformation.name: - for indexes in index_ranges: - if cls.name in datacube.complete_axes: - # Find the range of indexes between lower and upper - # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html - # Assumes the indexes are already sorted (could sort to be sure) and monotonically - # increasing - if method == "surrounding": - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end].to_list() - indexes_between_ranges.append(indexes_between) - else: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - indexes_between = indexes[start:end].to_list() - indexes_between_ranges.append(indexes_between) - else: - if method == "surrounding": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def remap(range): - return [range] - - cls.remap = remap - cls.find_indexes = find_indexes - cls.find_indices_between = find_indices_between - - return cls - - -def type_change(cls): - from .transformations.datacube_type_change import DatacubeAxisTypeChange - - if cls.type_change: - old_find_indexes = cls.find_indexes - - def find_indexes(path, datacube): - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - original_vals = old_find_indexes(path, datacube) - return transformation.change_val_type(cls.name, original_vals) - - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path, unwanted_path): - key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - if cls.name == transform.name: - unchanged_val = transform.make_str(value) - key_value_path[cls.name] = unchanged_val - return (key_value_path, leaf_path, unwanted_path) - - def unmap_to_datacube(path, unmapped_path): - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - changed_val = path.get(cls.name, None) - unchanged_val = transformation.make_str(changed_val) - if cls.name in path: - path.pop(cls.name, None) - unmapped_path[cls.name] = unchanged_val - return (path, unmapped_path) - - def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping - indexes_between_ranges = [] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - for indexes in index_ranges: - if method == "surrounding": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def remap(range): - return [range] - - cls.remap = remap - cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube - cls.find_indices_between = find_indices_between - cls.unmap_path_key = unmap_path_key - - return cls - - -def null(cls): - if cls.type_change: - old_find_indexes = cls.find_indexes - - def find_indexes(path, datacube): - return old_find_indexes(path, datacube) - - def find_indices_between(index_ranges, low, up, datacube, method=None): - indexes_between_ranges = [] - for indexes in index_ranges: - indexes_between = [i for i in indexes if low <= i <= up] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def remap(range): - return [range] - - cls.remap = remap - cls.find_indexes = find_indexes - cls.find_indices_between = find_indices_between - - return cls +from .transformations.datacube_cyclic import cyclic +from .transformations.datacube_mappers import mapper +from .transformations.datacube_merger import merge +from .transformations.datacube_reverse import reverse +from .transformations.datacube_type_change import type_change class DatacubeAxis(ABC): diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic.py index 802285c32..ba93de2e4 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic.py @@ -1,3 +1,8 @@ +import bisect +import math +from copy import deepcopy +from typing import List + from .datacube_transformations import DatacubeAxisTransformation @@ -23,3 +28,185 @@ def blocked_axes(self): def unwanted_axes(self): return [] + + +def cyclic(cls): + if cls.is_cyclic: + + def update_range(): + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisCyclic): + transformation = transform + cls.range = transformation.range + + def to_intervals(range): + update_range() + if range[0] == -math.inf: + range[0] = cls.range[0] + if range[1] == math.inf: + range[1] = cls.range[1] + axis_lower = cls.range[0] + axis_upper = cls.range[1] + axis_range = axis_upper - axis_lower + lower = range[0] + upper = range[1] + intervals = [] + if lower < axis_upper: + # In this case, we want to go from lower to the first remapped cyclic axis upper + # or the asked upper range value. + # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, + # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper + # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], + # where -180 is the asked upper range value. + loops = int((axis_upper - lower) / axis_range) + remapped_up = axis_upper - (loops) * axis_range + new_upper = min(upper, remapped_up) + else: + # In this case, since lower >= axis_upper, we need to either go to the asked upper range + # or we need to go to the first remapped cyclic axis upper which is higher than lower + new_upper = min(axis_upper + axis_range, upper) + while new_upper < lower: + new_upper = min(new_upper + axis_range, upper) + intervals.append([lower, new_upper]) + # Now that we have established what the first interval should be, we should just jump from cyclic range + # to cyclic range until we hit the asked upper range value. + new_up = deepcopy(new_upper) + while new_up < upper: + new_upper = new_up + new_up = min(upper, new_upper + axis_range) + intervals.append([new_upper, new_up]) + # Once we have added all the in-between ranges, we need to add the last interval + intervals.append([new_up, upper]) + return intervals + + def _remap_range_to_axis_range(range): + update_range() + axis_lower = cls.range[0] + axis_upper = cls.range[1] + axis_range = axis_upper - axis_lower + lower = range[0] + upper = range[1] + if lower < axis_lower: + # In this case we need to calculate the number of loops between the axis lower + # and the lower to recenter the lower + loops = int((axis_lower - lower - cls.tol) / axis_range) + return_lower = lower + (loops + 1) * axis_range + return_upper = upper + (loops + 1) * axis_range + elif lower >= axis_upper: + # In this case we need to calculate the number of loops between the axis upper + # and the lower to recenter the lower + loops = int((lower - axis_upper) / axis_range) + return_lower = lower - (loops + 1) * axis_range + return_upper = upper - (loops + 1) * axis_range + else: + # In this case, the lower value is already in the right range + return_lower = lower + return_upper = upper + return [return_lower, return_upper] + + def _remap_val_to_axis_range(value): + return_range = _remap_range_to_axis_range([value, value]) + return return_range[0] + + def remap(range: List): + update_range() + if cls.range[0] - cls.tol <= range[0] <= cls.range[1] + cls.tol: + if cls.range[0] - cls.tol <= range[1] <= cls.range[1] + cls.tol: + # If we are already in the cyclic range, return it + return [range] + elif abs(range[0] - range[1]) <= 2 * cls.tol: + # If we have a range that is just one point, then it should still be counted + # and so we should take a small interval around it to find values inbetween + range = [ + _remap_val_to_axis_range(range[0]) - cls.tol, + _remap_val_to_axis_range(range[0]) + cls.tol, + ] + return [range] + range_intervals = cls.to_intervals(range) + ranges = [] + for interval in range_intervals: + if abs(interval[0] - interval[1]) > 0: + # If the interval is not just a single point, we remap it to the axis range + range = _remap_range_to_axis_range([interval[0], interval[1]]) + up = range[1] + low = range[0] + if up < low: + # Make sure we remap in the right order + ranges.append([up - cls.tol, low + cls.tol]) + else: + ranges.append([low - cls.tol, up + cls.tol]) + return ranges + + old_find_indexes = cls.find_indexes + + def find_indexes(path, datacube): + return old_find_indexes(path, datacube) + + old_unmap_path_key = cls.unmap_path_key + + def unmap_path_key(key_value_path, leaf_path, unwanted_path): + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisCyclic): + if cls.name == transform.name: + new_val = _remap_val_to_axis_range(value) + key_value_path[cls.name] = new_val + key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) + return (key_value_path, leaf_path, unwanted_path) + + old_unmap_to_datacube = cls.unmap_to_datacube + + def unmap_to_datacube(path, unmapped_path): + (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) + return (path, unmapped_path) + + old_find_indices_between = cls.find_indices_between + + def find_indices_between(index_ranges, low, up, datacube, method=None): + update_range() + indexes_between_ranges = [] + + if method != "surrounding": + return old_find_indices_between(index_ranges, low, up, datacube, method) + else: + for indexes in index_ranges: + if cls.name in datacube.complete_axes: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + else: + start = bisect.bisect_left(indexes, low) + end = bisect.bisect_right(indexes, up) + + if start - 1 < 0: + index_val_found = indexes[-1:][0] + indexes_between_ranges.append([index_val_found]) + if end + 1 > len(indexes): + index_val_found = indexes[:2][0] + indexes_between_ranges.append([index_val_found]) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + if cls.name in datacube.complete_axes: + indexes_between = indexes[start:end].to_list() + else: + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + def offset(range): + # We first unpad the range by the axis tolerance to make sure that + # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. + # Also, it's safer that we find the offset of a value inside the range instead of on the border + unpadded_range = [range[0] + 1.5 * cls.tol, range[1] - 1.5 * cls.tol] + cyclic_range = _remap_range_to_axis_range(unpadded_range) + offset = unpadded_range[0] - cyclic_range[0] + return offset + + cls.to_intervals = to_intervals + cls.remap = remap + cls.offset = offset + cls.find_indexes = find_indexes + cls.unmap_to_datacube = unmap_to_datacube + cls.find_indices_between = find_indices_between + cls.unmap_path_key = unmap_path_key + + return cls diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 2e94c6494..d410f9911 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -1,6 +1,8 @@ +import bisect from copy import deepcopy from importlib import import_module +from ...utility.list_tools import bisect_left_cmp, bisect_right_cmp from .datacube_transformations import DatacubeAxisTransformation @@ -79,3 +81,107 @@ def unmap(self, first_val, second_val): "regular": "RegularGridMapper", "reduced_ll": "ReducedLatLonMapper", } + + +def mapper(cls): + if cls.has_mapper: + + def find_indexes(path, datacube): + # first, find the relevant transformation object that is a mapping in the cls.transformation dico + for transform in cls.transformations: + if isinstance(transform, DatacubeMapper): + transformation = transform + if cls.name == transformation._mapped_axes()[0]: + return transformation.first_axis_vals() + if cls.name == transformation._mapped_axes()[1]: + first_val = path[transformation._mapped_axes()[0]] + return transformation.second_axis_vals(first_val) + + old_unmap_to_datacube = cls.unmap_to_datacube + + def unmap_to_datacube(path, unmapped_path): + (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) + for transform in cls.transformations: + if isinstance(transform, DatacubeMapper): + if cls.name == transform._mapped_axes()[0]: + # if we are on the first axis, then need to add the first val to unmapped_path + first_val = path.get(cls.name, None) + path.pop(cls.name, None) + if cls.name not in unmapped_path: + # if for some reason, the unmapped_path already has the first axis val, then don't update + unmapped_path[cls.name] = first_val + if cls.name == transform._mapped_axes()[1]: + # if we are on the second axis, then the val of the first axis is stored + # inside unmapped_path so can get it from there + second_val = path.get(cls.name, None) + path.pop(cls.name, None) + first_val = unmapped_path.get(transform._mapped_axes()[0], None) + unmapped_path.pop(transform._mapped_axes()[0], None) + # if the first_val was not in the unmapped_path, then it's still in path + if first_val is None: + first_val = path.get(transform._mapped_axes()[0], None) + path.pop(transform._mapped_axes()[0], None) + if first_val is not None and second_val is not None: + unmapped_idx = transform.unmap(first_val, second_val) + unmapped_path[transform.old_axis] = unmapped_idx + return (path, unmapped_path) + + old_unmap_path_key = cls.unmap_path_key + + def unmap_path_key(key_value_path, leaf_path, unwanted_path): + key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeMapper): + if cls.name == transform._mapped_axes()[0]: + unwanted_val = key_value_path[transform._mapped_axes()[0]] + unwanted_path[cls.name] = unwanted_val + if cls.name == transform._mapped_axes()[1]: + first_val = unwanted_path[transform._mapped_axes()[0]] + unmapped_idx = transform.unmap(first_val, value) + leaf_path.pop(transform._mapped_axes()[0], None) + key_value_path.pop(cls.name) + key_value_path[transform.old_axis] = unmapped_idx + return (key_value_path, leaf_path, unwanted_path) + + def find_indices_between(index_ranges, low, up, datacube, method=None): + # TODO: add method for snappping + indexes_between_ranges = [] + for transform in cls.transformations: + if isinstance(transform, DatacubeMapper): + transformation = transform + if cls.name in transformation._mapped_axes(): + for idxs in index_ranges: + if method == "surrounding": + axis_reversed = transform._axis_reversed[cls.name] + if not axis_reversed: + start = bisect.bisect_left(idxs, low) + end = bisect.bisect_right(idxs, up) + else: + # TODO: do the custom bisect + end = bisect_left_cmp(idxs, low, cmp=lambda x, y: x > y) + 1 + start = bisect_right_cmp(idxs, up, cmp=lambda x, y: x > y) + start = max(start - 1, 0) + end = min(end + 1, len(idxs)) + indexes_between = idxs[start:end] + indexes_between_ranges.append(indexes_between) + else: + axis_reversed = transform._axis_reversed[cls.name] + if not axis_reversed: + lower_idx = bisect.bisect_left(idxs, low) + upper_idx = bisect.bisect_right(idxs, up) + indexes_between = idxs[lower_idx:upper_idx] + else: + # TODO: do the custom bisect + end_idx = bisect_left_cmp(idxs, low, cmp=lambda x, y: x > y) + 1 + start_idx = bisect_right_cmp(idxs, up, cmp=lambda x, y: x > y) + indexes_between = idxs[start_idx:end_idx] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + cls.find_indexes = find_indexes + cls.unmap_to_datacube = unmap_to_datacube + cls.find_indices_between = find_indices_between + cls.unmap_path_key = unmap_path_key + + return cls diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger.py index d60278671..883ea964a 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger.py @@ -1,3 +1,5 @@ +import bisect + import numpy as np import pandas as pd @@ -61,3 +63,77 @@ def unmerge(self, merged_val): def change_val_type(self, axis_name, values): new_values = pd.to_datetime(values) return new_values + + +def merge(cls): + if cls.has_merger: + + def find_indexes(path, datacube): + # first, find the relevant transformation object that is a mapping in the cls.transformation dico + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisMerger): + transformation = transform + if cls.name == transformation._first_axis: + return transformation.merged_values(datacube) + + old_unmap_path_key = cls.unmap_path_key + + def unmap_path_key(key_value_path, leaf_path, unwanted_path): + key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) + new_key_value_path = {} + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisMerger): + if cls.name == transform._first_axis: + (first_val, second_val) = transform.unmerge(value) + new_key_value_path[transform._first_axis] = first_val + new_key_value_path[transform._second_axis] = second_val + return (new_key_value_path, leaf_path, unwanted_path) + + old_unmap_to_datacube = cls.unmap_to_datacube + + def unmap_to_datacube(path, unmapped_path): + (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisMerger): + transformation = transform + if cls.name == transformation._first_axis: + old_val = path.get(cls.name, None) + (first_val, second_val) = transformation.unmerge(old_val) + path.pop(cls.name, None) + path[transformation._first_axis] = first_val + path[transformation._second_axis] = second_val + return (path, unmapped_path) + + def find_indices_between(index_ranges, low, up, datacube, method=None): + # TODO: add method for snappping + indexes_between_ranges = [] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisMerger): + transformation = transform + if cls.name in transformation._mapped_axes(): + for indexes in index_ranges: + if method == "surrounding": + start = indexes.index(low) + end = indexes.index(up) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + lower_idx = bisect.bisect_left(indexes, low) + upper_idx = bisect.bisect_right(indexes, up) + indexes_between = indexes[lower_idx:upper_idx] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + def remap(range): + return [range] + + cls.remap = remap + cls.find_indexes = find_indexes + cls.unmap_to_datacube = unmap_to_datacube + cls.find_indices_between = find_indices_between + cls.unmap_path_key = unmap_path_key + + return cls diff --git a/polytope/datacube/transformations/datacube_null_transformation.py b/polytope/datacube/transformations/datacube_null_transformation.py index 55c94277e..66af3f3a9 100644 --- a/polytope/datacube/transformations/datacube_null_transformation.py +++ b/polytope/datacube/transformations/datacube_null_transformation.py @@ -20,3 +20,27 @@ def blocked_axes(self): def unwanted_axes(self): return [] + + +def null(cls): + if cls.type_change: + old_find_indexes = cls.find_indexes + + def find_indexes(path, datacube): + return old_find_indexes(path, datacube) + + def find_indices_between(index_ranges, low, up, datacube, method=None): + indexes_between_ranges = [] + for indexes in index_ranges: + indexes_between = [i for i in indexes if low <= i <= up] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + def remap(range): + return [range] + + cls.remap = remap + cls.find_indexes = find_indexes + cls.find_indices_between = find_indices_between + + return cls diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse.py index 6a556907a..0e03006a7 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse.py @@ -1,3 +1,5 @@ +import bisect + from .datacube_transformations import DatacubeAxisTransformation @@ -20,3 +22,66 @@ def blocked_axes(self): def unwanted_axes(self): return [] + + +def reverse(cls): + if cls.reorder: + + def find_indexes(path, datacube): + # first, find the relevant transformation object that is a mapping in the cls.transformation dico + subarray = datacube.dataarray.sel(path, method="nearest") + unordered_indices = datacube.datacube_natural_indexes(cls, subarray) + if cls.name in datacube.complete_axes: + ordered_indices = unordered_indices.sort_values() + else: + ordered_indices = unordered_indices + return ordered_indices + + def find_indices_between(index_ranges, low, up, datacube, method=None): + # TODO: add method for snappping + indexes_between_ranges = [] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisReverse): + transformation = transform + if cls.name == transformation.name: + for indexes in index_ranges: + if cls.name in datacube.complete_axes: + # Find the range of indexes between lower and upper + # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html + # Assumes the indexes are already sorted (could sort to be sure) and monotonically + # increasing + if method == "surrounding": + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.append(indexes_between) + else: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.append(indexes_between) + else: + if method == "surrounding": + start = indexes.index(low) + end = indexes.index(up) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + lower_idx = bisect.bisect_left(indexes, low) + upper_idx = bisect.bisect_right(indexes, up) + indexes_between = indexes[lower_idx:upper_idx] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + def remap(range): + return [range] + + cls.remap = remap + cls.find_indexes = find_indexes + cls.find_indices_between = find_indices_between + + return cls diff --git a/polytope/datacube/transformations/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change.py index cdc046b76..1813efe24 100644 --- a/polytope/datacube/transformations/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change.py @@ -1,3 +1,4 @@ +import bisect from copy import deepcopy from importlib import import_module @@ -51,3 +52,73 @@ def make_str(self, value): _type_to_datacube_type_change_lookup = {"int": "TypeChangeStrToInt"} + + +def type_change(cls): + if cls.type_change: + old_find_indexes = cls.find_indexes + + def find_indexes(path, datacube): + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisTypeChange): + transformation = transform + if cls.name == transformation.name: + original_vals = old_find_indexes(path, datacube) + return transformation.change_val_type(cls.name, original_vals) + + old_unmap_path_key = cls.unmap_path_key + + def unmap_path_key(key_value_path, leaf_path, unwanted_path): + key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisTypeChange): + if cls.name == transform.name: + unchanged_val = transform.make_str(value) + key_value_path[cls.name] = unchanged_val + return (key_value_path, leaf_path, unwanted_path) + + def unmap_to_datacube(path, unmapped_path): + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisTypeChange): + transformation = transform + if cls.name == transformation.name: + changed_val = path.get(cls.name, None) + unchanged_val = transformation.make_str(changed_val) + if cls.name in path: + path.pop(cls.name, None) + unmapped_path[cls.name] = unchanged_val + return (path, unmapped_path) + + def find_indices_between(index_ranges, low, up, datacube, method=None): + # TODO: add method for snappping + indexes_between_ranges = [] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisTypeChange): + transformation = transform + if cls.name == transformation.name: + for indexes in index_ranges: + if method == "surrounding": + start = indexes.index(low) + end = indexes.index(up) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + lower_idx = bisect.bisect_left(indexes, low) + upper_idx = bisect.bisect_right(indexes, up) + indexes_between = indexes[lower_idx:upper_idx] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + def remap(range): + return [range] + + cls.remap = remap + cls.find_indexes = find_indexes + cls.unmap_to_datacube = unmap_to_datacube + cls.find_indices_between = find_indices_between + cls.unmap_path_key = unmap_path_key + + return cls From 8eeb0b5087aa7ece5ad406db06e6e0564b43fe6e Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 22 Jan 2024 16:15:27 +0100 Subject: [PATCH 287/332] refactor some transformations --- polytope/datacube/datacube_axis.py | 4 +- .../datacube_cyclic/cyclic_axis_decorator.py | 188 ++++++++++++++++++ .../{ => datacube_cyclic}/datacube_cyclic.py | 2 +- .../datacube_mappers/datacube_mappers.py | 81 ++++++++ .../mapper_axis_decorator.py} | 83 +------- .../mapper_types}/healpix.py | 0 .../mapper_types}/octahedral.py | 2 +- .../mapper_types}/reduced_ll.py | 0 .../mapper_types}/regular.py | 0 .../datacube_transformations.py | 4 +- tests/test_mappers.py | 4 +- 11 files changed, 280 insertions(+), 88 deletions(-) create mode 100644 polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py rename polytope/datacube/transformations/{ => datacube_cyclic}/datacube_cyclic.py (99%) create mode 100644 polytope/datacube/transformations/datacube_mappers/datacube_mappers.py rename polytope/datacube/transformations/{datacube_mappers.py => datacube_mappers/mapper_axis_decorator.py} (67%) rename polytope/datacube/transformations/{mappers => datacube_mappers/mapper_types}/healpix.py (100%) rename polytope/datacube/transformations/{mappers => datacube_mappers/mapper_types}/octahedral.py (99%) rename polytope/datacube/transformations/{mappers => datacube_mappers/mapper_types}/reduced_ll.py (100%) rename polytope/datacube/transformations/{mappers => datacube_mappers/mapper_types}/regular.py (100%) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 129581d00..2e88fb26f 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -5,8 +5,8 @@ import numpy as np import pandas as pd -from .transformations.datacube_cyclic import cyclic -from .transformations.datacube_mappers import mapper +from .transformations.datacube_cyclic.cyclic_axis_decorator import cyclic +from .transformations.datacube_mappers.mapper_axis_decorator import mapper from .transformations.datacube_merger import merge from .transformations.datacube_reverse import reverse from .transformations.datacube_type_change import type_change diff --git a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py new file mode 100644 index 000000000..8b1cf2927 --- /dev/null +++ b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py @@ -0,0 +1,188 @@ +import bisect +import math +from copy import deepcopy +from typing import List + +from .datacube_cyclic import DatacubeAxisCyclic + + +def cyclic(cls): + if cls.is_cyclic: + + def update_range(): + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisCyclic): + transformation = transform + cls.range = transformation.range + + def to_intervals(range): + update_range() + if range[0] == -math.inf: + range[0] = cls.range[0] + if range[1] == math.inf: + range[1] = cls.range[1] + axis_lower = cls.range[0] + axis_upper = cls.range[1] + axis_range = axis_upper - axis_lower + lower = range[0] + upper = range[1] + intervals = [] + if lower < axis_upper: + # In this case, we want to go from lower to the first remapped cyclic axis upper + # or the asked upper range value. + # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, + # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper + # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], + # where -180 is the asked upper range value. + loops = int((axis_upper - lower) / axis_range) + remapped_up = axis_upper - (loops) * axis_range + new_upper = min(upper, remapped_up) + else: + # In this case, since lower >= axis_upper, we need to either go to the asked upper range + # or we need to go to the first remapped cyclic axis upper which is higher than lower + new_upper = min(axis_upper + axis_range, upper) + while new_upper < lower: + new_upper = min(new_upper + axis_range, upper) + intervals.append([lower, new_upper]) + # Now that we have established what the first interval should be, we should just jump from cyclic range + # to cyclic range until we hit the asked upper range value. + new_up = deepcopy(new_upper) + while new_up < upper: + new_upper = new_up + new_up = min(upper, new_upper + axis_range) + intervals.append([new_upper, new_up]) + # Once we have added all the in-between ranges, we need to add the last interval + intervals.append([new_up, upper]) + return intervals + + def _remap_range_to_axis_range(range): + update_range() + axis_lower = cls.range[0] + axis_upper = cls.range[1] + axis_range = axis_upper - axis_lower + lower = range[0] + upper = range[1] + if lower < axis_lower: + # In this case we need to calculate the number of loops between the axis lower + # and the lower to recenter the lower + loops = int((axis_lower - lower - cls.tol) / axis_range) + return_lower = lower + (loops + 1) * axis_range + return_upper = upper + (loops + 1) * axis_range + elif lower >= axis_upper: + # In this case we need to calculate the number of loops between the axis upper + # and the lower to recenter the lower + loops = int((lower - axis_upper) / axis_range) + return_lower = lower - (loops + 1) * axis_range + return_upper = upper - (loops + 1) * axis_range + else: + # In this case, the lower value is already in the right range + return_lower = lower + return_upper = upper + return [return_lower, return_upper] + + def _remap_val_to_axis_range(value): + return_range = _remap_range_to_axis_range([value, value]) + return return_range[0] + + def remap(range: List): + update_range() + if cls.range[0] - cls.tol <= range[0] <= cls.range[1] + cls.tol: + if cls.range[0] - cls.tol <= range[1] <= cls.range[1] + cls.tol: + # If we are already in the cyclic range, return it + return [range] + elif abs(range[0] - range[1]) <= 2 * cls.tol: + # If we have a range that is just one point, then it should still be counted + # and so we should take a small interval around it to find values inbetween + range = [ + _remap_val_to_axis_range(range[0]) - cls.tol, + _remap_val_to_axis_range(range[0]) + cls.tol, + ] + return [range] + range_intervals = cls.to_intervals(range) + ranges = [] + for interval in range_intervals: + if abs(interval[0] - interval[1]) > 0: + # If the interval is not just a single point, we remap it to the axis range + range = _remap_range_to_axis_range([interval[0], interval[1]]) + up = range[1] + low = range[0] + if up < low: + # Make sure we remap in the right order + ranges.append([up - cls.tol, low + cls.tol]) + else: + ranges.append([low - cls.tol, up + cls.tol]) + return ranges + + old_find_indexes = cls.find_indexes + + def find_indexes(path, datacube): + return old_find_indexes(path, datacube) + + old_unmap_path_key = cls.unmap_path_key + + def unmap_path_key(key_value_path, leaf_path, unwanted_path): + value = key_value_path[cls.name] + for transform in cls.transformations: + if isinstance(transform, DatacubeAxisCyclic): + if cls.name == transform.name: + new_val = _remap_val_to_axis_range(value) + key_value_path[cls.name] = new_val + key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) + return (key_value_path, leaf_path, unwanted_path) + + old_unmap_to_datacube = cls.unmap_to_datacube + + def unmap_to_datacube(path, unmapped_path): + (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) + return (path, unmapped_path) + + old_find_indices_between = cls.find_indices_between + + def find_indices_between(index_ranges, low, up, datacube, method=None): + update_range() + indexes_between_ranges = [] + + if method != "surrounding": + return old_find_indices_between(index_ranges, low, up, datacube, method) + else: + for indexes in index_ranges: + if cls.name in datacube.complete_axes: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + else: + start = bisect.bisect_left(indexes, low) + end = bisect.bisect_right(indexes, up) + + if start - 1 < 0: + index_val_found = indexes[-1:][0] + indexes_between_ranges.append([index_val_found]) + if end + 1 > len(indexes): + index_val_found = indexes[:2][0] + indexes_between_ranges.append([index_val_found]) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + if cls.name in datacube.complete_axes: + indexes_between = indexes[start:end].to_list() + else: + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + def offset(range): + # We first unpad the range by the axis tolerance to make sure that + # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. + # Also, it's safer that we find the offset of a value inside the range instead of on the border + unpadded_range = [range[0] + 1.5 * cls.tol, range[1] - 1.5 * cls.tol] + cyclic_range = _remap_range_to_axis_range(unpadded_range) + offset = unpadded_range[0] - cyclic_range[0] + return offset + + cls.to_intervals = to_intervals + cls.remap = remap + cls.offset = offset + cls.find_indexes = find_indexes + cls.unmap_to_datacube = unmap_to_datacube + cls.find_indices_between = find_indices_between + cls.unmap_path_key = unmap_path_key + + return cls diff --git a/polytope/datacube/transformations/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py similarity index 99% rename from polytope/datacube/transformations/datacube_cyclic.py rename to polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py index ba93de2e4..fd4da7ae2 100644 --- a/polytope/datacube/transformations/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py @@ -3,7 +3,7 @@ from copy import deepcopy from typing import List -from .datacube_transformations import DatacubeAxisTransformation +from ..datacube_transformations import DatacubeAxisTransformation class DatacubeAxisCyclic(DatacubeAxisTransformation): diff --git a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py new file mode 100644 index 000000000..c42d04a77 --- /dev/null +++ b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py @@ -0,0 +1,81 @@ +from copy import deepcopy +from importlib import import_module + +from ..datacube_transformations import DatacubeAxisTransformation + + +class DatacubeMapper(DatacubeAxisTransformation): + # Needs to implements DatacubeAxisTransformation methods + + def __init__(self, name, mapper_options): + self.transformation_options = mapper_options + self.grid_type = mapper_options["type"] + self.grid_resolution = mapper_options["resolution"] + self.grid_axes = mapper_options["axes"] + self.old_axis = name + self._final_transformation = self.generate_final_transformation() + self._final_mapped_axes = self._final_transformation._mapped_axes + self._axis_reversed = self._final_transformation._axis_reversed + + def generate_final_transformation(self): + map_type = _type_to_datacube_mapper_lookup[self.grid_type] + module = import_module("polytope.datacube.transformations.datacube_mappers.mapper_types." + self.grid_type) + constructor = getattr(module, map_type) + transformation = deepcopy(constructor(self.old_axis, self.grid_axes, self.grid_resolution)) + return transformation + + def blocked_axes(self): + return [] + + def unwanted_axes(self): + return [self._final_mapped_axes[0]] + + def transformation_axes_final(self): + final_axes = self._final_mapped_axes + return final_axes + + # Needs to also implement its own methods + + def change_val_type(self, axis_name, values): + # the new axis_vals created will be floats + return [0.0] + + def _mapped_axes(self): + # NOTE: Each of the mapper method needs to call it's sub mapper method + final_axes = self._final_mapped_axes + return final_axes + + def _base_axis(self): + pass + + def _resolution(self): + pass + + def first_axis_vals(self): + return self._final_transformation.first_axis_vals() + + def second_axis_vals(self, first_val): + return self._final_transformation.second_axis_vals(first_val) + + def map_first_axis(self, lower, upper): + return self._final_transformation.map_first_axis(lower, upper) + + def map_second_axis(self, first_val, lower, upper): + return self._final_transformation.map_second_axis(first_val, lower, upper) + + def find_second_idx(self, first_val, second_val): + return self._final_transformation.find_second_idx(first_val, second_val) + + def unmap_first_val_to_start_line_idx(self, first_val): + return self._final_transformation.unmap_first_val_to_start_line_idx(first_val) + + def unmap(self, first_val, second_val): + return self._final_transformation.unmap(first_val, second_val) + + +_type_to_datacube_mapper_lookup = { + "octahedral": "OctahedralGridMapper", + "healpix": "HealpixGridMapper", + "regular": "RegularGridMapper", + "reduced_ll": "ReducedLatLonMapper", +} diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py similarity index 67% rename from polytope/datacube/transformations/datacube_mappers.py rename to polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py index d410f9911..30f95b619 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py @@ -1,86 +1,7 @@ import bisect -from copy import deepcopy -from importlib import import_module -from ...utility.list_tools import bisect_left_cmp, bisect_right_cmp -from .datacube_transformations import DatacubeAxisTransformation - - -class DatacubeMapper(DatacubeAxisTransformation): - # Needs to implements DatacubeAxisTransformation methods - - def __init__(self, name, mapper_options): - self.transformation_options = mapper_options - self.grid_type = mapper_options["type"] - self.grid_resolution = mapper_options["resolution"] - self.grid_axes = mapper_options["axes"] - self.old_axis = name - self._final_transformation = self.generate_final_transformation() - self._final_mapped_axes = self._final_transformation._mapped_axes - self._axis_reversed = self._final_transformation._axis_reversed - - def generate_final_transformation(self): - map_type = _type_to_datacube_mapper_lookup[self.grid_type] - module = import_module("polytope.datacube.transformations.mappers." + self.grid_type) - constructor = getattr(module, map_type) - transformation = deepcopy(constructor(self.old_axis, self.grid_axes, self.grid_resolution)) - return transformation - - def blocked_axes(self): - return [] - - def unwanted_axes(self): - return [self._final_mapped_axes[0]] - - def transformation_axes_final(self): - final_axes = self._final_mapped_axes - return final_axes - - # Needs to also implement its own methods - - def change_val_type(self, axis_name, values): - # the new axis_vals created will be floats - return [0.0] - - def _mapped_axes(self): - # NOTE: Each of the mapper method needs to call it's sub mapper method - final_axes = self._final_mapped_axes - return final_axes - - def _base_axis(self): - pass - - def _resolution(self): - pass - - def first_axis_vals(self): - return self._final_transformation.first_axis_vals() - - def second_axis_vals(self, first_val): - return self._final_transformation.second_axis_vals(first_val) - - def map_first_axis(self, lower, upper): - return self._final_transformation.map_first_axis(lower, upper) - - def map_second_axis(self, first_val, lower, upper): - return self._final_transformation.map_second_axis(first_val, lower, upper) - - def find_second_idx(self, first_val, second_val): - return self._final_transformation.find_second_idx(first_val, second_val) - - def unmap_first_val_to_start_line_idx(self, first_val): - return self._final_transformation.unmap_first_val_to_start_line_idx(first_val) - - def unmap(self, first_val, second_val): - return self._final_transformation.unmap(first_val, second_val) - - -_type_to_datacube_mapper_lookup = { - "octahedral": "OctahedralGridMapper", - "healpix": "HealpixGridMapper", - "regular": "RegularGridMapper", - "reduced_ll": "ReducedLatLonMapper", -} +from ....utility.list_tools import bisect_left_cmp, bisect_right_cmp +from .datacube_mappers import DatacubeMapper def mapper(cls): diff --git a/polytope/datacube/transformations/mappers/healpix.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/healpix.py similarity index 100% rename from polytope/datacube/transformations/mappers/healpix.py rename to polytope/datacube/transformations/datacube_mappers/mapper_types/healpix.py diff --git a/polytope/datacube/transformations/mappers/octahedral.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/octahedral.py similarity index 99% rename from polytope/datacube/transformations/mappers/octahedral.py rename to polytope/datacube/transformations/datacube_mappers/mapper_types/octahedral.py index 1c24703df..b2a3159bb 100644 --- a/polytope/datacube/transformations/mappers/octahedral.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_types/octahedral.py @@ -1,6 +1,6 @@ import math -from ....utility.list_tools import bisect_left_cmp, bisect_right_cmp +from .....utility.list_tools import bisect_left_cmp, bisect_right_cmp from ..datacube_mappers import DatacubeMapper diff --git a/polytope/datacube/transformations/mappers/reduced_ll.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/reduced_ll.py similarity index 100% rename from polytope/datacube/transformations/mappers/reduced_ll.py rename to polytope/datacube/transformations/datacube_mappers/mapper_types/reduced_ll.py diff --git a/polytope/datacube/transformations/mappers/regular.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/regular.py similarity index 100% rename from polytope/datacube/transformations/mappers/regular.py rename to polytope/datacube/transformations/datacube_mappers/mapper_types/regular.py diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 900ad16b6..4bd07a2dc 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -54,8 +54,8 @@ def change_val_type(self, axis_name, values): } _type_to_transformation_file_lookup = { - "mapper": "mappers", - "cyclic": "cyclic", + "mapper": "mappers.datacube_mappers", + "cyclic": "cyclic.datacube_cyclic", "merge": "merger", "reverse": "reverse", "type_change": "type_change", diff --git a/tests/test_mappers.py b/tests/test_mappers.py index b3661822b..1231f19d7 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -1,4 +1,6 @@ -from polytope.datacube.transformations.mappers.octahedral import OctahedralGridMapper +from polytope.datacube.transformations.datacube_mappers.mapper_types.octahedral import ( + OctahedralGridMapper, +) class TestMapper: From db04526d8b5e484100d1efd2242854f03dd0bdd1 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 22 Jan 2024 21:16:06 +0100 Subject: [PATCH 288/332] fix cyclic nearest method bug --- polytope/datacube/backends/fdb.py | 8 +++--- tests/test_point_nearest.py | 41 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 4d7b7aa57..661e52330 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -93,7 +93,10 @@ def get_2nd_last_values(self, requests, leaf_path={}): lat_child.remove_branch() else: possible_lons = [latlon[1] for latlon in nearest_latlons if latlon[0] == lat_child.value] - for lon_child in lat_child.children: + lon_children_values = [child.value for child in lat_child.children] + for j in range(len(lon_children_values)): + lon_child_val = lon_children_values[j] + lon_child = [child for child in lat_child.children if child.value == lon_child_val][0] if lon_child.value not in possible_lons: lon_child.remove_branch() @@ -182,10 +185,7 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) - print("REQUEST TO FDB") - print(fdb_requests) output_values = self.fdb.extract(fdb_requests) - print(output_values) return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index c1ae7dbd8..081050dad 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -16,6 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, "number": {"type_change": "int"}, + "longitude": {"cyclic": [0, 360]}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -76,3 +77,43 @@ def test_fdb_datacube_true_point_2(self): result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 3 + + @pytest.mark.fdb + def test_fdb_datacube_true_point_3(self): + request = Request( + Select("step", [21]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231102T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[0.035149384216, -0.01]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + assert result.leaves[0].value == 0 + assert result.leaves[0].axis.name == "longitude" + + @pytest.mark.fdb + def test_fdb_datacube_true_point_4(self): + request = Request( + Select("step", [21]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231102T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[0.035149384216, 359.97]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + assert result.leaves[0].value == 359.929906542056 + assert result.leaves[0].axis.name == "longitude" From b0fec5fc994f9407dc9f9f590c23e2323db2ced6 Mon Sep 17 00:00:00 2001 From: majh Date: Mon, 22 Jan 2024 20:31:19 +0000 Subject: [PATCH 289/332] add init.py and requirements --- polytope/datacube/transformations/__init__.py | 0 setup.py | 3 +++ 2 files changed, 3 insertions(+) create mode 100644 polytope/datacube/transformations/__init__.py diff --git a/polytope/datacube/transformations/__init__.py b/polytope/datacube/transformations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/setup.py b/setup.py index 6a5ca3c66..0bcb9ac6c 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,8 @@ io.open("polytope/version.py", encoding="utf_8_sig").read(), ).group(1) +with open('requirements.txt') as f: + requirements = f.read().splitlines() setup( name="polytope-python", @@ -23,4 +25,5 @@ packages=find_packages(), zip_safe=False, include_package_data=True, + install_requires=requirements ) From 9f50e5473ec092ffc8eb1b03faaafce3e45273b7 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 17 Jan 2024 17:31:42 +0100 Subject: [PATCH 290/332] start nearest method by finding indices between like for surrounding method --- polytope/datacube/datacube_axis.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 5c78c22f6..1119be943 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -147,7 +147,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): update_range() indexes_between_ranges = [] - if method != "surrounding": + if method != "surrounding" and method != "nearest": return old_find_indices_between(index_ranges, low, up, datacube, method) else: for indexes in index_ranges: @@ -264,7 +264,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): transformation = transform if cls.name in transformation._mapped_axes(): for idxs in index_ranges: - if method == "surrounding": + if method == "surrounding" or method == "nearest": axis_reversed = transform._axis_reversed[cls.name] if not axis_reversed: start = bisect.bisect_left(idxs, low) @@ -349,7 +349,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): transformation = transform if cls.name in transformation._mapped_axes(): for indexes in index_ranges: - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.index(low) end = indexes.index(up) start = max(start - 1, 0) @@ -403,7 +403,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically # increasing - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") start = max(start - 1, 0) @@ -416,7 +416,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between = indexes[start:end].to_list() indexes_between_ranges.append(indexes_between) else: - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.index(low) end = indexes.index(up) start = max(start - 1, 0) @@ -486,7 +486,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): transformation = transform if cls.name == transformation.name: for indexes in index_ranges: - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.index(low) end = indexes.index(up) start = max(start - 1, 0) @@ -598,7 +598,7 @@ def find_indices_between(self, index_ranges, low, up, datacube, method=None): # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") start = max(start - 1, 0) @@ -611,7 +611,7 @@ def find_indices_between(self, index_ranges, low, up, datacube, method=None): indexes_between = indexes[start:end].to_list() indexes_between_ranges.append(indexes_between) else: - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.index(low) end = indexes.index(up) start = max(start - 1, 0) From 2d303c5b299f757c745c3f910b321e4d093fbef2 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 18 Jan 2024 11:19:26 +0100 Subject: [PATCH 291/332] make nearest method return only nearest point for end lat/lon grid --- polytope/datacube/backends/fdb.py | 34 ++++++++++++++++ polytope/datacube/backends/xarray.py | 2 + polytope/engine/hullslicer.py | 2 + polytope/shapes.py | 12 ++++-- polytope/utility/geometry.py | 18 +++++++++ tests/test_fdb_datacube.py | 3 +- tests/test_point_nearest.py | 60 ++++++++++++++++++++++++++++ tests/test_point_shape.py | 7 ++++ 8 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 tests/test_point_nearest.py diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index cbc76893f..9e9bc1a4a 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -2,6 +2,7 @@ import pygribjump as pygj +from ...utility.geometry import nearest_pt from .datacube import Datacube, IndexTree @@ -15,6 +16,7 @@ def __init__(self, config={}, axis_options={}): self.blocked_axes = [] self.fake_axes = [] self.unwanted_path = {} + self.nearest_search = {} partial_request = config # Find values in the level 3 FDB datacube @@ -61,6 +63,38 @@ def get(self, requests: IndexTree, leaf_path={}): def get_2nd_last_values(self, requests, leaf_path={}): # In this function, we recursively loop over the last two layers of the tree and store the indices of the # request ranges in those layers + first_ax_name = requests.children[0].axis.name + second_ax_name = requests.children[0].children[0].axis.name + nearest_pts = [ + [lat_val, lon_val] + for (lat_val, lon_val) in zip(self.nearest_search[first_ax_name][0], self.nearest_search[second_ax_name][0]) + ] + # TODO: here find nearest point first before retrieving etc + if len(self.nearest_search) != 0: + # first collect the lat lon points found + found_latlon_pts = [] + for lat_child in requests.children: + for lon_child in lat_child.children: + found_latlon_pts.append([lat_child.value, lon_child.value]) + # now find the nearest lat lon to the points requested + nearest_latlons = [] + for pt in nearest_pts: + nearest_latlon = nearest_pt(found_latlon_pts, pt) + nearest_latlons.append(nearest_latlon) + # TODO: now combine with the rest of the function.... + # TODO: need to remove the branches that do not fit + copy_requests = deepcopy(requests) + for i in range(len(copy_requests.children)): + lat_child = copy_requests.children[i] + lat_child = [child for child in requests.children if child.value == lat_child.value][0] + if lat_child.value not in [latlon[0] for latlon in nearest_latlons]: + lat_child.remove_branch() + else: + possible_lons = [latlon[1] for latlon in nearest_latlons if latlon[0] == lat_child.value] + for lon_child in lat_child.children: + if lon_child.value not in possible_lons: + lon_child.remove_branch() + lat_length = len(requests.children) range_lengths = [False] * lat_length current_start_idxs = [False] * lat_length diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index f8ca1c2e2..34cd8066f 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -17,6 +17,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.complete_axes = [] self.blocked_axes = [] self.fake_axes = [] + self.nearest_search = None for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: @@ -52,6 +53,7 @@ def get(self, requests: IndexTree): for key in path_copy: axis = self._axes[key] (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) + # TODO: here do nearest point search path = self.fit_path(path) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmapped_path) diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index e6c8e3eb0..f95f71e2e 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -50,6 +50,8 @@ def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, nex upper = ax.from_float(upper + tol) flattened = node.flatten() method = polytope.method + if method == "nearest": + datacube.nearest_search[ax.name] = polytope.points # TODO: this hashing doesn't work because we need to know the latitude val for finding longitude values # TODO: Maybe create a coupled_axes list inside of datacube and add to it during axis formation, then here diff --git a/polytope/shapes.py b/polytope/shapes.py index 6af698ee2..c0378af06 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -80,9 +80,15 @@ def __init__(self, axes, values, method=None): self.values = values self.method = method self.polytopes = [] - for i in range(len(axes)): - polytope_points = [v[i] for v in self.values] - self.polytopes.append(ConvexPolytope([axes[i]], [polytope_points], self.method)) + # if method != "nearest": + if True: + # if the method is surrounding, need to treat as 1D polytopes + for i in range(len(axes)): + polytope_points = [v[i] for v in self.values] + # TODO: IS THIS RIGHT? FOR MULTIPLE POINTS, DOES IT CREATE A LINE SEGMENT INSTEAD? + self.polytopes.extend([ConvexPolytope([axes[i]], [[point]], self.method) for point in polytope_points]) + # if method == "nearest": + # self.polytopes.extend([ConvexPolytope(axes, [v], self.method) for v in self.values]) def axes(self): return self._axes diff --git a/polytope/utility/geometry.py b/polytope/utility/geometry.py index bbbb75152..2c88d9655 100644 --- a/polytope/utility/geometry.py +++ b/polytope/utility/geometry.py @@ -1,4 +1,22 @@ +import math + + def lerp(a, b, value): direction = [a - b for a, b in zip(a, b)] intersect = [b + value * d for b, d in zip(b, direction)] return intersect + + +def nearest_pt(pts_list, pt): + nearest_pt = pts_list[0] + distance = l2_norm(pts_list[0], pt) + for new_pt in pts_list[1:]: + new_distance = l2_norm(new_pt, pt) + if new_distance < distance: + distance = new_distance + nearest_pt = new_pt + return nearest_pt + + +def l2_norm(pt1, pt2): + return math.sqrt((pt1[0] - pt2[0]) * (pt1[0] - pt2[0]) + (pt1[1] - pt2[1]) * (pt1[1] - pt2[1])) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index a7c54816a..bf4452a35 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -20,7 +20,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc"} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) @@ -38,7 +38,6 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), - Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py new file mode 100644 index 000000000..9166fd794 --- /dev/null +++ b/tests/test_point_nearest.py @@ -0,0 +1,60 @@ +import pandas as pd +import pytest + +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Point, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.fdb + def test_fdb_datacube(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20230625T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Point(["latitude", "longitude"], [[0.16, 0.176]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + + @pytest.mark.fdb + def test_fdb_datacube_true_point(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20230625T120000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Point(["latitude", "longitude"], [[0.175746921078, 0.210608424337]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 diff --git a/tests/test_point_shape.py b/tests/test_point_shape.py index 0bc203d61..7dbbd9b52 100644 --- a/tests/test_point_shape.py +++ b/tests/test_point_shape.py @@ -30,6 +30,13 @@ def test_point(self): assert len(result.leaves) == 1 assert result.leaves[0].axis.name == "level" + def test_multiple_points(self): + request = Request(Point(["step", "level"], [[3, 10], [3, 12]]), Select("date", ["2000-01-01"])) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 2 + assert result.leaves[0].axis.name == "level" + def test_point_surrounding_step(self): request = Request(Point(["step", "level"], [[2, 10]], method="surrounding"), Select("date", ["2000-01-01"])) result = self.API.retrieve(request) From 544cf68cd4678874d53281f9364c3a102a6d486a Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 18 Jan 2024 13:45:14 +0100 Subject: [PATCH 292/332] fix small bug --- polytope/datacube/backends/fdb.py | 14 ++++++++------ tests/test_ecmwf_oper_data_fdb.py | 2 +- tests/test_slice_date_range_fdb.py | 1 - tests/test_slice_date_range_fdb_v2.py | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 9e9bc1a4a..f682b671f 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -63,14 +63,16 @@ def get(self, requests: IndexTree, leaf_path={}): def get_2nd_last_values(self, requests, leaf_path={}): # In this function, we recursively loop over the last two layers of the tree and store the indices of the # request ranges in those layers - first_ax_name = requests.children[0].axis.name - second_ax_name = requests.children[0].children[0].axis.name - nearest_pts = [ - [lat_val, lon_val] - for (lat_val, lon_val) in zip(self.nearest_search[first_ax_name][0], self.nearest_search[second_ax_name][0]) - ] # TODO: here find nearest point first before retrieving etc if len(self.nearest_search) != 0: + first_ax_name = requests.children[0].axis.name + second_ax_name = requests.children[0].children[0].axis.name + nearest_pts = [ + [lat_val, lon_val] + for (lat_val, lon_val) in zip( + self.nearest_search[first_ax_name][0], self.nearest_search[second_ax_name][0] + ) + ] # first collect the lat lon points found found_latlon_pts = [] for lat_child in requests.children: diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 7ec4a4659..553407eab 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -16,7 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, } - self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": 0, "type": "fc"} + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 624a77fe2..ec14ba48c 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -36,7 +36,6 @@ def test_fdb_datacube(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["an"]), - Select("number", [1]), Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), ) result = self.API.retrieve(request) diff --git a/tests/test_slice_date_range_fdb_v2.py b/tests/test_slice_date_range_fdb_v2.py index 24ae1a9a5..63ce5e678 100644 --- a/tests/test_slice_date_range_fdb_v2.py +++ b/tests/test_slice_date_range_fdb_v2.py @@ -16,7 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, } - self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": 0} + self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "stream": "enda"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) self.slicer = HullSlicer() self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) From 192bdcdb50a6ad309890ac2334d3bf9fdd13546e Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 18 Jan 2024 13:49:07 +0100 Subject: [PATCH 293/332] make sure nearest is only called on one point --- polytope/shapes.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/polytope/shapes.py b/polytope/shapes.py index c0378af06..0c170b3fc 100644 --- a/polytope/shapes.py +++ b/polytope/shapes.py @@ -80,15 +80,11 @@ def __init__(self, axes, values, method=None): self.values = values self.method = method self.polytopes = [] - # if method != "nearest": - if True: - # if the method is surrounding, need to treat as 1D polytopes - for i in range(len(axes)): - polytope_points = [v[i] for v in self.values] - # TODO: IS THIS RIGHT? FOR MULTIPLE POINTS, DOES IT CREATE A LINE SEGMENT INSTEAD? - self.polytopes.extend([ConvexPolytope([axes[i]], [[point]], self.method) for point in polytope_points]) - # if method == "nearest": - # self.polytopes.extend([ConvexPolytope(axes, [v], self.method) for v in self.values]) + if method == "nearest": + assert len(self.values) == 1 + for i in range(len(axes)): + polytope_points = [v[i] for v in self.values] + self.polytopes.extend([ConvexPolytope([axes[i]], [[point]], self.method) for point in polytope_points]) def axes(self): return self._axes From 0551725e97ec9746da3ce934e519834746b74871 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 18 Jan 2024 14:55:34 +0100 Subject: [PATCH 294/332] fix bug --- polytope/datacube/backends/fdb.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index f682b671f..4d7b7aa57 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -85,10 +85,10 @@ def get_2nd_last_values(self, requests, leaf_path={}): nearest_latlons.append(nearest_latlon) # TODO: now combine with the rest of the function.... # TODO: need to remove the branches that do not fit - copy_requests = deepcopy(requests) - for i in range(len(copy_requests.children)): - lat_child = copy_requests.children[i] - lat_child = [child for child in requests.children if child.value == lat_child.value][0] + lat_children_values = [child.value for child in requests.children] + for i in range(len(lat_children_values)): + lat_child_val = lat_children_values[i] + lat_child = [child for child in requests.children if child.value == lat_child_val][0] if lat_child.value not in [latlon[0] for latlon in nearest_latlons]: lat_child.remove_branch() else: @@ -182,7 +182,10 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) + print("REQUEST TO FDB") + print(fdb_requests) output_values = self.fdb.extract(fdb_requests) + print(output_values) return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): From e5dfb8e331225025e202c9156b0e7e2836661c21 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 19 Jan 2024 10:07:13 +0100 Subject: [PATCH 295/332] make tests work --- examples/healpix_grid_box_example.py | 6 +----- tests/test_point_nearest.py | 22 ++++++++++++++++++++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/examples/healpix_grid_box_example.py b/examples/healpix_grid_box_example.py index 8e676fa3b..6cfbb5e78 100644 --- a/examples/healpix_grid_box_example.py +++ b/examples/healpix_grid_box_example.py @@ -14,11 +14,7 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/healpix.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(time=0).isel(isobaricInhPa=0).z self.xarraydatacube = XArrayDatacube(self.latlon_array) - self.options = { - "values": { - "transformation": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}} - } - } + self.options = {"values": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}}} self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index 9166fd794..c1ae7dbd8 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -38,7 +38,7 @@ def test_fdb_datacube(self): Point(["latitude", "longitude"], [[0.16, 0.176]], method="nearest"), ) result = self.API.retrieve(request) - result.pprint() + # result.pprint() assert len(result.leaves) == 1 @pytest.mark.fdb @@ -56,5 +56,23 @@ def test_fdb_datacube_true_point(self): Point(["latitude", "longitude"], [[0.175746921078, 0.210608424337]], method="nearest"), ) result = self.API.retrieve(request) - result.pprint() + # result.pprint() assert len(result.leaves) == 1 + + @pytest.mark.fdb + def test_fdb_datacube_true_point_2(self): + request = Request( + Select("step", [21, 22, 23]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231102T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[0.035149384216, 0.0]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 3 From de9cae8f8a93129b1fda40541cbb6e5142daad03 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 19 Jan 2024 10:34:30 +0100 Subject: [PATCH 296/332] fix merging of multiple params --- polytope/datacube/index_tree.py | 14 +++++++---- tests/test_multiple_param_fdb.py | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 tests/test_multiple_param_fdb.py diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index 9054cd0c5..7af7f5422 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -84,12 +84,16 @@ def __eq__(self, other): else: if other.value == self.value: return True - if other.value - 2 * other.axis.tol <= self.value <= other.value + 2 * other.axis.tol: - return True - elif self.value - 2 * self.axis.tol <= other.value <= self.value + 2 * self.axis.tol: - return True else: - return False + if isinstance(self.value, str): + return False + else: + if other.value - 2 * other.axis.tol <= self.value <= other.value + 2 * other.axis.tol: + return True + elif self.value - 2 * self.axis.tol <= other.value <= self.value + 2 * self.axis.tol: + return True + else: + return False # return (self.axis.name, self.value) == (other.axis.name, other.value) def __lt__(self, other): diff --git a/tests/test_multiple_param_fdb.py b/tests/test_multiple_param_fdb.py new file mode 100644 index 000000000..04b8e7127 --- /dev/null +++ b/tests/test_multiple_param_fdb.py @@ -0,0 +1,41 @@ +import pandas as pd +import pytest + +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.fdb + def test_fdb_datacube(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240118T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["49", "167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 18 From d4746f36cb70fda7ae6c58d8d24112c48b9056c0 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 11 Jan 2024 14:25:22 +0100 Subject: [PATCH 297/332] replace |= operation with update --- polytope/datacube/backends/fdb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 4d7b7aa57..cf3c4beda 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -50,7 +50,7 @@ def get(self, requests: IndexTree, leaf_path={}): (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path ) - leaf_path |= key_value_path + leaf_path.update(key_value_path) if len(requests.children[0].children[0].children) == 0: # remap this last key self.get_2nd_last_values(requests, leaf_path) @@ -115,7 +115,7 @@ def get_2nd_last_values(self, requests, leaf_path={}): (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path ) - leaf_path |= key_value_path + leaf_path.update(key_value_path) (range_lengths[i], current_start_idxs[i], fdb_node_ranges[i]) = self.get_last_layer_before_leaf( lat_child, leaf_path, range_length, current_start_idx, fdb_range_nodes ) @@ -130,7 +130,7 @@ def get_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path ) - leaf_path |= key_value_path + leaf_path.update(key_value_path) last_idx = key_value_path["values"] if current_idx[i] is None: current_idx[i] = last_idx @@ -145,7 +145,7 @@ def get_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path ) - leaf_path |= key_value_path + leaf_path.update(key_value_path) i += 1 current_start_idx = key_value_path["values"] current_idx[i] = current_start_idx From 740e91766aae31fe42b5115c7a17d9bae7aff1ba Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 23 Jan 2024 09:54:52 +0100 Subject: [PATCH 298/332] black setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0bcb9ac6c..005e76dfd 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ io.open("polytope/version.py", encoding="utf_8_sig").read(), ).group(1) -with open('requirements.txt') as f: +with open("requirements.txt") as f: requirements = f.read().splitlines() setup( @@ -25,5 +25,5 @@ packages=find_packages(), zip_safe=False, include_package_data=True, - install_requires=requirements + install_requires=requirements, ) From aba829540c3a0cf15d3835f24d9aa74c715c0867 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 23 Jan 2024 10:05:04 +0100 Subject: [PATCH 299/332] fix CI --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 005e76dfd..e152f4899 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ io.open("polytope/version.py", encoding="utf_8_sig").read(), ).group(1) -with open("requirements.txt") as f: +with open(".requirements.txt") as f: requirements = f.read().splitlines() setup( From 4f5667dfc1b863aab86654bcc16c1e27f44ffa15 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 23 Jan 2024 10:07:21 +0100 Subject: [PATCH 300/332] fix CI --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e152f4899..18241b407 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ io.open("polytope/version.py", encoding="utf_8_sig").read(), ).group(1) -with open(".requirements.txt") as f: +with open("./requirements.txt") as f: requirements = f.read().splitlines() setup( From 2ae23da43384b8704723942e0275a4ab95e4b15b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 23 Jan 2024 10:14:03 +0100 Subject: [PATCH 301/332] fix CI --- MANIFEST.in | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..540b72040 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include requirements.txt \ No newline at end of file diff --git a/setup.py b/setup.py index 18241b407..005e76dfd 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ io.open("polytope/version.py", encoding="utf_8_sig").read(), ).group(1) -with open("./requirements.txt") as f: +with open("requirements.txt") as f: requirements = f.read().splitlines() setup( From 78531228f7dc96323f558edee223a3174d0418bf Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 23 Jan 2024 10:17:27 +0100 Subject: [PATCH 302/332] fix CI --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..540b72040 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include requirements.txt \ No newline at end of file From 196842cadc0ced67159a8f9de1bc7923e5e0ae6c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Tue, 23 Jan 2024 10:34:50 +0100 Subject: [PATCH 303/332] black setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0bcb9ac6c..005e76dfd 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ io.open("polytope/version.py", encoding="utf_8_sig").read(), ).group(1) -with open('requirements.txt') as f: +with open("requirements.txt") as f: requirements = f.read().splitlines() setup( @@ -25,5 +25,5 @@ packages=find_packages(), zip_safe=False, include_package_data=True, - install_requires=requirements + install_requires=requirements, ) From 56b10ad5453728da7be770429fa72043f3d042f9 Mon Sep 17 00:00:00 2001 From: majh Date: Wed, 24 Jan 2024 00:16:05 +0000 Subject: [PATCH 304/332] logging --- polytope/datacube/index_tree.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index 303bd4d84..b15e67fe2 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -1,5 +1,6 @@ import json from typing import OrderedDict +import logging from sortedcontainers import SortedList @@ -168,10 +169,12 @@ def intersect(self, other): def pprint(self, level=0): if self.axis.name == "root": - print("\n") - print("\t" * level + "\u21b3" + str(self)) + logging.debug("\n") + logging.debug("\t" * level + "\u21b3" + str(self)) for child in self.children: child.pprint(level + 1) + if len(self.children) == 0: + logging.debug("\t" * (level + 1) + "\u21b3" + str(self.result)) def remove_branch(self): if not self.is_root(): From 49838d05d04e8462404b189df9cefc13287c0fd8 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 10:21:45 +0100 Subject: [PATCH 305/332] better batching to fdb backend --- polytope/datacube/backends/fdb.py | 156 ++++++++++++++++++++++++++++-- 1 file changed, 149 insertions(+), 7 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index d0f4264ff..8e9300128 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -39,7 +39,14 @@ def __init__(self, config={}, axis_options={}): val = self._axes[name].type self._check_and_add_axes(options, name, val) - def get(self, requests: IndexTree, leaf_path={}): + def get(self, requests: IndexTree): + fdb_requests = [] + fdb_requests_decoding_info = [] + self.get_fdb_requests(requests, fdb_requests, fdb_requests_decoding_info) + output_values = self.fdb.extract(fdb_requests) + self.assign_fdb_output_to_nodes(output_values, fdb_requests_decoding_info) + + def old_get(self, requests: IndexTree, leaf_path={}): # First when request node is root, go to its children if requests.axis.name == "root": for c in requests.children: @@ -61,6 +68,39 @@ def get(self, requests: IndexTree, leaf_path={}): for c in requests.children: self.get(c, leaf_path) + def get_fdb_requests(self, requests: IndexTree, fdb_requests=[], fdb_requests_decoding_info=[], leaf_path={}): + # First when request node is root, go to its children + if requests.axis.name == "root": + for c in requests.children: + self.get_fdb_requests(c, fdb_requests, fdb_requests_decoding_info) + # If request node has no children, we have a leaf so need to assign fdb values to it + else: + key_value_path = {requests.axis.name: requests.value} + ax = requests.axis + (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( + key_value_path, leaf_path, self.unwanted_path + ) + leaf_path.update(key_value_path) + if len(requests.children[0].children[0].children) == 0: + # remap this last key + # TODO: here, find the fdb_requests and associated nodes to which to add results + + (path, range_lengths, current_start_idxs, fdb_node_ranges, lat_length) = self.get_2nd_last_values( + requests, leaf_path + ) + (original_indices, sorted_request_ranges) = self.sort_fdb_request_ranges( + range_lengths, current_start_idxs, lat_length + ) + fdb_requests.append(tuple((path, sorted_request_ranges))) + fdb_requests_decoding_info.append( + tuple((original_indices, fdb_node_ranges, lat_length, range_lengths, current_start_idxs)) + ) + + # Otherwise remap the path for this key and iterate again over children + else: + for c in requests.children: + self.get_fdb_requests(c, fdb_requests, fdb_requests_decoding_info, leaf_path) + def get_2nd_last_values(self, requests, leaf_path={}): # In this function, we recursively loop over the last two layers of the tree and store the indices of the # request ranges in those layers @@ -68,6 +108,75 @@ def get_2nd_last_values(self, requests, leaf_path={}): if len(self.nearest_search) != 0: first_ax_name = requests.children[0].axis.name second_ax_name = requests.children[0].children[0].axis.name + # TODO: throw error if first_ax_name or second_ax_name not in self.nearest_search.keys() + nearest_pts = [ + [lat_val, lon_val] + for (lat_val, lon_val) in zip( + self.nearest_search[first_ax_name][0], self.nearest_search[second_ax_name][0] + ) + ] + # first collect the lat lon points found + found_latlon_pts = [] + for lat_child in requests.children: + for lon_child in lat_child.children: + found_latlon_pts.append([lat_child.value, lon_child.value]) + # now find the nearest lat lon to the points requested + nearest_latlons = [] + for pt in nearest_pts: + nearest_latlon = nearest_pt(found_latlon_pts, pt) + nearest_latlons.append(nearest_latlon) + # TODO: now combine with the rest of the function.... + # TODO: need to remove the branches that do not fit + lat_children_values = [child.value for child in requests.children] + for i in range(len(lat_children_values)): + lat_child_val = lat_children_values[i] + lat_child = [child for child in requests.children if child.value == lat_child_val][0] + if lat_child.value not in [latlon[0] for latlon in nearest_latlons]: + lat_child.remove_branch() + else: + possible_lons = [latlon[1] for latlon in nearest_latlons if latlon[0] == lat_child.value] + lon_children_values = [child.value for child in lat_child.children] + for j in range(len(lon_children_values)): + lon_child_val = lon_children_values[j] + lon_child = [child for child in lat_child.children if child.value == lon_child_val][0] + if lon_child.value not in possible_lons: + lon_child.remove_branch() + + lat_length = len(requests.children) + range_lengths = [False] * lat_length + current_start_idxs = [False] * lat_length + fdb_node_ranges = [False] * lat_length + for i in range(len(requests.children)): + lat_child = requests.children[i] + lon_length = len(lat_child.children) + range_lengths[i] = [1] * lon_length + current_start_idxs[i] = [None] * lon_length + fdb_node_ranges[i] = [[IndexTree.root] * lon_length] * lon_length + range_length = deepcopy(range_lengths[i]) + current_start_idx = deepcopy(current_start_idxs[i]) + fdb_range_nodes = deepcopy(fdb_node_ranges[i]) + key_value_path = {lat_child.axis.name: lat_child.value} + ax = lat_child.axis + (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( + key_value_path, leaf_path, self.unwanted_path + ) + leaf_path.update(key_value_path) + (range_lengths[i], current_start_idxs[i], fdb_node_ranges[i]) = self.get_last_layer_before_leaf( + lat_child, leaf_path, range_length, current_start_idx, fdb_range_nodes + ) + # TODO: do we need to return all of this? + leaf_path_copy = deepcopy(leaf_path) + leaf_path_copy.pop("values") + return (leaf_path_copy, range_lengths, current_start_idxs, fdb_node_ranges, lat_length) + + def old_get_2nd_last_values(self, requests, leaf_path={}): + # In this function, we recursively loop over the last two layers of the tree and store the indices of the + # request ranges in those layers + # TODO: here find nearest point first before retrieving etc + if len(self.nearest_search) != 0: + first_ax_name = requests.children[0].axis.name + second_ax_name = requests.children[0].children[0].axis.name + # TODO: throw error if first_ax_name or second_ax_name not in self.nearest_search.keys() nearest_pts = [ [lat_val, lon_val] for (lat_val, lon_val) in zip( @@ -155,6 +264,30 @@ def get_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, current_idx[i] = current_start_idx return (range_l, current_idx, fdb_range_n) + def assign_fdb_output_to_nodes(self, output_values, fdb_requests_decoding_info): + for k in range(len(output_values)): + request_output_values = output_values[k] + ( + original_indices, + fdb_node_ranges, + lat_length, + range_lengths, + current_start_idxs, + ) = fdb_requests_decoding_info[k] + new_fdb_range_nodes = [] + new_range_lengths = [] + for j in range(lat_length): + for i in range(len(range_lengths[j])): + if current_start_idxs[j][i] is not None: + new_fdb_range_nodes.append(fdb_node_ranges[j][i]) + new_range_lengths.append(range_lengths[j][i]) + sorted_fdb_range_nodes = [new_fdb_range_nodes[i] for i in original_indices] + sorted_range_lengths = [new_range_lengths[i] for i in original_indices] + for i in range(len(sorted_fdb_range_nodes)): + for j in range(sorted_range_lengths[i]): + n = sorted_fdb_range_nodes[i][j] + n.result = request_output_values[0][i][0][j] + def give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_range_nodes, lat_length): (output_values, original_indices) = self.find_fdb_values( leaf_path, range_lengths, current_start_idx, lat_length @@ -169,9 +302,21 @@ def give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_ sorted_fdb_range_nodes = [new_fdb_range_nodes[i] for i in original_indices] sorted_range_lengths = [new_range_lengths[i] for i in original_indices] for i in range(len(sorted_fdb_range_nodes)): - for k in range(sorted_range_lengths[i]): - n = sorted_fdb_range_nodes[i][k] - n.result = output_values[0][0][i][0][k] + for j in range(sorted_range_lengths[i]): + n = sorted_fdb_range_nodes[i][j] + n.result = output_values[0][0][i][0][j] + + def sort_fdb_request_ranges(self, range_lengths, current_start_idx, lat_length): + interm_request_ranges = [] + for i in range(lat_length): + for j in range(len(range_lengths[i])): + if current_start_idx[i][j] is not None: + current_request_ranges = (current_start_idx[i][j], current_start_idx[i][j] + range_lengths[i][j]) + interm_request_ranges.append(current_request_ranges) + request_ranges_with_idx = list(enumerate(interm_request_ranges)) + sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) + original_indices, sorted_request_ranges = zip(*sorted_list) + return (original_indices, sorted_request_ranges) def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): path.pop("values") @@ -186,10 +331,7 @@ def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) original_indices, sorted_request_ranges = zip(*sorted_list) fdb_requests.append(tuple((path, sorted_request_ranges))) - print("REQUEST TO FDB") - print(fdb_requests) output_values = self.fdb.extract(fdb_requests) - print(output_values) return (output_values, original_indices) def datacube_natural_indexes(self, axis, subarray): From b95c500dfcb4a40dff5ca4f51cbf9c6cca2cb578 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 10:23:53 +0100 Subject: [PATCH 306/332] clean up unnecessary functions --- polytope/datacube/backends/fdb.py | 121 ------------------------------ 1 file changed, 121 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 8e9300128..7b2d19295 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -46,28 +46,6 @@ def get(self, requests: IndexTree): output_values = self.fdb.extract(fdb_requests) self.assign_fdb_output_to_nodes(output_values, fdb_requests_decoding_info) - def old_get(self, requests: IndexTree, leaf_path={}): - # First when request node is root, go to its children - if requests.axis.name == "root": - for c in requests.children: - self.get(c) - # If request node has no children, we have a leaf so need to assign fdb values to it - else: - key_value_path = {requests.axis.name: requests.value} - ax = requests.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( - key_value_path, leaf_path, self.unwanted_path - ) - leaf_path.update(key_value_path) - if len(requests.children[0].children[0].children) == 0: - # remap this last key - self.get_2nd_last_values(requests, leaf_path) - - # Otherwise remap the path for this key and iterate again over children - else: - for c in requests.children: - self.get(c, leaf_path) - def get_fdb_requests(self, requests: IndexTree, fdb_requests=[], fdb_requests_decoding_info=[], leaf_path={}): # First when request node is root, go to its children if requests.axis.name == "root": @@ -169,71 +147,6 @@ def get_2nd_last_values(self, requests, leaf_path={}): leaf_path_copy.pop("values") return (leaf_path_copy, range_lengths, current_start_idxs, fdb_node_ranges, lat_length) - def old_get_2nd_last_values(self, requests, leaf_path={}): - # In this function, we recursively loop over the last two layers of the tree and store the indices of the - # request ranges in those layers - # TODO: here find nearest point first before retrieving etc - if len(self.nearest_search) != 0: - first_ax_name = requests.children[0].axis.name - second_ax_name = requests.children[0].children[0].axis.name - # TODO: throw error if first_ax_name or second_ax_name not in self.nearest_search.keys() - nearest_pts = [ - [lat_val, lon_val] - for (lat_val, lon_val) in zip( - self.nearest_search[first_ax_name][0], self.nearest_search[second_ax_name][0] - ) - ] - # first collect the lat lon points found - found_latlon_pts = [] - for lat_child in requests.children: - for lon_child in lat_child.children: - found_latlon_pts.append([lat_child.value, lon_child.value]) - # now find the nearest lat lon to the points requested - nearest_latlons = [] - for pt in nearest_pts: - nearest_latlon = nearest_pt(found_latlon_pts, pt) - nearest_latlons.append(nearest_latlon) - # TODO: now combine with the rest of the function.... - # TODO: need to remove the branches that do not fit - lat_children_values = [child.value for child in requests.children] - for i in range(len(lat_children_values)): - lat_child_val = lat_children_values[i] - lat_child = [child for child in requests.children if child.value == lat_child_val][0] - if lat_child.value not in [latlon[0] for latlon in nearest_latlons]: - lat_child.remove_branch() - else: - possible_lons = [latlon[1] for latlon in nearest_latlons if latlon[0] == lat_child.value] - lon_children_values = [child.value for child in lat_child.children] - for j in range(len(lon_children_values)): - lon_child_val = lon_children_values[j] - lon_child = [child for child in lat_child.children if child.value == lon_child_val][0] - if lon_child.value not in possible_lons: - lon_child.remove_branch() - - lat_length = len(requests.children) - range_lengths = [False] * lat_length - current_start_idxs = [False] * lat_length - fdb_node_ranges = [False] * lat_length - for i in range(len(requests.children)): - lat_child = requests.children[i] - lon_length = len(lat_child.children) - range_lengths[i] = [1] * lon_length - current_start_idxs[i] = [None] * lon_length - fdb_node_ranges[i] = [[IndexTree.root] * lon_length] * lon_length - range_length = deepcopy(range_lengths[i]) - current_start_idx = deepcopy(current_start_idxs[i]) - fdb_range_nodes = deepcopy(fdb_node_ranges[i]) - key_value_path = {lat_child.axis.name: lat_child.value} - ax = lat_child.axis - (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( - key_value_path, leaf_path, self.unwanted_path - ) - leaf_path.update(key_value_path) - (range_lengths[i], current_start_idxs[i], fdb_node_ranges[i]) = self.get_last_layer_before_leaf( - lat_child, leaf_path, range_length, current_start_idx, fdb_range_nodes - ) - self.give_fdb_val_to_node(leaf_path, range_lengths, current_start_idxs, fdb_node_ranges, lat_length) - def get_last_layer_before_leaf(self, requests, leaf_path, range_l, current_idx, fdb_range_n): i = 0 for c in requests.children: @@ -288,24 +201,6 @@ def assign_fdb_output_to_nodes(self, output_values, fdb_requests_decoding_info): n = sorted_fdb_range_nodes[i][j] n.result = request_output_values[0][i][0][j] - def give_fdb_val_to_node(self, leaf_path, range_lengths, current_start_idx, fdb_range_nodes, lat_length): - (output_values, original_indices) = self.find_fdb_values( - leaf_path, range_lengths, current_start_idx, lat_length - ) - new_fdb_range_nodes = [] - new_range_lengths = [] - for j in range(lat_length): - for i in range(len(range_lengths[j])): - if current_start_idx[j][i] is not None: - new_fdb_range_nodes.append(fdb_range_nodes[j][i]) - new_range_lengths.append(range_lengths[j][i]) - sorted_fdb_range_nodes = [new_fdb_range_nodes[i] for i in original_indices] - sorted_range_lengths = [new_range_lengths[i] for i in original_indices] - for i in range(len(sorted_fdb_range_nodes)): - for j in range(sorted_range_lengths[i]): - n = sorted_fdb_range_nodes[i][j] - n.result = output_values[0][0][i][0][j] - def sort_fdb_request_ranges(self, range_lengths, current_start_idx, lat_length): interm_request_ranges = [] for i in range(lat_length): @@ -318,22 +213,6 @@ def sort_fdb_request_ranges(self, range_lengths, current_start_idx, lat_length): original_indices, sorted_request_ranges = zip(*sorted_list) return (original_indices, sorted_request_ranges) - def find_fdb_values(self, path, range_lengths, current_start_idx, lat_length): - path.pop("values") - fdb_requests = [] - interm_request_ranges = [] - for i in range(lat_length): - for j in range(len(range_lengths[i])): - if current_start_idx[i][j] is not None: - current_request_ranges = (current_start_idx[i][j], current_start_idx[i][j] + range_lengths[i][j]) - interm_request_ranges.append(current_request_ranges) - request_ranges_with_idx = list(enumerate(interm_request_ranges)) - sorted_list = sorted(request_ranges_with_idx, key=lambda x: x[1][0]) - original_indices, sorted_request_ranges = zip(*sorted_list) - fdb_requests.append(tuple((path, sorted_request_ranges))) - output_values = self.fdb.extract(fdb_requests) - return (output_values, original_indices) - def datacube_natural_indexes(self, axis, subarray): indexes = subarray[axis.name] return indexes From 38f2ade35c410b08abef454f3f30cebc24a90a49 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 25 Jan 2024 10:23:22 +0100 Subject: [PATCH 307/332] change structure of transformations --- polytope/datacube/datacube_axis.py | 6 +- .../datacube_cyclic/datacube_cyclic.py | 187 ------------------ .../datacube_merger/datacube_merger.py | 63 ++++++ .../merger_axis_decorator.py} | 64 +----- .../datacube_null_transformation.py | 22 +++ .../null_axis_decorator.py} | 24 --- .../datacube_reverse/datacube_reverse.py | 22 +++ .../reverse_axis_decorator.py} | 23 +-- .../datacube_transformations.py | 8 +- .../datacube_type_change.py | 53 +++++ .../type_change_axis_decorator.py} | 53 +---- 11 files changed, 170 insertions(+), 355 deletions(-) create mode 100644 polytope/datacube/transformations/datacube_merger/datacube_merger.py rename polytope/datacube/transformations/{datacube_merger.py => datacube_merger/merger_axis_decorator.py} (61%) create mode 100644 polytope/datacube/transformations/datacube_null_transformation/datacube_null_transformation.py rename polytope/datacube/transformations/{datacube_null_transformation.py => datacube_null_transformation/null_axis_decorator.py} (56%) create mode 100644 polytope/datacube/transformations/datacube_reverse/datacube_reverse.py rename polytope/datacube/transformations/{datacube_reverse.py => datacube_reverse/reverse_axis_decorator.py} (86%) create mode 100644 polytope/datacube/transformations/datacube_type_change/datacube_type_change.py rename polytope/datacube/transformations/{datacube_type_change.py => datacube_type_change/type_change_axis_decorator.py} (66%) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 2e88fb26f..7f9806e86 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -7,9 +7,9 @@ from .transformations.datacube_cyclic.cyclic_axis_decorator import cyclic from .transformations.datacube_mappers.mapper_axis_decorator import mapper -from .transformations.datacube_merger import merge -from .transformations.datacube_reverse import reverse -from .transformations.datacube_type_change import type_change +from .transformations.datacube_merger.merger_axis_decorator import merge +from .transformations.datacube_reverse.reverse_axis_decorator import reverse +from .transformations.datacube_type_change.type_change_axis_decorator import type_change class DatacubeAxis(ABC): diff --git a/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py index fd4da7ae2..86113aa2d 100644 --- a/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py @@ -1,8 +1,3 @@ -import bisect -import math -from copy import deepcopy -from typing import List - from ..datacube_transformations import DatacubeAxisTransformation @@ -28,185 +23,3 @@ def blocked_axes(self): def unwanted_axes(self): return [] - - -def cyclic(cls): - if cls.is_cyclic: - - def update_range(): - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisCyclic): - transformation = transform - cls.range = transformation.range - - def to_intervals(range): - update_range() - if range[0] == -math.inf: - range[0] = cls.range[0] - if range[1] == math.inf: - range[1] = cls.range[1] - axis_lower = cls.range[0] - axis_upper = cls.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - intervals = [] - if lower < axis_upper: - # In this case, we want to go from lower to the first remapped cyclic axis upper - # or the asked upper range value. - # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, - # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper - # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], - # where -180 is the asked upper range value. - loops = int((axis_upper - lower) / axis_range) - remapped_up = axis_upper - (loops) * axis_range - new_upper = min(upper, remapped_up) - else: - # In this case, since lower >= axis_upper, we need to either go to the asked upper range - # or we need to go to the first remapped cyclic axis upper which is higher than lower - new_upper = min(axis_upper + axis_range, upper) - while new_upper < lower: - new_upper = min(new_upper + axis_range, upper) - intervals.append([lower, new_upper]) - # Now that we have established what the first interval should be, we should just jump from cyclic range - # to cyclic range until we hit the asked upper range value. - new_up = deepcopy(new_upper) - while new_up < upper: - new_upper = new_up - new_up = min(upper, new_upper + axis_range) - intervals.append([new_upper, new_up]) - # Once we have added all the in-between ranges, we need to add the last interval - intervals.append([new_up, upper]) - return intervals - - def _remap_range_to_axis_range(range): - update_range() - axis_lower = cls.range[0] - axis_upper = cls.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - if lower < axis_lower: - # In this case we need to calculate the number of loops between the axis lower - # and the lower to recenter the lower - loops = int((axis_lower - lower - cls.tol) / axis_range) - return_lower = lower + (loops + 1) * axis_range - return_upper = upper + (loops + 1) * axis_range - elif lower >= axis_upper: - # In this case we need to calculate the number of loops between the axis upper - # and the lower to recenter the lower - loops = int((lower - axis_upper) / axis_range) - return_lower = lower - (loops + 1) * axis_range - return_upper = upper - (loops + 1) * axis_range - else: - # In this case, the lower value is already in the right range - return_lower = lower - return_upper = upper - return [return_lower, return_upper] - - def _remap_val_to_axis_range(value): - return_range = _remap_range_to_axis_range([value, value]) - return return_range[0] - - def remap(range: List): - update_range() - if cls.range[0] - cls.tol <= range[0] <= cls.range[1] + cls.tol: - if cls.range[0] - cls.tol <= range[1] <= cls.range[1] + cls.tol: - # If we are already in the cyclic range, return it - return [range] - elif abs(range[0] - range[1]) <= 2 * cls.tol: - # If we have a range that is just one point, then it should still be counted - # and so we should take a small interval around it to find values inbetween - range = [ - _remap_val_to_axis_range(range[0]) - cls.tol, - _remap_val_to_axis_range(range[0]) + cls.tol, - ] - return [range] - range_intervals = cls.to_intervals(range) - ranges = [] - for interval in range_intervals: - if abs(interval[0] - interval[1]) > 0: - # If the interval is not just a single point, we remap it to the axis range - range = _remap_range_to_axis_range([interval[0], interval[1]]) - up = range[1] - low = range[0] - if up < low: - # Make sure we remap in the right order - ranges.append([up - cls.tol, low + cls.tol]) - else: - ranges.append([low - cls.tol, up + cls.tol]) - return ranges - - old_find_indexes = cls.find_indexes - - def find_indexes(path, datacube): - return old_find_indexes(path, datacube) - - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path, unwanted_path): - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisCyclic): - if cls.name == transform.name: - new_val = _remap_val_to_axis_range(value) - key_value_path[cls.name] = new_val - key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) - return (key_value_path, leaf_path, unwanted_path) - - old_unmap_to_datacube = cls.unmap_to_datacube - - def unmap_to_datacube(path, unmapped_path): - (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) - return (path, unmapped_path) - - old_find_indices_between = cls.find_indices_between - - def find_indices_between(index_ranges, low, up, datacube, method=None): - update_range() - indexes_between_ranges = [] - - if method != "surrounding": - return old_find_indices_between(index_ranges, low, up, datacube, method) - else: - for indexes in index_ranges: - if cls.name in datacube.complete_axes: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - else: - start = bisect.bisect_left(indexes, low) - end = bisect.bisect_right(indexes, up) - - if start - 1 < 0: - index_val_found = indexes[-1:][0] - indexes_between_ranges.append([index_val_found]) - if end + 1 > len(indexes): - index_val_found = indexes[:2][0] - indexes_between_ranges.append([index_val_found]) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - if cls.name in datacube.complete_axes: - indexes_between = indexes[start:end].to_list() - else: - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def offset(range): - # We first unpad the range by the axis tolerance to make sure that - # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. - # Also, it's safer that we find the offset of a value inside the range instead of on the border - unpadded_range = [range[0] + 1.5 * cls.tol, range[1] - 1.5 * cls.tol] - cyclic_range = _remap_range_to_axis_range(unpadded_range) - offset = unpadded_range[0] - cyclic_range[0] - return offset - - cls.to_intervals = to_intervals - cls.remap = remap - cls.offset = offset - cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube - cls.find_indices_between = find_indices_between - cls.unmap_path_key = unmap_path_key - - return cls diff --git a/polytope/datacube/transformations/datacube_merger/datacube_merger.py b/polytope/datacube/transformations/datacube_merger/datacube_merger.py new file mode 100644 index 000000000..ecd533729 --- /dev/null +++ b/polytope/datacube/transformations/datacube_merger/datacube_merger.py @@ -0,0 +1,63 @@ +import numpy as np +import pandas as pd + +from ..datacube_transformations import DatacubeAxisTransformation + + +class DatacubeAxisMerger(DatacubeAxisTransformation): + def __init__(self, name, merge_options): + self.transformation_options = merge_options + self.name = name + self._first_axis = name + self._second_axis = merge_options["with"] + self._linkers = merge_options["linkers"] + + def blocked_axes(self): + return [self._second_axis] + + def unwanted_axes(self): + return [] + + def _mapped_axes(self): + return self._first_axis + + def merged_values(self, datacube): + first_ax_vals = datacube.ax_vals(self.name) + second_ax_name = self._second_axis + second_ax_vals = datacube.ax_vals(second_ax_name) + linkers = self._linkers + merged_values = [] + for i in range(len(first_ax_vals)): + first_val = first_ax_vals[i] + for j in range(len(second_ax_vals)): + second_val = second_ax_vals[j] + # TODO: check that the first and second val are strings + val_to_add = pd.to_datetime("".join([first_val, linkers[0], second_val, linkers[1]])) + val_to_add = val_to_add.to_numpy() + val_to_add = val_to_add.astype("datetime64[s]") + merged_values.append(val_to_add) + merged_values = np.array(merged_values) + return merged_values + + def transformation_axes_final(self): + return [self._first_axis] + + def generate_final_transformation(self): + return self + + def unmerge(self, merged_val): + merged_val = str(merged_val) + first_idx = merged_val.find(self._linkers[0]) + first_val = merged_val[:first_idx] + first_linker_size = len(self._linkers[0]) + second_linked_size = len(self._linkers[1]) + second_val = merged_val[first_idx + first_linker_size : -second_linked_size] + + # TODO: maybe replacing like this is too specific to time/dates? + first_val = str(first_val).replace("-", "") + second_val = second_val.replace(":", "") + return (first_val, second_val) + + def change_val_type(self, axis_name, values): + new_values = pd.to_datetime(values) + return new_values diff --git a/polytope/datacube/transformations/datacube_merger.py b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py similarity index 61% rename from polytope/datacube/transformations/datacube_merger.py rename to polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py index 883ea964a..eeb0cbe70 100644 --- a/polytope/datacube/transformations/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py @@ -1,68 +1,6 @@ import bisect -import numpy as np -import pandas as pd - -from .datacube_transformations import DatacubeAxisTransformation - - -class DatacubeAxisMerger(DatacubeAxisTransformation): - def __init__(self, name, merge_options): - self.transformation_options = merge_options - self.name = name - self._first_axis = name - self._second_axis = merge_options["with"] - self._linkers = merge_options["linkers"] - - def blocked_axes(self): - return [self._second_axis] - - def unwanted_axes(self): - return [] - - def _mapped_axes(self): - return self._first_axis - - def merged_values(self, datacube): - first_ax_vals = datacube.ax_vals(self.name) - second_ax_name = self._second_axis - second_ax_vals = datacube.ax_vals(second_ax_name) - linkers = self._linkers - merged_values = [] - for i in range(len(first_ax_vals)): - first_val = first_ax_vals[i] - for j in range(len(second_ax_vals)): - second_val = second_ax_vals[j] - # TODO: check that the first and second val are strings - val_to_add = pd.to_datetime("".join([first_val, linkers[0], second_val, linkers[1]])) - val_to_add = val_to_add.to_numpy() - val_to_add = val_to_add.astype("datetime64[s]") - merged_values.append(val_to_add) - merged_values = np.array(merged_values) - return merged_values - - def transformation_axes_final(self): - return [self._first_axis] - - def generate_final_transformation(self): - return self - - def unmerge(self, merged_val): - merged_val = str(merged_val) - first_idx = merged_val.find(self._linkers[0]) - first_val = merged_val[:first_idx] - first_linker_size = len(self._linkers[0]) - second_linked_size = len(self._linkers[1]) - second_val = merged_val[first_idx + first_linker_size : -second_linked_size] - - # TODO: maybe replacing like this is too specific to time/dates? - first_val = str(first_val).replace("-", "") - second_val = second_val.replace(":", "") - return (first_val, second_val) - - def change_val_type(self, axis_name, values): - new_values = pd.to_datetime(values) - return new_values +from .datacube_merger import DatacubeAxisMerger def merge(cls): diff --git a/polytope/datacube/transformations/datacube_null_transformation/datacube_null_transformation.py b/polytope/datacube/transformations/datacube_null_transformation/datacube_null_transformation.py new file mode 100644 index 000000000..43dccbbed --- /dev/null +++ b/polytope/datacube/transformations/datacube_null_transformation/datacube_null_transformation.py @@ -0,0 +1,22 @@ +from ..datacube_transformations import DatacubeAxisTransformation + + +class DatacubeNullTransformation(DatacubeAxisTransformation): + def __init__(self, name, mapper_options): + self.name = name + self.transformation_options = mapper_options + + def generate_final_transformation(self): + return self + + def transformation_axes_final(self): + return [self.name] + + def change_val_type(self, axis_name, values): + return values + + def blocked_axes(self): + return [] + + def unwanted_axes(self): + return [] diff --git a/polytope/datacube/transformations/datacube_null_transformation.py b/polytope/datacube/transformations/datacube_null_transformation/null_axis_decorator.py similarity index 56% rename from polytope/datacube/transformations/datacube_null_transformation.py rename to polytope/datacube/transformations/datacube_null_transformation/null_axis_decorator.py index 66af3f3a9..10e2644df 100644 --- a/polytope/datacube/transformations/datacube_null_transformation.py +++ b/polytope/datacube/transformations/datacube_null_transformation/null_axis_decorator.py @@ -1,27 +1,3 @@ -from .datacube_transformations import DatacubeAxisTransformation - - -class DatacubeNullTransformation(DatacubeAxisTransformation): - def __init__(self, name, mapper_options): - self.name = name - self.transformation_options = mapper_options - - def generate_final_transformation(self): - return self - - def transformation_axes_final(self): - return [self.name] - - def change_val_type(self, axis_name, values): - return values - - def blocked_axes(self): - return [] - - def unwanted_axes(self): - return [] - - def null(cls): if cls.type_change: old_find_indexes = cls.find_indexes diff --git a/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py new file mode 100644 index 000000000..6e60de872 --- /dev/null +++ b/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py @@ -0,0 +1,22 @@ +from ..datacube_transformations import DatacubeAxisTransformation + + +class DatacubeAxisReverse(DatacubeAxisTransformation): + def __init__(self, name, mapper_options): + self.name = name + self.transformation_options = mapper_options + + def generate_final_transformation(self): + return self + + def transformation_axes_final(self): + return [self.name] + + def change_val_type(self, axis_name, values): + return values + + def blocked_axes(self): + return [] + + def unwanted_axes(self): + return [] diff --git a/polytope/datacube/transformations/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py similarity index 86% rename from polytope/datacube/transformations/datacube_reverse.py rename to polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py index 0e03006a7..2e5eff960 100644 --- a/polytope/datacube/transformations/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py @@ -1,27 +1,6 @@ import bisect -from .datacube_transformations import DatacubeAxisTransformation - - -class DatacubeAxisReverse(DatacubeAxisTransformation): - def __init__(self, name, mapper_options): - self.name = name - self.transformation_options = mapper_options - - def generate_final_transformation(self): - return self - - def transformation_axes_final(self): - return [self.name] - - def change_val_type(self, axis_name, values): - return values - - def blocked_axes(self): - return [] - - def unwanted_axes(self): - return [] +from .datacube_reverse import DatacubeAxisReverse def reverse(cls): diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 4bd07a2dc..2077f3466 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -8,8 +8,8 @@ class DatacubeAxisTransformation(ABC): def create_transform(name, transformation_type_key, transformation_options): transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] - - module = import_module("polytope.datacube.transformations.datacube_" + transformation_file_name) + file_name = ".datacube_" + transformation_file_name + module = import_module("polytope.datacube.transformations" + file_name + file_name) constructor = getattr(module, transformation_type) transformation_type_option = transformation_options[transformation_type_key] new_transformation = deepcopy(constructor(name, transformation_type_option)) @@ -54,8 +54,8 @@ def change_val_type(self, axis_name, values): } _type_to_transformation_file_lookup = { - "mapper": "mappers.datacube_mappers", - "cyclic": "cyclic.datacube_cyclic", + "mapper": "mappers", + "cyclic": "cyclic", "merge": "merger", "reverse": "reverse", "type_change": "type_change", diff --git a/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py new file mode 100644 index 000000000..8ba2ef0f7 --- /dev/null +++ b/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py @@ -0,0 +1,53 @@ +from copy import deepcopy +from importlib import import_module + +from ..datacube_transformations import DatacubeAxisTransformation + + +class DatacubeAxisTypeChange(DatacubeAxisTransformation): + # The transformation here will be to point the old axes to the new cyclic axes + + def __init__(self, name, type_options): + self.name = name + self.transformation_options = type_options + self.new_type = type_options + self._final_transformation = self.generate_final_transformation() + + def generate_final_transformation(self): + map_type = _type_to_datacube_type_change_lookup[self.new_type] + module = import_module("polytope.datacube.transformations.datacube_type_change.datacube_type_change") + constructor = getattr(module, map_type) + transformation = deepcopy(constructor(self.name, self.new_type)) + return transformation + + def transformation_axes_final(self): + return [self._final_transformation.axis_name] + + def change_val_type(self, axis_name, values): + return_idx = [self._final_transformation.transform_type(val) for val in values] + return_idx.sort() + return return_idx + + def make_str(self, value): + return self._final_transformation.make_str(value) + + def blocked_axes(self): + return [] + + def unwanted_axes(self): + return [] + + +class TypeChangeStrToInt(DatacubeAxisTypeChange): + def __init__(self, axis_name, new_type): + self.axis_name = axis_name + self._new_type = new_type + + def transform_type(self, value): + return int(value) + + def make_str(self, value): + return str(value) + + +_type_to_datacube_type_change_lookup = {"int": "TypeChangeStrToInt"} diff --git a/polytope/datacube/transformations/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py similarity index 66% rename from polytope/datacube/transformations/datacube_type_change.py rename to polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py index 1813efe24..10c9a95aa 100644 --- a/polytope/datacube/transformations/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py @@ -1,57 +1,6 @@ import bisect -from copy import deepcopy -from importlib import import_module -from .datacube_transformations import DatacubeAxisTransformation - - -class DatacubeAxisTypeChange(DatacubeAxisTransformation): - # The transformation here will be to point the old axes to the new cyclic axes - - def __init__(self, name, type_options): - self.name = name - self.transformation_options = type_options - self.new_type = type_options - self._final_transformation = self.generate_final_transformation() - - def generate_final_transformation(self): - map_type = _type_to_datacube_type_change_lookup[self.new_type] - module = import_module("polytope.datacube.transformations.datacube_type_change") - constructor = getattr(module, map_type) - transformation = deepcopy(constructor(self.name, self.new_type)) - return transformation - - def transformation_axes_final(self): - return [self._final_transformation.axis_name] - - def change_val_type(self, axis_name, values): - return_idx = [self._final_transformation.transform_type(val) for val in values] - return_idx.sort() - return return_idx - - def make_str(self, value): - return self._final_transformation.make_str(value) - - def blocked_axes(self): - return [] - - def unwanted_axes(self): - return [] - - -class TypeChangeStrToInt(DatacubeAxisTypeChange): - def __init__(self, axis_name, new_type): - self.axis_name = axis_name - self._new_type = new_type - - def transform_type(self, value): - return int(value) - - def make_str(self, value): - return str(value) - - -_type_to_datacube_type_change_lookup = {"int": "TypeChangeStrToInt"} +from .datacube_type_change import DatacubeAxisTypeChange def type_change(cls): From 9faf412f6c5d5b8824076297234330e9a55bbb86 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 25 Jan 2024 17:34:01 +0100 Subject: [PATCH 308/332] merge develop into cache sliceable values branch --- polytope/datacube/backends/datacube.py | 2 ++ polytope/datacube/backends/fdb.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index efa20e93d..62620eb0b 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -37,6 +37,8 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt ) for blocked_axis in transformation.blocked_axes(): self.blocked_axes.append(blocked_axis) + if len(final_axis_names) > 1: + self.coupled_axes.append(final_axis_names) for axis_name in final_axis_names: self.fake_axes.append(axis_name) # if axis does not yet exist, create it diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index f40d0f174..daf1a7ef9 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -41,11 +41,9 @@ def __init__(self, config={}, axis_options={}): self._check_and_add_axes(options, name, val) def get(self, requests: IndexTree): - requests.pprint_2() fdb_requests = [] fdb_requests_decoding_info = [] self.get_fdb_requests(requests, fdb_requests, fdb_requests_decoding_info) - print(fdb_requests) output_values = self.fdb.extract(fdb_requests) self.assign_fdb_output_to_nodes(output_values, fdb_requests_decoding_info) From b5ce4831f2252c3260931bd46daee38c7d105035 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 25 Jan 2024 17:42:25 +0100 Subject: [PATCH 309/332] isort --- polytope/datacube/index_tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index 204cb854d..3e532bd00 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -1,6 +1,6 @@ import json -from typing import OrderedDict import logging +from typing import OrderedDict from sortedcontainers import SortedList From 10695a0a43095c90116785ab715d3f4c9cad7d19 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 26 Jan 2024 15:36:38 +0100 Subject: [PATCH 310/332] identical structure datacube --- polytope/datacube/backends/datacube.py | 4 +- polytope/datacube/backends/fdb.py | 21 +++++- polytope/datacube/backends/mock.py | 3 +- polytope/datacube/backends/xarray.py | 3 +- polytope/datacube/datacube_axis.py | 4 ++ polytope/datacube/index_tree.py | 8 +++ polytope/engine/hullslicer.py | 29 ++++++++ polytope/polytope.py | 4 +- tests/test_cyclic_nearest.py | 96 ++++++++++++++++++++++++++ tests/test_point_nearest.py | 2 +- tests/test_regular_grid.py | 10 ++- 11 files changed, 172 insertions(+), 12 deletions(-) create mode 100644 tests/test_cyclic_nearest.py diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 62620eb0b..6b4aa96a1 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -145,11 +145,11 @@ def remap_path(self, path: DatacubePath): return path @staticmethod - def create(datacube, axis_options: dict): + def create(datacube, axis_options: dict, datacube_options={}): if isinstance(datacube, (xr.core.dataarray.DataArray, xr.core.dataset.Dataset)): from .xarray import XArrayDatacube - xadatacube = XArrayDatacube(datacube, axis_options=axis_options) + xadatacube = XArrayDatacube(datacube, axis_options, datacube_options) return xadatacube else: return datacube diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index daf1a7ef9..e2a51a0bf 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -3,11 +3,12 @@ import pygribjump as pygj from ...utility.geometry import nearest_pt +from ..transformations.datacube_cyclic import DatacubeAxisCyclic from .datacube import Datacube, IndexTree class FDBDatacube(Datacube): - def __init__(self, config={}, axis_options={}): + def __init__(self, config={}, axis_options={}, datacube_options={}): self.axis_options = axis_options self.axis_counter = 0 self._axes = None @@ -19,6 +20,7 @@ def __init__(self, config={}, axis_options={}): self.nearest_search = {} self.nearest_search = {} self.coupled_axes = [] + self.axis_with_identical_structure_after = datacube_options.get("identical structure after") partial_request = config # Find values in the level 3 FDB datacube @@ -88,17 +90,30 @@ def get_2nd_last_values(self, requests, leaf_path={}): first_ax_name = requests.children[0].axis.name second_ax_name = requests.children[0].children[0].axis.name # TODO: throw error if first_ax_name or second_ax_name not in self.nearest_search.keys() + second_ax = requests.children[0].children[0].axis + print("other nearest point") + print( + [ + [lat_val, lon_val] + for (lat_val, lon_val) in zip( + self.nearest_search[first_ax_name][0], self.nearest_search[second_ax_name][0] + ) + ] + ) nearest_pts = [ - [lat_val, lon_val] + [lat_val, second_ax._remap_val_to_axis_range(lon_val)] for (lat_val, lon_val) in zip( self.nearest_search[first_ax_name][0], self.nearest_search[second_ax_name][0] ) ] - # first collect the lat lon points found + print("nearest point") + print(nearest_pts) found_latlon_pts = [] for lat_child in requests.children: for lon_child in lat_child.children: found_latlon_pts.append([lat_child.value, lon_child.value]) + print("found lat lons") + print(found_latlon_pts) # now find the nearest lat lon to the points requested nearest_latlons = [] for pt in nearest_pts: diff --git a/polytope/datacube/backends/mock.py b/polytope/datacube/backends/mock.py index 6281c9cf3..6d441fb94 100644 --- a/polytope/datacube/backends/mock.py +++ b/polytope/datacube/backends/mock.py @@ -7,7 +7,7 @@ class MockDatacube(Datacube): - def __init__(self, dimensions): + def __init__(self, dimensions, datacube_options={}): assert isinstance(dimensions, dict) self.dimensions = dimensions @@ -23,6 +23,7 @@ def __init__(self, dimensions): self.stride[k] = stride_cumulative stride_cumulative *= self.dimensions[k] self.coupled_axes = [] + self.axis_with_identical_structure_after = "" def get(self, requests: IndexTree): # Takes in a datacube and verifies the leaves of the tree are complete diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 17d35679c..7daf2a675 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -8,7 +8,7 @@ class XArrayDatacube(Datacube): """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0).""" - def __init__(self, dataarray: xr.DataArray, axis_options={}): + def __init__(self, dataarray: xr.DataArray, axis_options={}, datacube_options={}): self.axis_options = axis_options self.axis_counter = 0 self._axes = None @@ -19,6 +19,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options={}): self.fake_axes = [] self.nearest_search = None self.coupled_axes = [] + self.axis_with_identical_structure_after = datacube_options.get("identical structure after") for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 1119be943..fa6cc89eb 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -189,6 +189,7 @@ def offset(range): cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between cls.unmap_path_key = unmap_path_key + cls._remap_val_to_axis_range = _remap_val_to_axis_range return cls @@ -590,6 +591,9 @@ def offset(self, value): def unmap_path_key(self, key_value_path, leaf_path, unwanted_path): return (key_value_path, leaf_path, unwanted_path) + def _remap_val_to_axis_range(self, value): + return value + def find_indices_between(self, index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index 3e532bd00..8abcb57ab 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -1,3 +1,4 @@ +import copy import json import logging from typing import OrderedDict @@ -46,6 +47,13 @@ def leaves_with_ancestors(self): self._collect_leaf_nodes(leaves) return leaves + def copy_children_from_other(self, other): + for o in other.children: + c = IndexTree(o.axis, copy.copy(o.value)) + self.add_child(c) + c.copy_children_from_other(o) + return + def _collect_leaf_nodes_old(self, leaves): if len(self.children) == 0: leaves.append(self) diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 228b1ebf4..365334862 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -122,14 +122,43 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): # directly work on request and return it... for c in combinations: + cached_node = None + repeated_sub_nodes = [] + r = IndexTree() r["unsliced_polytopes"] = set(c) current_nodes = [r] for ax in datacube.axes.values(): next_nodes = [] for node in current_nodes: + # detect if node is for number == 1 + # store a reference to that node + # skip processing the other 49 numbers + # at the end, copy that initial reference 49 times and add to request with correct number + + stored_val = None + if node.axis.name == datacube.axis_with_identical_structure_after: + stored_val = node.value + cached_node = node + # logging.info("Caching number 1") + elif node.axis.name == datacube.axis_with_identical_structure_after and node.value != stored_val: + repeated_sub_nodes.append(node) + del node["unsliced_polytopes"] + # logging.info(f"Skipping number {node.value}") + continue + self._build_branch(ax, node, datacube, next_nodes) current_nodes = next_nodes + + # logging.info("=== BEFORE COPYING ===") + + for n in repeated_sub_nodes: + # logging.info(f"Copying children for number {n.value}") + n.copy_children_from_other(cached_node) + + # logging.info("=== AFTER COPYING ===") + # request.pprint() + request.merge(r) return request diff --git a/polytope/polytope.py b/polytope/polytope.py index f6d4a723e..dfd869c93 100644 --- a/polytope/polytope.py +++ b/polytope/polytope.py @@ -37,11 +37,11 @@ def __repr__(self): class Polytope: - def __init__(self, datacube, engine=None, axis_options={}): + def __init__(self, datacube, engine=None, axis_options={}, datacube_options={}): from .datacube import Datacube from .engine import Engine - self.datacube = Datacube.create(datacube, axis_options) + self.datacube = Datacube.create(datacube, axis_options, datacube_options) self.engine = engine if engine is not None else Engine.default() def slice(self, polytopes: List[ConvexPolytope]): diff --git a/tests/test_cyclic_nearest.py b/tests/test_cyclic_nearest.py new file mode 100644 index 000000000..9367ebfea --- /dev/null +++ b/tests/test_cyclic_nearest.py @@ -0,0 +1,96 @@ +import pandas as pd +import pytest +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file +from helper_functions import download_test_data + +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Point, Select + +# import geopandas as gpd +# import matplotlib.pyplot as plt + + +class TestRegularGrid: + def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + + nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" + download_test_data(nexus_url, "era5-levels-members.grib") + self.options = { + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "longitude": {"cyclic": [0, 360]}, + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"} + self.datacube_options = {"identical structure after": "number"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options, datacube_options=self.datacube_options) + self.slicer = HullSlicer() + self.API = Polytope( + datacube=self.fdbdatacube, + engine=self.slicer, + axis_options=self.options, + datacube_options=self.datacube_options, + ) + + def find_nearest_latlon(self, grib_file, target_lat, target_lon): + # Open the GRIB file + f = open(grib_file) + + # Load the GRIB messages from the file + messages = [] + while True: + message = codes_grib_new_from_file(f) + if message is None: + break + messages.append(message) + + # Find the nearest grid points + nearest_points = [] + for message in messages: + nearest_index = codes_grib_find_nearest(message, target_lat, target_lon) + nearest_points.append(nearest_index) + + # Close the GRIB file + f.close() + + return nearest_points + + @pytest.mark.fdb + @pytest.mark.internet + def test_regular_grid(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231102T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[39, 360 - 76.45]], method="nearest"), + ) + result = self.API.retrieve(request) + longitude_val_1 = result.leaves[0].flatten()["longitude"] + result.pprint_2() + assert longitude_val_1 == 283.561643835616 + + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231102T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[39, -76.45]], method="nearest"), + ) + result = self.API.retrieve(request) + longitude_val_1 = result.leaves[0].flatten()["longitude"] + result.pprint_2() + assert longitude_val_1 == 283.561643835616 diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index 081050dad..fa53b9809 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -95,7 +95,7 @@ def test_fdb_datacube_true_point_3(self): result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 1 - assert result.leaves[0].value == 0 + assert result.leaves[0].value == 359.929906542056 assert result.leaves[0].axis.name == "longitude" @pytest.mark.fdb diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 2bc51b1e0..66ed9ef28 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -25,9 +25,15 @@ def setup_method(self, method): "longitude": {"cyclic": [0, 360]}, } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": "0"} - self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.datacube_options = {"identical structure after": "number"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options, datacube_options=self.datacube_options) self.slicer = HullSlicer() - self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + self.API = Polytope( + datacube=self.fdbdatacube, + engine=self.slicer, + axis_options=self.options, + datacube_options=self.datacube_options, + ) def find_nearest_latlon(self, grib_file, target_lat, target_lon): # Open the GRIB file From 5235e7d7cab1cf590831b2ce166af6ed9968716d Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Fri, 26 Jan 2024 15:46:41 +0100 Subject: [PATCH 311/332] flake8 --- polytope/datacube/backends/fdb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index e2a51a0bf..4c2e271a4 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -3,7 +3,6 @@ import pygribjump as pygj from ...utility.geometry import nearest_pt -from ..transformations.datacube_cyclic import DatacubeAxisCyclic from .datacube import Datacube, IndexTree From 1efb65824f56bb08c9704c12447f42c0ebe4dafc Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 29 Jan 2024 09:49:12 +0000 Subject: [PATCH 312/332] clean up --- polytope/datacube/backends/fdb.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 4c2e271a4..964175599 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -62,8 +62,7 @@ def get_fdb_requests(self, requests: IndexTree, fdb_requests=[], fdb_requests_de ) leaf_path.update(key_value_path) if len(requests.children[0].children[0].children) == 0: - # remap this last key - # TODO: here, find the fdb_requests and associated nodes to which to add results + # find the fdb_requests and associated nodes to which to add results (path, range_lengths, current_start_idxs, fdb_node_ranges, lat_length) = self.get_2nd_last_values( requests, leaf_path @@ -84,42 +83,35 @@ def get_fdb_requests(self, requests: IndexTree, fdb_requests=[], fdb_requests_de def get_2nd_last_values(self, requests, leaf_path={}): # In this function, we recursively loop over the last two layers of the tree and store the indices of the # request ranges in those layers - # TODO: here find nearest point first before retrieving etc + + # Find nearest point first before retrieving if len(self.nearest_search) != 0: first_ax_name = requests.children[0].axis.name second_ax_name = requests.children[0].children[0].axis.name # TODO: throw error if first_ax_name or second_ax_name not in self.nearest_search.keys() second_ax = requests.children[0].children[0].axis - print("other nearest point") - print( - [ - [lat_val, lon_val] - for (lat_val, lon_val) in zip( - self.nearest_search[first_ax_name][0], self.nearest_search[second_ax_name][0] - ) - ] - ) + + # TODO: actually, here we should not remap the nearest_pts, we should instead unmap the + # found_latlon_pts and then remap them later once we have compared found_latlon_pts and nearest_pts nearest_pts = [ [lat_val, second_ax._remap_val_to_axis_range(lon_val)] for (lat_val, lon_val) in zip( self.nearest_search[first_ax_name][0], self.nearest_search[second_ax_name][0] ) ] - print("nearest point") - print(nearest_pts) + found_latlon_pts = [] for lat_child in requests.children: for lon_child in lat_child.children: found_latlon_pts.append([lat_child.value, lon_child.value]) - print("found lat lons") - print(found_latlon_pts) + # now find the nearest lat lon to the points requested nearest_latlons = [] for pt in nearest_pts: nearest_latlon = nearest_pt(found_latlon_pts, pt) nearest_latlons.append(nearest_latlon) - # TODO: now combine with the rest of the function.... - # TODO: need to remove the branches that do not fit + + # need to remove the branches that do not fit lat_children_values = [child.value for child in requests.children] for i in range(len(lat_children_values)): lat_child_val = lat_children_values[i] @@ -157,7 +149,7 @@ def get_2nd_last_values(self, requests, leaf_path={}): (range_lengths[i], current_start_idxs[i], fdb_node_ranges[i]) = self.get_last_layer_before_leaf( lat_child, leaf_path, range_length, current_start_idx, fdb_range_nodes ) - # TODO: do we need to return all of this? + leaf_path_copy = deepcopy(leaf_path) leaf_path_copy.pop("values") return (leaf_path_copy, range_lengths, current_start_idxs, fdb_node_ranges, lat_length) From 9ddbbd87d8a39d35290bb7ee289198bfde323b3c Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 29 Jan 2024 12:18:11 +0000 Subject: [PATCH 313/332] add local regular grid mapper --- .../datacube_mappers/datacube_mappers.py | 6 +- .../datacube_mappers/mapper_types/healpix.py | 3 +- .../mapper_types/local_regular.py | 69 +++++++++++++++++++ .../mapper_types/octahedral.py | 3 +- .../mapper_types/reduced_ll.py | 3 +- .../datacube_mappers/mapper_types/regular.py | 3 +- 6 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 polytope/datacube/transformations/datacube_mappers/mapper_types/local_regular.py diff --git a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py index c42d04a77..a2034bfba 100644 --- a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py @@ -12,6 +12,9 @@ def __init__(self, name, mapper_options): self.grid_type = mapper_options["type"] self.grid_resolution = mapper_options["resolution"] self.grid_axes = mapper_options["axes"] + self.local_area = [] + if "local" in mapper_options.keys(): + self.local_area = mapper_options["local"] self.old_axis = name self._final_transformation = self.generate_final_transformation() self._final_mapped_axes = self._final_transformation._mapped_axes @@ -21,7 +24,7 @@ def generate_final_transformation(self): map_type = _type_to_datacube_mapper_lookup[self.grid_type] module = import_module("polytope.datacube.transformations.datacube_mappers.mapper_types." + self.grid_type) constructor = getattr(module, map_type) - transformation = deepcopy(constructor(self.old_axis, self.grid_axes, self.grid_resolution)) + transformation = deepcopy(constructor(self.old_axis, self.grid_axes, self.grid_resolution, self.local_area)) return transformation def blocked_axes(self): @@ -78,4 +81,5 @@ def unmap(self, first_val, second_val): "healpix": "HealpixGridMapper", "regular": "RegularGridMapper", "reduced_ll": "ReducedLatLonMapper", + "local_regular": "LocalRegularGridMapper", } diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_types/healpix.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/healpix.py index 627a1be94..8589ec71a 100644 --- a/polytope/datacube/transformations/datacube_mappers/mapper_types/healpix.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_types/healpix.py @@ -5,7 +5,8 @@ class HealpixGridMapper(DatacubeMapper): - def __init__(self, base_axis, mapped_axes, resolution): + def __init__(self, base_axis, mapped_axes, resolution, local_area=[]): + # TODO: if local area is not empty list, raise NotImplemented self._mapped_axes = mapped_axes self._base_axis = base_axis self._resolution = resolution diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_types/local_regular.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/local_regular.py new file mode 100644 index 000000000..1c83c9884 --- /dev/null +++ b/polytope/datacube/transformations/datacube_mappers/mapper_types/local_regular.py @@ -0,0 +1,69 @@ +import bisect + +from ..datacube_mappers import DatacubeMapper + + +class LocalRegularGridMapper(DatacubeMapper): + def __init__(self, base_axis, mapped_axes, resolution, local_area=[]): + # TODO: if local area is not empty list, raise NotImplemented + self._mapped_axes = mapped_axes + self._base_axis = base_axis + self._first_axis_min = local_area[0] + self._first_axis_max = local_area[1] + self._second_axis_min = local_area[2] + self._second_axis_max = local_area[3] + if not isinstance(resolution, list): + self.first_resolution = resolution + self.second_resolution = resolution + else: + self.first_resolution = resolution[0] + self.second_resolution = resolution[1] + self._first_deg_increment = (local_area[1] - local_area[0]) / self.first_resolution + self._second_deg_increment = (local_area[3] - local_area[2]) / self.second_resolution + self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} + self._first_axis_vals = self.first_axis_vals() + + def first_axis_vals(self): + first_ax_vals = [self._first_axis_max - i * self._first_deg_increment for i in range(self.first_resolution)] + return first_ax_vals + + def map_first_axis(self, lower, upper): + axis_lines = self._first_axis_vals + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def second_axis_vals(self, first_val): + second_ax_vals = [ + self._second_axis_min + i * self._second_deg_increment for i in range(4 * self.second_resolution) + ] + return second_ax_vals + + def map_second_axis(self, first_val, lower, upper): + axis_lines = self.second_axis_vals(first_val) + return_vals = [val for val in axis_lines if lower <= val <= upper] + return return_vals + + def axes_idx_to_regular_idx(self, first_idx, second_idx): + final_idx = first_idx * self.second_resolution + second_idx + return final_idx + + def find_second_idx(self, first_val, second_val): + tol = 1e-10 + second_axis_vals = self.second_axis_vals(first_val) + second_idx = bisect.bisect_left(second_axis_vals, second_val - tol) + return second_idx + + def unmap_first_val_to_start_line_idx(self, first_val): + tol = 1e-8 + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + first_idx = self._first_axis_vals.index(first_val) + return first_idx * self.second_resolution + + def unmap(self, first_val, second_val): + tol = 1e-8 + first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0] + first_idx = self._first_axis_vals.index(first_val) + second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0] + second_idx = self.second_axis_vals(first_val).index(second_val) + final_index = self.axes_idx_to_regular_idx(first_idx, second_idx) + return final_index diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_types/octahedral.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/octahedral.py index b2a3159bb..730ac9592 100644 --- a/polytope/datacube/transformations/datacube_mappers/mapper_types/octahedral.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_types/octahedral.py @@ -5,7 +5,8 @@ class OctahedralGridMapper(DatacubeMapper): - def __init__(self, base_axis, mapped_axes, resolution): + def __init__(self, base_axis, mapped_axes, resolution, local_area=[]): + # TODO: if local area is not empty list, raise NotImplemented self._mapped_axes = mapped_axes self._base_axis = base_axis self._resolution = resolution diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_types/reduced_ll.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/reduced_ll.py index f056b2329..ece09b4ac 100644 --- a/polytope/datacube/transformations/datacube_mappers/mapper_types/reduced_ll.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_types/reduced_ll.py @@ -4,7 +4,8 @@ class ReducedLatLonMapper(DatacubeMapper): - def __init__(self, base_axis, mapped_axes, resolution): + def __init__(self, base_axis, mapped_axes, resolution, local_area=[]): + # TODO: if local area is not empty list, raise NotImplemented self._mapped_axes = mapped_axes self._base_axis = base_axis self._resolution = resolution diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_types/regular.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/regular.py index 3b0598a0b..c8f207fca 100644 --- a/polytope/datacube/transformations/datacube_mappers/mapper_types/regular.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_types/regular.py @@ -4,7 +4,8 @@ class RegularGridMapper(DatacubeMapper): - def __init__(self, base_axis, mapped_axes, resolution): + def __init__(self, base_axis, mapped_axes, resolution, local_area=[]): + # TODO: if local area is not empty list, raise NotImplemented self._mapped_axes = mapped_axes self._base_axis = base_axis self._resolution = resolution From aeb86eb9fb7c56802d5ed1e11ad13fc1f8e2af39 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 29 Jan 2024 12:56:52 +0000 Subject: [PATCH 314/332] add __init__ file --- polytope/datacube/transformations/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 polytope/datacube/transformations/__init__.py diff --git a/polytope/datacube/transformations/__init__.py b/polytope/datacube/transformations/__init__.py new file mode 100644 index 000000000..e69de29bb From d63d52dd679725f6fffee663fea4cc7ed9fbe0ee Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Mon, 29 Jan 2024 15:49:24 +0000 Subject: [PATCH 315/332] add nearest method to transformations --- polytope/datacube/index_tree.py | 2 +- .../transformations/datacube_cyclic/cyclic_axis_decorator.py | 2 +- .../transformations/datacube_mappers/mapper_axis_decorator.py | 2 +- .../transformations/datacube_merger/merger_axis_decorator.py | 2 +- .../datacube_reverse/reverse_axis_decorator.py | 4 ++-- .../datacube_type_change/type_change_axis_decorator.py | 2 +- tests/test_point_nearest.py | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index b15e67fe2..76ab48ab3 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -1,6 +1,6 @@ import json -from typing import OrderedDict import logging +from typing import OrderedDict from sortedcontainers import SortedList diff --git a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py index 8b1cf2927..76a2aa425 100644 --- a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py @@ -142,7 +142,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): update_range() indexes_between_ranges = [] - if method != "surrounding": + if method != "surrounding" or method != "nearest": return old_find_indices_between(index_ranges, low, up, datacube, method) else: for indexes in index_ranges: diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py index 30f95b619..994321860 100644 --- a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py @@ -73,7 +73,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): transformation = transform if cls.name in transformation._mapped_axes(): for idxs in index_ranges: - if method == "surrounding": + if method == "surrounding" or method == "nearest": axis_reversed = transform._axis_reversed[cls.name] if not axis_reversed: start = bisect.bisect_left(idxs, low) diff --git a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py index eeb0cbe70..16d816391 100644 --- a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py @@ -51,7 +51,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): transformation = transform if cls.name in transformation._mapped_axes(): for indexes in index_ranges: - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.index(low) end = indexes.index(up) start = max(start - 1, 0) diff --git a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py index 2e5eff960..18bc8bd63 100644 --- a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py @@ -29,7 +29,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically # increasing - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.searchsorted(low, "left") end = indexes.searchsorted(up, "right") start = max(start - 1, 0) @@ -42,7 +42,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between = indexes[start:end].to_list() indexes_between_ranges.append(indexes_between) else: - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.index(low) end = indexes.index(up) start = max(start - 1, 0) diff --git a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py index 10c9a95aa..0e3669825 100644 --- a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py @@ -47,7 +47,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): transformation = transform if cls.name == transformation.name: for indexes in index_ranges: - if method == "surrounding": + if method == "surrounding" or method == "nearest": start = indexes.index(low) end = indexes.index(up) start = max(start - 1, 0) diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index 081050dad..fa53b9809 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -95,7 +95,7 @@ def test_fdb_datacube_true_point_3(self): result = self.API.retrieve(request) result.pprint() assert len(result.leaves) == 1 - assert result.leaves[0].value == 0 + assert result.leaves[0].value == 359.929906542056 assert result.leaves[0].axis.name == "longitude" @pytest.mark.fdb From 72af10f305e948bd5fccfe12fff81b1d9c8537e9 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 11:28:15 +0100 Subject: [PATCH 316/332] isort --- .gitignore | 7 ++++++- polytope/datacube/index_tree.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index ca7e60eb6..921fe79b5 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,9 @@ polytope.egg-info *.xml site .coverage -*.grib \ No newline at end of file +*.grib +*.gif +*.html +example_eo +example_mri +.mypy_cache \ No newline at end of file diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index b15e67fe2..76ab48ab3 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -1,6 +1,6 @@ import json -from typing import OrderedDict import logging +from typing import OrderedDict from sortedcontainers import SortedList From b4bb7eaa2b5302b8c88841f451ca576bc0a18307 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 24 Jan 2024 13:35:42 +0100 Subject: [PATCH 317/332] update gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 921fe79b5..c28eebef8 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ site *.html example_eo example_mri -.mypy_cache \ No newline at end of file +.mypy_cache +*.req \ No newline at end of file From 28328d8f9b17937a249df5ef3143d1fcbb5bb005 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 1 Feb 2024 11:09:24 +0000 Subject: [PATCH 318/332] fix local grid mapper issues --- polytope/datacube/index_tree.py | 7 +++ .../mapper_types/local_regular.py | 6 +-- tests/test_local_regular_grid.py | 48 +++++++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 tests/test_local_regular_grid.py diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index 76ab48ab3..b4ce3bcec 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -46,6 +46,13 @@ def leaves_with_ancestors(self): self._collect_leaf_nodes(leaves) return leaves + def pprint_2(self, level=0): + if self.axis.name == "root": + print("\n") + print("\t" * level + "\u21b3" + str(self)) + for child in self.children: + child.pprint_2(level + 1) + def _collect_leaf_nodes_old(self, leaves): if len(self.children) == 0: leaves.append(self) diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_types/local_regular.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/local_regular.py index 1c83c9884..a1514778a 100644 --- a/polytope/datacube/transformations/datacube_mappers/mapper_types/local_regular.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_types/local_regular.py @@ -24,7 +24,7 @@ def __init__(self, base_axis, mapped_axes, resolution, local_area=[]): self._first_axis_vals = self.first_axis_vals() def first_axis_vals(self): - first_ax_vals = [self._first_axis_max - i * self._first_deg_increment for i in range(self.first_resolution)] + first_ax_vals = [self._first_axis_max - i * self._first_deg_increment for i in range(self.first_resolution + 1)] return first_ax_vals def map_first_axis(self, lower, upper): @@ -34,7 +34,7 @@ def map_first_axis(self, lower, upper): def second_axis_vals(self, first_val): second_ax_vals = [ - self._second_axis_min + i * self._second_deg_increment for i in range(4 * self.second_resolution) + self._second_axis_min + i * self._second_deg_increment for i in range(self.second_resolution + 1) ] return second_ax_vals @@ -44,7 +44,7 @@ def map_second_axis(self, first_val, lower, upper): return return_vals def axes_idx_to_regular_idx(self, first_idx, second_idx): - final_idx = first_idx * self.second_resolution + second_idx + final_idx = first_idx * (self.second_resolution + 1) + second_idx return final_idx def find_second_idx(self, first_val, second_val): diff --git a/tests/test_local_regular_grid.py b/tests/test_local_regular_grid.py new file mode 100644 index 000000000..e623c840a --- /dev/null +++ b/tests/test_local_regular_grid.py @@ -0,0 +1,48 @@ +import pandas as pd +import pytest +from earthkit import data + +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Point, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + + ds = data.from_source("file", "./local.grib") + self.latlon_array = ds.to_xarray().t2m + print(self.latlon_array) + + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": {"mapper": {"type": "local_regular", "resolution": [80, 80], "axes": ["latitude", "longitude"], "local": [-40, 40, -20, 60]}}, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "longitude": {"cyclic": [0, 360]}, + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.fdb + def test_fdb_datacube(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["an"]), + Point(["latitude", "longitude"], [[0.16, 0.176]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint_2() + assert len(result.leaves) == 1 From facc193e6fd649c1599716093fbddf85150398c1 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 1 Feb 2024 11:16:13 +0000 Subject: [PATCH 319/332] fix local grid test --- tests/test_local_regular_grid.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/test_local_regular_grid.py b/tests/test_local_regular_grid.py index e623c840a..03bcb22b3 100644 --- a/tests/test_local_regular_grid.py +++ b/tests/test_local_regular_grid.py @@ -1,6 +1,5 @@ import pandas as pd import pytest -from earthkit import data from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request @@ -11,13 +10,16 @@ class TestSlicingFDBDatacube: def setup_method(self, method): from polytope.datacube.backends.fdb import FDBDatacube - ds = data.from_source("file", "./local.grib") - self.latlon_array = ds.to_xarray().t2m - print(self.latlon_array) - # Create a dataarray with 3 labelled axes using different index types self.options = { - "values": {"mapper": {"type": "local_regular", "resolution": [80, 80], "axes": ["latitude", "longitude"], "local": [-40, 40, -20, 60]}}, + "values": { + "mapper": { + "type": "local_regular", + "resolution": [80, 80], + "axes": ["latitude", "longitude"], + "local": [-40, 40, -20, 60], + } + }, "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, "number": {"type_change": "int"}, @@ -40,9 +42,11 @@ def test_fdb_datacube(self): Select("param", ["167"]), Select("class", ["od"]), Select("stream", ["oper"]), - Select("type", ["an"]), + Select("type", ["fc"]), Point(["latitude", "longitude"], [[0.16, 0.176]], method="nearest"), ) result = self.API.retrieve(request) result.pprint_2() assert len(result.leaves) == 1 + assert result.leaves[0].flatten()["latitude"] == 0 + assert result.leaves[0].flatten()["longitude"] == 0 From 19b490644f78198d65113cb8cc1a56f475302a76 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 1 Feb 2024 13:43:58 +0000 Subject: [PATCH 320/332] test local grid with cyclicity --- tests/test_local_grid_cyclic.py | 71 +++++++++++++ tests/test_local_regular_grid.py | 177 ++++++++++++++++++++++++++++++- 2 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 tests/test_local_grid_cyclic.py diff --git a/tests/test_local_grid_cyclic.py b/tests/test_local_grid_cyclic.py new file mode 100644 index 000000000..cb1c57eb7 --- /dev/null +++ b/tests/test_local_grid_cyclic.py @@ -0,0 +1,71 @@ +import pandas as pd +import pytest + +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Point, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": { + "mapper": { + "type": "local_regular", + "resolution": [80, 80], + "axes": ["latitude", "longitude"], + "local": [-40, 40, -20, 60], + } + }, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "longitude": {"cyclic": [-180, 180]} + } + self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.fdb + def test_fdb_datacube(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[-20, -20]]), + ) + result = self.API.retrieve(request) + result.pprint_2() + assert len(result.leaves) == 1 + assert result.leaves[0].flatten()["latitude"] == -20 + assert result.leaves[0].flatten()["longitude"] == -20 + + def test_fdb_datacube_2(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[-20, 50+360]]), + ) + result = self.API.retrieve(request) + result.pprint_2() + assert len(result.leaves) == 1 + assert result.leaves[0].flatten()["latitude"] == -20 + assert result.leaves[0].flatten()["longitude"] == 50 diff --git a/tests/test_local_regular_grid.py b/tests/test_local_regular_grid.py index 03bcb22b3..dfac410e5 100644 --- a/tests/test_local_regular_grid.py +++ b/tests/test_local_regular_grid.py @@ -23,7 +23,6 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, "number": {"type_change": "int"}, - "longitude": {"cyclic": [0, 360]}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -50,3 +49,179 @@ def test_fdb_datacube(self): assert len(result.leaves) == 1 assert result.leaves[0].flatten()["latitude"] == 0 assert result.leaves[0].flatten()["longitude"] == 0 + + @pytest.mark.fdb + def test_point_outside_local_region(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[0.16, 61]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint_2() + assert len(result.leaves) == 1 + assert result.leaves[0].flatten()["latitude"] == 0 + assert result.leaves[0].flatten()["longitude"] == 60 + + @pytest.mark.fdb + def test_point_outside_local_region_2(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[41, 1]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint_2() + assert len(result.leaves) == 1 + assert result.leaves[0].flatten()["latitude"] == 40 + assert result.leaves[0].flatten()["longitude"] == 1 + + @pytest.mark.fdb + def test_point_outside_local_region_3(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[1, 61]]), + ) + result = self.API.retrieve(request) + result.pprint_2() + assert len(result.leaves) == 1 + assert result.is_root() + + @pytest.mark.fdb + def test_point_outside_local_region_4(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[41, 1]]), + ) + result = self.API.retrieve(request) + result.pprint_2() + assert len(result.leaves) == 1 + assert result.is_root() + + @pytest.mark.fdb + def test_point_outside_local_region_5(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[-41, 1]]), + ) + result = self.API.retrieve(request) + result.pprint_2() + assert len(result.leaves) == 1 + assert result.is_root() + + @pytest.mark.fdb + def test_point_outside_local_region_6(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[-30, -21]]), + ) + result = self.API.retrieve(request) + result.pprint_2() + assert len(result.leaves) == 1 + assert result.is_root() + + @pytest.mark.fdb + def test_point_outside_local_region_7(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[-41, 1]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint_2() + assert len(result.leaves) == 1 + assert result.leaves[0].flatten()["latitude"] == -40 + assert result.leaves[0].flatten()["longitude"] == 1 + + @pytest.mark.fdb + def test_point_outside_local_region_8(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[-30, -21]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint_2() + assert len(result.leaves) == 1 + assert result.leaves[0].flatten()["latitude"] == -30 + assert result.leaves[0].flatten()["longitude"] == -20 + + @pytest.mark.fdb + def test_point_outside_local_region_9(self): + request = Request( + Select("step", [0]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20240129T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[-30, -21]], method="surrounding"), + ) + result = self.API.retrieve(request) + result.pprint_2() + assert len(result.leaves) == 3 + assert result.leaves[0].flatten()["latitude"] == -31 + assert result.leaves[0].flatten()["longitude"] == -20 From df2ca21a731b0a826b747ac2c2db0ab0622c545b Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 1 Feb 2024 13:45:31 +0000 Subject: [PATCH 321/332] black --- tests/test_local_grid_cyclic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_local_grid_cyclic.py b/tests/test_local_grid_cyclic.py index cb1c57eb7..bf6f5713a 100644 --- a/tests/test_local_grid_cyclic.py +++ b/tests/test_local_grid_cyclic.py @@ -23,7 +23,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, "number": {"type_change": "int"}, - "longitude": {"cyclic": [-180, 180]} + "longitude": {"cyclic": [-180, 180]}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -62,7 +62,7 @@ def test_fdb_datacube_2(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["fc"]), - Point(["latitude", "longitude"], [[-20, 50+360]]), + Point(["latitude", "longitude"], [[-20, 50 + 360]]), ) result = self.API.retrieve(request) result.pprint_2() From c666d0fe428afad3772488acd2dc55a5f94fd242 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Thu, 1 Feb 2024 14:01:44 +0000 Subject: [PATCH 322/332] ignore pygribjump test --- tests/test_local_grid_cyclic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_local_grid_cyclic.py b/tests/test_local_grid_cyclic.py index bf6f5713a..025986321 100644 --- a/tests/test_local_grid_cyclic.py +++ b/tests/test_local_grid_cyclic.py @@ -51,6 +51,7 @@ def test_fdb_datacube(self): assert result.leaves[0].flatten()["latitude"] == -20 assert result.leaves[0].flatten()["longitude"] == -20 + @pytest.mark.fdb def test_fdb_datacube_2(self): request = Request( Select("step", [0]), From fc8c47bc71fb6d82a7252fd051b59f11779ab020 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 5 Feb 2024 12:56:11 +0000 Subject: [PATCH 323/332] remove unnecessary requirements and files --- .gitattributes | 4 ---- .gitconfig | 2 -- examples/requirements_examples.txt | 3 ++- requirements_example.txt | 1 - 4 files changed, 2 insertions(+), 8 deletions(-) delete mode 100644 .gitattributes delete mode 100644 .gitconfig delete mode 100644 requirements_example.txt diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index edc195ff1..000000000 --- a/.gitattributes +++ /dev/null @@ -1,4 +0,0 @@ -examples/data/*.grib filter=lfs diff=lfs merge=lfs -text -examples/data/*.jpg filter=lfs diff=lfs merge=lfs -text -examples/data/*.shp filter=lfs diff=lfs merge=lfs -text -tests/data/*.grib filter=lfs diff=lfs merge=lfs -text diff --git a/.gitconfig b/.gitconfig deleted file mode 100644 index e2bbefe3b..000000000 --- a/.gitconfig +++ /dev/null @@ -1,2 +0,0 @@ -[lfs] - fetchexclude = * \ No newline at end of file diff --git a/examples/requirements_examples.txt b/examples/requirements_examples.txt index 288d25581..cb2087e6e 100644 --- a/examples/requirements_examples.txt +++ b/examples/requirements_examples.txt @@ -9,4 +9,5 @@ shp==1.0.2 Fiona==1.8.22 geopandas==0.12.2 plotly==5.11.0 -pyshp==2.3.1 \ No newline at end of file +pyshp==2.3.1 +cfgrib==0.9.10.3 \ No newline at end of file diff --git a/requirements_example.txt b/requirements_example.txt deleted file mode 100644 index 19717dacf..000000000 --- a/requirements_example.txt +++ /dev/null @@ -1 +0,0 @@ -cfgrib==0.9.10.3 \ No newline at end of file From 73b532cfb1eb329643f71e5944789a3bf4674b6d Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 6 Feb 2024 10:56:43 +0000 Subject: [PATCH 324/332] test which requirements are truly needed --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 50594c4fc..035d29654 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -decorator==5.1.1 +# decorator==5.1.1 numpy==1.23.5 pandas==1.5.2 -pypi==2.1 -requests==2.28.1 +# pypi==2.1 +# requests==2.28.1 scipy==1.9.3 sortedcontainers==2.4.0 tripy==1.0.0 -typing==3.7.4.3 +# typing==3.7.4.3 xarray==2022.12.0 From 90b52af2676c2261d090b72ba693c62c7b74445f Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 6 Feb 2024 11:01:50 +0000 Subject: [PATCH 325/332] remove unnecessary requirements --- requirements.txt | 4 ---- tests/requirements_test.txt | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 035d29654..a6c7f3b39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,6 @@ -# decorator==5.1.1 numpy==1.23.5 pandas==1.5.2 -# pypi==2.1 -# requests==2.28.1 scipy==1.9.3 sortedcontainers==2.4.0 tripy==1.0.0 -# typing==3.7.4.3 xarray==2022.12.0 diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index 23b52cb64..e09e60e35 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -5,5 +5,4 @@ eccodes==1.5.2 h5netcdf==1.1.0 h5py==3.8.0 earthkit==0.0.1 -earthkit-data==0.1.3 -eccodes==1.5.2 \ No newline at end of file +earthkit-data==0.1.3 \ No newline at end of file From 2d154074f2b660a4228f7f4cbdf99e4dce810c1f Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 6 Feb 2024 16:23:16 +0000 Subject: [PATCH 326/332] flake8 --- polytope/datacube/index_tree.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index d95cc6cd5..2afb8416a 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -191,13 +191,6 @@ def pprint(self, level=0): if len(self.children) == 0: logging.debug("\t" * (level + 1) + "\u21b3" + str(self.result)) - def pprint_2(self, level=0): - if self.axis.name == "root": - print("\n") - print("\t" * level + "\u21b3" + str(self)) - for child in self.children: - child.pprint_2(level + 1) - def remove_branch(self): if not self.is_root(): old_parent = self._parent From df734b379c9e926fbaba68ef38cf3461b3c5b96c Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 1 Mar 2024 14:12:17 +0100 Subject: [PATCH 327/332] remove numpy version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6c7f3b39..b2bd6f0fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.23.5 +numpy pandas==1.5.2 scipy==1.9.3 sortedcontainers==2.4.0 From 23ce604fec2be3e6a7b45ea3ed966617d7348138 Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 1 Mar 2024 15:10:19 +0100 Subject: [PATCH 328/332] add __init__.py to subfolders --- .../datacube/transformations/datacube_cyclic/__init__.py | 2 ++ .../datacube/transformations/datacube_mappers/__init__.py | 2 ++ .../datacube_mappers/mapper_types/__init__.py | 5 +++++ .../datacube/transformations/datacube_merger/__init__.py | 2 ++ .../transformations/datacube_null_transformation/__init__.py | 2 ++ .../datacube/transformations/datacube_reverse/__init__.py | 2 ++ .../transformations/datacube_type_change/__init__.py | 2 ++ 7 files changed, 17 insertions(+) create mode 100644 polytope/datacube/transformations/datacube_cyclic/__init__.py create mode 100644 polytope/datacube/transformations/datacube_mappers/__init__.py create mode 100644 polytope/datacube/transformations/datacube_mappers/mapper_types/__init__.py create mode 100644 polytope/datacube/transformations/datacube_merger/__init__.py create mode 100644 polytope/datacube/transformations/datacube_null_transformation/__init__.py create mode 100644 polytope/datacube/transformations/datacube_reverse/__init__.py create mode 100644 polytope/datacube/transformations/datacube_type_change/__init__.py diff --git a/polytope/datacube/transformations/datacube_cyclic/__init__.py b/polytope/datacube/transformations/datacube_cyclic/__init__.py new file mode 100644 index 000000000..adfaf9d8f --- /dev/null +++ b/polytope/datacube/transformations/datacube_cyclic/__init__.py @@ -0,0 +1,2 @@ +from .cyclic_axis_decorator import * +from .datacube_cyclic import * diff --git a/polytope/datacube/transformations/datacube_mappers/__init__.py b/polytope/datacube/transformations/datacube_mappers/__init__.py new file mode 100644 index 000000000..ad69c9c5a --- /dev/null +++ b/polytope/datacube/transformations/datacube_mappers/__init__.py @@ -0,0 +1,2 @@ +from .datacube_mappers import * +from .mapper_axis_decorator import * diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_types/__init__.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/__init__.py new file mode 100644 index 000000000..ba9a7b339 --- /dev/null +++ b/polytope/datacube/transformations/datacube_mappers/mapper_types/__init__.py @@ -0,0 +1,5 @@ +from .healpix import * +from .local_regular import * +from .octahedral import * +from .reduced_ll import * +from .regular import * diff --git a/polytope/datacube/transformations/datacube_merger/__init__.py b/polytope/datacube/transformations/datacube_merger/__init__.py new file mode 100644 index 000000000..71a59acb1 --- /dev/null +++ b/polytope/datacube/transformations/datacube_merger/__init__.py @@ -0,0 +1,2 @@ +from .datacube_merger import * +from .merger_axis_decorator import * diff --git a/polytope/datacube/transformations/datacube_null_transformation/__init__.py b/polytope/datacube/transformations/datacube_null_transformation/__init__.py new file mode 100644 index 000000000..13395c4ad --- /dev/null +++ b/polytope/datacube/transformations/datacube_null_transformation/__init__.py @@ -0,0 +1,2 @@ +from .datacube_null_transformation import * +from .null_axis_decorator import * diff --git a/polytope/datacube/transformations/datacube_reverse/__init__.py b/polytope/datacube/transformations/datacube_reverse/__init__.py new file mode 100644 index 000000000..73cb9d862 --- /dev/null +++ b/polytope/datacube/transformations/datacube_reverse/__init__.py @@ -0,0 +1,2 @@ +from .datacube_reverse import * +from .reverse_axis_decorator import * diff --git a/polytope/datacube/transformations/datacube_type_change/__init__.py b/polytope/datacube/transformations/datacube_type_change/__init__.py new file mode 100644 index 000000000..89dab3aef --- /dev/null +++ b/polytope/datacube/transformations/datacube_type_change/__init__.py @@ -0,0 +1,2 @@ +from .datacube_type_change import * +from .type_change_axis_decorator import * From bd0eb3d6e578a1e7a090210b85500adaa99ca474 Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 1 Mar 2024 14:15:33 +0100 Subject: [PATCH 329/332] remove requirement versions --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index b2bd6f0fe..afbacd4ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ numpy -pandas==1.5.2 -scipy==1.9.3 -sortedcontainers==2.4.0 -tripy==1.0.0 -xarray==2022.12.0 +pandas +scipy +sortedcontainers +tripy +xarray From b117f83d90ff5328d6393b3bdeb97bf1b43911a6 Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 1 Mar 2024 14:21:34 +0100 Subject: [PATCH 330/332] remove test requirement versions --- tests/requirements_test.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index e09e60e35..0df9637b0 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -1,8 +1,8 @@ -r ../requirements.txt -pytest==7.2.0 -cffi==1.15.1 -eccodes==1.5.2 -h5netcdf==1.1.0 -h5py==3.8.0 -earthkit==0.0.1 -earthkit-data==0.1.3 \ No newline at end of file +pytest +cffi +eccodes +h5netcdf +h5py +earthkit +earthkit-data \ No newline at end of file From 2b8dc06b9a9a183137d93dc1d39fbf52b0c09b60 Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 1 Mar 2024 14:23:33 +0100 Subject: [PATCH 331/332] remove example requirement versions --- examples/requirements_examples.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/requirements_examples.txt b/examples/requirements_examples.txt index cb2087e6e..5c46d4d64 100644 --- a/examples/requirements_examples.txt +++ b/examples/requirements_examples.txt @@ -1,13 +1,13 @@ -r ../requirements.txt -r ../tests/requirements_test.txt -matplotlib==3.6.2 -matplotlib-inline==0.1.6 -Pillow==9.3.0 -Shapely==1.8.5.post1 -shp==1.0.2 -Fiona==1.8.22 -geopandas==0.12.2 -plotly==5.11.0 -pyshp==2.3.1 -cfgrib==0.9.10.3 \ No newline at end of file +matplotlib +matplotlib-inline +Pillow +Shapely +shp +Fiona +geopandas +plotly +pyshp +cfgrib \ No newline at end of file From a0c56db2d3c27fc12d43f0ff1a7aadcf0813c995 Mon Sep 17 00:00:00 2001 From: James Hawkes Date: Fri, 1 Mar 2024 15:17:55 +0000 Subject: [PATCH 332/332] Version 1.0.3 --- polytope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope/version.py b/polytope/version.py index 7863915fa..976498ab9 100644 --- a/polytope/version.py +++ b/polytope/version.py @@ -1 +1 @@ -__version__ = "1.0.2" +__version__ = "1.0.3"