diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 56b8e7ee0..fb1be91ea 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -155,8 +155,8 @@ jobs: pip install setuptools wheel twine - name: Build and publish env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + TWINE_USERNAME: "__token__" + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | python setup.py sdist twine upload dist/* \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index 03024ec5d..091f576d3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,3 @@ -jinja2<3.1.0 +jinja2>=3.1.3 Markdown<3.2 mkdocs>=1.0 \ No newline at end of file diff --git a/examples/requirements_examples.txt b/examples/requirements_examples.txt index cb2087e6e..5c46d4d64 100644 --- a/examples/requirements_examples.txt +++ b/examples/requirements_examples.txt @@ -1,13 +1,13 @@ -r ../requirements.txt -r ../tests/requirements_test.txt -matplotlib==3.6.2 -matplotlib-inline==0.1.6 -Pillow==9.3.0 -Shapely==1.8.5.post1 -shp==1.0.2 -Fiona==1.8.22 -geopandas==0.12.2 -plotly==5.11.0 -pyshp==2.3.1 -cfgrib==0.9.10.3 \ No newline at end of file +matplotlib +matplotlib-inline +Pillow +Shapely +shp +Fiona +geopandas +plotly +pyshp +cfgrib \ No newline at end of file diff --git a/performance/fdb_performance_3D.py b/performance/fdb_performance_3D.py index 547d865b0..88fb85c92 100644 --- a/performance/fdb_performance_3D.py +++ b/performance/fdb_performance_3D.py @@ -20,6 +20,7 @@ def setup_method(self, method): "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, "levelist": {"transformation": {"type_change": "int"}}, + "longitude": {"transformation": {"cyclic": [0, 360]}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index edc445927..76d0f939c 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -1,12 +1,10 @@ -import importlib import logging -import math from abc import ABC, abstractmethod from typing import Any import xarray as xr -from ...utility.combinatorics import unique, validate_axes +from ...utility.combinatorics import validate_axes from ..datacube_axis import DatacubeAxis from ..index_tree import DatacubePath, IndexTree from ..transformations.datacube_transformations import ( @@ -16,6 +14,24 @@ class Datacube(ABC): + def __init__(self, axis_options=None, datacube_options=None): + if axis_options is None: + self.axis_options = {} + else: + self.axis_options = axis_options + if datacube_options is None: + datacube_options = {} + self.axis_with_identical_structure_after = datacube_options.get("identical structure after") + self.coupled_axes = [] + self.axis_counter = 0 + self.complete_axes = [] + self.blocked_axes = [] + self.fake_axes = [] + self.treated_axes = [] + self.nearest_search = {} + self._axes = None + self.transformed_axes = [] + @abstractmethod def get(self, requests: IndexTree) -> Any: """Return data given a set of request trees""" @@ -46,23 +62,21 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt # first need to change the values so that we have right type values = transformation.change_val_type(axis_name, values) - if self._axes is None: - DatacubeAxis.create_standard(axis_name, values, self) - elif axis_name not in self._axes.keys(): + if self._axes is None or axis_name not in self._axes.keys(): DatacubeAxis.create_standard(axis_name, values, self) # add transformation tag to axis, as well as transformation options for later setattr(self._axes[axis_name], has_transform[transformation_type_key], True) # where has_transform is a # factory inside datacube_transformations to set the has_transform, is_cyclic etc axis properties # add the specific transformation handled here to the relevant axes # Modify the axis to update with the tag - decorator_module = importlib.import_module("polytope.datacube.datacube_axis") - decorator = getattr(decorator_module, transformation_type_key) - decorator(self._axes[axis_name]) + if transformation not in self._axes[axis_name].transformations: # Avoids duplicates being stored self._axes[axis_name].transformations.append(transformation) def _add_all_transformation_axes(self, options, name, values): for transformation_type_key in options.keys(): + if transformation_type_key != "cyclic": + self.transformed_axes.append(name) self._create_axes(name, values, transformation_type_key, options) def _check_and_add_axes(self, options, name, values): @@ -70,9 +84,7 @@ def _check_and_add_axes(self, options, name, values): self._add_all_transformation_axes(options, name, values) else: if name not in self.blocked_axes: - if self._axes is None: - DatacubeAxis.create_standard(name, values, self) - elif name not in self._axes.keys(): + if self._axes is None or name not in self._axes.keys(): DatacubeAxis.create_standard(name, values, self) def has_index(self, path: DatacubePath, axis, index): @@ -96,46 +108,12 @@ def get_indices(self, path: DatacubePath, axis, lower, upper, method=None): """ path = self.fit_path(path) indexes = axis.find_indexes(path, self) - search_ranges = axis.remap([lower, upper]) - original_search_ranges = axis.to_intervals([lower, upper]) - # Find the offsets for each interval in the requested range, which we will need later - search_ranges_offset = [] - for r in original_search_ranges: - offset = axis.offset(r) - search_ranges_offset.append(offset) - idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, method) - # Remove duplicates even if difference of the order of the axis tolerance - if offset is not None: - # Note that we can only do unique if not dealing with time values - idx_between = unique(idx_between) + idx_between = axis.find_indices_between(indexes, lower, upper, self, method) logging.info(f"For axis {axis.name} between {lower} and {upper}, found indices {idx_between}") return idx_between - def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, method): - idx_between = [] - for i in range(len(search_ranges)): - r = search_ranges[i] - offset = search_ranges_offset[i] - low = r[0] - up = r[1] - indexes_between = axis.find_indices_between([indexes], low, up, self, method) - # Now the indexes_between are values on the cyclic range so need to remap them to their original - # values before returning them - for j in range(len(indexes_between)): - # if we have a special indexes between range that needs additional offset, treat it here - if len(indexes_between[j]) == 0: - idx_between = idx_between - else: - for k in range(len(indexes_between[j])): - if offset is None: - indexes_between[j][k] = indexes_between[j][k] - else: - indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol))) - idx_between.append(indexes_between[j][k]) - return idx_between - def get_mapper(self, axis): """ Get the type mapper for a subaxis of the datacube given by label diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 30abd8819..39a2cab95 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -11,25 +11,13 @@ class FDBDatacube(Datacube): def __init__(self, config=None, axis_options=None, datacube_options=None, point_cloud_options=None): if config is None: config = {} - if axis_options is None: - axis_options = {} - if datacube_options is None: - datacube_options = {} + + super().__init__(axis_options, datacube_options) logging.info("Created an FDB datacube with options: " + str(axis_options)) - self.axis_options = axis_options - self.axis_counter = 0 - self._axes = None - treated_axes = [] - self.complete_axes = [] - self.blocked_axes = [] - self.fake_axes = [] self.unwanted_path = {} self.has_point_cloud = point_cloud_options # NOTE: here, will be True/False - self.coupled_axes = [] - self.axis_with_identical_structure_after = datacube_options.get("identical structure after") - self.nearest_search = {} partial_request = config # Find values in the level 3 FDB datacube @@ -44,12 +32,12 @@ def __init__(self, config=None, axis_options=None, datacube_options=None, point_ values.sort() options = axis_options.get(name, None) self._check_and_add_axes(options, name, values) - treated_axes.append(name) + self.treated_axes.append(name) self.complete_axes.append(name) # add other options to axis which were just created above like "lat" for the mapper transformations for eg for name in self._axes: - if name not in treated_axes: + if name not in self.treated_axes: options = axis_options.get(name, None) val = self._axes[name].type self._check_and_add_axes(options, name, val) @@ -250,7 +238,7 @@ def sort_fdb_request_ranges(self, range_lengths, current_start_idx, lat_length): return (original_indices, sorted_request_ranges) def datacube_natural_indexes(self, axis, subarray): - indexes = subarray[axis.name] + indexes = subarray.get(axis.name, None) return indexes def select(self, path, unmapped_path): diff --git a/polytope/datacube/backends/mock.py b/polytope/datacube/backends/mock.py index 3d70c885b..4fc93932e 100644 --- a/polytope/datacube/backends/mock.py +++ b/polytope/datacube/backends/mock.py @@ -8,6 +8,7 @@ class MockDatacube(Datacube): def __init__(self, dimensions, datacube_options={}): + super().__init__({}, datacube_options) assert isinstance(dimensions, dict) self.dimensions = dimensions @@ -22,8 +23,6 @@ def __init__(self, dimensions, datacube_options={}): for k, v in reversed(dimensions.items()): self.stride[k] = stride_cumulative stride_cumulative *= self.dimensions[k] - self.coupled_axes = [] - self.axis_with_identical_structure_after = "" def get(self, requests: IndexTree): # Takes in a datacube and verifies the leaves of the tree are complete diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 239c21bac..b3457c355 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -1,5 +1,6 @@ from copy import deepcopy +import numpy as np import xarray as xr from .datacube import Datacube, IndexTree @@ -9,44 +10,32 @@ class XArrayDatacube(Datacube): """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0).""" def __init__(self, dataarray: xr.DataArray, axis_options=None, datacube_options=None, point_cloud_options=None): - if axis_options is None: - axis_options = {} - if datacube_options is None: - datacube_options = {} - self.axis_options = axis_options - self.axis_counter = 0 - self._axes = None + super().__init__(axis_options, datacube_options) self.dataarray = dataarray treated_axes = [] - self.complete_axes = [] - self.blocked_axes = [] - self.fake_axes = [] - self.nearest_search = None - self.coupled_axes = [] - self.axis_with_identical_structure_after = datacube_options.get("identical structure after") self.has_point_cloud = point_cloud_options for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: - options = axis_options.get(name, None) + options = self.axis_options.get(name, None) self._check_and_add_axes(options, name, values) - treated_axes.append(name) + self.treated_axes.append(name) self.complete_axes.append(name) else: if self.dataarray[name].dims == (): - options = axis_options.get(name, None) + options = self.axis_options.get(name, None) self._check_and_add_axes(options, name, values) - treated_axes.append(name) + self.treated_axes.append(name) for name in dataarray.dims: - if name not in treated_axes: - options = axis_options.get(name, None) + if name not in self.treated_axes: + options = self.axis_options.get(name, None) val = dataarray[name].values[0] self._check_and_add_axes(options, name, val) - treated_axes.append(name) + self.treated_axes.append(name) # add other options to axis which were just created above like "lat" for the mapper transformations for eg for name in self._axes: - if name not in treated_axes: - options = axis_options.get(name, None) + if name not in self.treated_axes: + options = self.axis_options.get(name, None) val = self._axes[name].type self._check_and_add_axes(options, name, val) @@ -55,45 +44,23 @@ def find_point_cloud(self): if self.has_point_cloud: return self.has_point_cloud - def old_get(self, requests: IndexTree): - for r in requests.leaves: - path = r.flatten() - if len(path.items()) == self.axis_counter: - # first, find the grid mapper transform - unmapped_path = {} - path_copy = deepcopy(path) - for key in path_copy: - axis = self._axes[key] - (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) - path = self.fit_path(path) - subxarray = self.dataarray.sel(path, method="nearest") - subxarray = subxarray.sel(unmapped_path) - value = subxarray.item() - key = subxarray.name - r.result = (key, value) - else: - r.remove_branch() - def get(self, requests: IndexTree): - # TODO: change to work with the irregular grid - axis_counter = self.axis_counter + 1 - # if self.has_point_cloud: - # axis_counter = self.axis_counter - 1 - # else: - # axis_counter = self.axis_counter - for r in requests.leaves: - # path = r.flatten() path = r.flatten_with_result() - if len(path.items()) == axis_counter: + if len(path.items()) == self.axis_counter + 1: # first, find the grid mapper transform unmapped_path = {} path_copy = deepcopy(path) for key in path_copy: if key != "result": axis = self._axes[key] - (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) - path = self.fit_path(path) + key_value_path = {key: path_copy[key]} + (key_value_path, path, unmapped_path) = axis.unmap_path_key(key_value_path, path, unmapped_path) + path.update(key_value_path) + path.update(unmapped_path) + + unmapped_path = {} + self.refit_path(path, unmapped_path, path) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmapped_path) value = subxarray.item() @@ -107,13 +74,34 @@ def datacube_natural_indexes(self, axis, subarray): indexes = next(iter(subarray.xindexes.values())).to_pandas_index() else: if subarray[axis.name].values.ndim == 0: - indexes = [subarray[axis.name].values] + # NOTE how we handle the two special datetime and timedelta cases to conform with numpy arrays + if np.issubdtype(subarray[axis.name].values.dtype, np.datetime64): + indexes = [subarray[axis.name].astype("datetime64[us]").values] + elif np.issubdtype(subarray[axis.name].values.dtype, np.timedelta64): + indexes = [subarray[axis.name].astype("timedelta64[us]").values] + else: + indexes = [subarray[axis.name].values.tolist()] else: indexes = subarray[axis.name].values return indexes + def refit_path(self, path_copy, unmapped_path, path): + for key in path.keys(): + if key not in self.dataarray.dims: + path_copy.pop(key) + elif key not in self.dataarray.coords.dtypes: + unmapped_path.update({key: path[key]}) + path_copy.pop(key, None) + for key in self.dataarray.coords.dtypes: + key_dtype = self.dataarray.coords.dtypes[key] + if key_dtype.type is np.str_ and key in path.keys(): + unmapped_path.update({key: path[key]}) + path_copy.pop(key, None) + def select(self, path, unmapped_path): - subarray = self.dataarray.sel(path, method="nearest") + path_copy = deepcopy(path) + self.refit_path(path_copy, unmapped_path, path) + subarray = self.dataarray.sel(path_copy, method="nearest") subarray = subarray.sel(unmapped_path) return subarray diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 5bed5dfe8..4314fe7e9 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -1,3 +1,4 @@ +import bisect from abc import ABC, abstractmethod from copy import deepcopy from typing import Any, List @@ -5,11 +6,13 @@ import numpy as np import pandas as pd -from .transformations.datacube_cyclic.cyclic_axis_decorator import cyclic -from .transformations.datacube_mappers.mapper_axis_decorator import mapper -from .transformations.datacube_merger.merger_axis_decorator import merge -from .transformations.datacube_reverse.reverse_axis_decorator import reverse -from .transformations.datacube_type_change.type_change_axis_decorator import type_change +from .transformations.datacube_cyclic.datacube_cyclic import DatacubeAxisCyclic +from .transformations.datacube_mappers.datacube_mappers import DatacubeMapper +from .transformations.datacube_merger.datacube_merger import DatacubeAxisMerger +from .transformations.datacube_reverse.datacube_reverse import DatacubeAxisReverse +from .transformations.datacube_type_change.datacube_type_change import ( + DatacubeAxisTypeChange, +) class DatacubeAxis(ABC): @@ -19,10 +22,12 @@ class DatacubeAxis(ABC): reorder = False type_change = False - def update_axis(self): - if self.is_cyclic: - self = cyclic(self) - return self + def order_tranformations(self): + self.transformations = sorted(self.transformations, key=lambda x: transformations_order[type(x)]) + + def give_transformations_parents(self): + for i, transform in enumerate(self.transformations[1:]): + transform.parent = self.transformations[i - 1] # Convert from user-provided value to CONTINUOUS type (e.g. float, pd.timestamp) @abstractmethod @@ -43,15 +48,21 @@ def serialize(self, value: Any) -> Any: pass def to_intervals(self, range): - return [range] + intervals = [range] + for transformation in self.transformations[::-1]: + intervals = transformation.to_intervals(range, intervals, self) + return intervals def remap(self, range: List) -> Any: - return [range] + ranges = [range] + for transformation in self.transformations[::-1]: + ranges = transformation.remap(range, ranges, self) + return ranges def unmap_to_datacube(self, path, unmapped_path): return (path, unmapped_path) - def find_indexes(self, path, datacube): + def find_standard_indexes(self, path, datacube): unmapped_path = {} path_copy = deepcopy(path) for key in path_copy: @@ -60,46 +71,70 @@ def find_indexes(self, path, datacube): subarray = datacube.select(path, unmapped_path) return datacube.datacube_natural_indexes(self, subarray) + def find_indexes(self, path, datacube): + indexes = self.find_standard_indexes(path, datacube) + for transformation in self.transformations[::-1]: + indexes = transformation.find_modified_indexes(indexes, path, datacube, self) + return indexes + def offset(self, value): - return 0 + offset = 0 + for transformation in self.transformations[::-1]: + offset = transformation.offset(value, self, offset) + return offset def unmap_path_key(self, key_value_path, leaf_path, unwanted_path): + for transformation in self.transformations[::-1]: + (key_value_path, leaf_path, unwanted_path) = transformation.unmap_path_key( + key_value_path, leaf_path, unwanted_path, self + ) return (key_value_path, leaf_path, unwanted_path) def _remap_val_to_axis_range(self, value): + for transformation in self.transformations[::-1]: + value = transformation._remap_val_to_axis_range(value, self) return value - def find_indices_between(self, index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping + def find_standard_indices_between(self, indexes, low, up, datacube, method=None): indexes_between_ranges = [] - for indexes in index_ranges: - if self.name in datacube.complete_axes: - # Find the range of indexes between lower and upper - # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html - # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing - if method == "surrounding" or method == "nearest": - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end].to_list() - indexes_between_ranges.append(indexes_between) - else: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - indexes_between = indexes[start:end].to_list() - indexes_between_ranges.append(indexes_between) + + if self.name in datacube.complete_axes and self.name not in datacube.transformed_axes: + # Find the range of indexes between lower and upper + # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html + # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing + if method == "surrounding" or method == "nearest": + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.extend(indexes_between) else: - if method == "surrounding" or method == "nearest": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - indexes_between = [i for i in indexes if low <= i <= up] - indexes_between_ranges.append(indexes_between) + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.extend(indexes_between) + else: + if method == "surrounding" or method == "nearest": + start = bisect.bisect_left(indexes, low) + end = bisect.bisect_right(indexes, up) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end] + indexes_between_ranges.extend(indexes_between) + else: + lower_idx = bisect.bisect_left(indexes, low) + upper_idx = bisect.bisect_right(indexes, up) + indexes_between = indexes[lower_idx:upper_idx] + indexes_between_ranges.extend(indexes_between) + return indexes_between_ranges + + def find_indices_between(self, indexes_ranges, low, up, datacube, method=None): + indexes_between_ranges = self.find_standard_indices_between(indexes_ranges, low, up, datacube, method) + for transformation in self.transformations[::-1]: + indexes_between_ranges = transformation.find_indices_between( + indexes_ranges, low, up, datacube, method, indexes_between_ranges, self + ) return indexes_between_ranges @staticmethod @@ -120,15 +155,22 @@ def check_axis_type(name, values): raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") -@reverse -@cyclic -@mapper -@type_change +transformations_order = [ + DatacubeAxisMerger, + DatacubeAxisReverse, + DatacubeAxisCyclic, + DatacubeMapper, + DatacubeAxisTypeChange, +] +transformations_order = {key: i for i, key in enumerate(transformations_order)} + + class IntDatacubeAxis(DatacubeAxis): def __init__(self): self.name = None self.tol = 1e-12 self.range = None + # TODO: Maybe here, store transformations as a dico instead self.transformations = [] self.type = 0 @@ -145,10 +187,6 @@ def serialize(self, value): return value -@reverse -@cyclic -@mapper -@type_change class FloatDatacubeAxis(DatacubeAxis): def __init__(self): self.name = None @@ -170,7 +208,6 @@ def serialize(self, value): return value -@merge class PandasTimestampDatacubeAxis(DatacubeAxis): def __init__(self): self.name = None @@ -200,7 +237,6 @@ def offset(self, value): return None -@merge class PandasTimedeltaDatacubeAxis(DatacubeAxis): def __init__(self): self.name = None @@ -230,7 +266,6 @@ def offset(self, value): return None -@type_change class UnsliceableDatacubeAxis(DatacubeAxis): def __init__(self): self.name = None diff --git a/polytope/datacube/transformations/datacube_cyclic/__init__.py b/polytope/datacube/transformations/datacube_cyclic/__init__.py new file mode 100644 index 000000000..4d172088d --- /dev/null +++ b/polytope/datacube/transformations/datacube_cyclic/__init__.py @@ -0,0 +1 @@ +from .datacube_cyclic import * diff --git a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py deleted file mode 100644 index 972d6d1aa..000000000 --- a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py +++ /dev/null @@ -1,189 +0,0 @@ -import bisect -import math -from copy import deepcopy -from typing import List - -from .datacube_cyclic import DatacubeAxisCyclic - - -def cyclic(cls): - if cls.is_cyclic: - - def update_range(): - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisCyclic): - transformation = transform - cls.range = transformation.range - - def to_intervals(range): - update_range() - if range[0] == -math.inf: - range[0] = cls.range[0] - if range[1] == math.inf: - range[1] = cls.range[1] - axis_lower = cls.range[0] - axis_upper = cls.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - intervals = [] - if lower < axis_upper: - # In this case, we want to go from lower to the first remapped cyclic axis upper - # or the asked upper range value. - # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, - # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper - # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], - # where -180 is the asked upper range value. - loops = int((axis_upper - lower) / axis_range) - remapped_up = axis_upper - (loops) * axis_range - new_upper = min(upper, remapped_up) - else: - # In this case, since lower >= axis_upper, we need to either go to the asked upper range - # or we need to go to the first remapped cyclic axis upper which is higher than lower - new_upper = min(axis_upper + axis_range, upper) - while new_upper < lower: - new_upper = min(new_upper + axis_range, upper) - intervals.append([lower, new_upper]) - # Now that we have established what the first interval should be, we should just jump from cyclic range - # to cyclic range until we hit the asked upper range value. - new_up = deepcopy(new_upper) - while new_up < upper: - new_upper = new_up - new_up = min(upper, new_upper + axis_range) - intervals.append([new_upper, new_up]) - # Once we have added all the in-between ranges, we need to add the last interval - intervals.append([new_up, upper]) - return intervals - - def _remap_range_to_axis_range(range): - update_range() - axis_lower = cls.range[0] - axis_upper = cls.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - if lower < axis_lower: - # In this case we need to calculate the number of loops between the axis lower - # and the lower to recenter the lower - loops = int((axis_lower - lower - cls.tol) / axis_range) - return_lower = lower + (loops + 1) * axis_range - return_upper = upper + (loops + 1) * axis_range - elif lower >= axis_upper: - # In this case we need to calculate the number of loops between the axis upper - # and the lower to recenter the lower - loops = int((lower - axis_upper) / axis_range) - return_lower = lower - (loops + 1) * axis_range - return_upper = upper - (loops + 1) * axis_range - else: - # In this case, the lower value is already in the right range - return_lower = lower - return_upper = upper - return [return_lower, return_upper] - - def _remap_val_to_axis_range(value): - return_range = _remap_range_to_axis_range([value, value]) - return return_range[0] - - def remap(range: List): - update_range() - if cls.range[0] - cls.tol <= range[0] <= cls.range[1] + cls.tol: - if cls.range[0] - cls.tol <= range[1] <= cls.range[1] + cls.tol: - # If we are already in the cyclic range, return it - return [range] - elif abs(range[0] - range[1]) <= 2 * cls.tol: - # If we have a range that is just one point, then it should still be counted - # and so we should take a small interval around it to find values inbetween - range = [ - _remap_val_to_axis_range(range[0]) - cls.tol, - _remap_val_to_axis_range(range[0]) + cls.tol, - ] - return [range] - range_intervals = cls.to_intervals(range) - ranges = [] - for interval in range_intervals: - if abs(interval[0] - interval[1]) > 0: - # If the interval is not just a single point, we remap it to the axis range - range = _remap_range_to_axis_range([interval[0], interval[1]]) - up = range[1] - low = range[0] - if up < low: - # Make sure we remap in the right order - ranges.append([up - cls.tol, low + cls.tol]) - else: - ranges.append([low - cls.tol, up + cls.tol]) - return ranges - - old_find_indexes = cls.find_indexes - - def find_indexes(path, datacube): - return old_find_indexes(path, datacube) - - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path, unwanted_path): - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisCyclic): - if cls.name == transform.name: - new_val = _remap_val_to_axis_range(value) - key_value_path[cls.name] = new_val - key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) - return (key_value_path, leaf_path, unwanted_path) - - old_unmap_to_datacube = cls.unmap_to_datacube - - def unmap_to_datacube(path, unmapped_path): - (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) - return (path, unmapped_path) - - old_find_indices_between = cls.find_indices_between - - def find_indices_between(index_ranges, low, up, datacube, method=None): - update_range() - indexes_between_ranges = [] - - if method != "surrounding" or method != "nearest": - return old_find_indices_between(index_ranges, low, up, datacube, method) - else: - for indexes in index_ranges: - if cls.name in datacube.complete_axes: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - else: - start = bisect.bisect_left(indexes, low) - end = bisect.bisect_right(indexes, up) - - if start - 1 < 0: - index_val_found = indexes[-1:][0] - indexes_between_ranges.append([index_val_found]) - if end + 1 > len(indexes): - index_val_found = indexes[:2][0] - indexes_between_ranges.append([index_val_found]) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - if cls.name in datacube.complete_axes: - indexes_between = indexes[start:end].to_list() - else: - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def offset(range): - # We first unpad the range by the axis tolerance to make sure that - # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. - # Also, it's safer that we find the offset of a value inside the range instead of on the border - unpadded_range = [range[0] + 1.5 * cls.tol, range[1] - 1.5 * cls.tol] - cyclic_range = _remap_range_to_axis_range(unpadded_range) - offset = unpadded_range[0] - cyclic_range[0] - return offset - - cls.to_intervals = to_intervals - cls.remap = remap - cls.offset = offset - cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube - cls.find_indices_between = find_indices_between - cls.unmap_path_key = unmap_path_key - cls._remap_val_to_axis_range = _remap_val_to_axis_range - - return cls diff --git a/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py index 86113aa2d..b0dd6e1d0 100644 --- a/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py @@ -1,3 +1,7 @@ +import math +from copy import deepcopy + +from ....utility.combinatorics import unique from ..datacube_transformations import DatacubeAxisTransformation @@ -23,3 +27,145 @@ def blocked_axes(self): def unwanted_axes(self): return [] + + def update_range(self, axis): + axis.range = self.range + + def _remap_range_to_axis_range(self, range, axis): + self.update_range(axis) + axis_lower = axis.range[0] + axis_upper = axis.range[1] + axis_range = axis_upper - axis_lower + lower = range[0] + upper = range[1] + if lower < axis_lower: + # In this case we need to calculate the number of loops between the axis lower + # and the lower to recenter the lower + loops = int((axis_lower - lower - axis.tol) / axis_range) + return_lower = lower + (loops + 1) * axis_range + return_upper = upper + (loops + 1) * axis_range + elif lower >= axis_upper: + # In this case we need to calculate the number of loops between the axis upper + # and the lower to recenter the lower + loops = int((lower - axis_upper) / axis_range) + return_lower = lower - (loops + 1) * axis_range + return_upper = upper - (loops + 1) * axis_range + else: + # In this case, the lower value is already in the right range + return_lower = lower + return_upper = upper + return [return_lower, return_upper] + + def _remap_val_to_axis_range(self, value, axis): + value = self._remap_range_to_axis_range([value, value], axis) + return value[0] + + def offset(self, range, axis, offset): + # We first unpad the range by the axis tolerance to make sure that + # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. + # Also, it's safer that we find the offset of a value inside the range instead of on the border + unpadded_range = [range[0] + 1.5 * axis.tol, range[1] - 1.5 * axis.tol] + cyclic_range = self._remap_range_to_axis_range(unpadded_range, axis) + offset = unpadded_range[0] - cyclic_range[0] + return offset + + def remap(self, range, ranges, axis): + self.update_range(axis) + if axis.range[0] - axis.tol <= range[0] <= axis.range[1] + axis.tol: + if axis.range[0] - axis.tol <= range[1] <= axis.range[1] + axis.tol: + # If we are already in the cyclic range, return it + return [range] + elif abs(range[0] - range[1]) <= 2 * axis.tol: + # If we have a range that is just one point, then it should still be counted + # and so we should take a small interval around it to find values inbetween + range = [ + self._remap_val_to_axis_range(range[0], axis) - axis.tol, + self._remap_val_to_axis_range(range[0], axis) + axis.tol, + ] + return [range] + range_intervals = self.to_intervals(range, [[]], axis) + ranges = [] + for interval in range_intervals: + if abs(interval[0] - interval[1]) > 0: + # If the interval is not just a single point, we remap it to the axis range + range = self._remap_range_to_axis_range([interval[0], interval[1]], axis) + up = range[1] + low = range[0] + if up < low: + # Make sure we remap in the right order + ranges.append([up - axis.tol, low + axis.tol]) + else: + ranges.append([low - axis.tol, up + axis.tol]) + return ranges + + def to_intervals(self, range, intervals, axis): + self.update_range(axis) + if range[0] == -math.inf: + range[0] = axis.range[0] + if range[1] == math.inf: + range[1] = axis.range[1] + axis_lower = axis.range[0] + axis_upper = axis.range[1] + axis_range = axis_upper - axis_lower + lower = range[0] + upper = range[1] + intervals = [] + if lower < axis_upper: + # In this case, we want to go from lower to the first remapped cyclic axis upper + # or the asked upper range value. + # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, + # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper + # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], + # where -180 is the asked upper range value. + loops = int((axis_upper - lower) / axis_range) + remapped_up = axis_upper - (loops) * axis_range + new_upper = min(upper, remapped_up) + else: + # In this case, since lower >= axis_upper, we need to either go to the asked upper range + # or we need to go to the first remapped cyclic axis upper which is higher than lower + new_upper = min(axis_upper + axis_range, upper) + while new_upper < lower: + new_upper = min(new_upper + axis_range, upper) + intervals.append([lower, new_upper]) + # Now that we have established what the first interval should be, we should just jump from cyclic range + # to cyclic range until we hit the asked upper range value. + new_up = deepcopy(new_upper) + while new_up < upper: + new_upper = new_up + new_up = min(upper, new_upper + axis_range) + intervals.append([new_upper, new_up]) + # Once we have added all the in-between ranges, we need to add the last interval + intervals.append([new_up, upper]) + return intervals + + def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): + search_ranges = self.remap([low, up], [], axis) + original_search_ranges = self.to_intervals([low, up], [], axis) + # Find the offsets for each interval in the requested range, which we will need later + search_ranges_offset = [] + for r in original_search_ranges: + offset = self.offset(r, axis, 0) + search_ranges_offset.append(offset) + idx_between = [] + for i in range(len(search_ranges)): + r = search_ranges[i] + offset = search_ranges_offset[i] + low = r[0] + up = r[1] + indexes_between = axis.find_standard_indices_between(indexes_ranges, low, up, datacube, method) + # Now the indexes_between are values on the cyclic range so need to remap them to their original + # values before returning them + # if we have a special indexes between range that needs additional offset, treat it here + if len(indexes_between) == 0: + idx_between = idx_between + else: + for k in range(len(indexes_between)): + if offset is None: + indexes_between[k] = indexes_between[k] + else: + indexes_between[k] = round(indexes_between[k] + offset, int(-math.log10(axis.tol))) + idx_between.append(indexes_between[k]) + if offset is not None: + # Note that we can only do unique if not dealing with time values + idx_between = unique(idx_between) + return idx_between diff --git a/polytope/datacube/transformations/datacube_mappers/__init__.py b/polytope/datacube/transformations/datacube_mappers/__init__.py new file mode 100644 index 000000000..3cc477326 --- /dev/null +++ b/polytope/datacube/transformations/datacube_mappers/__init__.py @@ -0,0 +1 @@ +from .datacube_mappers import * diff --git a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py index 33bc615e9..7a681a783 100644 --- a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py @@ -75,6 +75,29 @@ def unmap_first_val_to_start_line_idx(self, first_val): def unmap(self, first_val, second_val, unmapped_idx=None): return self._final_transformation.unmap(first_val, second_val, unmapped_idx) + def find_modified_indexes(self, indexes, path, datacube, axis): + if axis.name == self._mapped_axes()[0]: + return self.first_axis_vals() + if axis.name == self._mapped_axes()[1]: + first_val = path[self._mapped_axes()[0]] + return self.second_axis_vals(first_val) + + def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): + value = key_value_path[axis.name] + if axis.name == self._mapped_axes()[0]: + unwanted_val = key_value_path[self._mapped_axes()[0]] + unwanted_path[axis.name] = unwanted_val + if axis.name == self._mapped_axes()[1]: + first_val = unwanted_path[self._mapped_axes()[0]] + unmapped_idx = leaf_path.get("result", None) + unmapped_idx = self.unmap(first_val, value, unmapped_idx) + print("THE UNMAPPED IDX IS NOT NONE?") + print(unmapped_idx) + leaf_path.pop(self._mapped_axes()[0], None) + key_value_path.pop(axis.name) + key_value_path[self.old_axis] = unmapped_idx + return (key_value_path, leaf_path, unwanted_path) + _type_to_datacube_mapper_lookup = { "octahedral": "OctahedralGridMapper", diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_types/__init__.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/__init__.py new file mode 100644 index 000000000..ba9a7b339 --- /dev/null +++ b/polytope/datacube/transformations/datacube_mappers/mapper_types/__init__.py @@ -0,0 +1,5 @@ +from .healpix import * +from .local_regular import * +from .octahedral import * +from .reduced_ll import * +from .regular import * diff --git a/polytope/datacube/transformations/datacube_merger/__init__.py b/polytope/datacube/transformations/datacube_merger/__init__.py new file mode 100644 index 000000000..085c4dfe1 --- /dev/null +++ b/polytope/datacube/transformations/datacube_merger/__init__.py @@ -0,0 +1 @@ +from .datacube_merger import * diff --git a/polytope/datacube/transformations/datacube_merger/datacube_merger.py b/polytope/datacube/transformations/datacube_merger/datacube_merger.py index 310036f04..91d680197 100644 --- a/polytope/datacube/transformations/datacube_merger/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger/datacube_merger.py @@ -71,3 +71,16 @@ def unmerge(self, merged_val): def change_val_type(self, axis_name, values): new_values = pd.to_datetime(values) return new_values + + def find_modified_indexes(self, indexes, path, datacube, axis): + if axis.name == self._first_axis: + return self.merged_values(datacube) + + def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): + new_key_value_path = {} + value = key_value_path[axis.name] + if axis.name == self._first_axis: + (first_val, second_val) = self.unmerge(value) + new_key_value_path[self._first_axis] = first_val + new_key_value_path[self._second_axis] = second_val + return (new_key_value_path, leaf_path, unwanted_path) diff --git a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py deleted file mode 100644 index 16d816391..000000000 --- a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py +++ /dev/null @@ -1,77 +0,0 @@ -import bisect - -from .datacube_merger import DatacubeAxisMerger - - -def merge(cls): - if cls.has_merger: - - def find_indexes(path, datacube): - # first, find the relevant transformation object that is a mapping in the cls.transformation dico - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name == transformation._first_axis: - return transformation.merged_values(datacube) - - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path, unwanted_path): - key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) - new_key_value_path = {} - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - if cls.name == transform._first_axis: - (first_val, second_val) = transform.unmerge(value) - new_key_value_path[transform._first_axis] = first_val - new_key_value_path[transform._second_axis] = second_val - return (new_key_value_path, leaf_path, unwanted_path) - - old_unmap_to_datacube = cls.unmap_to_datacube - - def unmap_to_datacube(path, unmapped_path): - (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name == transformation._first_axis: - old_val = path.get(cls.name, None) - (first_val, second_val) = transformation.unmerge(old_val) - path.pop(cls.name, None) - path[transformation._first_axis] = first_val - path[transformation._second_axis] = second_val - return (path, unmapped_path) - - def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping - indexes_between_ranges = [] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name in transformation._mapped_axes(): - for indexes in index_ranges: - if method == "surrounding" or method == "nearest": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def remap(range): - return [range] - - cls.remap = remap - cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube - cls.find_indices_between = find_indices_between - cls.unmap_path_key = unmap_path_key - - return cls diff --git a/polytope/datacube/transformations/datacube_null_transformation/datacube_null_transformation.py b/polytope/datacube/transformations/datacube_null_transformation/datacube_null_transformation.py deleted file mode 100644 index 43dccbbed..000000000 --- a/polytope/datacube/transformations/datacube_null_transformation/datacube_null_transformation.py +++ /dev/null @@ -1,22 +0,0 @@ -from ..datacube_transformations import DatacubeAxisTransformation - - -class DatacubeNullTransformation(DatacubeAxisTransformation): - def __init__(self, name, mapper_options): - self.name = name - self.transformation_options = mapper_options - - def generate_final_transformation(self): - return self - - def transformation_axes_final(self): - return [self.name] - - def change_val_type(self, axis_name, values): - return values - - def blocked_axes(self): - return [] - - def unwanted_axes(self): - return [] diff --git a/polytope/datacube/transformations/datacube_null_transformation/null_axis_decorator.py b/polytope/datacube/transformations/datacube_null_transformation/null_axis_decorator.py deleted file mode 100644 index 10e2644df..000000000 --- a/polytope/datacube/transformations/datacube_null_transformation/null_axis_decorator.py +++ /dev/null @@ -1,22 +0,0 @@ -def null(cls): - if cls.type_change: - old_find_indexes = cls.find_indexes - - def find_indexes(path, datacube): - return old_find_indexes(path, datacube) - - def find_indices_between(index_ranges, low, up, datacube, method=None): - indexes_between_ranges = [] - for indexes in index_ranges: - indexes_between = [i for i in indexes if low <= i <= up] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def remap(range): - return [range] - - cls.remap = remap - cls.find_indexes = find_indexes - cls.find_indices_between = find_indices_between - - return cls diff --git a/polytope/datacube/transformations/datacube_reverse/__init__.py b/polytope/datacube/transformations/datacube_reverse/__init__.py new file mode 100644 index 000000000..913c2c9bb --- /dev/null +++ b/polytope/datacube/transformations/datacube_reverse/__init__.py @@ -0,0 +1 @@ +from .datacube_reverse import * diff --git a/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py index 6e60de872..eed563c9e 100644 --- a/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py @@ -1,3 +1,4 @@ +from ....utility.list_tools import bisect_left_cmp, bisect_right_cmp from ..datacube_transformations import DatacubeAxisTransformation @@ -20,3 +21,45 @@ def blocked_axes(self): def unwanted_axes(self): return [] + + def find_modified_indexes(self, indexes, path, datacube, axis): + if axis.name in datacube.complete_axes: + ordered_indices = indexes.sort_values() + else: + ordered_indices = indexes + return ordered_indices + + def find_indices_between(self, indexes, low, up, datacube, method, indexes_between_ranges, axis): + indexes_between_ranges = [] + if axis.name == self.name: + if axis.name in datacube.complete_axes: + # Find the range of indexes between lower and upper + # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html + # Assumes the indexes are already sorted (could sort to be sure) and monotonically + # increasing + if method == "surrounding" or method == "nearest": + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.extend(indexes_between) + else: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.extend(indexes_between) + else: + if method == "surrounding" or method == "nearest": + end_idx = bisect_left_cmp(indexes, low, cmp=lambda x, y: x > y) + 1 + start_idx = bisect_right_cmp(indexes, up, cmp=lambda x, y: x > y) + start = max(start_idx - 1, 0) + end = min(end_idx + 1, len(indexes)) + indexes_between = indexes[start:end] + indexes_between_ranges.extend(indexes_between) + else: + end_idx = bisect_left_cmp(indexes, low, cmp=lambda x, y: x > y) + 1 + start_idx = bisect_right_cmp(indexes, up, cmp=lambda x, y: x > y) + indexes_between = indexes[start_idx:end_idx] + indexes_between_ranges.extend(indexes_between) + return indexes_between_ranges diff --git a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py deleted file mode 100644 index 18bc8bd63..000000000 --- a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py +++ /dev/null @@ -1,66 +0,0 @@ -import bisect - -from .datacube_reverse import DatacubeAxisReverse - - -def reverse(cls): - if cls.reorder: - - def find_indexes(path, datacube): - # first, find the relevant transformation object that is a mapping in the cls.transformation dico - subarray = datacube.dataarray.sel(path, method="nearest") - unordered_indices = datacube.datacube_natural_indexes(cls, subarray) - if cls.name in datacube.complete_axes: - ordered_indices = unordered_indices.sort_values() - else: - ordered_indices = unordered_indices - return ordered_indices - - def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping - indexes_between_ranges = [] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisReverse): - transformation = transform - if cls.name == transformation.name: - for indexes in index_ranges: - if cls.name in datacube.complete_axes: - # Find the range of indexes between lower and upper - # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html - # Assumes the indexes are already sorted (could sort to be sure) and monotonically - # increasing - if method == "surrounding" or method == "nearest": - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end].to_list() - indexes_between_ranges.append(indexes_between) - else: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - indexes_between = indexes[start:end].to_list() - indexes_between_ranges.append(indexes_between) - else: - if method == "surrounding" or method == "nearest": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def remap(range): - return [range] - - cls.remap = remap - cls.find_indexes = find_indexes - cls.find_indices_between = find_indices_between - - return cls diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 2077f3466..dabf4ff9d 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -4,6 +4,9 @@ class DatacubeAxisTransformation(ABC): + def __init__(self): + self.parent = None + @staticmethod def create_transform(name, transformation_type_key, transformation_options): transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] @@ -43,6 +46,27 @@ def transformation_axes_final(self): def change_val_type(self, axis_name, values): pass + def find_modified_indexes(self, indexes, path, datacube, axis): + return indexes + + def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): + return (key_value_path, leaf_path, unwanted_path) + + def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): + return indexes_between_ranges + + def _remap_val_to_axis_range(self, value, axis): + return value + + def offset(self, range, axis, offset): + return offset + + def remap(self, range, ranges, axis): + return ranges + + def to_intervals(self, range, intervals, axis): + return intervals + _type_to_datacube_transformation_lookup = { "mapper": "DatacubeMapper", @@ -50,7 +74,6 @@ def change_val_type(self, axis_name, values): "merge": "DatacubeAxisMerger", "reverse": "DatacubeAxisReverse", "type_change": "DatacubeAxisTypeChange", - "null": "DatacubeNullTransformation", } _type_to_transformation_file_lookup = { @@ -59,7 +82,6 @@ def change_val_type(self, axis_name, values): "merge": "merger", "reverse": "reverse", "type_change": "type_change", - "null": "null_transformation", } has_transform = { @@ -68,5 +90,4 @@ def change_val_type(self, axis_name, values): "merge": "has_merger", "reverse": "reorder", "type_change": "type_change", - "null": "null", } diff --git a/polytope/datacube/transformations/datacube_type_change/__init__.py b/polytope/datacube/transformations/datacube_type_change/__init__.py new file mode 100644 index 000000000..a209dcfb6 --- /dev/null +++ b/polytope/datacube/transformations/datacube_type_change/__init__.py @@ -0,0 +1 @@ +from .datacube_type_change import * diff --git a/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py index 8ba2ef0f7..137ed8a40 100644 --- a/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py @@ -37,6 +37,17 @@ def blocked_axes(self): def unwanted_axes(self): return [] + def find_modified_indexes(self, indexes, path, datacube, axis): + if axis.name == self.name: + return self.change_val_type(axis.name, indexes) + + def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): + value = key_value_path[axis.name] + if axis.name == self.name: + unchanged_val = self.make_str(value) + key_value_path[axis.name] = unchanged_val + return (key_value_path, leaf_path, unwanted_path) + class TypeChangeStrToInt(DatacubeAxisTypeChange): def __init__(self, axis_name, new_type): diff --git a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py deleted file mode 100644 index 0e3669825..000000000 --- a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py +++ /dev/null @@ -1,73 +0,0 @@ -import bisect - -from .datacube_type_change import DatacubeAxisTypeChange - - -def type_change(cls): - if cls.type_change: - old_find_indexes = cls.find_indexes - - def find_indexes(path, datacube): - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - original_vals = old_find_indexes(path, datacube) - return transformation.change_val_type(cls.name, original_vals) - - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path, unwanted_path): - key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - if cls.name == transform.name: - unchanged_val = transform.make_str(value) - key_value_path[cls.name] = unchanged_val - return (key_value_path, leaf_path, unwanted_path) - - def unmap_to_datacube(path, unmapped_path): - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - changed_val = path.get(cls.name, None) - unchanged_val = transformation.make_str(changed_val) - if cls.name in path: - path.pop(cls.name, None) - unmapped_path[cls.name] = unchanged_val - return (path, unmapped_path) - - def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping - indexes_between_ranges = [] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - for indexes in index_ranges: - if method == "surrounding" or method == "nearest": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def remap(range): - return [range] - - cls.remap = remap - cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube - cls.find_indices_between = find_indices_between - cls.unmap_path_key = unmap_path_key - - return cls diff --git a/polytope/engine/hullslicer.py b/polytope/engine/hullslicer.py index 8e7174591..0aa885026 100644 --- a/polytope/engine/hullslicer.py +++ b/polytope/engine/hullslicer.py @@ -46,7 +46,11 @@ def _build_unsliceable_child(self, polytope, ax, node, datacube, lower, next_nod next_nodes.append(child) else: # raise a value not found error - raise ValueError() + errmsg = ( + f"Datacube does not have expected index {lower} of type {type(lower)}" + f"on {ax.name} along the path {path}" + ) + raise ValueError(errmsg) def _build_sliceable_child(self, polytope, ax, node, datacube, lower, upper, next_nodes, slice_axis_idx): tol = ax.tol @@ -152,20 +156,15 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): if node.axis.name == datacube.axis_with_identical_structure_after: stored_val = node.value cached_node = node - # logging.info("Caching number 1") elif node.axis.name == datacube.axis_with_identical_structure_after and node.value != stored_val: repeated_sub_nodes.append(node) del node["unsliced_polytopes"] - # logging.info(f"Skipping number {node.value}") continue self._build_branch(ax, node, datacube, next_nodes) current_nodes = next_nodes - # logging.info("=== BEFORE COPYING ===") - for n in repeated_sub_nodes: - # logging.info(f"Copying children for number {n.value}") n.copy_children_from_other(cached_node) request.merge(r) diff --git a/polytope/utility/geometry.py b/polytope/utility/geometry.py index 2c88d9655..2906fed19 100644 --- a/polytope/utility/geometry.py +++ b/polytope/utility/geometry.py @@ -2,8 +2,7 @@ def lerp(a, b, value): - direction = [a - b for a, b in zip(a, b)] - intersect = [b + value * d for b, d in zip(b, direction)] + intersect = [b + (a - b) * value for a, b in zip(a, b)] return intersect diff --git a/polytope/version.py b/polytope/version.py index 7863915fa..976498ab9 100644 --- a/polytope/version.py +++ b/polytope/version.py @@ -1 +1 @@ -__version__ = "1.0.2" +__version__ = "1.0.3" diff --git a/readme.md b/readme.md index 48dba68f6..2f3ec4390 100644 --- a/readme.md +++ b/readme.md @@ -22,48 +22,41 @@ Documentation

-Polytope is a library for extracting complex data from datacubes. It provides an API for non-orthogonal access to data, where the stencil used to extract data from the datacube can be any arbitrary n-dimensional polygon (called a *polytope*). This can be used to efficiently extract complex features from a datacube, such as polygon regions or spatio-temporal paths. +Polytope is a library for extracting complex data from datacubes. It provides an API for non-orthogonal access to data, where the stencil used to extract data from the datacube can be any arbitrary *n*-dimensional polygon (called a *polytope*). This can be used to efficiently extract complex features from a datacube, such as polygon regions or spatio-temporal paths. -Polytope is designed to extend different datacube backends. -* Xarray dataarrays -* FDB object stores (coming soon) +Polytope is designed to extend different datacube backends: +* XArray dataarrays +* FDB object stores (through the GribJump software) -Polytope supports datacubes which have branching, non-uniform indexing, and even cyclic axes. If the datacube backend supports byte-addressability and efficient random access (either in-memory or direct from storage), *polytope* can be used to dramatically decrease overall I/O load. +Polytope supports datacubes which have branching, non-uniform indexing, and even cyclic axes. If the datacube backend supports byte-addressability and efficient random access (either in-memory or direct from storage), **Polytope** can be used to dramatically decrease overall I/O load. - -| :warning: This project is BETA and will be experimental for the foreseeable future. Interfaces and functionality are likely to change. DO NOT use this software in any project/software that is operational. | -|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +> [!WARNING] +> This project is BETA and will be experimental for the forseable future. Interfaces and functionality are likely to change, and the project itself may be scrapped. DO NOT use this software in any project/software that is operational. ## Concept -Polytope is designed to enable extraction of arbitrary extraction of data from a datacube. Instead of the typical range-based bounding-box approach, Polytope can extract any shape of data from a datacube using a "polytope" (n-dimensional polygon) stencil. +Polytope is designed to enable extraction of arbitrary extraction of data from a datacube. Instead of the typical range-based bounding-box approach, Polytope can extract any shape of data from a datacube using a "polytope" (*n*-dimensional polygon) stencil. -

Polytope Concept

- The Polytope algorithm can for example be used to extract: - 2D cut-outs, such as country cut-outs, from a datacube -

Greece cut-out

- - timeseries from a datacube

Timeseries

- - more complicated spatio-temporal paths, such as flight paths, from a datacube

Flight path

- - and many more high-dimensional shapes in arbitrary dimensions... @@ -93,11 +86,11 @@ Here is a step-by-step example of how to use this software. We then construct the Polytope object, passing in some additional metadata describing properties of the longitude axis. ```Python - options = {"longitude": {"Cyclic": [0, 360.0]}} + options = {"longitude": {"cyclic": [0, 360.0]}} from polytope.polytope import Polytope - p = Polytope(datacube=array, options=options) + p = Polytope(datacube=array, axis_options=options) ``` 2. Next, we create a request shape to extract from the datacube. @@ -139,25 +132,12 @@ Here is a step-by-step example of how to use this software. ↳longitude=1.0 ``` - - ## Testing -#### Git Large File Storage - -Polytope uses Git Large File Storage (LFS) to store large data files used in its tests and examples. -To run the tests and examples, it is thus necessary to install Git LFS, by following instructions provided [here](https://docs.github.com/en/repositories/working-with-files/managing-large-files/installing-git-large-file-storage) for example. -Once Git LFS is installed, individual data files can be downloaded using the command - - git lfs pull --include="*" --exclude="" - #### Additional Dependencies The Polytope tests and examples require additional Python packages compared to the main Polytope algorithm. -The additional dependencies are provided in the requirements_test.txt and requirements_examples.txt files, which can respectively be found in the examples and tests folders. +The additional dependencies are provided in the requirements_test.txt and requirements_examples.txt files, which can respectively be found in the tests and examples folders. Moreover, Polytope's tests and examples also require the installation of eccodes and GDAL. It is possible to install both of these dependencies using either a package manager or manually. diff --git a/requirements.txt b/requirements.txt index a6c7f3b39..afbacd4ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -numpy==1.23.5 -pandas==1.5.2 -scipy==1.9.3 -sortedcontainers==2.4.0 -tripy==1.0.0 -xarray==2022.12.0 +numpy +pandas +scipy +sortedcontainers +tripy +xarray diff --git a/requirements_example.txt b/requirements_example.txt new file mode 100644 index 000000000..e69de29bb diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index e09e60e35..0df9637b0 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -1,8 +1,8 @@ -r ../requirements.txt -pytest==7.2.0 -cffi==1.15.1 -eccodes==1.5.2 -h5netcdf==1.1.0 -h5py==3.8.0 -earthkit==0.0.1 -earthkit-data==0.1.3 \ No newline at end of file +pytest +cffi +eccodes +h5netcdf +h5py +earthkit +earthkit-data \ No newline at end of file diff --git a/tests/test_axis_mappers.py b/tests/test_axis_mappers.py index 991dbb1f0..6196251e8 100644 --- a/tests/test_axis_mappers.py +++ b/tests/test_axis_mappers.py @@ -1,6 +1,7 @@ import pandas as pd from polytope.datacube.datacube_axis import ( + DatacubeAxisCyclic, FloatDatacubeAxis, IntDatacubeAxis, PandasTimedeltaDatacubeAxis, @@ -31,30 +32,40 @@ def test_float_axis(self): def test_float_axis_cyclic(self): axis = FloatDatacubeAxis() axis.is_cyclic = True - axis = axis.update_axis() assert axis.parse(2) == 2.0 assert axis.to_float(2) == 2.0 assert axis.from_float(2) == 2.0 assert axis.serialize(2.0) == 2.0 + transformation = DatacubeAxisCyclic("", {}) # Test the to_intervals function - axis.range = [1, 3] - assert axis.to_intervals([4, 7]) == [[4, 5], [5, 7], [7, 7]] + transformation.range = [1, 3] + assert transformation.to_intervals([4, 7], [[]], axis) == [[4, 5], [5, 7], [7, 7]] # Test the cyclic_remap function - axis.range = [0, 1] - assert axis.remap([0, 2]) == [[-1e-12, 1.000000000001], [-1e-12, 1.000000000001]] - axis.range = [1, 2] - assert axis.remap([1, 3]) == [[0.999999999999, 2.000000000001], [0.999999999999, 2.000000000001]] - axis.range = [1, 3] - assert axis.remap([1, 4]) == [[0.999999999999, 3.000000000001], [0.999999999999, 2.000000000001]] - axis.range = [2, 4] - assert axis.remap([0, 5]) == [ + + transformation.range = [0, 1] + assert transformation.remap([0, 2], [[]], axis) == [[-1e-12, 1.000000000001], [-1e-12, 1.000000000001]] + + transformation.range = [1, 2] + assert transformation.remap([1, 3], [[]], axis) == [ + [0.999999999999, 2.000000000001], + [0.999999999999, 2.000000000001], + ] + + transformation.range = [1, 3] + assert transformation.remap([1, 4], [[]], axis) == [ + [0.999999999999, 3.000000000001], + [0.999999999999, 2.000000000001], + ] + + transformation.range = [2, 4] + assert transformation.remap([0, 5], [[]], axis) == [ [1.999999999999, 4.000000000001], [1.999999999999, 4.000000000001], [1.999999999999, 3.000000000001], ] - axis.range = [2.3, 4.6] - remapped = axis.remap([0.3, 5.7]) + transformation.range = [2.3, 4.6] + remapped = transformation.remap([0.3, 5.7], [[]], axis) assert remapped == [ [2.5999999999989996, 4.600000000001], [2.2999999999989997, 4.600000000001], @@ -62,22 +73,22 @@ def test_float_axis_cyclic(self): ] # Test the to_cyclic_value function - axis.range = [1, 3] - remapped = axis.remap([0, 7]) + transformation.range = [1, 3] + remapped = transformation.remap([0, 7], [[]], axis) assert remapped == [ [1.999999999999, 3.000000000001], [0.999999999999, 3.000000000001], [0.999999999999, 3.000000000001], [0.999999999999, 3.000000000001], ] - remapped = axis.remap([-1, 2]) + remapped = transformation.remap([-1, 2], [[]], axis) assert remapped == [[0.999999999999, 3.000000000001], [0.999999999999, 2.000000000001]] # Test the cyclic_offset function - assert axis.offset([3.05, 3.1]) == 2 - assert axis.offset([1.05, 1.1]) == 0 - assert axis.offset([-5.0, -4.95]) == -6 - assert axis.offset([5.05, 5.1]) == 4 + assert transformation.offset([3.05, 3.1], axis, 0) == 2 + assert transformation.offset([1.05, 1.1], axis, 0) == 0 + assert transformation.offset([-5.0, -4.95], axis, 0) == -6 + assert transformation.offset([5.05, 5.1], axis, 0) == 4 def test_timedelta_axis(self): axis = PandasTimedeltaDatacubeAxis() diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index 52cd41c21..c11bcb098 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -2,7 +2,6 @@ import pandas as pd import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -25,7 +24,6 @@ def setup_method(self, method): "long": {"cyclic": [-1.1, -0.1]}, "level": {"cyclic": [1, 129]}, } - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 8266c0a09..526473beb 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -2,7 +2,6 @@ import pandas as pd import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -25,7 +24,6 @@ def setup_method(self, method): "long": {"cyclic": [-1.1, -0.1]}, "level": {"cyclic": [1, 129]}, } - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index c0ac914f5..5a49be0b1 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -2,7 +2,6 @@ import pandas as pd import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -25,7 +24,6 @@ def setup_method(self, method): "long": {"cyclic": [0, 1.0]}, "level": {"cyclic": [1, 129]}, } - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_cyclic_nearest.py b/tests/test_cyclic_nearest.py index 9367ebfea..2dcf6aabd 100644 --- a/tests/test_cyclic_nearest.py +++ b/tests/test_cyclic_nearest.py @@ -23,6 +23,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, "longitude": {"cyclic": [0, 360]}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"} self.datacube_options = {"identical structure after": "number"} diff --git a/tests/test_cyclic_simple.py b/tests/test_cyclic_simple.py index f3bd1e28b..f900cac1b 100644 --- a/tests/test_cyclic_simple.py +++ b/tests/test_cyclic_simple.py @@ -2,7 +2,6 @@ import pandas as pd import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -22,7 +21,6 @@ def setup_method(self, method): }, ) options = {"long": {"cyclic": [0, 1.0]}, "level": {"cyclic": [1, 129]}} - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_cyclic_snapping.py b/tests/test_cyclic_snapping.py index adb3a7959..fa10fbbd3 100644 --- a/tests/test_cyclic_snapping.py +++ b/tests/test_cyclic_snapping.py @@ -1,6 +1,5 @@ import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select @@ -17,7 +16,6 @@ def setup_method(self, method): }, ) options = {"long": {"cyclic": [0, 1.0]}} - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index f3739e5bf..bedaf8af7 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -2,7 +2,6 @@ from earthkit import data from helper_functions import download_test_data -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.datacube.datacube_axis import FloatDatacubeAxis from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request @@ -17,9 +16,9 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m - self.xarraydatacube = XArrayDatacube(latlon_array) self.options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "latitude": {"reverse": {True}}, } self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=self.options) @@ -69,15 +68,16 @@ def test_created_axes(self): assert len(self.datacube._axes["longitude"].find_indexes({"latitude": 89.94618771566562}, self.datacube)) == 20 lon_ax = self.datacube._axes["longitude"] lat_ax = self.datacube._axes["latitude"] - (path, unmapped_path) = lat_ax.unmap_to_datacube({"latitude": 89.94618771566562}, {}) + (path_key, path, unmapped_path) = lat_ax.unmap_path_key({"latitude": 89.94618771566562}, {}, {}) assert path == {} assert unmapped_path == {"latitude": 89.94618771566562} - assert unmapped_path == {"latitude": 89.94618771566562} - (path, unmapped_path) = lon_ax.unmap_to_datacube({"longitude": 0.0}, {"latitude": 89.94618771566562}) + (path_key, path, unmapped_path) = lon_ax.unmap_path_key({"longitude": 0.0}, {}, {"latitude": 89.94618771566562}) assert path == {} - assert unmapped_path == {"values": 0} - assert lat_ax.find_indices_between([[89.94618771566562, 89.87647835333229]], 89.87, 90, self.datacube, 0) == [ - [89.94618771566562, 89.87647835333229] + assert unmapped_path == {"latitude": 89.94618771566562} + assert path_key == {"values": 0} + assert lat_ax.find_indices_between([89.94618771566562, 89.87647835333229], 89.87, 90, self.datacube, 0) == [ + 89.94618771566562, + 89.87647835333229, ] @pytest.mark.internet diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 692924f15..f52d41acf 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -15,6 +15,7 @@ def setup_method(self, method): "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "fc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index bf4452a35..093750926 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -19,6 +19,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, "number": {"type_change": "int"}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_float_type.py b/tests/test_float_type.py index 7fb6f94ab..6593234b3 100644 --- a/tests/test_float_type.py +++ b/tests/test_float_type.py @@ -2,7 +2,6 @@ import pytest import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select, Span @@ -20,7 +19,6 @@ def setup_method(self, method): "alt": np.arange(0.0, 20.0, 0.1), }, ) - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index ea6c4e6f9..8b6121da7 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -2,7 +2,6 @@ from earthkit import data from helper_functions import download_test_data, find_nearest_latlon -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -15,7 +14,6 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/healpix.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(time=0).isel(isobaricInhPa=0).z - self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { "values": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}}, "longitude": {"cyclic": [0, 360]}, diff --git a/tests/test_local_grid_cyclic.py b/tests/test_local_grid_cyclic.py index 025986321..da8b71f25 100644 --- a/tests/test_local_grid_cyclic.py +++ b/tests/test_local_grid_cyclic.py @@ -24,6 +24,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, "longitude": {"cyclic": [-180, 180]}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_local_regular_grid.py b/tests/test_local_regular_grid.py index dfac410e5..62be20962 100644 --- a/tests/test_local_regular_grid.py +++ b/tests/test_local_regular_grid.py @@ -23,6 +23,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, "number": {"type_change": "int"}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_local_swiss_grid.py b/tests/test_local_swiss_grid.py new file mode 100644 index 000000000..69d1984d3 --- /dev/null +++ b/tests/test_local_swiss_grid.py @@ -0,0 +1,89 @@ +# import geopandas as gpd +# import matplotlib.pyplot as plt +import pandas as pd +import pytest +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file + +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": { + "mapper": { + "type": "local_regular", + "resolution": [193, 417], + "axes": ["latitude", "longitude"], + "local": [45.485, 48.1, 5.28985, 10.9087], + } + }, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "levelist": {"type_change": "int"}, + } + + self.config = {"param": "3008"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.fdb + @pytest.mark.skip("Non-accessible data") + def test_fdb_datacube(self): + request = Request( + Select("step", [0]), + Select("levtype", ["unknown"]), + Select("date", [pd.Timestamp("20211102T120000")]), + Select("param", ["3008"]), + Select("levelist", [1]), + Box(["latitude", "longitude"], [47.38, 7], [47.5, 7.14]), + ) + result = self.API.retrieve(request) + # result.pprint_2() + assert len(result.leaves) == 99 + + lats = [] + lons = [] + eccodes_lats = [] + eccodes_lons = [] + tol = 1e-4 + f = open("./tests/data/hhl_geo.grib", "rb") + messages = [] + message = codes_grib_new_from_file(f) + messages.append(message) + + leaves = result.leaves + for i in range(len(leaves)): + cubepath = leaves[i].flatten() + lat = cubepath["latitude"] + lon = cubepath["longitude"] + del cubepath + lats.append(lat) + lons.append(lon) + nearest_points = codes_grib_find_nearest(message, lat, lon)[0] + eccodes_lat = nearest_points.lat + eccodes_lon = nearest_points.lon + eccodes_lats.append(eccodes_lat) + eccodes_lons.append(eccodes_lon) + assert eccodes_lat - tol <= lat + assert lat <= eccodes_lat + tol + assert eccodes_lon - tol <= lon + assert lon <= eccodes_lon + tol + f.close() + + # worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + # fig, ax = plt.subplots(figsize=(12, 6)) + # worldmap.plot(color="darkgrey", ax=ax) + + # plt.scatter(lons, lats, s=18, c="red", cmap="YlOrRd") + # plt.scatter(eccodes_lons, eccodes_lats, s=6, c="green") + # plt.colorbar(label="Temperature") + # plt.show() diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index 2714c36c6..e6cde2689 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -2,7 +2,6 @@ import pytest import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select, Span @@ -27,7 +26,6 @@ def setup_method(self, method): "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, "step": {"cyclic": [0, 2]}, } - self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index 7c3764904..82d95fb22 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -2,7 +2,6 @@ from earthkit import data from helper_functions import download_test_data -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -16,10 +15,10 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/foo.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) self.latlon_array = self.latlon_array.t2m - self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, "longitude": {"cyclic": [0, 360.0]}, + "latitude": {"reverse": {True}}, } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index 28f075d56..71d6f3129 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -2,7 +2,6 @@ import pandas as pd import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select @@ -20,7 +19,6 @@ def setup_method(self, method): }, ) self.options = {"date": {"merge": {"with": "time", "linkers": ["T", "00"]}}} - self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_multiple_param_fdb.py b/tests/test_multiple_param_fdb.py index 04b8e7127..dd1df749e 100644 --- a/tests/test_multiple_param_fdb.py +++ b/tests/test_multiple_param_fdb.py @@ -15,6 +15,7 @@ def setup_method(self, method): "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index adc251684..e041b0e53 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -2,7 +2,6 @@ from earthkit import data from helper_functions import download_test_data, find_nearest_latlon -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -16,9 +15,9 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/foo.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) self.latlon_array = self.latlon_array.t2m - self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "latitude": {"reverse": {True}}, } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_orca_irregular_grid.py b/tests/test_orca_irregular_grid.py index 503daa540..558adafcc 100644 --- a/tests/test_orca_irregular_grid.py +++ b/tests/test_orca_irregular_grid.py @@ -31,9 +31,6 @@ def setup_method(self, method): self.points = list(zip(self.latitudes, self.longitudes)) self.options = { "values": {"mapper": {"type": "irregular", "resolution": 1280, "axes": ["latitude", "longitude"]}}, - # "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, - # "step": {"type_change": "int"}, - # "number": {"type_change": "int"}, } # self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} # self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options, point_cloud_options=self.points) @@ -83,13 +80,13 @@ def test_quad_tree_slicer_extract(self): assert eccodes_lon - tol <= lon assert lon <= eccodes_lon + tol - worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) - fig, ax = plt.subplots(figsize=(12, 6)) - worldmap.plot(color="darkgrey", ax=ax) + # worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + # fig, ax = plt.subplots(figsize=(12, 6)) + # worldmap.plot(color="darkgrey", ax=ax) - plt.scatter(lons, lats, s=18, c="red", cmap="YlOrRd") - plt.scatter(eccodes_lons, eccodes_lats, s=6, c="green") - plt.colorbar(label="Temperature") - plt.show() + # plt.scatter(lons, lats, s=18, c="red", cmap="YlOrRd") + # plt.scatter(eccodes_lons, eccodes_lats, s=6, c="green") + # plt.colorbar(label="Temperature") + # plt.show() diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index fa53b9809..834b42d3f 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -17,6 +17,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, "longitude": {"cyclic": [0, 360]}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -39,7 +40,6 @@ def test_fdb_datacube(self): Point(["latitude", "longitude"], [[0.16, 0.176]], method="nearest"), ) result = self.API.retrieve(request) - # result.pprint() assert len(result.leaves) == 1 @pytest.mark.fdb @@ -98,6 +98,26 @@ def test_fdb_datacube_true_point_3(self): assert result.leaves[0].value == 359.929906542056 assert result.leaves[0].axis.name == "longitude" + @pytest.mark.fdb + def test_fdb_datacube_true_point_5(self): + request = Request( + Select("step", [21]), + Select("levtype", ["sfc"]), + Select("date", [pd.Timestamp("20231102T000000")]), + Select("domain", ["g"]), + Select("expver", ["0001"]), + Select("param", ["167"]), + Select("class", ["od"]), + Select("stream", ["oper"]), + Select("type", ["fc"]), + Point(["latitude", "longitude"], [[0.035149384216, 360 - 0.01]], method="nearest"), + ) + result = self.API.retrieve(request) + result.pprint() + assert len(result.leaves) == 1 + assert result.leaves[0].value == 359.929906542056 + assert result.leaves[0].axis.name == "longitude" + @pytest.mark.fdb def test_fdb_datacube_true_point_4(self): request = Request( @@ -113,7 +133,7 @@ def test_fdb_datacube_true_point_4(self): Point(["latitude", "longitude"], [[0.035149384216, 359.97]], method="nearest"), ) result = self.API.retrieve(request) - result.pprint() + # result.pprint_2() assert len(result.leaves) == 1 assert result.leaves[0].value == 359.929906542056 assert result.leaves[0].axis.name == "longitude" diff --git a/tests/test_point_shape.py b/tests/test_point_shape.py index aa1caf79d..95ce48f99 100644 --- a/tests/test_point_shape.py +++ b/tests/test_point_shape.py @@ -2,7 +2,6 @@ import pandas as pd import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Point, Select @@ -20,7 +19,6 @@ def setup_method(self, method): "level": range(1, 130), }, ) - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index b955ee827..a28f005a8 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -22,6 +22,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, "longitude": {"cyclic": [0, 360]}, + "latitude": {"reverse": {True}}, } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": "0"} self.datacube_options = {"identical structure after": "number"} diff --git a/tests/test_reverse_transformation.py b/tests/test_reverse_transformation.py index e501dfdfc..9e4d6f8b5 100644 --- a/tests/test_reverse_transformation.py +++ b/tests/test_reverse_transformation.py @@ -1,7 +1,6 @@ import numpy as np import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select @@ -18,7 +17,6 @@ def setup_method(self, method): }, ) options = {"lat": {"reverse": {True}}} - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_shapes.py b/tests/test_shapes.py index 092154bce..b0daa8b10 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -47,6 +47,7 @@ def test_all_mapper_cyclic(self): "step": {"type_change": "int"}, "number": {"type_change": "int"}, "longitude": {"cyclic": [0, 360]}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": "11"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 319bc2c24..fae61798e 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -16,6 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, "number": {"type_change": "int"}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_slice_date_range_fdb_v2.py b/tests/test_slice_date_range_fdb_v2.py index 63ce5e678..7fe005450 100644 --- a/tests/test_slice_date_range_fdb_v2.py +++ b/tests/test_slice_date_range_fdb_v2.py @@ -15,6 +15,7 @@ def setup_method(self, method): "values": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}}, "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, + "latitude": {"reverse": {True}}, } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "stream": "enda"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 15d355cb1..f9651edc4 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -3,7 +3,6 @@ from earthkit import data from helper_functions import download_test_data -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -16,7 +15,6 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() options = {"lat": {"reverse": {True}}} self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_slicing_unsliceable_axis.py b/tests/test_slicing_unsliceable_axis.py index 9d02abf76..a5fa1fcca 100644 --- a/tests/test_slicing_unsliceable_axis.py +++ b/tests/test_slicing_unsliceable_axis.py @@ -3,7 +3,6 @@ import pytest import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -18,7 +17,6 @@ def setup_method(self, method): dims=("date", "variable", "level"), coords={"date": pd.date_range("2000-01-01", "2000-01-03", 3), "variable": ["a"], "level": range(1, 130)}, ) - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/tests/test_slicing_xarray_4D.py b/tests/test_slicing_xarray_4D.py index 301bd874c..a19c260d4 100644 --- a/tests/test_slicing_xarray_4D.py +++ b/tests/test_slicing_xarray_4D.py @@ -3,7 +3,6 @@ import pytest import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.datacube.index_tree import IndexTree from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request @@ -34,7 +33,6 @@ def setup_method(self, method): "lat": np.around(np.arange(0.0, 10.0, 0.1), 15), }, ) - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/tests/test_snapping.py b/tests/test_snapping.py index 83d472203..41492f06f 100644 --- a/tests/test_snapping.py +++ b/tests/test_snapping.py @@ -1,7 +1,6 @@ import numpy as np import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select @@ -18,7 +17,6 @@ def setup_method(self, method): "step": [1, 3, 5], }, ) - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index acad24fbc..1f113dfca 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -6,7 +6,6 @@ from earthkit import data from helper_functions import download_test_data -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -19,7 +18,6 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() options = { "latitude": {"reverse": {True}}, diff --git a/tests/test_type_change_transformation.py b/tests/test_type_change_transformation.py index d88a03aca..5291e4180 100644 --- a/tests/test_type_change_transformation.py +++ b/tests/test_type_change_transformation.py @@ -1,7 +1,6 @@ import numpy as np import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select @@ -19,7 +18,6 @@ def setup_method(self, method): ) self.array = array options = {"step": {"type_change": "int"}} - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options)