Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Surrounding method for surrounding points in Select #70

Merged
merged 12 commits into from
Sep 7, 2023
Merged

Surrounding method for surrounding points in Select #70

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
5f39743
start doing surrounding points option
mathleur Aug 30, 2023
63857e3
make surrounding method work for all transformations
mathleur Aug 31, 2023
7917b68
try to make cyclic surrounding method work
mathleur Aug 31, 2023
3bf4f2d
make surrounding method work with cyclic axes
mathleur Sep 1, 2023
2b7cd43
make all tests work
mathleur Sep 5, 2023
ce0661f
try to make cyclic without offsets
mathleur Sep 5, 2023
abc896b
make surrounding method work with the cyclic axis
mathleur Sep 7, 2023
c03b7a5
clean up
mathleur Sep 7, 2023
78306d0
refactor find_indices_between for cyclic axis
mathleur Sep 7, 2023
f04ded5
refactor the surrounding method code in datacube.py _look_up_datacube…
mathleur Sep 7, 2023
f82c8e8
black
mathleur Sep 7, 2023
d0108da
merge with develop
mathleur Sep 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion polytope/datacube/backends/FDB_datacube.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ def update_fdb_dataarray(fdb_dataarray):


class FDBDatacube(Datacube):

def __init__(self, config={}, axis_options={}):
self.axis_options = axis_options
self.grid_mapper = None
Expand Down
34 changes: 20 additions & 14 deletions polytope/datacube/backends/datacube.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,12 @@ def validate(self, axes):

def _create_axes(self, name, values, transformation_type_key, transformation_options):
# first check what the final axes are for this axis name given transformations
final_axis_names = DatacubeAxisTransformation.get_final_axes(name, transformation_type_key,
transformation_options)
transformation = DatacubeAxisTransformation.create_transform(name, transformation_type_key,
transformation_options)
final_axis_names = DatacubeAxisTransformation.get_final_axes(
name, transformation_type_key, transformation_options
)
transformation = DatacubeAxisTransformation.create_transform(
name, transformation_type_key, transformation_options
)
for blocked_axis in transformation.blocked_axes():
self.blocked_axes.append(blocked_axis)
for axis_name in final_axis_names:
Expand Down Expand Up @@ -79,7 +81,7 @@ def fit_path(self, path):
path.pop(key)
return path

def get_indices(self, path: DatacubePath, axis, lower, upper):
def get_indices(self, path: DatacubePath, axis, lower, upper, method=None):
"""
Given a path to a subset of the datacube, return the discrete indexes which exist between
two non-discrete values (lower, upper) for a particular axis (given by label)
Expand All @@ -95,30 +97,34 @@ def get_indices(self, path: DatacubePath, axis, lower, upper):
for r in original_search_ranges:
offset = axis.offset(r)
search_ranges_offset.append(offset)
idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis)
idx_between = self._look_up_datacube(search_ranges, search_ranges_offset, indexes, axis, method)
# Remove duplicates even if difference of the order of the axis tolerance
if offset is not None:
# Note that we can only do unique if not dealing with time values
idx_between = unique(idx_between)
return idx_between

def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis):
def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, method):
idx_between = []
for i in range(len(search_ranges)):
r = search_ranges[i]
offset = search_ranges_offset[i]
low = r[0]
up = r[1]
indexes_between = axis.find_indices_between([indexes], low, up, self)
indexes_between = axis.find_indices_between([indexes], low, up, self, method)
# Now the indexes_between are values on the cyclic range so need to remap them to their original
# values before returning them
for j in range(len(indexes_between)):
for k in range(len(indexes_between[j])):
if offset is None:
indexes_between[j][k] = indexes_between[j][k]
else:
indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol)))
idx_between.append(indexes_between[j][k])
# if we have a special indexes between range that needs additional offset, treat it here
if len(indexes_between[j]) == 0:
idx_between = idx_between
else:
for k in range(len(indexes_between[j])):
if offset is None:
indexes_between[j][k] = indexes_between[j][k]
else:
indexes_between[j][k] = round(indexes_between[j][k] + offset, int(-math.log10(axis.tol)))
idx_between.append(indexes_between[j][k])
return idx_between

def get_mapper(self, axis):
Expand Down
2 changes: 1 addition & 1 deletion polytope/datacube/backends/mock.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ def get(self, requests: IndexTree):
def get_mapper(self, axis):
return self.mappers[axis]

def get_indices(self, path: DatacubePath, axis, lower, upper):
def get_indices(self, path: DatacubePath, axis, lower, upper, method=None):
if lower == upper == math.ceil(lower):
if lower >= 0:
return [int(lower)]
Expand Down
171 changes: 150 additions & 21 deletions polytope/datacube/datacube_axis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,37 @@ def unmap_to_datacube(path, unmapped_path):
(path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path)
return (path, unmapped_path)

old_find_indices_between = cls.find_indices_between

def find_indices_between(index_ranges, low, up, datacube, method=None):
update_range()
indexes_between_ranges = []

if method != "surrounding":
return old_find_indices_between(index_ranges, low, up, datacube, method)
else:
for indexes in index_ranges:
if cls.name in datacube.complete_axes:
start = indexes.searchsorted(low, "left")
end = indexes.searchsorted(up, "right")
else:
start = indexes.index(low)
end = indexes.index(up)
if start - 1 < 0:
index_val_found = indexes[-1:][0]
indexes_between_ranges.append([index_val_found])
if end + 1 > len(indexes):
index_val_found = indexes[:2][0]
indexes_between_ranges.append([index_val_found])
start = max(start - 1, 0)
end = min(end + 1, len(indexes))
if cls.name in datacube.complete_axes:
indexes_between = indexes[start:end].to_list()
else:
indexes_between = indexes[start:end]
indexes_between_ranges.append(indexes_between)
return indexes_between_ranges

def offset(range):
# We first unpad the range by the axis tolerance to make sure that
# we find the wanted range of the cyclic axis since we padded by the axis tolerance before.
Expand All @@ -150,6 +181,7 @@ def offset(range):
cls.find_indexes = find_indexes
cls.unmap_to_datacube = unmap_to_datacube
cls.unmap_total_path_to_datacube = unmap_total_path_to_datacube
cls.find_indices_between = find_indices_between

return cls

Expand All @@ -158,6 +190,7 @@ def mapper(cls):
from .transformations.datacube_mappers import DatacubeMapper

if cls.has_mapper:

def find_indexes(path, datacube):
# first, find the relevant transformation object that is a mapping in the cls.transformation dico
for transform in cls.transformations:
Expand Down Expand Up @@ -232,15 +265,27 @@ def unmap_total_path_to_datacube(path, unmapped_path):
def remap_to_requested(path, unmapped_path):
return (path, unmapped_path)

def find_indices_between(index_ranges, low, up, datacube):
def find_indices_between(index_ranges, low, up, datacube, method=None):
# TODO: add method for snappping
indexes_between_ranges = []
for transform in cls.transformations:
if isinstance(transform, DatacubeMapper):
transformation = transform
if cls.name in transformation._mapped_axes():
for idxs in index_ranges:
indexes_between = [i for i in idxs if low <= i <= up]
indexes_between_ranges.append(indexes_between)
if method == "surrounding":
start = idxs.index(low)
end = idxs.index(up)
start = max(start - 1, 0)
end = min(end + 1, len(idxs))
# indexes_between = [i for i in indexes if low <= i <= up]
indexes_between = idxs[start:end]
indexes_between_ranges.append(indexes_between)
else:
indexes_between = [i for i in idxs if low <= i <= up]
indexes_between_ranges.append(indexes_between)
# indexes_between = [i for i in idxs if low <= i <= up]
# indexes_between_ranges.append(indexes_between)
return indexes_between_ranges

old_remap = cls.remap
Expand All @@ -262,6 +307,7 @@ def merge(cls):
from .transformations.datacube_merger import DatacubeAxisMerger

if cls.has_merger:

def find_indexes(path, datacube):
# first, find the relevant transformation object that is a mapping in the cls.transformation dico
for transform in cls.transformations:
Expand Down Expand Up @@ -303,15 +349,27 @@ def unmap_to_datacube(path, unmapped_path):
def remap_to_requested(path, unmapped_path):
return (path, unmapped_path)

def find_indices_between(index_ranges, low, up, datacube):
def find_indices_between(index_ranges, low, up, datacube, method=None):
# TODO: add method for snappping
indexes_between_ranges = []
for transform in cls.transformations:
if isinstance(transform, DatacubeAxisMerger):
transformation = transform
if cls.name in transformation._mapped_axes():
for indexes in index_ranges:
indexes_between = [i for i in indexes if low <= i <= up]
indexes_between_ranges.append(indexes_between)
if method == "surrounding":
start = indexes.index(low)
end = indexes.index(up)
start = max(start - 1, 0)
end = min(end + 1, len(indexes))
# indexes_between = [i for i in indexes if low <= i <= up]
indexes_between = indexes[start:end]
indexes_between_ranges.append(indexes_between)
else:
indexes_between = [i for i in indexes if low <= i <= up]
indexes_between_ranges.append(indexes_between)
# indexes_between = [i for i in indexes if low <= i <= up]
# indexes_between_ranges.append(indexes_between)
return indexes_between_ranges

def remap(range):
Expand All @@ -331,6 +389,7 @@ def reverse(cls):
from .transformations.datacube_reverse import DatacubeAxisReverse

if cls.reorder:

def find_indexes(path, datacube):
# first, find the relevant transformation object that is a mapping in the cls.transformation dico
subarray = datacube.dataarray.sel(path, method="nearest")
Expand All @@ -347,15 +406,56 @@ def unmap_to_datacube(path, unmapped_path):
def remap_to_requested(path, unmapped_path):
return (path, unmapped_path)

def find_indices_between(index_ranges, low, up, datacube):
def find_indices_between(index_ranges, low, up, datacube, method=None):
# TODO: add method for snappping
indexes_between_ranges = []
for transform in cls.transformations:
if isinstance(transform, DatacubeAxisReverse):
transformation = transform
if cls.name == transformation.name:
for indexes in index_ranges:
indexes_between = [i for i in indexes if low <= i <= up]
indexes_between_ranges.append(indexes_between)
if cls.name in datacube.complete_axes:
# Find the range of indexes between lower and upper
# https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html
# Assumes the indexes are already sorted (could sort to be sure) and monotonically
# increasing
if method == "surrounding":
start = indexes.searchsorted(low, "left")
end = indexes.searchsorted(up, "right")
start = max(start - 1, 0)
end = min(end + 1, len(indexes))
indexes_between = indexes[start:end].to_list()
indexes_between_ranges.append(indexes_between)
else:
start = indexes.searchsorted(low, "left")
end = indexes.searchsorted(up, "right")
indexes_between = indexes[start:end].to_list()
indexes_between_ranges.append(indexes_between)
else:
if method == "surrounding":
start = indexes.index(low)
end = indexes.index(up)
start = max(start - 1, 0)
end = min(end + 1, len(indexes))
# indexes_between = [i for i in indexes if low <= i <= up]
indexes_between = indexes[start:end]
indexes_between_ranges.append(indexes_between)
else:
indexes_between = [i for i in indexes if low <= i <= up]
indexes_between_ranges.append(indexes_between)
# if method == "surrounding":
# start = indexes.index(low)
# end = indexes.index(up)
# start = max(start-1, 0)
# end = min(end+1, len(indexes))
# # indexes_between = [i for i in indexes if low <= i <= up]
# indexes_between = indexes[start:end]
# indexes_between_ranges.append(indexes_between)
# else:
# indexes_between = [i for i in indexes if low <= i <= up]
# indexes_between_ranges.append(indexes_between)
# indexes_between = [i for i in indexes if low <= i <= up]
# indexes_between_ranges.append(indexes_between)
return indexes_between_ranges

def remap(range):
Expand All @@ -374,7 +474,6 @@ def type_change(cls):
from .transformations.datacube_type_change import DatacubeAxisTypeChange

if cls.type_change:

old_find_indexes = cls.find_indexes

def find_indexes(path, datacube):
Expand Down Expand Up @@ -415,15 +514,27 @@ def unmap_to_datacube(path, unmapped_path):
def remap_to_requested(path, unmapped_path):
return (path, unmapped_path)

def find_indices_between(index_ranges, low, up, datacube):
def find_indices_between(index_ranges, low, up, datacube, method=None):
# TODO: add method for snappping
indexes_between_ranges = []
for transform in cls.transformations:
if isinstance(transform, DatacubeAxisTypeChange):
transformation = transform
if cls.name == transformation.name:
for indexes in index_ranges:
indexes_between = [i for i in indexes if low <= i <= up]
indexes_between_ranges.append(indexes_between)
if method == "surrounding":
start = indexes.index(low)
end = indexes.index(up)
start = max(start - 1, 0)
end = min(end + 1, len(indexes))
# indexes_between = [i for i in indexes if low <= i <= up]
indexes_between = indexes[start:end]
indexes_between_ranges.append(indexes_between)
else:
indexes_between = [i for i in indexes if low <= i <= up]
indexes_between_ranges.append(indexes_between)
# indexes_between = [i for i in indexes if low <= i <= up]
# indexes_between_ranges.append(indexes_between)
return indexes_between_ranges

def remap(range):
Expand Down Expand Up @@ -508,20 +619,38 @@ def unmap_total_path_to_datacube(self, path, unmapped_path):
def remap_to_requeest(path, unmapped_path):
return (path, unmapped_path)

def find_indices_between(self, index_ranges, low, up, datacube):
def find_indices_between(self, index_ranges, low, up, datacube, method=None):
# TODO: add method for snappping
indexes_between_ranges = []
for indexes in index_ranges:
if self.name in datacube.complete_axes:
# Find the range of indexes between lower and upper
# https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html
# Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing
start = indexes.searchsorted(low, "left")
end = indexes.searchsorted(up, "right")
indexes_between = indexes[start:end].to_list()
indexes_between_ranges.append(indexes_between)
if method == "surrounding":
start = indexes.searchsorted(low, "left")
end = indexes.searchsorted(up, "right")
start = max(start - 1, 0)
end = min(end + 1, len(indexes))
indexes_between = indexes[start:end].to_list()
indexes_between_ranges.append(indexes_between)
else:
start = indexes.searchsorted(low, "left")
end = indexes.searchsorted(up, "right")
indexes_between = indexes[start:end].to_list()
indexes_between_ranges.append(indexes_between)
else:
indexes_between = [i for i in indexes if low <= i <= up]
indexes_between_ranges.append(indexes_between)
if method == "surrounding":
start = indexes.index(low)
end = indexes.index(up)
start = max(start - 1, 0)
end = min(end + 1, len(indexes))
# indexes_between = [i for i in indexes if low <= i <= up]
indexes_between = indexes[start:end]
indexes_between_ranges.append(indexes_between)
else:
indexes_between = [i for i in indexes if low <= i <= up]
indexes_between_ranges.append(indexes_between)
return indexes_between_ranges

@staticmethod
Expand Down Expand Up @@ -572,7 +701,7 @@ class FloatDatacubeAxis(DatacubeAxis):
tol = 1e-12
range = None
transformations = []
type = 0.
type = 0.0

def parse(self, value: Any) -> Any:
return float(value)
Expand Down
Loading