Skip to content

Commit

Permalink
check changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@konstntokas
konstntokas committed Jan 23, 2025
1 parent 58986be commit 2ea86c6
Show file tree
Hide file tree
Showing 9 changed files with 114 additions and 93 deletions.
29 changes: 12 additions & 17 deletions test/test_store.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
from xcube_stac.constants import DATA_STORE_ID
from xcube_stac.constants import DATA_STORE_ID_XCUBE
from xcube_stac.constants import DATA_STORE_ID_CDSE
from xcube_stac.constants import SENITNEL2_L2A_BANDS
from xcube_stac.accessor.sen2 import SENITNEL2_L2A_BANDS
from xcube_stac.accessor.https import HttpsDataAccessor
from xcube_stac.accessor.s3 import S3DataAccessor
from xcube_stac._utils import reproject_bbox
Expand All @@ -51,13 +51,9 @@
)
SERVER_URL = "http://localhost:8080"
SERVER_ENDPOINT_URL = f"{SERVER_URL}/s3"
# CDSE_CREDENTIALS = {
# "key": "xxx",
# "secret": "xxx",
# }
CDSE_CREDENTIALS = {
"key": "O0M0CUQIDQO9TDZ4D8NR",
"secret": "qPUyXs9G6j8on6MY5KPhQNHuA5uZTqxEscrbBCGx",
"key": "xxx",
"secret": "xxx",
}


Expand Down Expand Up @@ -89,7 +85,7 @@ def setUp(self):
self.data_id_nonsearchable = "zanzibar/znz001.json"
self.data_id_searchable = (
"collections/sentinel-1-grd/items/"
"S1A_IW_GRDH_1SSH_20250120T103300_20250120T103325_057526_0715CD"
"S1A_EW_GRDM_1SDV_20250121T072110_20250121T072152_057539_071657"
)
self.data_id_time_range = (
"lcv_blue_landsat.glad.ard/lcv_blue_landsat.glad.ard_1999.12.02"
Expand Down Expand Up @@ -433,7 +429,7 @@ def test_open_data_tiff(self):
)
self.assertIsInstance(ds, xr.Dataset)
self.assertCountEqual(
["blue_p50", "blue_p25", "blue_p75", "qa_f", "crs"], list(ds.data_vars)
["blue_p50", "blue_p25", "blue_p75", "qa_f"], list(ds.data_vars)
)
self.assertCountEqual([151000, 188000], [ds.sizes["y"], ds.sizes["x"]])
self.assertCountEqual(
Expand Down Expand Up @@ -467,7 +463,7 @@ def test_open_data_tiff(self):
)
self.assertEqual(msg, str(cm.output[-1]))
self.assertIsInstance(mlds, MultiLevelDataset)
self.assertCountEqual(["blue_p25", "crs"], list(ds.data_vars))
self.assertCountEqual(["blue_p25"], list(ds.data_vars))
self.assertCountEqual([151000, 188000], [ds.sizes["y"], ds.sizes["x"]])
self.assertCountEqual(
[512, 512], [ds.chunksizes["x"][0], ds.chunksizes["y"][0]]
Expand All @@ -481,7 +477,7 @@ def test_open_data_tiff(self):
)
self.assertIsInstance(mlds, MultiLevelDataset)
ds = mlds.base_dataset
self.assertCountEqual(["blue_p25", "blue_p75", "crs"], list(ds.data_vars))
self.assertCountEqual(["blue_p25", "blue_p75"], list(ds.data_vars))
self.assertCountEqual([151000, 188000], [ds.sizes["y"], ds.sizes["x"]])
self.assertCountEqual(
[512, 512], [ds.chunksizes["x"][0], ds.chunksizes["y"][0]]
Expand Down Expand Up @@ -624,7 +620,6 @@ def test_open_data_xcube_server(self):
"analytic_multires_band_1",
"analytic_multires_band_2",
"analytic_multires_band_3",
"analytic_multires_spatial_ref",
],
list(ds.data_vars),
)
Expand Down Expand Up @@ -669,7 +664,7 @@ def test_open_data_cdse_sen2(self, mock_rioxarray_open):
)
self.assertIsInstance(ds, xr.Dataset)
self.assertCountEqual(
SENITNEL2_L2A_BANDS + ["crs", "solar_angle", "viewing_angle"],
SENITNEL2_L2A_BANDS + ["solar_angle", "viewing_angle"],
list(ds.data_vars),
)
self.assertCountEqual(
Expand All @@ -695,7 +690,7 @@ def test_open_data_cdse_sen2(self, mock_rioxarray_open):
ds = mlds.get_dataset(0)
self.assertIsInstance(mlds, MultiLevelDataset)
self.assertCountEqual(
["B01", "B02", "B03", "crs", "solar_angle", "viewing_angle"],
["B01", "B02", "B03", "solar_angle", "viewing_angle"],
list(ds.data_vars),
)
self.assertCountEqual(
Expand Down Expand Up @@ -727,7 +722,7 @@ def test_open_data_stack_mode(self):
open_params_dataset_geotiff=dict(tile_size=(512, 512)),
)
self.assertIsInstance(ds, xr.Dataset)
self.assertCountEqual(["red", "green", "blue", "crs"], list(ds.data_vars))
self.assertCountEqual(["red", "green", "blue"], list(ds.data_vars))
self.assertCountEqual(
[4, 16, 16],
[ds.sizes["time"], ds.sizes["y"], ds.sizes["x"]],
Expand Down Expand Up @@ -791,7 +786,7 @@ def test_open_data_stack_mode_cdse_sen2(self, mock_rioxarray_open):
self.assertIsInstance(ds, xr.Dataset)

self.assertCountEqual(
SENITNEL2_L2A_BANDS + ["crs", "solar_angle", "viewing_angle"],
SENITNEL2_L2A_BANDS + ["solar_angle", "viewing_angle"],
list(ds.data_vars),
)
self.assertCountEqual(
Expand Down Expand Up @@ -834,7 +829,7 @@ def test_open_data_stack_mode_cdse_sen2(self, mock_rioxarray_open):
self.assertIsInstance(ds, xr.Dataset)

self.assertCountEqual(
["B01", "B02", "B03", "crs", "solar_angle", "viewing_angle"],
["B01", "B02", "B03", "solar_angle", "viewing_angle"],
list(ds.data_vars),
)
self.assertCountEqual(
Expand Down
48 changes: 36 additions & 12 deletions xcube_stac/_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,13 +304,33 @@ def add_nominal_datetime(items: list[pystac.Item]) -> list[pystac.Item]:
return items


def get_processing_version(item: pystac.Item) -> float:
return float(
item.properties.get(
"processing:version",
item.properties.get("s2:processing_baseline", "1.0"),
)
)
def get_grid_mapping_name(ds: xr.Dataset) -> str | None:
gm_names = []
for var in ds.data_vars:
if "grid_mapping" in ds[var].attrs:
gm_names.append(ds[var].attrs["grid_mapping"])
if "crs" in ds:
gm_names.append("crs")
if "spatial_ref" in ds.coords:
gm_names.append("spatial_ref")
gm_names = np.unique(gm_names)
assert len(gm_names) <= 1, "Multiple grid mapping names found."
if len(gm_names) == 1:
return str(gm_names[0])
else:
return None


def normalize_grid_mapping(ds: xr.Dataset) -> xr.Dataset:
gm_name = get_grid_mapping_name(ds)
if gm_name is None:
return ds
for var in ds.data_vars:
ds[var].attrs["grid_mapping"] = "spatial_ref"
gm_var = ds[gm_name]
ds = ds.drop_vars(gm_name)
ds = ds.assign_coords(spatial_ref=xr.DataArray(0, attrs=gm_var.attrs))
return ds


def update_dict(dic: dict, dic_update: dict, inplace: bool = True) -> dict:
Expand Down Expand Up @@ -346,7 +366,11 @@ def get_url_from_pystac_object(
Returns:
the URL of an item.
"""
links = [link for link in pystac_obj.links if link.rel == "self"]
links = [
link
for link in pystac_obj.links
if link.rel == "self" and link.href.startswith("https://")
]
assert len(links) == 1
return links[0].href

Expand Down Expand Up @@ -595,7 +619,7 @@ def wrapper_resample_in_space(ds: xr.Dataset, target_gm: GridMapping) -> xr.Data
# and encoded crs in the data variable "crs". This is needed until the
# issue https://github.com/xcube-dev/xcube/issues/1013 is addressed.
ds = resample_in_space(ds, target_gm=target_gm, gm_name="crs", encode_cf=True)
vars = [
var_names = [
"x_bnds",
"y_bnds",
"lon_bnds",
Expand All @@ -604,7 +628,7 @@ def wrapper_resample_in_space(ds: xr.Dataset, target_gm: GridMapping) -> xr.Data
"transformed_y",
]
vars_sel = []
for var in vars:
if var in ds:
vars_sel.append(var)
for var_ame in var_names:
if var_ame in ds:
vars_sel.append(var_ame)
return ds.drop_vars(vars_sel)
39 changes: 38 additions & 1 deletion xcube_stac/accessor/sen2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
from xcube.core.store import DataTypeLike

from .._href_parse import decode_href
from ..constants import SENITNEL2_BANDS
from .._utils import is_valid_ml_data_type
from .._utils import get_gridmapping
from .._utils import get_spatial_dims
Expand All @@ -43,6 +42,44 @@
from ..stack import mosaic_spatial_take_first


SENITNEL2_BANDS = [
"B01",
"B02",
"B03",
"B04",
"B05",
"B06",
"B07",
"B08",
"B8A",
"B09",
"B10",
"B11",
"B12",
]
SENITNEL2_L2A_BANDS = SENITNEL2_BANDS + ["AOT", "SCL", "WVP"]
SENITNEL2_L2A_BANDS.remove("B10")
SENITNEL2_L2A_BAND_RESOLUTIONS = {
"B01": 60,
"B02": 10,
"B03": 10,
"B04": 10,
"B05": 20,
"B06": 20,
"B07": 20,
"B08": 10,
"B8A": 20,
"B09": 60,
"B11": 20,
"B12": 20,
"AOT": 10,
"SCL": 20,
"WVP": 10,
}
SENTINEL2_MIN_RESOLUTIONS = 10
SENTINEL2_REGEX_ASSET_NAME = "^[A-Z]{3}_[0-9]{2}m$"


class S3Sentinel2DataAccessor:
"""Implementation of the data accessor supporting
the jp2 format of Sentinel-2 data via the AWS S3 protocol.
Expand Down
38 changes: 0 additions & 38 deletions xcube_stac/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,41 +189,3 @@
spatial_res=SCHEMA_SPATIAL_RES,
query=SCHEMA_ADDITIONAL_QUERY,
)

# CDSE Sentinel-2 constants
SENITNEL2_BANDS = [
"B01",
"B02",
"B03",
"B04",
"B05",
"B06",
"B07",
"B08",
"B8A",
"B09",
"B10",
"B11",
"B12",
]
SENITNEL2_L2A_BANDS = SENITNEL2_BANDS + ["AOT", "SCL", "WVP"]
SENITNEL2_L2A_BANDS.remove("B10")
SENITNEL2_L2A_BAND_RESOLUTIONS = {
"B01": 60,
"B02": 10,
"B03": 10,
"B04": 10,
"B05": 20,
"B06": 20,
"B07": 20,
"B08": 10,
"B8A": 20,
"B09": 60,
"B11": 20,
"B12": 20,
"AOT": 10,
"SCL": 20,
"WVP": 10,
}
SENTINEL2_MIN_RESOLUTIONS = 10
SENTINEL2_REGEX_ASSET_NAME = "^[A-Z]{3}_[0-9]{2}m$"
6 changes: 3 additions & 3 deletions xcube_stac/helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@

from .accessor.s3 import S3DataAccessor
from .accessor.sen2 import S3Sentinel2DataAccessor
from .accessor.sen2 import SENITNEL2_L2A_BANDS
from .accessor.sen2 import SENTINEL2_REGEX_ASSET_NAME
from .accessor.sen2 import SENITNEL2_L2A_BAND_RESOLUTIONS
from .constants import MLDATASET_FORMATS
from .constants import SENITNEL2_L2A_BANDS
from .constants import SENTINEL2_REGEX_ASSET_NAME
from .constants import SENITNEL2_L2A_BAND_RESOLUTIONS
from ._href_parse import decode_href
from ._utils import get_format_id
from ._utils import get_format_from_path
Expand Down
19 changes: 10 additions & 9 deletions xcube_stac/mldataset/single_item.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,23 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from typing import Any, Optional
from typing import Any

import rasterio
import rasterio.session
import rioxarray
import pystac
import xarray as xr
from xcube.core.mldataset import MultiLevelDataset, LazyMultiLevelDataset
from xcube.core.gridmapping import GridMapping

from ._utils import rename_dataset
from ._utils import normalize_grid_mapping
from ._utils import merge_datasets
from ._utils import wrapper_clip_dataset_by_geometry
from .stac_extension.raster import apply_offset_scaling
from .accessor import S3DataAccessor
from .accessor import S3Sentinel2DataAccessor
from .._utils import rename_dataset
from .._utils import normalize_grid_mapping
from .._utils import merge_datasets
from .._utils import wrapper_clip_dataset_by_geometry
from ..stac_extension.raster import apply_offset_scaling
from ..accessor.s3 import S3DataAccessor
from ..accessor.sen2 import S3Sentinel2DataAccessor


class SingleItemMultiLevelDataset(LazyMultiLevelDataset):
Expand Down Expand Up @@ -88,7 +89,7 @@ def _get_dataset_lazily(
datasets.append(ds)
combined_dataset = merge_datasets(datasets, target_gm=self._target_gm)
if self._open_params.get("angles_sentinel2", False):
combined_dataset = self._s3_accessor.read_angles(
combined_dataset = self._s3_accessor.add_sen2_angles(
self._item, combined_dataset
)
combined_dataset.attrs = self._attrs
Expand Down
4 changes: 2 additions & 2 deletions xcube_stac/stack.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import xarray as xr

from ._utils import add_nominal_datetime
from ._utils import get_processing_version
from ._utils import get_spatial_dims
from .constants import LOG


Expand Down Expand Up @@ -117,7 +117,7 @@ def get_processing_version(item: pystac.Item) -> float:
)


def mosaic_2d_take_first(list_ds: list[xr.Dataset]) -> xr.Dataset:
def mosaic_spatial_take_first(list_ds: list[xr.Dataset]) -> xr.Dataset:
if len(list_ds) == 1:
return list_ds[0]
dim = "dummy"
Expand Down
2 changes: 1 addition & 1 deletion xcube_stac/store.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from typing import Any, Dict, Tuple, Union
from typing import Any
from collections.abc import Container, Iterator

import numpy as np
Expand Down
Loading

0 comments on commit 2ea86c6

Please sign in to comment.