From b139ebf0625598d3d3dbb465b0392a47685f734e Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Wed, 24 Jul 2019 18:37:47 +0200 Subject: [PATCH 01/10] about to implement lazy place group loading... --- test/webapi/controllers/test_places.py | 4 +- test/webapi/test_context.py | 71 +++++++++++------- xcube/webapi/context.py | 100 ++++++++++++++----------- xcube/webapi/controllers/places.py | 23 ++++-- xcube/webapi/handlers.py | 2 +- 5 files changed, 123 insertions(+), 77 deletions(-) diff --git a/test/webapi/controllers/test_places.py b/test/webapi/controllers/test_places.py index 472714e0d..8025b6956 100644 --- a/test/webapi/controllers/test_places.py +++ b/test/webapi/controllers/test_places.py @@ -46,12 +46,12 @@ def test_find_places_by_geojson(self): places = find_places(ctx, "all", geojson_obj=geojson_obj) self._assertPlaceGroup(places, 2, {'0', '3'}) - geojson_obj = {'type': 'FeatureCollection', 'places': [geojson_obj]} + geojson_obj = {'type': 'FeatureCollection', 'features': [geojson_obj]} places = find_places(ctx, "all", geojson_obj=geojson_obj) self._assertPlaceGroup(places, 2, {'0', '3'}) with self.assertRaises(ServiceBadRequestError) as cm: - geojson_obj = {'type': 'FeatureCollection', 'places': []} + geojson_obj = {'type': 'FeatureCollection', 'features': []} find_places(ctx, "all", geojson_obj=geojson_obj) self.assertEqual("HTTP 400: Received invalid GeoJSON object", f"{cm.exception}") diff --git a/test/webapi/test_context.py b/test/webapi/test_context.py index 890633f26..3828ffdd6 100644 --- a/test/webapi/test_context.py +++ b/test/webapi/test_context.py @@ -71,39 +71,60 @@ def test_get_color_mapping(self): cm = ctx.get_color_mapping('demo', '_') self.assertEqual(('jet', 0., 1.), cm) - def test_get_feature_collections(self): + def test_get_global_place_groups(self): ctx = new_test_service_context() - feature_collections = ctx.get_place_groups() - self.assertIsInstance(feature_collections, list) - self.assertEqual([{'id': 'inside-cube', 'title': 'Points inside the cube'}, - {'id': 'outside-cube', 'title': 'Points outside the cube'}], - feature_collections) - - def test_get_place_group(self): - ctx = new_test_service_context() - place_group = ctx.get_place_group() - self.assertIsInstance(place_group, dict) - self.assertIn("type", place_group) - self.assertEqual("FeatureCollection", place_group["type"]) - self.assertIn("features", place_group) - self.assertIsInstance(place_group["features"], list) - self.assertEqual(6, len(place_group["features"])) - self.assertIs(place_group, ctx.get_place_group()) - self.assertEqual([str(i) for i in range(6)], - [f["id"] for f in place_group["features"] if "id" in f]) - - def test_get_place_group_by_name(self): + place_groups = ctx.get_global_place_groups(load_features=False) + self.assertIsInstance(place_groups, list) + self.assertEqual(2, len(place_groups)) + for place_group in place_groups: + self.assertIn("type", place_group) + self.assertIn("id", place_group) + self.assertIn("title", place_group) + self.assertIn("propertyMapping", place_group) + self.assertIn("sourcePaths", place_group) + self.assertIn("sourceEncoding", place_group) + self.assertIn("features", place_group) + place_group = place_groups[0] + self.assertEqual('inside-cube', place_group['id']) + self.assertEqual('Points inside the cube', place_group['title']) + self.assertEqual(None, place_group['features']) + place_group = place_groups[1] + self.assertEqual('outside-cube', place_group['id']) + self.assertEqual('Points outside the cube', place_group['title']) + self.assertEqual(None, place_group['features']) + + place_groups = ctx.get_global_place_groups(load_features=True) + self.assertIsInstance(place_groups, list) + self.assertEqual(2, len(place_groups)) + for place_group in place_groups: + self.assertIn("type", place_group) + self.assertIn("id", place_group) + self.assertIn("title", place_group) + self.assertIn("propertyMapping", place_group) + self.assertIn("sourcePaths", place_group) + self.assertIn("sourceEncoding", place_group) + self.assertIn("features", place_group) + place_group = place_groups[0] + self.assertEqual('inside-cube', place_group['id']) + self.assertEqual('Points inside the cube', place_group['title']) + self.assertIsNotNone(place_group['features']) + place_group = place_groups[1] + self.assertEqual('outside-cube', place_group['id']) + self.assertEqual('Points outside the cube', place_group['title']) + self.assertIsNotNone(place_group['features']) + + def test_get_global_place_group(self): ctx = new_test_service_context() - place_group = ctx.get_place_group(place_group_id="inside-cube") + place_group = ctx.get_global_place_group("inside-cube", load_features=True) self.assertIsInstance(place_group, dict) self.assertIn("type", place_group) self.assertEqual("FeatureCollection", place_group["type"]) self.assertIn("features", place_group) self.assertIsInstance(place_group["features"], list) self.assertEqual(3, len(place_group["features"])) - self.assertIs(place_group, ctx.get_place_group(place_group_id="inside-cube")) - self.assertIsNot(place_group, ctx.get_place_group(place_group_id="outside-cube")) + self.assertIs(place_group, ctx.get_global_place_group(place_group_id="inside-cube")) + self.assertIsNot(place_group, ctx.get_global_place_group(place_group_id="outside-cube")) with self.assertRaises(ServiceResourceNotFoundError) as cm: - ctx.get_place_group(place_group_id="bibo") + ctx.get_global_place_group("bibo") self.assertEqual('HTTP 404: Place group "bibo" not found', f"{cm.exception}") diff --git a/xcube/webapi/context.py b/xcube/webapi/context.py index 66df1ea3b..c7ca7855e 100644 --- a/xcube/webapi/context.py +++ b/xcube/webapi/context.py @@ -110,6 +110,8 @@ def config(self, config: Config): self._image_cache.clear() if self._tile_cache: self._tile_cache.clear() + if self._place_group_cache: + self._place_group_cache.clear() self._config = config @@ -269,19 +271,15 @@ def get_legend_label(self, ds_name: str, var_name: str): return units raise ServiceResourceNotFoundError(f'Variable "{var_name}" not found in dataset "{ds_name}"') - def get_place_groups(self) -> List[Dict]: - place_group_configs = self._config.get("PlaceGroups", []) - place_groups = [] - for features_config in place_group_configs: - place_groups.append(dict(id=features_config.get("Identifier"), - title=features_config.get("Title"))) + def get_global_place_groups(self, load_features=True) -> List[Dict]: + place_groups = self._load_place_groups(self._config.get("PlaceGroups", []), is_global=True) + if load_features: + for place_group in place_groups: + self.load_place_group_features(place_group) return place_groups - def get_dataset_place_groups(self, ds_id: str) -> List[Dict]: + def get_dataset_place_groups(self, ds_id: str, load_features=True) -> List[Dict]: dataset_descriptor = self.get_dataset_descriptor(ds_id) - place_group_configs = dataset_descriptor.get("PlaceGroups") - if not place_group_configs: - return [] place_group_id_prefix = f"DS-{ds_id}-" @@ -289,44 +287,47 @@ def get_dataset_place_groups(self, ds_id: str) -> List[Dict]: for k, v in self._place_group_cache.items(): if k.startswith(place_group_id_prefix): place_groups.append(v) + if place_groups: return place_groups - place_groups = self._load_place_groups(place_group_configs) + place_groups = self._load_place_groups(dataset_descriptor.get("PlaceGroups", []), is_global=False) + if load_features: + for place_group in place_groups: + self.load_place_group_features(place_group) for place_group in place_groups: self._place_group_cache[place_group_id_prefix + place_group["id"]] = place_group return place_groups - def get_place_group(self, place_group_id: str = ALL_PLACES) -> Dict: - if ALL_PLACES not in self._place_group_cache: - place_group_configs = self._config.get("PlaceGroups", []) - place_groups = self._load_place_groups(place_group_configs) - - all_features = [] - for place_group in place_groups: - all_features.extend(place_group["features"]) - self._place_group_cache[place_group["id"]] = place_group - - self._place_group_cache[ALL_PLACES] = dict(type="FeatureCollection", features=all_features) - + def get_global_place_group(self, place_group_id: str, load_features: bool = False) -> Dict: + self._cache_global_place_groups() if place_group_id not in self._place_group_cache: raise ServiceResourceNotFoundError(f'Place group "{place_group_id}" not found') + place_group = self._place_group_cache[place_group_id] + if load_features: + self.load_place_group_features(place_group) + return place_group - return self._place_group_cache[place_group_id] + def _cache_global_place_groups(self): + if not self._place_group_cache: + for place_group in self.get_global_place_groups(): + self._place_group_cache[place_group['id']] = place_group - def _load_place_groups(self, place_group_configs: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + def _load_place_groups(self, place_group_configs: Dict, is_global: bool = False) -> List[Dict]: place_groups = [] for place_group_config in place_group_configs: - place_group = self._load_place_group(place_group_config) + place_group = self._load_place_group(place_group_config, is_global=is_global) place_groups.append(place_group) return place_groups - def _load_place_group(self, place_group_config: Dict[str, Any]) -> Dict[str, Any]: + def _load_place_group(self, place_group_config: Dict[str, Any], is_global: bool = False) -> Dict[str, Any]: ref_id = place_group_config.get("PlaceGroupRef") if ref_id: + if is_global: + raise ServiceError("'PlaceGroupRef' cannot be used in a global place group") # Trigger loading of all global "PlaceGroup" entries - self.get_place_group() + self._cache_global_place_groups() if len(place_group_config) > 1: raise ServiceError("'PlaceGroupRef' if present, must be the only entry in a 'PlaceGroups' item") if ref_id not in self._place_group_cache: @@ -346,33 +347,46 @@ def _load_place_group(self, place_group_config: Dict[str, Any]) -> Dict[str, Any raise ServiceError("Missing 'Path' entry in a 'PlaceGroups' item") if not os.path.isabs(place_path_wc): place_path_wc = os.path.join(self._base_dir, place_path_wc) + source_paths = glob.glob(place_path_wc) + source_encoding = place_group_config.get("CharacterEncoding", "utf-8") property_mapping = place_group_config.get("PropertyMapping") - character_encoding = place_group_config.get("CharacterEncoding", "utf-8") - - features = [] - collection_files = glob.glob(place_path_wc) - for collection_file in collection_files: - with fiona.open(collection_file, encoding=character_encoding) as feature_collection: - for feature in feature_collection: - self._remove_feature_id(feature) - feature["id"] = str(self._feature_index) - self._feature_index += 1 - features.append(feature) place_group = dict(type="FeatureCollection", - features=features, + features=None, id=place_group_id, title=place_group_title, - propertyMapping=property_mapping) + propertyMapping=property_mapping, + sourcePaths=source_paths, + sourceEncoding=source_encoding) sub_place_group_configs = place_group_config.get("Places") if sub_place_group_configs: - sub_place_groups = self._load_place_groups(sub_place_group_configs) - place_group["placeGroups"] = sub_place_groups + raise ServiceError("Invalid 'Places' entry in a 'PlaceGroups' item: not implemented yet") + # sub_place_group_configs = place_group_config.get("Places") + # if sub_place_group_configs: + # sub_place_groups = self._load_place_groups(sub_place_group_configs) + # place_group["placeGroups"] = sub_place_groups return place_group + def load_place_group_features(self, place_group: Dict[str, Any]) -> List[Dict[str, Any]]: + features = place_group.get('features') + if features is not None: + return features + source_files = place_group['sourcePaths'] + source_encoding = place_group['sourceEncoding'] + features = [] + for source_file in source_files: + with fiona.open(source_file, encoding=source_encoding) as feature_collection: + for feature in feature_collection: + self._remove_feature_id(feature) + feature["id"] = str(self._feature_index) + self._feature_index += 1 + features.append(feature) + place_group['features'] = features + return features + @classmethod def _remove_feature_id(cls, feature: Dict): cls._remove_id(feature) diff --git a/xcube/webapi/controllers/places.py b/xcube/webapi/controllers/places.py index ecf6d886d..bd10e12c2 100644 --- a/xcube/webapi/controllers/places.py +++ b/xcube/webapi/controllers/places.py @@ -5,7 +5,7 @@ import shapely.wkt from shapely.errors import WKTReadingError -from ..context import ServiceContext +from ..context import ServiceContext, ALL_PLACES from ..errors import ServiceBadRequestError from ...util.geom import get_dataset_geometry, get_box_split_bounds_geometry from ...util.perf import measure_time @@ -78,19 +78,30 @@ def __find_places(ctx: ServiceContext, comb_op: str = "and") -> GeoJsonFeatureCollection: if comb_op is not None and comb_op != "and": raise NotImplementedError("comb_op not yet supported") - place_group = ctx.get_place_group(place_group_id) + + if place_group_id == ALL_PLACES: + place_groups = ctx.get_global_place_groups() + features = [] + for pg in place_groups: + pg_features = ctx.load_place_group_features(pg) + features.extend(pg_features) + feature_collection = dict(type="FeatureCollection", features=features) + else: + feature_collection = ctx.get_global_place_group(place_group_id, load_features=True) + feature_collection = dict(type="FeatureCollection", features=feature_collection['features']) + if query_geometry is None: if query_expr is None: - return place_group + return feature_collection else: raise NotImplementedError() else: matching_places = [] if query_expr is None: - for place in place_group["features"]: - geometry = shapely.geometry.shape(place["geometry"]) + for feature in feature_collection['features']: + geometry = shapely.geometry.shape(feature['geometry']) if geometry.intersects(query_geometry): - matching_places.append(place) + matching_places.append(feature) else: raise NotImplementedError() return dict(type="FeatureCollection", features=matching_places) diff --git a/xcube/webapi/handlers.py b/xcube/webapi/handlers.py index 94061b080..9e8a82b8a 100644 --- a/xcube/webapi/handlers.py +++ b/xcube/webapi/handlers.py @@ -362,7 +362,7 @@ class GetPlaceGroupsHandler(ServiceRequestHandler): # noinspection PyShadowingBuiltins def get(self): - response = self.service_context.get_place_groups() + response = self.service_context.get_global_place_groups() self.set_header('Content-Type', "application/json") self.write(json.dumps(response, indent=2)) From 21c25368d6d18f6bbb351164d56cdd526748f873 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 25 Jul 2019 13:46:24 +0200 Subject: [PATCH 02/10] load_features=False --- xcube/webapi/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcube/webapi/context.py b/xcube/webapi/context.py index c7ca7855e..de54e1218 100644 --- a/xcube/webapi/context.py +++ b/xcube/webapi/context.py @@ -278,7 +278,7 @@ def get_global_place_groups(self, load_features=True) -> List[Dict]: self.load_place_group_features(place_group) return place_groups - def get_dataset_place_groups(self, ds_id: str, load_features=True) -> List[Dict]: + def get_dataset_place_groups(self, ds_id: str, load_features=False) -> List[Dict]: dataset_descriptor = self.get_dataset_descriptor(ds_id) place_group_id_prefix = f"DS-{ds_id}-" From c707a337ff054aacdc1d5a65f6cce94d69684328 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 25 Jul 2019 13:47:31 +0200 Subject: [PATCH 03/10] filtering place groups --- xcube/webapi/controllers/catalogue.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/xcube/webapi/controllers/catalogue.py b/xcube/webapi/controllers/catalogue.py index 50856ced3..3239b82bb 100644 --- a/xcube/webapi/controllers/catalogue.py +++ b/xcube/webapi/controllers/catalogue.py @@ -109,8 +109,16 @@ def get_dataset(ctx: ServiceContext, ds_id: str, client=None, base_url: str = No dim_names = ds.data_vars[list(ds.data_vars)[0]].dims if len(ds.data_vars) > 0 else ds.dims.keys() dataset_dict["dimensions"] = [get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names] - place_groups = ctx.get_dataset_place_groups(ds_id) + # place_groups = ctx.get_dataset_place_groups(ds_id, load_features=False) + place_groups = ctx.get_dataset_place_groups(ds_id, load_features=True) if place_groups: + def filter_place_group(place_group: Dict): + place_group = dict(place_group) + del place_group['sourcePaths'] + del place_group['sourceEncoding'] + # place_group['features'] = None + return place_group + place_groups = list(map(filter_place_group, place_groups)) dataset_dict["placeGroups"] = place_groups return dataset_dict From 4818493b18b83389b243895155cc8403b6fc3fef Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 25 Jul 2019 15:07:51 +0200 Subject: [PATCH 04/10] Fixed demo cube --- .../res/demo/cube-5-100-200.zarr/.zattrs | 2 +- .../cube-5-100-200.zarr/c2rcc_flags/0.0.0 | Bin 2100 -> 3297 bytes .../cube-5-100-200.zarr/c2rcc_flags/0.0.1 | Bin 2100 -> 3297 bytes .../cube-5-100-200.zarr/c2rcc_flags/0.0.2 | Bin 2100 -> 3297 bytes .../cube-5-100-200.zarr/c2rcc_flags/0.0.3 | Bin 2100 -> 3297 bytes .../res/demo/cube-5-100-200.zarr/lon_bnds/7.0 | Bin 542 -> 947 bytes .../res/demo/cube-5-100-200.zarr/lon_bnds/8.0 | Bin 623 -> 908 bytes .../res/demo/cube-5-100-200.zarr/time/.zarray | 2 +- .../res/demo/cube-5-100-200.zarr/time/0 | Bin 24 -> 56 bytes .../cube-5-100-200.zarr/time_bnds/.zarray | 2 +- .../demo/cube-5-100-200.zarr/time_bnds/0.0 | Bin 32 -> 96 bytes 11 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs b/xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs index ca0002993..e14351a44 100644 --- a/xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs +++ b/xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs @@ -2,7 +2,7 @@ "Conventions": "CF-1.6", "TileSize": "128:1190", "auto_grouping": "iop:conc:rtoa:rrs:rhown:kd:unc", - "coordinates": "time_bnds lat_bnds lon_bnds", + "coordinates": "lat_bnds lon_bnds time_bnds", "metadata_profile": "beam", "metadata_version": "0.5", "product_type": "C2RCC_OLCI", diff --git a/xcube/webapi/res/demo/cube-5-100-200.zarr/c2rcc_flags/0.0.0 b/xcube/webapi/res/demo/cube-5-100-200.zarr/c2rcc_flags/0.0.0 index 11cfccdd2fdaef26ec0b902b153f01dbf249be7c..d92dcc383ff8c4b2640236ff6b91ca6c39accf2e 100644 GIT binary patch literal 3297 zcmZQ#RODbV2^8ti@(hd&|3^4@9smrG(cplkg3;iBg#?%$ z4UW;^KnVd9&S>hu85}=GYnJB$KR^In(bkU!$Mb-C5b$LgZFivLno-VZ2;dEYYx0Aq PFcp>`JS7<)q(A`xho4AW literal 2100 zcmZQ#ROC=t$j$)5CL9b5A`C#l$iN^EB>s;?zz_fmfzdD+4Fhm87|jc#c>$adM$1G< WnfPPG)(byC09+o_kJvD%2LS*RI2^8ti@(hd&|3^4@9smrG(cplkg3;iBg#?%$ z4UW;^KnVd9&S>hu85}=GYnJB$KR^In(bkU!$Mb-C5b$LgZFivLno-VZ2;dEYYx0Aq PFcp>`JS7<)q(A`xho4AW literal 2100 zcmZQ#ROC=t$j$)5CL9b5A`C#l$iN^EB>s;?zz_fmfzdD+4Fhm87|jc#c>$adM$1G< WnfPPG)(byC09+o_kJvD%2LS*RI2^8ti@(hd&|3^4@9smrG(cplkg3;iBg#?%$ z4UW;^KnVd9&S>hu85}=GYnJB$KR^In(bkU!$Mb-C5b$LgZFivLno-VZ2;dEYYx0Aq PFcp>`JS7<)q(A`xho4AW literal 2100 zcmZQ#ROC=t$j$)5CL9b5A`C#l$iN^EB>s;?zz_fmfzdD+4Fhm87|jc#c>$adM$1G< WnfPPG)(byC09+o_kJvD%2LS*RI2^8ti@(hd&|3^4@9smrG(cplkg3;iBg#?%$ z4UW;^KnVd9&S>hu85}=GYnJB$KR^In(bkU!$Mb-C5b$LgZFivLno-VZ2;dEYYx0Aq PFcp>`JS7<)q(A`xho4AW literal 2100 zcmZQ#ROC=t$j$)5CL9b5A`C#l$iN^EB>s;?zz_fmfzdD+4Fhm87|jc#c>$adM$1G< WnfPPG)(byC09+o_kJvD%2LS*RI-*o}F%A@N5 zU;x1P(c+`GFahn|PG@0ZY3UwKlGW9Xje*$Q+}hgSex--Q-QE5D1ABNl9v>f{oc#EC zdU|$tetwyN`CoSqKKXX_Sgl>-IV9CSWBQ zqXf(irk$P9%li+9ZNILa%X2By6rm8x(wHn`4CAKpz9^L9oRAo+%37_3pfmyf+sp9D zfKrUxoza&mVlC%QV@#UXwbq1qpJfO+Cq#@z!B_}+PH9ysg>hL*IYXp$Zi4)tB1(lI zg!gG`Eyj5sW0uu5=VeK0Q6Qvs(@0rWj5$Y$F*C#%&fA^62XLw|rgd4yn5IfO*EE!7 znYAGhB4tt3HO4|1Q&k9Yp6A}r6z@Z5KkmJTFWx+B`v_r72&HMt80SJrnPp08o#)0B zg|*I=rT0};*C8}bj58(1i;K2DIrhq}E(C+#JOocy{#-&pdhJyK5Ii1C#{WK|w+_MQ N-WmjAy3+Nl_zQGk^y&Zr delta 171 zcmdnYK95C|iBXZGfro(sgyonR7(^Ht7@{YNx=QgIG4N}h4miO8WF+e^QlG}a74YRt zWz~PXiH|Kg|GNihXo!gXx0@Wx2qtDI5$PggU3nKVH1V1B#R{%c<03E|1@4(3Lzaqc^1ORvaG<*O6 diff --git a/xcube/webapi/res/demo/cube-5-100-200.zarr/lon_bnds/8.0 b/xcube/webapi/res/demo/cube-5-100-200.zarr/lon_bnds/8.0 index cfaf9f2f7223ef1ffd85f7a06fa88ff047af3104..6adaf2147709d079120e6bd69c5317c77e3e103c 100644 GIT binary patch delta 660 zcmYL{F-RRj5Qg_OlDGuKu#JYCi8g1W!c{;GiG%biK7T466zu#=?;EC^|ll<5P*TQiN!ork9uP_O;D(=P32QDC(wT z_*)F#{Jj47zj^>hV(@zQbbbxI=sQ?XDphU~0*AG(I@O01A#=%C)(%5!zw$e?EF&aI!Vp65t<_pe&a0{@@;ps3jv|EW z82qj$y?4%9W3*OEN+CFByPcPB$6E+N2=38o%DMN1Fyf`!>Oq#nwd$GxjBp%7YSKfO4F5DPvG8=Qa$q9xH0l$9z{mUQ@ lbc8;?03*YudO-m}5MX6k6#$}zfC0uJ@4(3Lzaqc^1OUtlWqbet diff --git a/xcube/webapi/res/demo/cube-5-100-200.zarr/time/.zarray b/xcube/webapi/res/demo/cube-5-100-200.zarr/time/.zarray index 670c15ee9..4fc171c80 100644 --- a/xcube/webapi/res/demo/cube-5-100-200.zarr/time/.zarray +++ b/xcube/webapi/res/demo/cube-5-100-200.zarr/time/.zarray @@ -1,6 +1,6 @@ { "chunks": [ - 1 + 5 ], "compressor": { "blocksize": 0, diff --git a/xcube/webapi/res/demo/cube-5-100-200.zarr/time/0 b/xcube/webapi/res/demo/cube-5-100-200.zarr/time/0 index 3138dae3865d14b9ef4c74655063c6c2340000c9..e72e033810c05740a0b3fcd6104585d5df1e51da 100644 GIT binary patch literal 56 zcmZQ#H0ID?U|;}Y3m`toY$`hY_yva#v!74DdGdn8TE`vFWKLahaB&crHVa7q{8ze1 I=JW*z0PCU`K>z>% literal 24 bcmZQ#H0I!7U|;}Y2_QbmY$`hY_yq?577hdF diff --git a/xcube/webapi/res/demo/cube-5-100-200.zarr/time_bnds/.zarray b/xcube/webapi/res/demo/cube-5-100-200.zarr/time_bnds/.zarray index 6871630c9..6f9745b09 100644 --- a/xcube/webapi/res/demo/cube-5-100-200.zarr/time_bnds/.zarray +++ b/xcube/webapi/res/demo/cube-5-100-200.zarr/time_bnds/.zarray @@ -1,6 +1,6 @@ { "chunks": [ - 1, + 5, 2 ], "compressor": { diff --git a/xcube/webapi/res/demo/cube-5-100-200.zarr/time_bnds/0.0 b/xcube/webapi/res/demo/cube-5-100-200.zarr/time_bnds/0.0 index 6a1c08725f1f37090d3974ba304a8274b960c5f8..42330d3a3f6f79c75bab99ef26c861de750c4b17 100644 GIT binary patch literal 96 zcmZQ#H0B6kU|;~@1R&nVEiN?s_yvbEzk?(|bYfn{l$$3nIMkg?n+>A>TrqhjbLxUa m+&b&$Ao|jQoJq5Q^tSz*XM*Uct;=@HoW9^N!^3qihz0<6W-c-S literal 32 hcmZQ#H0BUsU|;}Y1t8wWEiN?s_yvbEzk?(|GyqG*2z~$n From e93a9c26cb9943b23d9799bc6364209ad64ac4dc Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 25 Jul 2019 15:10:32 +0200 Subject: [PATCH 05/10] removed misleading global attributes --- xcube/webapi/res/demo/cube-1-250-250.zarr/.zattrs | 11 +---------- xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs | 11 +---------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/xcube/webapi/res/demo/cube-1-250-250.zarr/.zattrs b/xcube/webapi/res/demo/cube-1-250-250.zarr/.zattrs index 3a98a9629..8ebeaeec1 100644 --- a/xcube/webapi/res/demo/cube-1-250-250.zarr/.zattrs +++ b/xcube/webapi/res/demo/cube-1-250-250.zarr/.zattrs @@ -1,12 +1,3 @@ { - "Conventions": "CF-1.6", - "TileSize": "128:1190", - "auto_grouping": "iop:conc:rtoa:rrs:rhown:kd:unc", - "coordinates": "lon_bnds lat_bnds time_bnds", - "metadata_profile": "beam", - "metadata_version": "0.5", - "product_type": "C2RCC_OLCI", - "start_date": "16-JAN-2017 10:09:05.396603", - "stop_date": "16-JAN-2017 10:09:38.271909", - "tiepoint_coordinates": "TP_longitude TP_latitude" + "Conventions": "CF-1.7" } \ No newline at end of file diff --git a/xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs b/xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs index e14351a44..8ebeaeec1 100644 --- a/xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs +++ b/xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs @@ -1,12 +1,3 @@ { - "Conventions": "CF-1.6", - "TileSize": "128:1190", - "auto_grouping": "iop:conc:rtoa:rrs:rhown:kd:unc", - "coordinates": "lat_bnds lon_bnds time_bnds", - "metadata_profile": "beam", - "metadata_version": "0.5", - "product_type": "C2RCC_OLCI", - "start_date": "16-JAN-2017 10:09:05.396603", - "stop_date": "16-JAN-2017 10:09:38.271909", - "tiepoint_coordinates": "TP_longitude TP_latitude" + "Conventions": "CF-1.7" } \ No newline at end of file From 2576a8b1911cfbd570b41db250a5746d498c76d8 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 25 Jul 2019 15:28:59 +0200 Subject: [PATCH 06/10] added attribute "coordinate" required by xarray --- xcube/webapi/res/demo/cube-1-250-250.zarr/.zattrs | 3 ++- xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/xcube/webapi/res/demo/cube-1-250-250.zarr/.zattrs b/xcube/webapi/res/demo/cube-1-250-250.zarr/.zattrs index 8ebeaeec1..4b94b0e99 100644 --- a/xcube/webapi/res/demo/cube-1-250-250.zarr/.zattrs +++ b/xcube/webapi/res/demo/cube-1-250-250.zarr/.zattrs @@ -1,3 +1,4 @@ { - "Conventions": "CF-1.7" + "Conventions": "CF-1.7", + "coordinates": "time_bnds lat_bnds lon_bnds" } \ No newline at end of file diff --git a/xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs b/xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs index 8ebeaeec1..4b94b0e99 100644 --- a/xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs +++ b/xcube/webapi/res/demo/cube-5-100-200.zarr/.zattrs @@ -1,3 +1,4 @@ { - "Conventions": "CF-1.7" + "Conventions": "CF-1.7", + "coordinates": "time_bnds lat_bnds lon_bnds" } \ No newline at end of file From 93025217fb023f79ea2c6b5cbd86fffe2b4e480b Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 25 Jul 2019 18:46:26 +0200 Subject: [PATCH 07/10] fixed place group handling --- xcube/webapi/context.py | 132 ++++++++++++++--------------- xcube/webapi/controllers/places.py | 9 +- 2 files changed, 68 insertions(+), 73 deletions(-) diff --git a/xcube/webapi/context.py b/xcube/webapi/context.py index 0da4201e0..7bf35f1b4 100644 --- a/xcube/webapi/context.py +++ b/xcube/webapi/context.py @@ -270,13 +270,6 @@ def get_legend_label(self, ds_name: str, var_name: str): return units raise ServiceResourceNotFoundError(f'Variable "{var_name}" not found in dataset "{ds_name}"') - def get_global_place_groups(self, load_features=True) -> List[Dict]: - place_groups = self._load_place_groups(self._config.get("PlaceGroups", []), is_global=True) - if load_features: - for place_group in place_groups: - self.load_place_group_features(place_group) - return place_groups - def get_dataset_place_groups(self, ds_id: str, load_features=False) -> List[Dict]: dataset_descriptor = self.get_dataset_descriptor(ds_id) @@ -290,86 +283,89 @@ def get_dataset_place_groups(self, ds_id: str, load_features=False) -> List[Dict if place_groups: return place_groups - place_groups = self._load_place_groups(dataset_descriptor.get("PlaceGroups", []), is_global=False) - if load_features: - for place_group in place_groups: - self.load_place_group_features(place_group) + place_groups = self._load_place_groups(dataset_descriptor.get("PlaceGroups", []), + is_global=False, load_features=load_features) for place_group in place_groups: self._place_group_cache[place_group_id_prefix + place_group["id"]] = place_group return place_groups - def get_global_place_group(self, place_group_id: str, load_features: bool = False) -> Dict: - self._cache_global_place_groups() - if place_group_id not in self._place_group_cache: - raise ServiceResourceNotFoundError(f'Place group "{place_group_id}" not found') - place_group = self._place_group_cache[place_group_id] - if load_features: - self.load_place_group_features(place_group) - return place_group + def get_global_place_groups(self, load_features=False) -> List[Dict]: + return self._load_place_groups(self._config.get("PlaceGroups", []), is_global=True, load_features=load_features) - def _cache_global_place_groups(self): - if not self._place_group_cache: - for place_group in self.get_global_place_groups(): - self._place_group_cache[place_group['id']] = place_group - - def _load_place_groups(self, place_group_configs: Dict, is_global: bool = False) -> List[Dict]: + def get_global_place_group(self, place_group_id: str, load_features: bool = False) -> Dict: + place_group_descriptor = self._get_place_group_descriptor(place_group_id) + return self._load_place_group(place_group_descriptor, is_global=True, load_features=load_features) + + def _get_place_group_descriptor(self, place_group_id: str) -> Dict: + place_group_descriptors = self._config.get("PlaceGroups", []) + for place_group_descriptor in place_group_descriptors: + if place_group_descriptor['Identifier'] == place_group_id: + return place_group_descriptor + raise ServiceResourceNotFoundError(f'Place group "{place_group_id}" not found') + + def _load_place_groups(self, + place_group_descriptors: Dict, + is_global: bool = False, + load_features: bool = False) -> List[Dict]: place_groups = [] - for place_group_config in place_group_configs: - place_group = self._load_place_group(place_group_config, is_global=is_global) + for place_group_descriptor in place_group_descriptors: + place_group = self._load_place_group(place_group_descriptor, is_global=is_global, load_features=load_features) place_groups.append(place_group) return place_groups - def _load_place_group(self, place_group_config: Dict[str, Any], is_global: bool = False) -> Dict[str, Any]: - ref_id = place_group_config.get("PlaceGroupRef") - if ref_id: + def _load_place_group(self, place_group_descriptor: Dict[str, Any], is_global: bool = False, load_features: bool = False) -> Dict[str, Any]: + place_group_id = place_group_descriptor.get("PlaceGroupRef") + if place_group_id: if is_global: raise ServiceError("'PlaceGroupRef' cannot be used in a global place group") - # Trigger loading of all global "PlaceGroup" entries - self._cache_global_place_groups() - if len(place_group_config) > 1: + if len(place_group_descriptor) > 1: raise ServiceError("'PlaceGroupRef' if present, must be the only entry in a 'PlaceGroups' item") - if ref_id not in self._place_group_cache: - raise ServiceError("Invalid 'PlaceGroupRef' entry in a 'PlaceGroups' item") - return self._place_group_cache[ref_id] + return self.get_global_place_group(place_group_id, load_features=load_features) - place_group_id = place_group_config.get("Identifier") + place_group_id = place_group_descriptor.get("Identifier") if not place_group_id: raise ServiceError("Missing 'Identifier' entry in a 'PlaceGroups' item") - if place_group_id == ALL_PLACES: - raise ServiceError("Invalid 'Identifier' entry in a 'PlaceGroups' item") - - place_group_title = place_group_config.get("Title", place_group_id) - - place_path_wc = place_group_config.get("Path") - if not place_path_wc: - raise ServiceError("Missing 'Path' entry in a 'PlaceGroups' item") - if not os.path.isabs(place_path_wc): - place_path_wc = os.path.join(self._base_dir, place_path_wc) - source_paths = glob.glob(place_path_wc) - source_encoding = place_group_config.get("CharacterEncoding", "utf-8") - - property_mapping = place_group_config.get("PropertyMapping") - - place_group = dict(type="FeatureCollection", - features=None, - id=place_group_id, - title=place_group_title, - propertyMapping=property_mapping, - sourcePaths=source_paths, - sourceEncoding=source_encoding) - - sub_place_group_configs = place_group_config.get("Places") - if sub_place_group_configs: - raise ServiceError("Invalid 'Places' entry in a 'PlaceGroups' item: not implemented yet") - # sub_place_group_configs = place_group_config.get("Places") - # if sub_place_group_configs: - # sub_place_groups = self._load_place_groups(sub_place_group_configs) - # place_group["placeGroups"] = sub_place_groups + + if place_group_id in self._place_group_cache: + place_group = self._place_group_cache[place_group_id] + else: + place_group_title = place_group_descriptor.get("Title", place_group_id) + + place_path_wc = place_group_descriptor.get("Path") + if not place_path_wc: + raise ServiceError("Missing 'Path' entry in a 'PlaceGroups' item") + if not os.path.isabs(place_path_wc): + place_path_wc = os.path.join(self._base_dir, place_path_wc) + source_paths = glob.glob(place_path_wc) + source_encoding = place_group_descriptor.get("CharacterEncoding", "utf-8") + + property_mapping = place_group_descriptor.get("PropertyMapping") + + place_group = dict(type="FeatureCollection", + features=None, + id=place_group_id, + title=place_group_title, + propertyMapping=property_mapping, + sourcePaths=source_paths, + sourceEncoding=source_encoding) + + sub_place_group_configs = place_group_descriptor.get("Places") + if sub_place_group_configs: + raise ServiceError("Invalid 'Places' entry in a 'PlaceGroups' item: not implemented yet") + # sub_place_group_descriptors = place_group_config.get("Places") + # if sub_place_group_descriptors: + # sub_place_groups = self._load_place_groups(sub_place_group_descriptors) + # place_group["placeGroups"] = sub_place_groups + + self._place_group_cache[place_group_id] = place_group + + if load_features: + self._load_place_group_features(place_group) return place_group - def load_place_group_features(self, place_group: Dict[str, Any]) -> List[Dict[str, Any]]: + def _load_place_group_features(self, place_group: Dict[str, Any]) -> List[Dict[str, Any]]: features = place_group.get('features') if features is not None: return features diff --git a/xcube/webapi/controllers/places.py b/xcube/webapi/controllers/places.py index bd10e12c2..f98b79636 100644 --- a/xcube/webapi/controllers/places.py +++ b/xcube/webapi/controllers/places.py @@ -50,7 +50,7 @@ def find_places(ctx: ServiceContext, elif geojson_obj: try: if geojson_obj["type"] == "FeatureCollection": - query_geometry = shapely.geometry.shape(geojson_obj["places"][0]["geometry"]) + query_geometry = shapely.geometry.shape(geojson_obj["features"][0]["geometry"]) elif geojson_obj["type"] == "Feature": query_geometry = shapely.geometry.shape(geojson_obj["geometry"]) else: @@ -80,11 +80,10 @@ def __find_places(ctx: ServiceContext, raise NotImplementedError("comb_op not yet supported") if place_group_id == ALL_PLACES: - place_groups = ctx.get_global_place_groups() + place_groups = ctx.get_global_place_groups(load_features=True) features = [] - for pg in place_groups: - pg_features = ctx.load_place_group_features(pg) - features.extend(pg_features) + for place_group in place_groups: + features.extend(place_group['features']) feature_collection = dict(type="FeatureCollection", features=features) else: feature_collection = ctx.get_global_place_group(place_group_id, load_features=True) From 1375d39d2459e974ff9cafe6d9db3b56e71846e4 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 25 Jul 2019 18:47:58 +0200 Subject: [PATCH 08/10] no longer return features in response --- xcube/webapi/controllers/catalogue.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/xcube/webapi/controllers/catalogue.py b/xcube/webapi/controllers/catalogue.py index 3239b82bb..ba9b4ad8e 100644 --- a/xcube/webapi/controllers/catalogue.py +++ b/xcube/webapi/controllers/catalogue.py @@ -109,14 +109,13 @@ def get_dataset(ctx: ServiceContext, ds_id: str, client=None, base_url: str = No dim_names = ds.data_vars[list(ds.data_vars)[0]].dims if len(ds.data_vars) > 0 else ds.dims.keys() dataset_dict["dimensions"] = [get_dataset_coordinates(ctx, ds_id, dim_name) for dim_name in dim_names] - # place_groups = ctx.get_dataset_place_groups(ds_id, load_features=False) - place_groups = ctx.get_dataset_place_groups(ds_id, load_features=True) + place_groups = ctx.get_dataset_place_groups(ds_id) if place_groups: def filter_place_group(place_group: Dict): place_group = dict(place_group) del place_group['sourcePaths'] del place_group['sourceEncoding'] - # place_group['features'] = None + place_group['features'] = None return place_group place_groups = list(map(filter_place_group, place_groups)) dataset_dict["placeGroups"] = place_groups From e0958d0991fc6cfde9bac21116a82da845752020 Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Thu, 25 Jul 2019 19:27:30 +0200 Subject: [PATCH 09/10] new webapi: datasets/{dataset}/places --- test/webapi/test_handlers.py | 24 +++++++++++++---------- xcube/webapi/app.py | 4 +++- xcube/webapi/controllers/catalogue.py | 11 +++++++++-- xcube/webapi/handlers.py | 28 ++++++++++++++++++--------- 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/test/webapi/test_handlers.py b/test/webapi/test_handlers.py index c585ea0dc..e196df56e 100644 --- a/test/webapi/test_handlers.py +++ b/test/webapi/test_handlers.py @@ -135,11 +135,11 @@ def test_fetch_wmts_tile_with_params(self): response = self.fetch(self.prefix + '/wmts/1.0.0/tile/demo/conc_chl/0/0/0.png?time=current&cbar=jet') self.assertResponseOK(response) - def test_fetch_datasets_json(self): + def test_fetch_datasets(self): response = self.fetch(self.prefix + '/datasets') self.assertResponseOK(response) - def test_fetch_datasets_details_json(self): + def test_fetch_datasets_details(self): response = self.fetch(self.prefix + '/datasets?details=1') self.assertResponseOK(response) response = self.fetch(self.prefix + '/datasets?details=1&tiles=cesium') @@ -151,7 +151,7 @@ def test_fetch_datasets_details_json(self): "Parameter 'point' parameter must be a point using format" " ',', but was '2,8a,51.0'") - def test_fetch_dataset_json(self): + def test_fetch_dataset(self): response = self.fetch(self.prefix + '/datasets/demo') self.assertResponseOK(response) response = self.fetch(self.prefix + '/datasets/demo?tiles=ol4') @@ -159,6 +159,14 @@ def test_fetch_dataset_json(self): response = self.fetch(self.prefix + '/datasets/demo?tiles=cesium') self.assertResponseOK(response) + def test_fetch_dataset_places(self): + response = self.fetch(self.prefix + '/datasets/demo/places') + self.assertResponseOK(response) + + def test_fetch_dataset_coords(self): + response = self.fetch(self.prefix + '/datasets/demo/coords/time') + self.assertResponseOK(response) + def test_fetch_list_s3bucket(self): response = self.fetch(self.prefix + '/s3bucket') self.assertResponseOK(response) @@ -205,10 +213,6 @@ def _assert_fetch_head_s3bucket_object(self, method): response = self.fetch(self.prefix + '/s3bucket/demo/conc_chl/1.2.4', method=method) self.assertResponseOK(response) - def test_fetch_coords_json(self): - response = self.fetch(self.prefix + '/datasets/demo/coords/time') - self.assertResponseOK(response) - def test_fetch_dataset_tile(self): response = self.fetch(self.prefix + '/datasets/demo/vars/conc_chl/tiles/0/0/0.png') self.assertResponseOK(response) @@ -242,17 +246,17 @@ def test_fetch_color_bars_html(self): response = self.fetch(self.prefix + '/colorbars.html') self.assertResponseOK(response) - def test_fetch_feature_collections(self): + def test_fetch_all_places(self): response = self.fetch(self.prefix + '/places') self.assertResponseOK(response) - def test_fetch_features(self): + def test_fetch_places(self): response = self.fetch(self.prefix + '/places/all') self.assertResponseOK(response) response = self.fetch(self.prefix + '/places/all?bbox=10,10,20,20') self.assertResponseOK(response) - def test_fetch_features_for_dataset(self): + def test_fetch_dataset_places(self): response = self.fetch(self.prefix + '/places/all/demo') self.assertResponseOK(response) response = self.fetch(self.prefix + '/places/inside-cube/demo') diff --git a/xcube/webapi/app.py b/xcube/webapi/app.py index ca2930e46..625fdceca 100644 --- a/xcube/webapi/app.py +++ b/xcube/webapi/app.py @@ -30,7 +30,7 @@ GetDatasetCoordsHandler, GetTimeSeriesInfoHandler, GetTimeSeriesForPointHandler, WMTSKvpHandler, \ GetTimeSeriesForGeometryHandler, GetTimeSeriesForFeaturesHandler, GetTimeSeriesForGeometriesHandler, \ GetPlaceGroupsHandler, GetDatasetVarLegendHandler, GetDatasetHandler, GetWMTSTileHandler, GetS3BucketObjectHandler, \ - ListS3BucketHandler + ListS3BucketHandler, GetDatasetPlacesHandler from .service import url_pattern __author__ = "Norman Fomferra (Brockmann Consult GmbH)" @@ -56,6 +56,8 @@ def new_application(prefix: str = None): GetDatasetsHandler), (prefix + url_pattern('/datasets/{{ds_id}}'), GetDatasetHandler), + (prefix + url_pattern('/datasets/{{ds_id}}/places'), + GetDatasetPlacesHandler), (prefix + url_pattern('/datasets/{{ds_id}}/coords/{{dim_name}}'), GetDatasetCoordsHandler), (prefix + url_pattern('/datasets/{{ds_id}}/vars/{{var_name}}/legend.png'), diff --git a/xcube/webapi/controllers/catalogue.py b/xcube/webapi/controllers/catalogue.py index ba9b4ad8e..b36f36cc5 100644 --- a/xcube/webapi/controllers/catalogue.py +++ b/xcube/webapi/controllers/catalogue.py @@ -1,9 +1,10 @@ import functools import json -from typing import Dict, Tuple +from typing import Dict, Tuple, List import numpy as np +from xcube.webapi.controllers.places import GeoJsonFeatureCollection from ..context import ServiceContext from ..controllers.tiles import get_tile_source_options, get_dataset_tile_url from ..errors import ServiceBadRequestError @@ -62,7 +63,7 @@ def get_datasets(ctx: ServiceContext, return dict(datasets=dataset_dicts) -def get_dataset(ctx: ServiceContext, ds_id: str, client=None, base_url: str = None) -> Dict: +def get_dataset(ctx: ServiceContext, ds_id: str, client=None, base_url: str = None) -> GeoJsonFeatureCollection: dataset_descriptor = ctx.get_dataset_descriptor(ds_id) ds_id = dataset_descriptor['Identifier'] @@ -117,12 +118,18 @@ def filter_place_group(place_group: Dict): del place_group['sourceEncoding'] place_group['features'] = None return place_group + place_groups = list(map(filter_place_group, place_groups)) dataset_dict["placeGroups"] = place_groups return dataset_dict +def get_dataset_places(ctx: ServiceContext, ds_id: str) -> List[GeoJsonFeatureCollection]: + place_groups = ctx.get_dataset_place_groups(ds_id, load_features=True) + return place_groups + + def get_dataset_coordinates(ctx: ServiceContext, ds_id: str, dim_name: str) -> Dict: ds, var = ctx.get_dataset_and_coord_variable(ds_id, dim_name) values = list() diff --git a/xcube/webapi/handlers.py b/xcube/webapi/handlers.py index 9e8a82b8a..8a7523475 100644 --- a/xcube/webapi/handlers.py +++ b/xcube/webapi/handlers.py @@ -27,7 +27,8 @@ from tornado.ioloop import IOLoop -from .controllers.catalogue import get_datasets, get_dataset_coordinates, get_color_bars, get_dataset +from .controllers.catalogue import get_datasets, get_dataset_coordinates, get_color_bars, get_dataset, \ + get_dataset_places from .controllers.places import find_places, find_dataset_places from .controllers.tiles import get_dataset_tile, get_dataset_tile_grid, get_ne2_tile, get_ne2_tile_grid, get_legend from .controllers.time_series import get_time_series_info, get_time_series_for_point, get_time_series_for_geometry, \ @@ -49,6 +50,7 @@ _LOG_S3BUCKET_HANDLER = False + # noinspection PyAbstractClass class WMTSKvpHandler(ServiceRequestHandler): @@ -137,6 +139,22 @@ def get(self, ds_id: str): self.write(json.dumps(response, indent=2)) +class GetDatasetPlacesHandler(ServiceRequestHandler): + + def get(self, ds_id: str): + response = get_dataset_places(self.service_context, ds_id) + self.set_header('Content-Type', 'application/json') + self.write(json.dumps(response)) + + +class GetDatasetCoordsHandler(ServiceRequestHandler): + + def get(self, ds_id: str, dim_name: str): + response = get_dataset_coordinates(self.service_context, ds_id, dim_name) + self.set_header('Content-Type', 'application/json') + self.write(json.dumps(response)) + + # noinspection PyAbstractClass class ListS3BucketHandler(ServiceRequestHandler): @@ -250,14 +268,6 @@ def _get_key_and_local_path(self, ds_id: str, path: str): return key, pathlib.Path(local_path) -class GetDatasetCoordsHandler(ServiceRequestHandler): - - def get(self, ds_id: str, dim_name: str): - response = get_dataset_coordinates(self.service_context, ds_id, dim_name) - self.set_header('Content-Type', 'application/json') - self.write(json.dumps(response, indent=2)) - - # noinspection PyAbstractClass,PyBroadException class GetWMTSTileHandler(ServiceRequestHandler): From e86f28798afa153cd2897cf712bcef93de4e38db Mon Sep 17 00:00:00 2001 From: Norman Fomferra Date: Fri, 26 Jul 2019 16:30:03 +0200 Subject: [PATCH 10/10] closes #130 --- xcube/webapi/app.py | 6 ++-- xcube/webapi/context.py | 15 +++++++-- xcube/webapi/controllers/catalogue.py | 44 +++++++++++++++++++-------- xcube/webapi/handlers.py | 14 +++++++-- 4 files changed, 59 insertions(+), 20 deletions(-) diff --git a/xcube/webapi/app.py b/xcube/webapi/app.py index 625fdceca..d935bdc09 100644 --- a/xcube/webapi/app.py +++ b/xcube/webapi/app.py @@ -30,7 +30,7 @@ GetDatasetCoordsHandler, GetTimeSeriesInfoHandler, GetTimeSeriesForPointHandler, WMTSKvpHandler, \ GetTimeSeriesForGeometryHandler, GetTimeSeriesForFeaturesHandler, GetTimeSeriesForGeometriesHandler, \ GetPlaceGroupsHandler, GetDatasetVarLegendHandler, GetDatasetHandler, GetWMTSTileHandler, GetS3BucketObjectHandler, \ - ListS3BucketHandler, GetDatasetPlacesHandler + ListS3BucketHandler, GetDatasetPlaceGroupsHandler, GetDatasetPlaceGroupHandler from .service import url_pattern __author__ = "Norman Fomferra (Brockmann Consult GmbH)" @@ -57,7 +57,9 @@ def new_application(prefix: str = None): (prefix + url_pattern('/datasets/{{ds_id}}'), GetDatasetHandler), (prefix + url_pattern('/datasets/{{ds_id}}/places'), - GetDatasetPlacesHandler), + GetDatasetPlaceGroupsHandler), + (prefix + url_pattern('/datasets/{{ds_id}}/places/{{place_group_id}}'), + GetDatasetPlaceGroupHandler), (prefix + url_pattern('/datasets/{{ds_id}}/coords/{{dim_name}}'), GetDatasetCoordsHandler), (prefix + url_pattern('/datasets/{{ds_id}}/vars/{{var_name}}/legend.png'), diff --git a/xcube/webapi/context.py b/xcube/webapi/context.py index 7bf35f1b4..906e464f6 100644 --- a/xcube/webapi/context.py +++ b/xcube/webapi/context.py @@ -290,6 +290,15 @@ def get_dataset_place_groups(self, ds_id: str, load_features=False) -> List[Dict return place_groups + def get_dataset_place_group(self, ds_id: str, place_group_id: str, load_features=False) -> Dict: + place_groups = self.get_dataset_place_groups(ds_id, load_features=False) + for place_group in place_groups: + if place_group_id == place_group['id']: + if load_features: + self._load_place_group_features(place_group) + return place_group + raise ServiceResourceNotFoundError(f'Place group "{place_group_id}" not found') + def get_global_place_groups(self, load_features=False) -> List[Dict]: return self._load_place_groups(self._config.get("PlaceGroups", []), is_global=True, load_features=load_features) @@ -310,11 +319,13 @@ def _load_place_groups(self, load_features: bool = False) -> List[Dict]: place_groups = [] for place_group_descriptor in place_group_descriptors: - place_group = self._load_place_group(place_group_descriptor, is_global=is_global, load_features=load_features) + place_group = self._load_place_group(place_group_descriptor, is_global=is_global, + load_features=load_features) place_groups.append(place_group) return place_groups - def _load_place_group(self, place_group_descriptor: Dict[str, Any], is_global: bool = False, load_features: bool = False) -> Dict[str, Any]: + def _load_place_group(self, place_group_descriptor: Dict[str, Any], is_global: bool = False, + load_features: bool = False) -> Dict[str, Any]: place_group_id = place_group_descriptor.get("PlaceGroupRef") if place_group_id: if is_global: diff --git a/xcube/webapi/controllers/catalogue.py b/xcube/webapi/controllers/catalogue.py index b36f36cc5..8af3e6368 100644 --- a/xcube/webapi/controllers/catalogue.py +++ b/xcube/webapi/controllers/catalogue.py @@ -4,8 +4,8 @@ import numpy as np -from xcube.webapi.controllers.places import GeoJsonFeatureCollection from ..context import ServiceContext +from ..controllers.places import GeoJsonFeatureCollection from ..controllers.tiles import get_tile_source_options, get_dataset_tile_url from ..errors import ServiceBadRequestError from ..im.cmaps import get_cmaps @@ -93,7 +93,8 @@ def get_dataset(ctx: ServiceContext, ds_id: str, client=None, base_url: str = No if client is not None: tile_grid = ctx.get_tile_grid(ds_id) tile_xyz_source_options = get_tile_source_options(tile_grid, - get_dataset_tile_url(ctx, ds_id, var_name, + get_dataset_tile_url(ctx, ds_id, + var_name, base_url), client=client) variable_dict["tileSourceOptions"] = tile_xyz_source_options @@ -112,22 +113,19 @@ def get_dataset(ctx: ServiceContext, ds_id: str, client=None, base_url: str = No place_groups = ctx.get_dataset_place_groups(ds_id) if place_groups: - def filter_place_group(place_group: Dict): - place_group = dict(place_group) - del place_group['sourcePaths'] - del place_group['sourceEncoding'] - place_group['features'] = None - return place_group - - place_groups = list(map(filter_place_group, place_groups)) - dataset_dict["placeGroups"] = place_groups + dataset_dict["placeGroups"] = _filter_place_groups(place_groups, del_features=True) return dataset_dict -def get_dataset_places(ctx: ServiceContext, ds_id: str) -> List[GeoJsonFeatureCollection]: +def get_dataset_place_groups(ctx: ServiceContext, ds_id: str) -> List[GeoJsonFeatureCollection]: place_groups = ctx.get_dataset_place_groups(ds_id, load_features=True) - return place_groups + return _filter_place_groups(place_groups, del_features=False) + + +def get_dataset_place_group(ctx: ServiceContext, ds_id: str, place_group_id: str) -> GeoJsonFeatureCollection: + place_group = ctx.get_dataset_place_group(ds_id, place_group_id, load_features=False) + return _filter_place_group(place_group, del_features=False) def get_dataset_coordinates(ctx: ServiceContext, ds_id: str, dim_name: str) -> Dict: @@ -189,3 +187,23 @@ def _is_point_in_dataset_bbox(point: Tuple[float, float], dataset_dict: Dict): else: # Bounding box crosses antimeridian return x_min <= x <= 180.0 or -180.0 <= x <= x_max + + +def _filter_place_group(place_group: Dict, del_features: bool = False) -> Dict: + place_group = dict(place_group) + del place_group['sourcePaths'] + del place_group['sourceEncoding'] + if del_features: + del place_group['features'] + return place_group + + +def _filter_place_groups(place_groups, del_features: bool = False) -> List[Dict]: + if del_features: + def __filter_place_group(place_group): + return _filter_place_group(place_group, del_features=True) + else: + def __filter_place_group(place_group): + return _filter_place_group(place_group, del_features=False) + + return list(map(__filter_place_group, place_groups)) diff --git a/xcube/webapi/handlers.py b/xcube/webapi/handlers.py index 8a7523475..f317a73e1 100644 --- a/xcube/webapi/handlers.py +++ b/xcube/webapi/handlers.py @@ -28,7 +28,7 @@ from tornado.ioloop import IOLoop from .controllers.catalogue import get_datasets, get_dataset_coordinates, get_color_bars, get_dataset, \ - get_dataset_places + get_dataset_place_groups, get_dataset_place_group from .controllers.places import find_places, find_dataset_places from .controllers.tiles import get_dataset_tile, get_dataset_tile_grid, get_ne2_tile, get_ne2_tile_grid, get_legend from .controllers.time_series import get_time_series_info, get_time_series_for_point, get_time_series_for_geometry, \ @@ -139,10 +139,18 @@ def get(self, ds_id: str): self.write(json.dumps(response, indent=2)) -class GetDatasetPlacesHandler(ServiceRequestHandler): +class GetDatasetPlaceGroupsHandler(ServiceRequestHandler): def get(self, ds_id: str): - response = get_dataset_places(self.service_context, ds_id) + response = get_dataset_place_groups(self.service_context, ds_id) + self.set_header('Content-Type', 'application/json') + self.write(json.dumps(response)) + + +class GetDatasetPlaceGroupHandler(ServiceRequestHandler): + + def get(self, ds_id: str, place_group_id: str): + response = get_dataset_place_group(self.service_context, ds_id, place_group_id) self.set_header('Content-Type', 'application/json') self.write(json.dumps(response))