Forman 130 lazy load dataset details

test/webapi/controllers/test_places.py

@@ -46,12 +46,12 @@ def test_find_places_by_geojson(self):
         places = find_places(ctx, "all", geojson_obj=geojson_obj)
         self._assertPlaceGroup(places, 2, {'0', '3'})
 
-        geojson_obj = {'type': 'FeatureCollection', 'places': [geojson_obj]}
+        geojson_obj = {'type': 'FeatureCollection', 'features': [geojson_obj]}
         places = find_places(ctx, "all", geojson_obj=geojson_obj)
         self._assertPlaceGroup(places, 2, {'0', '3'})
 
         with self.assertRaises(ServiceBadRequestError) as cm:
-            geojson_obj = {'type': 'FeatureCollection', 'places': []}
+            geojson_obj = {'type': 'FeatureCollection', 'features': []}
             find_places(ctx, "all", geojson_obj=geojson_obj)
         self.assertEqual("HTTP 400: Received invalid GeoJSON object", f"{cm.exception}")
 

test/webapi/test_context.py

@@ -71,39 +71,60 @@ def test_get_color_mapping(self):
         cm = ctx.get_color_mapping('demo', '_')
         self.assertEqual(('jet', 0., 1.), cm)
 
-    def test_get_feature_collections(self):
+    def test_get_global_place_groups(self):
         ctx = new_test_service_context()
-        feature_collections = ctx.get_place_groups()
-        self.assertIsInstance(feature_collections, list)
-        self.assertEqual([{'id': 'inside-cube', 'title': 'Points inside the cube'},
-                          {'id': 'outside-cube', 'title': 'Points outside the cube'}],
-                         feature_collections)
-
-    def test_get_place_group(self):
-        ctx = new_test_service_context()
-        place_group = ctx.get_place_group()
-        self.assertIsInstance(place_group, dict)
-        self.assertIn("type", place_group)
-        self.assertEqual("FeatureCollection", place_group["type"])
-        self.assertIn("features", place_group)
-        self.assertIsInstance(place_group["features"], list)
-        self.assertEqual(6, len(place_group["features"]))
-        self.assertIs(place_group, ctx.get_place_group())
-        self.assertEqual([str(i) for i in range(6)],
-                         [f["id"] for f in place_group["features"] if "id" in f])
-
-    def test_get_place_group_by_name(self):
+        place_groups = ctx.get_global_place_groups(load_features=False)
+        self.assertIsInstance(place_groups, list)
+        self.assertEqual(2, len(place_groups))
+        for place_group in place_groups:
+            self.assertIn("type", place_group)
+            self.assertIn("id", place_group)
+            self.assertIn("title", place_group)
+            self.assertIn("propertyMapping", place_group)
+            self.assertIn("sourcePaths", place_group)
+            self.assertIn("sourceEncoding", place_group)
+            self.assertIn("features", place_group)
+        place_group = place_groups[0]
+        self.assertEqual('inside-cube', place_group['id'])
+        self.assertEqual('Points inside the cube', place_group['title'])
+        self.assertEqual(None, place_group['features'])
+        place_group = place_groups[1]
+        self.assertEqual('outside-cube', place_group['id'])
+        self.assertEqual('Points outside the cube', place_group['title'])
+        self.assertEqual(None, place_group['features'])
+
+        place_groups = ctx.get_global_place_groups(load_features=True)
+        self.assertIsInstance(place_groups, list)
+        self.assertEqual(2, len(place_groups))
+        for place_group in place_groups:
+            self.assertIn("type", place_group)
+            self.assertIn("id", place_group)
+            self.assertIn("title", place_group)
+            self.assertIn("propertyMapping", place_group)
+            self.assertIn("sourcePaths", place_group)
+            self.assertIn("sourceEncoding", place_group)
+            self.assertIn("features", place_group)
+        place_group = place_groups[0]
+        self.assertEqual('inside-cube', place_group['id'])
+        self.assertEqual('Points inside the cube', place_group['title'])
+        self.assertIsNotNone(place_group['features'])
+        place_group = place_groups[1]
+        self.assertEqual('outside-cube', place_group['id'])
+        self.assertEqual('Points outside the cube', place_group['title'])
+        self.assertIsNotNone(place_group['features'])
+
+    def test_get_global_place_group(self):
         ctx = new_test_service_context()
-        place_group = ctx.get_place_group(place_group_id="inside-cube")
+        place_group = ctx.get_global_place_group("inside-cube", load_features=True)
         self.assertIsInstance(place_group, dict)
         self.assertIn("type", place_group)
         self.assertEqual("FeatureCollection", place_group["type"])
         self.assertIn("features", place_group)
         self.assertIsInstance(place_group["features"], list)
         self.assertEqual(3, len(place_group["features"]))
-        self.assertIs(place_group, ctx.get_place_group(place_group_id="inside-cube"))
-        self.assertIsNot(place_group, ctx.get_place_group(place_group_id="outside-cube"))
+        self.assertIs(place_group, ctx.get_global_place_group(place_group_id="inside-cube"))
+        self.assertIsNot(place_group, ctx.get_global_place_group(place_group_id="outside-cube"))
 
         with self.assertRaises(ServiceResourceNotFoundError) as cm:
-            ctx.get_place_group(place_group_id="bibo")
+            ctx.get_global_place_group("bibo")
         self.assertEqual('HTTP 404: Place group "bibo" not found', f"{cm.exception}")

test/webapi/test_handlers.py

@@ -135,11 +135,11 @@ def test_fetch_wmts_tile_with_params(self):
         response = self.fetch(self.prefix + '/wmts/1.0.0/tile/demo/conc_chl/0/0/0.png?time=current&cbar=jet')
         self.assertResponseOK(response)
 
-    def test_fetch_datasets_json(self):
+    def test_fetch_datasets(self):
         response = self.fetch(self.prefix + '/datasets')
         self.assertResponseOK(response)
 
-    def test_fetch_datasets_details_json(self):
+    def test_fetch_datasets_details(self):
         response = self.fetch(self.prefix + '/datasets?details=1')
         self.assertResponseOK(response)
         response = self.fetch(self.prefix + '/datasets?details=1&tiles=cesium')
@@ -151,14 +151,22 @@ def test_fetch_datasets_details_json(self):
                                       "Parameter 'point' parameter must be a point using format"
                                       " '<lon>,<lat>', but was '2,8a,51.0'")
 
-    def test_fetch_dataset_json(self):
+    def test_fetch_dataset(self):
         response = self.fetch(self.prefix + '/datasets/demo')
         self.assertResponseOK(response)
         response = self.fetch(self.prefix + '/datasets/demo?tiles=ol4')
         self.assertResponseOK(response)
         response = self.fetch(self.prefix + '/datasets/demo?tiles=cesium')
         self.assertResponseOK(response)
 
+    def test_fetch_dataset_places(self):
+        response = self.fetch(self.prefix + '/datasets/demo/places')
+        self.assertResponseOK(response)
+
+    def test_fetch_dataset_coords(self):
+        response = self.fetch(self.prefix + '/datasets/demo/coords/time')
+        self.assertResponseOK(response)
+
     def test_fetch_list_s3bucket(self):
         response = self.fetch(self.prefix + '/s3bucket')
         self.assertResponseOK(response)
@@ -205,10 +213,6 @@ def _assert_fetch_head_s3bucket_object(self, method):
         response = self.fetch(self.prefix + '/s3bucket/demo/conc_chl/1.2.4', method=method)
         self.assertResponseOK(response)
 
-    def test_fetch_coords_json(self):
-        response = self.fetch(self.prefix + '/datasets/demo/coords/time')
-        self.assertResponseOK(response)
-
     def test_fetch_dataset_tile(self):
         response = self.fetch(self.prefix + '/datasets/demo/vars/conc_chl/tiles/0/0/0.png')
         self.assertResponseOK(response)
@@ -242,17 +246,17 @@ def test_fetch_color_bars_html(self):
         response = self.fetch(self.prefix + '/colorbars.html')
         self.assertResponseOK(response)
 
-    def test_fetch_feature_collections(self):
+    def test_fetch_all_places(self):
         response = self.fetch(self.prefix + '/places')
         self.assertResponseOK(response)
 
-    def test_fetch_features(self):
+    def test_fetch_places(self):
         response = self.fetch(self.prefix + '/places/all')
         self.assertResponseOK(response)
         response = self.fetch(self.prefix + '/places/all?bbox=10,10,20,20')
         self.assertResponseOK(response)
 
-    def test_fetch_features_for_dataset(self):
+    def test_fetch_dataset_places(self):
         response = self.fetch(self.prefix + '/places/all/demo')
         self.assertResponseOK(response)
         response = self.fetch(self.prefix + '/places/inside-cube/demo')

xcube/webapi/app.py

@@ -30,7 +30,7 @@
     GetDatasetCoordsHandler, GetTimeSeriesInfoHandler, GetTimeSeriesForPointHandler, WMTSKvpHandler, \
     GetTimeSeriesForGeometryHandler, GetTimeSeriesForFeaturesHandler, GetTimeSeriesForGeometriesHandler, \
     GetPlaceGroupsHandler, GetDatasetVarLegendHandler, GetDatasetHandler, GetWMTSTileHandler, GetS3BucketObjectHandler, \
-    ListS3BucketHandler
+    ListS3BucketHandler, GetDatasetPlaceGroupsHandler, GetDatasetPlaceGroupHandler
 from .service import url_pattern
 
 __author__ = "Norman Fomferra (Brockmann Consult GmbH)"
@@ -56,6 +56,10 @@ def new_application(prefix: str = None):
          GetDatasetsHandler),
         (prefix + url_pattern('/datasets/{{ds_id}}'),
          GetDatasetHandler),
+        (prefix + url_pattern('/datasets/{{ds_id}}/places'),
+         GetDatasetPlaceGroupsHandler),
+        (prefix + url_pattern('/datasets/{{ds_id}}/places/{{place_group_id}}'),
+         GetDatasetPlaceGroupHandler),
         (prefix + url_pattern('/datasets/{{ds_id}}/coords/{{dim_name}}'),
          GetDatasetCoordsHandler),
         (prefix + url_pattern('/datasets/{{ds_id}}/vars/{{var_name}}/legend.png'),

xcube/webapi/context.py

@@ -270,109 +270,128 @@ def get_legend_label(self, ds_name: str, var_name: str):
             return units
         raise ServiceResourceNotFoundError(f'Variable "{var_name}" not found in dataset "{ds_name}"')
 
-    def get_place_groups(self) -> List[Dict]:
-        place_group_configs = self._config.get("PlaceGroups", [])
-        place_groups = []
-        for features_config in place_group_configs:
-            place_groups.append(dict(id=features_config.get("Identifier"),
-                                     title=features_config.get("Title")))
-        return place_groups
-
-    def get_dataset_place_groups(self, ds_id: str) -> List[Dict]:
+    def get_dataset_place_groups(self, ds_id: str, load_features=False) -> List[Dict]:
         dataset_descriptor = self.get_dataset_descriptor(ds_id)
-        place_group_configs = dataset_descriptor.get("PlaceGroups")
-        if not place_group_configs:
-            return []
 
         place_group_id_prefix = f"DS-{ds_id}-"
 
         place_groups = []
         for k, v in self._place_group_cache.items():
             if k.startswith(place_group_id_prefix):
                 place_groups.append(v)
+
         if place_groups:
             return place_groups
 
-        place_groups = self._load_place_groups(place_group_configs)
+        place_groups = self._load_place_groups(dataset_descriptor.get("PlaceGroups", []),
+                                               is_global=False, load_features=load_features)
         for place_group in place_groups:
             self._place_group_cache[place_group_id_prefix + place_group["id"]] = place_group
 
         return place_groups
 
-    def get_place_group(self, place_group_id: str = ALL_PLACES) -> Dict:
-        if ALL_PLACES not in self._place_group_cache:
-            place_group_configs = self._config.get("PlaceGroups", [])
-            place_groups = self._load_place_groups(place_group_configs)
-
-            all_features = []
-            for place_group in place_groups:
-                all_features.extend(place_group["features"])
-                self._place_group_cache[place_group["id"]] = place_group
-
-            self._place_group_cache[ALL_PLACES] = dict(type="FeatureCollection", features=all_features)
-
-        if place_group_id not in self._place_group_cache:
-            raise ServiceResourceNotFoundError(f'Place group "{place_group_id}" not found')
-
-        return self._place_group_cache[place_group_id]
-
-    def _load_place_groups(self, place_group_configs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    def get_dataset_place_group(self, ds_id: str, place_group_id: str, load_features=False) -> Dict:
+        place_groups = self.get_dataset_place_groups(ds_id, load_features=False)
+        for place_group in place_groups:
+            if place_group_id == place_group['id']:
+                if load_features:
+                    self._load_place_group_features(place_group)
+                return place_group
+        raise ServiceResourceNotFoundError(f'Place group "{place_group_id}" not found')
+
+    def get_global_place_groups(self, load_features=False) -> List[Dict]:
+        return self._load_place_groups(self._config.get("PlaceGroups", []), is_global=True, load_features=load_features)
+
+    def get_global_place_group(self, place_group_id: str, load_features: bool = False) -> Dict:
+        place_group_descriptor = self._get_place_group_descriptor(place_group_id)
+        return self._load_place_group(place_group_descriptor, is_global=True, load_features=load_features)
+
+    def _get_place_group_descriptor(self, place_group_id: str) -> Dict:
+        place_group_descriptors = self._config.get("PlaceGroups", [])
+        for place_group_descriptor in place_group_descriptors:
+            if place_group_descriptor['Identifier'] == place_group_id:
+                return place_group_descriptor
+        raise ServiceResourceNotFoundError(f'Place group "{place_group_id}" not found')
+
+    def _load_place_groups(self,
+                           place_group_descriptors: Dict,
+                           is_global: bool = False,
+                           load_features: bool = False) -> List[Dict]:
         place_groups = []
-        for place_group_config in place_group_configs:
-            place_group = self._load_place_group(place_group_config)
+        for place_group_descriptor in place_group_descriptors:
+            place_group = self._load_place_group(place_group_descriptor, is_global=is_global,
+                                                 load_features=load_features)
             place_groups.append(place_group)
         return place_groups
 
-    def _load_place_group(self, place_group_config: Dict[str, Any]) -> Dict[str, Any]:
-        ref_id = place_group_config.get("PlaceGroupRef")
-        if ref_id:
-            # Trigger loading of all global "PlaceGroup" entries
-            self.get_place_group()
-            if len(place_group_config) > 1:
+    def _load_place_group(self, place_group_descriptor: Dict[str, Any], is_global: bool = False,
+                          load_features: bool = False) -> Dict[str, Any]:
+        place_group_id = place_group_descriptor.get("PlaceGroupRef")
+        if place_group_id:
+            if is_global:
+                raise ServiceError("'PlaceGroupRef' cannot be used in a global place group")
+            if len(place_group_descriptor) > 1:
                 raise ServiceError("'PlaceGroupRef' if present, must be the only entry in a 'PlaceGroups' item")
-            if ref_id not in self._place_group_cache:
-                raise ServiceError("Invalid 'PlaceGroupRef' entry in a 'PlaceGroups' item")
-            return self._place_group_cache[ref_id]
+            return self.get_global_place_group(place_group_id, load_features=load_features)
 
-        place_group_id = place_group_config.get("Identifier")
+        place_group_id = place_group_descriptor.get("Identifier")
         if not place_group_id:
             raise ServiceError("Missing 'Identifier' entry in a 'PlaceGroups' item")
-        if place_group_id == ALL_PLACES:
-            raise ServiceError("Invalid 'Identifier' entry in a 'PlaceGroups' item")
-
-        place_group_title = place_group_config.get("Title", place_group_id)
 
-        place_path_wc = place_group_config.get("Path")
-        if not place_path_wc:
-            raise ServiceError("Missing 'Path' entry in a 'PlaceGroups' item")
-        if not os.path.isabs(place_path_wc):
-            place_path_wc = os.path.join(self._base_dir, place_path_wc)
+        if place_group_id in self._place_group_cache:
+            place_group = self._place_group_cache[place_group_id]
+        else:
+            place_group_title = place_group_descriptor.get("Title", place_group_id)
+
+            place_path_wc = place_group_descriptor.get("Path")
+            if not place_path_wc:
+                raise ServiceError("Missing 'Path' entry in a 'PlaceGroups' item")
+            if not os.path.isabs(place_path_wc):
+                place_path_wc = os.path.join(self._base_dir, place_path_wc)
+            source_paths = glob.glob(place_path_wc)
+            source_encoding = place_group_descriptor.get("CharacterEncoding", "utf-8")
+
+            property_mapping = place_group_descriptor.get("PropertyMapping")
+
+            place_group = dict(type="FeatureCollection",
+                               features=None,
+                               id=place_group_id,
+                               title=place_group_title,
+                               propertyMapping=property_mapping,
+                               sourcePaths=source_paths,
+                               sourceEncoding=source_encoding)
+
+            sub_place_group_configs = place_group_descriptor.get("Places")
+            if sub_place_group_configs:
+                raise ServiceError("Invalid 'Places' entry in a 'PlaceGroups' item: not implemented yet")
+            # sub_place_group_descriptors = place_group_config.get("Places")
+            # if sub_place_group_descriptors:
+            #     sub_place_groups = self._load_place_groups(sub_place_group_descriptors)
+            #     place_group["placeGroups"] = sub_place_groups
+
+            self._place_group_cache[place_group_id] = place_group
+
+        if load_features:
+            self._load_place_group_features(place_group)
 
-        property_mapping = place_group_config.get("PropertyMapping")
-        character_encoding = place_group_config.get("CharacterEncoding", "utf-8")
+        return place_group
 
+    def _load_place_group_features(self, place_group: Dict[str, Any]) -> List[Dict[str, Any]]:
+        features = place_group.get('features')
+        if features is not None:
+            return features
+        source_files = place_group['sourcePaths']
+        source_encoding = place_group['sourceEncoding']
         features = []
-        collection_files = glob.glob(place_path_wc)
-        for collection_file in collection_files:
-            with fiona.open(collection_file, encoding=character_encoding) as feature_collection:
+        for source_file in source_files:
+            with fiona.open(source_file, encoding=source_encoding) as feature_collection:
                 for feature in feature_collection:
                     self._remove_feature_id(feature)
                     feature["id"] = str(self._feature_index)
                     self._feature_index += 1
                     features.append(feature)
-
-        place_group = dict(type="FeatureCollection",
-                           features=features,
-                           id=place_group_id,
-                           title=place_group_title,
-                           propertyMapping=property_mapping)
-
-        sub_place_group_configs = place_group_config.get("Places")
-        if sub_place_group_configs:
-            sub_place_groups = self._load_place_groups(sub_place_group_configs)
-            place_group["placeGroups"] = sub_place_groups
-
-        return place_group
+        place_group['features'] = features
+        return features
 
     @classmethod
     def _remove_feature_id(cls, feature: Dict):