From 61566d80ee88de61a59aff55704e6b76fe0a8740 Mon Sep 17 00:00:00 2001 From: Ernst de Vreede Date: Mon, 16 Sep 2024 18:09:48 +0200 Subject: [PATCH] Using getmetadata for EDR --- python/python_fastapi_server/routers/edr.py | 69 +-- .../routers/edr_covjson.py | 17 +- .../python_fastapi_server/routers/edr_cube.py | 124 +++-- .../routers/edr_instances.py | 21 +- .../routers/edr_locations.py | 2 +- .../routers/edr_position.py | 36 +- .../routers/edr_utils.py | 474 ++++++------------ 7 files changed, 309 insertions(+), 434 deletions(-) diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 9c7f9bc9..16458246 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -32,9 +32,9 @@ from .edr_utils import ( generate_max_age, get_base_url, - get_collectioninfo_for_id, - get_edr_collections, + get_collectioninfo_from_md, get_time_values_for_range, + get_metadata, ) logger = logging.getLogger(__name__) @@ -121,15 +121,19 @@ async def rest_get_edr_collections(request: Request, response: Response): links.append(self_link) collections: list[Collection] = [] ttl_set = set() - edr_collections = get_edr_collections() - for edr_coll in edr_collections: - coll, ttl = await get_collectioninfo_for_id(edr_coll) - if coll: - collections.append(coll) - if ttl is not None: - ttl_set.add(ttl) - else: - logger.warning("Unable to fetch WMS GetCapabilities for %s", edr_coll) + metadata = await get_metadata() + for coll_name in metadata.keys(): + print("COLL:", coll_name) + try: + coll = get_collectioninfo_from_md( + {coll_name: metadata[coll_name]}, coll_name + ) + if coll: + collections.append(coll) + else: + logger.warning("Unable to fetch WMS GetMetadata for %s", coll_name) + except Exception as exc: + print("ERR", exc) collections_data = Collections(links=links, collections=collections) if ttl_set: response.headers["cache-control"] = generate_max_age(min(ttl_set)) @@ -145,7 +149,10 @@ async def rest_get_edr_collection_by_id(collection_name: str, response: Response """ GET Returns collection information for given collection id """ - collection, ttl = await get_collectioninfo_for_id(collection_name) + metadata = await get_metadata(collection_name) + ttl = None + + collection = get_collectioninfo_from_md(metadata, collection_name) if ttl is not None: response.headers["cache-control"] = generate_max_age(ttl) if collection is None: @@ -219,25 +226,25 @@ def get_fixed_api(): version=edrApiApp.version, routes=edrApiApp.routes, ) - for pth in api["paths"].values(): - if "parameters" in pth["get"]: - for param in pth["get"]["parameters"]: - if param["in"] == "query" and param["name"] == "datetime": - param["style"] = "form" - param["explode"] = False - param["schema"] = { - "type": "string", - } - if "schema" in param: - if "anyOf" in param["schema"]: - for itany in param["schema"]["anyOf"]: - if itany.get("type") == "null": - print("NULL found p") - - if "CompactAxis" in api["components"]["schemas"]: - comp = api["components"]["schemas"]["CompactAxis"] - if "exclusiveMinimum" in comp["properties"]["num"]: - comp["properties"]["num"]["exclusiveMinimum"] = False + # for pth in api["paths"].values(): + # if "parameters" in pth["get"]: + # for param in pth["get"]["parameters"]: + # if param["in"] == "query" and param["name"] == "datetime": + # param["style"] = "form" + # param["explode"] = False + # param["schema"] = { + # "type": "string", + # } + # if "schema" in param: + # if "anyOf" in param["schema"]: + # for itany in param["schema"]["anyOf"]: + # if itany.get("type") == "null": + # print("NULL found p") + + # if "CompactAxis" in api["components"]["schemas"]: + # comp = api["components"]["schemas"]["CompactAxis"] + # if "exclusiveMinimum" in comp["properties"]["num"]: + # comp["properties"]["num"]["exclusiveMinimum"] = False return api diff --git a/python/python_fastapi_server/routers/edr_covjson.py b/python/python_fastapi_server/routers/edr_covjson.py index 829f4a44..7527c23c 100644 --- a/python/python_fastapi_server/routers/edr_covjson.py +++ b/python/python_fastapi_server/routers/edr_covjson.py @@ -31,7 +31,7 @@ SYMBOL_TYPE_URL = "http://www.opengis.net/def/uom/UCUM" -def covjson_from_resp(dats, vertical_name, custom_name, collection_name): +def covjson_from_resp(dats, metadata): """ Returns a coverage json from a Adaguc WMS GetFeatureInfo request """ @@ -39,6 +39,16 @@ def covjson_from_resp(dats, vertical_name, custom_name, collection_name): for dat in dats: if len(dat["data"]): (lon, lat) = dat["point"]["coords"].split(",") + custom_name = None + vertical_name = None + for param_dim in metadata[dat["name"]]["dims"].values(): + print("Checking dim ", param_dim["name"]) + if param_dim["name"] not in ["x", "y", "reference_time", "time"]: + if not param_dim["hidden"]: + if "isvertical" in param_dim: + vertical_name = param_dim["name"] + else: + custom_name = param_dim["name"] lat = float(lat) lon = float(lon) dims = makedims(dat["dims"], dat["data"]) @@ -49,7 +59,8 @@ def covjson_from_resp(dats, vertical_name, custom_name, collection_name): vertical_steps = getdimvals(dims, "elevation") custom_dim_values = [] - if custom_name is not None: + if custom_name is not None and len(custom_name) > 0: + print("CUSTOM_DIM:", custom_name, dat) custom_dim_values = getdimvals(dims, custom_name) valstack = [] @@ -72,7 +83,7 @@ def covjson_from_resp(dats, vertical_name, custom_name, collection_name): parameters: dict[str, CovJsonParameter] = {} ranges = {} - param_metadata = get_param_metadata(dat["name"], collection_name) + param_metadata = get_param_metadata(metadata[dat["name"]]) symbol = CovJsonSymbol( value=param_metadata["parameter_unit"], type=SYMBOL_TYPE_URL ) diff --git a/python/python_fastapi_server/routers/edr_cube.py b/python/python_fastapi_server/routers/edr_cube.py index b0259146..ab82445a 100644 --- a/python/python_fastapi_server/routers/edr_cube.py +++ b/python/python_fastapi_server/routers/edr_cube.py @@ -18,10 +18,9 @@ from .covjsonresponse import CovJSONResponse from .edr_utils import ( - parse_config_file, get_ref_times_for_coll, - get_edr_collections, instance_to_iso, + get_metadata, ) from .netcdf_to_covjson import netcdf_to_covjson from .ogcapi_tools import call_adaguc @@ -91,30 +90,34 @@ async def get_coll_inst_cube( "res_y", ] - edr_collectioninfo = get_edr_collections()[collection_name] + metadata = await get_metadata(collection_name) vertical_dim = "" + custom_name = None + first_layer_name = parameter_name.split(",")[0] + for param_dim in metadata[collection_name][first_layer_name]["dims"].values(): + if not param_dim["hidden"]: + if "isvertical" in param_dim: + vertical_name = param_dim["name"] + else: + custom_name = param_dim["name"] if z_par: - if edr_collectioninfo.get("vertical_name"): - vertical_name = edr_collectioninfo.get("vertical_name") + if vertical_name is not None: if vertical_name.upper() == "ELEVATION": - vertical_dim = f"{edr_collectioninfo.get('vertical_name')}={z_par}" + vertical_dim = f"{vertical_name}={z_par}" else: - vertical_dim = f"DIM_{edr_collectioninfo.get('vertical_name')}={z_par}" + vertical_dim = f"DIM_{vertical_name}={z_par}" custom_dims = [k for k in request.query_params if k not in allowed_params] custom_dim_parameter = "" - logger.info("custom dims: %s", custom_dims) + logger.info("custom dims: %s %s", custom_dims, custom_name) if len(custom_dims) > 0: for custom_dim in custom_dims: custom_dim_parameter += ( f"&DIM_{custom_dim}={request.query_params[custom_dim]}" ) - dataset = edr_collectioninfo["dataset"] - ref_times = await get_ref_times_for_coll( - dataset, edr_collectioninfo["parameters"][0]["name"] - ) + ref_times = get_ref_times_for_coll(metadata[collection_name]) if not instance and len(ref_times) > 0: instance = ref_times[-1] @@ -125,51 +128,64 @@ async def get_coll_inst_cube( else: res_queryterm = "" - collection_info = get_edr_collections().get(collection_name) - if "dataset" in collection_info: - logger.info("callADAGUC by dataset") - dataset = collection_info["dataset"] - translate_names, translate_dims = parse_config_file(dataset) - coveragejsons = [] - parameters = {} - for parameter_name in parameter_names: - if instance is None: - urlrequest = ( - f"dataset={dataset}&service=wcs&version=1.1.1&request=getcoverage&format=NetCDF4&crs=EPSG:4326&coverage={parameter_name}" - + f"&bbox={bbox}&time={datetime_par}" - + ( - f"&{custom_dim_parameter}" - if len(custom_dim_parameter) > 0 - else "" - ) - + (f"&{vertical_dim}" if len(vertical_dim) > 0 else "") - + res_queryterm - ) - else: - urlrequest = ( - f"dataset={dataset}&service=wcs&request=getcoverage&format=NetCDF4&crs=EPSG:4326&coverage={parameter_name}" - + f"&bbox={bbox}&time={datetime_par}&dim_reference_time={instance_to_iso(instance)}" - + ( - f"&{custom_dim_parameter}" - if len(custom_dim_parameter) > 0 - else "" - ) - + (f"&{vertical_dim}" if len(vertical_dim) > 0 else "") - + res_queryterm - ) - - status, response, _ = await call_adaguc(url=urlrequest.encode("UTF-8")) - logger.info("status: %d", status) - result_dataset = Dataset(f"{parameter_name}.nc", memory=response.getvalue()) - - coveragejson = netcdf_to_covjson( - result_dataset, translate_names, translate_dims + logger.info("callADAGUC by dataset") + dataset = collection_name + + translate_names = get_translate_names(metadata[collection_name]) + translate_dims = get_translate_dims(metadata[collection_name]) + + coveragejsons = [] + parameters = {} + datetime_arg = datetime_par + if datetime_arg is None: + datetime_arg = "*" + for parameter_name in parameter_names: + if instance is None: + urlrequest = ( + f"dataset={dataset}&service=wcs&version=1.1.1&request=getcoverage&format=NetCDF4&crs=EPSG:4326&coverage={parameter_name}" + + f"&bbox={bbox}&time={datetime_arg}" + + (f"&{custom_dim_parameter}" if len(custom_dim_parameter) > 0 else "") + + (f"&{vertical_dim}" if len(vertical_dim) > 0 else "") + + res_queryterm ) - if coveragejson is not None: - coveragejsons.append(coveragejson) - parameters = parameters | coveragejson.parameters + else: + urlrequest = ( + f"dataset={dataset}&service=wcs&request=getcoverage&format=NetCDF4&crs=EPSG:4326&coverage={parameter_name}" + + f"&bbox={bbox}&time={datetime_arg}&dim_reference_time={instance_to_iso(instance)}" + + (f"&{custom_dim_parameter}" if len(custom_dim_parameter) > 0 else "") + + (f"&{vertical_dim}" if len(vertical_dim) > 0 else "") + + res_queryterm + ) + + status, response, _ = await call_adaguc(url=urlrequest.encode("UTF-8")) + logger.info("status: %d", status) + result_dataset = Dataset(f"{parameter_name}.nc", memory=response.getvalue()) + + coveragejson = netcdf_to_covjson( + result_dataset, translate_names, translate_dims + ) + if coveragejson is not None: + coveragejsons.append(coveragejson) + parameters = parameters | coveragejson.parameters if len(coveragejsons) == 1: return coveragejsons[0] return CoverageCollection(coverages=coveragejsons, parameters=parameters) + + +def get_translate_names(metadata: dict) -> dict: + translated_names = {} + for layer_name in metadata: + var_name = metadata[layer_name]["layer"]["variables"][0]["variableName"] + translated_names[var_name] = layer_name + return {} + + +def get_translate_dims(metadata: dict) -> dict: + translated_dims = {} + for layer_name in metadata: + for dim_name in metadata[layer_name]["dims"]: + dim_netcdf_name = metadata[layer_name]["dims"]["name"] + translated_dims[dim_netcdf_name] = dim_name + return {} diff --git a/python/python_fastapi_server/routers/edr_instances.py b/python/python_fastapi_server/routers/edr_instances.py index 58a7e8a8..13bb904f 100644 --- a/python/python_fastapi_server/routers/edr_instances.py +++ b/python/python_fastapi_server/routers/edr_instances.py @@ -15,8 +15,8 @@ from .edr_utils import ( generate_max_age, get_base_url, - get_collectioninfo_for_id, - get_edr_collections, + get_metadata, + get_collectioninfo_from_md, get_ref_times_for_coll, ) @@ -39,12 +39,10 @@ async def rest_get_edr_inst_for_coll( ) instances: list[Instance] = [] - edr_collections = get_edr_collections() - ref_times = await get_ref_times_for_coll( - edr_collections[collection_name]["dataset"], - edr_collections[collection_name]["parameters"][0]["name"], - ) + metadata = await get_metadata(collection_name) + + ref_times = get_ref_times_for_coll(metadata[collection_name]) print("REF:", ref_times) links: list[Link] = [] links.append(Link(href=instances_url, rel="collection")) @@ -53,9 +51,7 @@ async def rest_get_edr_inst_for_coll( instance_links: list[Link] = [] instance_link = Link(href=f"{instances_url}/{instance}", rel="collection") instance_links.append(instance_link) - instance_info, ttl = await get_collectioninfo_for_id(collection_name, instance) - if ttl is not None: - ttl_set.add(ttl) + instance_info = get_collectioninfo_from_md(metadata, collection_name, instance) instances.append(instance_info) instances_data = Instances(instances=instances, links=links) @@ -73,7 +69,6 @@ async def rest_get_collection_info(collection_name: str, instance, response: Res """ GET "/collections/{collection_name}/instances/{instance}" """ - coll, ttl = await get_collectioninfo_for_id(collection_name, instance) - if ttl is not None: - response.headers["cache-control"] = generate_max_age(ttl) + metadata = await get_metadata(collection_name) + coll = get_collectioninfo_from_md(metadata, collection_name, instance) return coll diff --git a/python/python_fastapi_server/routers/edr_locations.py b/python/python_fastapi_server/routers/edr_locations.py index 5ccde457..39fa4061 100644 --- a/python/python_fastapi_server/routers/edr_locations.py +++ b/python/python_fastapi_server/routers/edr_locations.py @@ -18,7 +18,7 @@ @router.get("/collections/{_coll}/locations") @router.get("/collections/{_coll}/instances/{instance}/locations") -async def get_locations(_coll: str): +async def get_locations(_coll: str, instance: str): """ Returns locations where you could query data. """ diff --git a/python/python_fastapi_server/routers/edr_position.py b/python/python_fastapi_server/routers/edr_position.py index ee0797a0..553f5410 100644 --- a/python/python_fastapi_server/routers/edr_position.py +++ b/python/python_fastapi_server/routers/edr_position.py @@ -22,7 +22,7 @@ from .edr_utils import ( call_adaguc, generate_max_age, - get_edr_collections, + get_metadata, get_ttl_from_adaguc_headers, instance_to_iso, ) @@ -83,19 +83,30 @@ async def get_coll_inst_position( if len(custom_params) > 0: for custom_param in custom_params: custom_dims += f"&DIM_{custom_param}={request.query_params[custom_param]}" - edr_collections = get_edr_collections() - if collection_name in edr_collections: + metadata = await get_metadata(collection_name) + + if metadata is not None: parameter_names = parameter_name.split(",") + vertical_dim_name = "z" + for dim_name in metadata[collection_name][parameter_names[0]]["dims"]: + if ( + "isvertical" + in metadata[collection_name][parameter_names[0]]["dims"][dim_name] + and metadata[collection_name][parameter_names[0]]["dims"][dim_name] + == "true" + ): + vertical_dim_name = dim_name latlons = wkt.loads(coords) coord = {"lat": latlons["coordinates"][1], "lon": latlons["coordinates"][0]} resp, headers = await get_point_value( - edr_collections[collection_name], + collection_name, instance, [coord["lon"], coord["lat"]], parameter_names, datetime_par, z_par, + vertical_dim_name, custom_dims, ) if resp: @@ -105,28 +116,26 @@ async def get_coll_inst_position( response.headers["cache-control"] = generate_max_age(ttl) return covjson_from_resp( dat, - edr_collections[collection_name]["vertical_name"], - edr_collections[collection_name]["custom_name"], - collection_name, + metadata[collection_name], ) raise EdrException(code=400, description="No data") async def get_point_value( - edr_collectioninfo: dict, + collection_name: str, instance: str, coords: list[float], parameters: list[str], datetime_par: str, z_par: str = None, + z_name: str = None, custom_dims: str = None, ): """Returns information in EDR format for a given collection and position""" - dataset = edr_collectioninfo["dataset"] urlrequest = ( f"SERVICE=WMS&VERSION=1.3.0&REQUEST=GetPointValue&CRS=EPSG:4326" - f"&DATASET={dataset}&QUERY_LAYERS={','.join(parameters)}" + f"&DATASET={collection_name}&QUERY_LAYERS={','.join(parameters)}" f"&X={coords[0]}&Y={coords[1]}&INFO_FORMAT=application/json" ) if datetime_par: @@ -135,11 +144,8 @@ async def get_point_value( if instance: urlrequest += f"&DIM_reference_time={instance_to_iso(instance)}" if z_par: - if ( - "vertical_name" in edr_collectioninfo - and edr_collectioninfo["vertical_name"].upper() != "ELEVATION" - ): - urlrequest += f"&DIM_{edr_collectioninfo['vertical_name']}={z_par}" + if urlrequest.upper() != "ELEVATION": + urlrequest += f"&DIM_{z_name}={z_par}" else: urlrequest += f"&ELEVATION={z_par}" if custom_dims: diff --git a/python/python_fastapi_server/routers/edr_utils.py b/python/python_fastapi_server/routers/edr_utils.py index bbbaa1d3..d8eafeed 100644 --- a/python/python_fastapi_server/routers/edr_utils.py +++ b/python/python_fastapi_server/routers/edr_utils.py @@ -41,38 +41,6 @@ edr_cache = TTLCache(maxsize=100, ttl=120) # TODO: Redis? -def parse_config_file( - dataset_filename: str, adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DIR"] -): - """Return translation names for NetCDF data""" - translate = {} - dim_translate = {} - filename = dataset_filename - _, ext = os.path.splitext(filename) - if ext == "" or ext != ".xml": - filename = filename + ".xml" - - if os.path.isfile(os.path.join(adaguc_dataset_dir, filename)) and filename.endswith( - ".xml" - ): - tree = parse(os.path.join(adaguc_dataset_dir, filename)) - root = tree.getroot() - for lyr in root.iter("Layer"): - layer_variables = [l.text for l in lyr.iter("Variable")] - if len(layer_variables) == 1: - translate[layer_variables[0]] = ( - lyr.find("Name") or lyr.find("Variable") - ).text - for coll in root.iter("EdrCollection"): - if "vertical_name" in coll.attrib: - vertical_name = coll.attrib["vertical_name"] - for lyr in root.iter("Layer"): - for dimension in lyr.iter("Dimension"): - if dimension.text == vertical_name: - dim_translate[dimension.attrib["name"]] = "z" - return translate, dim_translate - - def parse_iso(dts: str) -> datetime: """ Converts a ISO8601 string into a datetime object @@ -87,16 +55,23 @@ def parse_iso(dts: str) -> datetime: return parsed_dt -async def get_ref_times_for_coll(dataset: str, layer: str) -> list[str]: +def get_ref_times_for_coll(metadata) -> list[str]: """ Returns available reference times for given collection """ - url = f"?DATASET={dataset}&SERVICE=WMS&VERSION=1.3.0&request=getreferencetimes&LAYER={layer}" - # logger.info("getreftime_url(%s,%s): %s", dataset, layer, url) - status, response, _ = await call_adaguc(url=url.encode("UTF-8")) - if status == 0: - ref_times = json.loads(response.getvalue()) - instance_ids = [parse_iso(reft).strftime("%Y%m%d%H%M") for reft in ref_times] + first_param = next(iter(metadata)) + print("MD:", first_param, metadata[first_param]) + if "reference_time" in metadata[first_param]["dims"]: + print( + "REFT:", + metadata[first_param]["dims"]["reference_time"]["values"], + ) + instance_ids = [ + parse_iso(reft).strftime("%Y%m%d%H%M") + for reft in metadata[first_param]["dims"]["reference_time"]["values"].split( + "," + ) + ] return instance_ids return [] @@ -132,93 +107,20 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI ] ) - edr_collections = {} + edr_collections = [] for dataset_file in dataset_files: dataset = dataset_file.replace(".xml", "") try: tree = parse(os.path.join(adaguc_dataset_dir, dataset_file)) root = tree.getroot() - for ogcapi_edr in root.iter("OgcApiEdr"): - for edr_collection in ogcapi_edr.iter("EdrCollection"): - edr_params = [] - for edr_parameter in edr_collection.iter("EdrParameter"): - if ( - "name" in edr_parameter.attrib - and "unit" in edr_parameter.attrib - ): - name = edr_parameter.attrib.get("name") - edr_param = { - "name": name, - "parameter_label": name, - "observed_property_label": name, - "description": name, - "standard_name": None, - "unit": "-", - } - # Try to take the standard name from the configuration - if "standard_name" in edr_parameter.attrib: - edr_param["standard_name"] = edr_parameter.attrib.get( - "standard_name" - ) - # If there is a standard name, set the label to the same as fallback - edr_param["observed_property_label"] = ( - edr_parameter.attrib.get("standard_name") - ) - - # Try to take the observed_property_label from the configuration - if "observed_property_label" in edr_parameter.attrib: - edr_param["observed_property_label"] = ( - edr_parameter.attrib.get("observed_property_label") - ) - # Try to take the parameter_label from the Layer configuration Title - if "parameter_label" in edr_parameter.attrib: - edr_param["parameter_label"] = edr_parameter.attrib.get( - "parameter_label" - ) - # Try to take the parameter_label from the Layer configuration Title - if "unit" in edr_parameter.attrib: - edr_param["unit"] = edr_parameter.attrib.get("unit") - - edr_params.append(edr_param) - - else: - logger.warning( - "In dataset %s, skipping parameter %s: has no name or units configured", - dataset_file, - edr_parameter, - ) - - if len(edr_params) == 0: - logger.warning( - "Skipping dataset %s: has no EdrParameter configured", - dataset_file, - ) - else: - collection_url = ( - get_base_url() - + f"/edr/collections/{edr_collection.attrib.get('name')}" - ) - edr_collections[edr_collection.attrib.get("name")] = { - "dataset": dataset, - "name": edr_collection.attrib.get("name"), - "service": f"{OWSLIB_DUMMY_URL}/wms", - "parameters": edr_params, - "vertical_name": edr_collection.attrib.get("vertical_name"), - "custom_name": edr_collection.attrib.get("custom_name"), - "base_url": collection_url, - "time_interval": edr_collection.attrib.get("time_interval"), - } + for _ in root.iter("OgcApiEdr"): + edr_collections.append(dataset) except ParseError: pass + # ['RADAR', 'ecmwf_ens_nl_0p1deg', 'multi_dim', 'uwcw_ha43_dini_5p5km', 'uwcw_ha43_nl_2km', 'uwcw_ha43ens_nl_2km'] return edr_collections -@cached(cache=edr_cache) -def get_edr_collections(): - """Returns all EDR collections""" - return init_edr_collections() - - def instance_to_iso(instance_dt: str): """ Converts EDR instance time in format %Y%m%d%H%M into iso8601 time string. @@ -270,22 +172,31 @@ def generate_max_age(ttl): return "max-age=0" -async def get_collectioninfo_for_id( - edr_collection: str, +def get_collectioninfo_from_md( + metadata: dict, + collection_name: str, instance: str = None, -) -> tuple[Collection, datetime]: +) -> Collection: """ Returns collection information for a given collection id and or instance Is used to obtain metadata from the dataset configuration and WMS GetCapabilities document. """ - logger.info("get_collectioninfo_for_id(%s, %s)", edr_collection, instance) - edr_collectionsinfo = get_edr_collections() - if edr_collection not in edr_collectionsinfo: - raise EdrException(code=400, description="Unknown or unconfigured collection") + logger.info("get_collectioninfo_for_id(%s, %s)", collection_name, instance) + + if metadata is None: + return None - edr_collectioninfo = edr_collectionsinfo[edr_collection] + first_param = next(iter(metadata[collection_name])) + if metadata[collection_name][first_param]["layer"]["variables"] is None: + return None - base_url = edr_collectioninfo["base_url"] + # edr_collectionsinfo = get_edr_collections() + # if edr_collection not in edr_collectionsinfo: + # raise EdrException(code=400, description="Unknown or unconfigured collection") + + # edr_collectioninfo = edr_collectionsinfo[edr_collection] + + base_url = get_base_url() + f"/edr/collections/{collection_name}" if instance is not None: base_url += f"/instances/{instance}" @@ -294,44 +205,38 @@ async def get_collectioninfo_for_id( links.append(Link(href=f"{base_url}", rel="collection", type="application/json")) ref_times = None - if not instance: - ref_times = await get_ref_times_for_coll( - edr_collectioninfo["dataset"], edr_collectioninfo["parameters"][0]["name"] - ) + ref_times = get_ref_times_for_coll(metadata[collection_name]) if ref_times and len(ref_times) > 0: instances_link = Link( href=f"{base_url}/instances", rel="collection", type="application/json" ) links.append(instances_link) - wmslayers, ttl = await get_capabilities(edr_collectioninfo["name"]) + print("MD:", metadata[collection_name][first_param]) - bbox = get_extent(edr_collectioninfo, wmslayers) + bbox = [metadata[collection_name][first_param]["layer"]["latlonbox"]] if bbox is None: return None, None crs = 'GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]' spatial = Spatial(bbox=bbox, crs=crs) - if ( - ref_times is None - or len(ref_times) == 0 - or edr_collectioninfo["time_interval"] is None - ): - (interval, time_values) = get_times_for_collection( - wmslayers, edr_collectioninfo["parameters"][0]["name"] - ) - else: - if instance is None: - instance = max(ref_times) # Default instance is latest instance - (interval, time_values) = create_times_for_instance( - edr_collectioninfo, instance - ) + # if ref_times is None or len(ref_times) == 0: + (interval, time_values) = get_times_for_collection( + metadata[collection_name], first_param + ) + # else: + # if instance is None: + # instance = max(ref_times) # Default instance is latest instance + # (interval, time_values) = create_times_for_instance( + # edr_collectioninfo, instance + # ) customlist: list = get_custom_dims_for_collection( - edr_collectioninfo, wmslayers, edr_collectioninfo["parameters"][0]["name"] + metadata[collection_name], first_param ) + print("CUSTOM:", customlist) # Custom can be a list of custom dimensions, like ensembles, thresholds custom = [] if customlist is not None: @@ -340,7 +245,7 @@ async def get_collectioninfo_for_id( vertical = None vertical_dim = get_vertical_dim_for_collection( - edr_collectioninfo, wmslayers, edr_collectioninfo["parameters"][0]["name"] + metadata[collection_name], first_param ) if vertical_dim: vertical = Vertical(**vertical_dim) @@ -350,6 +255,7 @@ async def get_collectioninfo_for_id( trs='TIMECRS["DateTime",TDATUM["Gregorian Calendar"],CS[TemporalDateTime,1],AXIS["Time (T)",future]', values=time_values, # ["R49/2022-06-30T06:00:00/PT60M"], ) + logger.info("TEMPORAL [%s]: %s, %s", collection_name, interval, time_values) extent = Extent( spatial=spatial, temporal=temporal, custom=custom, vertical=vertical @@ -422,9 +328,7 @@ async def get_collectioninfo_for_id( locations=EDRQuery(link=locations_link), ) - parameter_names = get_params_for_collection( - edr_collection=edr_collection, wmslayers=wmslayers - ) + parameter_info = get_params_for_collection(metadata[collection_name]) crs = ["EPSG:4326"] @@ -432,10 +336,10 @@ async def get_collectioninfo_for_id( if instance is None: collection = Collection( links=links, - id=edr_collection, + id=collection_name, extent=extent, data_queries=data_queries, - parameter_names=parameter_names, + parameter_names=parameter_info, crs=crs, output_formats=output_formats, ) @@ -445,12 +349,12 @@ async def get_collectioninfo_for_id( id=instance, extent=extent, data_queries=data_queries, - parameter_names=parameter_names, + parameter_names=parameter_info, crs=crs, output_formats=output_formats, ) - return collection, ttl + return collection def parse_period_string(period: str): @@ -544,161 +448,112 @@ def create_times_for_instance(edr_collectioninfo: dict, instance: str): return interval, times -def get_params_for_collection( - edr_collection: str, wmslayers: dict -) -> dict[str, Parameter]: +def get_params_for_collection(metadata: dict) -> dict[str, Parameter]: """ Returns a dictionary with parameters for given EDR collection """ parameter_names = {} - edr_collections = get_edr_collections() - for param_el in edr_collections[edr_collection]["parameters"]: - param_id = param_el["name"] - if not param_id in wmslayers: - logger.warning( - "EDR Parameter with name [%s] is not found in any of the adaguc Layer configurations. Available layers are %s", - param_id, - str(list(wmslayers.keys())), - ) - else: - param_metadata = get_param_metadata(param_id, edr_collection) - param = Parameter( - id=param_metadata["wms_layer_name"], - observedProperty=ObservedProperty( - id=param_metadata["observed_property_id"], - label=param_metadata["observed_property_label"], - ), - # description=param_metadata["wms_layer_title"], # TODO in follow up - type="Parameter", - unit=Unit( - symbol=Symbol( - value=param_metadata["parameter_unit"], type=SYMBOL_TYPE_URL - ) - ), - label=param_metadata["parameter_label"], - ) - parameter_names[param_el["name"]] = param + for param_id in metadata: + param_metadata = get_param_metadata(metadata[param_id]) + param = Parameter( + id=param_metadata["wms_layer_name"], + observedProperty=ObservedProperty( + id=param_metadata["observed_property_id"], + label=param_metadata["observed_property_label"], + ), + # description=param_metadata["wms_layer_title"], # TODO in follow up + type="Parameter", + unit=Unit( + symbol=Symbol( + value=param_metadata["parameter_unit"], type=SYMBOL_TYPE_URL + ) + ), + label=param_metadata["parameter_label"], + ) + parameter_names[param_id] = param return parameter_names -async def get_capabilities(collname): - """ - Get the collectioninfo from the WMS GetCapabilities - """ - collection_info = get_edr_collections().get(collname) - if "dataset" in collection_info: - logger.info("callADAGUC by dataset") - dataset = collection_info["dataset"] - urlrequest = ( - f"dataset={dataset}&service=wms&version=1.3.0&request=getcapabilities" - ) - status, response, headers = await call_adaguc(url=urlrequest.encode("UTF-8")) - ttl = get_ttl_from_adaguc_headers(headers) - logger.info("status: %d", status) - if status == 0: - xml = response.getvalue() - wms = WebMapService(collection_info["service"], xml=xml, version="1.3.0") - else: - logger.error("status: %d", status) - return {} +async def get_metadata(collname=None): + """get metadata from ADAGUC""" + logger.info("callADAGUC by dataset") + urlrequest = "service=wms&version=1.3.0&request=getmetadata&format=application/json" + if collname: + urlrequest = f"dataset={collname}&service=wms&version=1.3.0&request=getmetadata&format=application/json" + status, response, headers = await call_adaguc(url=urlrequest.encode("UTF-8")) + # ttl = get_ttl_from_adaguc_headers(headers) + logger.info("status: %d", status) + metadata = {} + if status == 0: + metadata = json.loads(response.getvalue().decode("UTF-8")) else: - logger.info("callADAGUC by service %s", dataset) - wms = WebMapService(dataset["service"], version="1.3.0") - ttl = None - - layers = {} - for layername, layerinfo in wms.contents.items(): - layers[layername] = { - "name": layername, - "title": layerinfo.title, - "dimensions": {**layerinfo.dimensions}, - "boundingBoxWGS84": layerinfo.boundingBoxWGS84, - } - return layers, ttl - - -def get_vertical_dim_for_collection( - edr_collectioninfo: dict, wmslayers, parameter: str = None -): + logger.error("status: %d", status) + return {} + return metadata + + +def get_vertical_dim_for_collection(metadata: dict, parameter: str = None): """ Return the verticel dimension the WMS GetCapabilities document. """ - if parameter and parameter in list(wmslayers): - layer = wmslayers[parameter] + if parameter and parameter in list(metadata): + layer = metadata[parameter] else: - layer = wmslayers[list(wmslayers)[0]] + layer = metadata[list(metadata)[0]] - for dim_name in layer["dimensions"]: + for dim_name in layer["dims"]: if dim_name in ["elevation"] or ( - "vertical_name" in edr_collectioninfo - and dim_name == edr_collectioninfo["vertical_name"] + "isvertical" in layer["dims"] and layer["dims"]["isvertical"] == "true" ): vertical_dim = { "interval": [], - "values": layer["dimensions"][dim_name]["values"], + "values": layer["dims"][dim_name]["values"], "vrs": "customvrs", } return vertical_dim return None -def get_custom_dims_for_collection( - edr_collectioninfo: dict, wmslayers, parameter: str = None -): +def get_custom_dims_for_collection(metadata: dict, parameter: str = None): """ Return the dimensions other then elevation or time from the WMS GetCapabilities document. """ custom = [] - if parameter and parameter in list(wmslayers): - layer = wmslayers[parameter] + if parameter and parameter in list(metadata): + layer = metadata[parameter] else: # default to first layer - layer = wmslayers[list(wmslayers)[0]] - for dim_name in layer["dimensions"]: - # Not needed for non custom dims: - if dim_name not in [ - "reference_time", - "time", - "elevation", - edr_collectioninfo.get("vertical_name"), - ]: - custom_dim = { - "id": dim_name, - "interval": [ - [ - layer["dimensions"][dim_name]["values"][0], - layer["dimensions"][dim_name]["values"][-1], - ] - ], - "values": layer["dimensions"][dim_name]["values"], - "reference": f"custom_{dim_name}", - } - if layer["dimensions"][dim_name]["values"] == ["1", "2", "3", "4", "5"]: - custom_dim["values"] = ["R5/1/1"] - # if dim_name == "member": - # custom_dim["id"] = "number" - custom.append(custom_dim) + layer = metadata[list(metadata)[0]] + for dim_name in layer["dims"]: + print("layer", layer["dims"][dim_name]) + if not layer["dims"][dim_name]["hidden"]: + # Not needed for non custom dims: + if dim_name not in [ + "reference_time", + "time", + "elevation", + ]: + custom_dim = { + "id": dim_name, + "interval": [ + [ + layer["dims"][dim_name]["values"][0], + layer["dims"][dim_name]["values"][-1], + ] + ], + "values": [layer["dims"][dim_name]["values"]], + "reference": f"custom_{dim_name}", + } + if layer["dims"][dim_name]["values"] == ["1", "2", "3", "4", "5"]: + custom_dim["values"] = ["R5/1/1"] + # if dim_name == "member": + # custom_dim["id"] = "number" + custom.append(custom_dim) return custom if len(custom) > 0 else None -def get_extent(edr_collectioninfo: dict, wmslayers): - """ - Get the boundingbox extent from the WMS GetCapabilities - """ - first_layer = edr_collectioninfo["parameters"][0]["name"] - if len(wmslayers): - if first_layer in wmslayers: - bbox = wmslayers[first_layer]["boundingBoxWGS84"] - else: - # Fallback to first layer in getcapabilities - bbox = wmslayers[next(iter(wmslayers))]["boundingBoxWGS84"] - - return [[bbox[0], bbox[1]], [bbox[2], bbox[3]]] - return None - - def get_times_for_collection( - wmslayers, parameter: str = None + metadata, parameter: str = None ) -> tuple[list[list[str]], list[str]]: """ Returns a list of times based on the time dimensions, it does a WMS GetCapabilities to the given dataset (cached) @@ -706,15 +561,16 @@ def get_times_for_collection( It does this for given parameter. When the parameter is not given it will do it for the first Layer in the GetCapabilities document. """ # logger.info("get_times_for_dataset(%s,%s)", edr_collectioninfo["name"], parameter) - if parameter and parameter in wmslayers: - layer = wmslayers[parameter] + if parameter and parameter in metadata: + layer = metadata[parameter] else: - layer = wmslayers[list(wmslayers)[0]] + layer = metadata[list(metadata)[0]] - if "time" in layer["dimensions"]: - time_dim = layer["dimensions"]["time"] - if "/" in time_dim["values"][0]: - terms = time_dim["values"][0].split("/") + if "time" in layer["dims"]: + time_dim = layer["dims"]["time"]["values"] + print("TIME_DIM:", time_dim) + if "/" in time_dim: + terms = time_dim.split("/") interval = [ [ datetime.strptime(terms[0], "%Y-%m-%dT%H:%M:%SZ").replace( @@ -725,19 +581,20 @@ def get_times_for_collection( ), ] ] - return interval, get_time_values_for_range(time_dim["values"][0]) + return interval, get_time_values_for_range(time_dim) + + terms = time_dim.split(",") interval = [ [ - datetime.strptime(time_dim["values"][0], "%Y-%m-%dT%H:%M:%SZ").replace( + datetime.strptime(terms[0], "%Y-%m-%dT%H:%M:%SZ").replace( tzinfo=timezone.utc ), - datetime.strptime(time_dim["values"][-1], "%Y-%m-%dT%H:%M:%SZ").replace( + datetime.strptime(terms[-1], "%Y-%m-%dT%H:%M:%SZ").replace( tzinfo=timezone.utc ), ] ] - return interval, time_dim["values"] - return None, None + return interval, terms def get_time_values_for_range(rng: str) -> list[str]: @@ -771,43 +628,26 @@ def get_time_values_for_range(rng: str) -> list[str]: VOCAB_ENDPOINT_URL = "https://vocab.nerc.ac.uk/standard_name/" -def get_parameter_config(edr_collection_name: str, param_id: str): - """Gets the EDRParameter configuration based on the edr collection name and parameter id - - Args: - edr_collection_name (str): edr collection name - param_id (str): parameter id - - Returns: - _type_: parameter element_ - """ - - edr_collections = get_edr_collections() - edr_collection_parameters = edr_collections[edr_collection_name]["parameters"] - for param_el in edr_collection_parameters: - if param_id == param_el["name"]: - return param_el - return None - - -def get_param_metadata(param_id: str, edr_collection_name) -> dict: +def get_param_metadata(param_metadata: dict) -> dict: """Composes parameter metadata based on the param_el and the wmslayer dictionaries Args: - param_id (str): The parameter / wms layer name to find - edr_collection_name (str): The collection name + param_metadata (dict): The parameter / wms layer name to find Returns: dict: dictionary with all metadata required to construct a Edr Parameter object. """ - param_el = get_parameter_config(edr_collection_name, param_id) - wms_layer_name = param_el["name"] + wms_layer_name = param_metadata["layer"]["name"] observed_property_id = wms_layer_name - parameter_label = param_el["parameter_label"] - parameter_unit = param_el["unit"] - observed_property_label = param_el["observed_property_label"] - if "standard_name" in param_el and param_el["standard_name"] is not None: - observed_property_id = VOCAB_ENDPOINT_URL + param_el["standard_name"] + parameter_label = param_metadata["layer"]["title"] + parameter_unit = param_metadata["layer"]["variables"][0]["units"] + if len(parameter_unit) == 0: + parameter_unit = "-^-" + observed_property_label = param_metadata["layer"]["variables"][0]["label"] + if "standard_name" in param_metadata["layer"]["variables"][0]: + observed_property_id = ( + VOCAB_ENDPOINT_URL + param_metadata["layer"]["variables"][0] + ) return { "wms_layer_name": wms_layer_name,