From 374451153c7f00525411c6eb7ca9dc1519317d74 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Mon, 21 Oct 2024 14:47:01 +0200 Subject: [PATCH] STAC, Collection and Workspace handling in openEO (#485) * Add `export_collection`, `export_workspace`, `stac_update`; `save_results` returns the STAC resource instead of boolean `true` https://github.com/Open-EO/openeo-api/issues/376 * Update stac_update/modify * Added details about STAC support. * Update meta/implementation.md Co-authored-by: Matthias Mohr --------- Co-authored-by: Michele Claus <31700619+clausmichele@users.noreply.github.com> --- CHANGELOG.md | 7 +++++ meta/implementation.md | 21 ++++++++++++++ meta/subtype-schemas.json | 24 ++++++++++++++++ proposals/export_collection.json | 45 ++++++++++++++++++++++++++++++ proposals/export_workspace.json | 48 ++++++++++++++++++++++++++++++++ proposals/stac_modify.json | 41 +++++++++++++++++++++++++++ save_result.json | 9 +++--- 7 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 proposals/export_collection.json create mode 100644 proposals/export_workspace.json create mode 100644 proposals/stac_modify.json diff --git a/CHANGELOG.md b/CHANGELOG.md index cf37ada6..5e6784e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased / Draft +### Added + +- `export_collection` +- `export_workspace` +- `stac_modify` + ### Changed - `clip`: Throw an exception if min > max [#472](https://github.com/Open-EO/openeo-processes/issues/472) +- `save_results`: Returns the STAC resource instead of boolean `true` [API#376](https://github.com/Open-EO/openeo-api/issues/376) - Added a uniqueness contraint to various array-typed parameters (e.g. lists of dimension names or labels) - Renamed `create_data_cube` to `create_cube`. [#68](https://github.com/Open-EO/openeo-processes/issues/68) - `apply_polygon`: Renamed `polygons` parameter to `geometries` for better alignment with other geometry handling processes. [#511](https://github.com/Open-EO/openeo-processes/issues/511) diff --git a/meta/implementation.md b/meta/implementation.md index c022ca52..ccdb8ed3 100644 --- a/meta/implementation.md +++ b/meta/implementation.md @@ -230,3 +230,24 @@ We have found some libraries that can be used for an implementation: - Julia: [Statistics.quantile](https://docs.julialang.org/en/v1/stdlib/Statistics/#Statistics.quantile!), type 7 is the default. - Python: [numpy](https://numpy.org/doc/stable/reference/generated/numpy.quantile.html), [pandas](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.quantile.html), [xarray](http://xarray.pydata.org/en/stable/generated/xarray.DataArray.quantile.html) - type 7 (called 'linear' for the interpolation parameter) is the default for all of them. - R: [quantile](https://stat.ethz.ch/R-manual/R-patched/library/stats/html/quantile.html) - type 7 is the default. + +## STAC support + +The subtype `stac` is an abstract type that refers to a STAC resource of any type (Catalog, Collection, or Item). +It can refer to: +- static STAC resources, e.g. hosted on cloud storage +- dynamic STAC resources made available via a STAC API +- a STAC JSON representation embedded as an argument into an openEO user-defined process + +### stac_modify + +The process `stac_modify` updates a given STAC resource based on +[RFC 7386: JSON Merge Patch](https://www.rfc-editor.org/rfc/rfc7386.html). +For static STAC resources, the content of the JSON files shall be updated according to RFC 7386. + +If the underlying STAC resource is part of an API, the following HTTP endpoints shall be used for the updates: +- For STAC Items: `PATCH /collections/{collectionId}/items/{featureId}` + according to the [Transaction Extension](https://github.com/stac-api-extensions/transaction) +- For STAC Collections: `PATCH /collections/{collectionId}` + according to the [Collection Transaction Extension](https://github.com/stac-api-extensions/collection-transaction) +- For STAC Catalogs there is no API support for updates. \ No newline at end of file diff --git a/meta/subtype-schemas.json b/meta/subtype-schemas.json index 347df234..f113ea24 100644 --- a/meta/subtype-schemas.json +++ b/meta/subtype-schemas.json @@ -278,6 +278,23 @@ "description": "A raster data cube, which is a data cube with two dimension of type spatial (x and y). This has been deprecated in favour of `datacube`.", "deprecated": true }, + "stac": { + "type": "object", + "subtype": "stac", + "title": "STAC resource", + "description": "A STAC Catalog, Collection, or Item, as defined by the [STAC specification](https://stacspec.org) version 0.9.0 or later.", + "oneOf": [ + { + "$ref": "http://schemas.stacspec.org/v1.0.0/catalog-spec/json-schema/catalog.json" + }, + { + "$ref": "http://schemas.stacspec.org/v1.0.0/collection-spec/json-schema/collection.json" + }, + { + "$ref": "http://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json" + } + ] + }, "temporal-interval": { "type": "array", "subtype": "temporal-interval", @@ -413,6 +430,13 @@ "title": "WKT2 definition", "description": "Specifies details about cartographic projections as WKT2 string. Refers to the latest WKT2 version (currently [WKT2:2018](http://docs.opengeospatial.org/is/18-010r7/18-010r7.html) / ISO 19162:2018) unless otherwise stated by the process." }, + "workspace-id": { + "type": "string", + "subtype": "workspace-id", + "title": "Workspace ID", + "description": "A workspace identifier from the list of available workspaces.", + "pattern": "^[\\w\\-\\.~]+$" + }, "year": { "type": "integer", "subtype": "year", diff --git a/proposals/export_collection.json b/proposals/export_collection.json new file mode 100644 index 00000000..e5014811 --- /dev/null +++ b/proposals/export_collection.json @@ -0,0 +1,45 @@ +{ + "id": "export_collection", + "summary": "Export data to an openEO collection", + "description": "Exports the given processing results made available through a STAC resource (e.g., a STAC Collection or Item) to the given openEO collection. The STAC resource itself is exported with all STAC resources and assets underneath.", + "categories": [ + "export", + "stac" + ], + "experimental": true, + "parameters": [ + { + "name": "data", + "description": "The data to export to the openEO collection.", + "schema": { + "type": "object", + "subtype": "stac" + } + }, + { + "name": "collection", + "description": "The identifier of the collection to export to. If the collection **exists**, the data will be added to the collection. If the collection **does not exist** yet, it will be created based on the given STAC metadata.", + "schema": [ + { + "title": "New Collection", + "type": "string", + "pattern": "^[A-Za-z0-9_\\-\\.~/]+$" + }, + { + "title": "Existing Collection", + "type": "string", + "subtype": "collection-id", + "pattern": "^[A-Za-z0-9_\\-\\.~/]+$" + } + ] + } + ], + "returns": { + "description": "Returns the collection identifier.", + "schema": { + "type": "string", + "subtype": "collection-id", + "pattern": "^[A-Za-z0-9_\\-\\.~/]+$" + } + } +} diff --git a/proposals/export_workspace.json b/proposals/export_workspace.json new file mode 100644 index 00000000..c469254f --- /dev/null +++ b/proposals/export_workspace.json @@ -0,0 +1,48 @@ +{ + "id": "export_workspace", + "summary": "Export data to a cloud user workspace", + "description": "Exports the given processing results made available through a STAC resource (e.g., a STAC Collection) to the given user workspace. The STAC resource itself is exported with all STAC resources and assets underneath.", + "categories": [ + "export", + "stac" + ], + "experimental": true, + "parameters": [ + { + "name": "data", + "description": "The data to export to the user workspace as a STAC resource.", + "schema": { + "type": "object", + "subtype": "stac" + } + }, + { + "name": "workspace", + "description": "The identifier of the workspace to export to.", + "schema": { + "type": "string", + "pattern": "^[\\w\\-\\.~]+$", + "subtype": "workspace-id" + } + }, + { + "name": "merge", + "description": "Provides a cloud-specific path identifier to a STAC resource to merge the given STAC resource into. If not provided, the STAC resource is kept separate from any other STAC resources in the workspace.", + "schema": { + "type": [ + "string", + "null" + ] + }, + "optional": true, + "default": null + } + ], + "returns": { + "description": "Returns the potentially updated STAC resource.", + "schema": { + "type": "object", + "subtype": "stac" + } + } +} diff --git a/proposals/stac_modify.json b/proposals/stac_modify.json new file mode 100644 index 00000000..8377c8a8 --- /dev/null +++ b/proposals/stac_modify.json @@ -0,0 +1,41 @@ +{ + "id": "stac_modify", + "summary": "Updates an existing STAC resource", + "description": "Modifies the given STAC resource (e.g., a STAC Collection or Item) based on the given changeset.", + "categories": [ + "stac" + ], + "experimental": true, + "parameters": [ + { + "name": "data", + "description": "The existing STAC resource.", + "schema": { + "type": "object", + "subtype": "stac" + } + }, + { + "name": "changes", + "description": "A potentially incomplete STAC resource that should be merged with the existing STAC resource. It follows the [RFC 7386: JSON Merge Patch](https://www.rfc-editor.org/rfc/rfc7386.html) specification.", + "schema": { + "type": "object" + } + } + ], + "returns": { + "description": "Returns the modified STAC resource.", + "schema": { + "type": "object", + "subtype": "stac" + } + }, + "links": [ + { + "href": "https://www.rfc-editor.org/rfc/rfc7386.html", + "title": "RFC 7386: JSON Merge Patch", + "type": "text/html", + "rel": "about" + } + ] +} diff --git a/save_result.json b/save_result.json index 7b952ead..418bafc7 100644 --- a/save_result.json +++ b/save_result.json @@ -4,7 +4,8 @@ "description": "Makes the processed data available in the given file format to the corresponding medium that is relevant for the context this processes is applied in:\n\n* For **batch jobs** the data is stored on the back-end. STAC-compatible metadata is usually made available with the processed data.\n* For **synchronous processing** the data is sent to the client as a direct response to the request.\n* **Secondary web services** are provided with the processed data so that it can make use of it (e.g., visualize it). Web service may require the data in a certain format. Please refer to the documentation of the individual service types for details.", "categories": [ "cubes", - "export" + "export", + "stac" ], "parameters": [ { @@ -35,10 +36,10 @@ } ], "returns": { - "description": "Always returns `true` as in case of an error an exception is thrown which aborts the execution of the process.", + "description": "Returns the STAC resource that was created in the process of saving the result.", "schema": { - "type": "boolean", - "const": true + "type": "object", + "subtype": "stac" } }, "exceptions": {