From b001730be94b806f50e248cbd6a63ca7ac249434 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 18 Oct 2024 14:51:40 +0200 Subject: [PATCH] Add type field and validation --- README.md | 24 ++++++++++++++++------- examples/catalog-link.json | 1 + examples/collection.json | 1 + examples/item-naip.json | 4 +++- json-schema/platforms/aws-s3.json | 29 ++++++++++++++++++++++++++++ json-schema/platforms/custom-s3.json | 16 +++++++++++++++ json-schema/platforms/ms-azure.json | 23 ++++++++++++++++++++++ json-schema/schema.json | 16 +++++++++++++++ package.json | 6 +++--- validator-config.json | 8 ++++++++ 10 files changed, 117 insertions(+), 11 deletions(-) create mode 100644 json-schema/platforms/aws-s3.json create mode 100644 json-schema/platforms/custom-s3.json create mode 100644 json-schema/platforms/ms-azure.json create mode 100644 validator-config.json diff --git a/README.md b/README.md index 1be2faf..f002a29 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ - **Field Name Prefix:** storage - **Scope:** Item, Catalog, Collection - **Extension [Maturity Classification](https://github.com/radiantearth/stac-spec/tree/master/extensions/README.md#extension-maturity):** Pilot -- **Owner**: @davidraleigh @matthewhanson +- **Owner**: @matthewhanson @m-mohr This document explains the Storage Extension to the [SpatioTemporal Asset Catalog](https://github.com/radiantearth/stac-spec) (STAC) specification. It allows adding details related to cloud object storage access and costs to be associated with STAC Assets. @@ -53,6 +53,7 @@ The fields in the table below can be used in these parts of STAC documents: | Field Name | Type | Description | | -------------- | ------- | ----------- | +| type | string | **REQUIRED.** Type identifier for the platform, see below. | | platform | string | **REQUIRED.** The cloud provider where data is stored as URI or URI template to the API. | | region | string | The region where the data is stored. Relevant to speed of access and inter region egress costs (as defined by PaaS provider). | | requester_pays | boolean | Is the data "requester pays" (`true`) or is it "data manager/cloud provider pays" (`false`). Defaults to `false`. | @@ -65,11 +66,12 @@ The properties `title` and `description` as defined in Common Metadata should be The `platform` field identifies the cloud provider where the data is stored as URI or URI template to the API of the service. If a URI template is provided, all variables must be defined in the Storage Scheme Object as a property with the same name. -For example, the URI template `https://{bucket}.{region}.example.com` must have at least the properties +For example, the URI template `https://{bucket}.{region}.example.com` must have at least the properties `bucket` and `region` defined: ```json { + "type": "example", "platform": "https://{bucket}.{region}.example.com", "region": "eu-fr", "bucket": "john-doe-stac", @@ -82,19 +84,27 @@ the `platform` property must identify the host so that the URL can be resolved w For example, this is especially useful to provide the endpoint URL for custom S3 providers. In this case the `platform` could effectively provide the endpoint URL. +#### type + We try to collect pre-defined templates and best pratices for as many providers as possible -in this repository, but be aware that these are not part of the official extension releases -and are not validated. This extension just provides the framework, the provider best pratices +in this repository, but be aware that these are not part of the official extension releases. +This extension just provides the framework, the provider best pratices may change at any time without a new version of this extension being released. The following providers have defined best pratices at this point: -- [AWS S3](platforms/aws-s3.md) -- [Generic S3 (non-AWS)](platforms/custom-s3.md) -- [Microsoft Azure](platforms/ms-azure.md) +| `type` | Provider and Documentation | +| ----------- | -------------------------- | +| `aws-s3` | [AWS S3](platforms/aws-s3.md) | +| `custom-s3` | [Generic S3 (non-AWS)](platforms/custom-s3.md) | +| `ms-azure` | [Microsoft Azure](platforms/ms-azure.md) | Feel encouraged to submit additional platform specifications via Pull Requests. +The `type` fields can be any value chosen by the implementor, +but the types defined in the table above should be used as defined in the best practices. +This ensures proper schema validation. + ## Contributing See the [Contributor documentation](CONTRIBUTING.md) for details. diff --git a/examples/catalog-link.json b/examples/catalog-link.json index 442189a..8094a2c 100644 --- a/examples/catalog-link.json +++ b/examples/catalog-link.json @@ -9,6 +9,7 @@ "description": "An example catalog with a link to documentation on object storage.", "storage:schemes": { "aws": { + "type": "aws-s3", "platform": "https://{bucket}.s3.{region}.amazonaws.com", "bucket": "mybucket", "region": "us-west-2", diff --git a/examples/collection.json b/examples/collection.json index 26b606e..4ca6980 100644 --- a/examples/collection.json +++ b/examples/collection.json @@ -11,6 +11,7 @@ "license": "CC-0", "storage:schemes": { "aws": { + "type": "aws-s3", "platform": "https://{bucket}.s3.{region}.amazonaws.com", "bucket": "mybucket", "region": "us-west-2", diff --git a/examples/item-naip.json b/examples/item-naip.json index 26a2727..2458121 100644 --- a/examples/item-naip.json +++ b/examples/item-naip.json @@ -43,21 +43,23 @@ "properties": { "datetime": "2016-09-28T00:00:00+00:00", "mission": "NAIP", - "platform": "UNKNOWN_PLATFORM", "gsd": 1, "storage:schemes": { "az-wus2-ar": { + "type": "ms-azure", "platform": "https://{account}.blob.core.windows.net", "account": "jon-doe-123", "region": "westus2" }, "aws-std": { + "type": "aws-s3", "platform": "https://{bucket}.s3.{region}.amazonaws.com", "bucket": "naip-visualization", "region": "us-west-2", "requester_pays": true }, "minio": { + "type": "custom-s3", "platform": "https://play.min.io:9000" } } diff --git a/json-schema/platforms/aws-s3.json b/json-schema/platforms/aws-s3.json new file mode 100644 index 0000000..3721741 --- /dev/null +++ b/json-schema/platforms/aws-s3.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json", + "title": "AWS S3", + "type": "object", + "if": { + "properties": { + "type": { + "const": "aws-s3" + } + } + }, + "then": { + "properties": { + "platform": { + "const": "https://{bucket}.s3.{region}.amazonaws.com" + }, + "bucket": { + "$comment": "See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html", + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-.]{1,61}[a-z0-9]$" + }, + "region": { + "type": "string", + "pattern": "^[a-z0-9-]+$" + } + } + } +} \ No newline at end of file diff --git a/json-schema/platforms/custom-s3.json b/json-schema/platforms/custom-s3.json new file mode 100644 index 0000000..25a89b7 --- /dev/null +++ b/json-schema/platforms/custom-s3.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json", + "title": "Generic S3", + "type": "object", + "if": { + "properties": { + "type": { + "const": "custom-s3" + } + } + }, + "then": { + "$comment": "No specific validation rules apply" + } +} \ No newline at end of file diff --git a/json-schema/platforms/ms-azure.json b/json-schema/platforms/ms-azure.json new file mode 100644 index 0000000..44132da --- /dev/null +++ b/json-schema/platforms/ms-azure.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json", + "title": "Microsoft Azure", + "type": "object", + "if": { + "properties": { + "type": { + "const": "ms-azure" + } + } + }, + "then": { + "properties": { + "platform": { + "const": "https://{account}.blob.core.windows.net" + }, + "account": { + "type": "string" + } + } + } +} \ No newline at end of file diff --git a/json-schema/schema.json b/json-schema/schema.json index 44c265f..1716e31 100644 --- a/json-schema/schema.json +++ b/json-schema/schema.json @@ -96,9 +96,14 @@ "patternProperties": { "^.{1,}$": { "required": [ + "type", "platform" ], "properties": { + "type": { + "title": "Type identifier", + "type": "string" + }, "platform": { "title": "Platform", "type": "string", @@ -115,6 +120,17 @@ "default": false } }, + "allOf": [ + { + "$ref": "./platforms/aws-s3.json" + }, + { + "$ref": "./platforms/custom-s3.json" + }, + { + "$ref": "./platforms/ms-azure.json" + } + ], "additionalProperties": true } }, diff --git a/package.json b/package.json index 948eb39..09ae103 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "scripts": { "test": "npm run check-markdown && npm run check-examples", "check-markdown": "remark . -f -r .github/remark.yaml", - "check-examples": "stac-node-validator . --lint --verbose --schemaMap https://stac-extensions.github.io/storage/v2.0.0/schema.json=./json-schema/schema.json", - "format-examples": "stac-node-validator . --format --schemaMap https://stac-extensions.github.io/storage/v2.0.0/schema.json=./json-schema/schema.json" + "check-examples": "stac-node-validator examples --verbose --lint --config ./validator-config.json", + "format-examples": "stac-node-validator examples --format --config ./validator-config.json" }, "dependencies": { "remark-cli": "^8.0.0", @@ -15,6 +15,6 @@ "remark-preset-lint-markdown-style-guide": "^3.0.0", "remark-preset-lint-recommended": "^4.0.0", "remark-validate-links": "^10.0.0", - "stac-node-validator": "^1.1.0" + "stac-node-validator": "^2.0.0-beta.12" } } diff --git a/validator-config.json b/validator-config.json new file mode 100644 index 0000000..e083d05 --- /dev/null +++ b/validator-config.json @@ -0,0 +1,8 @@ +{ + "schemaMap": { + "https://stac-extensions.github.io/storage/v2.0.0/schema.json": "./json-schema/schema.json", + "https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json": "./json-schema/platforms/aws-s3.json", + "https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json": "./json-schema/platforms/custom-s3.json", + "https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json": "./json-schema/platforms/ms-azure.json" + } +} \ No newline at end of file