diff --git a/api/README.md b/api/README.md index efecb80b21..7c7fb61fc0 100644 --- a/api/README.md +++ b/api/README.md @@ -31,6 +31,8 @@ This directory contains the API modules for Apache Polaris. Polaris Management API. - [`polaris-api-iceberg-service`](iceberg-service): contains the service classes for the Polaris Iceberg REST API. +- [`polaris-api-catalog-service`](polaris-catalog-service): contains the service classes for the Polaris + native Catalog REST API. The classes in these modules are generated from the OpenAPI specification files in the [`spec`](../spec) directory. \ No newline at end of file diff --git a/api/iceberg-service/build.gradle.kts b/api/iceberg-service/build.gradle.kts index f669f4a367..a2a73eadef 100644 --- a/api/iceberg-service/build.gradle.kts +++ b/api/iceberg-service/build.gradle.kts @@ -55,7 +55,7 @@ openApiGenerate { ignoreFileOverride = "$rootDir/.openapi-generator-ignore" removeOperationIdPrefix = true templateDir = "$rootDir/server-templates" - globalProperties.put("apis", "") + globalProperties.put("apis", "CatalogApi,ConfigurationApi,OAuth2Api") globalProperties.put("models", "false") globalProperties.put("apiDocs", "false") globalProperties.put("modelTests", "false") diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts new file mode 100644 index 0000000000..f231943335 --- /dev/null +++ b/api/polaris-catalog-service/build.gradle.kts @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + alias(libs.plugins.openapi.generator) + id("polaris-client") + alias(libs.plugins.jandex) +} + +val genericTableModels = + listOf( + "CreateGenericTableRequest", + "LoadGenericTableResponse", + "ListGenericTablesResponse", + "GenericTable", + ) + .joinToString(",") + +dependencies { + implementation(project(":polaris-core")) + + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + implementation("org.apache.iceberg:iceberg-core") + + implementation(libs.jakarta.annotation.api) + implementation(libs.jakarta.inject.api) + implementation(libs.jakarta.validation.api) + implementation(libs.swagger.annotations) + + implementation(libs.jakarta.servlet.api) + implementation(libs.jakarta.ws.rs.api) + + implementation(platform(libs.micrometer.bom)) + implementation("io.micrometer:micrometer-core") + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-annotations") + implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.core:jackson-databind") +} + +openApiGenerate { + inputSpec = "$rootDir/spec/polaris-catalog-service.yaml" + generatorName = "jaxrs-resteasy" + outputDir = "$projectDir/build/generated" + apiPackage = "org.apache.polaris.service.catalog.api" + modelPackage = "org.apache.polaris.service.types" + ignoreFileOverride = "$rootDir/.openapi-generator-ignore" + removeOperationIdPrefix = true + templateDir = "$rootDir/server-templates" + globalProperties.put("apis", "GenericTableApi") + globalProperties.put("models", genericTableModels) + globalProperties.put("apiDocs", "false") + globalProperties.put("modelTests", "false") + configOptions.put("resourceName", "catalog") + configOptions.put("useTags", "true") + configOptions.put("useBeanValidation", "false") + configOptions.put("sourceFolder", "src/main/java") + configOptions.put("useJakartaEe", "true") + configOptions.put("generateBuilders", "true") + configOptions.put("generateConstructorWithAllArgs", "true") + configOptions.put("openApiNullable", "false") + openapiNormalizer.put("REFACTOR_ALLOF_WITH_PROPERTIES_ONLY", "true") + additionalProperties.put("apiNamePrefix", "PolarisCatalog") + additionalProperties.put("apiNameSuffix", "") + additionalProperties.put("metricsPrefix", "polaris") + serverVariables.put("basePath", "api/catalog") + importMappings = + mapOf( + "ErrorModel" to "org.apache.iceberg.rest.responses.ErrorResponse", + "IcebergErrorResponse" to "org.apache.iceberg.rest.responses.ErrorResponse", + "TableIdentifier" to "org.apache.iceberg.catalog.TableIdentifier", + ) +} + +listOf("sourcesJar", "compileJava").forEach { task -> + tasks.named(task) { dependsOn("openApiGenerate") } +} + +sourceSets { + main { java { srcDir(project.layout.buildDirectory.dir("generated/src/main/java")) } } +} + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 4f13a8cd88..88fbcd0fa4 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -22,6 +22,7 @@ polaris-core=polaris-core polaris-api-iceberg-service=api/iceberg-service polaris-api-management-model=api/management-model polaris-api-management-service=api/management-service +polaris-api-catalog-service=api/polaris-catalog-service polaris-service-common=service/common polaris-quarkus-defaults=quarkus/defaults polaris-quarkus-service=quarkus/service diff --git a/spec/README.md b/spec/README.md index 7790d0ed34..a25010bf36 100644 --- a/spec/README.md +++ b/spec/README.md @@ -23,7 +23,7 @@ Polaris provides two sets of OpenAPI specifications: - `polaris-management-service.yml` - Defines the management APIs for using Polaris to create and manage Iceberg catalogs and their principals - `polaris-catalog-service.yaml` - Defines the specification for the Polaris Catalog API, which encompasses both the Iceberg REST Catalog API and Polaris-native API. - - `polaris-catalog-apis` - Contains the specifications of Polaris-native API + - `polaris-catalog-apis` - Contains the specification for Polaris-specific Catalog APIs - `iceberg-rest-catalog-open-api.yaml` - Contains the specification for Iceberg Rest Catalog API ## Generated Specification Files @@ -45,5 +45,6 @@ npm install @redocly/cli -g ``` redocly bundle spec/polaris-catalog-service.yaml -o spec/generated/bundled-polaris-catalog-service.yaml ``` +Note: the license header will be removed after the bundle generation, please manually add it back. diff --git a/spec/generated/bundled-polaris-catalog-service.yaml b/spec/generated/bundled-polaris-catalog-service.yaml index 1b10c66dec..2a5509ca9e 100644 --- a/spec/generated/bundled-polaris-catalog-service.yaml +++ b/spec/generated/bundled-polaris-catalog-service.yaml @@ -1359,6 +1359,147 @@ paths: $ref: '#/components/responses/ServiceUnavailableResponse' 5XX: $ref: '#/components/responses/ServerErrorResponse' + /polaris/v1/{prefix}/namespaces/{namespace}/generic-tables: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + get: + tags: + - Generic Table API + summary: List all generic tables identifiers underneath a given namespace + description: Return all generic table identifiers under this namespace + operationId: listGenericTables + parameters: + - $ref: '#/components/parameters/page-token' + - $ref: '#/components/parameters/page-size' + responses: + '200': + $ref: '#/components/responses/ListGenericTablesResponse' + '400': + $ref: '#/components/responses/BadRequestErrorResponse' + '401': + $ref: '#/components/responses/UnauthorizedResponse' + '403': + $ref: '#/components/responses/ForbiddenResponse' + '404': + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + '503': + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + post: + tags: + - Generic Table API + summary: Create a generic table under the given namespace + description: Create a generic table under the given namespace, and return the created table information as a response. + operationId: createGenericTable + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateGenericTableRequest' + responses: + '200': + $ref: '#/components/responses/CreateGenericTableResponse' + '400': + $ref: '#/components/responses/BadRequestErrorResponse' + '401': + $ref: '#/components/responses/UnauthorizedResponse' + '403': + $ref: '#/components/responses/ForbiddenResponse' + '404': + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '#/components/examples/NoSuchNamespaceError' + '409': + description: Conflict - The table already exists under the given namespace + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableAlreadyExists: + $ref: '#/components/examples/TableAlreadyExistsError' + '503': + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + /polaris/v1/{prefix}/namespaces/{namespace}/generic-tables/{generic-table}: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/generic-table' + get: + tags: + - Generic Table API + summary: Load a generic table under the given namespace from the catalog + operationId: loadGenericTable + description: |- + Load a generic table from the catalog under the given namespace. + The response contains all table information passed during create. + responses: + '200': + $ref: '#/components/responses/LoadGenericTableResponse' + '400': + $ref: '#/components/responses/BadRequestErrorResponse' + '401': + $ref: '#/components/responses/UnauthorizedResponse' + '403': + $ref: '#/components/responses/ForbiddenResponse' + '404': + description: Not Found - NoSuchTableError, generic table to load does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToLoadDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + '503': + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + delete: + tags: + - Generic Table API + summary: Drop a generic table under the given namespace from the catalog + operationId: dropGenericTable + description: Remove a table under the given namespace from the catalog + responses: + '204': + description: Success, no content + '400': + $ref: '#/components/responses/BadRequestErrorResponse' + '401': + $ref: '#/components/responses/UnauthorizedResponse' + '403': + $ref: '#/components/responses/ForbiddenResponse' + '404': + description: Not Found - NoSuchTableError, Generic table to drop does not exist + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + examples: + TableToDeleteDoesNotExist: + $ref: '#/components/examples/NoSuchTableError' + '503': + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' components: securitySchemes: OAuth2: @@ -3257,6 +3398,62 @@ components: $ref: '#/components/schemas/NotificationType' payload: $ref: '#/components/schemas/TableUpdateNotification' + ListGenericTablesResponse: + type: object + properties: + next-page-token: + $ref: '#/components/schemas/PageToken' + identifiers: + type: array + uniqueItems: true + items: + $ref: '#/components/schemas/TableIdentifier' + CreateGenericTableRequest: + type: object + required: + - name + - format + properties: + name: + type: string + format: + type: string + doc: + type: string + properties: + type: object + additionalProperties: + type: string + GenericTable: + type: object + description: | + Generic Table information. + - `name` name for the generic table + - `format` format for the generic table, i.e. "delta", "csv" + - `properties` properties for the generic table passed on creation + - `doc` comment or description for the generic table + required: + - name + - format + properties: + name: + type: string + format: + type: string + doc: + type: string + properties: + type: object + additionalProperties: + type: string + LoadGenericTableResponse: + description: Result used when a table is successfully loaded. + type: object + required: + - table + properties: + table: + $ref: '#/components/schemas/GenericTable' responses: BadRequestErrorResponse: description: Indicates a bad request error. It could be caused by an unexpected request body format or other forms of request validation failure, such as invalid json. Usually serves application/json content, although in some cases simple text/plain content might be returned by the server's middleware. @@ -3436,6 +3633,24 @@ components: application/json: schema: $ref: '#/components/schemas/LoadViewResult' + ListGenericTablesResponse: + description: List of generic table identifiers. + content: + application/json: + schema: + $ref: '#/components/schemas/ListGenericTablesResponse' + CreateGenericTableResponse: + description: Table result if successfully created a generic table. + content: + application/json: + schema: + $ref: '#/components/schemas/LoadGenericTableResponse' + LoadGenericTableResponse: + description: Table result if successfully load a generic table. + content: + application/json: + schema: + $ref: '#/components/schemas/LoadGenericTableResponse' parameters: prefix: name: prefix @@ -3505,6 +3720,14 @@ components: schema: type: string example: sales + generic-table: + name: generic-table + in: path + description: A generic table name + required: true + schema: + type: string + example: sales examples: ListNamespacesNonEmptyExample: summary: A non-empty list of namespaces diff --git a/spec/polaris-catalog-apis/generic-tables-api.yaml b/spec/polaris-catalog-apis/generic-tables-api.yaml new file mode 100644 index 0000000000..92ee7f9a59 --- /dev/null +++ b/spec/polaris-catalog-apis/generic-tables-api.yaml @@ -0,0 +1,273 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +--- +paths: + /polaris/v1/{prefix}/namespaces/{namespace}/generic-tables: + parameters: + - $ref: '../iceberg-rest-catalog-open-api.yaml#/components/parameters/prefix' + - $ref: '../iceberg-rest-catalog-open-api.yaml#/components/parameters/namespace' + + get: + tags: + - Generic Table API + summary: List all generic tables identifiers underneath a given namespace + description: Return all generic table identifiers under this namespace + operationId: listGenericTables + parameters: + - $ref: '../iceberg-rest-catalog-open-api.yaml#/components/parameters/page-token' + - $ref: '../iceberg-rest-catalog-open-api.yaml#/components/parameters/page-size' + responses: + 200: + $ref: '#/components/responses/ListGenericTablesResponse' + 400: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/BadRequestErrorResponse' + 401: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/UnauthorizedResponse' + 403: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/examples/NoSuchNamespaceError' + 503: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServerErrorResponse' + + post: + tags: + - Generic Table API + summary: Create a generic table under the given namespace + description: + Create a generic table under the given namespace, and return the created table information as a response. + operationId: createGenericTable + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateGenericTableRequest' + responses: + 200: + $ref: '#/components/responses/CreateGenericTableResponse' + 400: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/BadRequestErrorResponse' + 401: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/UnauthorizedResponse' + 403: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ForbiddenResponse' + 404: + description: Not Found - The namespace specified does not exist + content: + application/json: + schema: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/schemas/IcebergErrorResponse' + examples: + NamespaceNotFound: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/examples/NoSuchNamespaceError' + 409: + description: Conflict - The table already exists under the given namespace + content: + application/json: + schema: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/schemas/IcebergErrorResponse' + examples: + TableAlreadyExists: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/examples/TableAlreadyExistsError' + 503: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServerErrorResponse' + + /polaris/v1/{prefix}/namespaces/{namespace}/generic-tables/{generic-table}: + parameters: + - $ref: '../iceberg-rest-catalog-open-api.yaml#/components/parameters/prefix' + - $ref: '../iceberg-rest-catalog-open-api.yaml#/components/parameters/namespace' + - $ref: '#/components/parameters/generic-table' + + get: + tags: + - Generic Table API + summary: Load a generic table under the given namespace from the catalog + operationId: loadGenericTable + description: + Load a generic table from the catalog under the given namespace. + + The response contains all table information passed during create. + + responses: + 200: + $ref: '#/components/responses/LoadGenericTableResponse' + 400: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/BadRequestErrorResponse' + 401: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/UnauthorizedResponse' + 403: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableError, generic table to load does not exist + content: + application/json: + schema: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/schemas/IcebergErrorResponse' + examples: + TableToLoadDoesNotExist: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/examples/NoSuchTableError' + 503: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServerErrorResponse' + + delete: + tags: + - Generic Table API + summary: Drop a generic table under the given namespace from the catalog + operationId: dropGenericTable + description: Remove a table under the given namespace from the catalog + responses: + 204: + description: Success, no content + 400: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/BadRequestErrorResponse' + 401: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/UnauthorizedResponse' + 403: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ForbiddenResponse' + 404: + description: + Not Found - NoSuchTableError, Generic table to drop does not exist + content: + application/json: + schema: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/schemas/IcebergErrorResponse' + examples: + TableToDeleteDoesNotExist: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/examples/NoSuchTableError' + 503: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServerErrorResponse' + +components: + parameters: + generic-table: + name: generic-table + in: path + description: A generic table name + required: true + schema: + type: string + example: "sales" + + schemas: + CreateGenericTableRequest: + type: object + required: + - name + - format + properties: + name: + type: string + format: + type: string + doc: + type: string + properties: + type: object + additionalProperties: + type: string + + GenericTable: + type: object + description: > + Generic Table information. + + - `name` name for the generic table + + - `format` format for the generic table, i.e. "delta", "csv" + + - `properties` properties for the generic table passed on creation + + - `doc` comment or description for the generic table + required: + - name + - format + properties: + name: + type: string + format: + type: string + doc: + type: string + properties: + type: object + additionalProperties: + type: string + + LoadGenericTableResponse: + description: Result used when a table is successfully loaded. + type: object + required: + - table + properties: + table: + $ref: '#/components/schemas/GenericTable' + + ListGenericTablesResponse: + type: object + properties: + next-page-token: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/schemas/PageToken' + identifiers: + type: array + uniqueItems: true + items: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/schemas/TableIdentifier' + + responses: + + LoadGenericTableResponse: + description: Table result if successfully load a generic table. + content: + application/json: + schema: + $ref: '#/components/schemas/LoadGenericTableResponse' + + + CreateGenericTableResponse: + description: Table result if successfully created a generic table. + content: + application/json: + schema: + $ref: '#/components/schemas/LoadGenericTableResponse' + + + ListGenericTablesResponse: + description: List of generic table identifiers. + content: + application/json: + schema: + $ref: '#/components/schemas/ListGenericTablesResponse' diff --git a/spec/polaris-catalog-service.yaml b/spec/polaris-catalog-service.yaml index cedcf0f717..8299ec29c6 100644 --- a/spec/polaris-catalog-service.yaml +++ b/spec/polaris-catalog-service.yaml @@ -127,13 +127,21 @@ paths: # /v1/{prefix}/namespaces/{namespace}/tables/{table}/tasks: # Not implemented in Polaris - ###################### - # Polaris-native API # - ###################### + ############################ + # Polaris Notification API # + ############################ /v1/{prefix}/namespaces/{namespace}/tables/{table}/notifications: $ref: './polaris-catalog-apis/notifications-api.yaml#/paths/~1v1~1{prefix}~1namespaces~1{namespace}~1tables~1{table}~1notifications' + ############################# + # Polaris Generic Table API # + ############################# + /polaris/v1/{prefix}/namespaces/{namespace}/generic-tables: + $ref: './polaris-catalog-apis/generic-tables-api.yaml#/paths/~1polaris~1v1~1{prefix}~1namespaces~1{namespace}~1generic-tables' + + /polaris/v1/{prefix}/namespaces/{namespace}/generic-tables/{generic-table}: + $ref: './polaris-catalog-apis/generic-tables-api.yaml#/paths/~1polaris~1v1~1{prefix}~1namespaces~1{namespace}~1generic-tables~1{generic-table}' components: securitySchemes: