Polaris Catalog Spec: Generic Table APIs (#1150)

gh-yzou · web-flow · commit 9425a6e0cb5f · 2025-03-13T09:38:36.000-07:00
diff --git a/api/README.md b/api/README.md
@@ -31,6 +31,8 @@ This directory contains the API modules for Apache Polaris.
   Polaris Management API.
 - [`polaris-api-iceberg-service`](iceberg-service): contains the service classes for the Polaris
   Iceberg REST API.
+- [`polaris-api-catalog-service`](polaris-catalog-service): contains the service classes for the Polaris
+  native Catalog REST API.
 
 The classes in these modules are generated from the OpenAPI specification files in the
 [`spec`](../spec) directory.
diff --git a/api/iceberg-service/build.gradle.kts b/api/iceberg-service/build.gradle.kts
@@ -55,7 +55,7 @@ openApiGenerate {
   ignoreFileOverride = "$rootDir/.openapi-generator-ignore"
   removeOperationIdPrefix = true
   templateDir = "$rootDir/server-templates"
-  globalProperties.put("apis", "")
+  globalProperties.put("apis", "CatalogApi,ConfigurationApi,OAuth2Api")
   globalProperties.put("models", "false")
   globalProperties.put("apiDocs", "false")
   globalProperties.put("modelTests", "false")
diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+  alias(libs.plugins.openapi.generator)
+  id("polaris-client")
+  alias(libs.plugins.jandex)
+}
+
+val genericTableModels =
+  listOf(
+      "CreateGenericTableRequest",
+      "LoadGenericTableResponse",
+      "ListGenericTablesResponse",
+      "GenericTable",
+    )
+    .joinToString(",")
+
+dependencies {
+  implementation(project(":polaris-core"))
+
+  implementation(platform(libs.iceberg.bom))
+  implementation("org.apache.iceberg:iceberg-api")
+  implementation("org.apache.iceberg:iceberg-core")
+
+  implementation(libs.jakarta.annotation.api)
+  implementation(libs.jakarta.inject.api)
+  implementation(libs.jakarta.validation.api)
+  implementation(libs.swagger.annotations)
+
+  implementation(libs.jakarta.servlet.api)
+  implementation(libs.jakarta.ws.rs.api)
+
+  implementation(platform(libs.micrometer.bom))
+  implementation("io.micrometer:micrometer-core")
+
+  implementation(platform(libs.jackson.bom))
+  implementation("com.fasterxml.jackson.core:jackson-annotations")
+  implementation("com.fasterxml.jackson.core:jackson-core")
+  implementation("com.fasterxml.jackson.core:jackson-databind")
+}
+
+openApiGenerate {
+  inputSpec = "$rootDir/spec/polaris-catalog-service.yaml"
+  generatorName = "jaxrs-resteasy"
+  outputDir = "$projectDir/build/generated"
+  apiPackage = "org.apache.polaris.service.catalog.api"
+  modelPackage = "org.apache.polaris.service.types"
+  ignoreFileOverride = "$rootDir/.openapi-generator-ignore"
+  removeOperationIdPrefix = true
+  templateDir = "$rootDir/server-templates"
+  globalProperties.put("apis", "GenericTableApi")
+  globalProperties.put("models", genericTableModels)
+  globalProperties.put("apiDocs", "false")
+  globalProperties.put("modelTests", "false")
+  configOptions.put("resourceName", "catalog")
+  configOptions.put("useTags", "true")
+  configOptions.put("useBeanValidation", "false")
+  configOptions.put("sourceFolder", "src/main/java")
+  configOptions.put("useJakartaEe", "true")
+  configOptions.put("generateBuilders", "true")
+  configOptions.put("generateConstructorWithAllArgs", "true")
+  configOptions.put("openApiNullable", "false")
+  openapiNormalizer.put("REFACTOR_ALLOF_WITH_PROPERTIES_ONLY", "true")
+  additionalProperties.put("apiNamePrefix", "PolarisCatalog")
+  additionalProperties.put("apiNameSuffix", "")
+  additionalProperties.put("metricsPrefix", "polaris")
+  serverVariables.put("basePath", "api/catalog")
+  importMappings =
+    mapOf(
+      "ErrorModel" to "org.apache.iceberg.rest.responses.ErrorResponse",
+      "IcebergErrorResponse" to "org.apache.iceberg.rest.responses.ErrorResponse",
+      "TableIdentifier" to "org.apache.iceberg.catalog.TableIdentifier",
+    )
+}
+
+listOf("sourcesJar", "compileJava").forEach { task ->
+  tasks.named(task) { dependsOn("openApiGenerate") }
+}
+
+sourceSets {
+  main { java { srcDir(project.layout.buildDirectory.dir("generated/src/main/java")) } }
+}
+
+tasks.named("javadoc") { dependsOn("jandex") }
diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties
@@ -22,6 +22,7 @@ polaris-core=polaris-core
 polaris-api-iceberg-service=api/iceberg-service
 polaris-api-management-model=api/management-model
 polaris-api-management-service=api/management-service
+polaris-api-catalog-service=api/polaris-catalog-service
 polaris-service-common=service/common
 polaris-quarkus-defaults=quarkus/defaults
 polaris-quarkus-service=quarkus/service
diff --git a/spec/README.md b/spec/README.md
@@ -23,7 +23,7 @@ Polaris provides two sets of OpenAPI specifications:
 - `polaris-management-service.yml` - Defines the management APIs for using Polaris to create and manage Iceberg catalogs and their principals
 - `polaris-catalog-service.yaml` - Defines the specification for the Polaris Catalog API, which encompasses both the Iceberg REST Catalog API
    and Polaris-native API.
-  - `polaris-catalog-apis` - Contains the specifications of Polaris-native API
+  - `polaris-catalog-apis` - Contains the specification for Polaris-specific Catalog APIs
   - `iceberg-rest-catalog-open-api.yaml` - Contains the specification for Iceberg Rest Catalog API
 
 ## Generated Specification Files
@@ -45,5 +45,6 @@ npm install @redocly/cli -g
 ```
 redocly bundle spec/polaris-catalog-service.yaml -o spec/generated/bundled-polaris-catalog-service.yaml
 ```
+Note: the license header will be removed after the bundle generation, please manually add it back.
 
 
diff --git a/spec/generated/bundled-polaris-catalog-service.yaml b/spec/generated/bundled-polaris-catalog-service.yaml
@@ -1359,6 +1359,147 @@ paths:
           $ref: '#/components/responses/ServiceUnavailableResponse'
         5XX:
           $ref: '#/components/responses/ServerErrorResponse'
+  /polaris/v1/{prefix}/namespaces/{namespace}/generic-tables:
+    parameters:
+      - $ref: '#/components/parameters/prefix'
+      - $ref: '#/components/parameters/namespace'
+    get:
+      tags:
+        - Generic Table API
+      summary: List all generic tables identifiers underneath a given namespace
+      description: Return all generic table identifiers under this namespace
+      operationId: listGenericTables
+      parameters:
+        - $ref: '#/components/parameters/page-token'
+        - $ref: '#/components/parameters/page-size'
+      responses:
+        '200':
+          $ref: '#/components/responses/ListGenericTablesResponse'
+        '400':
+          $ref: '#/components/responses/BadRequestErrorResponse'
+        '401':
+          $ref: '#/components/responses/UnauthorizedResponse'
+        '403':
+          $ref: '#/components/responses/ForbiddenResponse'
+        '404':
+          description: Not Found - The namespace specified does not exist
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/IcebergErrorResponse'
+              examples:
+                NamespaceNotFound:
+                  $ref: '#/components/examples/NoSuchNamespaceError'
+        '503':
+          $ref: '#/components/responses/ServiceUnavailableResponse'
+        5XX:
+          $ref: '#/components/responses/ServerErrorResponse'
+    post:
+      tags:
+        - Generic Table API
+      summary: Create a generic table under the given namespace
+      description: Create a generic table under the given namespace, and return the created table information as a response.
+      operationId: createGenericTable
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CreateGenericTableRequest'
+      responses:
+        '200':
+          $ref: '#/components/responses/CreateGenericTableResponse'
+        '400':
+          $ref: '#/components/responses/BadRequestErrorResponse'
+        '401':
+          $ref: '#/components/responses/UnauthorizedResponse'
+        '403':
+          $ref: '#/components/responses/ForbiddenResponse'
+        '404':
+          description: Not Found - The namespace specified does not exist
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/IcebergErrorResponse'
+              examples:
+                NamespaceNotFound:
+                  $ref: '#/components/examples/NoSuchNamespaceError'
+        '409':
+          description: Conflict - The table already exists under the given namespace
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/IcebergErrorResponse'
+              examples:
+                TableAlreadyExists:
+                  $ref: '#/components/examples/TableAlreadyExistsError'
+        '503':
+          $ref: '#/components/responses/ServiceUnavailableResponse'
+        5XX:
+          $ref: '#/components/responses/ServerErrorResponse'
+  /polaris/v1/{prefix}/namespaces/{namespace}/generic-tables/{generic-table}:
+    parameters:
+      - $ref: '#/components/parameters/prefix'
+      - $ref: '#/components/parameters/namespace'
+      - $ref: '#/components/parameters/generic-table'
+    get:
+      tags:
+        - Generic Table API
+      summary: Load a generic table under the given namespace from the catalog
+      operationId: loadGenericTable
+      description: |-
+        Load a generic table from the catalog under the given namespace.
+        The response contains all table information passed during create.
+      responses:
+        '200':
+          $ref: '#/components/responses/LoadGenericTableResponse'
+        '400':
+          $ref: '#/components/responses/BadRequestErrorResponse'
+        '401':
+          $ref: '#/components/responses/UnauthorizedResponse'
+        '403':
+          $ref: '#/components/responses/ForbiddenResponse'
+        '404':
+          description: Not Found - NoSuchTableError, generic table to load does not exist
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/IcebergErrorResponse'
+              examples:
+                TableToLoadDoesNotExist:
+                  $ref: '#/components/examples/NoSuchTableError'
+        '503':
+          $ref: '#/components/responses/ServiceUnavailableResponse'
+        5XX:
+          $ref: '#/components/responses/ServerErrorResponse'
+    delete:
+      tags:
+        - Generic Table API
+      summary: Drop a generic table under the given namespace from the catalog
+      operationId: dropGenericTable
+      description: Remove a table under the given namespace from the catalog
+      responses:
+        '204':
+          description: Success, no content
+        '400':
+          $ref: '#/components/responses/BadRequestErrorResponse'
+        '401':
+          $ref: '#/components/responses/UnauthorizedResponse'
+        '403':
+          $ref: '#/components/responses/ForbiddenResponse'
+        '404':
+          description: Not Found - NoSuchTableError, Generic table to drop does not exist
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/IcebergErrorResponse'
+              examples:
+                TableToDeleteDoesNotExist:
+                  $ref: '#/components/examples/NoSuchTableError'
+        '503':
+          $ref: '#/components/responses/ServiceUnavailableResponse'
+        5XX:
+          $ref: '#/components/responses/ServerErrorResponse'
 components:
   securitySchemes:
     OAuth2:
@@ -3257,6 +3398,62 @@ components:
           $ref: '#/components/schemas/NotificationType'
         payload:
           $ref: '#/components/schemas/TableUpdateNotification'
+    ListGenericTablesResponse:
+      type: object
+      properties:
+        next-page-token:
+          $ref: '#/components/schemas/PageToken'
+        identifiers:
+          type: array
+          uniqueItems: true
+          items:
+            $ref: '#/components/schemas/TableIdentifier'
+    CreateGenericTableRequest:
+      type: object
+      required:
+        - name
+        - format
+      properties:
+        name:
+          type: string
+        format:
+          type: string
+        doc:
+          type: string
+        properties:
+          type: object
+          additionalProperties:
+            type: string
+    GenericTable:
+      type: object
+      description: |
+        Generic Table information.
+        - `name` name for the generic table
+        - `format` format for the generic table, i.e. "delta", "csv"
+        - `properties` properties for the generic table passed on creation
+        - `doc` comment or description for the generic table
+      required:
+        - name
+        - format
+      properties:
+        name:
+          type: string
+        format:
+          type: string
+        doc:
+          type: string
+        properties:
+          type: object
+          additionalProperties:
+            type: string
+    LoadGenericTableResponse:
+      description: Result used when a table is successfully loaded.
+      type: object
+      required:
+        - table
+      properties:
+        table:
+          $ref: '#/components/schemas/GenericTable'
   responses:
     BadRequestErrorResponse:
       description: Indicates a bad request error. It could be caused by an unexpected request body format or other forms of request validation failure, such as invalid json. Usually serves application/json content, although in some cases simple text/plain content might be returned by the server's middleware.
@@ -3436,6 +3633,24 @@ components:
         application/json:
           schema:
             $ref: '#/components/schemas/LoadViewResult'
+    ListGenericTablesResponse:
+      description: List of generic table identifiers.
+      content:
+        application/json:
+          schema:
+            $ref: '#/components/schemas/ListGenericTablesResponse'
+    CreateGenericTableResponse:
+      description: Table result if successfully created a generic table.
+      content:
+        application/json:
+          schema:
+            $ref: '#/components/schemas/LoadGenericTableResponse'
+    LoadGenericTableResponse:
+      description: Table result if successfully load a generic table.
+      content:
+        application/json:
+          schema:
+            $ref: '#/components/schemas/LoadGenericTableResponse'
   parameters:
     prefix:
       name: prefix
@@ -3505,6 +3720,14 @@ components:
       schema:
         type: string
       example: sales
+    generic-table:
+      name: generic-table
+      in: path
+      description: A generic table name
+      required: true
+      schema:
+        type: string
+      example: sales
   examples:
     ListNamespacesNonEmptyExample:
       summary: A non-empty list of namespaces
diff --git a/spec/polaris-catalog-apis/generic-tables-api.yaml b/spec/polaris-catalog-apis/generic-tables-api.yaml
diff --git a/spec/polaris-catalog-service.yaml b/spec/polaris-catalog-service.yaml
