From 0ec241bf21e31b1cee97cef5a728e5699a1f0576 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 31 Mar 2025 10:46:56 -0700 Subject: [PATCH 01/18] prep for review --- api/polaris-catalog-service/build.gradle.kts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index 2fd861d80b..64105739a7 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -82,7 +82,8 @@ openApiGenerate { removeOperationIdPrefix = true templateDir = "$rootDir/server-templates" globalProperties.put("apis", "GenericTableApi,PolicyApi") - globalProperties.put("models", models) + // TODO -- reset this value to `models` after the Iceberg 1.8 change has merged + globalProperties.put("models", "true") globalProperties.put("apiDocs", "false") globalProperties.put("modelTests", "false") configOptions.put("resourceName", "catalog") From 96d55ac5641b1c9db19d6c6dbc59a5f04770d97c Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 31 Mar 2025 10:48:36 -0700 Subject: [PATCH 02/18] reset --- spec/iceberg-rest-catalog-open-api.yaml | 218 ++++++++++++++---------- 1 file changed, 130 insertions(+), 88 deletions(-) diff --git a/spec/iceberg-rest-catalog-open-api.yaml b/spec/iceberg-rest-catalog-open-api.yaml index b4adb6be96..5fb9f88045 100644 --- a/spec/iceberg-rest-catalog-open-api.yaml +++ b/spec/iceberg-rest-catalog-open-api.yaml @@ -17,12 +17,8 @@ # under the License. # -# CODE_COPIED_TO_POLARIS - -# Apache Iceberg Version: 1.7.1 - --- -openapi: 3.0.3 +openapi: 3.1.1 info: title: Apache Iceberg REST Catalog API license: @@ -934,6 +930,15 @@ paths: key. For example, "urn:ietf:params:oauth:token-type:jwt=". parameters: - $ref: '#/components/parameters/data-access' + - name: If-None-Match + in: header + description: + An optional header that allows the server to return 304 (Not Modified) if the metadata + is current. The content is the value of the ETag received in a CreateTableResponse or + LoadTableResponse. + required: false + schema: + type: string - in: query name: snapshots description: @@ -949,6 +954,10 @@ paths: responses: 200: $ref: '#/components/responses/LoadTableResponse' + 304: + description: + Not Modified - Based on the content of the 'If-None-Match' header the table metadata has + not changed since. 400: $ref: '#/components/responses/BadRequestErrorResponse' 401: @@ -1877,6 +1886,14 @@ components: type: integer minimum: 1 + etag: + name: ETag + in: header + description: Identifies a unique version of the table metadata. + required: false + schema: + type: string + ############################## # Application Schema Objects # ############################## @@ -2052,6 +2069,10 @@ components: type: boolean doc: type: string + initial-default: + $ref: "#/components/schemas/PrimitiveTypeValue" + write-default: + $ref: "#/components/schemas/PrimitiveTypeValue" StructType: type: object @@ -2061,7 +2082,7 @@ components: properties: type: type: string - enum: ["struct"] + const: "struct" fields: type: array items: @@ -2077,7 +2098,7 @@ components: properties: type: type: string - enum: ["list"] + const: "list" element-id: type: integer element: @@ -2097,7 +2118,7 @@ components: properties: type: type: string - enum: ["map"] + const: "map" key-id: type: integer key: @@ -2169,7 +2190,7 @@ components: properties: type: $ref: '#/components/schemas/ExpressionType' - enum: ["true"] + const: "true" FalseExpression: type: object @@ -2178,7 +2199,7 @@ components: properties: type: $ref: '#/components/schemas/ExpressionType' - enum: ["false"] + const: "false" AndOrExpression: type: object @@ -2203,7 +2224,7 @@ components: properties: type: $ref: '#/components/schemas/ExpressionType' - enum: ["not"] + const: "not" child: $ref: '#/components/schemas/Expression' @@ -2273,7 +2294,7 @@ components: properties: type: type: string - enum: ["transform"] + const: "transform" transform: $ref: '#/components/schemas/Transform' term: @@ -2648,6 +2669,7 @@ components: set-partition-statistics: '#/components/schemas/SetPartitionStatisticsUpdate' remove-partition-statistics: '#/components/schemas/RemovePartitionStatisticsUpdate' remove-partition-specs: '#/components/schemas/RemovePartitionSpecsUpdate' + enable-row-lineage: '#/components/schemas/EnableRowLineageUpdate' type: object required: - action @@ -2660,12 +2682,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - uuid properties: action: type: string - enum: ["assign-uuid"] + const: "assign-uuid" uuid: type: string @@ -2673,12 +2694,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - format-version properties: action: type: string - enum: ["upgrade-format-version"] + const: "upgrade-format-version" format-version: type: integer @@ -2686,28 +2706,32 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - schema properties: action: type: string - enum: ["add-schema"] + const: "add-schema" schema: $ref: '#/components/schemas/Schema' last-column-id: type: integer - description: The highest assigned column ID for the table. This is used to ensure columns are always assigned an unused ID when evolving schemas. When omitted, it will be computed on the server side. + deprecated: true + description: + This optional field is **DEPRECATED for REMOVAL** since it more safe to handle this internally, + and shouldn't be exposed to the clients. + + The highest assigned column ID for the table. This is used to ensure columns are always + assigned an unused ID when evolving schemas. When omitted, it will be computed on the server side. SetCurrentSchemaUpdate: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - schema-id properties: action: type: string - enum: ["set-current-schema"] + const: "set-current-schema" schema-id: type: integer description: Schema ID to set as current, or -1 to set last added schema @@ -2716,12 +2740,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - spec properties: action: type: string - enum: ["add-spec"] + const: "add-spec" spec: $ref: '#/components/schemas/PartitionSpec' @@ -2729,12 +2752,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - spec-id properties: action: type: string - enum: [ "set-default-spec" ] + const: "set-default-spec" spec-id: type: integer description: Partition spec ID to set as the default, or -1 to set last added spec @@ -2743,12 +2765,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - sort-order properties: action: type: string - enum: [ "add-sort-order" ] + const: "add-sort-order" sort-order: $ref: '#/components/schemas/SortOrder' @@ -2756,12 +2777,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - sort-order-id properties: action: type: string - enum: [ "set-default-sort-order" ] + const: "set-default-sort-order" sort-order-id: type: integer description: Sort order ID to set as the default, or -1 to set last added sort order @@ -2770,12 +2790,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - snapshot properties: action: type: string - enum: [ "add-snapshot" ] + const: "add-snapshot" snapshot: $ref: '#/components/schemas/Snapshot' @@ -2784,12 +2803,11 @@ components: - $ref: '#/components/schemas/BaseUpdate' - $ref: '#/components/schemas/SnapshotReference' required: - - action - ref-name properties: action: type: string - enum: [ "set-snapshot-ref" ] + const: "set-snapshot-ref" ref-name: type: string @@ -2797,12 +2815,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - snapshot-ids properties: action: type: string - enum: [ "remove-snapshots" ] + const: "remove-snapshots" snapshot-ids: type: array items: @@ -2813,12 +2830,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - ref-name properties: action: type: string - enum: [ "remove-snapshot-ref" ] + const: "remove-snapshot-ref" ref-name: type: string @@ -2826,12 +2842,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - location properties: action: type: string - enum: [ "set-location" ] + const: "set-location" location: type: string @@ -2839,12 +2854,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - updates properties: action: type: string - enum: [ "set-properties" ] + const: "set-properties" updates: type: object additionalProperties: @@ -2854,12 +2868,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - removals properties: action: type: string - enum: [ "remove-properties" ] + const: "remove-properties" removals: type: array items: @@ -2869,12 +2882,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - view-version properties: action: type: string - enum: [ "add-view-version" ] + const: "add-view-version" view-version: $ref: '#/components/schemas/ViewVersion' @@ -2882,12 +2894,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - view-version-id properties: action: type: string - enum: [ "set-current-view-version" ] + const: "set-current-view-version" view-version-id: type: integer description: The view version id to set as current, or -1 to set last added view version id @@ -2896,16 +2907,18 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - - snapshot-id - statistics properties: action: type: string - enum: [ "set-statistics" ] + const: "set-statistics" snapshot-id: type: integer format: int64 + deprecated: true + description: + This optional field is **DEPRECATED for REMOVAL** since it contains redundant information. + Clients should use the `statistics.snapshot-id` field instead. statistics: $ref: '#/components/schemas/StatisticsFile' @@ -2913,12 +2926,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - snapshot-id properties: action: type: string - enum: [ "remove-statistics" ] + const: "remove-statistics" snapshot-id: type: integer format: int64 @@ -2927,12 +2939,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - partition-statistics properties: action: type: string - enum: [ "set-partition-statistics" ] + const: "set-partition-statistics" partition-statistics: $ref: '#/components/schemas/PartitionStatisticsFile' @@ -2940,12 +2951,11 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: - - action - snapshot-id properties: action: type: string - enum: [ "remove-partition-statistics" ] + const: "remove-partition-statistics" snapshot-id: type: integer format: int64 @@ -2958,12 +2968,21 @@ components: properties: action: type: string - enum: [ "remove-partition-specs" ] + const: "remove-partition-specs" spec-ids: type: array items: type: integer + # Disabling Row Lineage is Forbidden + EnableRowLineageUpdate: + allOf: + - $ref: '#/components/schemas/BaseUpdate' + properties: + action: + type: string + const: "enable-row-lineage" + TableUpdate: anyOf: - $ref: '#/components/schemas/AssignUUIDUpdate' @@ -2984,6 +3003,7 @@ components: - $ref: '#/components/schemas/SetStatisticsUpdate' - $ref: '#/components/schemas/RemoveStatisticsUpdate' - $ref: '#/components/schemas/RemovePartitionSpecsUpdate' + - $ref: '#/components/schemas/EnableRowLineageUpdate' ViewUpdate: anyOf: @@ -3009,35 +3029,35 @@ components: assert-last-assigned-partition-id: '#/components/schemas/AssertLastAssignedPartitionId' assert-default-spec-id: '#/components/schemas/AssertDefaultSpecId' assert-default-sort-order-id: '#/components/schemas/AssertDefaultSortOrderId' - oneOf: - - $ref: '#/components/schemas/AssertCreate' - - $ref: '#/components/schemas/AssertTableUUID' - - $ref: '#/components/schemas/AssertRefSnapshotId' - - $ref: '#/components/schemas/AssertLastAssignedFieldId' - - $ref: '#/components/schemas/AssertCurrentSchemaId' - - $ref: '#/components/schemas/AssertLastAssignedPartitionId' - - $ref: '#/components/schemas/AssertDefaultSpecId' - - $ref: '#/components/schemas/AssertDefaultSortOrderId' + properties: + type: + type: string + required: + - type AssertCreate: type: object description: The table must not already exist; used for create transactions + allOf: + - $ref: '#/components/schemas/TableRequirement' required: - type properties: type: type: string - enum: ["assert-create"] + const: "assert-create" AssertTableUUID: description: The table UUID must match the requirement's `uuid` + allOf: + - $ref: '#/components/schemas/TableRequirement' required: - type - uuid properties: type: type: string - enum: ["assert-table-uuid"] + const: "assert-table-uuid" uuid: type: string @@ -3045,14 +3065,15 @@ components: description: The table branch or tag identified by the requirement's `ref` must reference the requirement's `snapshot-id`; if `snapshot-id` is `null` or missing, the ref must not already exist + allOf: + - $ref: '#/components/schemas/TableRequirement' required: - - type - ref - snapshot-id properties: type: type: string - enum: [ "assert-ref-snapshot-id" ] + const: "assert-ref-snapshot-id" ref: type: string snapshot-id: @@ -3062,65 +3083,70 @@ components: AssertLastAssignedFieldId: description: The table's last assigned column id must match the requirement's `last-assigned-field-id` + allOf: + - $ref: '#/components/schemas/TableRequirement' required: - - type - last-assigned-field-id properties: type: type: string - enum: [ "assert-last-assigned-field-id" ] + const: "assert-last-assigned-field-id" last-assigned-field-id: type: integer AssertCurrentSchemaId: description: The table's current schema id must match the requirement's `current-schema-id` + allOf: + - $ref: '#/components/schemas/TableRequirement' required: - - type - current-schema-id properties: type: type: string - enum: [ "assert-current-schema-id" ] + const: "assert-current-schema-id" current-schema-id: type: integer AssertLastAssignedPartitionId: description: The table's last assigned partition id must match the requirement's `last-assigned-partition-id` + allOf: + - $ref: '#/components/schemas/TableRequirement' required: - - type - last-assigned-partition-id properties: type: type: string - enum: [ "assert-last-assigned-partition-id" ] + const: "assert-last-assigned-partition-id" last-assigned-partition-id: type: integer AssertDefaultSpecId: description: The table's default spec id must match the requirement's `default-spec-id` + allOf: + - $ref: '#/components/schemas/TableRequirement' required: - - type - default-spec-id properties: type: type: string - enum: [ "assert-default-spec-id" ] + const: "assert-default-spec-id" default-spec-id: type: integer AssertDefaultSortOrderId: description: The table's default sort order id must match the requirement's `default-sort-order-id` + allOf: + - $ref: '#/components/schemas/TableRequirement' required: - - type - default-sort-order-id properties: type: type: string - enum: [ "assert-default-sort-order-id" ] + const: "assert-default-sort-order-id" default-sort-order-id: type: integer @@ -3141,7 +3167,7 @@ components: properties: type: type: string - enum: [ "assert-view-uuid" ] + const: "assert-view-uuid" uuid: type: string @@ -3196,6 +3222,7 @@ components: - `s3.secret-access-key`: secret for credentials that provide access to data in S3 - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification + - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled ## Storage Credentials @@ -3264,7 +3291,7 @@ components: properties: status: $ref: '#/components/schemas/PlanStatus' - enum: ["completed"] + const: "completed" CompletedPlanningWithIDResult: type: object @@ -3287,7 +3314,7 @@ components: properties: status: $ref: '#/components/schemas/PlanStatus' - enum: ["failed"] + const: "failed" AsyncPlanningResult: type: object @@ -3296,7 +3323,7 @@ components: properties: status: $ref: '#/components/schemas/PlanStatus' - enum: ["submitted"] + const: "submitted" plan-id: description: ID used to track a planning request type: string @@ -4136,6 +4163,7 @@ components: - avro - orc - parquet + - puffin ContentFile: discriminator: @@ -4200,7 +4228,7 @@ components: properties: content: type: string - enum: [ "data" ] + const: "data" column-sizes: allOf: - $ref: '#/components/schemas/CountMap' @@ -4244,7 +4272,15 @@ components: properties: content: type: string - enum: [ "position-deletes" ] + const: "position-deletes" + content-offset: + type: integer + format: int64 + description: Offset within the delete file of delete content + content-size-in-bytes: + type: integer + format: int64 + description: Length, in bytes, of the delete content; required if content-offset is present EqualityDeleteFile: allOf: @@ -4254,7 +4290,7 @@ components: properties: content: type: string - enum: [ "equality-deletes" ] + const: "equality-deletes" equality-ids: type: array items: @@ -4576,6 +4612,9 @@ components: application/json: schema: $ref: '#/components/schemas/LoadTableResult' + headers: + etag: + $ref: '#/components/parameters/etag' PlanTableScanResponse: description: Result of submitting a table scan to plan @@ -4604,6 +4643,9 @@ components: application/json: schema: $ref: '#/components/schemas/LoadTableResult' + headers: + etag: + $ref: '#/components/parameters/etag' LoadViewResponse: description: View metadata result when loading a view @@ -4807,4 +4849,4 @@ components: catalog: Allows interacting with the Config and Catalog APIs BearerAuth: type: http - scheme: bearer + scheme: bearer \ No newline at end of file From c1b6bc4c6f44584d0d8fb592f58fd5de2e08cc87 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 31 Mar 2025 10:50:46 -0700 Subject: [PATCH 03/18] more changes --- api/polaris-catalog-service/build.gradle.kts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index 64105739a7..e9cb85f6eb 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -81,11 +81,12 @@ openApiGenerate { ignoreFileOverride = "$rootDir/.openapi-generator-ignore" removeOperationIdPrefix = true templateDir = "$rootDir/server-templates" - globalProperties.put("apis", "GenericTableApi,PolicyApi") - // TODO -- reset this value to `models` after the Iceberg 1.8 change has merged + // TODO -- revert the changes to these values after the Iceberg 1.8 change has merged + globalProperties.put("apis", "true") globalProperties.put("models", "true") - globalProperties.put("apiDocs", "false") - globalProperties.put("modelTests", "false") + globalProperties.put("apiDocs", "true") + globalProperties.put("modelTests", "true") + // TODO -- end of changes configOptions.put("resourceName", "catalog") configOptions.put("useTags", "true") configOptions.put("useBeanValidation", "false") From 6ba05130bffd32faadfbaa7d9e003b0d22efd9eb Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Mon, 31 Mar 2025 10:52:29 -0700 Subject: [PATCH 04/18] fixes --- spec/iceberg-rest-catalog-open-api.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spec/iceberg-rest-catalog-open-api.yaml b/spec/iceberg-rest-catalog-open-api.yaml index 5fb9f88045..728d0cb531 100644 --- a/spec/iceberg-rest-catalog-open-api.yaml +++ b/spec/iceberg-rest-catalog-open-api.yaml @@ -17,6 +17,10 @@ # under the License. # +# CODE_COPIED_TO_POLARIS + +# Apache Iceberg Version: 1.8.1 + --- openapi: 3.1.1 info: @@ -4849,4 +4853,4 @@ components: catalog: Allows interacting with the Config and Catalog APIs BearerAuth: type: http - scheme: bearer \ No newline at end of file + scheme: bearer From 42b42751bfe7eb29183f418a62db0382c27dedd5 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 1 Apr 2025 10:24:29 -0700 Subject: [PATCH 05/18] github action change --- .github/workflows/gradle.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index d18ab65108..4c3c001087 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -57,10 +57,10 @@ jobs: validate-wrappers: false - name: Code style checks and tests - run: ./gradlew --continue check + run: ./gradlew --continue check --no-build-cache - name: Check Maven publication - run: ./gradlew publishToMavenLocal sourceTarball + run: ./gradlew publishToMavenLocal sourceTarball --no-build-cache - name: Archive test results uses: actions/upload-artifact@v4 @@ -84,9 +84,9 @@ jobs: - name: Clean run: ./gradlew clean - name: Build - run: ./gradlew compileAll + run: ./gradlew compileAll --no-build-cache - name: Run selected tests - run: ./gradlew :polaris-quarkus-service:intTest + run: ./gradlew :polaris-quarkus-service:intTest --no-build-cache - name: Archive test results uses: actions/upload-artifact@v4 From 4ba83981021d5f3f4206325b8184bd0c1a4a978c Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 1 Apr 2025 10:32:00 -0700 Subject: [PATCH 06/18] another build change --- .github/workflows/regtest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regtest.yml b/.github/workflows/regtest.yml index c75da2d9b0..ce9a97a949 100644 --- a/.github/workflows/regtest.yml +++ b/.github/workflows/regtest.yml @@ -45,11 +45,11 @@ jobs: - name: Image build run: | - ./gradlew :polaris-quarkus-server:assemble -Dquarkus.container-image.build=true + ./gradlew :polaris-quarkus-server:assemble -Dquarkus.container-image.build=true --no-build-cache - name: Regression Test env: AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} run: | - docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest \ No newline at end of file + docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest From e25a68305c4077af2ca841d1f999cb7225f772c5 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 1 Apr 2025 10:33:03 -0700 Subject: [PATCH 07/18] try api revert --- api/polaris-catalog-service/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index e9cb85f6eb..3b608c191f 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -82,7 +82,7 @@ openApiGenerate { removeOperationIdPrefix = true templateDir = "$rootDir/server-templates" // TODO -- revert the changes to these values after the Iceberg 1.8 change has merged - globalProperties.put("apis", "true") + globalProperties.put("apis", "GenericTableApi,PolicyApi") globalProperties.put("models", "true") globalProperties.put("apiDocs", "true") globalProperties.put("modelTests", "true") From 0abbd3f5dcabbe42182d91c22f2fddfc23c0cddf Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Tue, 1 Apr 2025 10:48:35 -0700 Subject: [PATCH 08/18] re-all --- api/polaris-catalog-service/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index 3b608c191f..e9cb85f6eb 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -82,7 +82,7 @@ openApiGenerate { removeOperationIdPrefix = true templateDir = "$rootDir/server-templates" // TODO -- revert the changes to these values after the Iceberg 1.8 change has merged - globalProperties.put("apis", "GenericTableApi,PolicyApi") + globalProperties.put("apis", "true") globalProperties.put("models", "true") globalProperties.put("apiDocs", "true") globalProperties.put("modelTests", "true") From 5b8f1461d433cacacee0f7ce866250ff34a098fe Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 18 Apr 2025 12:40:16 -0700 Subject: [PATCH 09/18] custom type mappings, rebuild --- api/iceberg-service/build.gradle.kts | 6 ++++++ api/polaris-catalog-service/build.gradle.kts | 6 ++++++ .../service/catalog/iceberg/IcebergCatalogAdapter.java | 5 ++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/api/iceberg-service/build.gradle.kts b/api/iceberg-service/build.gradle.kts index 596864e7f7..09679e48a1 100644 --- a/api/iceberg-service/build.gradle.kts +++ b/api/iceberg-service/build.gradle.kts @@ -104,6 +104,12 @@ openApiGenerate { "NotificationRequest" to "org.apache.polaris.service.types.NotificationRequest", "TableUpdateNotification" to "org.apache.polaris.service.types.TableUpdateNotification", "NotificationType" to "org.apache.polaris.service.types.NotificationType", + + // GenericTable types + "CreateGenericTableRequest" to "org.apache.polaris.service.types.CreateGenericTableRequest", + "GenericTable" to "org.apache.polaris.service.types.GenericTable", + "ListGenericTablesResponse" to "org.apache.polaris.service.types.ListGenericTablesResponse", + "LoadGenericTableResponse" to "org.apache.polaris.service.types.LoadGenericTableResponse", ) } diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index 73e43b10a8..e5dae5d5d2 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -87,6 +87,12 @@ openApiGenerate { globalProperties.put("models", "true") globalProperties.put("apiDocs", "true") globalProperties.put("modelTests", "true") + + // Old values: +// globalProperties.put("apis", "GenericTableApi,PolicyApi") +// globalProperties.put("models", models) +// globalProperties.put("apiDocs", "false") +// globalProperties.put("modelTests", "false") // TODO -- end of changes configOptions.put("resourceName", "catalog") configOptions.put("useTags", "true") diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index 6895351395..05b2497850 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -369,6 +369,7 @@ public Response loadTable( String namespace, String table, String accessDelegationMode, + String ifNoneMatchString, String snapshots, RealmContext realmContext, SecurityContext securityContext) { @@ -377,9 +378,7 @@ public Response loadTable( Namespace ns = decodeNamespace(namespace); TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table)); - // TODO: Populate with header value from parameter once the generated interface - // contains the if-none-match header - IfNoneMatch ifNoneMatch = IfNoneMatch.fromHeader(null); + IfNoneMatch ifNoneMatch = IfNoneMatch.fromHeader(ifNoneMatchString); if (ifNoneMatch.isWildcard()) { throw new BadRequestException("If-None-Match may not take the value of '*'"); From 993178f79864c52e4bab5e3b1979a344cc37cd79 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 18 Apr 2025 12:40:18 -0700 Subject: [PATCH 10/18] autolint --- api/polaris-catalog-service/build.gradle.kts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index e5dae5d5d2..3f3c2f9275 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -89,10 +89,10 @@ openApiGenerate { globalProperties.put("modelTests", "true") // Old values: -// globalProperties.put("apis", "GenericTableApi,PolicyApi") -// globalProperties.put("models", models) -// globalProperties.put("apiDocs", "false") -// globalProperties.put("modelTests", "false") + // globalProperties.put("apis", "GenericTableApi,PolicyApi") + // globalProperties.put("models", models) + // globalProperties.put("apiDocs", "false") + // globalProperties.put("modelTests", "false") // TODO -- end of changes configOptions.put("resourceName", "catalog") configOptions.put("useTags", "true") From dc39aa4e557c01f4ef9ec1d84baf06be2b9bd07c Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 18 Apr 2025 13:10:52 -0700 Subject: [PATCH 11/18] polish --- api/polaris-catalog-service/build.gradle.kts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index 3f3c2f9275..e4f92fad7e 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -82,18 +82,10 @@ openApiGenerate { ignoreFileOverride = "$rootDir/.openapi-generator-ignore" removeOperationIdPrefix = true templateDir = "$rootDir/server-templates" - // TODO -- revert the changes to these values after the Iceberg 1.8 change has merged - globalProperties.put("apis", "true") + globalProperties.put("apis", "CatalogApi,ConfigurationApi,OAuth2Api,PolicyApi,GenericTableApi") globalProperties.put("models", "true") globalProperties.put("apiDocs", "true") globalProperties.put("modelTests", "true") - - // Old values: - // globalProperties.put("apis", "GenericTableApi,PolicyApi") - // globalProperties.put("models", models) - // globalProperties.put("apiDocs", "false") - // globalProperties.put("modelTests", "false") - // TODO -- end of changes configOptions.put("resourceName", "catalog") configOptions.put("useTags", "true") configOptions.put("useBeanValidation", "false") From baa24b001c3b4dd6399a62a99c3db0df11e4c3e2 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 18 Apr 2025 13:11:37 -0700 Subject: [PATCH 12/18] yank custom types --- api/iceberg-service/build.gradle.kts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/api/iceberg-service/build.gradle.kts b/api/iceberg-service/build.gradle.kts index 09679e48a1..596864e7f7 100644 --- a/api/iceberg-service/build.gradle.kts +++ b/api/iceberg-service/build.gradle.kts @@ -104,12 +104,6 @@ openApiGenerate { "NotificationRequest" to "org.apache.polaris.service.types.NotificationRequest", "TableUpdateNotification" to "org.apache.polaris.service.types.TableUpdateNotification", "NotificationType" to "org.apache.polaris.service.types.NotificationType", - - // GenericTable types - "CreateGenericTableRequest" to "org.apache.polaris.service.types.CreateGenericTableRequest", - "GenericTable" to "org.apache.polaris.service.types.GenericTable", - "ListGenericTablesResponse" to "org.apache.polaris.service.types.ListGenericTablesResponse", - "LoadGenericTableResponse" to "org.apache.polaris.service.types.LoadGenericTableResponse", ) } From 389d5934004e042bfa99137980ead1a2dc0713a1 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 18 Apr 2025 13:18:06 -0700 Subject: [PATCH 13/18] update --- api/polaris-catalog-service/build.gradle.kts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index e4f92fad7e..6cabdc9bcc 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -1,3 +1,5 @@ +import org.openapitools.generator.gradle.plugin.tasks.GenerateTask + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -73,7 +75,10 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-databind") } -openApiGenerate { +tasks.named("openApiGenerate") { + doFirst { + delete(fileTree("$projectDir/build/generated")) + } inputSpec = "$rootDir/spec/polaris-catalog-service.yaml" generatorName = "jaxrs-resteasy" outputDir = "$projectDir/build/generated" @@ -82,10 +87,11 @@ openApiGenerate { ignoreFileOverride = "$rootDir/.openapi-generator-ignore" removeOperationIdPrefix = true templateDir = "$rootDir/server-templates" - globalProperties.put("apis", "CatalogApi,ConfigurationApi,OAuth2Api,PolicyApi,GenericTableApi") + globalProperties.put("apis", "true") globalProperties.put("models", "true") globalProperties.put("apiDocs", "true") globalProperties.put("modelTests", "true") + globalProperties.put("supportingFiles", "true") configOptions.put("resourceName", "catalog") configOptions.put("useTags", "true") configOptions.put("useBeanValidation", "false") From c088768c1fb37f775062ed479adac3907aab7eb6 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 18 Apr 2025 13:18:08 -0700 Subject: [PATCH 14/18] autolint --- api/polaris-catalog-service/build.gradle.kts | 23 ++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index 6cabdc9bcc..c1bb82e16a 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -1,4 +1,21 @@ -import org.openapitools.generator.gradle.plugin.tasks.GenerateTask +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ /* * Licensed to the Apache Software Foundation (ASF) under one @@ -76,9 +93,7 @@ dependencies { } tasks.named("openApiGenerate") { - doFirst { - delete(fileTree("$projectDir/build/generated")) - } + doFirst { delete(fileTree("$projectDir/build/generated")) } inputSpec = "$rootDir/spec/polaris-catalog-service.yaml" generatorName = "jaxrs-resteasy" outputDir = "$projectDir/build/generated" From cbbd49a71576dad88a023706915c5362b60a2d36 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 18 Apr 2025 13:22:08 -0700 Subject: [PATCH 15/18] wip --- api/polaris-catalog-service/build.gradle.kts | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index c1bb82e16a..1ec62ba3ae 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -17,24 +17,7 @@ * under the License. */ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ +import org.openapitools.generator.gradle.plugin.tasks.GenerateTask plugins { alias(libs.plugins.openapi.generator) From cff50ab7553835ee2c8e759cfb21f254f1a56717 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Fri, 18 Apr 2025 14:01:14 -0700 Subject: [PATCH 16/18] Revert build changes --- api/polaris-catalog-service/build.gradle.kts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/api/polaris-catalog-service/build.gradle.kts b/api/polaris-catalog-service/build.gradle.kts index 1ec62ba3ae..6cd15f88a7 100644 --- a/api/polaris-catalog-service/build.gradle.kts +++ b/api/polaris-catalog-service/build.gradle.kts @@ -17,8 +17,6 @@ * under the License. */ -import org.openapitools.generator.gradle.plugin.tasks.GenerateTask - plugins { alias(libs.plugins.openapi.generator) id("polaris-client") @@ -75,8 +73,7 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-databind") } -tasks.named("openApiGenerate") { - doFirst { delete(fileTree("$projectDir/build/generated")) } +openApiGenerate { inputSpec = "$rootDir/spec/polaris-catalog-service.yaml" generatorName = "jaxrs-resteasy" outputDir = "$projectDir/build/generated" @@ -85,11 +82,10 @@ tasks.named("openApiGenerate") { ignoreFileOverride = "$rootDir/.openapi-generator-ignore" removeOperationIdPrefix = true templateDir = "$rootDir/server-templates" - globalProperties.put("apis", "true") - globalProperties.put("models", "true") - globalProperties.put("apiDocs", "true") - globalProperties.put("modelTests", "true") - globalProperties.put("supportingFiles", "true") + globalProperties.put("apis", "GenericTableApi,PolicyApi") + globalProperties.put("models", models) + globalProperties.put("apiDocs", "false") + globalProperties.put("modelTests", "false") configOptions.put("resourceName", "catalog") configOptions.put("useTags", "true") configOptions.put("useBeanValidation", "false") From 5c26ec6e244d44ea56b2992b56e6158fbdd3386d Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 23 Apr 2025 11:09:01 -0700 Subject: [PATCH 17/18] example --- .github/workflows/gradle.yml | 8 ++++---- .github/workflows/regtest.yml | 5 ++--- api/iceberg-service/build.gradle.kts | 4 ++++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 4c3c001087..d18ab65108 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -57,10 +57,10 @@ jobs: validate-wrappers: false - name: Code style checks and tests - run: ./gradlew --continue check --no-build-cache + run: ./gradlew --continue check - name: Check Maven publication - run: ./gradlew publishToMavenLocal sourceTarball --no-build-cache + run: ./gradlew publishToMavenLocal sourceTarball - name: Archive test results uses: actions/upload-artifact@v4 @@ -84,9 +84,9 @@ jobs: - name: Clean run: ./gradlew clean - name: Build - run: ./gradlew compileAll --no-build-cache + run: ./gradlew compileAll - name: Run selected tests - run: ./gradlew :polaris-quarkus-service:intTest --no-build-cache + run: ./gradlew :polaris-quarkus-service:intTest - name: Archive test results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/regtest.yml b/.github/workflows/regtest.yml index ef9e7abda2..c814a93665 100644 --- a/.github/workflows/regtest.yml +++ b/.github/workflows/regtest.yml @@ -48,12 +48,11 @@ jobs: ./gradlew \ :polaris-quarkus-server:assemble \ :polaris-quarkus-server:quarkusAppPartsBuild --rerun \ - -Dquarkus.container-image.build=true \ - --no-build-cache + -Dquarkus.container-image.build=true - name: Regression Test env: AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} run: | - docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest + docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest \ No newline at end of file diff --git a/api/iceberg-service/build.gradle.kts b/api/iceberg-service/build.gradle.kts index 596864e7f7..3adf8da8a2 100644 --- a/api/iceberg-service/build.gradle.kts +++ b/api/iceberg-service/build.gradle.kts @@ -118,3 +118,7 @@ sourceSets { tasks.named("javadoc") { dependsOn("jandex") } tasks.named("processResources") { dependsOn("openApiGenerate") } + +tasks.named("openApiGenerate") { + outputs.cacheIf { false } +} From 5f9b559cc7359e6e95a35fdd445e60efceb49b30 Mon Sep 17 00:00:00 2001 From: Eric Maynard Date: Wed, 23 Apr 2025 11:09:08 -0700 Subject: [PATCH 18/18] autolint --- api/iceberg-service/build.gradle.kts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/iceberg-service/build.gradle.kts b/api/iceberg-service/build.gradle.kts index 3adf8da8a2..59c7bd2583 100644 --- a/api/iceberg-service/build.gradle.kts +++ b/api/iceberg-service/build.gradle.kts @@ -119,6 +119,4 @@ tasks.named("javadoc") { dependsOn("jandex") } tasks.named("processResources") { dependsOn("openApiGenerate") } -tasks.named("openApiGenerate") { - outputs.cacheIf { false } -} +tasks.named("openApiGenerate") { outputs.cacheIf { false } }