Merge branch 'main' into issue-1273-fix-custom-metadata-location

Author: liamzwbao

.asf.yaml

@@ -56,6 +56,7 @@ github:
           - markdown-link-check
           - build
           - regtest
+          - spark-plugin-regtest
           - site
           - "Helm tests"
 
@@ -71,7 +72,6 @@ github:
   collaborators:
     # Adding renovate-bot as a collaborator, so CI doesn't need to be manually approved
     - renovate-bot
-    - singhpk234
     - gh-yzou
     - nssalian
     - adnanhemani

.github/CODEOWNERS

@@ -17,4 +17,4 @@
 # under the License.
 #
 
-* @adutra @annafil @anoopj @ashvina @dennishuo @dimas-b @eric-maynard @jackye1995 @jbonofre @vvcephei @collado-mike @snazy @RussellSpitzer @takidau @MonkeyCanCode @flyrain @ebyhr
+* @adutra @annafil @anoopj @ashvina @dennishuo @dimas-b @eric-maynard @jackye1995 @jbonofre @vvcephei @collado-mike @snazy @RussellSpitzer @takidau @MonkeyCanCode @flyrain @ebyhr @ajantha-bhat @HonahX @singhpk234 @pingtimeout

.github/workflows/helm.yml

@@ -57,16 +57,7 @@ jobs:
       - name: Set up chart-testing
         uses: helm/chart-testing-action@v2.6.1
 
-      - name: Run chart-testing (list-changed)
-        id: list-changed
-        run: |
-          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }} --chart-dirs helm)
-          if [[ -n "$changed" ]]; then
-            echo "changed=true" >> "$GITHUB_OUTPUT"
-          fi
-
       - name: Run 'helm template' validation
-        if: steps.list-changed.outputs.changed == 'true'
         run: |
           cd helm/polaris
           for f in values.yaml ci/*.yaml; do
@@ -76,27 +67,22 @@ jobs:
           done
 
       - name: Run Helm unit tests
-        if: steps.list-changed.outputs.changed == 'true'
         run: |
           helm plugin install https://github.com/helm-unittest/helm-unittest.git || true
           helm unittest helm/polaris
 
       - name: Run chart-testing (lint)
-        if: steps.list-changed.outputs.changed == 'true'
         run: ct lint --target-branch ${{ github.event.repository.default_branch }} --debug --charts ./helm/polaris
 
       - name: Set up Minikube
-        if: steps.list-changed.outputs.changed == 'true'
         uses: medyagh/setup-minikube@v0.0.18
 
       - name: Print Docker info
-        if: steps.list-changed.outputs.changed == 'true'
         run: |
           docker -v
           minikube docker-env
 
       - name: Image build
-        if: steps.list-changed.outputs.changed == 'true'
         run: |
           eval $(minikube -p minikube docker-env)
           ./gradlew \
@@ -108,13 +94,11 @@ jobs:
           minikube image ls
 
       - name: Install fixtures
-        if: steps.list-changed.outputs.changed == 'true'
         run: |
           kubectl create namespace polaris-ns
           kubectl apply --namespace polaris-ns -f helm/polaris/ci/fixtures 
 
       - name: Run chart-testing (install)
-        if: steps.list-changed.outputs.changed == 'true'
         run: |
           ct install --target-branch ${{ github.event.repository.default_branch }} \
             --namespace polaris-ns \

.github/workflows/nightly.yml

@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Nightly Build
+
+on:
+  schedule:
+    - cron: '15 12 * * *'
+  workflow_dispatch:
+
+# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
+permissions:
+  actions: read
+  pull-requests: read
+  checks: read
+  contents: read
+  deployments: read
+  id-token: none
+  issues: read
+  discussions: read
+  packages: read
+  pages: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+
+jobs:
+  nightly_build:
+    runs-on: ubuntu-latest
+    if: github.repository == 'apache/polaris'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+        with:
+          validate-wrappers: false
+      - name: Publish SNAPSHOTs to Apache Nexus Repository
+        run: ./gradlew publishToApache
+        env:
+          ORG_GRADLE_PROJECT_apacheUsername: ${{ secrets.NEXUS_USER }}
+          ORG_GRADLE_PROJECT_apachePassword: ${{ secrets.NEXUS_PW }} 

.github/workflows/python-client.yml

@@ -0,0 +1,70 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will build a Python project with Poetry and cache/restore any dependencies to improve the workflow execution time
+# For more information see: https://docs.github.com/en/actions/use-cases-and-examples/building-and-testing/building-and-testing-python
+
+name: Python Client CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
+
+    steps:
+      - name: Checkout Polaris project
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        run: |
+          pip install --user --upgrade -r regtests/requirements.txt
+
+      # TODO: add cache for poetry dependencies once we have poetry.lock in the repo
+      - name: Install dependencies
+        working-directory: client/python
+        run: poetry install --all-extras
+
+      - name: Lint with flake8
+        working-directory: client/python
+        run: |
+          poetry run flake8 --select=E9,F63,F7,F82 --max-line-length=127 --statistics
+
+      - name: Test with pytest
+        working-directory: client/python
+        run: |
+          export SCRIPT_DIR="non-existing-mock-directory"
+          poetry run pytest

.github/workflows/spark_client_regtests.yml

@@ -25,7 +25,7 @@ on:
     branches: [ "main" ]
 
 jobs:
-  regtest:
+  spark-plugin-regtest:
 
     runs-on: ubuntu-latest
     permissions:

.github/workflows/stale.yml

@@ -22,7 +22,7 @@ jobs:
   stale:
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/stale@816d9db1aba399a7f70277f1a2b01a4d21497fdd
+      - uses: actions/stale@f78de9780efb7a789cf4745957fa3374cbb94fd5
         with:
           days-before-close: 5
           days-before-stale: 30

LICENSE

@@ -219,6 +219,11 @@ This product includes code from Apache Iceberg.
 * spec/polaris-catalog-apis/oauth-tokens-api.yaml
 * integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
 * service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java
+* service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/CatalogHandlerUtils.java
+* plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/PolarisSparkCatalog.java
+* plugins/spark/v3.5/spark/src/test/java/org/apache/polaris/spark/PolarisInMemoryCatalog.java
+* plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/PolarisRESTCatalog.java
+* plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/SparkCatalog.java
 
 Copyright: Copyright 2017-2025 The Apache Software Foundation
 Home page: https://iceberg.apache.org
@@ -261,11 +266,12 @@ License: https://www.apache.org/licenses/LICENSE-2.0
 This product includes code from Project Nessie.
 
 * build-logic/src/main/kotlin/LicenseFileValidation.kt
-* build-logic/src/main/kotlin/publishing/MemoizedGitInfo.kt
-* build-logic/src/main/kotlin/publishing/PublishingHelperExtension.kt
+* build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerPlugin.kt
+* build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerExtension.kt
 * build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt
-* build-logic/src/main/kotlin/publishing/shadowPub.kt
-* build-logic/src/main/kotlin/publishing/util.kt
+* build-logic/src/main/kotlin/Utilities.kt
+* build-logic/src/main/kotlin/polaris-shadow-jar.gradle.kts
+* build-logic/src/main/kotlin/polaris-quarkus.gradle.kts
 * tools/config-docs/annotations/src/main/java/org/apache/polaris/docs/ConfigDocs.java
 * tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/DocGenDoclet.java
 * tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/MarkdownFormatter.java
@@ -277,8 +283,8 @@ This product includes code from Project Nessie.
 * tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/PropertyInfo.java
 * tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/SmallRyeConfigMappingInfo.java
 * tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/SmallRyeConfigPropertyInfo.java
-* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/SmallRyeConfigSectionPage.java
 * tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/SmallRyeConfigs.java
+* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/ReferenceConfigDocsGenerator.java
 * tools/config-docs/generator/src/test/java/org/apache/polaris/docs/generator/TestDocGenTool.java
 * tools/config-docs/generator/src/test/java/tests/properties/ConfigProps.java
 * tools/config-docs/generator/src/test/java/tests/properties/MoreProps.java
@@ -304,6 +310,15 @@ This product includes code from Project Nessie.
 * tools/config-docs/generator/src/test/java/tests/smallrye/OtherMapped.java
 * tools/config-docs/generator/src/test/java/tests/smallrye/SomeEnum.java
 * tools/config-docs/generator/src/test/java/tests/smallrye/VeryNested.java
+* tools/container-spec-helper/src/main/java/org/apache/polaris/containerspec/ContainerSpecHelper.java
+* quarkus/admin/src/main/java/org/apache/polaris/admintool/PolarisAdminTool.java
+* helm/polaris/tests/logging_storage_test.yaml
+* helm/polaris/tests/quantity_test.yaml
+* helm/polaris/tests/service_monitor_test.yaml
+* helm/polaris/templates/_helpers.tpl
+* helm/polaris/templates/serviceaccount.yaml
+* helm/polaris/templates/servicemonitor.yaml
+* helm/polaris/templates/storage.yaml
 
 Copyright: Copyright 2015-2025 Dremio Corporation
 Home page: https://projectnessie.org/

NOTICE

@@ -7,3 +7,10 @@ The Apache Software Foundation (http://www.apache.org/).
 The initial code for the Polaris project was donated
 to the ASF by Snowflake Inc. (https://www.snowflake.com/) copyright 2024.
 
+--------------------------------------------------------------------------------
+
+This project includes code from Project Nessie, developed at Dremio,
+with the following copyright notice:
+
+| Nessie
+| Copyright 2015-2025 Dremio Corporation

api/iceberg-service/build.gradle.kts

@@ -66,7 +66,6 @@ openApiGenerate {
   configOptions.put("useBeanValidation", "false")
   configOptions.put("sourceFolder", "src/main/java")
   configOptions.put("useJakartaEe", "true")
-  openapiNormalizer.put("REFACTOR_ALLOF_WITH_PROPERTIES_ONLY", "true")
   additionalProperties.put("apiNamePrefix", "IcebergRest")
   additionalProperties.put("apiNameSuffix", "")
   additionalProperties.put("metricsPrefix", "polaris")
@@ -120,3 +119,5 @@ sourceSets {
 tasks.named("javadoc") { dependsOn("jandex") }
 
 tasks.named("processResources") { dependsOn("openApiGenerate") }
+
+tasks.named("openApiGenerate") { outputs.cacheIf { false } }