Create a wrapper script to generate python client; regenerate the python client (#1347)

As noted in #755 and elsewhere, the generated types in client/python are currently out of date. This introduces a script to regenerate them and a gradle task to run that script.

I've also run the script, which necessitated several things to get tests passing:

1. There were small nonfunctional spec changes needed in order to keep the Python client working
2. The CLI and its tests required a few fixes to work with the updated Python client
3. Many of the regtests required fixes to work with the updated Python client

Author: eric-maynard

build.gradle.kts

@@ -141,6 +141,18 @@ tasks.named<RatTask>("rat").configure {
   excludes.add("plugins/**/*.ref")
 }
 
+tasks.register<Exec>("regeneratePythonClient") {
+  description = "Regenerates the python client"
+
+  workingDir = project.projectDir
+  commandLine("bash", "client/templates/regenerate.sh")
+
+  dependsOn(":polaris-api-iceberg-service:processResources")
+  dependsOn(":polaris-api-management-service:processResources")
+  dependsOn(":polaris-api-catalog-service:processResources")
+  dependsOn(":polaris-api-management-model:processResources")
+}
+
 // Pass environment variables:
 //    ORG_GRADLE_PROJECT_apacheUsername
 //    ORG_GRADLE_PROJECT_apachePassword

client/python/.github/workflows/python.yml

@@ -22,7 +22,7 @@
 #
 # ref: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
 
-name: polaris.management Python package
+name: polaris.catalog Python package
 
 on: [push, pull_request]
 
@@ -35,9 +35,9 @@ jobs:
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies

client/python/.gitlab-ci.yml

@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# NOTE: This file is auto generated by OpenAPI Generator.
+# URL: https://openapi-generator.tech
+#
+# ref: https://docs.gitlab.com/ee/ci/README.html
+# ref: https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Python.gitlab-ci.yml
+
+stages:
+  - test
+
+.pytest:
+  stage: test
+  script:
+   - pip install -r requirements.txt
+   - pip install -r test-requirements.txt
+   - pytest --cov=polaris.catalog
+
+pytest-3.7:
+  extends: .pytest
+  image: python:3.7-alpine
+pytest-3.8:
+  extends: .pytest
+  image: python:3.8-alpine
+pytest-3.9:
+  extends: .pytest
+  image: python:3.9-alpine
+pytest-3.10:
+  extends: .pytest
+  image: python:3.10-alpine
+pytest-3.11:
+  extends: .pytest
+  image: python:3.11-alpine

client/python/.openapi-generator/FILES

@@ -19,6 +19,7 @@ docs/AssertRefSnapshotId.md
 docs/AssertTableUUID.md
 docs/AssertViewUUID.md
 docs/AssignUUIDUpdate.md
+docs/AsyncPlanningResult.md
 docs/BaseUpdate.md
 docs/BlobMetadata.md
 docs/CatalogConfig.md
@@ -27,6 +28,8 @@ docs/CommitTableRequest.md
 docs/CommitTableResponse.md
 docs/CommitTransactionRequest.md
 docs/CommitViewRequest.md
+docs/CompletedPlanningResult.md
+docs/CompletedPlanningWithIDResult.md
 docs/ContentFile.md
 docs/CountMap.md
 docs/CounterResult.md
@@ -35,10 +38,19 @@ docs/CreateNamespaceResponse.md
 docs/CreateTableRequest.md
 docs/CreateViewRequest.md
 docs/DataFile.md
+docs/DeleteFile.md
+docs/EmptyPlanningResult.md
+docs/EnableRowLineageUpdate.md
 docs/EqualityDeleteFile.md
 docs/ErrorModel.md
 docs/Expression.md
+docs/FailedPlanningResult.md
+docs/FalseExpression.md
+docs/FetchPlanningResult.md
+docs/FetchScanTasksRequest.md
+docs/FetchScanTasksResult.md
 docs/FileFormat.md
+docs/FileScanTask.md
 docs/GetNamespaceResponse.md
 docs/IcebergCatalogAPI.md
 docs/IcebergConfigurationAPI.md
@@ -48,24 +60,27 @@ docs/ListNamespacesResponse.md
 docs/ListTablesResponse.md
 docs/ListType.md
 docs/LiteralExpression.md
+docs/LoadCredentialsResponse.md
 docs/LoadTableResult.md
 docs/LoadViewResult.md
 docs/MapType.md
 docs/MetadataLogInner.md
 docs/MetricResult.md
 docs/ModelSchema.md
 docs/NotExpression.md
-docs/NotificationRequest.md
-docs/NotificationType.md
 docs/NullOrder.md
 docs/OAuthError.md
 docs/OAuthTokenResponse.md
 docs/PartitionField.md
 docs/PartitionSpec.md
 docs/PartitionStatisticsFile.md
+docs/PlanStatus.md
+docs/PlanTableScanRequest.md
+docs/PlanTableScanResult.md
 docs/PositionDeleteFile.md
 docs/PrimitiveTypeValue.md
 docs/RegisterTableRequest.md
+docs/RemovePartitionSpecsUpdate.md
 docs/RemovePartitionStatisticsUpdate.md
 docs/RemovePropertiesUpdate.md
 docs/RemoveSnapshotRefUpdate.md
@@ -75,6 +90,7 @@ docs/RenameTableRequest.md
 docs/ReportMetricsRequest.md
 docs/SQLViewRepresentation.md
 docs/ScanReport.md
+docs/ScanTasks.md
 docs/SetCurrentSchemaUpdate.md
 docs/SetCurrentViewVersionUpdate.md
 docs/SetDefaultSortOrderUpdate.md
@@ -93,17 +109,18 @@ docs/SortDirection.md
 docs/SortField.md
 docs/SortOrder.md
 docs/StatisticsFile.md
+docs/StorageCredential.md
 docs/StructField.md
 docs/StructType.md
 docs/TableIdentifier.md
 docs/TableMetadata.md
 docs/TableRequirement.md
 docs/TableUpdate.md
-docs/TableUpdateNotification.md
 docs/Term.md
 docs/TimerResult.md
 docs/TokenType.md
 docs/TransformTerm.md
+docs/TrueExpression.md
 docs/Type.md
 docs/UnaryExpression.md
 docs/UpdateNamespacePropertiesRequest.md
@@ -144,6 +161,7 @@ polaris/catalog/models/assert_ref_snapshot_id.py
 polaris/catalog/models/assert_table_uuid.py
 polaris/catalog/models/assert_view_uuid.py
 polaris/catalog/models/assign_uuid_update.py
+polaris/catalog/models/async_planning_result.py
 polaris/catalog/models/base_update.py
 polaris/catalog/models/blob_metadata.py
 polaris/catalog/models/catalog_config.py
@@ -152,6 +170,8 @@ polaris/catalog/models/commit_table_request.py
 polaris/catalog/models/commit_table_response.py
 polaris/catalog/models/commit_transaction_request.py
 polaris/catalog/models/commit_view_request.py
+polaris/catalog/models/completed_planning_result.py
+polaris/catalog/models/completed_planning_with_id_result.py
 polaris/catalog/models/content_file.py
 polaris/catalog/models/count_map.py
 polaris/catalog/models/counter_result.py
@@ -160,34 +180,46 @@ polaris/catalog/models/create_namespace_response.py
 polaris/catalog/models/create_table_request.py
 polaris/catalog/models/create_view_request.py
 polaris/catalog/models/data_file.py
+polaris/catalog/models/delete_file.py
+polaris/catalog/models/empty_planning_result.py
+polaris/catalog/models/enable_row_lineage_update.py
 polaris/catalog/models/equality_delete_file.py
 polaris/catalog/models/error_model.py
 polaris/catalog/models/expression.py
+polaris/catalog/models/failed_planning_result.py
+polaris/catalog/models/false_expression.py
+polaris/catalog/models/fetch_planning_result.py
+polaris/catalog/models/fetch_scan_tasks_request.py
+polaris/catalog/models/fetch_scan_tasks_result.py
 polaris/catalog/models/file_format.py
+polaris/catalog/models/file_scan_task.py
 polaris/catalog/models/get_namespace_response.py
 polaris/catalog/models/iceberg_error_response.py
 polaris/catalog/models/list_namespaces_response.py
 polaris/catalog/models/list_tables_response.py
 polaris/catalog/models/list_type.py
 polaris/catalog/models/literal_expression.py
+polaris/catalog/models/load_credentials_response.py
 polaris/catalog/models/load_table_result.py
 polaris/catalog/models/load_view_result.py
 polaris/catalog/models/map_type.py
 polaris/catalog/models/metadata_log_inner.py
 polaris/catalog/models/metric_result.py
 polaris/catalog/models/model_schema.py
 polaris/catalog/models/not_expression.py
-polaris/catalog/models/notification_request.py
-polaris/catalog/models/notification_type.py
 polaris/catalog/models/null_order.py
 polaris/catalog/models/o_auth_error.py
 polaris/catalog/models/o_auth_token_response.py
 polaris/catalog/models/partition_field.py
 polaris/catalog/models/partition_spec.py
 polaris/catalog/models/partition_statistics_file.py
+polaris/catalog/models/plan_status.py
+polaris/catalog/models/plan_table_scan_request.py
+polaris/catalog/models/plan_table_scan_result.py
 polaris/catalog/models/position_delete_file.py
 polaris/catalog/models/primitive_type_value.py
 polaris/catalog/models/register_table_request.py
+polaris/catalog/models/remove_partition_specs_update.py
 polaris/catalog/models/remove_partition_statistics_update.py
 polaris/catalog/models/remove_properties_update.py
 polaris/catalog/models/remove_snapshot_ref_update.py
@@ -196,6 +228,7 @@ polaris/catalog/models/remove_statistics_update.py
 polaris/catalog/models/rename_table_request.py
 polaris/catalog/models/report_metrics_request.py
 polaris/catalog/models/scan_report.py
+polaris/catalog/models/scan_tasks.py
 polaris/catalog/models/set_current_schema_update.py
 polaris/catalog/models/set_current_view_version_update.py
 polaris/catalog/models/set_default_sort_order_update.py
@@ -215,17 +248,18 @@ polaris/catalog/models/sort_field.py
 polaris/catalog/models/sort_order.py
 polaris/catalog/models/sql_view_representation.py
 polaris/catalog/models/statistics_file.py
+polaris/catalog/models/storage_credential.py
 polaris/catalog/models/struct_field.py
 polaris/catalog/models/struct_type.py
 polaris/catalog/models/table_identifier.py
 polaris/catalog/models/table_metadata.py
 polaris/catalog/models/table_requirement.py
 polaris/catalog/models/table_update.py
-polaris/catalog/models/table_update_notification.py
 polaris/catalog/models/term.py
 polaris/catalog/models/timer_result.py
 polaris/catalog/models/token_type.py
 polaris/catalog/models/transform_term.py
+polaris/catalog/models/true_expression.py
 polaris/catalog/models/type.py
 polaris/catalog/models/unary_expression.py
 polaris/catalog/models/update_namespace_properties_request.py
@@ -246,4 +280,28 @@ setup.cfg
 setup.py
 test-requirements.txt
 test/__init__.py
+test/test_async_planning_result.py
+test/test_completed_planning_result.py
+test/test_completed_planning_with_id_result.py
+test/test_content_file.py
+test/test_count_map.py
+test/test_data_file.py
+test/test_delete_file.py
+test/test_empty_planning_result.py
+test/test_equality_delete_file.py
+test/test_failed_planning_result.py
+test/test_fetch_planning_result.py
+test/test_fetch_scan_tasks_request.py
+test/test_fetch_scan_tasks_result.py
+test/test_file_format.py
+test/test_file_scan_task.py
+test/test_iceberg_catalog_api.py
+test/test_iceberg_configuration_api.py
+test/test_iceberg_o_auth2_api.py
+test/test_plan_status.py
+test/test_plan_table_scan_request.py
+test/test_plan_table_scan_result.py
+test/test_position_delete_file.py
+test/test_scan_tasks.py
+test/test_value_map.py
 tox.ini

client/python/.travis.yml

@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# ref: https://docs.travis-ci.com/user/languages/python
+language: python
+python:
+  - "3.7"
+  - "3.8"
+  - "3.9"
+  - "3.10"
+  - "3.11"
+  # uncomment the following if needed
+  #- "3.11-dev"  # 3.11 development branch
+  #- "nightly"  # nightly build
+# command to install dependencies
+install:
+  - "pip install -r requirements.txt"
+  - "pip install -r test-requirements.txt"
+# command to run tests
+script: pytest --cov=polaris.catalog