Skip to content

Commit 9c455ed

Browse files
MonkeyCanCodeMonkeyCanCode
authored
Python client auto generate (apache#2192)
* Python client auto generate * Python client auto generate * Python client auto generate * Python client auto generate * Python client auto generate * Python client auto generate * Remove auto generated doc * undo * Fix doc * Fix docker ref from CONTAINER_TOOL to DOCKER * Add client help manual to GH action
1 parent c937a30 commit 9c455ed

File tree

14 files changed

+289
-253
lines changed

14 files changed

+289
-253
lines changed

‎.github/workflows/python-client.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,6 @@ jobs:
5050
distribution: 'temurin'
5151
java-version: '21'
5252

53-
- name: Run regeneratePythonClient
54-
run: ./gradlew regeneratePythonClient
55-
5653
- name: Set up Python ${{ matrix.python-version }}
5754
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
5855
with:
@@ -76,3 +73,7 @@ jobs:
7673
- name: Integration Tests
7774
run: |
7875
make client-integration-test
76+
77+
- name: Run Polaris Client help maual
78+
run: |
79+
./polaris --help

‎.github/workflows/regtest.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,6 @@ jobs:
5151
- name: Fix permissions
5252
run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/*
5353

54-
- name: Run regeneratePythonClient
55-
run: ./gradlew regeneratePythonClient
56-
5754
- name: Image build
5855
run: |
5956
./gradlew \

‎.github/workflows/spark_client_regtests.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,6 @@ jobs:
5151
- name: Fix permissions
5252
run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/*
5353

54-
- name: Run regeneratePythonClient
55-
run: ./gradlew regeneratePythonClient
56-
5754
- name: Project build without testing
5855
env:
5956
# publishToMavenLocal causes a GH API requests, use the token for those requests

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ client-lint: client-setup-env ## Run linting checks for Polaris client
134134
.PHONY: client-regenerate
135135
client-regenerate: client-setup-env ## Regenerate the client code
136136
@echo "--- Regenerating client code ---"
137-
@client/templates/regenerate.sh
137+
@$(ACTIVATE_AND_CD) && python3 generate_clients.py
138138
@echo "--- Client code regeneration complete ---"
139139

140140
.PHONY: client-unit-test

‎build.gradle.kts‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -142,18 +142,6 @@ tasks.named<RatTask>("rat").configure {
142142
excludes.add("**/*.png")
143143
}
144144

145-
tasks.register<Exec>("regeneratePythonClient") {
146-
description = "Regenerates the python client"
147-
148-
workingDir = project.projectDir
149-
commandLine("bash", "client/templates/regenerate.sh")
150-
151-
dependsOn(":polaris-api-iceberg-service:processResources")
152-
dependsOn(":polaris-api-management-service:processResources")
153-
dependsOn(":polaris-api-catalog-service:processResources")
154-
dependsOn(":polaris-api-management-model:processResources")
155-
}
156-
157145
// Pass environment variables:
158146
// ORG_GRADLE_PROJECT_apacheUsername
159147
// ORG_GRADLE_PROJECT_apachePassword

‎client/python/.openapi-generator-ignore‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,5 +52,4 @@ git_push.sh
5252
setup.cfg
5353
tox.ini
5454
README.md
55-
56-
55+
pyproject.toml

‎client/python/generate_clients.py‎

Lines changed: 267 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,267 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
# http://www.apache.org/licenses/LICENSE-2.0
19+
#
20+
# Unless required by applicable law or agreed to in writing,
21+
# software distributed under the License is distributed on an
22+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
23+
# KIND, either express or implied. See the License for the
24+
# specific language governing permissions and limitations
25+
# under the License.
26+
27+
import sys
28+
import os.path
29+
import subprocess
30+
from pathlib import Path
31+
import fnmatch
32+
33+
# Paths
34+
CLIENT_DIR = Path(__file__).parent
35+
PROJECT_ROOT = CLIENT_DIR.parent.parent
36+
HEADER_DIR = CLIENT_DIR.parent / "templates"
37+
SPEC_DIR = os.path.join(PROJECT_ROOT, "spec")
38+
POLARIS_MANAGEMENT_SPEC = os.path.join(SPEC_DIR, "polaris-management-service.yml")
39+
ICEBERG_CATALOG_SPEC = os.path.join(SPEC_DIR, "iceberg-rest-catalog-open-api.yaml")
40+
POLARIS_CATALOG_SPEC = os.path.join(SPEC_DIR, "polaris-catalog-service.yaml")
41+
OPEN_API_GENERATOR_IGNORE = os.path.join(CLIENT_DIR, ".openapi-generator-ignore")
42+
43+
# Open API Generator Configs
44+
PACKAGE_NAME_POLARIS_MANAGEMENT = (
45+
"--additional-properties=packageName=polaris.management"
46+
)
47+
PACKAGE_NAME_POLARIS_CATALOG = "--additional-properties=packageName=polaris.catalog"
48+
PYTHON_VERSION = "--additional-properties=pythonVersion=3.9"
49+
50+
# Cleanup
51+
KEEP_TEST_FILES = [
52+
Path("test/test_cli_parsing.py"),
53+
]
54+
EXCLUDE_PATHS = [
55+
Path(".gitignore"),
56+
Path(".openapi-generator/"),
57+
Path(".openapi-generator-ignore"),
58+
Path(".pytest_cache/"),
59+
Path("test/test_cli_parsing.py"),
60+
Path("cli/"),
61+
Path("polaris/__pycache__/"),
62+
Path("polaris/catalog/__pycache__/"),
63+
Path("polaris/catalog/models/__pycache__/"),
64+
Path("polaris/catalog/api/__pycache__/"),
65+
Path("polaris/management/__pycache__/"),
66+
Path("polaris/management/models/__pycache__/"),
67+
Path("polaris/management/api/__pycache__/"),
68+
Path("integration_tests/"),
69+
Path(".github/workflows/python.yml"),
70+
Path(".gitlab-ci.yml"),
71+
Path("pyproject.toml"),
72+
Path("requirements.txt"),
73+
Path("test-requirements.txt"),
74+
Path("setup.py"),
75+
Path(".DS_Store"),
76+
Path("Makefile"),
77+
Path("poetry.lock"),
78+
Path("docker-compose.yml"),
79+
Path(".pre-commit-config.yaml"),
80+
Path("README.md"),
81+
Path("generate_clients.py"),
82+
Path(".venv"),
83+
]
84+
EXCLUDE_EXTENSIONS = [
85+
"json",
86+
"iml",
87+
"keep",
88+
"gitignore",
89+
]
90+
91+
92+
def clean_old_tests() -> None:
93+
print("Deleting old tests...")
94+
test_dir = CLIENT_DIR / "test"
95+
if not test_dir.exists():
96+
print(f"Test directory {test_dir} does not exist, skipping test cleanup.")
97+
return
98+
99+
for item in test_dir.rglob("*"):
100+
if item.is_file():
101+
# Check if the file should be kept relative to CLIENT_DIR
102+
relative_path = item.relative_to(CLIENT_DIR)
103+
if relative_path not in KEEP_TEST_FILES:
104+
try:
105+
os.remove(item)
106+
print(f"{relative_path}: removed")
107+
except OSError as e:
108+
print(f"Error removing {relative_path}: {e}")
109+
else:
110+
print(f"{relative_path}: skipped")
111+
112+
init_py_to_delete = CLIENT_DIR / "test" / "__init__.py"
113+
if init_py_to_delete.exists():
114+
try:
115+
os.remove(init_py_to_delete)
116+
print(f"{init_py_to_delete.relative_to(CLIENT_DIR)}: removed")
117+
except OSError as e:
118+
print(f"Error removing {init_py_to_delete.relative_to(CLIENT_DIR)}: {e}")
119+
print("Old test deletion complete.")
120+
121+
122+
def generate_polaris_management_client() -> None:
123+
subprocess.check_call(
124+
[
125+
"openapi-generator-cli",
126+
"generate",
127+
"-i",
128+
POLARIS_MANAGEMENT_SPEC,
129+
"-g",
130+
"python",
131+
"-o",
132+
CLIENT_DIR,
133+
PACKAGE_NAME_POLARIS_MANAGEMENT,
134+
"--additional-properties=apiNamePrefix=polaris",
135+
PYTHON_VERSION,
136+
"--additional-properties=generateSourceCodeOnly=true",
137+
"--skip-validate-spec",
138+
"--ignore-file-override",
139+
OPEN_API_GENERATOR_IGNORE,
140+
],
141+
stdout=subprocess.DEVNULL,
142+
)
143+
144+
145+
def generate_polaris_catalog_client() -> None:
146+
subprocess.check_call(
147+
[
148+
"openapi-generator-cli",
149+
"generate",
150+
"-i",
151+
POLARIS_CATALOG_SPEC,
152+
"-g",
153+
"python",
154+
"-o",
155+
CLIENT_DIR,
156+
PACKAGE_NAME_POLARIS_CATALOG,
157+
"--additional-properties=apiNameSuffix=",
158+
PYTHON_VERSION,
159+
"--additional-properties=generateSourceCodeOnly=true",
160+
"--skip-validate-spec",
161+
"--ignore-file-override",
162+
OPEN_API_GENERATOR_IGNORE,
163+
],
164+
stdout=subprocess.DEVNULL,
165+
)
166+
167+
168+
def generate_iceberg_catalog_client() -> None:
169+
subprocess.check_call(
170+
[
171+
"openapi-generator-cli",
172+
"generate",
173+
"-i",
174+
ICEBERG_CATALOG_SPEC,
175+
"-g",
176+
"python",
177+
"-o",
178+
CLIENT_DIR,
179+
PACKAGE_NAME_POLARIS_CATALOG,
180+
"--additional-properties=apiNameSuffix=",
181+
"--additional-properties=apiNamePrefix=Iceberg",
182+
PYTHON_VERSION,
183+
"--additional-properties=generateSourceCodeOnly=true",
184+
"--skip-validate-spec",
185+
"--ignore-file-override",
186+
OPEN_API_GENERATOR_IGNORE,
187+
],
188+
stdout=subprocess.DEVNULL,
189+
)
190+
191+
192+
def _prepend_header_to_file(file_path: Path, header_file_path: Path) -> None:
193+
try:
194+
with open(header_file_path, "r") as hf:
195+
header_content = hf.read()
196+
with open(file_path, "r+") as f:
197+
original_content = f.read()
198+
f.seek(0)
199+
f.write(header_content + original_content)
200+
except IOError as e:
201+
print(f"Error prepending header to {file_path}: {e}")
202+
203+
204+
def prepend_licenses() -> None:
205+
print("Re-applying license headers...")
206+
for file_path in CLIENT_DIR.rglob("*"):
207+
if file_path.is_file():
208+
relative_file_path = file_path.relative_to(CLIENT_DIR)
209+
file_extension = ""
210+
# If it's a "dotfile" like .keep
211+
if (
212+
relative_file_path.name.startswith(".")
213+
and "." not in relative_file_path.name[1:]
214+
):
215+
# e.g., for '.keep', this is 'keep'
216+
file_extension = relative_file_path.name.lstrip(".")
217+
else:
218+
# For standard files like generate_clients.py
219+
file_extension = file_path.suffix.lstrip(".")
220+
221+
# Check if extension is excluded
222+
if file_extension in EXCLUDE_EXTENSIONS:
223+
print(f"{relative_file_path}: skipped (extension excluded)")
224+
continue
225+
226+
is_excluded = False
227+
# Combine EXCLUDE_PATHS and KEEP_TEST_FILES for comprehensive exclusion check
228+
# Convert Path objects in EXCLUDE_PATHS to strings for fnmatch compatibility
229+
# Ensure patterns ending with '/' are handled for directory matching
230+
all_exclude_patterns = [
231+
str(p) + ("/" if p.is_dir() else "") for p in EXCLUDE_PATHS
232+
] + [str(p) for p in KEEP_TEST_FILES]
233+
234+
for exclude_pattern_str in all_exclude_patterns:
235+
# Handle direct file match or if the file is within an excluded directory
236+
if fnmatch.fnmatch(str(relative_file_path), exclude_pattern_str) or (
237+
exclude_pattern_str.endswith("/")
238+
and str(relative_file_path).startswith(exclude_pattern_str)
239+
):
240+
is_excluded = True
241+
break
242+
243+
if is_excluded:
244+
print(f"{relative_file_path}: skipped (path excluded)")
245+
continue
246+
247+
header_file_path = HEADER_DIR / f"header-{file_extension}.txt"
248+
249+
if header_file_path.is_file():
250+
_prepend_header_to_file(file_path, header_file_path)
251+
print(f"{relative_file_path}: updated")
252+
else:
253+
print(f"No header compatible with file {relative_file_path}")
254+
sys.exit(2)
255+
print("License fix complete.")
256+
257+
258+
def build() -> None:
259+
clean_old_tests()
260+
generate_polaris_management_client()
261+
generate_polaris_catalog_client()
262+
generate_iceberg_catalog_client()
263+
prepend_licenses()
264+
265+
266+
if __name__ == "__main__":
267+
build()

‎client/python/pyproject.toml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ include = [
5454
"polaris/**"
5555
]
5656

57-
[tool.poetry.group.test.dependencies]
57+
[tool.poetry.group.dev.dependencies]
5858
pytest = ">= 7.2.1"
5959
pytest-cov = ">= 2.8.1"
6060
tox = ">= 3.9.0"
@@ -63,7 +63,12 @@ types-python-dateutil = ">= 2.8.19.14"
6363
mypy = ">=1.17, <=1.17.1"
6464
pyiceberg = "==0.9.1"
6565
pre-commit = "==4.3.0"
66+
openapi-generator-cli = "==7.11.0.post0"
6667

6768
[build-system]
6869
requires = ["poetry-core>=2.0.0,<3.0.0"]
6970
build-backend = "poetry.core.masonry.api"
71+
72+
[tool.poetry.build]
73+
generate-setup-file = false
74+
script = "generate_clients.py"

0 commit comments

Comments
 (0)