Skip to content

Commit e8a2f9a

Browse files
committed
fix conflict
2 parents fe84057 + 980e206 commit e8a2f9a

File tree

147 files changed

+6121
-1457
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

147 files changed

+6121
-1457
lines changed

.github/workflows/stale.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
stale:
2323
runs-on: ubuntu-24.04
2424
steps:
25-
- uses: actions/stale@3a9db7e6a41a89f618792c92c0e97cc736e1b13f
25+
- uses: actions/stale@5f858e3efba33a5ca4407a664cc011ad407f2008
2626
with:
2727
days-before-close: 5
2828
days-before-stale: 30

.gitignore

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ gradle/wrapper/gradle-wrapper-*.sha256
6464
.env
6565

6666
# IntelliJ
67-
/.idea
67+
.idea/
6868
*.iml
6969
*.ipr
7070
*.iws
@@ -100,3 +100,9 @@ hs_err_pid*
100100

101101
# macOS
102102
*.DS_Store
103+
104+
# vscode
105+
.vscode/
106+
107+
# Python Virtual Env
108+
.venv

CHANGELOG.md

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,13 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
3232
### Upgrade Notes
3333

3434
- Amazon RDS plugin enabled, this allows polaris to connect to AWS Aurora PostgreSQL using IAM authentication.
35-
- The EclipseLink Persistence implementation has been deprecated since 1.0.0 and will be completely removed
36-
in 1.3.0 or in 2.0.0 (whichever happens earlier).
3735

3836
### Breaking Changes
3937

4038
### New Features
4139

40+
- Added a finer grained authorization model for UpdateTable requests. Existing privileges continue to work for granting UpdateTable, such as `TABLE_WRITE_PROPERTIES`.
41+
However, you can now instead grant privileges just for specific operations, such as `TABLE_ADD_SNAPSHOT`
4242
- Added a Management API endpoint to reset principal credentials, controlled by the `ENABLE_CREDENTIAL_RESET` (default: true) feature flag.
4343
- The `ENABLE_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS` was added to support sub-catalog (initially namespace and table) RBAC for federated catalogs.
4444
The setting can be configured on a per-catalog basis by setting the catalog property: `polaris.config.enable-sub-catalog-rbac-for-federated-catalogs`.
@@ -53,6 +53,11 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
5353
### Deprecations
5454

5555
* The property `polaris.active-roles-provider.type` is deprecated and has no effect anymore.
56+
- The EclipseLink Persistence implementation has been deprecated since 1.0.0 and will be completely removed
57+
in 1.3.0 or in 2.0.0 (whichever happens earlier).
58+
- The legacy management endpoints at `/metrics` and `/healthcheck` have been deprecated in 1.2.0 and will be
59+
completely removed in 1.3.0 or in 2.0.0 (whichever happens earlier). Please use the standard management
60+
endpoints at `/q/metrics` and `/q/health` instead.
5661

5762
### Fixes
5863

build-logic/src/main/kotlin/Utilities.kt

Lines changed: 0 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,6 @@
1414
* limitations under the License.
1515
*/
1616

17-
import java.io.File
18-
import java.io.FileOutputStream
19-
import java.nio.file.attribute.FileTime
20-
import java.util.zip.ZipFile
21-
import java.util.zip.ZipOutputStream
2217
import org.gradle.api.Project
2318
import org.gradle.process.JavaForkOptions
2419

@@ -66,41 +61,3 @@ fun JavaForkOptions.addSparkJvmOptions() {
6661
"-Djdk.reflect.useDirectMethodHandle=false",
6762
)
6863
}
69-
70-
/**
71-
* Rewrites the given ZIP file.
72-
*
73-
* The timestamps of all entries are set to `1980-02-01 00:00`, zip entries appear in a
74-
* deterministic order.
75-
*/
76-
fun makeZipReproducible(source: File) {
77-
val t = FileTime.fromMillis(318211200_000) // 1980-02-01 00:00 GMT
78-
79-
val outFile = File(source.absolutePath + ".tmp.out")
80-
81-
val names = mutableListOf<String>()
82-
ZipFile(source).use { zip -> zip.stream().forEach { e -> names.add(e.name) } }
83-
names.sort()
84-
85-
ZipOutputStream(FileOutputStream(outFile)).use { dst ->
86-
ZipFile(source).use { zip ->
87-
names.forEach { n ->
88-
val e = zip.getEntry(n)
89-
zip.getInputStream(e).use { src ->
90-
e.setCreationTime(t)
91-
e.setLastAccessTime(t)
92-
e.setLastModifiedTime(t)
93-
dst.putNextEntry(e)
94-
src.copyTo(dst)
95-
dst.closeEntry()
96-
src.close()
97-
}
98-
}
99-
}
100-
}
101-
102-
val origFile = File(source.absolutePath + ".tmp.orig")
103-
source.renameTo(origFile)
104-
outFile.renameTo(source)
105-
origFile.delete()
106-
}

build-logic/src/main/kotlin/polaris-java.gradle.kts

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,12 @@
1818
*/
1919

2020
import java.util.Properties
21+
import kotlin.jvm.java
2122
import net.ltgt.gradle.errorprone.CheckSeverity
2223
import net.ltgt.gradle.errorprone.errorprone
2324
import org.gradle.api.tasks.compile.JavaCompile
2425
import org.gradle.api.tasks.testing.Test
26+
import org.gradle.kotlin.dsl.assign
2527
import org.gradle.kotlin.dsl.named
2628
import org.kordamp.gradle.plugin.jandex.JandexExtension
2729
import org.kordamp.gradle.plugin.jandex.JandexPlugin
@@ -254,20 +256,34 @@ configurations.all {
254256
}
255257
}
256258

257-
if (plugins.hasPlugin("io.quarkus")) {
258-
tasks.named("quarkusBuild") {
259-
actions.addLast {
260-
listOf(
261-
"quarkus-app/quarkus-run.jar",
262-
"quarkus-app/quarkus/generated-bytecode.jar",
263-
"quarkus-app/quarkus/transformed-bytecode.jar",
264-
)
265-
.forEach { name ->
266-
val file = project.layout.buildDirectory.get().file(name).asFile
267-
if (file.exists()) {
268-
makeZipReproducible(file)
269-
}
270-
}
271-
}
272-
}
259+
gradle.sharedServices.registerIfAbsent(
260+
"intTestParallelismConstraint",
261+
TestingParallelismHelper::class.java,
262+
) {
263+
val intTestParallelism =
264+
Integer.getInteger(
265+
"polaris.intTestParallelism",
266+
(Runtime.getRuntime().availableProcessors() / 4).coerceAtLeast(1),
267+
)
268+
maxParallelUsages = intTestParallelism
269+
}
270+
271+
gradle.sharedServices.registerIfAbsent(
272+
"testParallelismConstraint",
273+
TestingParallelismHelper::class.java,
274+
) {
275+
val testParallelism =
276+
Integer.getInteger(
277+
"polaris.testParallelism",
278+
(Runtime.getRuntime().availableProcessors() / 2).coerceAtLeast(1),
279+
)
280+
maxParallelUsages = testParallelism
281+
}
282+
283+
abstract class TestingParallelismHelper : BuildService<BuildServiceParameters.None>
284+
285+
tasks.withType<Test>().configureEach {
286+
val constraintName =
287+
if ("test" == name) "testParallelismConstraint" else "intTestParallelismConstraint"
288+
usesService(gradle.sharedServices.registrations.named(constraintName).get().service)
273289
}

build-logic/src/main/kotlin/polaris-runtime.gradle.kts

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,26 @@ tasks.named("javadoc") { dependsOn("jandex") }
9595
tasks.named("quarkusDependenciesBuild") { dependsOn("jandex") }
9696

9797
tasks.named("imageBuild") { dependsOn("jandex") }
98+
99+
tasks.withType(Test::class.java).configureEach {
100+
// Gradle's Jacoco plugin doesn't work well with Quarkus's test coverage
101+
extensions.configure(JacocoTaskExtension::class) { isEnabled = false }
102+
103+
// Quarkus tests run "in isolated class loaders", which means that class-statically active
104+
// resources pile up used JVM, as those classes cannot be GC'd.
105+
// Examples of those statically held active resources are:
106+
// - Iceberg's worker pools (thread pools, executors, etc.)
107+
// - Hadoop's stats-cleaner (org.apache.hadoop.fs.FileSystem.Statistics.STATS_DATA_CLEANER)
108+
// - Guava's 'MoreExecutors' (via Iceberg `ThreadPools`)`
109+
// Forcing a new JVM after each test class works around this issue.
110+
forkEvery = 1
111+
112+
maxParallelForks = 1
113+
114+
// enlarge the max heap size to avoid out of memory error
115+
maxHeapSize = "4g"
116+
117+
// Silence the 'OpenJDK 64-Bit Server VM warning: Sharing is only supported for boot loader
118+
// classes because bootstrap classpath has been appended' warning from OpenJDK.
119+
jvmArgs("-Xshare:off")
120+
}
Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
import json
20+
import os
21+
from argparse import Namespace
22+
from functools import cached_property
23+
from typing import Optional, Dict
24+
25+
from cli.constants import (
26+
CONFIG_FILE,
27+
CLIENT_PROFILE_ENV,
28+
CLIENT_ID_ENV,
29+
CLIENT_SECRET_ENV,
30+
REALM_ENV,
31+
HEADER_ENV,
32+
Arguments,
33+
DEFAULT_HOSTNAME,
34+
DEFAULT_PORT
35+
)
36+
from cli.options.option_tree import Argument
37+
from polaris.management import ApiClient, Configuration
38+
39+
40+
def _load_profiles() -> Dict[str, str]:
41+
if not os.path.exists(CONFIG_FILE):
42+
return {}
43+
with open(CONFIG_FILE, "r") as f:
44+
return json.load(f)
45+
46+
47+
class BuilderConfig:
48+
def __init__(self, options: Namespace):
49+
self.options = options
50+
self.proxy = options.proxy
51+
self.access_token = options.access_token
52+
53+
@cached_property
54+
def profile(self) -> Dict[str, str]:
55+
profile = {}
56+
client_profile = self.options.profile or os.getenv(CLIENT_PROFILE_ENV)
57+
if client_profile:
58+
profiles = _load_profiles()
59+
profile = profiles.get(client_profile)
60+
if not profile:
61+
raise Exception(f"Polaris profile {client_profile} not found")
62+
return profile
63+
64+
@cached_property
65+
def base_url(self) -> str:
66+
if self.options.base_url:
67+
if self.options.host is not None or self.options.port is not None:
68+
raise Exception(
69+
f"Please provide either {Argument.to_flag_name(Arguments.BASE_URL)} or"
70+
f" {Argument.to_flag_name(Arguments.HOST)} &"
71+
f" {Argument.to_flag_name(Arguments.PORT)}, but not both"
72+
)
73+
return self.options.base_url
74+
75+
host = self.options.host or self.profile.get(Arguments.HOST) or DEFAULT_HOSTNAME
76+
port = self.options.port or self.profile.get(Arguments.PORT) or DEFAULT_PORT
77+
return f"http://{host}:{port}"
78+
79+
@cached_property
80+
def management_url(self) -> str:
81+
return f"{self.base_url}/api/management/v1"
82+
83+
@cached_property
84+
def catalog_url(self) -> str:
85+
return f"{self.base_url}/api/catalog/v1"
86+
87+
@cached_property
88+
def client_id(self) -> Optional[str]:
89+
return (
90+
self.options.client_id
91+
or os.getenv(CLIENT_ID_ENV)
92+
or self.profile.get(Arguments.CLIENT_ID)
93+
)
94+
95+
@cached_property
96+
def client_secret(self) -> Optional[str]:
97+
return (
98+
self.options.client_secret
99+
or os.getenv(CLIENT_SECRET_ENV)
100+
or self.profile.get(Arguments.CLIENT_SECRET)
101+
)
102+
103+
@cached_property
104+
def realm(self) -> Optional[str]:
105+
realms = (
106+
self.options.realm
107+
or os.getenv(REALM_ENV)
108+
or self.profile.get(Arguments.REALM)
109+
)
110+
return realms
111+
112+
@cached_property
113+
def header(self) -> Optional[str]:
114+
return (
115+
self.options.header
116+
or os.getenv(HEADER_ENV)
117+
or self.profile.get(Arguments.HEADER)
118+
)
119+
120+
121+
class ApiClientBuilder:
122+
def __init__(self, options: Namespace, *, direct_authentication: bool = False):
123+
self.conf = BuilderConfig(options)
124+
self.direct_auth_enabled = direct_authentication
125+
126+
def _get_token(self):
127+
header_params = {"Content-Type": "application/x-www-form-urlencoded"}
128+
if self.conf.header and self.conf.realm:
129+
header_params[self.conf.header] = self.conf.realm
130+
131+
conf = Configuration(host=self.conf.management_url)
132+
conf.proxy = self.conf.proxy
133+
api_client = ApiClient(conf)
134+
response = api_client.call_api(
135+
"POST",
136+
f"{self.conf.catalog_url}/oauth/tokens",
137+
header_params=header_params,
138+
post_params={
139+
"grant_type": "client_credentials",
140+
"client_id": self.conf.client_id,
141+
"client_secret": self.conf.client_secret,
142+
"scope": "PRINCIPAL_ROLE:ALL",
143+
},
144+
).response.data
145+
if "access_token" not in json.loads(response):
146+
raise Exception("Failed to get access token")
147+
return json.loads(response)["access_token"]
148+
149+
def _build(self) -> ApiClient:
150+
has_access_token = self.conf.access_token is not None
151+
has_client_secret = self.conf.client_id is not None and self.conf.client_secret is not None
152+
if has_access_token and (self.conf.client_id or self.conf.client_secret):
153+
raise Exception(
154+
f"Please provide credentials via either {Argument.to_flag_name(Arguments.CLIENT_ID)} &"
155+
f" {Argument.to_flag_name(Arguments.CLIENT_SECRET)} or"
156+
f" {Argument.to_flag_name(Arguments.ACCESS_TOKEN)}, but not both"
157+
)
158+
if not has_access_token and not has_client_secret:
159+
raise Exception(
160+
f"Please provide credentials via either {Argument.to_flag_name(Arguments.CLIENT_ID)} &"
161+
f" {Argument.to_flag_name(Arguments.CLIENT_SECRET)} or"
162+
f" {Argument.to_flag_name(Arguments.ACCESS_TOKEN)}."
163+
f" Alternatively, you may set the environment variables {CLIENT_ID_ENV} &"
164+
f" {CLIENT_SECRET_ENV}."
165+
)
166+
167+
config = Configuration(host=self.conf.management_url)
168+
config.proxy = self.conf.proxy
169+
if has_access_token:
170+
config.access_token = self.conf.access_token
171+
elif has_client_secret:
172+
config.username = self.conf.client_id
173+
config.password = self.conf.client_secret
174+
175+
if not has_access_token and not self.direct_auth_enabled:
176+
config.username = None
177+
config.password = None
178+
config.access_token = self._get_token()
179+
180+
client_params = {}
181+
if self.conf.realm and self.conf.header:
182+
client_params["header_name"] = self.conf.header
183+
client_params["header_value"] = self.conf.realm
184+
185+
return ApiClient(config, **client_params)
186+
187+
def get_api_client(self) -> ApiClient:
188+
return self._build()

0 commit comments

Comments
 (0)