Skip to content

Commit c95bf58

Browse files
authored
Add REST Compatibility Kit (#10908)
* Add REST Compatibility Kit * Publish test artifacts for open api module
1 parent bf00d51 commit c95bf58

11 files changed

+701
-5
lines changed

build.gradle

+45
Original file line numberDiff line numberDiff line change
@@ -953,6 +953,51 @@ project(':iceberg-snowflake') {
953953
}
954954

955955
project(':iceberg-open-api') {
956+
apply plugin: 'java-test-fixtures'
957+
958+
dependencies {
959+
testImplementation project(':iceberg-api')
960+
testImplementation project(':iceberg-core')
961+
testImplementation project(':iceberg-core').sourceSets.test.runtimeClasspath
962+
testImplementation(testFixtures(project(':iceberg-open-api')))
963+
964+
testImplementation libs.junit.jupiter
965+
testImplementation libs.junit.suite.api
966+
testImplementation libs.junit.suite.engine
967+
testImplementation libs.assertj.core
968+
969+
testImplementation project(':iceberg-aws-bundle')
970+
testImplementation project(':iceberg-gcp-bundle')
971+
testImplementation project(':iceberg-azure-bundle')
972+
973+
testFixturesImplementation project(':iceberg-api')
974+
testFixturesImplementation project(':iceberg-core')
975+
testFixturesImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
976+
testFixturesImplementation project(':iceberg-core').sourceSets.test.runtimeClasspath
977+
testFixturesImplementation project(':iceberg-aws')
978+
testFixturesImplementation project(':iceberg-gcp')
979+
testFixturesImplementation project(':iceberg-azure')
980+
981+
testFixturesImplementation libs.jetty.servlet
982+
testFixturesImplementation libs.jetty.server
983+
testFixturesImplementation libs.sqlite.jdbc
984+
}
985+
986+
test {
987+
useJUnitPlatform()
988+
989+
// Always rerun the compatibility tests
990+
outputs.upToDateWhen {false}
991+
maxParallelForks = 1
992+
993+
// Pass through any system properties that start with "rck" (REST Compatibility Kit)
994+
// Note: only pass through specific properties so they do not affect other build/test
995+
// configurations
996+
systemProperties System.properties
997+
.findAll { k, v -> k.startsWith("rck") }
998+
.collectEntries { k, v -> { [(k):v, (k.replaceFirst("rck.", "")):v] }} // strip prefix
999+
}
1000+
9561001
def restCatalogSpec = "$projectDir/rest-catalog-open-api.yaml"
9571002
tasks.register('validateRESTCatalogSpec', org.openapitools.generator.gradle.plugin.tasks.ValidateTask) {
9581003
inputSpec.set(restCatalogSpec)

deploy.gradle

+4-5
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,8 @@ if (project.hasProperty('release') && jdkVersion != '11') {
2222
}
2323

2424
subprojects {
25-
if (it.name == 'iceberg-open-api') {
26-
// don't publish iceberg-open-api
27-
return
28-
}
29-
3025
def isBom = it.name == 'iceberg-bom'
26+
def isOpenApi = it.name == 'iceberg-open-api'
3127

3228
apply plugin: 'maven-publish'
3329
apply plugin: 'signing'
@@ -76,6 +72,9 @@ subprojects {
7672
apache(MavenPublication) {
7773
if (isBom) {
7874
from components.javaPlatform
75+
} else if (isOpenApi) {
76+
artifact testJar
77+
artifact testFixturesJar
7978
} else {
8079
if (tasks.matching({task -> task.name == 'shadowJar'}).isEmpty()) {
8180
from components.java

gradle/libs.versions.toml

+3
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ jaxb-api = "2.3.1"
6262
jaxb-runtime = "2.3.9"
6363
jetty = "11.0.23"
6464
junit = "5.10.1"
65+
junit-platform = "1.10.3"
6566
kafka = "3.8.0"
6667
kryo-shaded = "4.0.3"
6768
microprofile-openapi-api = "3.1.1"
@@ -202,6 +203,8 @@ jetty-server = { module = "org.eclipse.jetty:jetty-server", version.ref = "jetty
202203
jetty-servlet = { module = "org.eclipse.jetty:jetty-servlet", version.ref = "jetty" }
203204
junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" }
204205
junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit" }
206+
junit-suite-api = { module = "org.junit.platform:junit-platform-suite-api", version.ref = "junit-platform" }
207+
junit-suite-engine = { module = "org.junit.platform:junit-platform-suite-engine", version.ref = "junit-platform" }
205208
junit-vintage-engine = { module = "org.junit.vintage:junit-vintage-engine", version.ref = "junit" }
206209
kryo-shaded = { module = "com.esotericsoftware:kryo-shaded", version.ref = "kryo-shaded" }
207210
mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" }

open-api/README.md

+63
Original file line numberDiff line numberDiff line change
@@ -40,3 +40,66 @@ make generate
4040
```
4141

4242
The generated code is not being used in the project, but helps to see what the changes in the open-API definition are in the generated code.
43+
44+
# REST Compatibility Kit (RCK)
45+
46+
The REST Compatibility Kit (RCK) is a Technology Compatibility Kit (TCK) implementation for the
47+
Iceberg REST Specification. This includes a series of tests based on the Java reference
48+
implementation of the REST Catalog that can be executed against any REST server that implements the
49+
spec.
50+
51+
## Test Configuration
52+
53+
The RCK can be configured using either environment variables or java system properties and allows
54+
for configuring both the tests and the REST client. Environment variables prefixed by `CATALOG_`
55+
are passed through the catalog configuring with the following mutations:
56+
57+
1. The `CATALOG_` prefix is stripped from the key name
58+
2. Single underscore (`_`) is replaced with a dot (`.`)
59+
3. Double underscore (`__`) is replaced with a dash (`-`)
60+
4. The key names are converted to lowercase
61+
62+
A basic environment configuration would look like the following:
63+
64+
```shell
65+
CATALOG_URI=https://my_rest_server.io/ ## -> uri=https://my_rest_server.io/
66+
CATALOG_WAREHOUSE=test_warehouse ## -> warehouse=test_warehouse
67+
CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO ## -> io-impl=org.apache.iceberg.aws.s3.S3FileIO
68+
CATALOG_CREDENTIAL=<oauth_key>:<oauth_secret> ## -> credential=<oauth_key>:<oauth_secret>
69+
```
70+
71+
Java properties passed to the test must be prefixed with `rck.`, which can be used to configure some
72+
test configurations described below and any catalog client properties.
73+
74+
An example of the same configuration using java system properties would look like the following:
75+
```shell
76+
rck.uri=https://my_rest_server.io/ ## -> uri=https://my_rest_server.io/
77+
rck.warehouse=test_warehouse ## -> warehouse=test_warehouse
78+
rck.io-impl=org.apache.iceberg.aws.s3.S3FileIO ## -> io-impl=org.apache.iceberg.aws.s3.S3FileIO
79+
rck.credential=<oauth_key>:<oauth_secret> ## -> credential=<oauth_key>:<oauth_secret>
80+
```
81+
82+
Some test behaviors are configurable depending on the catalog implementations. Not all behaviors
83+
are strictly defined by the REST Specification. The following are currently configurable:
84+
85+
| config | default |
86+
|-------------------------------|---------|
87+
| rck.requires-namespace-create | true |
88+
| rck.supports-serverside-retry | true |
89+
90+
91+
## Running Compatibility Tests
92+
93+
The compatibility tests can be invoked via gradle with the following:
94+
95+
Note: The default behavior is to run a local http server with a jdbc backend for testing purposes,
96+
so `-Drck.local=false` must be set to point to an external REST server.
97+
98+
```shell
99+
./gradlew :iceberg-open-api:test --tests RESTCompatibilityKitSuite \
100+
-Drck.local=false \
101+
-Drck.requires-namespace-create=true \
102+
-Drck.uri=https://my_rest_server.io/ \
103+
-Drck.warehouse=test_warehouse \
104+
-Drck.credential=<oauth_key>:<oauth_secret>
105+
```
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.iceberg.rest;
20+
21+
import static org.assertj.core.api.Assertions.assertThat;
22+
23+
import org.apache.iceberg.catalog.CatalogTests;
24+
import org.apache.iceberg.util.PropertyUtil;
25+
import org.junit.jupiter.api.AfterAll;
26+
import org.junit.jupiter.api.BeforeAll;
27+
import org.junit.jupiter.api.BeforeEach;
28+
import org.junit.jupiter.api.extension.ExtendWith;
29+
import org.slf4j.Logger;
30+
import org.slf4j.LoggerFactory;
31+
32+
@ExtendWith(RESTServerExtension.class)
33+
public class RESTCompatibilityKitCatalogTests extends CatalogTests<RESTCatalog> {
34+
private static final Logger LOG = LoggerFactory.getLogger(RESTCompatibilityKitCatalogTests.class);
35+
36+
private static RESTCatalog restCatalog;
37+
38+
@BeforeAll
39+
static void beforeClass() throws Exception {
40+
restCatalog = RCKUtils.initCatalogClient();
41+
42+
assertThat(restCatalog.listNamespaces())
43+
.withFailMessage("Namespaces list should not contain: %s", RCKUtils.TEST_NAMESPACES)
44+
.doesNotContainAnyElementsOf(RCKUtils.TEST_NAMESPACES);
45+
}
46+
47+
@BeforeEach
48+
void before() {
49+
try {
50+
RCKUtils.purgeCatalogTestEntries(restCatalog);
51+
} catch (Exception e) {
52+
LOG.warn("Failure during test setup", e);
53+
}
54+
}
55+
56+
@AfterAll
57+
static void afterClass() throws Exception {
58+
restCatalog.close();
59+
}
60+
61+
@Override
62+
protected RESTCatalog catalog() {
63+
return restCatalog;
64+
}
65+
66+
@Override
67+
protected boolean requiresNamespaceCreate() {
68+
return PropertyUtil.propertyAsBoolean(
69+
restCatalog.properties(),
70+
RESTCompatibilityKitSuite.RCK_REQUIRES_NAMESPACE_CREATE,
71+
super.requiresNamespaceCreate());
72+
}
73+
74+
@Override
75+
protected boolean supportsServerSideRetry() {
76+
return PropertyUtil.propertyAsBoolean(
77+
restCatalog.properties(), RESTCompatibilityKitSuite.RCK_SUPPORTS_SERVERSIDE_RETRY, true);
78+
}
79+
80+
@Override
81+
protected boolean overridesRequestedLocation() {
82+
return PropertyUtil.propertyAsBoolean(
83+
restCatalog.properties(),
84+
RESTCompatibilityKitSuite.RCK_OVERRIDES_REQUESTED_LOCATION,
85+
false);
86+
}
87+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.iceberg.rest;
20+
21+
import org.junit.platform.suite.api.SelectClasses;
22+
import org.junit.platform.suite.api.Suite;
23+
import org.junit.platform.suite.api.SuiteDisplayName;
24+
25+
/**
26+
* Iceberg REST Compatibility Kit
27+
*
28+
* <p>This test suite provides the ability to run the Iceberg catalog tests against a remote REST
29+
* catalog implementation to verify the behaviors against the reference implementation catalog
30+
* tests.
31+
*
32+
* <p>The tests can be configured through environment variables or system properties. By default,
33+
* the tests will run using a local http server using a servlet implementation that leverages the
34+
* {@link RESTCatalogAdapter}.
35+
*/
36+
@Suite
37+
@SuiteDisplayName("Iceberg REST Compatibility Kit")
38+
@SelectClasses({RESTCompatibilityKitCatalogTests.class, RESTCompatibilityKitViewCatalogTests.class})
39+
public class RESTCompatibilityKitSuite {
40+
static final String RCK_REQUIRES_NAMESPACE_CREATE = "rck.requires-namespace-create";
41+
static final String RCK_SUPPORTS_SERVERSIDE_RETRY = "rck.supports-serverside-retry";
42+
static final String RCK_OVERRIDES_REQUESTED_LOCATION = "rck.overrides-requested-location";
43+
44+
protected RESTCompatibilityKitSuite() {}
45+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.iceberg.rest;
20+
21+
import static org.assertj.core.api.Assertions.assertThat;
22+
23+
import org.apache.iceberg.catalog.Catalog;
24+
import org.apache.iceberg.util.PropertyUtil;
25+
import org.apache.iceberg.view.ViewCatalogTests;
26+
import org.junit.jupiter.api.AfterAll;
27+
import org.junit.jupiter.api.BeforeAll;
28+
import org.junit.jupiter.api.BeforeEach;
29+
import org.junit.jupiter.api.extension.ExtendWith;
30+
import org.slf4j.Logger;
31+
import org.slf4j.LoggerFactory;
32+
33+
@ExtendWith(RESTServerExtension.class)
34+
public class RESTCompatibilityKitViewCatalogTests extends ViewCatalogTests<RESTCatalog> {
35+
private static final Logger LOG =
36+
LoggerFactory.getLogger(RESTCompatibilityKitViewCatalogTests.class);
37+
private static RESTCatalog restCatalog;
38+
39+
@BeforeAll
40+
static void beforeClass() throws Exception {
41+
restCatalog = RCKUtils.initCatalogClient();
42+
43+
assertThat(restCatalog.listNamespaces())
44+
.withFailMessage("Namespaces list should not contain: %s", RCKUtils.TEST_NAMESPACES)
45+
.doesNotContainAnyElementsOf(RCKUtils.TEST_NAMESPACES);
46+
}
47+
48+
@BeforeEach
49+
void before() {
50+
try {
51+
RCKUtils.purgeCatalogTestEntries(restCatalog);
52+
} catch (Exception e) {
53+
LOG.warn("Failure during test setup", e);
54+
}
55+
}
56+
57+
@AfterAll
58+
static void afterClass() throws Exception {
59+
restCatalog.close();
60+
}
61+
62+
@Override
63+
protected RESTCatalog catalog() {
64+
return restCatalog;
65+
}
66+
67+
@Override
68+
protected Catalog tableCatalog() {
69+
return restCatalog;
70+
}
71+
72+
@Override
73+
protected boolean requiresNamespaceCreate() {
74+
return PropertyUtil.propertyAsBoolean(
75+
restCatalog.properties(), RESTCompatibilityKitSuite.RCK_REQUIRES_NAMESPACE_CREATE, true);
76+
}
77+
78+
@Override
79+
protected boolean supportsServerSideRetry() {
80+
return PropertyUtil.propertyAsBoolean(
81+
restCatalog.properties(), RESTCompatibilityKitSuite.RCK_SUPPORTS_SERVERSIDE_RETRY, true);
82+
}
83+
84+
@Override
85+
protected boolean overridesRequestedLocation() {
86+
return PropertyUtil.propertyAsBoolean(
87+
restCatalog.properties(),
88+
RESTCompatibilityKitSuite.RCK_OVERRIDES_REQUESTED_LOCATION,
89+
false);
90+
}
91+
}

0 commit comments

Comments
 (0)