Skip to content

Commit f042880

Browse files
adutraadutra
authored
Upgrade Polaris to Iceberg 1.9.0 (#1309)
1 parent 52ab374 commit f042880

File tree

11 files changed

+88
-50
lines changed

11 files changed

+88
-50
lines changed

‎gradle/libs.versions.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
[versions]
2121
hadoop = "3.4.1"
22-
iceberg = "1.8.1" # Ensure to update the iceberg version in regtests to keep regtests up-to-date
22+
iceberg = "1.9.0" # Ensure to update the iceberg version in regtests to keep regtests up-to-date
2323
quarkus = "3.21.2"
2424
immutables = "2.10.1"
2525
picocli = "4.7.7"

‎integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@
2020

2121
import com.google.common.collect.ImmutableMap;
2222
import java.util.Map;
23-
import org.apache.iceberg.catalog.SessionCatalog;
24-
import org.apache.iceberg.rest.HTTPClient;
2523
import org.apache.iceberg.rest.RESTCatalog;
2624
import org.apache.iceberg.rest.auth.OAuth2Properties;
2725
import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
@@ -36,14 +34,7 @@ public static RESTCatalog restCatalog(
3634
String catalog,
3735
Map<String, String> extraProperties) {
3836
String authToken = client.obtainToken(credentials);
39-
SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty();
40-
RESTCatalog restCatalog =
41-
new RESTCatalog(
42-
context,
43-
(config) ->
44-
HTTPClient.builder(config)
45-
.uri(config.get(org.apache.iceberg.CatalogProperties.URI))
46-
.build());
37+
RESTCatalog restCatalog = new RESTCatalog();
4738

4839
ImmutableMap.Builder<String, String> propertiesBuilder =
4940
ImmutableMap.<String, String>builder()

‎integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -371,8 +371,8 @@ public void testIcebergCreateTablesInExternalCatalog() throws IOException {
371371
TableIdentifier.of(ns, "the_table"),
372372
new Schema(
373373
List.of(
374-
Types.NestedField.of(
375-
1, false, "theField", Types.StringType.get()))))
374+
Types.NestedField.required(
375+
1, "theField", Types.StringType.get()))))
376376
.withLocation("file:///tmp/tables")
377377
.withSortOrder(SortOrder.unsorted())
378378
.withPartitionSpec(PartitionSpec.unpartitioned())
@@ -399,8 +399,8 @@ public void testIcebergCreateTablesWithWritePathBlocked() throws IOException {
399399
TableIdentifier.of(ns, "the_table"),
400400
new Schema(
401401
List.of(
402-
Types.NestedField.of(
403-
1, false, "theField", Types.StringType.get()))))
402+
Types.NestedField.required(
403+
1, "theField", Types.StringType.get()))))
404404
.withSortOrder(SortOrder.unsorted())
405405
.withPartitionSpec(PartitionSpec.unpartitioned())
406406
.withProperties(Map.of("write.data.path", "s3://my-bucket/path/to/data"))
@@ -416,8 +416,8 @@ public void testIcebergCreateTablesWithWritePathBlocked() throws IOException {
416416
TableIdentifier.of(ns, "the_table"),
417417
new Schema(
418418
List.of(
419-
Types.NestedField.of(
420-
1, false, "theField", Types.StringType.get()))))
419+
Types.NestedField.required(
420+
1, "theField", Types.StringType.get()))))
421421
.withSortOrder(SortOrder.unsorted())
422422
.withPartitionSpec(PartitionSpec.unpartitioned())
423423
.withProperties(Map.of("write.metadata.path", "s3://my-bucket/path/to/data"))
@@ -454,7 +454,7 @@ public void testIcebergRegisterTableInExternalCatalog() throws IOException {
454454
.assignUUID()
455455
.addPartitionSpec(PartitionSpec.unpartitioned())
456456
.addSortOrder(SortOrder.unsorted())
457-
.addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())))
457+
.addSchema(new Schema(Types.NestedField.required(1, "col1", Types.StringType.get())))
458458
.build();
459459
TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation));
460460

@@ -490,7 +490,7 @@ public void testIcebergUpdateTableInExternalCatalog() throws IOException {
490490
String location = baseLocation.resolve("testIcebergUpdateTableInExternalCatalog").toString();
491491
String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json";
492492

493-
Types.NestedField col1 = Types.NestedField.of(1, false, "col1", Types.StringType.get());
493+
Types.NestedField col1 = Types.NestedField.required(1, "col1", Types.StringType.get());
494494
TableMetadata tableMetadata =
495495
TableMetadata.buildFromEmpty()
496496
.setLocation(location)
@@ -545,7 +545,7 @@ public void testIcebergDropTableInExternalCatalog() throws IOException {
545545
.assignUUID()
546546
.addPartitionSpec(PartitionSpec.unpartitioned())
547547
.addSortOrder(SortOrder.unsorted())
548-
.addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())))
548+
.addSchema(new Schema(Types.NestedField.required(1, "col1", Types.StringType.get())))
549549
.build();
550550
TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation));
551551

‎integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,19 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog>
148148
private final String catalogBaseLocation =
149149
s3BucketBase + "/" + System.getenv("USER") + "/path/to/data";
150150

151+
private static final Map<String, String> DEFAULT_REST_CATALOG_CONFIG =
152+
Map.of(
153+
org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
154+
"catalog-default-key1",
155+
org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
156+
"catalog-default-key2",
157+
org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
158+
"catalog-default-key3",
159+
org.apache.iceberg.CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
160+
"catalog-override-key3",
161+
org.apache.iceberg.CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
162+
"catalog-override-key4");
163+
151164
private static final String[] DEFAULT_CATALOG_PROPERTIES = {
152165
"allow.unstructured.table.location", "true",
153166
"allow.external.table.location", "true"
@@ -235,7 +248,7 @@ public void before(TestInfo testInfo) {
235248

236249
managementApi.createCatalog(principalRoleName, catalog);
237250

238-
restCatalogConfig =
251+
Map<String, String> dynamicConfig =
239252
testInfo
240253
.getTestMethod()
241254
.map(m -> m.getAnnotation(RestCatalogConfig.class))
@@ -254,6 +267,12 @@ public void before(TestInfo testInfo) {
254267
})
255268
.orElse(ImmutableMap.of());
256269

270+
restCatalogConfig =
271+
ImmutableMap.<String, String>builder()
272+
.putAll(DEFAULT_REST_CATALOG_CONFIG)
273+
.putAll(dynamicConfig)
274+
.build();
275+
257276
restCatalog = initCatalog(currentCatalogName, ImmutableMap.of());
258277
}
259278

@@ -581,7 +600,7 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisable
581600
restCatalog.createNamespace(ns1);
582601
TableMetadata tableMetadata =
583602
TableMetadata.newTableMetadata(
584-
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
603+
new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))),
585604
PartitionSpec.unpartitioned(),
586605
externalCatalogBase + "/ns1/my_table",
587606
Map.of());
@@ -616,7 +635,7 @@ public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisa
616635
restCatalog.createNamespace(ns1);
617636
TableMetadata tableMetadata =
618637
TableMetadata.newTableMetadata(
619-
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
638+
new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))),
620639
PartitionSpec.unpartitioned(),
621640
externalCatalogBase + "/ns1/my_table",
622641
Map.of());
@@ -650,7 +669,7 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigEnabled
650669
restCatalog.createNamespace(ns1);
651670
TableMetadata tableMetadata =
652671
TableMetadata.newTableMetadata(
653-
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
672+
new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))),
654673
PartitionSpec.unpartitioned(),
655674
externalCatalogBase + "/ns1/my_table",
656675
Map.of());
@@ -681,7 +700,7 @@ public void testLoadTableTwiceWithETag() {
681700
restCatalog.createNamespace(ns1);
682701
TableMetadata tableMetadata =
683702
TableMetadata.newTableMetadata(
684-
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
703+
new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))),
685704
PartitionSpec.unpartitioned(),
686705
"file:///tmp/ns1/my_table",
687706
Map.of());
@@ -728,7 +747,7 @@ public void testRegisterAndLoadTableWithReturnedETag() {
728747
restCatalog.createNamespace(ns1);
729748
TableMetadata tableMetadata =
730749
TableMetadata.newTableMetadata(
731-
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
750+
new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))),
732751
PartitionSpec.unpartitioned(),
733752
"file:///tmp/ns1/my_table",
734753
Map.of());
@@ -774,7 +793,7 @@ public void testCreateAndLoadTableWithReturnedEtag() {
774793
restCatalog.createNamespace(ns1);
775794
TableMetadata tableMetadata =
776795
TableMetadata.newTableMetadata(
777-
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
796+
new Schema(List.of(Types.NestedField.required(1, "col1", new Types.StringType()))),
778797
PartitionSpec.unpartitioned(),
779798
"file:///tmp/ns1/my_table",
780799
Map.of());

‎integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,16 @@ public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogT
6262
Assumptions.setPreferredAssumptionException(PreferredAssumptionException.JUNIT5);
6363
}
6464

65+
public static Map<String, String> DEFAULT_REST_CATALOG_CONFIG =
66+
Map.of(
67+
org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1",
68+
org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2",
69+
org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key3", "catalog-default-key3",
70+
org.apache.iceberg.CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3",
71+
"catalog-override-key3",
72+
org.apache.iceberg.CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4",
73+
"catalog-override-key4");
74+
6575
private static ClientCredentials adminCredentials;
6676
private static PolarisApiEndpoints endpoints;
6777
private static PolarisClient client;
@@ -120,15 +130,7 @@ public void before(TestInfo testInfo) {
120130

121131
restCatalog =
122132
IcebergHelper.restCatalog(
123-
client,
124-
endpoints,
125-
principalCredentials,
126-
catalogName,
127-
Map.of(
128-
org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key1",
129-
"catalog-default-key1",
130-
org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key2",
131-
"catalog-default-key2"));
133+
client, endpoints, principalCredentials, catalogName, DEFAULT_REST_CATALOG_CONFIG);
132134
}
133135

134136
@AfterEach

‎plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkCatalogBaseIT.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import java.util.Arrays;
2828
import java.util.Map;
2929
import org.apache.iceberg.exceptions.BadRequestException;
30+
import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
3031
import org.apache.iceberg.spark.SupportsReplaceView;
3132
import org.apache.spark.sql.catalyst.analysis.NoSuchNamespaceException;
3233
import org.apache.spark.sql.catalyst.analysis.NoSuchViewException;
@@ -107,7 +108,10 @@ void testNamespaceOperations() throws Exception {
107108

108109
// directly drop lv1ns[0] should fail
109110
assertThatThrownBy(() -> namespaceCatalog.dropNamespace(lv1ns[0], true))
110-
.isInstanceOf(BadRequestException.class);
111+
.isInstanceOfAny(
112+
BadRequestException.class, // Iceberg < 1.9.0
113+
NamespaceNotEmptyException.class // Iceberg >= 1.9.0
114+
);
111115
for (String[] namespace : lv2ns1) {
112116
namespaceCatalog.dropNamespace(namespace, true);
113117
}
@@ -249,7 +253,10 @@ void testListViews() throws Exception {
249253

250254
// drop namespace fails since there are views under it
251255
assertThatThrownBy(() -> namespaceCatalog.dropNamespace(l2ns, true))
252-
.isInstanceOf(BadRequestException.class);
256+
.isInstanceOfAny(
257+
BadRequestException.class, // Iceberg < 1.9.0
258+
NamespaceNotEmptyException.class // Iceberg >= 1.9.0
259+
);
253260
// drop the views
254261
for (String name : nsl2ViewNames) {
255262
viewCatalog.dropView(Identifier.of(l2ns, name));

‎quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,19 @@ public Map<String, String> getConfigOverrides() {
180180
public static final String SECRET_ACCESS_KEY = "secret_access_key";
181181
public static final String SESSION_TOKEN = "session_token";
182182

183+
public static Map<String, String> TABLE_PREFIXES =
184+
Map.of(
185+
CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1",
186+
"catalog-default-key1",
187+
CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2",
188+
"catalog-default-key2",
189+
CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3",
190+
"catalog-default-key3",
191+
CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3",
192+
"catalog-override-key3",
193+
CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4",
194+
"catalog-override-key4");
195+
183196
@Inject MetaStoreManagerFactory managerFactory;
184197
@Inject PolarisConfigurationStore configurationStore;
185198
@Inject PolarisStorageIntegrationProvider storageIntegrationProvider;
@@ -344,6 +357,7 @@ protected IcebergCatalog initCatalog(
344357
ImmutableMap.Builder<String, String> propertiesBuilder =
345358
ImmutableMap.<String, String>builder()
346359
.put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")
360+
.putAll(TABLE_PREFIXES)
347361
.putAll(additionalProperties);
348362
icebergCatalog.initialize(CATALOG_NAME, propertiesBuilder.buildKeepingLast());
349363
return icebergCatalog;

‎quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,14 @@ public Map<String, String> getConfigOverrides() {
104104

105105
public static final String CATALOG_NAME = "polaris-catalog";
106106

107+
public static Map<String, String> VIEW_PREFIXES =
108+
Map.of(
109+
CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1",
110+
CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2",
111+
CatalogProperties.VIEW_DEFAULT_PREFIX + "key3", "catalog-default-key3",
112+
CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3", "catalog-override-key3",
113+
CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4", "catalog-override-key4");
114+
107115
@Inject MetaStoreManagerFactory managerFactory;
108116
@Inject UserSecretsManagerFactory userSecretsManagerFactory;
109117
@Inject PolarisConfigurationStore configurationStore;
@@ -211,15 +219,12 @@ public void before(TestInfo testInfo) {
211219
securityContext,
212220
Mockito.mock(),
213221
fileIOFactory);
214-
this.catalog.initialize(
215-
CATALOG_NAME,
216-
ImmutableMap.of(
217-
CatalogProperties.FILE_IO_IMPL,
218-
"org.apache.iceberg.inmemory.InMemoryFileIO",
219-
CatalogProperties.VIEW_DEFAULT_PREFIX + "key1",
220-
"catalog-default-key1",
221-
CatalogProperties.VIEW_DEFAULT_PREFIX + "key2",
222-
"catalog-default-key2"));
222+
Map<String, String> properties =
223+
ImmutableMap.<String, String>builder()
224+
.put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")
225+
.putAll(VIEW_PREFIXES)
226+
.build();
227+
this.catalog.initialize(CATALOG_NAME, properties);
223228
}
224229

225230
@AfterEach

‎quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ static TableMetadata writeTableMetadata(
100100
tmBuilder
101101
.setLocation("path/to/table")
102102
.addSchema(
103-
new Schema(List.of(Types.NestedField.of(1, false, "field1", Types.StringType.get()))))
103+
new Schema(List.of(Types.NestedField.required(1, "field1", Types.StringType.get()))))
104104
.addSortOrder(SortOrder.unsorted())
105105
.assignUUID(UUID.randomUUID().toString())
106106
.addPartitionSpec(PartitionSpec.unpartitioned());

‎regtests/setup.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ if [ -z "${SPARK_HOME}" ]; then
3131
fi
3232
SPARK_CONF="${SPARK_HOME}/conf/spark-defaults.conf"
3333
DERBY_HOME="/tmp/derby"
34-
ICEBERG_VERSION="1.8.1"
34+
ICEBERG_VERSION="1.9.0"
3535
export PYTHONPATH="${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.9.7-src.zip:$PYTHONPATH"
3636

3737
# Ensure binaries are downloaded locally

0 commit comments

Comments
 (0)