Skip to content

Commit a5552a1

Browse files
eric-maynarderic-maynard
authored
Implement GenericTableCatalog (#1231)
* add missing apis * more tests, fixes * clean up drop * autolint * changes per review * revert iceberg messages to comply with oss tests * another revert * more iceberg catalog changes * autolint * wip * refactor to subtype * autolint * rebase * add another assert * autolint * add another best effort check * autolint * reduce metastore trips * autolint
1 parent 94e2676 commit a5552a1

File tree

24 files changed

+616
-270
lines changed

24 files changed

+616
-270
lines changed

‎polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntitySubType.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,9 @@ public enum PolarisEntitySubType {
3030
ANY_SUBTYPE(-1, null),
3131
// the NULL value is used when an entity has no subtype, i.e. NOT_APPLICABLE really
3232
NULL_SUBTYPE(0, null),
33-
TABLE(2, PolarisEntityType.ICEBERG_TABLE_LIKE),
34-
VIEW(3, PolarisEntityType.ICEBERG_TABLE_LIKE);
33+
ICEBERG_TABLE(2, PolarisEntityType.TABLE_LIKE),
34+
ICEBERG_VIEW(3, PolarisEntityType.TABLE_LIKE),
35+
GENERIC_TABLE(4, PolarisEntityType.TABLE_LIKE);
3536

3637
// to efficiently map the code of a subtype to its corresponding subtype enum, use a reverse
3738
// array which is initialized below

‎polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,10 @@ public enum PolarisEntityType {
3131
CATALOG(4, ROOT, false, false),
3232
CATALOG_ROLE(5, CATALOG, true, false),
3333
NAMESPACE(6, CATALOG, false, true),
34-
// generic table is either a view or a real table
35-
ICEBERG_TABLE_LIKE(7, NAMESPACE, false, false),
34+
TABLE_LIKE(7, NAMESPACE, false, false),
3635
TASK(8, ROOT, false, false),
37-
FILE(9, ICEBERG_TABLE_LIKE, false, false),
38-
POLICY(10, NAMESPACE, false, false),
39-
GENERIC_TABLE(11, NAMESPACE, false, false);
36+
FILE(9, TABLE_LIKE, false, false),
37+
POLICY(10, NAMESPACE, false, false);
4038

4139
// to efficiently map a code to its corresponding entity type, use a reverse array which
4240
// is initialized below

‎polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -41,22 +41,22 @@ public enum PolarisPrivilege {
4141
TABLE_CREATE(6, PolarisEntityType.NAMESPACE),
4242
VIEW_CREATE(7, PolarisEntityType.NAMESPACE),
4343
NAMESPACE_DROP(8, PolarisEntityType.NAMESPACE),
44-
TABLE_DROP(9, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE),
45-
VIEW_DROP(10, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW),
44+
TABLE_DROP(9, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE),
45+
VIEW_DROP(10, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_VIEW),
4646
NAMESPACE_LIST(11, PolarisEntityType.NAMESPACE),
4747
TABLE_LIST(12, PolarisEntityType.NAMESPACE),
4848
VIEW_LIST(13, PolarisEntityType.NAMESPACE),
4949
NAMESPACE_READ_PROPERTIES(14, PolarisEntityType.NAMESPACE),
50-
TABLE_READ_PROPERTIES(15, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE),
51-
VIEW_READ_PROPERTIES(16, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW),
50+
TABLE_READ_PROPERTIES(15, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE),
51+
VIEW_READ_PROPERTIES(16, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_VIEW),
5252
NAMESPACE_WRITE_PROPERTIES(17, PolarisEntityType.NAMESPACE),
53-
TABLE_WRITE_PROPERTIES(18, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE),
54-
VIEW_WRITE_PROPERTIES(19, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW),
55-
TABLE_READ_DATA(20, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE),
56-
TABLE_WRITE_DATA(21, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE),
53+
TABLE_WRITE_PROPERTIES(18, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE),
54+
VIEW_WRITE_PROPERTIES(19, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_VIEW),
55+
TABLE_READ_DATA(20, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE),
56+
TABLE_WRITE_DATA(21, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE),
5757
NAMESPACE_FULL_METADATA(22, PolarisEntityType.NAMESPACE),
58-
TABLE_FULL_METADATA(23, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE),
59-
VIEW_FULL_METADATA(24, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW),
58+
TABLE_FULL_METADATA(23, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE),
59+
VIEW_FULL_METADATA(24, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_VIEW),
6060
CATALOG_CREATE(25, PolarisEntityType.ROOT),
6161
CATALOG_DROP(26, PolarisEntityType.CATALOG),
6262
CATALOG_LIST(27, PolarisEntityType.ROOT),
@@ -70,14 +70,14 @@ public enum PolarisPrivilege {
7070
CATALOG_ROLE_LIST_GRANTS(35, PolarisEntityType.PRINCIPAL),
7171
CATALOG_LIST_GRANTS(36, PolarisEntityType.CATALOG),
7272
NAMESPACE_LIST_GRANTS(37, PolarisEntityType.NAMESPACE),
73-
TABLE_LIST_GRANTS(38, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE),
74-
VIEW_LIST_GRANTS(39, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW),
73+
TABLE_LIST_GRANTS(38, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE),
74+
VIEW_LIST_GRANTS(39, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_VIEW),
7575
CATALOG_MANAGE_GRANTS_ON_SECURABLE(40, PolarisEntityType.CATALOG),
7676
NAMESPACE_MANAGE_GRANTS_ON_SECURABLE(41, PolarisEntityType.NAMESPACE),
7777
TABLE_MANAGE_GRANTS_ON_SECURABLE(
78-
42, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE),
78+
42, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE),
7979
VIEW_MANAGE_GRANTS_ON_SECURABLE(
80-
43, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW),
80+
43, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_VIEW),
8181
PRINCIPAL_CREATE(44, PolarisEntityType.ROOT),
8282
PRINCIPAL_DROP(45, PolarisEntityType.PRINCIPAL),
8383
PRINCIPAL_LIST(46, PolarisEntityType.ROOT),

‎polaris-core/src/main/java/org/apache/polaris/core/entity/GenericTableEntity.java‎ renamed to ‎polaris-core/src/main/java/org/apache/polaris/core/entity/table/GenericTableEntity.java‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,23 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19-
package org.apache.polaris.core.entity;
19+
package org.apache.polaris.core.entity.table;
2020

2121
import com.fasterxml.jackson.annotation.JsonIgnore;
2222
import org.apache.iceberg.catalog.Namespace;
2323
import org.apache.iceberg.catalog.TableIdentifier;
2424
import org.apache.iceberg.rest.RESTUtil;
25+
import org.apache.polaris.core.entity.NamespaceEntity;
26+
import org.apache.polaris.core.entity.PolarisBaseEntity;
27+
import org.apache.polaris.core.entity.PolarisEntity;
28+
import org.apache.polaris.core.entity.PolarisEntitySubType;
29+
import org.apache.polaris.core.entity.PolarisEntityType;
2530

2631
/**
27-
* A {@link PolarisEntity} implementation for generic tables. These tables are not Iceberg-like in
28-
* that they may not have a schema or base location. Similarly to {@link IcebergTableLikeEntity}
29-
* however, these tables have an identifier and a parent namespace.
32+
* A {@link TableLikeEntity} implementation for generic tables. These tables are not Iceberg-like in
33+
* that they may not have a schema or base location.
3034
*/
31-
public class GenericTableEntity extends PolarisEntity {
35+
public class GenericTableEntity extends TableLikeEntity {
3236

3337
public static final String FORMAT_KEY = "format";
3438

@@ -52,7 +56,8 @@ public static class Builder
5256
extends PolarisEntity.BaseBuilder<GenericTableEntity, GenericTableEntity.Builder> {
5357
public Builder(TableIdentifier tableIdentifier, String format) {
5458
super();
55-
setType(PolarisEntityType.GENERIC_TABLE);
59+
setType(PolarisEntityType.TABLE_LIKE);
60+
setSubType(PolarisEntitySubType.GENERIC_TABLE);
5661
setTableIdentifier(tableIdentifier);
5762
setFormat(format);
5863
}

‎polaris-core/src/main/java/org/apache/polaris/core/entity/IcebergTableLikeEntity.java‎ renamed to ‎polaris-core/src/main/java/org/apache/polaris/core/entity/table/IcebergTableLikeEntity.java‎

Lines changed: 13 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,24 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19-
package org.apache.polaris.core.entity;
19+
package org.apache.polaris.core.entity.table;
2020

2121
import com.fasterxml.jackson.annotation.JsonIgnore;
2222
import java.util.Optional;
2323
import org.apache.iceberg.catalog.Namespace;
2424
import org.apache.iceberg.catalog.TableIdentifier;
2525
import org.apache.iceberg.rest.RESTUtil;
26-
27-
public class IcebergTableLikeEntity extends PolarisEntity {
26+
import org.apache.polaris.core.entity.NamespaceEntity;
27+
import org.apache.polaris.core.entity.PolarisBaseEntity;
28+
import org.apache.polaris.core.entity.PolarisEntity;
29+
import org.apache.polaris.core.entity.PolarisEntityConstants;
30+
import org.apache.polaris.core.entity.PolarisEntityType;
31+
32+
/**
33+
* An entity type for {@link TableLikeEntity} instances that conform to iceberg semantics around
34+
* locations. This includes both Iceberg tables and Iceberg views.
35+
*/
36+
public class IcebergTableLikeEntity extends TableLikeEntity {
2837
// For applicable types, this key on the "internalProperties" map will return the location
2938
// of the internalProperties JSON file.
3039
public static final String METADATA_LOCATION_KEY = "metadata-location";
@@ -46,22 +55,6 @@ public static IcebergTableLikeEntity of(PolarisBaseEntity sourceEntity) {
4655
return null;
4756
}
4857

49-
@JsonIgnore
50-
public TableIdentifier getTableIdentifier() {
51-
Namespace parent = getParentNamespace();
52-
return TableIdentifier.of(parent, getName());
53-
}
54-
55-
@JsonIgnore
56-
public Namespace getParentNamespace() {
57-
String encodedNamespace =
58-
getInternalPropertiesAsMap().get(NamespaceEntity.PARENT_NAMESPACE_KEY);
59-
if (encodedNamespace == null) {
60-
return Namespace.empty();
61-
}
62-
return RESTUtil.decodeNamespace(encodedNamespace);
63-
}
64-
6558
@JsonIgnore
6659
public String getMetadataLocation() {
6760
return getInternalPropertiesAsMap().get(METADATA_LOCATION_KEY);
@@ -82,7 +75,7 @@ public String getBaseLocation() {
8275
public static class Builder extends PolarisEntity.BaseBuilder<IcebergTableLikeEntity, Builder> {
8376
public Builder(TableIdentifier identifier, String metadataLocation) {
8477
super();
85-
setType(PolarisEntityType.ICEBERG_TABLE_LIKE);
78+
setType(PolarisEntityType.TABLE_LIKE);
8679
setTableIdentifier(identifier);
8780
setMetadataLocation(metadataLocation);
8881
}

‎polaris-core/src/main/java/org/apache/polaris/core/entity/table/TableLikeEntity.java‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.core.entity.table;
20+
21+
import com.fasterxml.jackson.annotation.JsonIgnore;
22+
import jakarta.annotation.Nonnull;
23+
import org.apache.iceberg.catalog.Namespace;
24+
import org.apache.iceberg.catalog.TableIdentifier;
25+
import org.apache.iceberg.rest.RESTUtil;
26+
import org.apache.polaris.core.entity.NamespaceEntity;
27+
import org.apache.polaris.core.entity.PolarisBaseEntity;
28+
import org.apache.polaris.core.entity.PolarisEntity;
29+
import org.apache.polaris.core.entity.PolarisEntityType;
30+
31+
/**
32+
* An entity type for all table-like entities including Iceberg tables, Iceberg views, and generic
33+
* tables. This entity maps to {@link PolarisEntityType#TABLE_LIKE}
34+
*/
35+
public abstract class TableLikeEntity extends PolarisEntity {
36+
37+
public TableLikeEntity(@Nonnull PolarisBaseEntity sourceEntity) {
38+
super(sourceEntity);
39+
}
40+
41+
@JsonIgnore
42+
public TableIdentifier getTableIdentifier() {
43+
Namespace parent = getParentNamespace();
44+
return TableIdentifier.of(parent, getName());
45+
}
46+
47+
@JsonIgnore
48+
public Namespace getParentNamespace() {
49+
String encodedNamespace =
50+
getInternalPropertiesAsMap().get(NamespaceEntity.PARENT_NAMESPACE_KEY);
51+
if (encodedNamespace == null) {
52+
return Namespace.empty();
53+
}
54+
return RESTUtil.decodeNamespace(encodedNamespace);
55+
}
56+
}

‎polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -373,8 +373,7 @@ public ResolverStatus resolveAll() {
373373

374374
// validate input
375375
diagnostics.check(
376-
entityType != PolarisEntityType.NAMESPACE
377-
&& entityType != PolarisEntityType.ICEBERG_TABLE_LIKE,
376+
entityType != PolarisEntityType.NAMESPACE && entityType != PolarisEntityType.TABLE_LIKE,
378377
"cannot_be_path");
379378
diagnostics.check(
380379
entityType.isTopLevel() || this.referenceCatalogName != null, "reference_catalog_expected");

‎polaris-core/src/main/java/org/apache/polaris/core/policy/validator/datacompaction/DataCompactionPolicyValidator.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919
package org.apache.polaris.core.policy.validator.datacompaction;
2020

2121
import static org.apache.polaris.core.entity.PolarisEntityType.CATALOG;
22-
import static org.apache.polaris.core.entity.PolarisEntityType.ICEBERG_TABLE_LIKE;
2322
import static org.apache.polaris.core.entity.PolarisEntityType.NAMESPACE;
23+
import static org.apache.polaris.core.entity.PolarisEntityType.TABLE_LIKE;
2424

2525
import java.util.Set;
2626
import org.apache.polaris.core.entity.PolarisEntitySubType;
@@ -32,7 +32,7 @@ public class DataCompactionPolicyValidator implements PolicyValidator {
3232
public static final DataCompactionPolicyValidator INSTANCE = new DataCompactionPolicyValidator();
3333

3434
private static final Set<PolarisEntityType> ATTACHABLE_ENTITY_TYPES =
35-
Set.of(CATALOG, NAMESPACE, ICEBERG_TABLE_LIKE);
35+
Set.of(CATALOG, NAMESPACE, TABLE_LIKE);
3636

3737
@Override
3838
public void validate(String content) throws InvalidPolicyException {
@@ -49,7 +49,7 @@ public boolean canAttach(PolarisEntityType entityType, PolarisEntitySubType enti
4949
return false;
5050
}
5151

52-
if (entityType == ICEBERG_TABLE_LIKE && entitySubType != PolarisEntitySubType.TABLE) {
52+
if (entityType == TABLE_LIKE && entitySubType != PolarisEntitySubType.ICEBERG_TABLE) {
5353
return false;
5454
}
5555

‎polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,9 @@
4141
import org.apache.polaris.core.config.FeatureConfiguration;
4242
import org.apache.polaris.core.context.CallContext;
4343
import org.apache.polaris.core.entity.CatalogEntity;
44-
import org.apache.polaris.core.entity.IcebergTableLikeEntity;
4544
import org.apache.polaris.core.entity.PolarisEntity;
4645
import org.apache.polaris.core.entity.PolarisEntityConstants;
46+
import org.apache.polaris.core.entity.table.IcebergTableLikeEntity;
4747
import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo;
4848
import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo;
4949
import org.apache.polaris.core.storage.gcp.GcpStorageConfigurationInfo;

‎polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ public Map<String, String> getIfPresent(StorageCredentialCacheKey key) {
154154
private boolean isTypeSupported(PolarisEntityType type) {
155155
return type == PolarisEntityType.CATALOG
156156
|| type == PolarisEntityType.NAMESPACE
157-
|| type == PolarisEntityType.ICEBERG_TABLE_LIKE
157+
|| type == PolarisEntityType.TABLE_LIKE
158158
|| type == PolarisEntityType.TASK;
159159
}
160160

0 commit comments

Comments
 (0)