Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[AMORO-2296] Add cache for loading tables in core and trino #2298

Merged
merged 8 commits into from
Nov 23, 2023
Merged

[AMORO-2296] Add cache for loading tables in core and trino #2298

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
20821cc
add cache for loading tables
HuangFru Nov 13, 2023
257af14
fix checkstyle
HuangFru Nov 13, 2023
181f9c6
implement caching catalog in BasicMixedIcebergCatalog
HuangFru Nov 14, 2023
e06b3fe
extract namespace catalog
HuangFru Nov 14, 2023
1258b08
extract namespace catalog
HuangFru Nov 14, 2023
52db1aa
Merge remote-tracking branch 'upstream/master' into trino-cache4load
HuangFru Nov 16, 2023
78c62f9
resolve conflict
HuangFru Nov 16, 2023
611c4de
resolve conflict
HuangFru Nov 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 35 additions & 3 deletions core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
import com.netease.arctic.table.TableProperties;
import com.netease.arctic.table.blocker.BasicTableBlockerManager;
import com.netease.arctic.table.blocker.TableBlockerManager;
import org.apache.iceberg.CachingCatalog;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.SortOrder;
Expand All @@ -45,6 +47,7 @@
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
import org.apache.iceberg.util.PropertyUtil;

import java.util.List;
import java.util.Map;
Expand All @@ -55,6 +58,7 @@
public class BasicMixedIcebergCatalog implements ArcticCatalog {

private Catalog icebergCatalog;
private SupportsNamespaces asNamespaceCatalog;
private TableMetaStore tableMetaStore;
private Map<String, String> catalogProperties;
private String name;
Expand All @@ -69,6 +73,28 @@

@Override
public void initialize(String name, Map<String, String> properties, TableMetaStore metaStore) {
boolean cacheEnabled =
PropertyUtil.propertyAsBoolean(
properties, CatalogProperties.CACHE_ENABLED, CatalogProperties.CACHE_ENABLED_DEFAULT);

boolean cacheCaseSensitive =
PropertyUtil.propertyAsBoolean(
properties,
CatalogProperties.CACHE_CASE_SENSITIVE,
CatalogProperties.CACHE_CASE_SENSITIVE_DEFAULT);

long cacheExpirationIntervalMs =
PropertyUtil.propertyAsLong(
properties,
CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS,
CatalogProperties.CACHE_EXPIRATION_INTERVAL_MS_DEFAULT);

// An expiration interval of 0ms effectively disables caching.
// Do not wrap with CachingCatalog.
if (cacheExpirationIntervalMs == 0) {
cacheEnabled = false;

Check warning on line 95 in core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java

View check run for this annotation

Codecov / codecov/patch

core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java#L95 

Added line #L95 was not covered by tests
}

Catalog icebergCatalog =
metaStore.doAs(
() ->
Expand All @@ -84,7 +110,13 @@
synchronized (this) {
this.name = name;
this.tableMetaStore = metaStore;
this.icebergCatalog = icebergCatalog;
this.icebergCatalog =
cacheEnabled
? CachingCatalog.wrap(icebergCatalog, cacheCaseSensitive, cacheExpirationIntervalMs)
: icebergCatalog;
if (icebergCatalog instanceof SupportsNamespaces) {
this.asNamespaceCatalog = (SupportsNamespaces) icebergCatalog;
}
this.databaseFilterPattern = databaseFilterPattern;
this.catalogProperties = properties;
this.tables = tables;
Expand Down Expand Up @@ -224,13 +256,13 @@
}

private SupportsNamespaces asNamespaceCatalog() {
if (!(icebergCatalog instanceof SupportsNamespaces)) {
if (asNamespaceCatalog == null) {
throw new UnsupportedOperationException(
String.format(
"Iceberg catalog: %s doesn't implement SupportsNamespaces",
icebergCatalog.getClass().getName()));
}
return (SupportsNamespaces) icebergCatalog;
return asNamespaceCatalog;
}

private class MixedIcebergTableBuilder implements TableBuilder {
Expand Down
11 changes: 7 additions & 4 deletions trino/src/main/java/com/netease/arctic/trino/ArcticConnectorMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
import java.util.Map;
import java.util.Optional;
import java.util.OptionalLong;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;

/**
Expand All @@ -81,6 +82,8 @@ public class ArcticConnectorMetadata implements ConnectorMetadata {

private final ArcticCatalog arcticCatalog;

private final Map<SchemaTableName, ArcticTable> tableCache = new ConcurrentHashMap<>();

public ArcticConnectorMetadata(
HuangFru marked this conversation as resolved.
Show resolved Hide resolved
KeyedConnectorMetadata keyedConnectorMetadata,
IcebergMetadata icebergMetadata,
Expand All @@ -99,8 +102,7 @@ public List<String> listSchemaNames(ConnectorSession session) {

@Override
public ConnectorTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) {
// 需要缓存
ArcticTable arcticTable = null;
ArcticTable arcticTable;
try {
arcticTable = getArcticTable(tableName);
} catch (NoSuchTableException e) {
Expand Down Expand Up @@ -169,7 +171,7 @@ public ColumnMetadata getColumnMetadata(
public Iterator<TableColumnsMetadata> streamTableColumns(
ConnectorSession session, SchemaTablePrefix prefix) {
if (prefix.getTable().isPresent()) {
ArcticTable arcticTable = null;
ArcticTable arcticTable;
try {
arcticTable =
getArcticTable(new SchemaTableName(prefix.getSchema().get(), prefix.getTable().get()));
Expand Down Expand Up @@ -452,7 +454,8 @@ public void rollback() {
}

public ArcticTable getArcticTable(SchemaTableName schemaTableName) {
return arcticCatalog.loadTable(getTableIdentifier(schemaTableName));
return tableCache.computeIfAbsent(
schemaTableName, ignore -> arcticCatalog.loadTable(getTableIdentifier(schemaTableName)));
}

private TableIdentifier getTableIdentifier(SchemaTableName schemaTableName) {
Expand Down
Loading