From 3600570f1c43016b95f3e56cbe9b00cbc942703a Mon Sep 17 00:00:00 2001 From: cell Date: Thu, 31 Aug 2023 13:46:07 +0800 Subject: [PATCH] [AMORO-1870]: Support table filters to make AMS ignore tables that are not needed (#1879) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [AMORO-1870]: table filter #1870 \n * add table filter * fix check style * fix gitignore * fix db table filter * fix table filter * Update managing-catalogs.md and change docs * fix checkstyle * fix change table filter key * change table filter md * Update managing-catalogs.md * Add Deprecated annotation * Update docs/admin-guides/managing-catalogs.md Co-authored-by: ZhouJinsong * Update ams/api/src/main/java/com/netease/arctic/ams/api/properties/CatalogMetaProperties.java Co-authored-by: ZhouJinsong * fix check style --------- Co-authored-by: 刘政 Co-authored-by: wangtaohz <103108928+wangtaohz@users.noreply.github.com> Co-authored-by: wangtao Co-authored-by: ZhouJinsong --- .../ams/api/properties/CatalogMetaProperties.java | 5 +++++ .../arctic/catalog/IcebergCatalogWrapper.java | 14 +++++++++++++- docs/admin-guides/managing-catalogs.md | 3 ++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/ams/api/src/main/java/com/netease/arctic/ams/api/properties/CatalogMetaProperties.java b/ams/api/src/main/java/com/netease/arctic/ams/api/properties/CatalogMetaProperties.java index ae18ba8783..20c05f2564 100644 --- a/ams/api/src/main/java/com/netease/arctic/ams/api/properties/CatalogMetaProperties.java +++ b/ams/api/src/main/java/com/netease/arctic/ams/api/properties/CatalogMetaProperties.java @@ -41,7 +41,12 @@ public class CatalogMetaProperties { @Deprecated public static final String KEY_WAREHOUSE_DIR = "warehouse.dir"; public static final String KEY_WAREHOUSE = "warehouse"; + /** + * @deprecated since 0.6.0, will be removed in 0.7.0; use {@link CatalogMetaProperties#KEY_TABLE_FILTER} instead. + */ + @Deprecated public static final String KEY_DATABASE_FILTER_REGULAR_EXPRESSION = "database.filter-regular-expression"; + public static final String KEY_TABLE_FILTER = "table-filter"; public static final String CATALOG_TYPE_HADOOP = "hadoop"; public static final String CATALOG_TYPE_HIVE = "hive"; diff --git a/core/src/main/java/com/netease/arctic/catalog/IcebergCatalogWrapper.java b/core/src/main/java/com/netease/arctic/catalog/IcebergCatalogWrapper.java index 78a3a45554..c6101f01b2 100644 --- a/core/src/main/java/com/netease/arctic/catalog/IcebergCatalogWrapper.java +++ b/core/src/main/java/com/netease/arctic/catalog/IcebergCatalogWrapper.java @@ -57,6 +57,7 @@ public class IcebergCatalogWrapper implements ArcticCatalog { private CatalogMeta meta; private Map customProperties; private Pattern databaseFilterPattern; + private Pattern tableFilterPattern; private transient TableMetaStore tableMetaStore; private transient Catalog icebergCatalog; @@ -90,6 +91,15 @@ private void initialize(CatalogMeta meta, Map properties) { } else { databaseFilterPattern = null; } + + if (meta.getCatalogProperties().containsKey(CatalogMetaProperties.KEY_TABLE_FILTER)) { + String tableFilter = + meta.getCatalogProperties().get(CatalogMetaProperties.KEY_TABLE_FILTER); + tableFilterPattern = Pattern.compile(tableFilter); + } else { + tableFilterPattern = null; + } + } public IcebergCatalogWrapper(CatalogMeta meta, Map properties) { @@ -148,7 +158,9 @@ public void dropDatabase(String databaseName) { @Override public List listTables(String database) { return tableMetaStore.doAs(() -> icebergCatalog.listTables(Namespace.of(database)).stream() - .filter(tableIdentifier -> tableIdentifier.namespace().levels().length == 1) + .filter(tableIdentifier -> tableIdentifier.namespace().levels().length == 1 && + (tableFilterPattern == null || + tableFilterPattern.matcher((database + "." + tableIdentifier.name())).matches())) .map(tableIdentifier -> TableIdentifier.of(name(), database, tableIdentifier.name())) .collect(Collectors.toList())); } diff --git a/docs/admin-guides/managing-catalogs.md b/docs/admin-guides/managing-catalogs.md index baf45aa713..25429cabeb 100644 --- a/docs/admin-guides/managing-catalogs.md +++ b/docs/admin-guides/managing-catalogs.md @@ -48,6 +48,7 @@ You can create a catalog in the AMS frontend: Common properties include: - warehouse: Warehouse **must be configured**, as it determines where our database and table files should be placed - catalog-impl: when the metastore is **Custom**, an additional catalog-impl must be defined, and the user must put the jar package for the custom catalog implementation into the **{ARCTIC_HOME}/lib** directory, **and the service must be restarted to take effect** +- table-filter: Configure a regular expression to filter tables in the catalog. The matching will be done in the format of `database.table`. For example, if it is set to `(A\.a)|(B\.b)`, it will ignore all tables except for table `a` in database `A` and table `b` in database `B` - table.*: If you want to add the same table configuration to all tables under a catalog, you can add `table.` before the configuration key to indicate that it is a table-level configuration. For example, `table.self-optimizing.group` We recommend users to create a Catalog following the guidelines below： @@ -61,4 +62,4 @@ When a user needs to delete a Catalog, they can go to the details page of the Ca {{< hint info >}} Before deleting a Catalog, AMS will verify whether there is metadata for tables under that Catalog. If there are still tables under that Catalog, AMS will prompt that the deletion failed. -{{< /hint >}} \ No newline at end of file +{{< /hint >}}