From 7fece64168c3b486f097916d8d3fb9cc0bc600f9 Mon Sep 17 00:00:00 2001 From: huangjianmin <531493269@qq.com> Date: Wed, 15 Nov 2023 15:28:32 +0800 Subject: [PATCH 01/11] re add table filter --- .../netease/arctic/CommonUnifiedCatalog.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/netease/arctic/CommonUnifiedCatalog.java b/core/src/main/java/com/netease/arctic/CommonUnifiedCatalog.java index 96dae7bcdf..16dc9d02f0 100644 --- a/core/src/main/java/com/netease/arctic/CommonUnifiedCatalog.java +++ b/core/src/main/java/com/netease/arctic/CommonUnifiedCatalog.java @@ -20,6 +20,7 @@ import com.netease.arctic.ams.api.CatalogMeta; import com.netease.arctic.ams.api.TableFormat; +import com.netease.arctic.ams.api.properties.CatalogMetaProperties; import com.netease.arctic.table.TableIdentifier; import com.netease.arctic.table.TableMetaStore; import com.netease.arctic.utils.CatalogUtil; @@ -31,6 +32,7 @@ import java.util.ServiceLoader; import java.util.Set; import java.util.function.Supplier; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -40,6 +42,7 @@ public class CommonUnifiedCatalog implements UnifiedCatalog { private CatalogMeta meta; private Map formatCatalogs = Maps.newHashMap(); private final Map properties = Maps.newHashMap(); + private Pattern tableFilterPattern; public CommonUnifiedCatalog( Supplier catalogMetaSupplier, Map properties) { @@ -48,6 +51,12 @@ public CommonUnifiedCatalog( this.meta = catalogMeta; this.properties.putAll(properties); this.metaSupplier = catalogMetaSupplier; + if (meta.getCatalogProperties().containsKey(CatalogMetaProperties.KEY_TABLE_FILTER)) { + String tableFilter = meta.getCatalogProperties().get(CatalogMetaProperties.KEY_TABLE_FILTER); + tableFilterPattern = Pattern.compile(tableFilter); + } else { + tableFilterPattern = null; + } initializeFormatCatalogs(); } @@ -129,9 +138,11 @@ public List listTables(String database) { Map tableNameToFormat = Maps.newHashMap(); for (TableFormat format : formats) { if (formatCatalogs.containsKey(format)) { - formatCatalogs - .get(format) - .listTables(database) + formatCatalogs.get(format).listTables(database).stream() + .filter( + tableName -> + tableFilterPattern == null + || tableFilterPattern.matcher((database + "." + tableName)).matches()) .forEach(table -> tableNameToFormat.putIfAbsent(table, format)); } } From 788570c52783f84d65bb5a37cd9ac19530e6d907 Mon Sep 17 00:00:00 2001 From: huangjianmin <531493269@qq.com> Date: Thu, 16 Nov 2023 10:14:35 +0800 Subject: [PATCH 02/11] implement in external catalog --- .../server/catalog/ExternalCatalog.java | 25 ++++++++++++++++++- .../netease/arctic/CommonUnifiedCatalog.java | 17 +++---------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java b/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java index afaf2308ce..a1b954d88e 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java +++ b/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java @@ -6,6 +6,7 @@ import com.netease.arctic.UnifiedCatalog; import com.netease.arctic.ams.api.CatalogMeta; import com.netease.arctic.ams.api.TableFormat; +import com.netease.arctic.ams.api.properties.CatalogMetaProperties; import com.netease.arctic.server.persistence.mapper.TableMetaMapper; import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.table.TableMetaStore; @@ -15,12 +16,14 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; +import java.util.regex.Pattern; import java.util.stream.Collectors; public class ExternalCatalog extends ServerCatalog { UnifiedCatalog unifiedCatalog; TableMetaStore tableMetaStore; + private Pattern tableFilterPattern; protected ExternalCatalog(CatalogMeta metadata) { super(metadata); @@ -28,6 +31,13 @@ protected ExternalCatalog(CatalogMeta metadata) { this.unifiedCatalog = this.tableMetaStore.doAs( () -> new CommonUnifiedCatalog(this::getMetadata, Maps.newHashMap())); + if (metadata.getCatalogProperties().containsKey(CatalogMetaProperties.KEY_TABLE_FILTER)) { + String tableFilter = + metadata.getCatalogProperties().get(CatalogMetaProperties.KEY_TABLE_FILTER); + tableFilterPattern = Pattern.compile(tableFilter); + } else { + tableFilterPattern = null; + } } public void syncTable(String database, String tableName, TableFormat format) { @@ -83,7 +93,20 @@ public List listTables() { @Override public List listTables(String database) { - return doAs(() -> new ArrayList<>(unifiedCatalog.listTables(database))); + return doAs( + () -> + new ArrayList<>( + unifiedCatalog.listTables(database).stream() + .filter( + tableIDWithFormat -> + tableFilterPattern == null + || tableFilterPattern + .matcher( + (database + + "." + + tableIDWithFormat.getIdentifier().getTableName())) + .matches()) + .collect(Collectors.toList()))); } @Override diff --git a/core/src/main/java/com/netease/arctic/CommonUnifiedCatalog.java b/core/src/main/java/com/netease/arctic/CommonUnifiedCatalog.java index 16dc9d02f0..96dae7bcdf 100644 --- a/core/src/main/java/com/netease/arctic/CommonUnifiedCatalog.java +++ b/core/src/main/java/com/netease/arctic/CommonUnifiedCatalog.java @@ -20,7 +20,6 @@ import com.netease.arctic.ams.api.CatalogMeta; import com.netease.arctic.ams.api.TableFormat; -import com.netease.arctic.ams.api.properties.CatalogMetaProperties; import com.netease.arctic.table.TableIdentifier; import com.netease.arctic.table.TableMetaStore; import com.netease.arctic.utils.CatalogUtil; @@ -32,7 +31,6 @@ import java.util.ServiceLoader; import java.util.Set; import java.util.function.Supplier; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -42,7 +40,6 @@ public class CommonUnifiedCatalog implements UnifiedCatalog { private CatalogMeta meta; private Map formatCatalogs = Maps.newHashMap(); private final Map properties = Maps.newHashMap(); - private Pattern tableFilterPattern; public CommonUnifiedCatalog( Supplier catalogMetaSupplier, Map properties) { @@ -51,12 +48,6 @@ public CommonUnifiedCatalog( this.meta = catalogMeta; this.properties.putAll(properties); this.metaSupplier = catalogMetaSupplier; - if (meta.getCatalogProperties().containsKey(CatalogMetaProperties.KEY_TABLE_FILTER)) { - String tableFilter = meta.getCatalogProperties().get(CatalogMetaProperties.KEY_TABLE_FILTER); - tableFilterPattern = Pattern.compile(tableFilter); - } else { - tableFilterPattern = null; - } initializeFormatCatalogs(); } @@ -138,11 +129,9 @@ public List listTables(String database) { Map tableNameToFormat = Maps.newHashMap(); for (TableFormat format : formats) { if (formatCatalogs.containsKey(format)) { - formatCatalogs.get(format).listTables(database).stream() - .filter( - tableName -> - tableFilterPattern == null - || tableFilterPattern.matcher((database + "." + tableName)).matches()) + formatCatalogs + .get(format) + .listTables(database) .forEach(table -> tableNameToFormat.putIfAbsent(table, format)); } } From 400c5b76955bb3da1d69ba04f0cdd56fb98ff586 Mon Sep 17 00:00:00 2001 From: huangjianmin <531493269@qq.com> Date: Mon, 20 Nov 2023 10:17:13 +0800 Subject: [PATCH 03/11] add ut case --- .../server/catalog/ExternalCatalogTest.java | 90 ------------------- .../server/catalog/TestServerCatalog.java | 77 +++++++++++++--- 2 files changed, 64 insertions(+), 103 deletions(-) delete mode 100644 ams/server/src/test/java/com/netease/arctic/server/catalog/ExternalCatalogTest.java diff --git a/ams/server/src/test/java/com/netease/arctic/server/catalog/ExternalCatalogTest.java b/ams/server/src/test/java/com/netease/arctic/server/catalog/ExternalCatalogTest.java deleted file mode 100644 index 8568104405..0000000000 --- a/ams/server/src/test/java/com/netease/arctic/server/catalog/ExternalCatalogTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netease.arctic.server.catalog; - -import com.netease.arctic.formats.AmoroCatalogTestHelper; -import com.netease.arctic.formats.IcebergHadoopCatalogTestHelper; -import com.netease.arctic.formats.PaimonHadoopCatalogTestHelper; -import com.netease.arctic.hive.formats.IcebergHiveCatalogTestHelper; -import com.netease.arctic.hive.formats.PaimonHiveCatalogTestHelper; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -@RunWith(Parameterized.class) -public class ExternalCatalogTest extends TableCatalogTestBase { - - private final String testDatabaseName = "test_database"; - - private final String testTableName = "test_table"; - - public ExternalCatalogTest(AmoroCatalogTestHelper amoroCatalogTestHelper) { - super(amoroCatalogTestHelper); - } - - @Parameterized.Parameters(name = "{0}") - public static Object[] parameters() { - return new Object[] { - PaimonHadoopCatalogTestHelper.defaultHelper(), - PaimonHiveCatalogTestHelper.defaultHelper(), - IcebergHadoopCatalogTestHelper.defaultHelper(), - IcebergHiveCatalogTestHelper.defaultHelper() - }; - } - - @Before - public void setUp() throws Exception { - getAmoroCatalog().createDatabase(testDatabaseName); - getAmoroCatalogTestHelper().createTable(testDatabaseName, testTableName); - } - - @Test - public void listDatabases() { - Assert.assertTrue(getExternalCatalog().listDatabases().contains(testDatabaseName)); - } - - @Test - public void dataBaseExists() { - Assert.assertTrue(getExternalCatalog().exist(testDatabaseName)); - } - - @Test - public void tableExists() { - Assert.assertTrue(getExternalCatalog().exist(testDatabaseName, testTableName)); - } - - @Test - public void listTables() { - Assert.assertEquals(1, getExternalCatalog().listTables(testDatabaseName).size()); - Assert.assertEquals( - testTableName, - getExternalCatalog().listTables(testDatabaseName).get(0).getIdentifier().getTableName()); - } - - @Test - public void loadTable() { - Assert.assertNotNull(getExternalCatalog().loadTable(testDatabaseName, testTableName)); - } - - private ServerCatalog getExternalCatalog() { - return tableService().getServerCatalog(getAmoroCatalogTestHelper().catalogName()); - } -} diff --git a/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java b/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java index edae2e21e3..db04de4fa2 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java +++ b/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java @@ -18,10 +18,13 @@ package com.netease.arctic.server.catalog; +import com.netease.arctic.ams.api.properties.CatalogMetaProperties; import com.netease.arctic.formats.AmoroCatalogTestHelper; import com.netease.arctic.formats.IcebergHadoopCatalogTestHelper; +import com.netease.arctic.formats.MixedIcebergHadoopCatalogTestHelper; import com.netease.arctic.formats.PaimonHadoopCatalogTestHelper; import com.netease.arctic.hive.formats.IcebergHiveCatalogTestHelper; +import com.netease.arctic.hive.formats.MixedIcebergHiveCatalogTestHelper; import com.netease.arctic.hive.formats.PaimonHiveCatalogTestHelper; import org.junit.Assert; import org.junit.Before; @@ -29,6 +32,8 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.util.HashMap; + @RunWith(Parameterized.class) public class TestServerCatalog extends TableCatalogTestBase { @@ -36,6 +41,8 @@ public class TestServerCatalog extends TableCatalogTestBase { private final String testTableName = "test_table"; + private final String testTableNameFilter = "test_table_filter"; + public TestServerCatalog(AmoroCatalogTestHelper amoroCatalogTestHelper) { super(amoroCatalogTestHelper); } @@ -43,10 +50,48 @@ public TestServerCatalog(AmoroCatalogTestHelper amoroCatalogTestHelper) { @Parameterized.Parameters(name = "{0}") public static Object[] parameters() { return new Object[] { - PaimonHadoopCatalogTestHelper.defaultHelper(), - PaimonHiveCatalogTestHelper.defaultHelper(), - IcebergHadoopCatalogTestHelper.defaultHelper(), - IcebergHiveCatalogTestHelper.defaultHelper() + new PaimonHadoopCatalogTestHelper( + "test_paimon_catalog", + new HashMap() { + { + put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); + } + }), + new PaimonHiveCatalogTestHelper( + "test_paimon_catalog", + new HashMap() { + { + put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); + } + }), + new IcebergHadoopCatalogTestHelper( + "test_iceberg_catalog", + new HashMap() { + { + put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); + } + }), + new IcebergHiveCatalogTestHelper( + "test_iceberg_catalog", + new HashMap() { + { + put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); + } + }), + new MixedIcebergHadoopCatalogTestHelper( + "test_mixed_catalog", + new HashMap() { + { + put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); + } + }), + new MixedIcebergHiveCatalogTestHelper( + "test_mixed_catalog", + new HashMap() { + { + put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); + } + }) }; } @@ -54,37 +99,43 @@ public static Object[] parameters() { public void setUp() throws Exception { getAmoroCatalog().createDatabase(testDatabaseName); getAmoroCatalogTestHelper().createTable(testDatabaseName, testTableName); + getAmoroCatalogTestHelper().createTable(testDatabaseName, testTableNameFilter); } @Test public void listDatabases() { - Assert.assertTrue(getExternalCatalog().listDatabases().contains(testDatabaseName)); + Assert.assertTrue(getServerCatalog().listDatabases().contains(testDatabaseName)); } @Test public void dataBaseExists() { - Assert.assertTrue(getExternalCatalog().exist(testDatabaseName)); + Assert.assertTrue(getServerCatalog().exist(testDatabaseName)); } @Test public void tableExists() { - Assert.assertTrue(getExternalCatalog().exist(testDatabaseName, testTableName)); + Assert.assertTrue(getServerCatalog().exist(testDatabaseName, testTableName)); } @Test public void listTables() { - Assert.assertEquals(1, getExternalCatalog().listTables(testDatabaseName).size()); - Assert.assertEquals( - testTableName, - getExternalCatalog().listTables(testDatabaseName).get(0).getIdentifier().getTableName()); + ServerCatalog serverCatalog = getServerCatalog(); + if (serverCatalog instanceof ExternalCatalog) { + Assert.assertEquals(1, serverCatalog.listTables(testDatabaseName).size()); + Assert.assertEquals( + testTableName, + serverCatalog.listTables(testDatabaseName).get(0).getIdentifier().getTableName()); + } else { + Assert.assertEquals(2, serverCatalog.listTables(testDatabaseName).size()); + } } @Test public void loadTable() { - Assert.assertNotNull(getExternalCatalog().loadTable(testDatabaseName, testTableName)); + Assert.assertNotNull(getServerCatalog().loadTable(testDatabaseName, testTableName)); } - private ServerCatalog getExternalCatalog() { + private ServerCatalog getServerCatalog() { return tableService().getServerCatalog(getAmoroCatalogTestHelper().catalogName()); } } From b5597d7fdfa2103fb272ed08daae2e682bb00eff Mon Sep 17 00:00:00 2001 From: huangjianmin <531493269@qq.com> Date: Tue, 21 Nov 2023 10:53:39 +0800 Subject: [PATCH 04/11] fix comment --- .../com/netease/arctic/server/catalog/ExternalCatalog.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java b/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java index a1b954d88e..79b2e0aacf 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java +++ b/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java @@ -31,9 +31,9 @@ protected ExternalCatalog(CatalogMeta metadata) { this.unifiedCatalog = this.tableMetaStore.doAs( () -> new CommonUnifiedCatalog(this::getMetadata, Maps.newHashMap())); - if (metadata.getCatalogProperties().containsKey(CatalogMetaProperties.KEY_TABLE_FILTER)) { - String tableFilter = - metadata.getCatalogProperties().get(CatalogMetaProperties.KEY_TABLE_FILTER); + String tableFilter = + metadata.getCatalogProperties().get(CatalogMetaProperties.KEY_TABLE_FILTER); + if (tableFilter != null) { tableFilterPattern = Pattern.compile(tableFilter); } else { tableFilterPattern = null; From 825c31ed06cc9de6ba26c8891e1f2d4eee1a3efb Mon Sep 17 00:00:00 2001 From: huangjianmin <531493269@qq.com> Date: Thu, 23 Nov 2023 11:00:26 +0800 Subject: [PATCH 05/11] fix comment --- .../arctic/mixed/BasicMixedIcebergCatalog.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java b/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java index 57190441ef..eddeb6afaa 100644 --- a/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java +++ b/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java @@ -59,6 +59,7 @@ public class BasicMixedIcebergCatalog implements ArcticCatalog { private Map catalogProperties; private String name; private Pattern databaseFilterPattern; + private Pattern tableFilterPattern; private AmsClient client; private MixedTables tables; @@ -80,12 +81,18 @@ public void initialize(String name, Map properties, TableMetaSto properties.get(CatalogMetaProperties.KEY_DATABASE_FILTER_REGULAR_EXPRESSION); databaseFilterPattern = Pattern.compile(databaseFilter); } + Pattern tableFilterPattern = null; + String tableFilter = properties.get(CatalogMetaProperties.KEY_TABLE_FILTER); + if (tableFilter != null) { + tableFilterPattern = Pattern.compile(tableFilter); + } MixedTables tables = new MixedTables(metaStore, properties, icebergCatalog); synchronized (this) { this.name = name; this.tableMetaStore = metaStore; this.icebergCatalog = icebergCatalog; this.databaseFilterPattern = databaseFilterPattern; + this.tableFilterPattern = tableFilterPattern; this.catalogProperties = properties; this.tables = tables; if (properties.containsKey(CatalogMetaProperties.AMS_URI)) { @@ -124,7 +131,16 @@ public void dropDatabase(String databaseName) { @Override public List listTables(String database) { List icebergTableList = - tableMetaStore.doAs(() -> icebergCatalog.listTables(Namespace.of(database))); + tableMetaStore.doAs( + () -> + icebergCatalog.listTables(Namespace.of(database)).stream() + .filter( + tableIdentifier -> + tableFilterPattern == null + || tableFilterPattern + .matcher(database + "." + tableIdentifier.name()) + .matches()) + .collect(Collectors.toList())); List mixedTables = Lists.newArrayList(); Set visited = Sets.newHashSet(); for (org.apache.iceberg.catalog.TableIdentifier identifier : icebergTableList) { From 8915d442e3046c727b561cfc8093d5d6ca4b0418 Mon Sep 17 00:00:00 2001 From: huangjianmin <531493269@qq.com> Date: Thu, 23 Nov 2023 11:07:48 +0800 Subject: [PATCH 06/11] fix comment --- .../arctic/mixed/BasicMixedIcebergCatalog.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java b/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java index a6158063de..aec3936310 100644 --- a/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java +++ b/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java @@ -64,6 +64,7 @@ public class BasicMixedIcebergCatalog implements ArcticCatalog { private Map catalogProperties; private String name; private Pattern databaseFilterPattern; + private Pattern tableFilterPattern; private AmsClient client; private MixedTables tables; private SupportsNamespaces asNamespaceCatalog; @@ -103,6 +104,12 @@ public void initialize(String name, Map properties, TableMetaSto databaseFilterPattern = Pattern.compile(databaseFilter); } Catalog catalog = buildIcebergCatalog(name, properties, metaStore.getConfiguration()); + String tableFilter = properties.get(CatalogMetaProperties.KEY_TABLE_FILTER); + if (tableFilter != null) { + this.tableFilterPattern = Pattern.compile(tableFilter); + } else { + this.tableFilterPattern = null; + } this.name = name; this.tableMetaStore = metaStore; this.icebergCatalog = @@ -150,7 +157,16 @@ public void dropDatabase(String databaseName) { @Override public List listTables(String database) { List icebergTableList = - tableMetaStore.doAs(() -> icebergCatalog().listTables(Namespace.of(database))); + tableMetaStore.doAs( + () -> + icebergCatalog().listTables(Namespace.of(database)).stream() + .filter( + tableIdentifier -> + tableFilterPattern == null + || tableFilterPattern + .matcher((database + "." + tableIdentifier.name())) + .matches()) + .collect(Collectors.toList())); List mixedTables = Lists.newArrayList(); Set visited = Sets.newHashSet(); for (org.apache.iceberg.catalog.TableIdentifier identifier : icebergTableList) { From 791bf36f514430c2596942f636606d3b7314c136 Mon Sep 17 00:00:00 2001 From: huangjianmin <531493269@qq.com> Date: Thu, 23 Nov 2023 15:59:04 +0800 Subject: [PATCH 07/11] fix ut --- .../arctic/server/catalog/TestServerCatalog.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java b/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java index db04de4fa2..9974909e17 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java +++ b/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java @@ -18,6 +18,7 @@ package com.netease.arctic.server.catalog; +import com.netease.arctic.AmoroCatalog; import com.netease.arctic.ams.api.properties.CatalogMetaProperties; import com.netease.arctic.formats.AmoroCatalogTestHelper; import com.netease.arctic.formats.IcebergHadoopCatalogTestHelper; @@ -26,6 +27,7 @@ import com.netease.arctic.hive.formats.IcebergHiveCatalogTestHelper; import com.netease.arctic.hive.formats.MixedIcebergHiveCatalogTestHelper; import com.netease.arctic.hive.formats.PaimonHiveCatalogTestHelper; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -102,6 +104,14 @@ public void setUp() throws Exception { getAmoroCatalogTestHelper().createTable(testDatabaseName, testTableNameFilter); } + @After + public void cleanFilterTable() { + AmoroCatalog amoroCatalog = getAmoroCatalog(); + if (amoroCatalog.exist(testDatabaseName, testTableNameFilter)) { + amoroCatalog.dropTable(testDatabaseName, testTableNameFilter, true); + } + } + @Test public void listDatabases() { Assert.assertTrue(getServerCatalog().listDatabases().contains(testDatabaseName)); From 3b79b5757d1a980a4e2a96e5f1ea8cdb1da2c707 Mon Sep 17 00:00:00 2001 From: huangjianmin <531493269@qq.com> Date: Mon, 27 Nov 2023 09:50:49 +0800 Subject: [PATCH 08/11] fix update properties --- .../server/catalog/ExternalCatalog.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java b/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java index 79b2e0aacf..68facb4f86 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java +++ b/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java @@ -31,13 +31,7 @@ protected ExternalCatalog(CatalogMeta metadata) { this.unifiedCatalog = this.tableMetaStore.doAs( () -> new CommonUnifiedCatalog(this::getMetadata, Maps.newHashMap())); - String tableFilter = - metadata.getCatalogProperties().get(CatalogMetaProperties.KEY_TABLE_FILTER); - if (tableFilter != null) { - tableFilterPattern = Pattern.compile(tableFilter); - } else { - tableFilterPattern = null; - } + updateTableFilter(metadata); } public void syncTable(String database, String tableName, TableFormat format) { @@ -64,6 +58,7 @@ public void updateMetadata(CatalogMeta metadata) { super.updateMetadata(metadata); this.tableMetaStore = CatalogUtil.buildMetaStore(metadata); this.unifiedCatalog.refresh(); + updateTableFilter(metadata); } @Override @@ -114,6 +109,16 @@ public AmoroTable loadTable(String database, String tableName) { return doAs(() -> unifiedCatalog.loadTable(database, tableName)); } + private void updateTableFilter(CatalogMeta metadata) { + String tableFilter = + metadata.getCatalogProperties().get(CatalogMetaProperties.KEY_TABLE_FILTER); + if (tableFilter != null) { + tableFilterPattern = Pattern.compile(tableFilter); + } else { + tableFilterPattern = null; + } + } + private T doAs(Callable callable) { return tableMetaStore.doAs(callable); } From 1ddf92f28b2511fb6479d63837d3c01d8b82ecae Mon Sep 17 00:00:00 2001 From: huangjianmin <531493269@qq.com> Date: Wed, 29 Nov 2023 16:00:34 +0800 Subject: [PATCH 09/11] roll back the engine side's filter --- .../arctic/mixed/BasicMixedIcebergCatalog.java | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java b/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java index aec3936310..a6158063de 100644 --- a/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java +++ b/core/src/main/java/com/netease/arctic/mixed/BasicMixedIcebergCatalog.java @@ -64,7 +64,6 @@ public class BasicMixedIcebergCatalog implements ArcticCatalog { private Map catalogProperties; private String name; private Pattern databaseFilterPattern; - private Pattern tableFilterPattern; private AmsClient client; private MixedTables tables; private SupportsNamespaces asNamespaceCatalog; @@ -104,12 +103,6 @@ public void initialize(String name, Map properties, TableMetaSto databaseFilterPattern = Pattern.compile(databaseFilter); } Catalog catalog = buildIcebergCatalog(name, properties, metaStore.getConfiguration()); - String tableFilter = properties.get(CatalogMetaProperties.KEY_TABLE_FILTER); - if (tableFilter != null) { - this.tableFilterPattern = Pattern.compile(tableFilter); - } else { - this.tableFilterPattern = null; - } this.name = name; this.tableMetaStore = metaStore; this.icebergCatalog = @@ -157,16 +150,7 @@ public void dropDatabase(String databaseName) { @Override public List listTables(String database) { List icebergTableList = - tableMetaStore.doAs( - () -> - icebergCatalog().listTables(Namespace.of(database)).stream() - .filter( - tableIdentifier -> - tableFilterPattern == null - || tableFilterPattern - .matcher((database + "." + tableIdentifier.name())) - .matches()) - .collect(Collectors.toList())); + tableMetaStore.doAs(() -> icebergCatalog().listTables(Namespace.of(database))); List mixedTables = Lists.newArrayList(); Set visited = Sets.newHashSet(); for (org.apache.iceberg.catalog.TableIdentifier identifier : icebergTableList) { From ffd82bdd95b0690dc4e0589f1815ec37d8970d9f Mon Sep 17 00:00:00 2001 From: huangjianmin <531493269@qq.com> Date: Wed, 6 Dec 2023 11:07:23 +0800 Subject: [PATCH 10/11] resolve conflicts --- .../server/catalog/TestServerCatalog.java | 99 +++++++------------ 1 file changed, 33 insertions(+), 66 deletions(-) diff --git a/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java b/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java index 9974909e17..1c374050b0 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java +++ b/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java @@ -18,7 +18,7 @@ package com.netease.arctic.server.catalog; -import com.netease.arctic.AmoroCatalog; +import com.netease.arctic.ams.api.CatalogMeta; import com.netease.arctic.ams.api.properties.CatalogMetaProperties; import com.netease.arctic.formats.AmoroCatalogTestHelper; import com.netease.arctic.formats.IcebergHadoopCatalogTestHelper; @@ -27,15 +27,13 @@ import com.netease.arctic.hive.formats.IcebergHiveCatalogTestHelper; import com.netease.arctic.hive.formats.MixedIcebergHiveCatalogTestHelper; import com.netease.arctic.hive.formats.PaimonHiveCatalogTestHelper; -import org.junit.After; import org.junit.Assert; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import java.util.HashMap; - @RunWith(Parameterized.class) public class TestServerCatalog extends TableCatalogTestBase { @@ -43,8 +41,6 @@ public class TestServerCatalog extends TableCatalogTestBase { private final String testTableName = "test_table"; - private final String testTableNameFilter = "test_table_filter"; - public TestServerCatalog(AmoroCatalogTestHelper amoroCatalogTestHelper) { super(amoroCatalogTestHelper); } @@ -52,48 +48,12 @@ public TestServerCatalog(AmoroCatalogTestHelper amoroCatalogTestHelper) { @Parameterized.Parameters(name = "{0}") public static Object[] parameters() { return new Object[] { - new PaimonHadoopCatalogTestHelper( - "test_paimon_catalog", - new HashMap() { - { - put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); - } - }), - new PaimonHiveCatalogTestHelper( - "test_paimon_catalog", - new HashMap() { - { - put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); - } - }), - new IcebergHadoopCatalogTestHelper( - "test_iceberg_catalog", - new HashMap() { - { - put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); - } - }), - new IcebergHiveCatalogTestHelper( - "test_iceberg_catalog", - new HashMap() { - { - put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); - } - }), - new MixedIcebergHadoopCatalogTestHelper( - "test_mixed_catalog", - new HashMap() { - { - put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); - } - }), - new MixedIcebergHiveCatalogTestHelper( - "test_mixed_catalog", - new HashMap() { - { - put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); - } - }) + PaimonHadoopCatalogTestHelper.defaultHelper(), + PaimonHiveCatalogTestHelper.defaultHelper(), + IcebergHadoopCatalogTestHelper.defaultHelper(), + IcebergHiveCatalogTestHelper.defaultHelper(), + MixedIcebergHadoopCatalogTestHelper.defaultHelper(), + MixedIcebergHiveCatalogTestHelper.defaultHelper() }; } @@ -101,15 +61,6 @@ public static Object[] parameters() { public void setUp() throws Exception { getAmoroCatalog().createDatabase(testDatabaseName); getAmoroCatalogTestHelper().createTable(testDatabaseName, testTableName); - getAmoroCatalogTestHelper().createTable(testDatabaseName, testTableNameFilter); - } - - @After - public void cleanFilterTable() { - AmoroCatalog amoroCatalog = getAmoroCatalog(); - if (amoroCatalog.exist(testDatabaseName, testTableNameFilter)) { - amoroCatalog.dropTable(testDatabaseName, testTableNameFilter, true); - } } @Test @@ -129,15 +80,31 @@ public void tableExists() { @Test public void listTables() { - ServerCatalog serverCatalog = getServerCatalog(); - if (serverCatalog instanceof ExternalCatalog) { - Assert.assertEquals(1, serverCatalog.listTables(testDatabaseName).size()); - Assert.assertEquals( - testTableName, - serverCatalog.listTables(testDatabaseName).get(0).getIdentifier().getTableName()); - } else { - Assert.assertEquals(2, serverCatalog.listTables(testDatabaseName).size()); - } + Assert.assertEquals(1, getServerCatalog().listTables(testDatabaseName).size()); + Assert.assertEquals( + testTableName, + getServerCatalog().listTables(testDatabaseName).get(0).getIdentifier().getTableName()); + } + + @Test + public void listTablesWithTableFilter() { + // Table filter only affects ExternalCatalog + Assume.assumeTrue(getServerCatalog() instanceof ExternalCatalog); + String dbWithFilter = "db_with_filter"; + String tableWithFilter = "table_with_filter"; + getAmoroCatalog().createDatabase(dbWithFilter); + + getAmoroCatalogTestHelper().createTable(dbWithFilter, tableWithFilter); + // 1.create some databases and tables + // 2.check the tables without filter + // 3.set table filter + CatalogMeta metadata = getServerCatalog().getMetadata(); + metadata + .getCatalogProperties() + .put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); + getServerCatalog().updateMetadata(metadata); + // 4.check the table list with filter + // 5.finally unset the table filter, remove the databases and tables in this test case } @Test From 24a40a530bde3fb56596594ab1510ce539dfc49d Mon Sep 17 00:00:00 2001 From: huangjianmin <531493269@qq.com> Date: Wed, 6 Dec 2023 15:21:00 +0800 Subject: [PATCH 11/11] add ut --- .../server/catalog/TestServerCatalog.java | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java b/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java index 1c374050b0..7de000b0bd 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java +++ b/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java @@ -87,24 +87,46 @@ public void listTables() { } @Test - public void listTablesWithTableFilter() { + public void listTablesWithTableFilter() throws Exception { // Table filter only affects ExternalCatalog Assume.assumeTrue(getServerCatalog() instanceof ExternalCatalog); String dbWithFilter = "db_with_filter"; - String tableWithFilter = "table_with_filter"; + String tableWithFilter1 = "test_table1"; + String tableWithFilter2 = "test_table2"; getAmoroCatalog().createDatabase(dbWithFilter); + getAmoroCatalogTestHelper().createTable(dbWithFilter, tableWithFilter1); + getAmoroCatalogTestHelper().createTable(dbWithFilter, tableWithFilter2); + // without table filter + Assert.assertEquals(2, getServerCatalog().listTables(dbWithFilter).size()); - getAmoroCatalogTestHelper().createTable(dbWithFilter, tableWithFilter); - // 1.create some databases and tables - // 2.check the tables without filter - // 3.set table filter CatalogMeta metadata = getServerCatalog().getMetadata(); metadata .getCatalogProperties() - .put(CatalogMetaProperties.KEY_TABLE_FILTER, "test_database.test_table"); + .put(CatalogMetaProperties.KEY_TABLE_FILTER, dbWithFilter + "." + tableWithFilter1); getServerCatalog().updateMetadata(metadata); - // 4.check the table list with filter - // 5.finally unset the table filter, remove the databases and tables in this test case + Assert.assertEquals(1, getServerCatalog().listTables(dbWithFilter).size()); + Assert.assertEquals( + tableWithFilter1, + getServerCatalog().listTables(dbWithFilter).get(0).getIdentifier().getTableName()); + + CatalogMeta metadata2 = getServerCatalog().getMetadata(); + metadata + .getCatalogProperties() + .put(CatalogMetaProperties.KEY_TABLE_FILTER, dbWithFilter + "\\." + ".+"); + getServerCatalog().updateMetadata(metadata2); + Assert.assertEquals(2, getServerCatalog().listTables(dbWithFilter).size()); + + CatalogMeta metadata3 = getServerCatalog().getMetadata(); + metadata + .getCatalogProperties() + .put(CatalogMetaProperties.KEY_TABLE_FILTER, testDatabaseName + "\\." + ".+"); + getServerCatalog().updateMetadata(metadata3); + Assert.assertEquals(1, getServerCatalog().listTables(testDatabaseName).size()); + Assert.assertTrue(getServerCatalog().listTables(dbWithFilter).isEmpty()); + + CatalogMeta metadata4 = getServerCatalog().getMetadata(); + metadata.getCatalogProperties().remove(CatalogMetaProperties.KEY_TABLE_FILTER); + getServerCatalog().updateMetadata(metadata4); } @Test