diff --git a/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java b/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java index afaf2308ce..68facb4f86 100644 --- a/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java +++ b/ams/server/src/main/java/com/netease/arctic/server/catalog/ExternalCatalog.java @@ -6,6 +6,7 @@ import com.netease.arctic.UnifiedCatalog; import com.netease.arctic.ams.api.CatalogMeta; import com.netease.arctic.ams.api.TableFormat; +import com.netease.arctic.ams.api.properties.CatalogMetaProperties; import com.netease.arctic.server.persistence.mapper.TableMetaMapper; import com.netease.arctic.server.table.ServerTableIdentifier; import com.netease.arctic.table.TableMetaStore; @@ -15,12 +16,14 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; +import java.util.regex.Pattern; import java.util.stream.Collectors; public class ExternalCatalog extends ServerCatalog { UnifiedCatalog unifiedCatalog; TableMetaStore tableMetaStore; + private Pattern tableFilterPattern; protected ExternalCatalog(CatalogMeta metadata) { super(metadata); @@ -28,6 +31,7 @@ protected ExternalCatalog(CatalogMeta metadata) { this.unifiedCatalog = this.tableMetaStore.doAs( () -> new CommonUnifiedCatalog(this::getMetadata, Maps.newHashMap())); + updateTableFilter(metadata); } public void syncTable(String database, String tableName, TableFormat format) { @@ -54,6 +58,7 @@ public void updateMetadata(CatalogMeta metadata) { super.updateMetadata(metadata); this.tableMetaStore = CatalogUtil.buildMetaStore(metadata); this.unifiedCatalog.refresh(); + updateTableFilter(metadata); } @Override @@ -83,7 +88,20 @@ public List listTables() { @Override public List listTables(String database) { - return doAs(() -> new ArrayList<>(unifiedCatalog.listTables(database))); + return doAs( + () -> + new ArrayList<>( + unifiedCatalog.listTables(database).stream() + .filter( + tableIDWithFormat -> + tableFilterPattern == null + || tableFilterPattern + .matcher( + (database + + "." + + tableIDWithFormat.getIdentifier().getTableName())) + .matches()) + .collect(Collectors.toList()))); } @Override @@ -91,6 +109,16 @@ public AmoroTable loadTable(String database, String tableName) { return doAs(() -> unifiedCatalog.loadTable(database, tableName)); } + private void updateTableFilter(CatalogMeta metadata) { + String tableFilter = + metadata.getCatalogProperties().get(CatalogMetaProperties.KEY_TABLE_FILTER); + if (tableFilter != null) { + tableFilterPattern = Pattern.compile(tableFilter); + } else { + tableFilterPattern = null; + } + } + private T doAs(Callable callable) { return tableMetaStore.doAs(callable); } diff --git a/ams/server/src/test/java/com/netease/arctic/server/catalog/ExternalCatalogTest.java b/ams/server/src/test/java/com/netease/arctic/server/catalog/ExternalCatalogTest.java deleted file mode 100644 index 8568104405..0000000000 --- a/ams/server/src/test/java/com/netease/arctic/server/catalog/ExternalCatalogTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.netease.arctic.server.catalog; - -import com.netease.arctic.formats.AmoroCatalogTestHelper; -import com.netease.arctic.formats.IcebergHadoopCatalogTestHelper; -import com.netease.arctic.formats.PaimonHadoopCatalogTestHelper; -import com.netease.arctic.hive.formats.IcebergHiveCatalogTestHelper; -import com.netease.arctic.hive.formats.PaimonHiveCatalogTestHelper; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -@RunWith(Parameterized.class) -public class ExternalCatalogTest extends TableCatalogTestBase { - - private final String testDatabaseName = "test_database"; - - private final String testTableName = "test_table"; - - public ExternalCatalogTest(AmoroCatalogTestHelper amoroCatalogTestHelper) { - super(amoroCatalogTestHelper); - } - - @Parameterized.Parameters(name = "{0}") - public static Object[] parameters() { - return new Object[] { - PaimonHadoopCatalogTestHelper.defaultHelper(), - PaimonHiveCatalogTestHelper.defaultHelper(), - IcebergHadoopCatalogTestHelper.defaultHelper(), - IcebergHiveCatalogTestHelper.defaultHelper() - }; - } - - @Before - public void setUp() throws Exception { - getAmoroCatalog().createDatabase(testDatabaseName); - getAmoroCatalogTestHelper().createTable(testDatabaseName, testTableName); - } - - @Test - public void listDatabases() { - Assert.assertTrue(getExternalCatalog().listDatabases().contains(testDatabaseName)); - } - - @Test - public void dataBaseExists() { - Assert.assertTrue(getExternalCatalog().exist(testDatabaseName)); - } - - @Test - public void tableExists() { - Assert.assertTrue(getExternalCatalog().exist(testDatabaseName, testTableName)); - } - - @Test - public void listTables() { - Assert.assertEquals(1, getExternalCatalog().listTables(testDatabaseName).size()); - Assert.assertEquals( - testTableName, - getExternalCatalog().listTables(testDatabaseName).get(0).getIdentifier().getTableName()); - } - - @Test - public void loadTable() { - Assert.assertNotNull(getExternalCatalog().loadTable(testDatabaseName, testTableName)); - } - - private ServerCatalog getExternalCatalog() { - return tableService().getServerCatalog(getAmoroCatalogTestHelper().catalogName()); - } -} diff --git a/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java b/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java index edae2e21e3..7de000b0bd 100644 --- a/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java +++ b/ams/server/src/test/java/com/netease/arctic/server/catalog/TestServerCatalog.java @@ -18,12 +18,17 @@ package com.netease.arctic.server.catalog; +import com.netease.arctic.ams.api.CatalogMeta; +import com.netease.arctic.ams.api.properties.CatalogMetaProperties; import com.netease.arctic.formats.AmoroCatalogTestHelper; import com.netease.arctic.formats.IcebergHadoopCatalogTestHelper; +import com.netease.arctic.formats.MixedIcebergHadoopCatalogTestHelper; import com.netease.arctic.formats.PaimonHadoopCatalogTestHelper; import com.netease.arctic.hive.formats.IcebergHiveCatalogTestHelper; +import com.netease.arctic.hive.formats.MixedIcebergHiveCatalogTestHelper; import com.netease.arctic.hive.formats.PaimonHiveCatalogTestHelper; import org.junit.Assert; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -46,7 +51,9 @@ public static Object[] parameters() { PaimonHadoopCatalogTestHelper.defaultHelper(), PaimonHiveCatalogTestHelper.defaultHelper(), IcebergHadoopCatalogTestHelper.defaultHelper(), - IcebergHiveCatalogTestHelper.defaultHelper() + IcebergHiveCatalogTestHelper.defaultHelper(), + MixedIcebergHadoopCatalogTestHelper.defaultHelper(), + MixedIcebergHiveCatalogTestHelper.defaultHelper() }; } @@ -58,33 +65,76 @@ public void setUp() throws Exception { @Test public void listDatabases() { - Assert.assertTrue(getExternalCatalog().listDatabases().contains(testDatabaseName)); + Assert.assertTrue(getServerCatalog().listDatabases().contains(testDatabaseName)); } @Test public void dataBaseExists() { - Assert.assertTrue(getExternalCatalog().exist(testDatabaseName)); + Assert.assertTrue(getServerCatalog().exist(testDatabaseName)); } @Test public void tableExists() { - Assert.assertTrue(getExternalCatalog().exist(testDatabaseName, testTableName)); + Assert.assertTrue(getServerCatalog().exist(testDatabaseName, testTableName)); } @Test public void listTables() { - Assert.assertEquals(1, getExternalCatalog().listTables(testDatabaseName).size()); + Assert.assertEquals(1, getServerCatalog().listTables(testDatabaseName).size()); Assert.assertEquals( testTableName, - getExternalCatalog().listTables(testDatabaseName).get(0).getIdentifier().getTableName()); + getServerCatalog().listTables(testDatabaseName).get(0).getIdentifier().getTableName()); + } + + @Test + public void listTablesWithTableFilter() throws Exception { + // Table filter only affects ExternalCatalog + Assume.assumeTrue(getServerCatalog() instanceof ExternalCatalog); + String dbWithFilter = "db_with_filter"; + String tableWithFilter1 = "test_table1"; + String tableWithFilter2 = "test_table2"; + getAmoroCatalog().createDatabase(dbWithFilter); + getAmoroCatalogTestHelper().createTable(dbWithFilter, tableWithFilter1); + getAmoroCatalogTestHelper().createTable(dbWithFilter, tableWithFilter2); + // without table filter + Assert.assertEquals(2, getServerCatalog().listTables(dbWithFilter).size()); + + CatalogMeta metadata = getServerCatalog().getMetadata(); + metadata + .getCatalogProperties() + .put(CatalogMetaProperties.KEY_TABLE_FILTER, dbWithFilter + "." + tableWithFilter1); + getServerCatalog().updateMetadata(metadata); + Assert.assertEquals(1, getServerCatalog().listTables(dbWithFilter).size()); + Assert.assertEquals( + tableWithFilter1, + getServerCatalog().listTables(dbWithFilter).get(0).getIdentifier().getTableName()); + + CatalogMeta metadata2 = getServerCatalog().getMetadata(); + metadata + .getCatalogProperties() + .put(CatalogMetaProperties.KEY_TABLE_FILTER, dbWithFilter + "\\." + ".+"); + getServerCatalog().updateMetadata(metadata2); + Assert.assertEquals(2, getServerCatalog().listTables(dbWithFilter).size()); + + CatalogMeta metadata3 = getServerCatalog().getMetadata(); + metadata + .getCatalogProperties() + .put(CatalogMetaProperties.KEY_TABLE_FILTER, testDatabaseName + "\\." + ".+"); + getServerCatalog().updateMetadata(metadata3); + Assert.assertEquals(1, getServerCatalog().listTables(testDatabaseName).size()); + Assert.assertTrue(getServerCatalog().listTables(dbWithFilter).isEmpty()); + + CatalogMeta metadata4 = getServerCatalog().getMetadata(); + metadata.getCatalogProperties().remove(CatalogMetaProperties.KEY_TABLE_FILTER); + getServerCatalog().updateMetadata(metadata4); } @Test public void loadTable() { - Assert.assertNotNull(getExternalCatalog().loadTable(testDatabaseName, testTableName)); + Assert.assertNotNull(getServerCatalog().loadTable(testDatabaseName, testTableName)); } - private ServerCatalog getExternalCatalog() { + private ServerCatalog getServerCatalog() { return tableService().getServerCatalog(getAmoroCatalogTestHelper().catalogName()); } }