Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ARCTIC-1336]: Support mixed iceberg in external catalog based on iceberg catalog. #1885

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
3434905
mixed iceberg
baiyangtx Aug 23, 2023
ab1dc08
Core Unit Tests Passed
baiyangtx Aug 24, 2023
581993b
Fix checkstyle problems
baiyangtx Aug 24, 2023
924eda3
Fix hive unit tests
baiyangtx Aug 24, 2023
39526f8
fix flink unit tests
baiyangtx Aug 24, 2023
fe8f4ed
fix unit test in AMS
baiyangtx Aug 25, 2023
e239b85
fix unit test in Flink
baiyangtx Aug 25, 2023
17e33b7
trino unit tests
baiyangtx Aug 25, 2023
4edd638
Merge branch 'master' into mixed-iceberg-based-on-iceberg-catalog
baiyangtx Aug 25, 2023
120041b
Merge branch 'master' into mixed-iceberg-based-on-iceberg-catalog
baiyangtx Aug 25, 2023
4aa6468
trino unit tests
baiyangtx Aug 25, 2023
8e70b62
Merge branch 'master' into mixed-iceberg-based-on-iceberg-catalog
baiyangtx Aug 28, 2023
8fe8c1d
trino unit tests
baiyangtx Aug 28, 2023
28c2ffa
Merge remote-tracking branch 'origin/mixed-iceberg-based-on-iceberg-c…
baiyangtx Aug 28, 2023
840e4bc
add mixed-iceberg to dashboard
baiyangtx Aug 28, 2023
0f7a1a7
add doAs for new mixed-iceberg
baiyangtx Aug 28, 2023
6c77e29
do not check resource
HuangFru Aug 29, 2023
2d59ce8
Merge pull request #12 from HuangFru/trino-ut: do not check resource
baiyangtx Aug 29, 2023
629ded7
do not check resource
HuangFru Aug 29, 2023
ea218f0
Merge pull request #13 from HuangFru/trino-ut: do not close AMS
baiyangtx Aug 29, 2023
8787f4e
Merge branch 'master' into mixed-iceberg-based-on-iceberg-catalog
baiyangtx Aug 29, 2023
321bb67
Merge branch 'master' into mixed-iceberg-based-on-iceberg-catalog
baiyangtx Aug 31, 2023
c0b3aae
fix the review comments.
baiyangtx Aug 31, 2023
5831750
Merge branch 'master' into mixed-iceberg-based-on-iceberg-catalog
baiyangtx Aug 31, 2023
5c85797
remove duplicate class
baiyangtx Aug 31, 2023
9008895
Merge remote-tracking branch 'origin/mixed-iceberg-based-on-iceberg-c…
baiyangtx Aug 31, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -74,4 +74,7 @@ public class CatalogMetaProperties {
public static final String LOG_STORE_PROPERTIES_PREFIX = "log-store.";
public static final String OPTIMIZE_PROPERTIES_PREFIX = "self-optimizing.";

// mixed-format properties
public static final String MIXED_FORMAT_TABLE_STORE_SEPARATOR = "mixed-format.table-store.separator";
public static final String MIXED_FORMAT_TABLE_STORE_SEPARATOR_DEFAULT = "_";
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

package com.netease.arctic.ams.api;

import com.netease.arctic.ams.api.properties.CatalogMetaProperties;
import org.apache.thrift.TException;
import org.apache.thrift.TMultiplexedProcessor;
import org.apache.thrift.protocol.TBinaryProtocol;
Expand Down Expand Up @@ -53,7 +52,6 @@
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;

import static com.netease.arctic.ams.api.properties.CatalogMetaProperties.CATALOG_TYPE_HADOOP;

public class MockArcticMetastoreServer implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(MockArcticMetastoreServer.class);
Expand All @@ -74,44 +72,10 @@ public class MockArcticMetastoreServer implements Runnable {
public static MockArcticMetastoreServer getInstance() {
if (!INSTANCE.isStarted()) {
INSTANCE.start();
Map<String, String> storageConfig = new HashMap<>();
storageConfig.put(
CatalogMetaProperties.STORAGE_CONFIGS_KEY_TYPE,
CatalogMetaProperties.STORAGE_CONFIGS_VALUE_TYPE_HDFS);
storageConfig.put(CatalogMetaProperties.STORAGE_CONFIGS_KEY_CORE_SITE, getHadoopSite());
storageConfig.put(CatalogMetaProperties.STORAGE_CONFIGS_KEY_HDFS_SITE, getHadoopSite());

Map<String, String> authConfig = new HashMap<>();
authConfig.put(
CatalogMetaProperties.AUTH_CONFIGS_KEY_TYPE,
CatalogMetaProperties.AUTH_CONFIGS_VALUE_TYPE_SIMPLE);
authConfig.put(
CatalogMetaProperties.AUTH_CONFIGS_KEY_HADOOP_USERNAME,
System.getProperty("user.name"));

Map<String, String> catalogProperties = new HashMap<>();
catalogProperties.put(CatalogMetaProperties.KEY_WAREHOUSE, "/tmp");

CatalogMeta catalogMeta = new CatalogMeta(TEST_CATALOG_NAME, CATALOG_TYPE_HADOOP,
storageConfig, authConfig, catalogProperties);
INSTANCE.handler().createCatalog(catalogMeta);

try {
INSTANCE.handler().createDatabase(TEST_CATALOG_NAME, TEST_DB_NAME);
} catch (TException e) {
throw new RuntimeException(e);
}
}
return INSTANCE;
}

public void createCatalogIfAbsent(CatalogMeta catalogMeta) {
MockArcticMetastoreServer server = getInstance();
if (server.handler().getCatalogs().stream()
.noneMatch(meta -> meta.getCatalogName().equals(catalogMeta.getCatalogName()))) {
server.handler().createCatalog(catalogMeta);
}
}

public static String getHadoopSite() {
return Base64.getEncoder().encodeToString("<configuration></configuration>".getBytes(StandardCharsets.UTF_8));
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

This file was deleted.

Loading