Skip to content

Commit

Permalink
Hive: Refactor TestHiveCatalog tests to use the core CatalogTests
Browse files Browse the repository at this point in the history
  • Loading branch information
nk1506 committed Oct 26, 2023
1 parent e2b56da commit e9668f8
Show file tree
Hide file tree
Showing 6 changed files with 253 additions and 156 deletions.
1 change: 1 addition & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -717,6 +717,7 @@ project(':iceberg-hive-metastore') {
}

testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,10 @@ protected boolean supportsNamesWithSlashes() {
return true;
}

protected boolean supportsNamesWithDot() {
return true;
}

@Test
public void testCreateNamespace() {
C catalog = catalog();
Expand Down Expand Up @@ -469,6 +473,8 @@ public void testNamespaceWithSlash() {

@Test
public void testNamespaceWithDot() {
Assumptions.assumeTrue(supportsNamesWithDot());

C catalog = catalog();

Namespace withDot = Namespace.of("new.db");
Expand Down Expand Up @@ -546,6 +552,8 @@ public void testTableNameWithSlash() {

@Test
public void testTableNameWithDot() {
Assumptions.assumeTrue(supportsNamesWithDot());

C catalog = catalog();

TableIdentifier ident = TableIdentifier.of("ns", "ta.ble");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,15 @@ public void renameTable(TableIdentifier from, TableIdentifier originalTo) {
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException("Interrupted in call to rename", e);
} catch (RuntimeException e) {
// in case of table already exists,
// Hive rename operation throws exception as
// java.lang.RuntimeException:InvalidOperationException(message:new table <> already exists)
if (e.getMessage().contains(String.format("new table %s already exists)", to))) {
throw new org.apache.iceberg.exceptions.AlreadyExistsException(
"Table already exists: %s", to);
}
throw new RuntimeException("Failed to rename " + from + " to " + to, e);
}
}

Expand Down Expand Up @@ -288,7 +297,7 @@ public void createNamespace(Namespace namespace, Map<String, String> meta) {

} catch (AlreadyExistsException e) {
throw new org.apache.iceberg.exceptions.AlreadyExistsException(
e, "Namespace '%s' already exists!", namespace);
e, "Namespace already exists: %s", namespace);

} catch (TException e) {
throw new RuntimeException(
Expand Down Expand Up @@ -500,6 +509,9 @@ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
return String.format("%s/%s", databaseData.getLocationUri(), tableIdentifier.name());
}

} catch (NoSuchObjectException e) {
throw new NoSuchNamespaceException(
e, "Namespace does not exist: %s", tableIdentifier.namespace().levels()[0]);
} catch (TException e) {
throw new RuntimeException(
String.format("Metastore operation failed for %s", tableIdentifier), e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ protected void doCommit(TableMetadata base, TableMetadata metadata) {
String baseMetadataLocation = base != null ? base.metadataFileLocation() : null;
if (!Objects.equals(baseMetadataLocation, metadataLocation)) {
throw new CommitFailedException(
"Base metadata location '%s' is not same as the current table metadata location '%s' for %s.%s",
"Cannot commit, Base metadata location '%s' is not same as the current table metadata location '%s' for %s.%s",
baseMetadataLocation, metadataLocation, database, tableName);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iceberg.hive;

import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;

/**
* Setup HiveMetastore. It does not create any database. All the tests should create a database
* accordingly. It should replace the existing setUp class {@link HiveMetastoreTest}
*/
class HiveMetastoreSetup {

protected HiveMetaStoreClient metastoreClient;
protected TestHiveMetastore metastore;
protected HiveConf hiveConf;
HiveCatalog catalog;

HiveMetastoreSetup(Map<String, String> hiveConfOverride) throws Exception {
metastore = new TestHiveMetastore();
HiveConf hiveConfWithOverrides = new HiveConf(TestHiveMetastore.class);
if (hiveConfOverride != null) {
for (Map.Entry<String, String> kv : hiveConfOverride.entrySet()) {
hiveConfWithOverrides.set(kv.getKey(), kv.getValue());
}
}

metastore.start(hiveConfWithOverrides);
hiveConf = metastore.hiveConf();
metastoreClient = new HiveMetaStoreClient(hiveConfWithOverrides);
catalog =
(HiveCatalog)
CatalogUtil.loadCatalog(
HiveCatalog.class.getName(),
CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE,
ImmutableMap.of(
CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS,
String.valueOf(TimeUnit.SECONDS.toMillis(10))),
hiveConfWithOverrides);
}

void stopMetastore() throws Exception {
try {
metastoreClient.close();
metastore.stop();
} finally {
catalog = null;
metastoreClient = null;
metastore = null;
}
}
}
Loading

0 comments on commit e9668f8

Please sign in to comment.