Skip to content

Commit

Permalink
Hive: Refactor TestHiveCatalog tests to use the core CatalogTests
Browse files Browse the repository at this point in the history
  • Loading branch information
nk1506 committed Oct 31, 2023
1 parent fceea89 commit 530933f
Show file tree
Hide file tree
Showing 7 changed files with 263 additions and 133 deletions.
1 change: 1 addition & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -719,6 +719,7 @@ project(':iceberg-hive-metastore') {
}

testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
testImplementation libs.awaitility
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,10 @@ protected boolean supportsNamesWithSlashes() {
return true;
}

protected boolean supportsNamesWithDot() {
return true;
}

@Test
public void testCreateNamespace() {
C catalog = catalog();
Expand Down Expand Up @@ -470,6 +474,8 @@ public void testNamespaceWithSlash() {

@Test
public void testNamespaceWithDot() {
Assumptions.assumeTrue(supportsNamesWithDot());

C catalog = catalog();

Namespace withDot = Namespace.of("new.db");
Expand Down Expand Up @@ -547,6 +553,8 @@ public void testTableNameWithSlash() {

@Test
public void testTableNameWithDot() {
Assumptions.assumeTrue(supportsNamesWithDot());

C catalog = catalog();

TableIdentifier ident = TableIdentifier.of("ns", "ta.ble");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,15 @@ public void renameTable(TableIdentifier from, TableIdentifier originalTo) {
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException("Interrupted in call to rename", e);
} catch (RuntimeException e) {
// in case of table already exists,
// Hive rename operation throws exception as
// java.lang.RuntimeException:InvalidOperationException(message:new table <> already exists)
if (e.getMessage().contains(String.format("new table %s already exists)", to))) {
throw new org.apache.iceberg.exceptions.AlreadyExistsException(
"Table already exists: %s", to);
}
throw new RuntimeException("Failed to rename " + from + " to " + to, e);
}
}

Expand Down Expand Up @@ -288,7 +297,7 @@ public void createNamespace(Namespace namespace, Map<String, String> meta) {

} catch (AlreadyExistsException e) {
throw new org.apache.iceberg.exceptions.AlreadyExistsException(
e, "Namespace '%s' already exists!", namespace);
e, "Namespace already exists: %s", namespace);

} catch (TException e) {
throw new RuntimeException(
Expand Down Expand Up @@ -500,6 +509,9 @@ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
return String.format("%s/%s", databaseData.getLocationUri(), tableIdentifier.name());
}

} catch (NoSuchObjectException e) {
throw new NoSuchNamespaceException(
e, "Namespace does not exist: %s", tableIdentifier.namespace().levels()[0]);
} catch (TException e) {
throw new RuntimeException(
String.format("Metastore operation failed for %s", tableIdentifier), e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ protected void doCommit(TableMetadata base, TableMetadata metadata) {
String baseMetadataLocation = base != null ? base.metadataFileLocation() : null;
if (!Objects.equals(baseMetadataLocation, metadataLocation)) {
throw new CommitFailedException(
"Base metadata location '%s' is not same as the current table metadata location '%s' for %s.%s",
"Cannot commit, Base metadata location '%s' is not same as the current table metadata location '%s' for %s.%s",
baseMetadataLocation, metadataLocation, database, tableName);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iceberg.hive;

import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;

public final class HiveMetastoreExtension implements AfterEachCallback, BeforeEachCallback {

HiveCatalog catalog;
HiveMetaStoreClient metastoreClient;
TestHiveMetastore metastore;
HiveConf hiveConf;
private final Map<String, String> hiveConfOverride;
static final String DB_NAME = "hivedb";

public HiveMetastoreExtension(Map<String, String> hiveConfOverride) {
this.hiveConfOverride = hiveConfOverride;
}

@Override
public void beforeEach(ExtensionContext extensionContext) throws Exception {
this.metastore = new TestHiveMetastore();
HiveConf hiveConfWithOverrides = new HiveConf(TestHiveMetastore.class);
if (hiveConfOverride != null) {
for (Map.Entry<String, String> kv : hiveConfOverride.entrySet()) {
hiveConfWithOverrides.set(kv.getKey(), kv.getValue());
}
}

metastore.start(hiveConfWithOverrides);
this.hiveConf = metastore.hiveConf();
this.metastoreClient = new HiveMetaStoreClient(hiveConfWithOverrides);

String dbPath = metastore.getDatabasePath(DB_NAME);
Database db = new Database(DB_NAME, "description", dbPath, Maps.newHashMap());
metastoreClient.createDatabase(db);

this.catalog =
(HiveCatalog)
CatalogUtil.loadCatalog(
HiveCatalog.class.getName(),
CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE,
ImmutableMap.of(
CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS,
String.valueOf(TimeUnit.SECONDS.toMillis(10))),
hiveConfWithOverrides);
}

@Override
public void afterEach(ExtensionContext extensionContext) throws Exception {
this.catalog = null;
metastoreClient.close();
this.metastoreClient = null;
metastore.stop();
this.metastore = null;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;

/*
* This meta-setup has been deprecated use {@link HiveMetastoreExtension} instead.
* */
@Deprecated
public abstract class HiveMetastoreTest {

protected static final String DB_NAME = "hivedb";
Expand Down
Loading

0 comments on commit 530933f

Please sign in to comment.