Skip to content

Commit

Permalink
reuse hive docker container
Browse files Browse the repository at this point in the history
  • Loading branch information
@mchades
mchades committed Aug 9, 2024
1 parent 97839ee commit 967cfb7
Show file tree
Hide file tree
Showing 20 changed files with 458 additions and 139 deletions.
6 changes: 1 addition & 5 deletions ...op/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,11 @@ public static void setup() throws IOException {
keytabPath = file.getAbsolutePath() + "/client.keytab";
containerSuite
.getKerberosHiveContainer()
.getContainer()
.copyFileFromContainer("/etc/admin.keytab", keytabPath);

String krb5TmpPath = file.getAbsolutePath() + "/krb5.conf_tmp";
String krb5Path = file.getAbsolutePath() + "/krb5.conf";
containerSuite
.getKerberosHiveContainer()
.getContainer()
.copyFileFromContainer("/etc/krb5.conf", krb5TmpPath);
containerSuite.getKerberosHiveContainer().copyFileFromContainer("/etc/krb5.conf", krb5TmpPath);

// Modify the krb5.conf and change the kdc and admin_server to the container IP
String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress();
Expand Down
39 changes: 15 additions & 24 deletions ...java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,25 +137,22 @@ public static void stop() {

private static void prepareKerberosConfig() throws IOException, KrbException {
// Keytab of the Gravitino SDK client
kerberosHiveContainer
.getContainer()
.copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
"/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);

createKeyTableForSchemaAndFileset();

// Keytab of the Gravitino server
kerberosHiveContainer
.getContainer()
.copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
"/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);

// Keytab of Gravitino server to connector to HDFS
kerberosHiveContainer
.getContainer()
.copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HADOOP_CLIENT_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
"/etc/admin.keytab", TMP_DIR + HADOOP_CLIENT_KEYTAB);

String tmpKrb5Path = TMP_DIR + "krb5.conf_tmp";
String krb5Path = TMP_DIR + "krb5.conf";
kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
kerberosHiveContainer.copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);

// Modify the krb5.conf and change the kdc and admin_server to the container IP
String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress();
Expand Down Expand Up @@ -190,10 +187,8 @@ private static void createKeyTableForSchemaAndFileset() throws IOException {
HADOOP_SCHEMA_PRINCIPAL,
REALM),
StandardCharsets.UTF_8);
kerberosHiveContainer
.getContainer()
.copyFileToContainer(
MountableFile.forHostPath(TMP_DIR + createSchemaShellFile), createSchemaShellFile);
kerberosHiveContainer.copyFileToContainer(
MountableFile.forHostPath(TMP_DIR + createSchemaShellFile), createSchemaShellFile);
kerberosHiveContainer.executeInContainer("bash", createSchemaShellFile);

FileUtils.writeStringToFile(
Expand All @@ -208,18 +203,14 @@ private static void createKeyTableForSchemaAndFileset() throws IOException {
HADOOP_FILESET_PRINCIPAL,
REALM),
StandardCharsets.UTF_8);
kerberosHiveContainer
.getContainer()
.copyFileToContainer(
MountableFile.forHostPath(TMP_DIR + createFileSetShellFile), createFileSetShellFile);
kerberosHiveContainer.copyFileToContainer(
MountableFile.forHostPath(TMP_DIR + createFileSetShellFile), createFileSetShellFile);
kerberosHiveContainer.executeInContainer("bash", createFileSetShellFile);

kerberosHiveContainer
.getContainer()
.copyFileFromContainer(HADOOP_SCHEMA_KEYTAB, TMP_DIR + HADOOP_SCHEMA_KEYTAB);
kerberosHiveContainer
.getContainer()
.copyFileFromContainer(HADOOP_FILESET_KEYTAB, TMP_DIR + HADOOP_FILESET_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
HADOOP_SCHEMA_KEYTAB, TMP_DIR + HADOOP_SCHEMA_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
HADOOP_FILESET_KEYTAB, TMP_DIR + HADOOP_FILESET_KEYTAB);
}

private static void addKerberosConfig() {
Expand Down
20 changes: 10 additions & 10 deletions ...est/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,23 +137,20 @@ public static void stop() {

private static void prepareKerberosConfig() throws Exception {
// Keytab of the Gravitino SDK client
kerberosHiveContainer
.getContainer()
.copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
"/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);

// Keytab of the Gravitino server
kerberosHiveContainer
.getContainer()
.copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
"/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);

// Keytab of Gravitino server to connector to Hive
kerberosHiveContainer
.getContainer()
.copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
"/etc/admin.keytab", TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB);

String tmpKrb5Path = TMP_DIR + "/krb5.conf_tmp";
String krb5Path = TMP_DIR + "/krb5.conf";
kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
kerberosHiveContainer.copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);

// Modify the krb5.conf and change the kdc and admin_server to the container IP
String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress();
Expand Down Expand Up @@ -235,6 +232,9 @@ public void testUserAuthentication() {
Assertions.assertTrue(
exceptionMessage.contains("Permission denied: user=gravitino_client, access=WRITE"));

// make sure the directory exists
kerberosHiveContainer.executeInContainer(
"hadoop", "fs", "-mkdir", "-p", "/user/hive/warehouse");
// Now try to give the user the permission to create schema again
kerberosHiveContainer.executeInContainer(
"hadoop", "fs", "-chmod", "-R", "777", "/user/hive/warehouse");
Expand Down
17 changes: 7 additions & 10 deletions ...he/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,23 +150,20 @@ public static void stop() {

private static void prepareKerberosConfig() throws Exception {
// Keytab of the Gravitino SDK client
kerberosHiveContainer
.getContainer()
.copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
"/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);

// Keytab of the Gravitino server
kerberosHiveContainer
.getContainer()
.copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
"/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);

// Keytab of Gravitino server to connector to Hive
kerberosHiveContainer
.getContainer()
.copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
"/etc/admin.keytab", TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB);

String tmpKrb5Path = TMP_DIR + "/krb5.conf_tmp";
String krb5Path = TMP_DIR + "/krb5.conf";
kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
kerberosHiveContainer.copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);

// Modify the krb5.conf and change the kdc and admin_server to the container IP
String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress();
Expand Down
9 changes: 5 additions & 4 deletions ...org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.gravitino.NameIdentifier;
import org.apache.gravitino.client.GravitinoMetalake;
import org.apache.gravitino.iceberg.common.IcebergConfig;
import org.apache.gravitino.integration.test.container.ContainerSuite;
import org.apache.gravitino.integration.test.container.MySQLContainer;
import org.apache.gravitino.integration.test.container.PostgreSQLContainer;
import org.apache.gravitino.integration.test.util.AbstractIT;
Expand All @@ -56,10 +57,10 @@ public class TestMultipleJDBCLoad extends AbstractIT {

@BeforeAll
public static void startup() throws IOException {
containerSuite.startMySQLContainer(TEST_DB_NAME);
mySQLContainer = containerSuite.getMySQLContainer();
containerSuite.startPostgreSQLContainer(TEST_DB_NAME);
postgreSQLContainer = containerSuite.getPostgreSQLContainer();
ContainerSuite.getInstance().startMySQLContainer(TEST_DB_NAME);
mySQLContainer = ContainerSuite.getInstance().getMySQLContainer();
ContainerSuite.getInstance().startPostgreSQLContainer(TEST_DB_NAME);
postgreSQLContainer = ContainerSuite.getInstance().getPostgreSQLContainer();
}

@Test
Expand Down
16 changes: 6 additions & 10 deletions ...ravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,23 +143,19 @@ public static void stop() {

private static void prepareKerberosConfig() throws Exception {
// Keytab of the Gravitino SDK client
kerberosHiveContainer
.getContainer()
.copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
"/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);

// Keytab of the Gravitino server
kerberosHiveContainer
.getContainer()
.copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
kerberosHiveContainer.copyFileFromContainer(
"/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);

// Keytab of Gravitino server to connector to Hive
kerberosHiveContainer
.getContainer()
.copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HDFS_CLIENT_KEYTAB);
kerberosHiveContainer.copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HDFS_CLIENT_KEYTAB);

String tmpKrb5Path = TMP_DIR + "/krb5.conf_tmp";
String krb5Path = TMP_DIR + "/krb5.conf";
kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
kerberosHiveContainer.copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);

// Modify the krb5.conf and change the kdc and admin_server to the container IP
String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress();
Expand Down
1 change: 0 additions & 1 deletion clients/client-python/gravitino/exceptions/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,4 +101,3 @@ class UnauthorizedException(GravitinoRuntimeException):

class BadRequestException(GravitinoRuntimeException):
"""An exception thrown when the request is invalid."""

10 changes: 9 additions & 1 deletion ...t-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.gravitino.integration.test.container;

import static java.util.Objects.requireNonNull;
import static org.apache.gravitino.integration.test.util.AbstractIT.ACTIVE_CI;
import static org.testcontainers.utility.MountableFile.forHostPath;

import com.github.dockerjava.api.DockerClient;
Expand Down Expand Up @@ -50,14 +51,20 @@
*/
public abstract class BaseContainer implements AutoCloseable {
public static final Logger LOG = LoggerFactory.getLogger(BaseContainer.class);

protected static final boolean activeCI = "true".equals(System.getenv(ACTIVE_CI));

// Host name of the container
protected final String hostName;
// Exposed ports of the container
private final Set<Integer> ports;
// Files to mount in the container
private final Map<String, String> filesToMount;
// environment variables of the container
private final Map<String, String> envVars;
protected final Map<String, String> envVars;

protected final String image;

// Additional host and IP address mapping
private final Map<String, String> extraHosts;
// Network of the container
Expand All @@ -81,6 +88,7 @@ protected BaseContainer(
.withSysctls(
Collections.singletonMap(
"net.ipv4.ip_local_port_range", "20000 40000")));
this.image = image;
this.ports = requireNonNull(ports, "ports is null");
this.hostName = requireNonNull(hostName, "hostName is null");
this.extraHosts = extraHosts;
Expand Down
6 changes: 6 additions & 0 deletions ...-common/src/test/java/org/apache/gravitino/integration/test/container/ContainerSuite.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
public class ContainerSuite implements Closeable {
public static final Logger LOG = LoggerFactory.getLogger(ContainerSuite.class);
private static volatile ContainerSuite instance = null;
private static volatile boolean initialized = false;

// The subnet must match the configuration in
// `dev/docker/tools/mac-docker-connector.conf`
Expand Down Expand Up @@ -87,11 +88,16 @@ private static void init() {
}
}

public static boolean initialized() {
return initialized;
}

public static ContainerSuite getInstance() {
if (instance == null) {
synchronized (ContainerSuite.class) {
if (instance == null) {
init();
initialized = true;
instance = new ContainerSuite();
}
}
Expand Down
Loading

0 comments on commit 967cfb7

Please sign in to comment.