From fd390d033eff331a75e83e2c97ad2dff18947abd Mon Sep 17 00:00:00 2001 From: yuqi Date: Fri, 17 May 2024 17:16:46 +0800 Subject: [PATCH 01/16] Add kerberos IT --- catalogs/catalog-hadoop/build.gradle.kts | 1 + .../test/KerberosizedHDFSTest.java | 100 +++++++++++++ .../test/container/BaseContainer.java | 2 +- .../test/container/ContainerSuite.java | 24 ++++ .../container/KerberosizedHDFSContainer.java | 131 ++++++++++++++++++ 5 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/KerberosizedHDFSTest.java create mode 100644 integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosizedHDFSContainer.java diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 605c827a345..4719a8126ea 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -99,6 +99,7 @@ tasks.test { doFirst { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.10") + environment("GRAVITINO_CI_KERBEROSIZED_HDFS_DOCKER_IMAGE", "hdfs_ci:test") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/KerberosizedHDFSTest.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/KerberosizedHDFSTest.java new file mode 100644 index 00000000000..0b021db595e --- /dev/null +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/KerberosizedHDFSTest.java @@ -0,0 +1,100 @@ +/* + * Copyright 2024 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.catalog.hadoop.integration.test; + +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.container.KerberosizedHDFSContainer; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.PrivilegedAction; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Tag("gravitino-docker-it") +public class KerberosizedHDFSTest { + private static final Logger LOG = LoggerFactory.getLogger(KerberosizedHDFSTest.class); + + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + private static final String CLIENT_PRINCIPAL = "client@EXAMPLE.COM"; + private static UserGroupInformation clientUGI; + + @BeforeAll + public static void setup() throws IOException { + containerSuite.startHDFSContainer(); + + containerSuite + .getHdfsContainer() + .getContainer() + .copyFileFromContainer("/tmp/client.keytab", "/tmp/client.keytab"); + + containerSuite + .getHdfsContainer() + .getContainer() + .copyFileFromContainer("/etc/krb5.conf", "/tmp/krb5.conf_tmp"); + + String ip = containerSuite.getHdfsContainer().getContainerIpAddress(); + + String content = + FileUtils.readFileToString(new File("/tmp/krb5.conf_tmp"), StandardCharsets.UTF_8); + content = content.replace("kdc = localhost", "kdc = " + ip + ":88"); + content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749"); + FileUtils.write(new File("/tmp/krb5.conf"), content, StandardCharsets.UTF_8); + + LOG.info("Kerberos kdc config:\n{}", content); + + System.setProperty("java.security.krb5.conf", "/tmp/krb5.conf"); + System.setProperty("sun.security.krb5.debug", "true"); + } + + @Test + public void testKerberoizeHDFS() throws IOException { + Configuration conf = new Configuration(); + conf.set("fs.defaultFS", defaultBaseLocation()); + conf.setBoolean("fs.hdfs.impl.disable.cache", true); + conf.set("hadoop.security.authentication", "kerberos"); + + UserGroupInformation.setConfiguration(conf); + clientUGI = + UserGroupInformation.loginUserFromKeytabAndReturnUGI( + CLIENT_PRINCIPAL, "/tmp/client.keytab"); + clientUGI.doAs( + (PrivilegedAction) + () -> { + try { + FileSystem fs = FileSystem.get(conf); + Path path = new Path("/user/client"); + Assertions.assertTrue(fs.exists(path)); + + path = new Path("/user/client/test1"); + fs.mkdirs(path); + + FileStatus status = fs.getFileStatus(path); + Assertions.assertEquals(755, status.getPermission().toOctal()); + return null; + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + private static String defaultBaseLocation() { + return String.format( + "hdfs://%s:%d/user/", + containerSuite.getHdfsContainer().getContainerIpAddress(), + KerberosizedHDFSContainer.HDFS_DEFAULTFS_PORT); + } +} diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java index 35cc6f54cc3..b68bc57bc98 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java @@ -88,7 +88,7 @@ protected void setupContainer() { .withCreateContainerCmdModifier(c -> c.withHostName(hostName)) .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) .waitingFor(Wait.forListeningPort()) - .withStartupTimeout(Duration.ofMinutes(5)); + .withStartupTimeout(Duration.ofMinutes(1)); network.ifPresent(net -> container.withNetwork(net).withNetworkAliases(hostName)); } diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java index 22b10306ab8..e5f20b39c5f 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java @@ -42,6 +42,7 @@ public class ContainerSuite implements Closeable { private static volatile TrinoITContainers trinoITContainers; private static volatile KafkaContainer kafkaContainer; private static volatile DorisContainer dorisContainer; + private static volatile KerberosizedHDFSContainer hdfsContainer; private static volatile MySQLContainer mySQLContainer; private static volatile MySQLContainer mySQLVersion5Container; @@ -65,6 +66,10 @@ private ContainerSuite() { } } + public KerberosizedHDFSContainer getHdfsContainer() { + return hdfsContainer; + } + public static ContainerSuite getInstance() { if (instance == null) { synchronized (ContainerSuite.class) { @@ -101,6 +106,25 @@ public void startHiveContainer() { } } + public void startHDFSContainer() { + if (hdfsContainer == null) { + synchronized (ContainerSuite.class) { + if (hdfsContainer == null) { + // Start HDFS container + KerberosizedHDFSContainer.Builder hiveBuilder = + KerberosizedHDFSContainer.builder() + .withHostName(KerberosizedHDFSContainer.HOST_NAME) + .withEnvVars(ImmutableMap.builder().build()) + .withExposePorts(ImmutableSet.of()) + .withNetwork(network); + KerberosizedHDFSContainer container = closer.register(hiveBuilder.build()); + container.start(); + hdfsContainer = container; + } + } + } + } + public void startTrinoContainer( String trinoConfDir, String trinoConnectorLibDir, diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosizedHDFSContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosizedHDFSContainer.java new file mode 100644 index 00000000000..22810356a7b --- /dev/null +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosizedHDFSContainer.java @@ -0,0 +1,131 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.integration.test.container; + +import static java.lang.String.format; +import static org.testcontainers.shaded.org.awaitility.Awaitility.await; + +import com.google.common.collect.ImmutableSet; +import java.time.Duration; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.rnorth.ducttape.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.Network; + +public class KerberosizedHDFSContainer extends BaseContainer { + public static final Logger LOG = LoggerFactory.getLogger(KerberosizedHDFSContainer.class); + + public static final String DEFAULT_IMAGE = + System.getenv("GRAVITINO_CI_KERBEROSIZED_HDFS_DOCKER_IMAGE"); + public static final String HOST_NAME = "hdfs_host"; + + private static final String HDFS_LOG_PATH = "/usr/local/hadoop/logs"; + public static final int HDFS_DEFAULTFS_PORT = 9000; + public static final int KDC_PORT = 88; + public static final int KDC_ADMIN_PORT = 749; + + public static KerberosizedHDFSContainer.Builder builder() { + return new KerberosizedHDFSContainer.Builder(); + } + + protected KerberosizedHDFSContainer( + String image, + String hostName, + Set ports, + Map extraHosts, + Map filesToMount, + Map envVars, + Optional network) { + super(image, hostName, ports, extraHosts, filesToMount, envVars, network); + } + + @Override + protected void setupContainer() { + super.setupContainer(); + withLogConsumer(new PrintingContainerLog(format("%-14s| ", "HDFSContainer"))); + withStartupTimeout(Duration.ofMinutes(5)); + } + + @Override + public void start() { + super.start(); + Preconditions.check("HDFS container startup failed!", checkContainerStatus(5)); + } + + @Override + public void close() { + copyHDFSLog(); + super.close(); + } + + private void copyHDFSLog() { + try { + String destPath = System.getenv("IT_PROJECT_DIR"); + LOG.info("Copy HDFS log file to {}", destPath); + + String hdfsTarFileName = "/hdfs.tar"; + + // Pack the jar files + container.execInContainer("tar", "cf", hdfsTarFileName, HDFS_LOG_PATH); + + // Copy the jar files + container.copyFileFromContainer(hdfsTarFileName, destPath + hdfsTarFileName); + } catch (Exception e) { + LOG.error("Failed to copy container log to local", e); + } + } + + @Override + protected boolean checkContainerStatus(int retryLimit) { + await() + .atMost(100, TimeUnit.SECONDS) + .pollInterval(100 / retryLimit, TimeUnit.SECONDS) + .until( + () -> { + try { + String[] commandAndArgs = new String[] {"bash", "/tmp/check-status.sh"}; + Container.ExecResult execResult = executeInContainer(commandAndArgs); + if (execResult.getExitCode() != 0) { + String message = + format( + "Command [%s] exited with %s", + String.join(" ", commandAndArgs), execResult.getExitCode()); + LOG.error("{}", message); + LOG.error("stderr: {}", execResult.getStderr()); + LOG.error("stdout: {}", execResult.getStdout()); + } else { + LOG.info("HDFS container startup success!"); + return true; + } + } catch (RuntimeException e) { + LOG.error(e.getMessage(), e); + } + return false; + }); + + return true; + } + + public static class Builder + extends BaseContainer.Builder { + private Builder() { + this.image = DEFAULT_IMAGE; + this.hostName = HOST_NAME; + this.exposePorts = ImmutableSet.of(HDFS_DEFAULTFS_PORT, KDC_PORT, KDC_ADMIN_PORT); + } + + @Override + public KerberosizedHDFSContainer build() { + return new KerberosizedHDFSContainer( + image, hostName, exposePorts, extraHosts, filesToMount, envVars, network); + } + } +} From 780e3756014a972f9d9f774b02dc1503802049c8 Mon Sep 17 00:00:00 2001 From: yuqi Date: Sat, 18 May 2024 08:30:49 +0800 Subject: [PATCH 02/16] Fix --- catalogs/catalog-hadoop/build.gradle.kts | 2 +- .../test/{KerberosizedHDFSTest.java => HDFSKerberosIT.java} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/{KerberosizedHDFSTest.java => HDFSKerberosIT.java} (96%) diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 4719a8126ea..09361dc7144 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -99,7 +99,7 @@ tasks.test { doFirst { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.10") - environment("GRAVITINO_CI_KERBEROSIZED_HDFS_DOCKER_IMAGE", "hdfs_ci:test") + environment("GRAVITINO_CI_KERBEROSIZED_HDFS_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hdfs:0.1.0") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/KerberosizedHDFSTest.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java similarity index 96% rename from catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/KerberosizedHDFSTest.java rename to catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java index 0b021db595e..70b4497e457 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/KerberosizedHDFSTest.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java @@ -25,8 +25,8 @@ import org.slf4j.LoggerFactory; @Tag("gravitino-docker-it") -public class KerberosizedHDFSTest { - private static final Logger LOG = LoggerFactory.getLogger(KerberosizedHDFSTest.class); +public class HDFSKerberosIT { + private static final Logger LOG = LoggerFactory.getLogger(HDFSKerberosIT.class); private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private static final String CLIENT_PRINCIPAL = "client@EXAMPLE.COM"; From 9c0a2edf182dc956fa5863ff539006cc282d3f0f Mon Sep 17 00:00:00 2001 From: yuqi Date: Sat, 18 May 2024 12:17:51 +0800 Subject: [PATCH 03/16] Fix test error --- .../catalog/hadoop/integration/test/HDFSKerberosIT.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java index 70b4497e457..d57ba98e18c 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java @@ -17,6 +17,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; @@ -60,6 +61,12 @@ public static void setup() throws IOException { System.setProperty("sun.security.krb5.debug", "true"); } + @AfterAll + public static void tearDown() { + // Reset the UGI + UserGroupInformation.reset(); + } + @Test public void testKerberoizeHDFS() throws IOException { Configuration conf = new Configuration(); From 3f06416fec1457198eb9e0f60924abba4c8538f3 Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 20 May 2024 09:54:12 +0800 Subject: [PATCH 04/16] Add fix. --- .../hadoop/integration/test/HDFSKerberosIT.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java index d57ba98e18c..b90454cbaeb 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java @@ -16,6 +16,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -68,7 +69,7 @@ public static void tearDown() { } @Test - public void testKerberoizeHDFS() throws IOException { + public void testKerberosHDFS() throws IOException { Configuration conf = new Configuration(); conf.set("fs.defaultFS", defaultBaseLocation()); conf.setBoolean("fs.hdfs.impl.disable.cache", true); @@ -78,7 +79,7 @@ public void testKerberoizeHDFS() throws IOException { clientUGI = UserGroupInformation.loginUserFromKeytabAndReturnUGI( CLIENT_PRINCIPAL, "/tmp/client.keytab"); - clientUGI.doAs( + PrivilegedAction action = (PrivilegedAction) () -> { try { @@ -95,7 +96,14 @@ public void testKerberoizeHDFS() throws IOException { } catch (IOException e) { throw new RuntimeException(e); } - }); + }; + + clientUGI.doAs(action); + + // Clear UGI, It will throw exception + UserGroupInformation.reset(); + Exception e = Assertions.assertThrows(Exception.class, action::run); + Assertions.assertTrue(e.getCause() instanceof AccessControlException); } private static String defaultBaseLocation() { From 85e6b733a86bc9bf32eeea425d5398c2fe3f415d Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 20 May 2024 09:54:53 +0800 Subject: [PATCH 05/16] fix --- .../catalog/hadoop/integration/test/HDFSKerberosIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java index b90454cbaeb..4e41d2d2839 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java @@ -103,7 +103,7 @@ public void testKerberosHDFS() throws IOException { // Clear UGI, It will throw exception UserGroupInformation.reset(); Exception e = Assertions.assertThrows(Exception.class, action::run); - Assertions.assertTrue(e.getCause() instanceof AccessControlException); + Assertions.assertInstanceOf(AccessControlException.class, e.getCause()); } private static String defaultBaseLocation() { From 66640fc5310076fb540a5a7c04a925ce919c84a3 Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 20 May 2024 10:16:56 +0800 Subject: [PATCH 06/16] fix --- catalogs/catalog-hadoop/build.gradle.kts | 2 +- .../integration/test/HDFSKerberosIT.java | 4 ++-- .../test/container/BaseContainer.java | 2 +- .../test/container/ContainerSuite.java | 12 ++++++------ ...ntainer.java => KerberosHDFSContainer.java} | 18 +++++++++--------- 5 files changed, 19 insertions(+), 19 deletions(-) rename integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/{KerberosizedHDFSContainer.java => KerberosHDFSContainer.java} (88%) diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 09361dc7144..0dea9098147 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -99,7 +99,7 @@ tasks.test { doFirst { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.10") - environment("GRAVITINO_CI_KERBEROSIZED_HDFS_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hdfs:0.1.0") + environment("GRAVITINO_CI_KERBEROS_HDFS_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hdfs:0.1.0") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java index 4e41d2d2839..1b32a5d2520 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java @@ -6,7 +6,7 @@ package com.datastrato.gravitino.catalog.hadoop.integration.test; import com.datastrato.gravitino.integration.test.container.ContainerSuite; -import com.datastrato.gravitino.integration.test.container.KerberosizedHDFSContainer; +import com.datastrato.gravitino.integration.test.container.KerberosHDFSContainer; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -110,6 +110,6 @@ private static String defaultBaseLocation() { return String.format( "hdfs://%s:%d/user/", containerSuite.getHdfsContainer().getContainerIpAddress(), - KerberosizedHDFSContainer.HDFS_DEFAULTFS_PORT); + KerberosHDFSContainer.HDFS_DEFAULTFS_PORT); } } diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java index b68bc57bc98..35cc6f54cc3 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java @@ -88,7 +88,7 @@ protected void setupContainer() { .withCreateContainerCmdModifier(c -> c.withHostName(hostName)) .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) .waitingFor(Wait.forListeningPort()) - .withStartupTimeout(Duration.ofMinutes(1)); + .withStartupTimeout(Duration.ofMinutes(5)); network.ifPresent(net -> container.withNetwork(net).withNetworkAliases(hostName)); } diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java index e5f20b39c5f..e8ca5edd2b2 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java @@ -42,7 +42,7 @@ public class ContainerSuite implements Closeable { private static volatile TrinoITContainers trinoITContainers; private static volatile KafkaContainer kafkaContainer; private static volatile DorisContainer dorisContainer; - private static volatile KerberosizedHDFSContainer hdfsContainer; + private static volatile KerberosHDFSContainer hdfsContainer; private static volatile MySQLContainer mySQLContainer; private static volatile MySQLContainer mySQLVersion5Container; @@ -66,7 +66,7 @@ private ContainerSuite() { } } - public KerberosizedHDFSContainer getHdfsContainer() { + public KerberosHDFSContainer getHdfsContainer() { return hdfsContainer; } @@ -111,13 +111,13 @@ public void startHDFSContainer() { synchronized (ContainerSuite.class) { if (hdfsContainer == null) { // Start HDFS container - KerberosizedHDFSContainer.Builder hiveBuilder = - KerberosizedHDFSContainer.builder() - .withHostName(KerberosizedHDFSContainer.HOST_NAME) + KerberosHDFSContainer.Builder hdfsBuilder = + KerberosHDFSContainer.builder() + .withHostName(KerberosHDFSContainer.HOST_NAME) .withEnvVars(ImmutableMap.builder().build()) .withExposePorts(ImmutableSet.of()) .withNetwork(network); - KerberosizedHDFSContainer container = closer.register(hiveBuilder.build()); + KerberosHDFSContainer container = closer.register(hdfsBuilder.build()); container.start(); hdfsContainer = container; } diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosizedHDFSContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosHDFSContainer.java similarity index 88% rename from integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosizedHDFSContainer.java rename to integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosHDFSContainer.java index 22810356a7b..3b29bfe1b74 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosizedHDFSContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosHDFSContainer.java @@ -20,11 +20,11 @@ import org.testcontainers.containers.Container; import org.testcontainers.containers.Network; -public class KerberosizedHDFSContainer extends BaseContainer { - public static final Logger LOG = LoggerFactory.getLogger(KerberosizedHDFSContainer.class); +public class KerberosHDFSContainer extends BaseContainer { + public static final Logger LOG = LoggerFactory.getLogger(KerberosHDFSContainer.class); public static final String DEFAULT_IMAGE = - System.getenv("GRAVITINO_CI_KERBEROSIZED_HDFS_DOCKER_IMAGE"); + System.getenv("GRAVITINO_CI_KERBEROS_HDFS_DOCKER_IMAGE"); public static final String HOST_NAME = "hdfs_host"; private static final String HDFS_LOG_PATH = "/usr/local/hadoop/logs"; @@ -32,11 +32,11 @@ public class KerberosizedHDFSContainer extends BaseContainer { public static final int KDC_PORT = 88; public static final int KDC_ADMIN_PORT = 749; - public static KerberosizedHDFSContainer.Builder builder() { - return new KerberosizedHDFSContainer.Builder(); + public static KerberosHDFSContainer.Builder builder() { + return new KerberosHDFSContainer.Builder(); } - protected KerberosizedHDFSContainer( + protected KerberosHDFSContainer( String image, String hostName, Set ports, @@ -115,7 +115,7 @@ protected boolean checkContainerStatus(int retryLimit) { } public static class Builder - extends BaseContainer.Builder { + extends BaseContainer.Builder { private Builder() { this.image = DEFAULT_IMAGE; this.hostName = HOST_NAME; @@ -123,8 +123,8 @@ private Builder() { } @Override - public KerberosizedHDFSContainer build() { - return new KerberosizedHDFSContainer( + public KerberosHDFSContainer build() { + return new KerberosHDFSContainer( image, hostName, exposePorts, extraHosts, filesToMount, envVars, network); } } From 01dbf609d0c83027a3bf51aff71f1aac42bbccff Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 20 May 2024 21:10:49 +0800 Subject: [PATCH 07/16] Fix --- catalogs/catalog-hadoop/build.gradle.kts | 2 +- .../integration/test/HDFSKerberosIT.java | 25 +++++------- .../test/container/BaseContainer.java | 8 +++- .../test/container/ContainerSuite.java | 27 +++++++++++++ .../test/container/HiveContainer.java | 39 +++++++------------ 5 files changed, 59 insertions(+), 42 deletions(-) diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 0dea9098147..d630e60a193 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -99,7 +99,7 @@ tasks.test { doFirst { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.10") - environment("GRAVITINO_CI_KERBEROS_HDFS_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hdfs:0.1.0") + environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.0") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java index 1b32a5d2520..f6a6ec454fe 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java @@ -13,7 +13,6 @@ import java.security.PrivilegedAction; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.AccessControlException; @@ -31,28 +30,28 @@ public class HDFSKerberosIT { private static final Logger LOG = LoggerFactory.getLogger(HDFSKerberosIT.class); private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private static final String CLIENT_PRINCIPAL = "client@EXAMPLE.COM"; + private static final String CLIENT_PRINCIPAL = "cli@HADOOPKRB"; private static UserGroupInformation clientUGI; @BeforeAll public static void setup() throws IOException { - containerSuite.startHDFSContainer(); + containerSuite.startKerberosHiveContainer(); containerSuite - .getHdfsContainer() + .getKerberosHiveContainer() .getContainer() - .copyFileFromContainer("/tmp/client.keytab", "/tmp/client.keytab"); + .copyFileFromContainer("/etc/admin.keytab", "/tmp/client.keytab"); containerSuite - .getHdfsContainer() + .getKerberosHiveContainer() .getContainer() .copyFileFromContainer("/etc/krb5.conf", "/tmp/krb5.conf_tmp"); - String ip = containerSuite.getHdfsContainer().getContainerIpAddress(); + String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress(); String content = FileUtils.readFileToString(new File("/tmp/krb5.conf_tmp"), StandardCharsets.UTF_8); - content = content.replace("kdc = localhost", "kdc = " + ip + ":88"); + content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88"); content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749"); FileUtils.write(new File("/tmp/krb5.conf"), content, StandardCharsets.UTF_8); @@ -84,14 +83,8 @@ public void testKerberosHDFS() throws IOException { () -> { try { FileSystem fs = FileSystem.get(conf); - Path path = new Path("/user/client"); + Path path = new Path("/"); Assertions.assertTrue(fs.exists(path)); - - path = new Path("/user/client/test1"); - fs.mkdirs(path); - - FileStatus status = fs.getFileStatus(path); - Assertions.assertEquals(755, status.getPermission().toOctal()); return null; } catch (IOException e) { throw new RuntimeException(e); @@ -109,7 +102,7 @@ public void testKerberosHDFS() throws IOException { private static String defaultBaseLocation() { return String.format( "hdfs://%s:%d/user/", - containerSuite.getHdfsContainer().getContainerIpAddress(), + containerSuite.getKerberosHiveContainer().getContainerIpAddress(), KerberosHDFSContainer.HDFS_DEFAULTFS_PORT); } } diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java index 35cc6f54cc3..059f48dab6e 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java @@ -36,7 +36,7 @@ public abstract class BaseContainer implements AutoCloseable { public static final Logger LOG = LoggerFactory.getLogger(BaseContainer.class); // Host name of the container - private final String hostName; + protected final String hostName; // Exposed ports of the container private final Set ports; // Files to mount in the container @@ -153,6 +153,7 @@ protected abstract static class Builder< SELF extends Builder, CONTAINER extends BaseContainer> { protected String image; protected String hostName; + protected boolean kerberosEnabled; protected Set exposePorts = ImmutableSet.of(); protected Map extraHosts = ImmutableMap.of(); protected Map filesToMount = ImmutableMap.of(); @@ -201,6 +202,11 @@ public SELF withNetwork(Network network) { return self; } + public SELF withKerberosEnabled(boolean kerberosEnabled) { + this.kerberosEnabled = kerberosEnabled; + return self; + } + public abstract CONTAINER build(); } } diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java index e8ca5edd2b2..969ace827d8 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java @@ -43,6 +43,7 @@ public class ContainerSuite implements Closeable { private static volatile KafkaContainer kafkaContainer; private static volatile DorisContainer dorisContainer; private static volatile KerberosHDFSContainer hdfsContainer; + private static volatile HiveContainer kerberosHiveContainer; private static volatile MySQLContainer mySQLContainer; private static volatile MySQLContainer mySQLVersion5Container; @@ -106,6 +107,28 @@ public void startHiveContainer() { } } + public void startKerberosHiveContainer() { + if (kerberosHiveContainer == null) { + synchronized (ContainerSuite.class) { + if (kerberosHiveContainer == null) { + // Start Hive container + HiveContainer.Builder hiveBuilder = + HiveContainer.builder() + .withHostName("gravitino-ci-kerberos-hive") + .withEnvVars( + ImmutableMap.builder() + .put("HADOOP_USER_NAME", "datastrato") + .build()) + .withKerberosEnabled(true) + .withNetwork(network); + HiveContainer container = closer.register(hiveBuilder.build()); + container.start(); + kerberosHiveContainer = container; + } + } + } + } + public void startHDFSContainer() { if (hdfsContainer == null) { synchronized (ContainerSuite.class) { @@ -308,6 +331,10 @@ public HiveContainer getHiveContainer() { return hiveContainer; } + public HiveContainer getKerberosHiveContainer() { + return kerberosHiveContainer; + } + public DorisContainer getDorisContainer() { return dorisContainer; } diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java index 1006b957dbd..1ad029f8a73 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java @@ -25,6 +25,8 @@ public class HiveContainer extends BaseContainer { public static final Logger LOG = LoggerFactory.getLogger(HiveContainer.class); public static final String DEFAULT_IMAGE = System.getenv("GRAVITINO_CI_HIVE_DOCKER_IMAGE"); + public static final String KERBEROS_IMAGE = + System.getenv("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE"); public static final String HOST_NAME = "gravitino-ci-hive"; private static final int MYSQL_PORT = 3306; public static final int HDFS_DEFAULTFS_PORT = 9000; @@ -51,7 +53,7 @@ protected HiveContainer( @Override protected void setupContainer() { super.setupContainer(); - withLogConsumer(new PrintingContainerLog(format("%-14s| ", "HiveContainer"))); + withLogConsumer(new PrintingContainerLog(format("%-14s| ", "HiveContainer-" + hostName))); } @Override @@ -113,35 +115,18 @@ protected boolean checkContainerStatus(int retryLimit) { return false; }); - String sql = "show databases"; - await() - .atMost(30, TimeUnit.SECONDS) - .pollInterval(30 / retryLimit, TimeUnit.SECONDS) - .until( - () -> { - try { - Container.ExecResult result = executeInContainer("hive", "-e", sql); - if (result.getStdout().contains("default")) { - return true; - } - } catch (Exception e) { - LOG.error("Failed to execute sql: {}", sql, e); - } - return false; - }); + // Kerberized Hive can't execute '', so we remove the related check. + String sql = + "CREATE TABLE IF NOT EXISTS default.employee ( eid int, name String, " + + "salary String, destination String) "; await() .atMost(30, TimeUnit.SECONDS) .pollInterval(30 / retryLimit, TimeUnit.SECONDS) .until( () -> { try { - Container.ExecResult result = - executeInContainer( - "hive", - "-e", - "CREATE TABLE IF NOT EXISTS default.employee ( eid int, name String, " - + "salary String, destination String) "); + Container.ExecResult result = executeInContainer("hive", "-e", sql); if (result.getExitCode() == 0) { return true; } @@ -183,7 +168,13 @@ private Builder() { @Override public HiveContainer build() { return new HiveContainer( - image, hostName, exposePorts, extraHosts, filesToMount, envVars, network); + kerberosEnabled ? KERBEROS_IMAGE : image, + kerberosEnabled ? "kerberos-" + hostName : hostName, + exposePorts, + extraHosts, + filesToMount, + envVars, + network); } } } From e71dde30edf4736a83fc6310f759a6ea86fb0f30 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 21 May 2024 11:53:47 +0800 Subject: [PATCH 08/16] Remove unused code. --- .../integration/test/HDFSKerberosIT.java | 4 +- .../test/container/ContainerSuite.java | 24 ---- .../test/container/KerberosHDFSContainer.java | 131 ------------------ 3 files changed, 2 insertions(+), 157 deletions(-) delete mode 100644 integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosHDFSContainer.java diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java index f6a6ec454fe..def653cc9c2 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java @@ -6,7 +6,7 @@ package com.datastrato.gravitino.catalog.hadoop.integration.test; import com.datastrato.gravitino.integration.test.container.ContainerSuite; -import com.datastrato.gravitino.integration.test.container.KerberosHDFSContainer; +import com.datastrato.gravitino.integration.test.container.HiveContainer; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -103,6 +103,6 @@ private static String defaultBaseLocation() { return String.format( "hdfs://%s:%d/user/", containerSuite.getKerberosHiveContainer().getContainerIpAddress(), - KerberosHDFSContainer.HDFS_DEFAULTFS_PORT); + HiveContainer.HDFS_DEFAULTFS_PORT); } } diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java index 969ace827d8..0846718dee3 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java @@ -42,7 +42,6 @@ public class ContainerSuite implements Closeable { private static volatile TrinoITContainers trinoITContainers; private static volatile KafkaContainer kafkaContainer; private static volatile DorisContainer dorisContainer; - private static volatile KerberosHDFSContainer hdfsContainer; private static volatile HiveContainer kerberosHiveContainer; private static volatile MySQLContainer mySQLContainer; @@ -67,10 +66,6 @@ private ContainerSuite() { } } - public KerberosHDFSContainer getHdfsContainer() { - return hdfsContainer; - } - public static ContainerSuite getInstance() { if (instance == null) { synchronized (ContainerSuite.class) { @@ -129,25 +124,6 @@ public void startKerberosHiveContainer() { } } - public void startHDFSContainer() { - if (hdfsContainer == null) { - synchronized (ContainerSuite.class) { - if (hdfsContainer == null) { - // Start HDFS container - KerberosHDFSContainer.Builder hdfsBuilder = - KerberosHDFSContainer.builder() - .withHostName(KerberosHDFSContainer.HOST_NAME) - .withEnvVars(ImmutableMap.builder().build()) - .withExposePorts(ImmutableSet.of()) - .withNetwork(network); - KerberosHDFSContainer container = closer.register(hdfsBuilder.build()); - container.start(); - hdfsContainer = container; - } - } - } - } - public void startTrinoContainer( String trinoConfDir, String trinoConnectorLibDir, diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosHDFSContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosHDFSContainer.java deleted file mode 100644 index 3b29bfe1b74..00000000000 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/KerberosHDFSContainer.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino.integration.test.container; - -import static java.lang.String.format; -import static org.testcontainers.shaded.org.awaitility.Awaitility.await; - -import com.google.common.collect.ImmutableSet; -import java.time.Duration; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import org.rnorth.ducttape.Preconditions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.Container; -import org.testcontainers.containers.Network; - -public class KerberosHDFSContainer extends BaseContainer { - public static final Logger LOG = LoggerFactory.getLogger(KerberosHDFSContainer.class); - - public static final String DEFAULT_IMAGE = - System.getenv("GRAVITINO_CI_KERBEROS_HDFS_DOCKER_IMAGE"); - public static final String HOST_NAME = "hdfs_host"; - - private static final String HDFS_LOG_PATH = "/usr/local/hadoop/logs"; - public static final int HDFS_DEFAULTFS_PORT = 9000; - public static final int KDC_PORT = 88; - public static final int KDC_ADMIN_PORT = 749; - - public static KerberosHDFSContainer.Builder builder() { - return new KerberosHDFSContainer.Builder(); - } - - protected KerberosHDFSContainer( - String image, - String hostName, - Set ports, - Map extraHosts, - Map filesToMount, - Map envVars, - Optional network) { - super(image, hostName, ports, extraHosts, filesToMount, envVars, network); - } - - @Override - protected void setupContainer() { - super.setupContainer(); - withLogConsumer(new PrintingContainerLog(format("%-14s| ", "HDFSContainer"))); - withStartupTimeout(Duration.ofMinutes(5)); - } - - @Override - public void start() { - super.start(); - Preconditions.check("HDFS container startup failed!", checkContainerStatus(5)); - } - - @Override - public void close() { - copyHDFSLog(); - super.close(); - } - - private void copyHDFSLog() { - try { - String destPath = System.getenv("IT_PROJECT_DIR"); - LOG.info("Copy HDFS log file to {}", destPath); - - String hdfsTarFileName = "/hdfs.tar"; - - // Pack the jar files - container.execInContainer("tar", "cf", hdfsTarFileName, HDFS_LOG_PATH); - - // Copy the jar files - container.copyFileFromContainer(hdfsTarFileName, destPath + hdfsTarFileName); - } catch (Exception e) { - LOG.error("Failed to copy container log to local", e); - } - } - - @Override - protected boolean checkContainerStatus(int retryLimit) { - await() - .atMost(100, TimeUnit.SECONDS) - .pollInterval(100 / retryLimit, TimeUnit.SECONDS) - .until( - () -> { - try { - String[] commandAndArgs = new String[] {"bash", "/tmp/check-status.sh"}; - Container.ExecResult execResult = executeInContainer(commandAndArgs); - if (execResult.getExitCode() != 0) { - String message = - format( - "Command [%s] exited with %s", - String.join(" ", commandAndArgs), execResult.getExitCode()); - LOG.error("{}", message); - LOG.error("stderr: {}", execResult.getStderr()); - LOG.error("stdout: {}", execResult.getStdout()); - } else { - LOG.info("HDFS container startup success!"); - return true; - } - } catch (RuntimeException e) { - LOG.error(e.getMessage(), e); - } - return false; - }); - - return true; - } - - public static class Builder - extends BaseContainer.Builder { - private Builder() { - this.image = DEFAULT_IMAGE; - this.hostName = HOST_NAME; - this.exposePorts = ImmutableSet.of(HDFS_DEFAULTFS_PORT, KDC_PORT, KDC_ADMIN_PORT); - } - - @Override - public KerberosHDFSContainer build() { - return new KerberosHDFSContainer( - image, hostName, exposePorts, extraHosts, filesToMount, envVars, network); - } - } -} From 8aa9b4dc0a0d161d422afe6e56527e1893be45d0 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 21 May 2024 16:34:30 +0800 Subject: [PATCH 09/16] optimize code. --- catalogs/catalog-hadoop/build.gradle.kts | 2 +- .../catalog/hadoop/integration/test/HDFSKerberosIT.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index d630e60a193..72b58e35300 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -99,7 +99,7 @@ tasks.test { doFirst { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.10") - environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.0") + environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "kerberos-hive:test") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java index def653cc9c2..2b62fe29fb4 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java @@ -37,6 +37,7 @@ public class HDFSKerberosIT { public static void setup() throws IOException { containerSuite.startKerberosHiveContainer(); + // Copy the keytab and krb5.conf from the container containerSuite .getKerberosHiveContainer() .getContainer() @@ -49,6 +50,7 @@ public static void setup() throws IOException { String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress(); + // Modify the krb5.conf and change the kdc and admin_server to the container IP String content = FileUtils.readFileToString(new File("/tmp/krb5.conf_tmp"), StandardCharsets.UTF_8); content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88"); From 30d4c60662c9b559806c3a2e19145e112d0c0974 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 21 May 2024 16:44:36 +0800 Subject: [PATCH 10/16] Fix mistake --- catalogs/catalog-hadoop/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 72b58e35300..d630e60a193 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -99,7 +99,7 @@ tasks.test { doFirst { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.10") - environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "kerberos-hive:test") + environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.0") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit From 965f63a020bd3678c4cfb955e05d1ddc9a9a32c2 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 21 May 2024 17:14:30 +0800 Subject: [PATCH 11/16] Revert the code that check status of `show databases` for Hive container. --- .../test/container/HiveContainer.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java index 1ad029f8a73..884846ee149 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java @@ -115,9 +115,23 @@ protected boolean checkContainerStatus(int retryLimit) { return false; }); - // Kerberized Hive can't execute '', so we remove the related check. - - String sql = + final String showDatabaseSQL = "show databases"; + await() + .atMost(30, TimeUnit.SECONDS) + .pollInterval(30 / retryLimit, TimeUnit.SECONDS) + .until( + () -> { + try { + Container.ExecResult result = executeInContainer("hive", "-e", showDatabaseSQL); + if (result.getStdout().contains("default")) { + return true; + } + } catch (Exception e) { + LOG.error("Failed to execute sql: {}", showDatabaseSQL, e); + } + return false; + }); + final String createTableSQL = "CREATE TABLE IF NOT EXISTS default.employee ( eid int, name String, " + "salary String, destination String) "; await() @@ -126,12 +140,12 @@ protected boolean checkContainerStatus(int retryLimit) { .until( () -> { try { - Container.ExecResult result = executeInContainer("hive", "-e", sql); + Container.ExecResult result = executeInContainer("hive", "-e", createTableSQL); if (result.getExitCode() == 0) { return true; } } catch (Exception e) { - LOG.error("Failed to execute sql: {}", sql, e); + LOG.error("Failed to execute sql: {}", createTableSQL, e); } return false; }); From 2cb6eb6d9fffe90968b1ac8be805c42758515255 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 22 May 2024 19:56:39 +0800 Subject: [PATCH 12/16] Resolve comments --- .../integration/test/HDFSKerberosIT.java | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java index 2b62fe29fb4..a8d3f657f24 100644 --- a/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java +++ b/catalogs/catalog-hadoop/src/test/java/com/datastrato/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java @@ -10,6 +10,7 @@ import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.security.PrivilegedAction; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -33,33 +34,40 @@ public class HDFSKerberosIT { private static final String CLIENT_PRINCIPAL = "cli@HADOOPKRB"; private static UserGroupInformation clientUGI; + private static String keytabPath; + @BeforeAll public static void setup() throws IOException { containerSuite.startKerberosHiveContainer(); + File baseDir = new File(System.getProperty("java.io.tmpdir")); + File file = Files.createTempDirectory(baseDir.toPath(), "test").toFile(); + file.deleteOnExit(); + // Copy the keytab and krb5.conf from the container + keytabPath = file.getAbsolutePath() + "/client.keytab"; containerSuite .getKerberosHiveContainer() .getContainer() - .copyFileFromContainer("/etc/admin.keytab", "/tmp/client.keytab"); + .copyFileFromContainer("/etc/admin.keytab", keytabPath); + String krb5TmpPath = file.getAbsolutePath() + "/krb5.conf_tmp"; + String krb5Path = file.getAbsolutePath() + "/krb5.conf"; containerSuite .getKerberosHiveContainer() .getContainer() - .copyFileFromContainer("/etc/krb5.conf", "/tmp/krb5.conf_tmp"); - - String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress(); + .copyFileFromContainer("/etc/krb5.conf", krb5TmpPath); // Modify the krb5.conf and change the kdc and admin_server to the container IP - String content = - FileUtils.readFileToString(new File("/tmp/krb5.conf_tmp"), StandardCharsets.UTF_8); + String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress(); + String content = FileUtils.readFileToString(new File(krb5TmpPath), StandardCharsets.UTF_8); content = content.replace("kdc = localhost:88", "kdc = " + ip + ":88"); content = content.replace("admin_server = localhost", "admin_server = " + ip + ":749"); - FileUtils.write(new File("/tmp/krb5.conf"), content, StandardCharsets.UTF_8); + FileUtils.write(new File(krb5Path), content, StandardCharsets.UTF_8); LOG.info("Kerberos kdc config:\n{}", content); - System.setProperty("java.security.krb5.conf", "/tmp/krb5.conf"); + System.setProperty("java.security.krb5.conf", krb5Path); System.setProperty("sun.security.krb5.debug", "true"); } @@ -67,6 +75,10 @@ public static void setup() throws IOException { public static void tearDown() { // Reset the UGI UserGroupInformation.reset(); + + // Clean up the kerberos configuration + System.clearProperty("java.security.krb5.conf"); + System.clearProperty("sun.security.krb5.debug"); } @Test @@ -77,9 +89,7 @@ public void testKerberosHDFS() throws IOException { conf.set("hadoop.security.authentication", "kerberos"); UserGroupInformation.setConfiguration(conf); - clientUGI = - UserGroupInformation.loginUserFromKeytabAndReturnUGI( - CLIENT_PRINCIPAL, "/tmp/client.keytab"); + clientUGI = UserGroupInformation.loginUserFromKeytabAndReturnUGI(CLIENT_PRINCIPAL, keytabPath); PrivilegedAction action = (PrivilegedAction) () -> { From 83d661aea9596c1bb17543e882998c9c0a750091 Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 22 May 2024 21:03:16 +0800 Subject: [PATCH 13/16] Remove used environment --- .../gravitino/integration/test/container/ContainerSuite.java | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java index 0846718dee3..06e77df2394 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java @@ -112,7 +112,6 @@ public void startKerberosHiveContainer() { .withHostName("gravitino-ci-kerberos-hive") .withEnvVars( ImmutableMap.builder() - .put("HADOOP_USER_NAME", "datastrato") .build()) .withKerberosEnabled(true) .withNetwork(network); From 64dbfc407e2666dd255c10e75d91c85998c96b5b Mon Sep 17 00:00:00 2001 From: yuqi Date: Wed, 22 May 2024 23:01:02 +0800 Subject: [PATCH 14/16] Fix style --- .../gravitino/integration/test/container/ContainerSuite.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java index 06e77df2394..082a4a82c19 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java @@ -110,9 +110,7 @@ public void startKerberosHiveContainer() { HiveContainer.Builder hiveBuilder = HiveContainer.builder() .withHostName("gravitino-ci-kerberos-hive") - .withEnvVars( - ImmutableMap.builder() - .build()) + .withEnvVars(ImmutableMap.builder().build()) .withKerberosEnabled(true) .withNetwork(network); HiveContainer container = closer.register(hiveBuilder.build()); From 7a2b2a39e52bd1b6a89188d77a20f1b47ee7dfae Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 23 May 2024 09:23:40 +0800 Subject: [PATCH 15/16] Increase retry number. --- .../gravitino/integration/test/container/HiveContainer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java index 884846ee149..e5a33f378db 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java @@ -59,7 +59,7 @@ protected void setupContainer() { @Override public void start() { super.start(); - Preconditions.check("Hive container startup failed!", checkContainerStatus(10)); + Preconditions.check("Hive container startup failed!", checkContainerStatus(15)); } @Override @@ -90,7 +90,7 @@ private void copyHiveLog() { @Override protected boolean checkContainerStatus(int retryLimit) { await() - .atMost(100, TimeUnit.SECONDS) + .atMost(150, TimeUnit.SECONDS) .pollInterval(100 / retryLimit, TimeUnit.SECONDS) .until( () -> { From 2089149a30c35556b863d48551a9cdcdba85cb66 Mon Sep 17 00:00:00 2001 From: yuqi Date: Thu, 23 May 2024 09:24:12 +0800 Subject: [PATCH 16/16] fix --- .../gravitino/integration/test/container/HiveContainer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java index e5a33f378db..14e5cb484dd 100644 --- a/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java +++ b/integration-test-common/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java @@ -91,7 +91,7 @@ private void copyHiveLog() { protected boolean checkContainerStatus(int retryLimit) { await() .atMost(150, TimeUnit.SECONDS) - .pollInterval(100 / retryLimit, TimeUnit.SECONDS) + .pollInterval(150 / retryLimit, TimeUnit.SECONDS) .until( () -> { try {