reuse hive docker container in CI

mchades · Sep 1, 2024 · 2e2e378 · 2e2e378
1 parent 07cc7a6
commit 2e2e378
Show file tree

Hide file tree

Showing 27 changed files with 684 additions and 133 deletions.
diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
@@ -161,10 +161,31 @@ jobs:
         run: |
           dev/ci/util_free_space.sh
 
+      #      - name: Setup debug Github Action
+      #        if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }}
+      #        uses: csexton/debugger-action@master
+
+      - name: Setup reused hive services
+        run: |
+          free -h
+          docker compose -f integration-test-common/docker-script/docker-compose.yaml up -d hive hive_with_kerberos || true
+          if [ $? -ne 0 ]; then
+            docker compose integration-test/trino-it/docker-compose.yaml ps
+            docker compose integration-test/trino-it/docker-compose.yaml logs
+          
+            exit 1
+          else
+          echo "Services started successfully"
+          fi
+
+      - name: check services status
+        run: |
+          docker compose -f integration-test-common/docker-script/docker-compose.yaml ps
+
       - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }})
         id: integrationTest
         run: >
-          ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
+          export ACTIVE_CI=true && ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false
           -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test 
           -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test 
           -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test

diff --git a/.github/workflows/spark-integration-test.yml b/.github/workflows/spark-integration-test.yml
@@ -91,7 +91,7 @@ jobs:
         id: integrationTest
         run: |
           if [ "${{ matrix.scala-version }}" == "2.12" ];then 
-            ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
+          ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
           fi
           ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "org.apache.gravitino.spark.connector.integration.test.**"
           ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**"

diff --git a/...test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java b/...test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java
@@ -46,6 +46,7 @@
 import org.apache.gravitino.catalog.hive.HiveConstants;
 import org.apache.gravitino.client.GravitinoMetalake;
 import org.apache.gravitino.connector.AuthorizationPropertiesMeta;
+import org.apache.gravitino.integration.test.container.ContainerSuite;
 import org.apache.gravitino.integration.test.container.HiveContainer;
 import org.apache.gravitino.integration.test.container.RangerContainer;
 import org.apache.gravitino.integration.test.util.AbstractIT;
@@ -73,6 +74,7 @@
 @Tag("gravitino-docker-test")
 public class RangerHiveE2EIT extends AbstractIT {
   private static final Logger LOG = LoggerFactory.getLogger(RangerHiveE2EIT.class);
+  private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
 
   private static RangerAuthorizationPlugin rangerAuthPlugin;
   public static final String metalakeName =

diff --git a/...op/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java b/...op/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HDFSKerberosIT.java
@@ -63,15 +63,11 @@ public static void setup() throws IOException {
     keytabPath = file.getAbsolutePath() + "/client.keytab";
     containerSuite
         .getKerberosHiveContainer()
-        .getContainer()
         .copyFileFromContainer("/etc/admin.keytab", keytabPath);
 
     String krb5TmpPath = file.getAbsolutePath() + "/krb5.conf_tmp";
     String krb5Path = file.getAbsolutePath() + "/krb5.conf";
-    containerSuite
-        .getKerberosHiveContainer()
-        .getContainer()
-        .copyFileFromContainer("/etc/krb5.conf", krb5TmpPath);
+    containerSuite.getKerberosHiveContainer().copyFileFromContainer("/etc/krb5.conf", krb5TmpPath);
 
     // Modify the krb5.conf and change the kdc and admin_server to the container IP
     String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress();

diff --git a/...java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java b/...java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java
@@ -137,25 +137,22 @@ public static void stop() {
 
   private static void prepareKerberosConfig() throws IOException, KrbException {
     // Keytab of the Gravitino SDK client
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        "/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
 
     createKeyTableForSchemaAndFileset();
 
     // Keytab of the Gravitino server
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        "/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
 
     // Keytab of Gravitino server to connector to HDFS
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HADOOP_CLIENT_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        "/etc/admin.keytab", TMP_DIR + HADOOP_CLIENT_KEYTAB);
 
     String tmpKrb5Path = TMP_DIR + "krb5.conf_tmp";
     String krb5Path = TMP_DIR + "krb5.conf";
-    kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
+    kerberosHiveContainer.copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
 
     // Modify the krb5.conf and change the kdc and admin_server to the container IP
     String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress();
@@ -190,10 +187,8 @@ private static void createKeyTableForSchemaAndFileset() throws IOException {
             HADOOP_SCHEMA_PRINCIPAL,
             REALM),
         StandardCharsets.UTF_8);
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileToContainer(
-            MountableFile.forHostPath(TMP_DIR + createSchemaShellFile), createSchemaShellFile);
+    kerberosHiveContainer.copyFileToContainer(
+        MountableFile.forHostPath(TMP_DIR + createSchemaShellFile), createSchemaShellFile);
     kerberosHiveContainer.executeInContainer("bash", createSchemaShellFile);
 
     FileUtils.writeStringToFile(
@@ -208,18 +203,14 @@ private static void createKeyTableForSchemaAndFileset() throws IOException {
             HADOOP_FILESET_PRINCIPAL,
             REALM),
         StandardCharsets.UTF_8);
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileToContainer(
-            MountableFile.forHostPath(TMP_DIR + createFileSetShellFile), createFileSetShellFile);
+    kerberosHiveContainer.copyFileToContainer(
+        MountableFile.forHostPath(TMP_DIR + createFileSetShellFile), createFileSetShellFile);
     kerberosHiveContainer.executeInContainer("bash", createFileSetShellFile);
 
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer(HADOOP_SCHEMA_KEYTAB, TMP_DIR + HADOOP_SCHEMA_KEYTAB);
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer(HADOOP_FILESET_KEYTAB, TMP_DIR + HADOOP_FILESET_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        HADOOP_SCHEMA_KEYTAB, TMP_DIR + HADOOP_SCHEMA_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        HADOOP_FILESET_KEYTAB, TMP_DIR + HADOOP_FILESET_KEYTAB);
   }
 
   private static void addKerberosConfig() {
@@ -256,7 +247,7 @@ public void testUserAuthentication() {
     properties.put(PRINCIPAL_KEY, HADOOP_CLIENT_PRINCIPAL);
     properties.put("location", HDFS_URL + "/user/hadoop/");
 
-    kerberosHiveContainer.executeInContainer("hadoop", "fs", "-mkdir", "/user/hadoop");
+    kerberosHiveContainer.executeInContainer("hadoop", "fs", "-mkdir", "-p", "/user/hadoop");
 
     Catalog catalog =
         gravitinoMetalake.createCatalog(

diff --git a/...est/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java b/...est/java/org/apache/gravitino/catalog/hive/integration/test/HiveUserAuthenticationIT.java
@@ -137,23 +137,20 @@ public static void stop() {
 
   private static void prepareKerberosConfig() throws Exception {
     // Keytab of the Gravitino SDK client
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        "/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
 
     // Keytab of the Gravitino server
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        "/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
 
     // Keytab of Gravitino server to connector to Hive
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        "/etc/admin.keytab", TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB);
 
     String tmpKrb5Path = TMP_DIR + "/krb5.conf_tmp";
     String krb5Path = TMP_DIR + "/krb5.conf";
-    kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
+    kerberosHiveContainer.copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
 
     // Modify the krb5.conf and change the kdc and admin_server to the container IP
     String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress();
@@ -235,6 +232,9 @@ public void testUserAuthentication() {
     Assertions.assertTrue(
         exceptionMessage.contains("Permission denied: user=gravitino_client, access=WRITE"));
 
+    // make sure the directory exists
+    kerberosHiveContainer.executeInContainer(
+        "hadoop", "fs", "-mkdir", "-p", "/user/hive/warehouse");
     // Now try to give the user the permission to create schema again
     kerberosHiveContainer.executeInContainer(
         "hadoop", "fs", "-chmod", "-R", "777", "/user/hive/warehouse");

diff --git a/...he/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java b/...he/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergKerberosHiveIT.java
@@ -150,23 +150,20 @@ public static void stop() {
 
   private static void prepareKerberosConfig() throws Exception {
     // Keytab of the Gravitino SDK client
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        "/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
 
     // Keytab of the Gravitino server
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        "/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
 
     // Keytab of Gravitino server to connector to Hive
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        "/etc/admin.keytab", TMP_DIR + HIVE_METASTORE_CLIENT_KEYTAB);
 
     String tmpKrb5Path = TMP_DIR + "/krb5.conf_tmp";
     String krb5Path = TMP_DIR + "/krb5.conf";
-    kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
+    kerberosHiveContainer.copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
 
     // Modify the krb5.conf and change the kdc and admin_server to the container IP
     String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress();
@@ -258,7 +255,7 @@ void testIcebergWithKerberosAndUserImpersonation() throws IOException {
 
     // Now try to permit the user to create the schema again
     kerberosHiveContainer.executeInContainer(
-        "hadoop", "fs", "-mkdir", "/user/hive/warehouse-catalog-iceberg");
+        "hadoop", "fs", "-mkdir", "-p", "/user/hive/warehouse-catalog-iceberg");
     kerberosHiveContainer.executeInContainer(
         "hadoop", "fs", "-chmod", "-R", "777", "/user/hive/warehouse-catalog-iceberg");
     Assertions.assertDoesNotThrow(

diff --git a/...org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java b/...org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/TestMultipleJDBCLoad.java
@@ -32,6 +32,7 @@
 import org.apache.gravitino.NameIdentifier;
 import org.apache.gravitino.client.GravitinoMetalake;
 import org.apache.gravitino.iceberg.common.IcebergConfig;
+import org.apache.gravitino.integration.test.container.ContainerSuite;
 import org.apache.gravitino.integration.test.container.MySQLContainer;
 import org.apache.gravitino.integration.test.container.PostgreSQLContainer;
 import org.apache.gravitino.integration.test.util.AbstractIT;
@@ -56,10 +57,10 @@ public class TestMultipleJDBCLoad extends AbstractIT {
 
   @BeforeAll
   public static void startup() throws IOException {
-    containerSuite.startMySQLContainer(TEST_DB_NAME);
-    mySQLContainer = containerSuite.getMySQLContainer();
-    containerSuite.startPostgreSQLContainer(TEST_DB_NAME);
-    postgreSQLContainer = containerSuite.getPostgreSQLContainer();
+    ContainerSuite.getInstance().startMySQLContainer(TEST_DB_NAME);
+    mySQLContainer = ContainerSuite.getInstance().getMySQLContainer();
+    ContainerSuite.getInstance().startPostgreSQLContainer(TEST_DB_NAME);
+    postgreSQLContainer = ContainerSuite.getInstance().getPostgreSQLContainer();
   }
 
   @Test

diff --git a/...ravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java b/...ravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonKerberosFilesystemIT.java
@@ -143,23 +143,19 @@ public static void stop() {
 
   private static void prepareKerberosConfig() throws Exception {
     // Keytab of the Gravitino SDK client
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer("/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        "/gravitino_client.keytab", TMP_DIR + GRAVITINO_CLIENT_KEYTAB);
 
     // Keytab of the Gravitino server
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer("/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer(
+        "/gravitino_server.keytab", TMP_DIR + GRAVITINO_SERVER_KEYTAB);
 
     // Keytab of Gravitino server to connector to Hive
-    kerberosHiveContainer
-        .getContainer()
-        .copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HDFS_CLIENT_KEYTAB);
+    kerberosHiveContainer.copyFileFromContainer("/etc/admin.keytab", TMP_DIR + HDFS_CLIENT_KEYTAB);
 
     String tmpKrb5Path = TMP_DIR + "/krb5.conf_tmp";
     String krb5Path = TMP_DIR + "/krb5.conf";
-    kerberosHiveContainer.getContainer().copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
+    kerberosHiveContainer.copyFileFromContainer("/etc/krb5.conf", tmpKrb5Path);
 
     // Modify the krb5.conf and change the kdc and admin_server to the container IP
     String ip = containerSuite.getKerberosHiveContainer().getContainerIpAddress();
@@ -243,7 +239,7 @@ void testPaimonWithKerberos() {
 
     // Now try to permit the user to create the schema again
     kerberosHiveContainer.executeInContainer(
-        "hadoop", "fs", "-mkdir", "/user/hive/paimon_catalog_warehouse");
+        "hadoop", "fs", "-mkdir", "-p", "/user/hive/paimon_catalog_warehouse");
     kerberosHiveContainer.executeInContainer(
         "hadoop", "fs", "-chmod", "-R", "777", "/user/hive/paimon_catalog_warehouse");
     Assertions.assertDoesNotThrow(

diff --git a/clients/client-python/apache_gravitino.egg-info/PKG-INFO b/clients/client-python/apache_gravitino.egg-info/PKG-INFO
@@ -0,0 +1,48 @@
+Metadata-Version: 2.1
+Name: apache-gravitino
+Version: 0.7.0.dev0
+Summary: Python lib/client for Apache Gravitino
+Home-page: https://github.com/apache/gravitino
+Author: Apache Software Foundation
+Author-email: dev@gravitino.apache.org
+Maintainer: Apache Gravitino Community
+Maintainer-email: dev@gravitino.apache.org
+License: Apache-2.0
+Project-URL: Homepage, https://gravitino.apache.org/
+Project-URL: Source Code, https://github.com/apache/gravitino
+Project-URL: Documentation, https://gravitino.apache.org/docs/overview
+Project-URL: Bug Tracker, https://github.com/apache/gravitino/issues
+Project-URL: Slack Chat, https://the-asf.slack.com/archives/C078RESTT19
+Keywords: Data,AI,metadata,catalog
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: requests==2.32.2
+Requires-Dist: dataclasses-json==0.6.6
+Requires-Dist: readerwriterlock==1.0.9
+Requires-Dist: fsspec==2024.3.1
+Requires-Dist: pyarrow==15.0.2
+Requires-Dist: cachetools==5.3.3
+Provides-Extra: dev
+Requires-Dist: requests==2.32.2; extra == "dev"
+Requires-Dist: dataclasses-json==0.6.6; extra == "dev"
+Requires-Dist: pylint==3.2.2; extra == "dev"
+Requires-Dist: black==24.4.2; extra == "dev"
+Requires-Dist: twine==5.1.1; extra == "dev"
+Requires-Dist: coverage==7.5.1; extra == "dev"
+Requires-Dist: pandas==2.0.3; extra == "dev"
+Requires-Dist: pyarrow==15.0.2; extra == "dev"
+Requires-Dist: llama-index==0.10.40; extra == "dev"
+Requires-Dist: tenacity==8.3.0; extra == "dev"
+Requires-Dist: cachetools==5.3.3; extra == "dev"
+Requires-Dist: readerwriterlock==1.0.9; extra == "dev"
+Requires-Dist: docker==7.1.0; extra == "dev"
+Requires-Dist: pyjwt[crypto]==2.8.0; extra == "dev"
+
+Apache Gravitino Python client