Merge remote-tracking branch 'upstream/main' into ISSUE-2238

apache · Mar 29, 2024 · 999fa25 · 999fa25
2 parents 7ca2389 + 0d36d2f
commit 999fa25
Show file tree

Hide file tree

Showing 46 changed files with 3,521 additions and 1,722 deletions.
diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
@@ -9,8 +9,8 @@ on:
     branches: [ "main", "branch-*" ]
 
 concurrency:
-  group: ${{ github.worklfow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_requests' }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   changes:

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -9,8 +9,8 @@ on:
     branches: [ "main", "branch-*" ]
 
 concurrency:
-  group: ${{ github.worklfow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_requests' }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:

diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml
@@ -9,8 +9,8 @@ on:
     branches: [ "main", "branch-*" ]
 
 concurrency:
-  group: ${{ github.worklfow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_requests' }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   changes:

diff --git a/...hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTablePropertiesMetadata.java b/...hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTablePropertiesMetadata.java
@@ -57,11 +57,11 @@ public class HiveTablePropertiesMetadata extends BasePropertiesMetadata {
   @VisibleForTesting
   public static final String ORC_SERDE_CLASS = "org.apache.hadoop.hive.ql.io.orc.OrcSerde";
 
-  private static final String PARQUET_INPUT_FORMAT_CLASS =
+  public static final String PARQUET_INPUT_FORMAT_CLASS =
       "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat";
-  private static final String PARQUET_OUTPUT_FORMAT_CLASS =
+  public static final String PARQUET_OUTPUT_FORMAT_CLASS =
       "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat";
-  private static final String PARQUET_SERDE_CLASS =
+  public static final String PARQUET_SERDE_CLASS =
       "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe";
   private static final String COLUMNAR_SERDE_CLASS =
       "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe";
@@ -89,7 +89,10 @@ public enum TableType {
     VIRTUAL_INDEX,
   }
 
-  enum StorageFormat {
+  // In embedded test mode, HiveTablePropertiesMetadata will be loaded by spark connector which has
+  // different classloaders with Hive catalog. If StorageFormat is package scope, it couldn't
+  // be accessed by Hive catalog related classes in same package, so making it public.
+  public enum StorageFormat {
     SEQUENCEFILE(
         SEQUENCEFILE_INPUT_FORMAT_CLASS, SEQUENCEFILE_OUTPUT_FORMAT_CLASS, LAZY_SIMPLE_SERDE_CLASS),
     TEXTFILE(TEXT_INPUT_FORMAT_CLASS, IGNORE_KEY_OUTPUT_FORMAT_CLASS, LAZY_SIMPLE_SERDE_CLASS),

diff --git a/...mmon/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTablePropertiesMetadata.java b/...mmon/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcTablePropertiesMetadata.java
@@ -6,6 +6,7 @@
 
 import com.datastrato.gravitino.StringIdentifier;
 import com.datastrato.gravitino.connector.BasePropertiesMetadata;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Maps;
 import java.util.HashMap;
 import java.util.Map;
@@ -14,13 +15,15 @@ public abstract class JdbcTablePropertiesMetadata extends BasePropertiesMetadata
 
   public static final String COMMENT_KEY = "comment";
 
-  protected Map<String, String> transformToJdbcProperties(Map<String, String> properties) {
+  @VisibleForTesting
+  public Map<String, String> transformToJdbcProperties(Map<String, String> properties) {
     HashMap<String, String> resultProperties = Maps.newHashMap(properties);
     resultProperties.remove(StringIdentifier.ID_KEY);
     return resultProperties;
   }
 
-  protected Map<String, String> convertFromJdbcProperties(Map<String, String> properties) {
+  @VisibleForTesting
+  public Map<String, String> convertFromJdbcProperties(Map<String, String> properties) {
     return properties;
   }
 }
diff --git a/...ql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlTablePropertiesMetadata.java b/...ql/src/main/java/com/datastrato/gravitino/catalog/mysql/MysqlTablePropertiesMetadata.java
@@ -105,7 +105,7 @@ public Map<String, String> transformToJdbcProperties(Map<String, String> propert
   }
 
   @Override
-  protected Map<String, String> convertFromJdbcProperties(Map<String, String> properties) {
+  public Map<String, String> convertFromJdbcProperties(Map<String, String> properties) {
     BidiMap<String, String> mysqlConfigToGravitino = GRAVITINO_CONFIG_TO_MYSQL.inverseBidiMap();
     return Collections.unmodifiableMap(
         new HashMap<String, String>() {

diff --git a/catalogs/catalog-messaging-kafka/build.gradle.kts b/catalogs/catalog-messaging-kafka/build.gradle.kts
@@ -16,10 +16,14 @@ dependencies {
   implementation(project(":common"))
 
   implementation(libs.guava)
+  implementation(libs.kafka.clients)
+  implementation(libs.slf4j.api)
 
+  testImplementation(libs.commons.io)
+  testImplementation(libs.curator.test)
   testImplementation(libs.junit.jupiter.api)
+  testImplementation(libs.kafka)
   testImplementation(libs.mockito.core)
-  testImplementation(libs.commons.io)
 
   testRuntimeOnly(libs.junit.jupiter.engine)
 }