From 90efd06e92c2e65aee2961af7c3b88e23a492f88 Mon Sep 17 00:00:00 2001 From: fanng Date: Thu, 29 Aug 2024 20:37:13 +0800 Subject: [PATCH 1/6] fix comment --- .../catalog-lakehouse-iceberg/build.gradle.kts | 1 - .../apache/gravitino}/OverwriteDefaultConfig.java | 2 +- .../apache/gravitino/config/ConfigConstants.java | 5 +++++ iceberg/iceberg-common/build.gradle.kts | 2 +- .../gravitino/iceberg/common/IcebergConfig.java | 14 ++++++++------ .../iceberg/common/TestIcebergConfig.java | 5 ++--- .../gravitino/server/web/JettyServerConfig.java | 7 +++---- 7 files changed, 20 insertions(+), 16 deletions(-) rename {server-common/src/main/java/org/apache/gravitino/server/web => core/src/main/java/org/apache/gravitino}/OverwriteDefaultConfig.java (96%) diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts index bab879f64d9..1361dcaeae2 100644 --- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts +++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts @@ -36,7 +36,6 @@ dependencies { implementation(project(":common")) implementation(project(":core")) implementation(project(":iceberg:iceberg-common")) - implementation(project(":server-common")) implementation(libs.bundles.iceberg) implementation(libs.bundles.jersey) implementation(libs.bundles.jetty) diff --git a/server-common/src/main/java/org/apache/gravitino/server/web/OverwriteDefaultConfig.java b/core/src/main/java/org/apache/gravitino/OverwriteDefaultConfig.java similarity index 96% rename from server-common/src/main/java/org/apache/gravitino/server/web/OverwriteDefaultConfig.java rename to core/src/main/java/org/apache/gravitino/OverwriteDefaultConfig.java index 98c797c1efa..f742d4bdb8e 100644 --- a/server-common/src/main/java/org/apache/gravitino/server/web/OverwriteDefaultConfig.java +++ b/core/src/main/java/org/apache/gravitino/OverwriteDefaultConfig.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.gravitino.server.web; +package org.apache.gravitino; import java.util.Map; diff --git a/core/src/main/java/org/apache/gravitino/config/ConfigConstants.java b/core/src/main/java/org/apache/gravitino/config/ConfigConstants.java index 5317b19a4dd..b5760c171a8 100644 --- a/core/src/main/java/org/apache/gravitino/config/ConfigConstants.java +++ b/core/src/main/java/org/apache/gravitino/config/ConfigConstants.java @@ -24,6 +24,11 @@ public final class ConfigConstants { private ConfigConstants() {} + /** HTTP Server port, reused by Gravitino server and Iceberg REST server */ + public static final String WEBSERVER_HTTP_PORT = "httpPort"; + /** HTTPS Server port, reused by Gravitino server and Iceberg REST server */ + public static final String WEBSERVER_HTTPS_PORT = "httpsPort"; + /** The value of messages used to indicate that the configuration is not set. */ public static final String NOT_BLANK_ERROR_MSG = "The value can't be blank"; diff --git a/iceberg/iceberg-common/build.gradle.kts b/iceberg/iceberg-common/build.gradle.kts index d0964e2dacb..b42885df2c2 100644 --- a/iceberg/iceberg-common/build.gradle.kts +++ b/iceberg/iceberg-common/build.gradle.kts @@ -28,7 +28,6 @@ dependencies { implementation(project(":catalogs:catalog-common")) implementation(project(":core")) implementation(project(":common")) - implementation(project(":server-common")) implementation(libs.bundles.iceberg) implementation(libs.bundles.log4j) implementation(libs.caffeine) @@ -83,6 +82,7 @@ dependencies { annotationProcessor(libs.lombok) compileOnly(libs.lombok) + testImplementation(project(":server-common")) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.sqlite.jdbc) diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java index ed4306b8c84..97f91d210ec 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java @@ -24,13 +24,12 @@ import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Config; +import org.apache.gravitino.OverwriteDefaultConfig; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergPropertiesUtils; import org.apache.gravitino.config.ConfigBuilder; import org.apache.gravitino.config.ConfigConstants; import org.apache.gravitino.config.ConfigEntry; -import org.apache.gravitino.server.web.JettyServerConfig; -import org.apache.gravitino.server.web.OverwriteDefaultConfig; import org.apache.gravitino.storage.OSSProperties; import org.apache.gravitino.storage.S3Properties; @@ -38,6 +37,9 @@ public class IcebergConfig extends Config implements OverwriteDefaultConfig { public static final String ICEBERG_CONFIG_PREFIX = "gravitino.iceberg-rest."; + public static final int DEFAULT_ICEBERG_REST_SERVICE_HTTP_PORT = 9001; + public static final int DEFAULT_ICEBERG_REST_SERVICE_HTTPS_PORT = 9433; + public static final ConfigEntry CATALOG_BACKEND = new ConfigBuilder(IcebergConstants.CATALOG_BACKEND) .doc("Catalog backend of Gravitino Iceberg catalog") @@ -248,9 +250,9 @@ public Map getIcebergCatalogProperties() { @Override public Map getOverwriteDefaultConfig() { return ImmutableMap.of( - JettyServerConfig.WEBSERVER_HTTP_PORT.getKey(), - String.valueOf(JettyServerConfig.DEFAULT_ICEBERG_REST_SERVICE_HTTP_PORT), - JettyServerConfig.WEBSERVER_HTTPS_PORT.getKey(), - String.valueOf(JettyServerConfig.DEFAULT_ICEBERG_REST_SERVICE_HTTPS_PORT)); + ConfigConstants.WEBSERVER_HTTP_PORT, + String.valueOf(DEFAULT_ICEBERG_REST_SERVICE_HTTP_PORT), + ConfigConstants.WEBSERVER_HTTPS_PORT, + String.valueOf(DEFAULT_ICEBERG_REST_SERVICE_HTTPS_PORT)); } } diff --git a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/TestIcebergConfig.java b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/TestIcebergConfig.java index 0c0b06458a3..3294c72b26f 100644 --- a/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/TestIcebergConfig.java +++ b/iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/TestIcebergConfig.java @@ -47,10 +47,9 @@ public void testIcebergHttpPort() { IcebergConfig icebergConfig = new IcebergConfig(properties); JettyServerConfig jettyServerConfig = JettyServerConfig.fromConfig(icebergConfig); Assertions.assertEquals( - JettyServerConfig.DEFAULT_ICEBERG_REST_SERVICE_HTTP_PORT, jettyServerConfig.getHttpPort()); + IcebergConfig.DEFAULT_ICEBERG_REST_SERVICE_HTTP_PORT, jettyServerConfig.getHttpPort()); Assertions.assertEquals( - JettyServerConfig.DEFAULT_ICEBERG_REST_SERVICE_HTTPS_PORT, - jettyServerConfig.getHttpsPort()); + IcebergConfig.DEFAULT_ICEBERG_REST_SERVICE_HTTPS_PORT, jettyServerConfig.getHttpsPort()); properties = ImmutableMap.of( diff --git a/server-common/src/main/java/org/apache/gravitino/server/web/JettyServerConfig.java b/server-common/src/main/java/org/apache/gravitino/server/web/JettyServerConfig.java index ead9645e8db..ec77fa95488 100644 --- a/server-common/src/main/java/org/apache/gravitino/server/web/JettyServerConfig.java +++ b/server-common/src/main/java/org/apache/gravitino/server/web/JettyServerConfig.java @@ -31,6 +31,7 @@ import javax.net.ssl.SSLContext; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Config; +import org.apache.gravitino.OverwriteDefaultConfig; import org.apache.gravitino.config.ConfigBuilder; import org.apache.gravitino.config.ConfigConstants; import org.apache.gravitino.config.ConfigEntry; @@ -41,8 +42,6 @@ public final class JettyServerConfig { private static final Logger LOG = LoggerFactory.getLogger(JettyServerConfig.class); public static final String GRAVITINO_SERVER_CONFIG_PREFIX = "gravitino.server.webserver."; private static final String SPLITTER = ","; - public static final int DEFAULT_ICEBERG_REST_SERVICE_HTTP_PORT = 9001; - public static final int DEFAULT_ICEBERG_REST_SERVICE_HTTPS_PORT = 9433; public static final int DEFAULT_GRAVITINO_WEBSERVER_HTTP_PORT = 8090; public static final int DEFAULT_GRAVITINO_WEBSERVER_HTTPS_PORT = 8433; @@ -54,7 +53,7 @@ public final class JettyServerConfig { .createWithDefault("0.0.0.0"); public static final ConfigEntry WEBSERVER_HTTP_PORT = - new ConfigBuilder("httpPort") + new ConfigBuilder(ConfigConstants.WEBSERVER_HTTP_PORT) .doc("The http port number of the Jetty web server") .version(ConfigConstants.VERSION_0_1_0) .intConf() @@ -126,7 +125,7 @@ public final class JettyServerConfig { .createWithDefault(false); public static final ConfigEntry WEBSERVER_HTTPS_PORT = - new ConfigBuilder("httpsPort") + new ConfigBuilder(ConfigConstants.WEBSERVER_HTTPS_PORT) .doc("The https port number of the Jetty web server") .version(ConfigConstants.VERSION_0_3_0) .intConf() From f7c3dc790cabdf2aa11ec1448032d549bcd556d4 Mon Sep 17 00:00:00 2001 From: fanng Date: Fri, 6 Sep 2024 19:42:00 +0800 Subject: [PATCH 2/6] iceberg-common succ --- .../build.gradle.kts | 38 ++++++++++--------- .../iceberg/converter/TestBaseConvert.java | 22 ++++++----- gradle/libs.versions.toml | 1 + iceberg/iceberg-common/build.gradle.kts | 30 ++++++++++++--- iceberg/iceberg-rest-server/build.gradle.kts | 14 +++++-- ...estConfigBasedIcebergTableOpsProvider.java | 12 ++---- ...GravitinoBasedIcebergTableOpsProvider.java | 10 +---- 7 files changed, 74 insertions(+), 53 deletions(-) diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts index 1361dcaeae2..d52c8fbb84d 100644 --- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts +++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts @@ -31,21 +31,25 @@ val icebergVersion: String = libs.versions.iceberg.get() val scalaCollectionCompatVersion: String = libs.versions.scala.collection.compat.get() dependencies { - implementation(project(":api")) + implementation(project(":api")) { + exclude("*") + } implementation(project(":catalogs:catalog-common")) - implementation(project(":common")) - implementation(project(":core")) + implementation(project(":common")) { + exclude("*") + } + implementation(project(":core")) { + exclude("*") + } implementation(project(":iceberg:iceberg-common")) implementation(libs.bundles.iceberg) - implementation(libs.bundles.jersey) - implementation(libs.bundles.jetty) + implementation(libs.bundles.log4j) implementation(libs.cglib) implementation(libs.commons.collections4) implementation(libs.commons.io) implementation(libs.commons.lang3) implementation(libs.guava) - implementation(libs.sqlite.jdbc) annotationProcessor(libs.lombok) @@ -59,7 +63,9 @@ dependencies { testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion") testImplementation("org.apache.iceberg:iceberg-spark-runtime-${sparkMajorVersion}_$scalaVersion:$icebergVersion") - testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") + testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") { + exclude("org.apache.hive") + } testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") { exclude("org.apache.avro") exclude("org.apache.hadoop") @@ -68,22 +74,14 @@ dependencies { exclude("org.rocksdb") } - testImplementation(libs.hadoop2.common) { - exclude("com.github.spotbugs") - } - testImplementation(libs.jersey.test.framework.core) { - exclude(group = "org.junit.jupiter") - } - testImplementation(libs.jersey.test.framework.provider.jetty) { - exclude(group = "org.junit.jupiter") - } + testImplementation(libs.bundles.jersey) + testImplementation(libs.bundles.jetty) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.mockito.core) // For test TestMultipleJDBCLoad, it was depended on testcontainers.mysql and testcontainers.postgresql testImplementation(libs.mysql.driver) testImplementation(libs.postgresql.driver) - testImplementation(libs.slf4j.api) testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) @@ -104,7 +102,11 @@ tasks { val copyCatalogLibs by registering(Copy::class) { dependsOn("jar", "runtimeJars") - from("build/libs") + from("build/libs") { + exclude("guava-*.jar") + exclude("log4j-*.jar") + exclude("slf4j-*.jar") + } into("$rootDir/distribution/package/catalogs/lakehouse-iceberg/libs") } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java index 04f0aac8ce1..004e73aac12 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/converter/TestBaseConvert.java @@ -26,7 +26,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; -import org.apache.commons.lang.math.RandomUtils; +import java.util.Random; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergColumn; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.expressions.Expression; @@ -54,6 +54,8 @@ public class TestBaseConvert { protected static final Map GRAVITINO_TYPE = new HashMap<>(); protected static final Map ICEBERG_TYPE = new HashMap<>(); + private static Random random = new Random(System.currentTimeMillis()); + static { GRAVITINO_TYPE.put("BOOLEAN", org.apache.gravitino.rel.types.Types.BooleanType.get()); // Types not supported by iceberg @@ -113,8 +115,8 @@ protected static SortOrder[] createSortOrder(String... colNames) { results.add( SortOrders.of( field(colName), - RandomUtils.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING, - RandomUtils.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST)); + random.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING, + random.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST)); } return results.toArray(new SortOrder[0]); } @@ -122,15 +124,15 @@ protected static SortOrder[] createSortOrder(String... colNames) { protected static SortOrder createSortOrder(String name, String colName) { return SortOrders.of( FunctionExpression.of(name, field(colName)), - RandomUtils.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING, - RandomUtils.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST); + random.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING, + random.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST); } protected static SortOrder createSortOrder(String name, int width, String colName) { return SortOrders.of( FunctionExpression.of(name, Literals.integerLiteral(width), field(colName)), - RandomUtils.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING, - RandomUtils.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST); + random.nextBoolean() ? SortDirection.DESCENDING : SortDirection.ASCENDING, + random.nextBoolean() ? NullOrdering.NULLS_FIRST : NullOrdering.NULLS_LAST); } protected static Types.NestedField createNestedField( @@ -143,7 +145,7 @@ protected static Types.NestedField[] createNestedField(String... colNames) { for (int i = 0; i < colNames.length; i++) { results.add( Types.NestedField.of( - i + 1, RandomUtils.nextBoolean(), colNames[i], getRandomIcebergType(), TEST_COMMENT)); + i + 1, random.nextBoolean(), colNames[i], getRandomIcebergType(), TEST_COMMENT)); } return results.toArray(new Types.NestedField[0]); } @@ -211,7 +213,7 @@ protected static String getGravitinoSortOrderExpressionString(Expression sortOrd private static Type getRandomGravitinoType() { Collection values = GRAVITINO_TYPE.values(); return values.stream() - .skip(RandomUtils.nextInt(values.size())) + .skip(random.nextInt(values.size())) .findFirst() .orElseThrow(() -> new RuntimeException("No type found")); } @@ -219,7 +221,7 @@ private static Type getRandomGravitinoType() { private static org.apache.iceberg.types.Type getRandomIcebergType() { Collection values = ICEBERG_TYPE.values(); return values.stream() - .skip(RandomUtils.nextInt(values.size())) + .skip(random.nextInt(values.size())) .findFirst() .orElseThrow(() -> new RuntimeException("No type found")); } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4efb10eb220..af0b1becb5f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -138,6 +138,7 @@ hive2-common = { group = "org.apache.hive", name = "hive-common", version.ref = hive2-jdbc = { group = "org.apache.hive", name = "hive-jdbc", version.ref = "hive2"} hadoop2-auth = { group = "org.apache.hadoop", name = "hadoop-auth", version.ref = "hadoop2" } hadoop2-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref = "hadoop2" } +hadoop2-hdfs-client = { group = "org.apache.hadoop", name = "hadoop-hdfs-client", version.ref = "hadoop2" } hadoop2-common = { group = "org.apache.hadoop", name = "hadoop-common", version.ref = "hadoop2"} hadoop2-mapreduce-client-core = { group = "org.apache.hadoop", name = "hadoop-mapreduce-client-core", version.ref = "hadoop2"} hadoop3-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref = "hadoop3" } diff --git a/iceberg/iceberg-common/build.gradle.kts b/iceberg/iceberg-common/build.gradle.kts index b42885df2c2..85250ca1a3d 100644 --- a/iceberg/iceberg-common/build.gradle.kts +++ b/iceberg/iceberg-common/build.gradle.kts @@ -26,11 +26,17 @@ plugins { dependencies { implementation(project(":catalogs:catalog-common")) - implementation(project(":core")) - implementation(project(":common")) + implementation(project(":core")) { + exclude("*") + } + implementation(project(":common")) { + exclude("*") + } implementation(libs.bundles.iceberg) implementation(libs.bundles.log4j) + implementation(libs.bundles.kerby) implementation(libs.caffeine) + implementation(libs.cglib) implementation(libs.commons.lang3) implementation(libs.guava) implementation(libs.iceberg.aliyun) @@ -41,17 +47,22 @@ dependencies { exclude("com.github.spotbugs") exclude("com.sun.jersey") exclude("javax.servlet") + exclude("org.apache.curator") + exclude("org.apache.zookeeper") exclude("org.mortbay.jetty") } + // use hdfs-default.xml implementation(libs.hadoop2.hdfs) { + exclude("*") + } + implementation(libs.hadoop2.hdfs.client) { exclude("com.sun.jersey") exclude("javax.servlet") + exclude("org.fusesource.leveldbjni") exclude("org.mortbay.jetty") } implementation(libs.hadoop2.mapreduce.client.core) { - exclude("com.sun.jersey") - exclude("javax.servlet") - exclude("org.mortbay.jetty") + exclude("*") } implementation(libs.hive2.metastore) { exclude("co.cask.tephra") @@ -60,10 +71,17 @@ dependencies { exclude("com.sun.jersey") exclude("com.tdunning", "json") exclude("com.zaxxer", "HikariCP") + exclude("com.github.joshelser") + exclude("io.dropwizard.metrics") exclude("javax.servlet") exclude("javax.transaction", "transaction-api") + exclude("org.apache.ant") exclude("org.apache.avro", "avro") + exclude("org.apache.curator") + exclude("org.apache.derby") exclude("org.apache.hbase") + exclude("org.apache.hive", "hive-service-rpc") + exclude("org.apache.hadoop") exclude("org.apache.hadoop", "hadoop-yarn-api") exclude("org.apache.hadoop", "hadoop-yarn-server-applicationhistoryservice") exclude("org.apache.hadoop", "hadoop-yarn-server-common") @@ -71,7 +89,9 @@ dependencies { exclude("org.apache.hadoop", "hadoop-yarn-server-web-proxy") exclude("org.apache.logging.log4j") exclude("org.apache.parquet", "parquet-hadoop-bundle") + exclude("org.apache.orc") exclude("org.apache.zookeeper") + exclude("org.datanucleus") exclude("org.eclipse.jetty.aggregate", "jetty-all") exclude("org.eclipse.jetty.orbit", "javax.servlet") exclude("org.mortbay.jetty") diff --git a/iceberg/iceberg-rest-server/build.gradle.kts b/iceberg/iceberg-rest-server/build.gradle.kts index a90b85b96ff..594e6d04208 100644 --- a/iceberg/iceberg-rest-server/build.gradle.kts +++ b/iceberg/iceberg-rest-server/build.gradle.kts @@ -34,14 +34,22 @@ dependencies { implementation(project(":api")) implementation(project(":catalogs:catalog-common")) implementation(project(":clients:client-java")) - implementation(project(":core")) - implementation(project(":common")) + implementation(project(":core")) { + exclude("*") + } + implementation(project(":common")) { + exclude("*") + } implementation(project(":iceberg:iceberg-common")) - implementation(project(":server-common")) + implementation(project(":server-common")) { + exclude("*") + } implementation(libs.bundles.iceberg) implementation(libs.bundles.jetty) implementation(libs.bundles.jersey) implementation(libs.bundles.log4j) + implementation(libs.bundles.metrics) + implementation(libs.bundles.prometheus) implementation(libs.caffeine) implementation(libs.commons.lang3) implementation(libs.guava) diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestConfigBasedIcebergTableOpsProvider.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestConfigBasedIcebergTableOpsProvider.java index 70d9a834ffc..be97e03d045 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestConfigBasedIcebergTableOpsProvider.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestConfigBasedIcebergTableOpsProvider.java @@ -20,7 +20,6 @@ import com.google.common.collect.Maps; import java.util.Map; -import java.util.UUID; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.iceberg.common.ops.IcebergTableOps; @@ -33,9 +32,6 @@ import org.junit.jupiter.params.provider.ValueSource; public class TestConfigBasedIcebergTableOpsProvider { - private static final String STORE_PATH = - "/tmp/gravitino_test_iceberg_jdbc_backend_" + UUID.randomUUID().toString().replace("-", ""); - @Test public void testValidIcebergTableOps() { String hiveCatalogName = "hive_backend"; @@ -51,13 +47,11 @@ public void testValidIcebergTableOps() { // jdbc backend catalog config.put("catalog.jdbc_backend.catalog-backend-name", jdbcCatalogName); config.put("catalog.jdbc_backend.catalog-backend", "jdbc"); - config.put( - "catalog.jdbc_backend.uri", - String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", STORE_PATH)); + config.put("catalog.jdbc_backend.uri", "jdbc:sqlite::memory:"); config.put("catalog.jdbc_backend.warehouse", "/tmp/usr/jdbc/warehouse"); config.put("catalog.jdbc_backend.jdbc.password", "gravitino"); config.put("catalog.jdbc_backend.jdbc.user", "gravitino"); - config.put("catalog.jdbc_backend.jdbc-driver", "org.h2.Driver"); + config.put("catalog.jdbc_backend.jdbc-driver", "org.sqlite.JDBC"); config.put("catalog.jdbc_backend.jdbc-initialize", "true"); // default catalog config.put("catalog-backend-name", defaultCatalogName); @@ -87,7 +81,7 @@ public void testValidIcebergTableOps() { Assertions.assertEquals("jdbc", jdbcIcebergConfig.get(IcebergConfig.CATALOG_BACKEND)); Assertions.assertEquals( "/tmp/usr/jdbc/warehouse", jdbcIcebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE)); - Assertions.assertEquals("org.h2.Driver", jdbcIcebergConfig.get(IcebergConfig.JDBC_DRIVER)); + Assertions.assertEquals("org.sqlite.JDBC", jdbcIcebergConfig.get(IcebergConfig.JDBC_DRIVER)); Assertions.assertEquals(true, jdbcIcebergConfig.get(IcebergConfig.JDBC_INIT_TABLES)); Assertions.assertEquals( diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestGravitinoBasedIcebergTableOpsProvider.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestGravitinoBasedIcebergTableOpsProvider.java index b95a430f409..c26ceeb2f31 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestGravitinoBasedIcebergTableOpsProvider.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestGravitinoBasedIcebergTableOpsProvider.java @@ -19,7 +19,6 @@ package org.apache.gravitino.iceberg.provider; import java.util.HashMap; -import java.util.UUID; import org.apache.gravitino.Catalog; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.client.GravitinoAdminClient; @@ -32,9 +31,6 @@ import org.mockito.Mockito; public class TestGravitinoBasedIcebergTableOpsProvider { - private static final String STORE_PATH = - "/tmp/gravitino_test_iceberg_jdbc_backend_" + UUID.randomUUID().toString().replace("-", ""); - @Test public void testValidIcebergTableOps() { String hiveCatalogName = "hive_backend"; @@ -65,13 +61,11 @@ public void testValidIcebergTableOps() { new HashMap() { { put(IcebergConstants.CATALOG_BACKEND, "jdbc"); - put( - IcebergConstants.URI, - String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MYSQL", STORE_PATH)); + put(IcebergConstants.URI, "jdbc:sqlite::memory:"); put(IcebergConstants.WAREHOUSE, "/tmp/user/hive/warehouse-jdbc"); put(IcebergConstants.GRAVITINO_JDBC_USER, "gravitino"); put(IcebergConstants.GRAVITINO_JDBC_PASSWORD, "gravitino"); - put(IcebergConstants.GRAVITINO_JDBC_DRIVER, "org.h2.Driver"); + put(IcebergConstants.GRAVITINO_JDBC_DRIVER, "org.sqlite.JDBC"); put(IcebergConstants.ICEBERG_JDBC_INITIALIZE, "true"); put(IcebergConstants.CATALOG_BACKEND_NAME, jdbcCatalogName); } From 433e07fa9386373ea899d6bb2f6bd895fb847890 Mon Sep 17 00:00:00 2001 From: fanng Date: Tue, 10 Sep 2024 08:21:35 +0800 Subject: [PATCH 3/6] remove jline --- iceberg/iceberg-common/build.gradle.kts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/iceberg/iceberg-common/build.gradle.kts b/iceberg/iceberg-common/build.gradle.kts index 85250ca1a3d..23b3d30db28 100644 --- a/iceberg/iceberg-common/build.gradle.kts +++ b/iceberg/iceberg-common/build.gradle.kts @@ -34,7 +34,9 @@ dependencies { } implementation(libs.bundles.iceberg) implementation(libs.bundles.log4j) - implementation(libs.bundles.kerby) + implementation(libs.bundles.kerby) { + exclude("org.jline") + } implementation(libs.caffeine) implementation(libs.cglib) implementation(libs.commons.lang3) @@ -75,6 +77,7 @@ dependencies { exclude("io.dropwizard.metrics") exclude("javax.servlet") exclude("javax.transaction", "transaction-api") + exclude("jline") exclude("org.apache.ant") exclude("org.apache.avro", "avro") exclude("org.apache.curator") From 3ee3f241474bf33e32ab3cc2cbf66c1b94aca217 Mon Sep 17 00:00:00 2001 From: fanng Date: Wed, 11 Sep 2024 19:23:33 +0800 Subject: [PATCH 4/6] remove jersey jetty --- catalogs/catalog-lakehouse-iceberg/build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts index d52c8fbb84d..68f1edcabdf 100644 --- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts +++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts @@ -74,8 +74,8 @@ dependencies { exclude("org.rocksdb") } - testImplementation(libs.bundles.jersey) - testImplementation(libs.bundles.jetty) + // testImplementation(libs.bundles.jersey) + // testImplementation(libs.bundles.jetty) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.mockito.core) From 4058510aa90af06f3e1ef41b9e8e30b5e318b9a0 Mon Sep 17 00:00:00 2001 From: fanng Date: Wed, 11 Sep 2024 22:08:46 +0800 Subject: [PATCH 5/6] remove jetty jersey --- catalogs/catalog-lakehouse-iceberg/build.gradle.kts | 2 -- 1 file changed, 2 deletions(-) diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts index 68f1edcabdf..25f750a6be9 100644 --- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts +++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts @@ -74,8 +74,6 @@ dependencies { exclude("org.rocksdb") } - // testImplementation(libs.bundles.jersey) - // testImplementation(libs.bundles.jetty) testImplementation(libs.junit.jupiter.api) testImplementation(libs.junit.jupiter.params) testImplementation(libs.mockito.core) From 8259c85993cd825c9fb7088e2b8549cdce80ed4f Mon Sep 17 00:00:00 2001 From: fanng Date: Fri, 13 Sep 2024 09:05:58 +0800 Subject: [PATCH 6/6] update licence.bin --- LICENSE.bin | 1 + 1 file changed, 1 insertion(+) diff --git a/LICENSE.bin b/LICENSE.bin index 14d44b7d31e..46b097b53f9 100644 --- a/LICENSE.bin +++ b/LICENSE.bin @@ -286,6 +286,7 @@ Apache Hadoop Auth Apache Hadoop Client Aggregator Apache Hadoop Common + Apache Hadoop HDFS Apache Hadoop HDFS Client Apache Hadoop MapReduce Common Apache Hadoop MapReduce Core