feat: MDT Test framework without writing data files

LICENSE

@@ -358,13 +358,3 @@ Copyright (c) 2005, European Commission project OneLab under contract 034819 (ht
 
   Home page: https://github.com/streamsets/datacollector-oss
   License: http://www.apache.org/licenses/LICENSE-2.0
-
- -------------------------------------------------------------------------------
-
- This product includes code from Apache Avro
-
- * org.apache.hudi.avro.JsonEncoder adapted from org.apache.avro.io.JsonEncoder
-
- Copyright: 2010-2019 The Apache Software Foundation
- Home page: https://avro.apache.org
- License: http://www.apache.org/licenses/LICENSE-2.0

azure-pipelines-20230430.yml

@@ -104,8 +104,8 @@ variables:
   PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn'
   MVN_OPTS_INSTALL: '-T 3 -Phudi-platform-service -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5'
   MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)'
-  JAVA_MVN_TEST_FILTER: '-DwildcardSuites=skipScalaTests -DfailIfNoTests=false'
-  SCALA_MVN_TEST_FILTER: '-Dtest=skipJavaTests -DfailIfNoTests=false'
+  JAVA_MVN_TEST_FILTER: '-DwildcardSuites=skipScalaTests -Dsurefire.failIfNoSpecifiedTests=false'
+  SCALA_MVN_TEST_FILTER: '-Dtest=skipJavaTests -Dsurefire.failIfNoSpecifiedTests=false'
   JOB3456_MODULES: ${{ join(',',parameters.job3456UTModules) }}
   JAVA_FUNCTIONAL_PACKAGE_TEST_FILTER: '**/org/apache/hudi/functional/**/*'
   MVN_ARG_FUNCTIONAL_PACKAGE_TEST: "-Dtest=\"$(JAVA_FUNCTIONAL_PACKAGE_TEST_FILTER)\""
@@ -126,7 +126,7 @@ stages:
     jobs:
       - job: UT_FT_1
         displayName: UT hudi-hadoop-common & UT FT client/spark-client
-        timeoutInMinutes: '90'
+        timeoutInMinutes: '120'
         steps:
           - task: Maven@4
             displayName: maven install
@@ -180,7 +180,7 @@ stages:
             displayName: Top 100 long-running testcases
       - job: UT_FT_2
         displayName: FTA hudi-spark
-        timeoutInMinutes: '90'
+        timeoutInMinutes: '120'
         steps:
           - task: Maven@4
             displayName: maven install
@@ -250,7 +250,7 @@ stages:
             displayName: Top 100 long-running testcases
       - job: UT_FT_4
         displayName: UT spark-datasource Java Test 2
-        timeoutInMinutes: '90'
+        timeoutInMinutes: '120'
         steps:
           - task: Maven@4
             displayName: maven install
@@ -285,7 +285,7 @@ stages:
             displayName: Top 100 long-running testcases
       - job: UT_FT_5
         displayName: UT spark-datasource DML
-        timeoutInMinutes: '90'
+        timeoutInMinutes: '120'
         steps:
           - task: Maven@4
             displayName: maven install
@@ -320,7 +320,7 @@ stages:
             displayName: Top 100 long-running testcases
       - job: UT_FT_6
         displayName: UT spark-datasource DDL & Others
-        timeoutInMinutes: '90'
+        timeoutInMinutes: '120'
         steps:
           - task: Maven@4
             displayName: maven install
@@ -355,7 +355,7 @@ stages:
             displayName: Top 100 long-running testcases
       - job: UT_FT_7
         displayName: UT Hudi Streamer & FT utilities
-        timeoutInMinutes: '90'
+        timeoutInMinutes: '120'
         steps:
           - task: Docker@2
             displayName: "login to docker hub"
@@ -378,10 +378,11 @@ stages:
               command: 'run'
               arguments: >
                 -v $(Build.SourcesDirectory):/hudi
+                -v /var/run/docker.sock:/var/run/docker.sock
                 -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
                 /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source -pl hudi-utilities -am
-                && mvn test  $(MVN_OPTS_TEST) -Punit-tests $(JACOCO_AGENT_DESTFILE1_ARG) -Dtest="TestHoodie*" -DfailIfNoTests=false -DargLine="-Xmx4g" -pl hudi-utilities
-                && mvn test  $(MVN_OPTS_TEST) -Pfunctional-tests $(JACOCO_AGENT_DESTFILE2_ARG) -DfailIfNoTests=false -DargLine="-Xmx4g" -pl hudi-utilities"
+                && mvn test  $(MVN_OPTS_TEST) -Punit-tests $(JACOCO_AGENT_DESTFILE1_ARG) -Dtest="TestHoodie*" -Dsurefire.failIfNoSpecifiedTests=false -pl hudi-utilities
+                && mvn test  $(MVN_OPTS_TEST) -Pfunctional-tests $(JACOCO_AGENT_DESTFILE2_ARG) -Dsurefire.failIfNoSpecifiedTests=false -pl hudi-utilities"
           - task: PublishTestResults@2
             displayName: 'Publish Test Results'
             inputs:
@@ -404,7 +405,7 @@ stages:
             displayName: Top 100 long-running testcases
       - job: UT_FT_8
         displayName: UT FT Spark and SQL (additional)
-        timeoutInMinutes: '110'
+        timeoutInMinutes: '120'
         steps:
           - task: Maven@4
             displayName: maven install
@@ -457,7 +458,7 @@ stages:
             displayName: Top 100 long-running testcases
       - job: UT_FT_9
         displayName: FT spark 2
-        timeoutInMinutes: '90'
+        timeoutInMinutes: '120'
         steps:
           - task: Maven@4
             displayName: maven install
@@ -492,7 +493,7 @@ stages:
             displayName: Top 100 long-running testcases
       - job: UT_FT_10
         displayName: UT FT common & other modules
-        timeoutInMinutes: '90'
+        timeoutInMinutes: '120'
         steps:
           - task: Docker@2
             displayName: "login to docker hub"
@@ -515,11 +516,12 @@ stages:
               command: 'run'
               arguments: >
                 -v $(Build.SourcesDirectory):/hudi
+                -v /var/run/docker.sock:/var/run/docker.sock
                 -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
                 /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source
-                && mvn test  $(MVN_OPTS_TEST) -Punit-tests -DfailIfNoTests=false -DargLine="-Xmx4g" $(JACOCO_AGENT_DESTFILE1_ARG) -pl $(JOB10_UT_MODULES)
-                && mvn test  $(MVN_OPTS_TEST) -Punit-tests $(JACOCO_AGENT_DESTFILE2_ARG) -Dtest="!TestHoodie*" -DfailIfNoTests=false -DargLine="-Xmx4g" -pl hudi-utilities
-                && mvn test  $(MVN_OPTS_TEST) -Pfunctional-tests -DfailIfNoTests=false -DargLine="-Xmx4g" $(JACOCO_AGENT_DESTFILE3_ARG) -pl $(JOB10_FT_MODULES)"
+                && mvn test  $(MVN_OPTS_TEST) -Punit-tests -Dsurefire.failIfNoSpecifiedTests=false $(JACOCO_AGENT_DESTFILE1_ARG) -pl $(JOB10_UT_MODULES)
+                && mvn test  $(MVN_OPTS_TEST) -Punit-tests $(JACOCO_AGENT_DESTFILE2_ARG) -Dtest="!TestHoodie*" -Dsurefire.failIfNoSpecifiedTests=false -pl hudi-utilities
+                && mvn test  $(MVN_OPTS_TEST) -Pfunctional-tests -Dsurefire.failIfNoSpecifiedTests=false  $(JACOCO_AGENT_DESTFILE3_ARG) -pl $(JOB10_FT_MODULES)"
           - task: PublishTestResults@2
             displayName: 'Publish Test Results'
             inputs:

hudi-aws/pom.xml

@@ -247,13 +247,13 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-inline</artifactId>
+            <groupId>com.esotericsoftware</groupId>
+            <artifactId>kryo-shaded</artifactId>
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>com.esotericsoftware</groupId>
-            <artifactId>kryo-shaded</artifactId>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>localstack</artifactId>
             <scope>test</scope>
         </dependency>
     </dependencies>

hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java

@@ -46,8 +46,7 @@ public DynamoDBBasedLockProvider(final LockConfiguration lockConfiguration, fina
 
   @Override
   public String getDynamoDBPartitionKey(LockConfiguration lockConfiguration) {
-    DynamoDbBasedLockConfig config = new DynamoDbBasedLockConfig.Builder()
-        .fromProperties(lockConfiguration.getConfig()).build();
+    DynamoDbBasedLockConfig config = DynamoDbBasedLockConfig.from(lockConfiguration.getConfig());
     ValidationUtils.checkArgument(
         config.contains(DYNAMODB_LOCK_PARTITION_KEY),
         "Config key is not found: " + DYNAMODB_LOCK_PARTITION_KEY.key());

hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProviderBase.java

@@ -80,9 +80,7 @@ public abstract class DynamoDBBasedLockProviderBase implements LockProvider<Lock
   protected volatile LockItem lock;
 
   protected DynamoDBBasedLockProviderBase(final LockConfiguration lockConfiguration, final StorageConfiguration<?> conf, DynamoDbClient dynamoDB) {
-    this.dynamoDbBasedLockConfig = new DynamoDbBasedLockConfig.Builder()
-        .fromProperties(lockConfiguration.getConfig())
-        .build();
+    this.dynamoDbBasedLockConfig = DynamoDbBasedLockConfig.from(lockConfiguration.getConfig());
     this.tableName = dynamoDbBasedLockConfig.getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_TABLE_NAME);
     long leaseDuration = dynamoDbBasedLockConfig.getInt(DynamoDbBasedLockConfig.LOCK_ACQUIRE_WAIT_TIMEOUT_MS_PROP_KEY);
     dynamoDBPartitionKey = getDynamoDBPartitionKey(lockConfiguration);

hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java

@@ -28,6 +28,8 @@
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.common.util.ValidationUtils;
 
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.regions.RegionMetadata;
 import software.amazon.awssdk.services.dynamodb.model.BillingMode;
@@ -41,12 +43,9 @@
     description = "Configs that control DynamoDB based locking mechanisms required for concurrency control "
         + " between writers to a Hudi table. Concurrency between Hudi's own table services "
         + " are auto managed internally.")
+@NoArgsConstructor(access = AccessLevel.PACKAGE)
 public class DynamoDbBasedLockConfig extends HoodieConfig {
 
-  public static DynamoDbBasedLockConfig.Builder newBuilder() {
-    return new DynamoDbBasedLockConfig.Builder();
-  }
-
   // configs for DynamoDb based locks
   public static final String DYNAMODB_BASED_LOCK_PROPERTY_PREFIX = LockConfiguration.LOCK_PREFIX + "dynamodb.";
 
@@ -132,31 +131,12 @@ public static DynamoDbBasedLockConfig.Builder newBuilder() {
       .sinceVersion("0.10.0")
       .withDocumentation("Lock Acquire Wait Timeout in milliseconds");
 
-  /**
-   * Builder for {@link DynamoDbBasedLockConfig}.
-   */
-  public static class Builder {
-    private final DynamoDbBasedLockConfig lockConfig = new DynamoDbBasedLockConfig();
-
-    public DynamoDbBasedLockConfig build() {
-      lockConfig.setDefaults(DynamoDbBasedLockConfig.class.getName());
-      checkRequiredProps();
-      return lockConfig;
-    }
-
-    public Builder fromProperties(TypedProperties props) {
-      lockConfig.getProps().putAll(props);
-      return this;
-    }
-
-    private void checkRequiredProps() {
-      String errorMsg = "Config key is not found: ";
-      ValidationUtils.checkArgument(
-          lockConfig.contains(DYNAMODB_LOCK_TABLE_NAME.key()),
-          errorMsg + DYNAMODB_LOCK_TABLE_NAME.key());
-      ValidationUtils.checkArgument(
-          lockConfig.contains(DYNAMODB_LOCK_REGION.key()),
-          errorMsg + DYNAMODB_LOCK_REGION.key());
-    }
+  public static DynamoDbBasedLockConfig from(TypedProperties properties) {
+    DynamoDbBasedLockConfig config = new DynamoDbBasedLockConfig();
+    config.getProps().putAll(properties);
+    config.setDefaults(DynamoDbBasedLockConfig.class.getName());
+    ValidationUtils.checkArgument(config.contains(DYNAMODB_LOCK_TABLE_NAME.key()), "Config key is not found: " + DYNAMODB_LOCK_TABLE_NAME.key());
+    ValidationUtils.checkArgument(config.contains(DYNAMODB_LOCK_REGION.key()), "Config key is not found: " + DYNAMODB_LOCK_REGION.key());
+    return config;
   }
-}
+}

hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java

@@ -26,7 +26,7 @@
 import org.apache.hudi.hadoop.fs.HadoopFSUtils;
 import org.apache.hudi.storage.HoodieStorage;
 import org.apache.hudi.storage.StorageConfiguration;
-import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -84,7 +84,9 @@ public static void initFS(boolean force) throws IOException {
     if (storage == null || force) {
       storage = (tableMetadata != null)
           ? tableMetadata.getStorage()
-          : new HoodieHadoopStorage(FileSystem.get(conf.unwrap()));
+          : HoodieStorageUtils.getStorage(
+              HadoopFSUtils.convertToStoragePath(FileSystem.get(conf.unwrap()).getWorkingDirectory()),
+              conf);
     }
   }
 

hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java

@@ -41,7 +41,7 @@
 import org.apache.hudi.hadoop.fs.HadoopFSUtils;
 import org.apache.hudi.keygen.SimpleKeyGenerator;
 import org.apache.hudi.storage.StorageConfiguration;
-import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
+import org.apache.hudi.storage.HoodieStorageUtils;
 import org.apache.hudi.testutils.Assertions;
 
 import org.apache.avro.generic.GenericRecord;
@@ -141,7 +141,10 @@ public void testAddPartitionMetaWithDryRun() throws IOException {
     assertTrue(ShellEvaluationResultUtil.isSuccess(result));
 
     // expected all 'No'.
-    String[][] rows = FSUtils.getAllPartitionFoldersThreeLevelsDown(new HoodieHadoopStorage(fs), tablePath)
+    String[][] rows = FSUtils.getAllPartitionFoldersThreeLevelsDown(
+        HoodieStorageUtils.getStorage(
+            HadoopFSUtils.convertToStoragePath(new Path(tablePath)),
+            HadoopFSUtils.getStorageConf(fs.getConf())), tablePath)
         .stream()
         .map(partition -> new String[] {partition, "No", "None"})
         .toArray(String[][]::new);
@@ -171,7 +174,10 @@ public void testAddPartitionMetaWithRealRun() throws IOException {
     Object result = shell.evaluate(() -> "repair addpartitionmeta --dryrun false");
     assertTrue(ShellEvaluationResultUtil.isSuccess(result));
 
-    List<String> paths = FSUtils.getAllPartitionFoldersThreeLevelsDown(new HoodieHadoopStorage(fs), tablePath);
+    List<String> paths = FSUtils.getAllPartitionFoldersThreeLevelsDown(
+        HoodieStorageUtils.getStorage(
+            HadoopFSUtils.convertToStoragePath(new Path(tablePath)),
+            HadoopFSUtils.getStorageConf(fs.getConf())), tablePath);
     // after dry run, the action will be 'Repaired'
     String[][] rows = paths.stream()
         .map(partition -> new String[] {partition, "No", "Repaired"})

hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java

@@ -18,7 +18,6 @@
 
 package org.apache.hudi.cli.commands;
 
-import org.apache.hudi.avro.HoodieAvroUtils;
 import org.apache.hudi.cli.HoodieCLI;
 import org.apache.hudi.cli.functional.CLIFunctionalTestHarness;
 import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator;
@@ -27,6 +26,8 @@
 import org.apache.hudi.common.fs.ConsistencyGuardConfig;
 import org.apache.hudi.common.model.HoodieCommitMetadata;
 import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.schema.HoodieSchema;
+import org.apache.hudi.common.schema.HoodieSchemaUtils;
 import org.apache.hudi.common.table.HoodieTableConfig;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.HoodieTableVersion;
@@ -36,7 +37,6 @@
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.storage.StoragePath;
 
-import org.apache.avro.Schema;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.junit.jupiter.api.BeforeEach;
@@ -259,18 +259,18 @@ public void testFetchTableSchema() throws Exception {
     assertTrue(ShellEvaluationResultUtil.isSuccess(result));
 
     String actualSchemaStr = result.toString().substring(result.toString().indexOf("{"));
-    Schema actualSchema = new Schema.Parser().parse(actualSchemaStr);
+    HoodieSchema actualSchema = HoodieSchema.parse(actualSchemaStr);
 
-    Schema expectedSchema = new Schema.Parser().parse(schemaStr);
-    expectedSchema = HoodieAvroUtils.addMetadataFields(expectedSchema);
+    HoodieSchema expectedSchema = HoodieSchema.parse(schemaStr);
+    expectedSchema = HoodieSchemaUtils.addMetadataFields(expectedSchema);
     assertEquals(actualSchema, expectedSchema);
 
     File file = File.createTempFile("temp", null);
     result = shell.evaluate(() -> "fetch table schema --outputFilePath " + file.getAbsolutePath());
     assertTrue(ShellEvaluationResultUtil.isSuccess(result));
 
     actualSchemaStr = getFileContent(file.getAbsolutePath());
-    actualSchema = new Schema.Parser().parse(actualSchemaStr);
+    actualSchema = HoodieSchema.parse(actualSchemaStr);
     assertEquals(actualSchema, expectedSchema);
   }
 

hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java

@@ -104,7 +104,7 @@ public void init() throws Exception {
         "", HoodieTableVersion.current().versionCode(),
         "org.apache.hudi.common.model.HoodieAvroPayload");
 
-    HoodieSparkWriteableTestTable cowTable = HoodieSparkWriteableTestTable.of(HoodieCLI.getTableMetaClient(), schema.toAvroSchema());
+    HoodieSparkWriteableTestTable cowTable = HoodieSparkWriteableTestTable.of(HoodieCLI.getTableMetaClient(), schema);
 
     cowTable.addCommit("20160401010101")
         .withInserts(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "1", hoodieRecords1)
@@ -127,7 +127,7 @@ public void init() throws Exception {
         morTablePath, "mor_table", HoodieTableType.MERGE_ON_READ.name(),
         "", HoodieTableVersion.current().versionCode(),
         "org.apache.hudi.common.model.HoodieAvroPayload");
-    HoodieSparkWriteableTestTable morTable = HoodieSparkWriteableTestTable.of(HoodieCLI.getTableMetaClient(), schema.toAvroSchema());
+    HoodieSparkWriteableTestTable morTable = HoodieSparkWriteableTestTable.of(HoodieCLI.getTableMetaClient(), schema);
 
     morTable.addDeltaCommit("20160401010101");
     morTable.withInserts(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "1", hoodieRecords1)
@@ -151,7 +151,7 @@ public void init() throws Exception {
         "", HoodieTableVersion.current().versionCode(),
         "org.apache.hudi.common.model.HoodieAvroPayload");
 
-    HoodieSparkWriteableTestTable cowNonPartitionedTable = HoodieSparkWriteableTestTable.of(HoodieCLI.getTableMetaClient(), schema.toAvroSchema());
+    HoodieSparkWriteableTestTable cowNonPartitionedTable = HoodieSparkWriteableTestTable.of(HoodieCLI.getTableMetaClient(), schema);
 
     cowNonPartitionedTable.addCommit("20160401010101")
         .withInserts(HoodieTestDataGenerator.NO_PARTITION_PATH, "1", hoodieRecords1)

hudi-client/hudi-client-common/pom.xml

@@ -187,11 +187,6 @@
         </exclusion>
       </exclusions>
     </dependency>
-    <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-inline</artifactId>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 
   <build>