Delete buildV9Directly in Kafka and Kinesis Indexing Service (apache#…

…11351) * delete_buildV9Directly_in_kafka_and_kinesis_indexing_service * delete * delete them from server * delete buildV9Directly from hadoop indexing * bug fixed Co-authored-by: yuanyi <yuanyi@freewheel.tv>
jihoonson · Jul 11, 2021 · 02a4e7a · 02a4e7a
1 parent 580e7d0
commit 02a4e7a
Show file tree

Hide file tree

Showing 39 changed files with 1 addition and 127 deletions.
diff --git a/docs/ingestion/native-batch.md b/docs/ingestion/native-batch.md
@@ -605,8 +605,7 @@ An example of the result is
         "logParseExceptions": false,
         "maxParseExceptions": 2147483647,
         "maxSavedParseExceptions": 0,
-        "forceGuaranteedRollup": false,
-        "buildV9Directly": true
+        "forceGuaranteedRollup": false
       }
     }
   },

diff --git a/.../main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java b/.../main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java
@@ -192,7 +192,6 @@ public HadoopIndexTask createTask(Interval interval, String version, List<DataSe
         tuningConfig.isCombineText(),
         tuningConfig.getUseCombiner(),
         tuningConfig.getMaxRowsInMemory(),
-        tuningConfig.getBuildV9Directly(),
         tuningConfig.getNumBackgroundPersistThreads(),
         tuningConfig.isForceExtendableShardSpecs(),
         true,

diff --git a/...ing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTuningConfig.java b/...ing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTuningConfig.java
@@ -45,8 +45,6 @@ public KafkaIndexTaskTuningConfig(
       @JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists,
       @JsonProperty("indexSpec") @Nullable IndexSpec indexSpec,
       @JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists,
-      // This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12.
-      @JsonProperty("buildV9Directly") @Nullable Boolean buildV9Directly,
       @Deprecated @JsonProperty("reportParseExceptions") @Nullable Boolean reportParseExceptions,
       @JsonProperty("handoffConditionTimeout") @Nullable Long handoffConditionTimeout,
       @JsonProperty("resetOffsetAutomatically") @Nullable Boolean resetOffsetAutomatically,
@@ -69,7 +67,6 @@ public KafkaIndexTaskTuningConfig(
         maxPendingPersists,
         indexSpec,
         indexSpecForIntermediatePersists,
-        true,
         reportParseExceptions,
         handoffConditionTimeout,
         resetOffsetAutomatically,
@@ -97,7 +94,6 @@ public KafkaIndexTaskTuningConfig withBasePersistDirectory(File dir)
         getMaxPendingPersists(),
         getIndexSpec(),
         getIndexSpecForIntermediatePersists(),
-        true,
         isReportParseExceptions(),
         getHandoffConditionTimeout(),
         isResetOffsetAutomatically(),

diff --git a/...src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTuningConfig.java b/...src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTuningConfig.java
@@ -68,7 +68,6 @@ public static KafkaSupervisorTuningConfig defaultConfig()
         null,
         null,
         null,
-        null,
         null
     );
   }
@@ -85,8 +84,6 @@ public KafkaSupervisorTuningConfig(
       @JsonProperty("maxPendingPersists") Integer maxPendingPersists,
       @JsonProperty("indexSpec") IndexSpec indexSpec,
       @JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists,
-      // This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12.
-      @JsonProperty("buildV9Directly") Boolean buildV9Directly,
       @JsonProperty("reportParseExceptions") Boolean reportParseExceptions,
       @JsonProperty("handoffConditionTimeout") Long handoffConditionTimeout,
       @JsonProperty("resetOffsetAutomatically") Boolean resetOffsetAutomatically,
@@ -115,7 +112,6 @@ public KafkaSupervisorTuningConfig(
         maxPendingPersists,
         indexSpec,
         indexSpecForIntermediatePersists,
-        true,
         reportParseExceptions,
         handoffConditionTimeout,
         resetOffsetAutomatically,
@@ -237,7 +233,6 @@ public KafkaIndexTaskTuningConfig convertToTaskTuningConfig()
         getMaxPendingPersists(),
         getIndexSpec(),
         getIndexSpecForIntermediatePersists(),
-        true,
         isReportParseExceptions(),
         getHandoffConditionTimeout(),
         isResetOffsetAutomatically(),

diff --git a/...ka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java b/...ka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java
@@ -2685,7 +2685,6 @@ private KafkaIndexTask createTask(
         null,
         null,
         null,
-        true,
         reportParseExceptions,
         handoffConditionTimeout,
         resetOffsetAutomatically,

diff --git a/...service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTuningConfigTest.java b/...service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTuningConfigTest.java
@@ -132,7 +132,6 @@ public void testConvert()
         new IndexSpec(),
         new IndexSpec(),
         true,
-        true,
         5L,
         null,
         null,
@@ -178,7 +177,6 @@ public void testSerdeWithModifiedTuningConfigAddedField() throws IOException
         new IndexSpec(),
         new IndexSpec(),
         true,
-        true,
         5L,
         null,
         null,
@@ -202,7 +200,6 @@ public void testSerdeWithModifiedTuningConfigAddedField() throws IOException
     Assert.assertEquals(base.getBasePersistDirectory(), deserialized.getBasePersistDirectory());
     Assert.assertEquals(base.getMaxPendingPersists(), deserialized.getMaxPendingPersists());
     Assert.assertEquals(base.getIndexSpec(), deserialized.getIndexSpec());
-    Assert.assertEquals(base.getBuildV9Directly(), deserialized.getBuildV9Directly());
     Assert.assertEquals(base.isReportParseExceptions(), deserialized.isReportParseExceptions());
     Assert.assertEquals(base.getHandoffConditionTimeout(), deserialized.getHandoffConditionTimeout());
     Assert.assertEquals(base.isResetOffsetAutomatically(), deserialized.isResetOffsetAutomatically());
@@ -229,7 +226,6 @@ public void testSerdeWithModifiedTuningConfigRemovedField() throws IOException
         new IndexSpec(),
         new IndexSpec(),
         true,
-        true,
         5L,
         null,
         null,
@@ -253,7 +249,6 @@ public void testSerdeWithModifiedTuningConfigRemovedField() throws IOException
     Assert.assertEquals(base.getBasePersistDirectory(), deserialized.getBasePersistDirectory());
     Assert.assertEquals(base.getMaxPendingPersists(), deserialized.getMaxPendingPersists());
     Assert.assertEquals(base.getIndexSpec(), deserialized.getIndexSpec());
-    Assert.assertEquals(base.getBuildV9Directly(), deserialized.getBuildV9Directly());
     Assert.assertEquals(base.isReportParseExceptions(), deserialized.isReportParseExceptions());
     Assert.assertEquals(base.getHandoffConditionTimeout(), deserialized.getHandoffConditionTimeout());
     Assert.assertEquals(base.isResetOffsetAutomatically(), deserialized.isResetOffsetAutomatically());

diff --git a/...service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/...service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java
@@ -311,7 +311,6 @@ public KafkaIndexTaskClient build(
             null,
             null,
             null,
-            true,
             false,
             null,
             false,
@@ -468,7 +467,6 @@ public void testCreateBaseTaskContexts() throws JsonProcessingException
             null,
             null,
             null,
-            null,
             null
         ),
         null
@@ -3280,7 +3278,6 @@ public void testIsTaskCurrent()
             null,
             null,
             null,
-            true,
             false,
             null,
             false,
@@ -3321,7 +3318,6 @@ public void testIsTaskCurrent()
         null,
         null,
         null,
-        true,
         false,
         null,
         null,
@@ -3619,7 +3615,6 @@ public KafkaIndexTaskClient build(
         null,
         null,
         null,
-        true,
         false,
         null,
         resetOffsetAutomatically,
@@ -3732,7 +3727,6 @@ public KafkaIndexTaskClient build(
         null,
         null,
         null,
-        true,
         false,
         null,
         false,

diff --git a/...est/java/org/apache/druid/indexing/kafka/test/TestModifiedKafkaIndexTaskTuningConfig.java b/...est/java/org/apache/druid/indexing/kafka/test/TestModifiedKafkaIndexTaskTuningConfig.java
@@ -49,8 +49,6 @@ public TestModifiedKafkaIndexTaskTuningConfig(
       @JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists,
       @JsonProperty("indexSpec") @Nullable IndexSpec indexSpec,
       @JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists,
-      // This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12.
-      @JsonProperty("buildV9Directly") @Nullable Boolean buildV9Directly,
       @Deprecated @JsonProperty("reportParseExceptions") @Nullable Boolean reportParseExceptions,
       @JsonProperty("handoffConditionTimeout") @Nullable Long handoffConditionTimeout,
       @JsonProperty("resetOffsetAutomatically") @Nullable Boolean resetOffsetAutomatically,
@@ -74,7 +72,6 @@ public TestModifiedKafkaIndexTaskTuningConfig(
         maxPendingPersists,
         indexSpec,
         indexSpecForIntermediatePersists,
-        true,
         reportParseExceptions,
         handoffConditionTimeout,
         resetOffsetAutomatically,

diff --git a/...service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTuningConfig.java b/...service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTuningConfig.java
@@ -62,7 +62,6 @@ public KinesisIndexTaskTuningConfig(
       @JsonProperty("maxPendingPersists") Integer maxPendingPersists,
       @JsonProperty("indexSpec") IndexSpec indexSpec,
       @JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists,
-      @JsonProperty("buildV9Directly") Boolean buildV9Directly,
       @JsonProperty("reportParseExceptions") Boolean reportParseExceptions,
       @JsonProperty("handoffConditionTimeout") Long handoffConditionTimeout,
       @JsonProperty("resetOffsetAutomatically") Boolean resetOffsetAutomatically,
@@ -92,7 +91,6 @@ public KinesisIndexTaskTuningConfig(
         maxPendingPersists,
         indexSpec,
         indexSpecForIntermediatePersists,
-        true,
         reportParseExceptions,
         handoffConditionTimeout,
         resetOffsetAutomatically,
@@ -170,7 +168,6 @@ public KinesisIndexTaskTuningConfig withBasePersistDirectory(File dir)
         getMaxPendingPersists(),
         getIndexSpec(),
         getIndexSpecForIntermediatePersists(),
-        true,
         isReportParseExceptions(),
         getHandoffConditionTimeout(),
         isResetOffsetAutomatically(),

diff --git a/...main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTuningConfig.java b/...main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTuningConfig.java
@@ -77,7 +77,6 @@ public static KinesisSupervisorTuningConfig defaultConfig()
         null,
         null,
         null,
-        null,
         null
     );
   }
@@ -94,7 +93,6 @@ public KinesisSupervisorTuningConfig(
       @JsonProperty("maxPendingPersists") Integer maxPendingPersists,
       @JsonProperty("indexSpec") IndexSpec indexSpec,
       @JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists,
-      @JsonProperty("buildV9Directly") Boolean buildV9Directly,
       @JsonProperty("reportParseExceptions") Boolean reportParseExceptions,
       @JsonProperty("handoffConditionTimeout") Long handoffConditionTimeout,
       @JsonProperty("resetOffsetAutomatically") Boolean resetOffsetAutomatically,
@@ -131,7 +129,6 @@ public KinesisSupervisorTuningConfig(
         maxPendingPersists,
         indexSpec,
         indexSpecForIntermediatePersists,
-        buildV9Directly,
         reportParseExceptions,
         handoffConditionTimeout,
         resetOffsetAutomatically,
@@ -267,7 +264,6 @@ public KinesisIndexTaskTuningConfig convertToTaskTuningConfig()
         getMaxPendingPersists(),
         getIndexSpec(),
         getIndexSpecForIntermediatePersists(),
-        true,
         isReportParseExceptions(),
         getHandoffConditionTimeout(),
         isResetOffsetAutomatically(),

diff --git a/...ng-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskSerdeTest.java b/...ng-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskSerdeTest.java
@@ -73,7 +73,6 @@ public class KinesisIndexTaskSerdeTest
       null,
       null,
       null,
-      null,
       null
   );
   private static final KinesisIndexTaskIOConfig IO_CONFIG = new KinesisIndexTaskIOConfig(

diff --git a/...ndexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java b/...ndexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java
@@ -2766,7 +2766,6 @@ private KinesisIndexTask createTask(
         null,
         null,
         null,
-        true,
         reportParseExceptions,
         handoffConditionTimeout,
         resetOffsetAutomatically,

diff --git a/...ice/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTuningConfigTest.java b/...ice/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTuningConfigTest.java
@@ -74,7 +74,6 @@ public void testSerdeWithDefaults() throws Exception
     Assert.assertEquals(new Period("PT10M"), config.getIntermediatePersistPeriod());
     Assert.assertEquals(0, config.getMaxPendingPersists());
     Assert.assertEquals(new IndexSpec(), config.getIndexSpec());
-    Assert.assertTrue(config.getBuildV9Directly());
     Assert.assertFalse(config.isReportParseExceptions());
     Assert.assertEquals(0, config.getHandoffConditionTimeout());
     Assert.assertEquals(10000, config.getRecordBufferSize());
@@ -96,7 +95,6 @@ public void testSerdeWithNonDefaults() throws Exception
                      + "  \"maxRowsPerSegment\": 100,\n"
                      + "  \"intermediatePersistPeriod\": \"PT1H\",\n"
                      + "  \"maxPendingPersists\": 100,\n"
-                     + "  \"buildV9Directly\": true,\n"
                      + "  \"reportParseExceptions\": true,\n"
                      + "  \"handoffConditionTimeout\": 100,\n"
                      + "  \"recordBufferSize\": 1000,\n"
@@ -125,7 +123,6 @@ public void testSerdeWithNonDefaults() throws Exception
     Assert.assertEquals(100, config.getMaxRowsPerSegment().intValue());
     Assert.assertEquals(new Period("PT1H"), config.getIntermediatePersistPeriod());
     Assert.assertEquals(100, config.getMaxPendingPersists());
-    Assert.assertTrue(config.getBuildV9Directly());
     Assert.assertTrue(config.isReportParseExceptions());
     Assert.assertEquals(100, config.getHandoffConditionTimeout());
     Assert.assertEquals(1000, config.getRecordBufferSize());
@@ -153,7 +150,6 @@ public void testSerdeWithModifiedTuningConfigAddedField() throws IOException
         new IndexSpec(),
         new IndexSpec(),
         true,
-        true,
         5L,
         true,
         false,
@@ -184,7 +180,6 @@ public void testSerdeWithModifiedTuningConfigAddedField() throws IOException
     Assert.assertEquals(base.getBasePersistDirectory(), deserialized.getBasePersistDirectory());
     Assert.assertEquals(base.getMaxPendingPersists(), deserialized.getMaxPendingPersists());
     Assert.assertEquals(base.getIndexSpec(), deserialized.getIndexSpec());
-    Assert.assertEquals(base.getBuildV9Directly(), deserialized.getBuildV9Directly());
     Assert.assertEquals(base.isReportParseExceptions(), deserialized.isReportParseExceptions());
     Assert.assertEquals(base.getHandoffConditionTimeout(), deserialized.getHandoffConditionTimeout());
     Assert.assertEquals(base.isResetOffsetAutomatically(), deserialized.isResetOffsetAutomatically());
@@ -215,7 +210,6 @@ public void testSerdeWithModifiedTuningConfigRemovedField() throws IOException
         new IndexSpec(),
         new IndexSpec(),
         true,
-        true,
         5L,
         true,
         false,
@@ -245,7 +239,6 @@ public void testSerdeWithModifiedTuningConfigRemovedField() throws IOException
     Assert.assertEquals(base.getBasePersistDirectory(), deserialized.getBasePersistDirectory());
     Assert.assertEquals(base.getMaxPendingPersists(), deserialized.getMaxPendingPersists());
     Assert.assertEquals(base.getIndexSpec(), deserialized.getIndexSpec());
-    Assert.assertEquals(base.getBuildV9Directly(), deserialized.getBuildV9Directly());
     Assert.assertEquals(base.isReportParseExceptions(), deserialized.isReportParseExceptions());
     Assert.assertEquals(base.getHandoffConditionTimeout(), deserialized.getHandoffConditionTimeout());
     Assert.assertEquals(base.isResetOffsetAutomatically(), deserialized.isResetOffsetAutomatically());
@@ -270,7 +263,6 @@ public void testResetOffsetAndSkipSequenceNotBothTrue() throws Exception
                      + "  \"maxRowsPerSegment\": 100,\n"
                      + "  \"intermediatePersistPeriod\": \"PT1H\",\n"
                      + "  \"maxPendingPersists\": 100,\n"
-                     + "  \"buildV9Directly\": true,\n"
                      + "  \"reportParseExceptions\": true,\n"
                      + "  \"handoffConditionTimeout\": 100,\n"
                      + "  \"recordBufferSize\": 1000,\n"
@@ -305,7 +297,6 @@ public void testConvert()
         new IndexSpec(),
         new IndexSpec(),
         true,
-        true,
         5L,
         true,
         false,
@@ -339,7 +330,6 @@ public void testConvert()
     Assert.assertEquals(new File("/tmp/xxx"), copy.getBasePersistDirectory());
     Assert.assertEquals(4, copy.getMaxPendingPersists());
     Assert.assertEquals(new IndexSpec(), copy.getIndexSpec());
-    Assert.assertTrue(copy.getBuildV9Directly());
     Assert.assertTrue(copy.isReportParseExceptions());
     Assert.assertEquals(5L, copy.getHandoffConditionTimeout());
     Assert.assertEquals(1000, copy.getRecordBufferSize());

diff --git a/...ice/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java b/...ice/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java
@@ -181,7 +181,6 @@ public void setupTest()
         null,
         null,
         null,
-        true,
         false,
         null,
         null,
@@ -3772,7 +3771,6 @@ public void testIsTaskCurrent()
         null,
         null,
         null,
-        true,
         false,
         null,
         null,
@@ -4827,7 +4825,6 @@ public KinesisIndexTaskClient build(
         null,
         null,
         null,
-        true,
         false,
         null,
         resetOffsetAutomatically,

diff --git a/.../java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTuningConfigTest.java b/.../java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTuningConfigTest.java
@@ -65,7 +65,6 @@ public void testSerdeWithDefaults() throws Exception
     Assert.assertEquals(new Period("PT10M"), config.getIntermediatePersistPeriod());
     Assert.assertEquals(0, config.getMaxPendingPersists());
     Assert.assertEquals(new IndexSpec(), config.getIndexSpec());
-    Assert.assertEquals(true, config.getBuildV9Directly());
     Assert.assertEquals(false, config.isReportParseExceptions());
     Assert.assertEquals(0, config.getHandoffConditionTimeout());
     Assert.assertNull(config.getWorkerThreads());
@@ -86,7 +85,6 @@ public void testSerdeWithNonDefaults() throws Exception
                      + "  \"maxRowsPerSegment\": 100,\n"
                      + "  \"intermediatePersistPeriod\": \"PT1H\",\n"
                      + "  \"maxPendingPersists\": 100,\n"
-                     + "  \"buildV9Directly\": false,\n"
                      + "  \"reportParseExceptions\": true,\n"
                      + "  \"handoffConditionTimeout\": 100,\n"
                      + "  \"workerThreads\": 12,\n"
@@ -114,7 +112,6 @@ public void testSerdeWithNonDefaults() throws Exception
     Assert.assertEquals(100, config.getMaxRowsPerSegment().intValue());
     Assert.assertEquals(new Period("PT1H"), config.getIntermediatePersistPeriod());
     Assert.assertEquals(100, config.getMaxPendingPersists());
-    Assert.assertEquals(true, config.getBuildV9Directly());
     Assert.assertEquals(true, config.isReportParseExceptions());
     Assert.assertEquals(100, config.getHandoffConditionTimeout());
     Assert.assertEquals(12, (int) config.getWorkerThreads());
-Original file line number
+Diff line change
@@ Expand Up / @@ -2685,7 +2685,6 @@ private KafkaIndexTask createTask( @@
             null,
             null,
             null,
-            true,
             reportParseExceptions,
             handoffConditionTimeout,
             resetOffsetAutomatically,
@@ Expand Down @@
-Original file line number
+Diff line change
@@ Expand Up / @@ -311,7 +311,6 @@ public KafkaIndexTaskClient build( @@
                 null,
                 null,
                 null,
-                true,
                 false,
                 null,
                 false,
@@ Expand Down Expand Up @@
                 null,
                 null,
                 null,
-                null,
                 null
             ),
             null
@@ Expand Down Expand Up / @@ -3280,7 +3278,6 @@ public void testIsTaskCurrent() @@
                 null,
                 null,
                 null,
-                true,
                 false,
                 null,
                 false,
@@ Expand Down Expand Up / @@ -3321,7 +3318,6 @@ public void testIsTaskCurrent() @@
             null,
             null,
             null,
-            true,
             false,
             null,
             null,
@@ Expand Down Expand Up / @@ -3619,7 +3615,6 @@ public KafkaIndexTaskClient build( @@
             null,
             null,
             null,
-            true,
             false,
             null,
             resetOffsetAutomatically,
@@ Expand Down Expand Up / @@ -3732,7 +3727,6 @@ public KafkaIndexTaskClient build( @@
             null,
             null,
             null,
-            true,
             false,
             null,
             false,
@@ Expand Down @@