[AMORO-1951] Support parallelized planning in one optimizer group (ap…

…ache#2282) * [AMORO-1951] Support parallelized planning in one optimizer group * [AMORO-1951] add unit test for OptimizingQueue and DefaultOptimizingService * [AMORO-1951] optimize default parameters * fix bugs * fix warnings and spotless issues * merge from apache#2290 * add apache license and fix spotless * fix config error * Update ams/server/src/main/java/com/netease/arctic/server/DefaultOptimizingService.java Co-authored-by: ZhouJinsong <zhoujinsong0505@163.com> * add annotations * fix compile errors * fix import problem * remove isDebugEnabled() * spotless apply * Update ArcticManagementConf.java * fix reboot bug and supply document content * use MoreObjects.toStringHelper for OptimizerThread.java * Merged from [AMORO-2376] Print right log info after calculating and sorting tables * fix import problem * remove unused codes * spotless * remove incorrect comments * add max-planning-parallelism to config --------- Co-authored-by: majin1102 <majin1102@163.com> Co-authored-by: ZhouJinsong <zhoujinsong0505@163.com>
ShawHee · Dec 29, 2023 · 90d845e · 90d845e
1 parent a149dc0
commit 90d845e
Show file tree

Hide file tree

Showing 21 changed files with 1,362 additions and 991 deletions.
diff --git a/ams/api/src/main/java/com/netease/arctic/ams/api/resource/ResourceManager.java b/ams/api/src/main/java/com/netease/arctic/ams/api/resource/ResourceManager.java
@@ -23,4 +23,7 @@ public interface ResourceManager {
   List<Resource> listResourcesByGroup(String groupName);
 
   Resource getResource(String resourceId);
+
+  /** close this resource manager */
+  void dispose();
 }
diff --git a/ams/dist/src/main/arctic-bin/conf/config.yaml b/ams/dist/src/main/arctic-bin/conf/config.yaml
@@ -32,6 +32,8 @@ ams:
   optimizer:
     heart-beat-timeout: 60000 # 1min
     task-ack-timeout: 30000 # 30s
+    polling-timeout: 3000 # 3s
+    max-planning-parallelism: 1 # default 1
 
   blocker:
     timeout: 60000 # 1min

diff --git a/ams/server/src/main/java/com/netease/arctic/server/ArcticManagementConf.java b/ams/server/src/main/java/com/netease/arctic/server/ArcticManagementConf.java
@@ -254,6 +254,18 @@ public class ArcticManagementConf {
           .defaultValue(30000L)
           .withDescription("Timeout duration for task acknowledgment.");
 
+  public static final ConfigOption<Integer> OPTIMIZER_MAX_PLANNING_PARALLELISM =
+      ConfigOptions.key("optimizer.max-planning-parallelism")
+          .intType()
+          .defaultValue(1)
+          .withDescription("Max planning parallelism in one optimizer group.");
+
+  public static final ConfigOption<Long> OPTIMIZER_POLLING_TIMEOUT =
+      ConfigOptions.key("optimizer.polling-timeout")
+          .longType()
+          .defaultValue(3000L)
+          .withDescription("Optimizer polling task timeout.");
+
   /** config key prefix of terminal */
   public static final String TERMINAL_PREFIX = "terminal.";
 
@@ -317,12 +329,6 @@ public class ArcticManagementConf {
   public static final String CONTAINER_IMPL = "container-impl";
   public static final String CONTAINER_PROPERTIES = "properties";
 
-  // optimizer config
-  public static final String OPTIMIZER_GROUP_LIST = "optimizer_groups";
-  public static final String OPTIMIZER_GROUP_NAME = "name";
-  public static final String OPTIMIZER_GROUP_CONTAINER = "container";
-  public static final String OPTIMIZER_GROUP_PROPERTIES = "properties";
-
   public static final String DB_TYPE_DERBY = "derby";
   public static final String DB_TYPE_MYSQL = "mysql";
   public static final String DB_TYPE_POSTGRES = "postgres";

diff --git a/ams/server/src/main/java/com/netease/arctic/server/ArcticServiceConstants.java b/ams/server/src/main/java/com/netease/arctic/server/ArcticServiceConstants.java
@@ -2,8 +2,6 @@
 
 public class ArcticServiceConstants {
 
-  public static final long OPTIMIZER_CHECK_INTERVAL = 10 * 1000;
-
   public static final long INVALID_TIME = 0;
 
   public static final long QUOTA_LOOK_BACK_TIME = 60 * 60 * 1000;