graceful shutdown

Author: morningman

fe/fe-common/src/main/java/org/apache/doris/common/Config.java

@@ -3620,12 +3620,6 @@ public static int metaServiceRpcRetryTimes() {
     })
     public static int first_error_msg_max_length = 256;
 
-    @ConfField(mutable = true, masterOnly = false, description = {
-            "如果设置为 false，则 health 接口将返回 503，表示此 FE 不应再接收服务请求",
-            "If set to false, the health endpoint will return 503, "
-                    + "indicating that this FE should no longer receive service requests"})
-    public static boolean frontend_service_available = true;
-
     @ConfField
     public static String cloud_snapshot_handler_class = "org.apache.doris.cloud.snapshot.CloudSnapshotHandler";
     @ConfField

fe/fe-core/src/main/java/org/apache/doris/DorisFE.java

@@ -36,6 +36,8 @@
 import org.apache.doris.journal.bdbje.BDBTool;
 import org.apache.doris.journal.bdbje.BDBToolOptions;
 import org.apache.doris.persist.meta.MetaReader;
+import org.apache.doris.qe.Coordinator;
+import org.apache.doris.qe.QeProcessorImpl;
 import org.apache.doris.qe.QeService;
 import org.apache.doris.qe.SimpleScheduler;
 import org.apache.doris.service.ExecuteEnv;
@@ -63,6 +65,7 @@
 import java.nio.channels.OverlappingFileLockException;
 import java.nio.file.StandardOpenOption;
 import java.time.LocalDate;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -76,15 +79,14 @@ public class DorisFE {
     public static final String DORIS_HOME_DIR = System.getenv("DORIS_HOME");
     public static final String PID_DIR = System.getenv("PID_DIR");
 
-
     private static String LOCK_FILE_PATH;
 
     private static final String LOCK_FILE_NAME = "process.lock";
     private static FileChannel processLockFileChannel;
     private static FileLock processFileLock;
 
     // set to true when all servers are ready.
-    private static AtomicBoolean serverReady = new AtomicBoolean(false);
+    private final static AtomicBoolean serverReady = new AtomicBoolean(false);
 
     public static void main(String[] args) {
         // Every doris version should have a final meta version, it should not change
@@ -148,7 +150,14 @@ public static void start(String dorisHomeDir, String pidDir, String[] args, Star
             }
 
             Log4jConfig.initLogging(dorisHomeDir + "/conf/");
-            Runtime.getRuntime().addShutdownHook(new Thread(LogManager::shutdown));
+
+            // Add shutdown hook for graceful exit
+            Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+                LOG.info("Received shutdown signal, starting graceful shutdown...");
+                serverReady.set(false);
+                gracefulShutdown();
+                LogManager.shutdown();
+            }));
 
             // set dns cache ttl
             java.security.Security.setProperty("networkaddress.cache.ttl", "60");
@@ -237,9 +246,10 @@ public static void start(String dorisHomeDir, String pidDir, String[] args, Star
             startMonitor();
 
             serverReady.set(true);
-            while (true) {
+            while (serverReady.get()) {
                 Thread.sleep(2000);
             }
+            LOG.info("Doris FE main loop exited, shutting down gracefully...");
         } catch (Throwable e) {
             // Some exception may thrown before LOG is inited.
             // So need to print to stdout
@@ -589,4 +599,21 @@ public static class StartupOptions {
     public static boolean isServerReady() {
         return serverReady.get();
     }
+
+    private static void gracefulShutdown() {
+        // wait for all queries to finish
+        try {
+            long now = System.currentTimeMillis();
+            List<Coordinator> allCoordinators = QeProcessorImpl.INSTANCE.getAllCoordinators();
+            while (!allCoordinators.isEmpty() && System.currentTimeMillis() - now < 300 * 1000L) {
+                Thread.sleep(1000);
+                allCoordinators = QeProcessorImpl.INSTANCE.getAllCoordinators();
+                LOG.info("waiting {} queries to finish before shutdown", allCoordinators.size());
+            }
+        } catch (Throwable t) {
+            LOG.error("", t);
+        }
+
+        LOG.info("graceful shutdown finished");
+    }
 }

fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/HealthAction.java

@@ -44,10 +44,6 @@ public Object execute(HttpServletRequest request, HttpServletResponse response)
             return ResponseEntityBuilder.serviceUnavailable("Server is not ready");
         }
 
-        if (!Config.frontend_service_available) {
-            return ResponseEntityBuilder.serviceUnavailable("Server is not available");
-        }
-
         Map<String, Object> result = new HashMap<>();
         result.put("total_backend_num", Env.getCurrentSystemInfo().getAllBackendIds(false).size());
         result.put("online_backend_num", Env.getCurrentSystemInfo().getAllBackendIds(true).size());