Track and prioritize routing according to application restart generations (#2564)

Author: CamJN

CHANGELOG

@@ -1,5 +1,6 @@
 Release 6.0.24 (Not yet released)
 -------------
+ * [Enterprise] Smarter rolling restarts for better performance and reliability. We changed the way we route requests. Instead of picking the least-busy process, we now first prioritize new processes first. During a rolling restart, this new behavior leads to more efficient utilization of application caches, faster validation of new rollouts, and faster recovery from problematic deployments. Closes GH-2551.
  * Fix a regression from 6.0.10 where running `passenger-config system-properties` would throw an error. Closes GH-2565.
  * [Enterprise] Fix a memory corruption-related crash that could occur during rolling restarting.
  * [Ubuntu] Add packages for Ubuntu 24.10 "oracular".

src/agent/Core/ApplicationPool/Group.h

@@ -92,8 +92,18 @@ class Group: public boost::enable_shared_from_this<Group> {
 	};
 
 	struct RouteResult {
-		Process *process;
-		bool finished;
+		/** The Process to route the request to, or nullptr if no process can be routed to. */
+		Process * const process;
+		/**
+		 * If `process` is nullptr, then `finished` indicates whether another `Group::route()`
+		 * call on a different request *could* succeed, meaning that the caller should continue
+		 * calling `Group::route()` if there are more queued requests that need to be processed.
+		 *
+		 * Usually `finished` is false because all processes are totally busy. But in some cases,
+		 * for example when using sticky sessions, it could be true because other requests can
+		 * potentially be routed to other processes.
+		 */
+		const bool finished;
 
 		RouteResult(Process *p, bool _finished = false)
 			: process(p),
@@ -223,9 +233,9 @@ class Group: public boost::enable_shared_from_this<Group> {
 	/****** Process list management ******/
 
 	Process *findProcessWithStickySessionId(unsigned int id) const;
-	Process *findProcessWithStickySessionIdOrLowestBusyness(unsigned int id) const;
-	Process *findProcessWithLowestBusyness(const ProcessList &processes) const;
-	Process *findEnabledProcessWithLowestBusyness() const;
+	Process *findBestProcessPreferringStickySessionId(unsigned int id) const;
+	Process *findBestProcess(const ProcessList &processes) const;
+	Process *findBestEnabledProcess() const;
 
 	void addProcessToList(const ProcessPtr &process, ProcessList &destination);
 	void removeProcessFromList(const ProcessPtr &process, ProcessList &source);

src/agent/Core/ApplicationPool/Group/InternalUtils.cpp

@@ -184,7 +184,7 @@ Group::createNullProcessObject() {
 	LockGuard l(context->memoryManagementSyncher);
 	Process *process = context->processObjectPool.malloc();
 	Guard guard(context, process);
-	process = new (process) Process(&info, args);
+	process = new (process) Process(&info, info.group->restartsInitiated, args);
 	process->shutdownNotRequired();
 	guard.clear();
 	return ProcessPtr(process, false);
@@ -221,7 +221,7 @@ Group::createProcessObject(const SpawningKit::Spawner &spawner,
 	LockGuard l(context->memoryManagementSyncher);
 	Process *process = context->processObjectPool.malloc();
 	Guard guard(context, process);
-	process = new (process) Process(&info, spawnResult, args);
+	process = new (process) Process(&info, info.group->restartsInitiated, spawnResult, args);
 	guard.clear();
 	return ProcessPtr(process, false);
 }

src/agent/Core/ApplicationPool/Group/ProcessListManagement.cpp

@@ -63,72 +63,75 @@ Group::findProcessWithStickySessionId(unsigned int id) const {
 	return NULL;
 }
 
+/**
+ * Return the process with the given sticky session ID if it exists.
+ * If not, then find the "best" enabled process to route a request to,
+ * according to the same criteria documented for findBestProcess().
+ *
+ * - If the process with the given sticky session ID exists, then always
+ *   returns that process. Meaning that this process could be `!canBeRoutedTo()`.
+ * - If there is no process that can be routed to, then returns nullptr.
+ */
 Process *
-Group::findProcessWithStickySessionIdOrLowestBusyness(unsigned int id) const {
-	int leastBusyProcessIndex = -1;
-	int lowestBusyness = 0;
-	unsigned int i, size = enabledProcessBusynessLevels.size();
-	const int *enabledProcessBusynessLevels = &this->enabledProcessBusynessLevels[0];
-
-	for (i = 0; i < size; i++) {
-		Process *process = enabledProcesses[i].get();
+Group::findBestProcessPreferringStickySessionId(unsigned int id) const {
+	Process *bestProcess = nullptr;
+	ProcessList::const_iterator it;
+	ProcessList::const_iterator end = enabledProcesses.end();
+
+	for (it = enabledProcesses.begin(); it != end; it++) {
+		Process *process = it->get();
 		if (process->getStickySessionId() == id) {
 			return process;
-		} else if (leastBusyProcessIndex == -1 || enabledProcessBusynessLevels[i] < lowestBusyness) {
-			leastBusyProcessIndex = i;
-			lowestBusyness = enabledProcessBusynessLevels[i];
+		} else if (!process->isTotallyBusy()
+			&& (
+				bestProcess == nullptr
+				|| process->generation > bestProcess->generation
+				|| (process->generation == bestProcess->generation && process->spawnStartTime < bestProcess->spawnStartTime)
+				|| (process->generation == bestProcess->generation && process->spawnStartTime == bestProcess->spawnStartTime && process->busyness() < bestProcess->busyness())
+			)
+		) {
+			bestProcess = process;
 		}
 	}
 
-	if (leastBusyProcessIndex == -1) {
-		return NULL;
-	} else {
-		return enabledProcesses[leastBusyProcessIndex].get();
-	}
+	return bestProcess;
 }
 
+/**
+ * Given a ProcessList, find the "best" process to route a request to.
+ * At the moment, "best" is defined as the process with the highest generation,
+ * lowest start time, and lowest busyness, in that order of priority.
+ *
+ * If there is no process that can be routed to, then returns nullptr.
+ *
+ * @post result != nullptr || result.canBeRoutedTo()
+ */
 Process *
-Group::findProcessWithLowestBusyness(const ProcessList &processes) const {
+Group::findBestProcess(const ProcessList &processes) const {
 	if (processes.empty()) {
-		return NULL;
+		return nullptr;
 	}
 
-	int lowestBusyness = -1;
-	Process *leastBusyProcess = NULL;
+	Process *bestProcess = nullptr;
 	ProcessList::const_iterator it;
 	ProcessList::const_iterator end = processes.end();
-	for (it = processes.begin(); it != end; it++) {
-		Process *process = (*it).get();
-		int busyness = process->busyness();
-		if (lowestBusyness == -1 || lowestBusyness > busyness) {
-			lowestBusyness = busyness;
-			leastBusyProcess = process;
-		}
-	}
-	return leastBusyProcess;
-}
-
-/**
- * Cache-optimized version of findProcessWithLowestBusyness() for the common case.
- */
-Process *
-Group::findEnabledProcessWithLowestBusyness() const {
-	if (enabledProcesses.empty()) {
-		return NULL;
-	}
 
-	int leastBusyProcessIndex = -1;
-	int lowestBusyness = 0;
-	unsigned int i, size = enabledProcessBusynessLevels.size();
-	const int *enabledProcessBusynessLevels = &this->enabledProcessBusynessLevels[0];
+	for (it = processes.begin(); it != end; it++) {
+		Process *process = it->get();
 
-	for (i = 0; i < size; i++) {
-		if (leastBusyProcessIndex == -1 || enabledProcessBusynessLevels[i] < lowestBusyness) {
-			leastBusyProcessIndex = i;
-			lowestBusyness = enabledProcessBusynessLevels[i];
+		if (!process->isTotallyBusy()
+			&& (
+				bestProcess == nullptr
+				|| process->generation > bestProcess->generation
+				|| (process->generation == bestProcess->generation && process->spawnStartTime < bestProcess->spawnStartTime)
+				|| (process->generation == bestProcess->generation && process->spawnStartTime == bestProcess->spawnStartTime && process->busyness() < bestProcess->busyness())
+			)
+		) {
+			bestProcess = process;
 		}
 	}
-	return enabledProcesses[leastBusyProcessIndex].get();
+
+	return bestProcess;
 }
 
 /**

src/agent/Core/ApplicationPool/Group/SessionManagement.cpp

@@ -30,6 +30,7 @@
 	#include <Core/ApplicationPool/Pool.h>
 #endif
 #include <Core/ApplicationPool/Group.h>
+#include <cassert>
 
 /*************************************************************************
  *
@@ -64,16 +65,17 @@ Group::RouteResult
 Group::route(const Options &options) const {
 	if (OXT_LIKELY(enabledCount > 0)) {
 		if (options.stickySessionId == 0) {
-			Process *process = findEnabledProcessWithLowestBusyness();
-			if (process->canBeRoutedTo()) {
+			Process *process = findBestProcess(enabledProcesses);
+			if (process != nullptr) {
+				assert(process->canBeRoutedTo());
 				return RouteResult(process);
 			} else {
 				return RouteResult(NULL, true);
 			}
 		} else {
-			Process *process = findProcessWithStickySessionIdOrLowestBusyness(
+			Process *process = findBestProcessPreferringStickySessionId(
 				options.stickySessionId);
-			if (process != NULL) {
+			if (process != nullptr) {
 				if (process->canBeRoutedTo()) {
 					return RouteResult(process);
 				} else {
@@ -84,8 +86,9 @@ Group::route(const Options &options) const {
 			}
 		}
 	} else {
-		Process *process = findProcessWithLowestBusyness(disablingProcesses);
-		if (process->canBeRoutedTo()) {
+		Process *process = findBestProcess(disablingProcesses);
+		if (process != nullptr) {
+			assert(process->canBeRoutedTo());
 			return RouteResult(process);
 		} else {
 			return RouteResult(NULL, true);
@@ -310,9 +313,8 @@ Group::get(const Options &newOptions, const GetCallback &callback,
 		assert(m_spawning || restarting() || poolAtFullCapacity());
 
 		if (disablingCount > 0 && !restarting()) {
-			Process *process = findProcessWithLowestBusyness(disablingProcesses);
-			assert(process != NULL);
-			if (!process->isTotallyBusy()) {
+			Process *process = findBestProcess(disablingProcesses);
+			if (process != nullptr && !process->isTotallyBusy()) {
 				return newSession(process, newOptions.currentTime);
 			}
 		}

src/agent/Core/ApplicationPool/Pool.h

@@ -28,10 +28,8 @@
 
 #include <string>
 #include <vector>
-#include <algorithm>
 #include <utility>
 #include <sstream>
-#include <iomanip>
 #include <boost/thread.hpp>
 #include <boost/bind/bind.hpp>
 #include <boost/shared_ptr.hpp>

src/agent/Core/ApplicationPool/Process.h

@@ -53,6 +53,10 @@
 #include <Core/SpawningKit/Result.h>
 #include <Shared/ApplicationPoolApiKey.h>
 
+namespace tut {
+	template<class Data> class test_object;
+}
+
 namespace Passenger {
 namespace ApplicationPool2 {
 
@@ -99,6 +103,9 @@ typedef boost::container::vector<ProcessPtr> ProcessList;
  */
 class Process {
 public:
+	friend class Group;
+	template<class Data> friend class tut::test_object;
+
 	static const unsigned int MAX_SOCKETS_ACCEPTING_HTTP_REQUESTS = 3;
 
 private:
@@ -388,6 +395,10 @@ class Process {
 
 	/** Last time when a session was opened for this Process. */
 	unsigned long long lastUsed;
+    /** Which generation of app processes this one belongs to,
+        inherited from the app group, incremented when a restart
+		is initiated*/
+	const unsigned int generation;
 	/** Number of sessions currently open.
 	 * @invariant session >= 0
 	 */
@@ -450,8 +461,7 @@ class Process {
 	/** Collected by Pool::collectAnalytics(). */
 	ProcessMetrics metrics;
 
-
-	Process(const BasicGroupInfo *groupInfo, const Json::Value &args)
+	Process(const BasicGroupInfo *groupInfo, const unsigned int gen, const Json::Value &args)
 		: info(this, groupInfo, args),
 		  socketsAcceptingHttpRequestsCount(0),
 		  spawnerCreationTime(getJsonUint64Field(args, "spawner_creation_time")),
@@ -462,6 +472,7 @@ class Process {
 		  refcount(1),
 		  index(-1),
 		  lastUsed(spawnEndTime),
+		  generation(gen),
 		  sessions(0),
 		  processed(0),
 		  lifeStatus(ALIVE),
@@ -475,7 +486,7 @@ class Process {
 		indexSocketsAcceptingHttpRequests();
 	}
 
-	Process(const BasicGroupInfo *groupInfo, const SpawningKit::Result &skResult,
+	Process(const BasicGroupInfo *groupInfo, const unsigned int gen, const SpawningKit::Result &skResult,
 		const Json::Value &args)
 		: info(this, groupInfo, skResult),
 		  socketsAcceptingHttpRequestsCount(0),
@@ -487,6 +498,7 @@ class Process {
 		  refcount(1),
 		  index(-1),
 		  lastUsed(spawnEndTime),
+		  generation(gen),
 		  sessions(0),
 		  processed(0),
 		  lifeStatus(ALIVE),