Merge pull request #3884 from terascope/shutdown-error-logging-improv…

…ements [teraslice] add logs to debug shutdown failures in execution controller
terascope · Dec 18, 2024 · a2daaa4 · a2daaa4
2 parents cda74ec + 30a4bfc
commit a2daaa4
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 4 deletions.
diff --git a/packages/teraslice/src/lib/workers/execution-controller/index.ts b/packages/teraslice/src/lib/workers/execution-controller/index.ts
@@ -561,9 +561,9 @@ export class ExecutionController {
 
         // start creating / dispatching slices, this will block until done
         await Promise.all([
-            this.client.sendAvailable(),
-            this._runDispatch(),
-            this.scheduler.run()
+            this.client.sendAvailable().then(() => this.logger.debug('client.sendAvailable() promise resolved')),
+            this._runDispatch().then(() => this.logger.debug('_runDispatch() promise resolved')),
+            this.scheduler.run().then(() => this.logger.debug('scheduler.run() promise resolved'))
         ]);
 
         const schedulerSuccessful = this.scheduler.isFinished && this.scheduler.slicersDone;
@@ -665,6 +665,7 @@ export class ExecutionController {
         clearInterval(dispatchInterval);
 
         this.isDoneDispatching = true;
+        this.logger.debug('done dispatching slices');
     }
 
     _dispatchSlice(slice: Slice, workerId: string) {
@@ -916,6 +917,11 @@ export class ExecutionController {
                         this.exId
                     } to finish...`
                 );
+                this.logger.debug(`Execution controller state vars at timeout:\nisExecutionDone: ${this.isExecutionDone}\nclient.ready: ${this.client.ready}\n`
+                    + `onlineClientCount: ${this.server.onlineClientCount}\nserver.isShuttingDown: ${this.server.isShuttingDown}`
+                    + `isShuttingDown: ${this.isShuttingDown}\nisShutdown: ${this.isShutdown}\n`
+                    + `isDoneDispatching: ${this.isDoneDispatching}\npendingDispatches: ${this.pendingDispatches}\n`
+                    + `scheduler.isFinished: ${this.scheduler.isFinished}\npendingSlices: ${this.pendingSlices}\n`);
                 return null;
             }
 

diff --git a/packages/teraslice/src/lib/workers/execution-controller/scheduler.ts b/packages/teraslice/src/lib/workers/execution-controller/scheduler.ts
@@ -130,7 +130,11 @@ export class Scheduler {
         );
 
         await pWhile(async () => {
-            if (!this._creating) return true;
+            if (!this._creating) {
+                this.logger.debug('done creating remaining slices');
+                return true;
+            }
+            this.logger.debug(`waiting for ${this._creating} remaining slices to be created...`);
             await pDelay(100);
             return false;
         });