diff --git a/packages/teraslice/src/lib/cluster/services/execution.ts b/packages/teraslice/src/lib/cluster/services/execution.ts index db7df820443..86a916907e8 100644 --- a/packages/teraslice/src/lib/cluster/services/execution.ts +++ b/packages/teraslice/src/lib/cluster/services/execution.ts @@ -240,6 +240,19 @@ export class ExecutionService { return; } + const runningStatuses = this.executionStorage.getRunningStatuses(); + + if (runningStatuses.includes(status)) { + // This should never happen. If we get here with a running status + // something has gone wrong. Mark execution as failed before shutdown. + this.logger.warn(`Cluster_master is changing status of execution ${exId} from ${status} to failed`); + await this.executionStorage.setStatus( + exId, + 'failed', + this.executionStorage.executionMetaData(null, getFullErrorStack(err)) + ); + } + this.logger.debug(`execution ${exId} finished, shutting down execution`); try { diff --git a/packages/teraslice/src/lib/workers/execution-controller/index.ts b/packages/teraslice/src/lib/workers/execution-controller/index.ts index 7b6eeb09e96..36b1e964416 100644 --- a/packages/teraslice/src/lib/workers/execution-controller/index.ts +++ b/packages/teraslice/src/lib/workers/execution-controller/index.ts @@ -899,13 +899,22 @@ export class ExecutionController { const invalidStateMsg = (state: string) => { const prefix = `Execution ${this.exId} was starting in ${state} status`; - return `${prefix} sending execution:finished event to cluster master`; + return `${prefix}, sending execution:finished event to cluster master`; }; if (includes(terminalStatuses, status)) { error = new Error(invalidStateMsg('terminal')); } else if (includes(runningStatuses, status)) { error = new Error(invalidStateMsg('running')); + // If in a running status the execution process + // crashed and k8s is trying to restart the pod, + // e.g. execution controller OOM. + this.logger.warn(`Changing execution status from ${status} to failed`); + await this.executionStorage.setStatus( + this.exId, + 'failed', + this.executionStorage.executionMetaData(null, getFullErrorStack(error)) + ); } else { return true; }