From 86cbbfe2bf2192e770192f270e78cbb66814b270 Mon Sep 17 00:00:00 2001 From: aelassas Date: Fri, 1 Dec 2023 16:45:52 +0100 Subject: [PATCH] Add retryCount and retryTimeout options #64 --- src/backend/Wexflow.Backend/css/designer.css | 1 + src/backend/Wexflow.Backend/designer.html | 6 ++ src/backend/Wexflow.Backend/js/designer.js | 66 +++++++++++++-- src/backend/Wexflow.Backend/js/language.js | 27 +++++- src/net/Wexflow.Core/Workflow.cs | 42 ++++++++-- src/net/Wexflow.Core/Workflow.xml | 8 +- .../Contracts/Workflow/WorkflowInfo.cs | 4 +- .../Wexflow.Server/Contracts/WorkflowInfo.cs | 27 +++++- src/net/Wexflow.Server/WexflowService.cs | 83 ++++++++++++++++--- src/netcore/Wexflow.Core/Workflow.cs | 42 ++++++++-- src/netcore/Wexflow.Core/Workflow.xml | 8 +- .../Contracts/Workflow/WorkflowInfo.cs | 4 +- .../Wexflow.Server/Contracts/WorkflowInfo.cs | 27 +++++- src/netcore/Wexflow.Server/WexflowService.cs | 80 +++++++++++++++--- 14 files changed, 373 insertions(+), 52 deletions(-) diff --git a/src/backend/Wexflow.Backend/css/designer.css b/src/backend/Wexflow.Backend/css/designer.css index 81cedf27..2e1457e7 100644 --- a/src/backend/Wexflow.Backend/css/designer.css +++ b/src/backend/Wexflow.Backend/css/designer.css @@ -674,6 +674,7 @@ color: #393C44; font-size: 14px; font-weight: bold; color: #253134; + margin-top: 3px; } .inputtext { diff --git a/src/backend/Wexflow.Backend/designer.html b/src/backend/Wexflow.Backend/designer.html index 4cf5cedd..2dfd160c 100644 --- a/src/backend/Wexflow.Backend/designer.html +++ b/src/backend/Wexflow.Backend/designer.html @@ -127,6 +127,12 @@

EnableParallelJobs

+ +

Task Retries

+ +

Task Retries Timeout

+ +

Local Variables diff --git a/src/backend/Wexflow.Backend/js/designer.js b/src/backend/Wexflow.Backend/js/designer.js index bad7ec17..1f655ac0 100644 --- a/src/backend/Wexflow.Backend/js/designer.js +++ b/src/backend/Wexflow.Backend/js/designer.js @@ -51,10 +51,17 @@ document.getElementById("wfenabled-label").innerHTML = language.get("wfenabled-label"); document.getElementById("wfapproval-label").innerHTML = language.get("wfapproval-label"); document.getElementById("wfenablepj-label").innerHTML = language.get("wfenablepj-label"); + + document.getElementById("wfretrycount-label").innerHTML = language.get("wfretrycount-label"); + document.getElementById("wfretrytimeout-label").innerHTML = language.get("wfretrytimeout-label"); + document.getElementById("wfretrycount-label").title = language.get("wfretrycount-title"); + document.getElementById("wfretrytimeout-label").title = language.get("wfretrytimeout-title"); + document.getElementById("wf-local-vars-label").innerHTML = language.get("wf-local-vars-label"); document.getElementById("wf-add-var").value = language.get("wf-add-var"); document.getElementById("removeblock").innerHTML = language.get("removeblock"); document.getElementById("removeworkflow").innerHTML = language.get("removeworkflow"); + let removeVariableButtons = document.getElementsByClassName("wf-remove-var"); for (let i = 0; i < removeVariableButtons.length; i++) { removeVariableButtons[i].innerHTML = language.get("wf-remove-var"); @@ -293,7 +300,7 @@ searchtasks.onkeyup = function (event) { event.preventDefault(); - if (event.keyCode === 13) { // Enter + if (event.key === 'Enter') { // Enter loadTasks(); } }; @@ -352,6 +359,8 @@ "IsEnabled": document.getElementById("wfenabled").checked, "IsApproval": document.getElementById("wfapproval").checked, "EnableParallelJobs": document.getElementById("wfenablepj").checked, + "RetryCount": document.getElementById("wfretrycount").value, + "RetryTimeout": document.getElementById("wfretrytimeout").value, "LocalVariables": [] }, "Tasks": [] @@ -590,6 +599,8 @@ "IsEnabled": document.getElementById("wfenabled").checked, "IsApproval": document.getElementById("wfapproval").checked, "EnableParallelJobs": document.getElementById("wfenablepj").checked, + "RetryCount": document.getElementById("wfretrycount").value, + "RetryTimeout": document.getElementById("wfretrytimeout").value, "LocalVariables": [] }, "Tasks": [] @@ -644,6 +655,8 @@ "IsEnabled": document.getElementById("wfenabled").checked, "IsApproval": document.getElementById("wfapproval").checked, "EnableParallelJobs": document.getElementById("wfenablepj").checked, + "RetryCount": document.getElementById("wfretrycount").value, + "RetryTimeout": document.getElementById("wfretrytimeout").value, "LocalVariables": [] }, "Tasks": [] @@ -880,7 +893,6 @@ self.parentNode.parentNode.nextSibling.firstChild.innerHTML += settingValueHtml; self.parentNode.parentNode.nextSibling.querySelector(".wf-setting-type").value = settingType; - // bind events let settingValueInput = self.parentNode.parentNode.nextSibling.querySelector(".wf-setting-value"); if (settingValueInput) { @@ -1832,7 +1844,6 @@ } }; - wfclose.onclick = function () { if (wfpropHidden === false) { document.getElementById("wfpropwrap").style.right = -wfpropwidth + "px"; @@ -1863,6 +1874,8 @@ "IsEnabled": document.getElementById("wfenabled").checked, "IsApproval": document.getElementById("wfapproval").checked, "EnableParallelJobs": document.getElementById("wfenablepj").checked, + "RetryCount": document.getElementById("wfretrycount").value, + "RetryTimeout": document.getElementById("wfretrytimeout").value, "LocalVariables": [] }, "Tasks": [] @@ -1899,6 +1912,24 @@ document.getElementById("wfenablepj").onchange = function () { workflow.WorkflowInfo.EnableParallelJobs = this.checked; }; + document.getElementById("wfretrycount").onchange = function () { + if (isInt(this.value) === false) { + this.style.borderColor = "#FF0000"; + } else { + this.style.borderColor = "#CCCCCC"; + } + + workflow.WorkflowInfo.RetryCount = this.value; + }; + document.getElementById("wfretrytimeout").onchange = function () { + if (isInt(this.value) === false) { + this.style.borderColor = "#FF0000"; + } else { + this.style.borderColor = "#CCCCCC"; + } + + workflow.WorkflowInfo.RetryTimeout = this.value; + }; // main function for updating tasks function updateTasks() { @@ -2071,6 +2102,16 @@ }, workflow, auth); }; + if (isInt(document.getElementById("wfretrycount").value) === false) { + window.Common.toastInfo(language.get("toast-workflow-retry-count-error")); + return; + } + + if (isInt(document.getElementById("wfretrytimeout").value) === false) { + window.Common.toastInfo(language.get("toast-workflow-retry-timeout-error")); + return; + } + let wfIdStr = document.getElementById("wfid").value; if (isInt(wfIdStr)) { let workflowId = parseInt(wfIdStr); @@ -2209,7 +2250,7 @@ initialWorkflow = JSON.parse(JSON.stringify(workflow)); removeworkflow.style.display = "block"; jsonEditorChanged = false; - openJsonView(JSON.stringify(workflow, null, '\t')); + //openJsonView(JSON.stringify(workflow, null, '\t')); if (callback) { callback(); } else { @@ -2945,7 +2986,15 @@ let graph = val; let xmlVal = '\r\n'; - xmlVal += '\t\r\n\t\t' + (workflow.WorkflowInfo.Period !== '' && workflow.WorkflowInfo.Period !== '00:00:00' ? ('\r\n\t\t') : '') + (workflow.WorkflowInfo.CronExpression !== '' && workflow.WorkflowInfo.CronExpression !== null ? ('\r\n\t\t') : '') + '\r\n\t\t\r\n\t\t\r\n\t\t\r\n\t\r\n'; + xmlVal += '\t\r\n\t\t' + + (workflow.WorkflowInfo.Period !== '' && workflow.WorkflowInfo.Period !== '00:00:00' ? ('\r\n\t\t') : '') + + (workflow.WorkflowInfo.CronExpression !== '' && workflow.WorkflowInfo.CronExpression !== null ? ('\r\n\t\t') : '') + + '\r\n\t\t\r\n\t\t' + + '\r\n\t\t' + + '\r\n\t\t' + + '\r\n\t\t' + + '\r\n\t\r\n'; if (workflow.WorkflowInfo.LocalVariables.length > 0) { xmlVal += '\t\r\n'; for (let i = 0; i < workflow.WorkflowInfo.LocalVariables.length; i++) { @@ -3160,6 +3209,9 @@ document.getElementById("wfapproval").checked = workflow.WorkflowInfo.IsApproval; document.getElementById("wfenablepj").checked = workflow.WorkflowInfo.EnableParallelJobs; + document.getElementById("wfretrycount").value = workflow.WorkflowInfo.RetryCount; + document.getElementById("wfretrytimeout").value = workflow.WorkflowInfo.RetryTimeout; + // Local variables document.getElementsByClassName("wf-local-vars")[0].innerHTML = ""; if (workflow.WorkflowInfo.LocalVariables.length > 0) { @@ -3380,7 +3432,7 @@ searchworkflows.select(); searchworkflows.onkeyup = function (event) { event.preventDefault(); - if (event.keyCode === 13) { // Enter + if (event.key === 'Enter') { // Enter let jbox = document.getElementsByClassName("jBox-content")[0]; window.Common.get(uri + "/search?s=" + searchworkflows.value, @@ -3580,6 +3632,8 @@ "IsEnabled": document.getElementById("wfenabled").checked, "IsApproval": document.getElementById("wfapproval").checked, "EnableParallelJobs": document.getElementById("wfenablepj").checked, + "RetryCount": document.getElementById("wfretrycount").value, + "RetryTimeout": document.getElementById("wfretrytimeout").value, "LocalVariables": [] }, "Tasks": [] diff --git a/src/backend/Wexflow.Backend/js/language.js b/src/backend/Wexflow.Backend/js/language.js index d3a44a07..7393ccb0 100644 --- a/src/backend/Wexflow.Backend/js/language.js +++ b/src/backend/Wexflow.Backend/js/language.js @@ -72,7 +72,7 @@ languages["en"]["job-part-1"] = "The job "; languages["en"]["job-approved"] = " was approved."; languages["en"]["job-rejected"] = " was rejected."; - languages["en"]["workflows-server-error"] = "An error occurred while retrieving workflows. Check that Wexflow server is running correctly." + languages["en"]["workflows-server-error"] = "An error occurred while retrieving workflows. Check that Wexflow server is running correctly."; languages["en"]["job-approved-error-part-1"] = "An error occurred while approving the job "; languages["en"]["job-rejected-error-part-1"] = "An error occurred while rejecting the job "; languages["en"]["job-approved-error-part-2"] = " of the workflow "; @@ -236,6 +236,13 @@ languages["en"]["th-approved-on"] = "Approved on"; languages["en"]["toast-save-and-run"] = "Workflow saved and started successfully."; + languages["en"]["wfretrycount-label"] = "Task Retries"; + languages["en"]["wfretrytimeout-label"] = "Task Retries Timeout"; + languages["en"]["wfretrycount-title"] = "Number of task retries in case of failure"; + languages["en"]["wfretrytimeout-title"] = "Waiting time between two tries in milliseconds"; + languages["en"]["toast-workflow-retry-count-error"] = "Task Retries is invalid."; + languages["en"]["toast-workflow-retry-timeout-error"] = "Task Retries Timeout is invalid."; + // // fr // @@ -296,7 +303,7 @@ languages["fr"]["job-part-1"] = "Le job "; languages["fr"]["job-approved"] = " a été approuvé."; languages["fr"]["job-rejected"] = " a été rejeté."; - languages["fr"]["workflows-server-error"] = "Une erreur s'est produite lors de la récupération de workflows. Vérifiez que le serveur tourne." + languages["fr"]["workflows-server-error"] = "Une erreur s'est produite lors de la récupération de workflows. Vérifiez que le serveur tourne."; languages["fr"]["job-approved-error-part-1"] = "Une erreur s'est produite lors de l'approbation du job "; languages["fr"]["job-rejected-error-part-1"] = "Une erreur s'est produite lors du rejet du job "; languages["fr"]["job-approved-error-part-2"] = " du workflow "; @@ -460,6 +467,13 @@ languages["fr"]["th-approved-on"] = "Approuvé le"; languages["fr"]["toast-save-and-run"] = "Workflow enregistré et démarré avec succès."; + languages["fr"]["wfretrycount-label"] = "Nombre de tentatives des tâches"; + languages["fr"]["wfretrytimeout-label"] = "Timeout des tentatives des tâches"; + languages["fr"]["wfretrycount-title"] = "Nombre de tentatives de tâche en cas d'échec"; + languages["fr"]["wfretrytimeout-title"] = "Délai d'attente de tâche entre deux tentatives en millisecondes"; + languages["fr"]["toast-workflow-retry-count-error"] = "Le nombre d'essais n'est pas au bon format."; + languages["fr"]["toast-workflow-retry-timeout-error"] = "Le nombre d'essais n'est pas au bon format."; + // // da // @@ -520,7 +534,7 @@ languages["da"]["job-part-1"] = "Jobbet"; languages["da"]["job-approved"] = "blev godkendt."; languages["da"]["job-rejected"] = "blev afvist."; - languages["da"]["workflows-server-error"] = "Der opstod en fejl under hentning af workflows. Kontroller, at Wexflow-server kører korrekt." + languages["da"]["workflows-server-error"] = "Der opstod en fejl under hentning af workflows. Kontroller, at Wexflow-server kører korrekt."; languages["da"]["job-approved-error-part-1"] = "Der opstod en fejl under godkendelse af jobbet"; languages["da"]["job-rejected-error-part-1"] = "En fejl opstod under afvisning af jobbet"; languages["da"]["job-approved-error-part-2"] = "af opgavern"; @@ -684,6 +698,13 @@ languages["da"]["th-approved-on"] = "Godkendt på"; languages["da"]["toast-save-and-run"] = "Workflow gemt og startet med succes."; + languages["da"]["wfretrycount-label"] = "Opgaveforsøg igen"; + languages["da"]["wfretrytimeout-label"] = "Timeout for Opgaveforsøg igen"; + languages["da"]["wfretrycount-title"] = "Antal mislykkede opgaveforsøg"; + languages["da"]["wfretrytimeout-title"] = "Ventetid mellem to forsøg i millisekunder"; + languages["da"]["toast-workflow-retry-count-error"] = "Opgaveforsøg er ugyldige."; + languages["da"]["toast-workflow-retry-timeout-error"] = "Timeout for opgavegenforsøg er ugyldig."; + return { codes: codes, languages: languages diff --git a/src/net/Wexflow.Core/Workflow.cs b/src/net/Wexflow.Core/Workflow.cs index 9d4ae513..68845b7f 100644 --- a/src/net/Wexflow.Core/Workflow.cs +++ b/src/net/Wexflow.Core/Workflow.cs @@ -225,6 +225,14 @@ public class Workflow /// Started on date time. /// public DateTime StartedOn { get; private set; } + ///

+ /// Number of retry times in case of failures. Default is 0. + /// + public int RetryCount { get; private set; } + /// + /// The retry timeout between two tries. Default is 1500ms. + /// + public int RetryTimeout { get; private set; } private readonly Queue _jobsQueue; private Thread _thread; @@ -513,6 +521,12 @@ private void Load(string xml) var enableParallelJobsStr = GetWorkflowSetting(xdoc, "enableParallelJobs", false); EnableParallelJobs = bool.Parse(string.IsNullOrEmpty(enableParallelJobsStr) ? "true" : enableParallelJobsStr); + var retryCount = GetWorkflowSetting(xdoc, "retryCount", false); + RetryCount = int.Parse(string.IsNullOrEmpty(retryCount) ? "0" : retryCount); + + var retryTimeout = GetWorkflowSetting(xdoc, "retryTimeout", false); + RetryTimeout = int.Parse(string.IsNullOrEmpty(retryTimeout) ? "1500" : retryTimeout); + if (xdoc.Root != null) { var xExecutionGraph = xdoc.Root.Element(XNamespaceWf + "ExecutionGraph"); @@ -1299,6 +1313,22 @@ private Status RunTasks(Node[] nodes, Task[] tasks, bool force) return IsRejected ? Status.Rejected : success ? Status.Success : atLeastOneSucceed || warning ? Status.Warning : Status.Error; } + private TaskStatus RunTask(Task task) + { + var status = task.Run(); + + var retries = 0; + while (status.Status != Status.Success && retries < RetryCount) + { + Thread.Sleep(RetryTimeout); + task.InfoFormat("Retry attempt {0}", retries + 1); + status = task.Run(); + retries++; + } + + return status; + } + private void RunSequentialTasks(IEnumerable tasks, ref bool success, ref bool warning, ref bool error) { var atLeastOneSucceed = false; @@ -1320,7 +1350,7 @@ private void RunSequentialTasks(IEnumerable tasks, ref bool success, ref b Logs.AddRange(task.Logs); continue; } - var status = task.Run(); + var status = RunTask(task); Logs.AddRange(task.Logs); success &= status.Status == Status.Success; warning |= status.Status == Status.Warning; @@ -1365,7 +1395,7 @@ private void RunTasks(Task[] tasks, Node[] nodes, Node node, bool force, ref boo { if (task.IsEnabled && !task.IsStopped && (!IsApproval || (IsApproval && !IsRejected) || force)) { - var status = task.Run(); + var status = RunTask(task); Logs.AddRange(task.Logs); success &= status.Status == Status.Success; @@ -1399,7 +1429,7 @@ private void RunTasks(Task[] tasks, Node[] nodes, Node node, bool force, ref boo { if (childTask.IsEnabled && !childTask.IsStopped && (!IsApproval || (IsApproval && !IsRejected) || force)) { - var childStatus = childTask.Run(); + var childStatus = RunTask(childTask); Logs.AddRange(childTask.Logs); success &= childStatus.Status == Status.Success; @@ -1455,7 +1485,7 @@ private void RunIf(Task[] tasks, Node[] nodes, If @if, bool force, ref bool succ { if (ifTask.IsEnabled && !ifTask.IsStopped && (!IsApproval || (IsApproval && !IsRejected))) { - var status = ifTask.Run(); + var status = RunTask(ifTask); Logs.AddRange(ifTask.Logs); success &= status.Status == Status.Success; @@ -1520,7 +1550,7 @@ private void RunWhile(Task[] tasks, Node[] nodes, While @while, bool force, ref { while (true) { - var status = whileTask.Run(); + var status = RunTask(whileTask); Logs.AddRange(whileTask.Logs); success &= status.Status == Status.Success; @@ -1572,7 +1602,7 @@ private void RunSwitch(Task[] tasks, Node[] nodes, Switch @switch, bool force, r { if (switchTask.IsEnabled && !switchTask.IsStopped && (!IsApproval || (IsApproval && !IsRejected))) { - var status = switchTask.Run(); + var status = RunTask(switchTask); Logs.AddRange(switchTask.Logs); success &= status.Status == Status.Success; diff --git a/src/net/Wexflow.Core/Workflow.xml b/src/net/Wexflow.Core/Workflow.xml index 254a08db..f8f19bdc 100644 --- a/src/net/Wexflow.Core/Workflow.xml +++ b/src/net/Wexflow.Core/Workflow.xml @@ -22,7 +22,11 @@ The possible values are true or false. An approval workflow must contain at least one Approval task or more. - The enableParallelJobs option Shows whether workflow jobs are executed in parallel. - Otherwise jobs are queued. Defaults to true. + Otherwise jobs are queued. Defaults to true. + - The retryCount option allows to retry a task a certain number of times in case of + failure. Defaults to 0 (no retry). + - The retryTimeout option indicates the waiting time between two tries in milliseconds. + Defaults to 1500ms. - A LocalVariables section which contains local variables. - A Tasks section which contains the tasks that will be executed by the workflow one after the other. @@ -43,6 +47,8 @@ + + diff --git a/src/net/Wexflow.Server/Contracts/Workflow/WorkflowInfo.cs b/src/net/Wexflow.Server/Contracts/Workflow/WorkflowInfo.cs index 2740a503..b96676ed 100644 --- a/src/net/Wexflow.Server/Contracts/Workflow/WorkflowInfo.cs +++ b/src/net/Wexflow.Server/Contracts/Workflow/WorkflowInfo.cs @@ -12,6 +12,8 @@ public class WorkflowInfo public string Description { get; set; } public string Period { get; set; } public string CronExpression { get; set; } + public int RetryCount { get; set; } + public int RetryTimeout { get; set; } public Variable[] LocalVariables { get; set; } } -} +} \ No newline at end of file diff --git a/src/net/Wexflow.Server/Contracts/WorkflowInfo.cs b/src/net/Wexflow.Server/Contracts/WorkflowInfo.cs index 1199990f..8af1b535 100644 --- a/src/net/Wexflow.Server/Contracts/WorkflowInfo.cs +++ b/src/net/Wexflow.Server/Contracts/WorkflowInfo.cs @@ -48,7 +48,30 @@ public class WorkflowInfo : IComparable public string StartedOn { get; set; } - public WorkflowInfo(string dbId, int id, Guid instanceId, string name, string filePath, LaunchType launchType, bool isEnabled, bool isApproval, bool enableParallelJobs, bool isWaitingForApproval, string desc, bool isRunning, bool isPaused, string period, string cronExpression, bool isExecutionGraphEmpty, Variable[] localVariables, string startedOn) + public int RetryCount { get; set; } + + public int RetryTimeout { get; set; } + + public WorkflowInfo(string dbId, + int id, + Guid instanceId, + string name, + string filePath, + LaunchType launchType, + bool isEnabled, + bool isApproval, + bool enableParallelJobs, + bool isWaitingForApproval, + string desc, + bool isRunning, + bool isPaused, + string period, + string cronExpression, + bool isExecutionGraphEmpty, + Variable[] localVariables, + string startedOn, + int retryCount, + int retryTimeout) { DbId = dbId; Id = id; @@ -68,6 +91,8 @@ public WorkflowInfo(string dbId, int id, Guid instanceId, string name, string fi IsExecutionGraphEmpty = isExecutionGraphEmpty; LocalVariables = localVariables; StartedOn = startedOn; + RetryCount = retryCount; + RetryTimeout = retryTimeout; } public int CompareTo(object obj) diff --git a/src/net/Wexflow.Server/WexflowService.cs b/src/net/Wexflow.Server/WexflowService.cs index 57c18df4..4948aace 100644 --- a/src/net/Wexflow.Server/WexflowService.cs +++ b/src/net/Wexflow.Server/WexflowService.cs @@ -180,7 +180,11 @@ private void Search() wf.Period.ToString(@"dd\.hh\:mm\:ss"), wf.CronExpression, wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() - , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]))) + , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout + ) + ) .ToArray(); } else if (user.UserProfile == Core.Db.UserProfile.Administrator) @@ -196,7 +200,11 @@ private void Search() wf.Period.ToString(@"dd\.hh\:mm\:ss"), wf.CronExpression, wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() - , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]))) + , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout + ) + ) .ToArray(); } } @@ -244,7 +252,11 @@ private void SearchApprovalWorkflows() wf.Period.ToString(@"dd\.hh\:mm\:ss"), wf.CronExpression, wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() - , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]))) + , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout + ) + ) .ToArray(); } else if (user.UserProfile == Core.Db.UserProfile.Administrator) @@ -261,7 +273,10 @@ private void SearchApprovalWorkflows() wf.Period.ToString(@"dd\.hh\:mm\:ss"), wf.CronExpression, wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() - , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]))) + , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout + )) .ToArray(); } } @@ -298,6 +313,8 @@ private void GetWorkflow() wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout ); var user = WexflowServer.WexflowEngine.GetUser(username); @@ -369,6 +386,8 @@ private void GetJob() wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout ); var user = WexflowServer.WexflowEngine.GetUser(username); @@ -434,6 +453,8 @@ private void GetJobs() w.IsExecutionGraphEmpty , w.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() , w.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout )); var user = WexflowServer.WexflowEngine.GetUser(username); @@ -998,7 +1019,9 @@ private void GetWorkflowJson() IsApproval = wf.IsApproval, EnableParallelJobs = wf.EnableParallelJobs, Description = wf.Description, - LocalVariables = variables.ToArray() + LocalVariables = variables.ToArray(), + RetryCount = wf.RetryCount, + RetryTimeout = wf.RetryTimeout }; var tasks = new List(); @@ -1820,6 +1843,9 @@ private SaveResult SaveJsonWorkflow(Core.Db.User user, string json) var enableParallelJobs = (bool)wi.SelectToken("EnableParallelJobs"); var workflowDesc = (string)wi.SelectToken("Description"); + var retryCount = (int)wi.SelectToken("RetryCount"); + var retryTimeout = (int)wi.SelectToken("RetryTimeout"); + // Local variables var xLocalVariables = new XElement(Xn + "LocalVariables"); var variables = wi.SelectToken("LocalVariables"); @@ -1918,12 +1944,13 @@ private SaveResult SaveJsonWorkflow(Core.Db.User user, string json) , new XElement(Xn + "Setting" , new XAttribute("name", "enableParallelJobs") , new XAttribute("value", enableParallelJobs.ToString().ToLower())) - //, new XElement(xn + "Setting" - // , new XAttribute("name", "period") - // , new XAttribute("value", workflowPeriod.ToString(@"dd\.hh\:mm\:ss"))) - //, new XElement(xn + "Setting" - // , new XAttribute("name", "cronExpression") - // , new XAttribute("value", cronExpression)) + , new XElement(Xn + "Setting" + , new XAttribute("name", "retryCount") + , new XAttribute("value", retryCount)) + , new XElement(Xn + "Setting" + , new XAttribute("name", "retryTimeout") + , new XAttribute("value", retryTimeout)) + ) , xLocalVariables , xtasks @@ -2003,6 +2030,9 @@ private SaveResult SaveJsonWorkflow(Core.Db.User user, string json) var enableParallelJobs = (bool)(wi.SelectToken("EnableParallelJobs") ?? true); var workflowDesc = (string)wi.SelectToken("Description") ?? string.Empty; + var retryCount = (int)wi.SelectToken("RetryCount"); + var retryTimeout = (int)wi.SelectToken("RetryTimeout"); + if (xdoc.Root == null) throw new InvalidOperationException("Root is null"); (xdoc.Root.Attribute("id") ?? throw new InvalidOperationException()).Value = workflowId.ToString(); (xdoc.Root.Attribute("name") ?? throw new InvalidOperationException()).Value = workflowName; @@ -2098,6 +2128,32 @@ private SaveResult SaveJsonWorkflow(Core.Db.User user, string json) // } //} + var xwfRetryCount = xdoc.Root.XPathSelectElement("wf:Settings/wf:Setting[@name='retryCount']", + wf.XmlNamespaceManager); + if (xwfRetryCount != null) + { + (xwfRetryCount.Attribute("value") ?? throw new InvalidOperationException()).Value = retryCount.ToString(); + } + else + { + (xdoc.Root.XPathSelectElement("wf:Settings", wf.XmlNamespaceManager) ?? throw new InvalidOperationException()) + .Add(new XElement(wf.XNamespaceWf + "Setting", new XAttribute("name", "retryCount"), + new XAttribute("value", retryCount))); + } + + var xwfRetryTimeout = xdoc.Root.XPathSelectElement("wf:Settings/wf:Setting[@name='retryTimeout']", + wf.XmlNamespaceManager); + if (xwfRetryTimeout != null) + { + (xwfRetryTimeout.Attribute("value") ?? throw new InvalidOperationException()).Value = retryTimeout.ToString(); + } + else + { + (xdoc.Root.XPathSelectElement("wf:Settings", wf.XmlNamespaceManager) ?? throw new InvalidOperationException()) + .Add(new XElement(wf.XNamespaceWf + "Setting", new XAttribute("name", "retryTimeout"), + new XAttribute("value", retryTimeout))); + } + // Local variables var xLocalVariables = xdoc.Root.Element(wf.XNamespaceWf + "LocalVariables"); if (xLocalVariables != null) @@ -3251,7 +3307,10 @@ private void GetUserWorkflows() wf.Period.ToString(@"dd\.hh\:mm\:ss"), wf.CronExpression, wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() - , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]))) + , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout + )) .ToArray(); } catch (Exception e) diff --git a/src/netcore/Wexflow.Core/Workflow.cs b/src/netcore/Wexflow.Core/Workflow.cs index 84f4c1e9..47221d76 100644 --- a/src/netcore/Wexflow.Core/Workflow.cs +++ b/src/netcore/Wexflow.Core/Workflow.cs @@ -226,6 +226,14 @@ public class Workflow /// Started on date time. /// public DateTime StartedOn { get; private set; } + /// + /// Number of retry times in case of failures. Default is 0. + /// + public int RetryCount { get; private set; } + /// + /// The retry timeout between two tries. Default is 1500ms. + /// + public int RetryTimeout { get; private set; } private readonly Queue _jobsQueue; private Thread _thread; @@ -508,6 +516,12 @@ private void Load(string xml) var enableParallelJobsStr = GetWorkflowSetting(xdoc, "enableParallelJobs", false); EnableParallelJobs = bool.Parse(string.IsNullOrEmpty(enableParallelJobsStr) ? "true" : enableParallelJobsStr); + var retryCount = GetWorkflowSetting(xdoc, "retryCount", false); + RetryCount = int.Parse(string.IsNullOrEmpty(retryCount) ? "0" : retryCount); + + var retryTimeout = GetWorkflowSetting(xdoc, "retryTimeout", false); + RetryTimeout = int.Parse(string.IsNullOrEmpty(retryTimeout) ? "1500" : retryTimeout); + if (xdoc.Root != null) { var xExecutionGraph = xdoc.Root.Element(XNamespaceWf + "ExecutionGraph"); @@ -1353,6 +1367,22 @@ private Status RunTasks(Node[] nodes, Task[] tasks, bool force) return IsRejected ? Status.Rejected : success ? Status.Success : atLeastOneSucceed || warning ? Status.Warning : Status.Error; } + private TaskStatus RunTask(Task task) + { + var status = task.Run(); + + var retries = 0; + while (status.Status != Status.Success && retries < RetryCount) + { + Thread.Sleep(RetryTimeout); + task.InfoFormat("Retry attempt {0}", retries + 1); + status = task.Run(); + retries++; + } + + return status; + } + private void RunSequentialTasks(IEnumerable tasks, ref bool success, ref bool warning, ref bool error) { var atLeastOneSucceed = false; @@ -1374,7 +1404,7 @@ private void RunSequentialTasks(IEnumerable tasks, ref bool success, ref b Logs.AddRange(task.Logs); continue; } - var status = task.Run(); + var status = RunTask(task); Logs.AddRange(task.Logs); success &= status.Status == Status.Success; warning |= status.Status == Status.Warning; @@ -1419,7 +1449,7 @@ private void RunTasks(Task[] tasks, Node[] nodes, Node node, bool force, ref boo { if (task.IsEnabled && !task.IsStopped && (!IsApproval || (IsApproval && !IsRejected) || force)) { - var status = task.Run(); + var status = RunTask(task); Logs.AddRange(task.Logs); success &= status.Status == Status.Success; @@ -1453,7 +1483,7 @@ private void RunTasks(Task[] tasks, Node[] nodes, Node node, bool force, ref boo { if (childTask.IsEnabled && !childTask.IsStopped && (!IsApproval || (IsApproval && !IsRejected) || force)) { - var childStatus = childTask.Run(); + var childStatus = RunTask(childTask); Logs.AddRange(childTask.Logs); success &= childStatus.Status == Status.Success; @@ -1509,7 +1539,7 @@ private void RunIf(Task[] tasks, Node[] nodes, If @if, bool force, ref bool succ { if (ifTask.IsEnabled && !ifTask.IsStopped && (!IsApproval || (IsApproval && !IsRejected))) { - var status = ifTask.Run(); + var status = RunTask(ifTask); Logs.AddRange(ifTask.Logs); success &= status.Status == Status.Success; @@ -1574,7 +1604,7 @@ private void RunWhile(Task[] tasks, Node[] nodes, While @while, bool force, ref { while (true) { - var status = whileTask.Run(); + var status = RunTask(whileTask); Logs.AddRange(whileTask.Logs); success &= status.Status == Status.Success; @@ -1626,7 +1656,7 @@ private void RunSwitch(Task[] tasks, Node[] nodes, Switch @switch, bool force, r { if (switchTask.IsEnabled && !switchTask.IsStopped && (!IsApproval || (IsApproval && !IsRejected))) { - var status = switchTask.Run(); + var status = RunTask(switchTask); Logs.AddRange(switchTask.Logs); success &= status.Status == Status.Success; diff --git a/src/netcore/Wexflow.Core/Workflow.xml b/src/netcore/Wexflow.Core/Workflow.xml index 254a08db..1f1d3aa9 100644 --- a/src/netcore/Wexflow.Core/Workflow.xml +++ b/src/netcore/Wexflow.Core/Workflow.xml @@ -22,7 +22,11 @@ The possible values are true or false. An approval workflow must contain at least one Approval task or more. - The enableParallelJobs option Shows whether workflow jobs are executed in parallel. - Otherwise jobs are queued. Defaults to true. + Otherwise jobs are queued. Defaults to true. + - The retryCount option allows to retry a task a certain number of times in case of + failure. Defaults to 0 (no retry). + - The retryTimeout option indicates the waiting time between two tries in milliseconds. + Defaults to 1500. - A LocalVariables section which contains local variables. - A Tasks section which contains the tasks that will be executed by the workflow one after the other. @@ -43,6 +47,8 @@ + + diff --git a/src/netcore/Wexflow.Server/Contracts/Workflow/WorkflowInfo.cs b/src/netcore/Wexflow.Server/Contracts/Workflow/WorkflowInfo.cs index 2740a503..b96676ed 100644 --- a/src/netcore/Wexflow.Server/Contracts/Workflow/WorkflowInfo.cs +++ b/src/netcore/Wexflow.Server/Contracts/Workflow/WorkflowInfo.cs @@ -12,6 +12,8 @@ public class WorkflowInfo public string Description { get; set; } public string Period { get; set; } public string CronExpression { get; set; } + public int RetryCount { get; set; } + public int RetryTimeout { get; set; } public Variable[] LocalVariables { get; set; } } -} +} \ No newline at end of file diff --git a/src/netcore/Wexflow.Server/Contracts/WorkflowInfo.cs b/src/netcore/Wexflow.Server/Contracts/WorkflowInfo.cs index 1199990f..8af1b535 100644 --- a/src/netcore/Wexflow.Server/Contracts/WorkflowInfo.cs +++ b/src/netcore/Wexflow.Server/Contracts/WorkflowInfo.cs @@ -48,7 +48,30 @@ public class WorkflowInfo : IComparable public string StartedOn { get; set; } - public WorkflowInfo(string dbId, int id, Guid instanceId, string name, string filePath, LaunchType launchType, bool isEnabled, bool isApproval, bool enableParallelJobs, bool isWaitingForApproval, string desc, bool isRunning, bool isPaused, string period, string cronExpression, bool isExecutionGraphEmpty, Variable[] localVariables, string startedOn) + public int RetryCount { get; set; } + + public int RetryTimeout { get; set; } + + public WorkflowInfo(string dbId, + int id, + Guid instanceId, + string name, + string filePath, + LaunchType launchType, + bool isEnabled, + bool isApproval, + bool enableParallelJobs, + bool isWaitingForApproval, + string desc, + bool isRunning, + bool isPaused, + string period, + string cronExpression, + bool isExecutionGraphEmpty, + Variable[] localVariables, + string startedOn, + int retryCount, + int retryTimeout) { DbId = dbId; Id = id; @@ -68,6 +91,8 @@ public WorkflowInfo(string dbId, int id, Guid instanceId, string name, string fi IsExecutionGraphEmpty = isExecutionGraphEmpty; LocalVariables = localVariables; StartedOn = startedOn; + RetryCount = retryCount; + RetryTimeout = retryTimeout; } public int CompareTo(object obj) diff --git a/src/netcore/Wexflow.Server/WexflowService.cs b/src/netcore/Wexflow.Server/WexflowService.cs index 9d294ab5..86425bcb 100644 --- a/src/netcore/Wexflow.Server/WexflowService.cs +++ b/src/netcore/Wexflow.Server/WexflowService.cs @@ -215,7 +215,10 @@ private void Search() wf.Period.ToString(@"dd\.hh\:mm\:ss"), wf.CronExpression, wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() - , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]))) + , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout + )) .ToArray(); } else if (user.UserProfile == Core.Db.UserProfile.Administrator) @@ -231,7 +234,10 @@ private void Search() wf.Period.ToString(@"dd\.hh\:mm\:ss"), wf.CronExpression, wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() - , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]))) + , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout + )) .ToArray(); } } @@ -272,7 +278,10 @@ private void SearchApprovalWorkflows() wf.Period.ToString(@"dd\.hh\:mm\:ss"), wf.CronExpression, wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() - , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]))) + , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout + )) .ToArray(); } else if (user.UserProfile == Core.Db.UserProfile.Administrator) @@ -289,7 +298,10 @@ private void SearchApprovalWorkflows() wf.Period.ToString(@"dd\.hh\:mm\:ss"), wf.CronExpression, wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() - , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]))) + , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout + )) .ToArray(); } } @@ -319,6 +331,8 @@ private void GetWorkflow() wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout ); var user = WexflowServer.WexflowEngine.GetUser(username); @@ -375,6 +389,8 @@ private void GetJob() wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout ); var user = WexflowServer.WexflowEngine.GetUser(username); @@ -425,6 +441,8 @@ private void GetJobs() w.IsExecutionGraphEmpty , w.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() , w.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout )); var user = WexflowServer.WexflowEngine.GetUser(username); @@ -910,7 +928,9 @@ private void GetWorkflowJson() IsApproval = wf.IsApproval, EnableParallelJobs = wf.EnableParallelJobs, Description = wf.Description, - LocalVariables = variables.ToArray() + LocalVariables = variables.ToArray(), + RetryCount = wf.RetryCount, + RetryTimeout = wf.RetryTimeout }; List tasks = new(); @@ -1624,6 +1644,9 @@ private static SaveResult SaveJsonWorkflow(Core.Db.User user, string json) var enableParallelJobs = (bool)wi.SelectToken("EnableParallelJobs"); var workflowDesc = (string)wi.SelectToken("Description"); + var retryCount = (int)wi.SelectToken("RetryCount"); + var retryTimeout = (int)wi.SelectToken("RetryTimeout"); + // Local variables XElement xLocalVariables = new(Xn + "LocalVariables"); var variables = wi.SelectToken("LocalVariables"); @@ -1722,12 +1745,12 @@ private static SaveResult SaveJsonWorkflow(Core.Db.User user, string json) , new XElement(Xn + "Setting" , new XAttribute("name", "enableParallelJobs") , new XAttribute("value", enableParallelJobs.ToString().ToLower())) - //, new XElement(xn + "Setting" - // , new XAttribute("name", "period") - // , new XAttribute("value", workflowPeriod.ToString(@"dd\.hh\:mm\:ss"))) - //, new XElement(xn + "Setting" - // , new XAttribute("name", "cronExpression") - // , new XAttribute("value", cronExpression)) + , new XElement(Xn + "Setting" + , new XAttribute("name", "retryCount") + , new XAttribute("value", retryCount)) + , new XElement(Xn + "Setting" + , new XAttribute("name", "retryTimeout") + , new XAttribute("value", retryTimeout)) ) , xLocalVariables , xtasks @@ -1807,6 +1830,9 @@ private static SaveResult SaveJsonWorkflow(Core.Db.User user, string json) var enableParallelJobs = (bool)(wi.SelectToken("EnableParallelJobs") ?? true); var workflowDesc = (string)wi.SelectToken("Description"); + var retryCount = (int)wi.SelectToken("RetryCount"); + var retryTimeout = (int)wi.SelectToken("RetryTimeout"); + if (xdoc.Root == null) throw new InvalidOperationException("Root is null"); xdoc.Root.Attribute("id")!.Value = workflowId.ToString(); xdoc.Root.Attribute("name")!.Value = workflowName ?? throw new InvalidOperationException(); @@ -1902,6 +1928,32 @@ private static SaveResult SaveJsonWorkflow(Core.Db.User user, string json) // } //} + var xwfRetryCount = xdoc.Root.XPathSelectElement("wf:Settings/wf:Setting[@name='retryCount']", + wf.XmlNamespaceManager); + if (xwfRetryCount != null) + { + (xwfRetryCount.Attribute("value") ?? throw new InvalidOperationException()).Value = retryCount.ToString(); + } + else + { + (xdoc.Root.XPathSelectElement("wf:Settings", wf.XmlNamespaceManager) ?? throw new InvalidOperationException()) + .Add(new XElement(wf.XNamespaceWf + "Setting", new XAttribute("name", "retryCount"), + new XAttribute("value", retryCount))); + } + + var xwfRetryTimeout = xdoc.Root.XPathSelectElement("wf:Settings/wf:Setting[@name='retryTimeout']", + wf.XmlNamespaceManager); + if (xwfRetryTimeout != null) + { + (xwfRetryTimeout.Attribute("value") ?? throw new InvalidOperationException()).Value = retryTimeout.ToString(); + } + else + { + (xdoc.Root.XPathSelectElement("wf:Settings", wf.XmlNamespaceManager) ?? throw new InvalidOperationException()) + .Add(new XElement(wf.XNamespaceWf + "Setting", new XAttribute("name", "retryTimeout"), + new XAttribute("value", retryTimeout))); + } + // Local variables var xLocalVariables = xdoc.Root.Element(wf.XNamespaceWf + "LocalVariables"); if (xLocalVariables != null) @@ -2932,8 +2984,10 @@ private void GetUserWorkflows() wf.Period.ToString(@"dd\.hh\:mm\:ss"), wf.CronExpression, wf.IsExecutionGraphEmpty , wf.LocalVariables.Select(v => new Contracts.Variable { Key = v.Key, Value = v.Value }).ToArray() - , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]))) - .ToArray(); + , wf.StartedOn.ToString(WexflowServer.Config["DateTimeFormat"]) + , wf.RetryCount + , wf.RetryTimeout + )).ToArray(); } catch (Exception e) {