From 7d7313fcc0ed6a7bb0a397c63b4bb7e03854f2df Mon Sep 17 00:00:00 2001 From: Yichao Yang <1048262223@qq.com> Date: Sun, 16 Aug 2020 18:57:34 +0800 Subject: [PATCH 01/53] [Fix][checkstyle] Fix checkstyle static type order checker (#3520) --- style/checkstyle.xml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/style/checkstyle.xml b/style/checkstyle.xml index 2a74b99a275e..d7283abafa16 100644 --- a/style/checkstyle.xml +++ b/style/checkstyle.xml @@ -219,12 +219,15 @@ ^sun\.misc\.BASE64Decoder, ^jdk\.internal\.jline\.internal\.Nullable"/> - + - + - + + + + @@ -269,7 +272,9 @@ - + + + From ad89f433f16d65ddcab3186baa84e14762d03e52 Mon Sep 17 00:00:00 2001 From: dengc367 Date: Mon, 17 Aug 2020 17:28:46 +0800 Subject: [PATCH 02/53] [Feature]modify some cases from rockxsj:Feature-presto to add presto datasource support (#3468) * Feature presto (#1) * * add presto datasource support update .gitigonre to igonre some files * * use another presto driver * * add LICENSE files about presto-jdbc * * just for test sonar Co-authored-by: rockxsj * modify the io.prestosql.jdbc.PrestoDriver to com.facebook.presto.jdbc.PrestoDriver * add presto connection in sql node Co-authored-by: rockxsj --- .../org/apache/dolphinscheduler/common/Constants.java | 2 +- dolphinscheduler-ui/src/js/conf/home/store/dag/state.js | 5 +++++ script/scp-hosts.sh | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 4f22e012410e..072a67f44f29 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -898,7 +898,7 @@ private Constants() { public static final String COM_ORACLE_JDBC_DRIVER = "oracle.jdbc.driver.OracleDriver"; public static final String COM_SQLSERVER_JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; public static final String COM_DB2_JDBC_DRIVER = "com.ibm.db2.jcc.DB2Driver"; - public static final String COM_PRESTO_JDBC_DRIVER = "io.prestosql.jdbc.PrestoDriver"; + public static final String COM_PRESTO_JDBC_DRIVER = "com.facebook.presto.jdbc.PrestoDriver"; /** * database type diff --git a/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js b/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js index 05dfa7716148..e3c75b838fe0 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/dag/state.js @@ -96,6 +96,11 @@ export default { id: 7, code: 'DB2', disabled: false + }, + { + id: 8, + code: 'PRESTO', + disabled: false } ], // Alarm interface diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh index 4a94cffa2950..9da94ab79cbc 100644 --- a/script/scp-hosts.sh +++ b/script/scp-hosts.sh @@ -33,8 +33,8 @@ for workerGroup in ${workersGroup[@]} do echo $workerGroup; worker=`echo $workerGroup|awk -F':' '{print $1}'` - groupName=`echo $workerGroup|awk -F':' '{print $2}'` - workersGroupMap+=([$worker]=$groupName) + groupsName=`echo $workerGroup|awk -F':' '{print $2}'` + workersGroupMap+=([$worker]=$groupsName) done @@ -53,7 +53,7 @@ do do # if worker in workersGroupMap if [[ "${workersGroupMap[${host}]}" ]] && [[ "${dsDir}" == "conf" ]]; then - sed -i ${txt} "s#worker.group.*#worker.group=${workersGroupMap[${host}]}#g" ${dsDir}/worker.properties + sed -i ${txt} "s:.*worker.groups.*:worker.groups=${workersGroupMap[${host}]}:g" ${dsDir}/worker.properties fi echo "start to scp $dsDir to $host/$installPath" @@ -61,4 +61,4 @@ do done echo "scp dirs to $host/$installPath complete" -done \ No newline at end of file +done From 0b7b6d4e2a600bb3af36363c4814da6b169317bf Mon Sep 17 00:00:00 2001 From: XiaotaoYi Date: Tue, 18 Aug 2020 07:09:39 +0800 Subject: [PATCH 03/53] =?UTF-8?q?[bug-3480][server]fix=20ds=20muti-level?= =?UTF-8?q?=20directory=20in=20zk,=20which=20lead=20to=20fa=E2=80=A6=20(#3?= =?UTF-8?q?532)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [bug-3480][server]fix ds muti-level directory in zk, which lead to fail to assign work * miss whitespace for if statement --- .../server/registry/ZookeeperNodeManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManager.java index f039fb55326d..b1a5edee383f 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManager.java @@ -151,10 +151,10 @@ protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String private String parseGroup(String path){ String[] parts = path.split("\\/"); - if(parts.length != 6){ + if (parts.length < 6) { throw new IllegalArgumentException(String.format("worker group path : %s is not valid, ignore", path)); } - String group = parts[4]; + String group = parts[parts.length - 2]; return group; } } From 59610a56610a4c7131818e3d1c70366d2bdd25ab Mon Sep 17 00:00:00 2001 From: Hsu Pu Date: Tue, 18 Aug 2020 14:46:00 +0800 Subject: [PATCH 04/53] [Improvement][dao,server] unit test for ConditionsTask (#3385) --- .../dao/entity/TaskInstance.java | 29 ++- .../dao/entity/TaskInstanceTest.java | 32 ++++ .../server/master/ConditionsTaskTest.java | 171 ++++++++++++------ pom.xml | 3 +- 4 files changed, 160 insertions(+), 75 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index a90d9271547c..b82da62b0229 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -16,24 +16,23 @@ */ package org.apache.dolphinscheduler.dao.entity; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.fasterxml.jackson.annotation.JsonFormat; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.model.TaskNode; -import org.apache.dolphinscheduler.common.utils.*; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; import org.apache.dolphinscheduler.common.utils.JSONUtils; import java.io.Serializable; import java.util.Date; import java.util.Map; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; + /** * task instance */ @@ -382,16 +381,16 @@ public void setAppLink(String appLink) { this.appLink = appLink; } - - - public String getDependency(){ - - if(this.dependency != null){ + public String getDependency() { + if (this.dependency != null) { return this.dependency; } TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); + return taskNode == null ? null : taskNode.getDependence(); + } - return taskNode.getDependence(); + public void setDependency(String dependency) { + this.dependency = dependency; } public Flag getFlag() { @@ -495,10 +494,6 @@ public boolean taskCanRetry() { } } - public void setDependency(String dependency) { - this.dependency = dependency; - } - public Priority getTaskInstancePriority() { return taskInstancePriority; } diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java index 9c596708725a..5742c95a5de1 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/entity/TaskInstanceTest.java @@ -16,6 +16,9 @@ */ package org.apache.dolphinscheduler.dao.entity; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.utils.JSONUtils; + import org.junit.Assert; import org.junit.Test; @@ -43,7 +46,36 @@ public void testTaskInstanceIsSubProcess() { //sub process taskInstance.setTaskType("DEPENDENT"); Assert.assertTrue(taskInstance.isDependTask()); + } + + /** + * test for TaskInstance.getDependence + */ + @Test + public void testTaskInstanceGetDependence() { + TaskInstance taskInstance; + TaskNode taskNode; + + taskInstance = new TaskInstance(); + taskInstance.setTaskJson(null); + Assert.assertNull(taskInstance.getDependency()); + + taskInstance = new TaskInstance(); + taskNode = new TaskNode(); + taskNode.setDependence(null); + taskInstance.setTaskJson(JSONUtils.toJsonString(taskNode)); + Assert.assertNull(taskInstance.getDependency()); + taskInstance = new TaskInstance(); + taskNode = new TaskNode(); + // expect a JSON here, and will be unwrap when toJsonString + taskNode.setDependence("\"A\""); + taskInstance.setTaskJson(JSONUtils.toJsonString(taskNode)); + Assert.assertEquals("A", taskInstance.getDependency()); + taskInstance = new TaskInstance(); + taskInstance.setTaskJson(null); + taskInstance.setDependency("{}"); + Assert.assertEquals("{}", taskInstance.getDependency()); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java index f1ee8ccf11c1..61058de8640f 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java @@ -16,14 +16,26 @@ */ package org.apache.dolphinscheduler.server.master; - +import org.apache.dolphinscheduler.common.enums.DependentRelation; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.TaskType; +import org.apache.dolphinscheduler.common.model.DependentItem; +import org.apache.dolphinscheduler.common.model.DependentTaskModel; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; +import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.runner.ConditionsTaskExecThread; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -34,99 +46,144 @@ import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; -import java.util.ArrayList; -import java.util.List; - @RunWith(MockitoJUnitRunner.Silent.class) public class ConditionsTaskTest { - private static final Logger logger = LoggerFactory.getLogger(DependentTaskTest.class); - private ProcessService processService; - private ApplicationContext applicationContext; + /** + * TaskNode.runFlag : task can be run normally + */ + public static final String FLOWNODE_RUN_FLAG_NORMAL = "NORMAL"; + private ProcessService processService; - private MasterConfig config; + private ProcessInstance processInstance; @Before public void before() { - config = new MasterConfig(); + ApplicationContext applicationContext = Mockito.mock(ApplicationContext.class); + SpringApplicationContext springApplicationContext = new SpringApplicationContext(); + springApplicationContext.setApplicationContext(applicationContext); + + MasterConfig config = new MasterConfig(); + Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config); config.setMasterTaskCommitRetryTimes(3); config.setMasterTaskCommitInterval(1000); + processService = Mockito.mock(ProcessService.class); - applicationContext = Mockito.mock(ApplicationContext.class); - SpringApplicationContext springApplicationContext = new SpringApplicationContext(); - springApplicationContext.setApplicationContext(applicationContext); Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService); - Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config); + processInstance = getProcessInstance(); Mockito.when(processService - .findTaskInstanceById(252612)) - .thenReturn(getTaskInstance()); + .findProcessInstanceById(processInstance.getId())) + .thenReturn(processInstance); + } - Mockito.when(processService.saveTaskInstance(getTaskInstance())) - .thenReturn(true); + private TaskInstance testBasicInit(ExecutionStatus expectResult) { + TaskInstance taskInstance = getTaskInstance(getTaskNode(), processInstance); - Mockito.when(processService.findProcessInstanceById(10112)) - .thenReturn(getProcessInstance()); + // for MasterBaseTaskExecThread.submit + Mockito.when(processService + .submitTask(taskInstance)) + .thenReturn(taskInstance); + // for MasterBaseTaskExecThread.call + Mockito.when(processService + .findTaskInstanceById(taskInstance.getId())) + .thenReturn(taskInstance); + // for ConditionsTaskExecThread.initTaskParameters + Mockito.when(processService + .saveTaskInstance(taskInstance)) + .thenReturn(true); + // for ConditionsTaskExecThread.updateTaskState + Mockito.when(processService + .updateTaskInstance(taskInstance)) + .thenReturn(true); + // for ConditionsTaskExecThread.waitTaskQuit + List conditions = Stream.of( + getTaskInstanceForValidTaskList(1001, "1", expectResult) + ).collect(Collectors.toList()); Mockito.when(processService - .findValidTaskListByProcessId(10112)) - .thenReturn(getTaskInstances()); + .findValidTaskListByProcessId(processInstance.getId())) + .thenReturn(conditions); + + return taskInstance; } @Test - public void testCondition(){ - TaskInstance taskInstance = getTaskInstance(); - String dependString = "{\"dependTaskList\":[{\"dependItemList\":[{\"depTasks\":\"1\",\"status\":\"SUCCESS\"}],\"relation\":\"AND\"}],\"relation\":\"AND\"}"; - String conditionResult = "{\"successNode\":[\"2\"],\"failedNode\":[\"3\"]}"; + public void testBasicSuccess() throws Exception { + TaskInstance taskInstance = testBasicInit(ExecutionStatus.SUCCESS); + ConditionsTaskExecThread taskExecThread = new ConditionsTaskExecThread(taskInstance); + taskExecThread.call(); + Assert.assertEquals(ExecutionStatus.SUCCESS, taskExecThread.getTaskInstance().getState()); + } - taskInstance.setDependency(dependString); - Mockito.when(processService.submitTask(taskInstance)) - .thenReturn(taskInstance); - ConditionsTaskExecThread conditions = - new ConditionsTaskExecThread(taskInstance); + @Test + public void testBasicFailure() throws Exception { + TaskInstance taskInstance = testBasicInit(ExecutionStatus.FAILURE); + ConditionsTaskExecThread taskExecThread = new ConditionsTaskExecThread(taskInstance); + taskExecThread.call(); + Assert.assertEquals(ExecutionStatus.FAILURE, taskExecThread.getTaskInstance().getState()); + } - try { - conditions.call(); - } catch (Exception e) { - e.printStackTrace(); - } + private TaskNode getTaskNode() { + TaskNode taskNode = new TaskNode(); + taskNode.setId("tasks-1000"); + taskNode.setName("C"); + taskNode.setType(TaskType.CONDITIONS.toString()); + taskNode.setRunFlag(FLOWNODE_RUN_FLAG_NORMAL); - Assert.assertEquals(ExecutionStatus.SUCCESS, conditions.getTaskInstance().getState()); - } + DependentItem dependentItem = new DependentItem(); + dependentItem.setDepTasks("1"); + dependentItem.setStatus(ExecutionStatus.SUCCESS); + DependentTaskModel dependentTaskModel = new DependentTaskModel(); + dependentTaskModel.setDependItemList(Stream.of(dependentItem).collect(Collectors.toList())); + dependentTaskModel.setRelation(DependentRelation.AND); - private TaskInstance getTaskInstance(){ - TaskInstance taskInstance = new TaskInstance(); - taskInstance.setId(252612); - taskInstance.setName("C"); - taskInstance.setTaskType("CONDITIONS"); - taskInstance.setProcessInstanceId(10112); - taskInstance.setProcessDefinitionId(100001); - return taskInstance; - } + DependentParameters dependentParameters = new DependentParameters(); + dependentParameters.setDependTaskList(Stream.of(dependentTaskModel).collect(Collectors.toList())); + dependentParameters.setRelation(DependentRelation.AND); + // in: AND(AND(1 is SUCCESS)) + taskNode.setDependence(JSONUtils.toJsonString(dependentParameters)); + ConditionsParameters conditionsParameters = new ConditionsParameters(); + conditionsParameters.setSuccessNode(Stream.of("2").collect(Collectors.toList())); + conditionsParameters.setFailedNode(Stream.of("3").collect(Collectors.toList())); - private List getTaskInstances(){ - List list = new ArrayList<>(); - TaskInstance taskInstance = new TaskInstance(); - taskInstance.setId(199999); - taskInstance.setName("1"); - taskInstance.setState(ExecutionStatus.SUCCESS); - list.add(taskInstance); - return list; + // out: SUCCESS => 2, FAILED => 3 + taskNode.setConditionResult(JSONUtils.toJsonString(conditionsParameters)); + + return taskNode; } - private ProcessInstance getProcessInstance(){ + private ProcessInstance getProcessInstance() { ProcessInstance processInstance = new ProcessInstance(); - processInstance.setId(10112); - processInstance.setProcessDefinitionId(100001); + processInstance.setId(1000); + processInstance.setProcessDefinitionId(1000); processInstance.setState(ExecutionStatus.RUNNING_EXECUTION); return processInstance; } + private TaskInstance getTaskInstance(TaskNode taskNode, ProcessInstance processInstance) { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(1000); + taskInstance.setTaskJson(JSONUtils.toJsonString(taskNode)); + taskInstance.setName(taskNode.getName()); + taskInstance.setTaskType(taskNode.getType()); + taskInstance.setProcessInstanceId(processInstance.getId()); + taskInstance.setProcessDefinitionId(processInstance.getProcessDefinitionId()); + return taskInstance; + } + + private TaskInstance getTaskInstanceForValidTaskList(int id, String name, ExecutionStatus state) { + TaskInstance taskInstance = new TaskInstance(); + taskInstance.setId(id); + taskInstance.setName(name); + taskInstance.setState(state); + return taskInstance; + } } diff --git a/pom.xml b/pom.xml index 5a3590affd48..2937a814fc1e 100644 --- a/pom.xml +++ b/pom.xml @@ -797,6 +797,7 @@ **/dao/mapper/CommandMapperTest.java **/dao/mapper/ConnectionFactoryTest.java **/dao/mapper/DataSourceMapperTest.java + **/dao/entity/TaskInstanceTest.java **/dao/entity/UdfFuncTest.java **/remote/JsonSerializerTest.java **/remote/RemoveTaskLogResponseCommandTest.java @@ -820,7 +821,7 @@ **/server/master/AlertManagerTest.java **/server/master/MasterCommandTest.java **/server/master/DependentTaskTest.java - + **/server/master/ConditionsTaskTest.java **/server/master/MasterExecThreadTest.java **/server/master/ParamsTest.java **/server/register/ZookeeperNodeManagerTest.java From 00e0003aed207adc33d1c4068977b172fa21d3f0 Mon Sep 17 00:00:00 2001 From: xingchun-chen <55787491+xingchun-chen@users.noreply.github.com> Date: Tue, 18 Aug 2020 17:27:36 +0800 Subject: [PATCH 05/53] [test-2995][e2e]Add task connection (#3524) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add task connection  * Optimize test cases   * Modify variable format     * Optimize test cases   * Update BrowserCommon.java * Update BrowserCommon.java * Update WorkflowDefineLocator.java Co-authored-by: chenxingchun <438044805@qq.com> --- .../common/BrowserCommon.java | 40 +++++++++++++++---- .../project/WorkflowDefineLocator.java | 12 +++++- .../page/project/WorkflowDefinePage.java | 5 +++ 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java b/e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java index 437e81cfbb0d..2eade95488fd 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java @@ -223,22 +223,46 @@ public Actions moveToElement(By locator) { /** * mouse drag element * - * @param source_locator BY - * @param target_locator BY + * @param sourceLocator BY + * @param targetLocator BY */ - public void dragAndDrop(By source_locator, By target_locator) { - WebElement sourceElement = locateElement(source_locator); - WebElement targetElement = locateElement(target_locator); + public void dragAndDrop(By sourceLocator, By targetLocator) { + WebElement sourceElement = locateElement(sourceLocator); + WebElement targetElement = locateElement(targetLocator); actions.dragAndDrop(sourceElement, targetElement).perform(); actions.release(); } - public void moveToDragElement(By target_locator, int X, int Y) { - WebElement targetElement = locateElement(target_locator); - actions.dragAndDropBy(targetElement, X, Y).perform(); + public void moveToDragElement(By targetLocator, int x, int y) { + WebElement targetElement = locateElement(targetLocator); + actions.dragAndDropBy(targetElement, x, y).perform(); actions.release(); } + /** + * Right mouse click on the element + * + * @param locator By + * @return actions + */ + public void mouseRightClickElement(By locator) { + WebElement mouseRightClickElement = locateElement(locator); + actions.contextClick(mouseRightClickElement).perform(); + } + + /** + * The mouse moves from a position to a specified positionØ + * + * @param sourceLocator BY + * @param targetLocator BY + * @return actions + */ + public void mouseMovePosition(By sourceLocator, By targetLocator) throws InterruptedException { + WebElement sourceElement = locateElement(sourceLocator); + WebElement targetElement = locateElement(targetLocator); + actions.dragAndDrop(sourceElement,targetElement).perform(); + actions.click(); + } /** * jump page diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java index bfb237ccd27b..a70b22eacf7a 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/locator/project/WorkflowDefineLocator.java @@ -120,12 +120,22 @@ public class WorkflowDefineLocator { //click submit button public static final By CLICK_SUBMIT_BUTTON = By.xpath("//div[3]/div/button[2]/span"); + //copy task + public static final By MOUSE_RIGHT_CLICK = By.xpath("//div[2]/div[2]/div/div/div/div/div[2]"); + public static final By COPY_TASK = By.xpath("//a[3]/span"); + + //click line + public static final By CLICK_LINE = By.xpath("//a[@id='line']/button/i"); + + public static final By LINE_SOURCES_TASK = By.xpath("//div[@id='canvas']/div[1]/div[2]"); + + public static final By LINE_TARGET_TASK = By.xpath("//div[@id='canvas']/div[2]/div[2]"); /** * save workflow */ //click save workflow button - public static final By CLICK_SAVE_WORKFLOW_BUTTON = By.xpath("//div[2]/div[1]/div[2]/button[2]"); + public static final By CLICK_SAVE_WORKFLOW_BUTTON = By.xpath("//div[2]/div[1]/div[2]/button[2]/span"); //input workflow name public static final By INPUT_WORKFLOW_NAME = By.xpath("//input"); diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java index 9e462805da45..83442562ca45 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/page/project/WorkflowDefinePage.java @@ -131,6 +131,11 @@ public boolean createWorkflow() throws InterruptedException { System.out.println("move to Dag Element "); moveToDragElement(WorkflowDefineLocator.MOUSE_MOVE_SHELL_AT_DAG,-300,-100); + System.out.println("copy task"); + mouseRightClickElement(WorkflowDefineLocator.MOUSE_RIGHT_CLICK); + clickButton(WorkflowDefineLocator.COPY_TASK); + clickButton(WorkflowDefineLocator.CLICK_LINE); + mouseMovePosition(WorkflowDefineLocator.LINE_SOURCES_TASK,WorkflowDefineLocator.LINE_TARGET_TASK); return ifTitleContains(WorkflowDefineData.CREATE_WORKFLOW_TITLE); } From a4ee351a3af1b05bafab48699f846e2b3ac226eb Mon Sep 17 00:00:00 2001 From: vanilla111 <1115690319@qq.com> Date: Wed, 19 Aug 2020 11:36:30 +0800 Subject: [PATCH 06/53] delay execution of tasks and improve some designs (#3427) --- .../api/dto/TaskCountDto.java | 24 +- .../dolphinscheduler/common/Constants.java | 17 +- .../common/enums/ExecutionStatus.java | 109 +++-- .../common/enums/TaskStateType.java | 10 +- .../common/model/TaskNode.java | 50 +- .../common/utils/DateUtils.java | 444 +++++++++++++++++- .../common/enums/ExecutionStatusTest.java | 32 ++ .../dao/entity/TaskInstance.java | 127 +++-- .../remote/codec/NettyDecoder.java | 1 + .../builder/TaskExecutionContextBuilder.java | 2 + .../server/entity/TaskExecutionContext.java | 171 ++++--- .../runner/ConditionsTaskExecThread.java | 1 + .../runner/DependentTaskExecThread.java | 1 + .../runner/MasterBaseTaskExecThread.java | 21 +- .../master/runner/MasterExecThread.java | 54 ++- .../master/runner/MasterTaskExecThread.java | 17 +- .../processor/TaskExecuteProcessor.java | 39 +- .../worker/runner/TaskExecuteThread.java | 98 +++- .../worker/task/AbstractCommandExecutor.java | 22 +- .../TaskPriorityQueueConsumerTest.java | 3 +- .../dispatch/ExecutorDispatcherTest.java | 3 +- .../executor/NettyExecutorManagerTest.java | 3 +- .../host/RoundRobinHostManagerTest.java | 3 +- .../queue/TaskResponseServiceTest.java | 4 +- .../master/registry/MasterRegistryTest.java | 4 +- .../worker/runner/TaskExecuteThreadTest.java | 171 +++++++ .../service/process/ProcessService.java | 98 +++- pom.xml | 2 + sql/dolphinscheduler-postgre.sql | 4 +- sql/dolphinscheduler_mysql.sql | 2 + .../mysql/dolphinscheduler_ddl.sql | 1 - .../mysql/dolphinscheduler_ddl.sql | 58 +++ .../mysql/dolphinscheduler_dml.sql | 16 + .../postgresql/dolphinscheduler_ddl.sql | 52 ++ .../postgresql/dolphinscheduler_dml.sql | 16 + 35 files changed, 1383 insertions(+), 297 deletions(-) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/enums/ExecutionStatusTest.java create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java create mode 100644 sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql create mode 100644 sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_dml.sql create mode 100644 sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql create mode 100644 sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_dml.sql diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java index fa7588f2edd4..35aaaf34dd41 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/TaskCountDto.java @@ -42,9 +42,10 @@ public TaskCountDto(List taskInstanceStateCounts) { countTaskDtos(taskInstanceStateCounts); } - private void countTaskDtos(List taskInstanceStateCounts){ + private void countTaskDtos(List taskInstanceStateCounts) { int submittedSuccess = 0; - int runningExeution = 0; + int runningExecution = 0; + int delayExecution = 0; int readyPause = 0; int pause = 0; int readyStop = 0; @@ -55,15 +56,18 @@ private void countTaskDtos(List taskInstanceStateCounts){ int kill = 0; int waittingThread = 0; - for(ExecuteStatusCount taskInstanceStateCount : taskInstanceStateCounts){ + for (ExecuteStatusCount taskInstanceStateCount : taskInstanceStateCounts) { ExecutionStatus status = taskInstanceStateCount.getExecutionStatus(); totalCount += taskInstanceStateCount.getCount(); - switch (status){ + switch (status) { case SUBMITTED_SUCCESS: submittedSuccess += taskInstanceStateCount.getCount(); break; case RUNNING_EXECUTION: - runningExeution += taskInstanceStateCount.getCount(); + runningExecution += taskInstanceStateCount.getCount(); + break; + case DELAY_EXECUTION: + delayExecution += taskInstanceStateCount.getCount(); break; case READY_PAUSE: readyPause += taskInstanceStateCount.getCount(); @@ -93,13 +97,14 @@ private void countTaskDtos(List taskInstanceStateCounts){ waittingThread += taskInstanceStateCount.getCount(); break; - default: - break; + default: + break; } } this.taskCountDtos = new ArrayList<>(); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.SUBMITTED_SUCCESS, submittedSuccess)); - this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXECUTION, runningExeution)); + this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.RUNNING_EXECUTION, runningExecution)); + this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.DELAY_EXECUTION, delayExecution)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_PAUSE, readyPause)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.PAUSE, pause)); this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.READY_STOP, readyStop)); @@ -111,8 +116,7 @@ private void countTaskDtos(List taskInstanceStateCounts){ this.taskCountDtos.add(new TaskStateCount(ExecutionStatus.WAITTING_THREAD, waittingThread)); } - - public List getTaskCountDtos(){ + public List getTaskCountDtos() { return taskCountDtos; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 072a67f44f29..cee83e73bcbc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -138,7 +138,7 @@ private Constants() { /** * python home */ - public static final String PYTHON_HOME="PYTHON_HOME"; + public static final String PYTHON_HOME = "PYTHON_HOME"; /** * resource.view.suffixs @@ -366,7 +366,6 @@ private Constants() { public static final double DEFAULT_WORKER_RESERVED_MEMORY = OSUtils.totalMemorySize() / 10; - /** * default log cache rows num,output when reach the number */ @@ -752,7 +751,7 @@ private Constants() { /** - * preview schedule execute count + * preview schedule execute count */ public static final int PREVIEW_SCHEDULE_EXECUTE_COUNT = 5; @@ -832,6 +831,7 @@ private Constants() { public static final int[] NOT_TERMINATED_STATES = new int[]{ ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), ExecutionStatus.RUNNING_EXECUTION.ordinal(), + ExecutionStatus.DELAY_EXECUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), ExecutionStatus.READY_STOP.ordinal(), ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(), @@ -852,18 +852,17 @@ private Constants() { /** * data total */ - public static final String COUNT = "count"; + public static final String COUNT = "count"; /** * page size */ - public static final String PAGE_SIZE = "pageSize"; + public static final String PAGE_SIZE = "pageSize"; /** * current page no */ - public static final String PAGE_NUMBER = "pageNo"; - + public static final String PAGE_NUMBER = "pageNo"; /** @@ -966,11 +965,11 @@ private Constants() { /** * authorize writable perm */ - public static final int AUTHORIZE_WRITABLE_PERM=7; + public static final int AUTHORIZE_WRITABLE_PERM = 7; /** * authorize readable perm */ - public static final int AUTHORIZE_READABLE_PERM=4; + public static final int AUTHORIZE_READABLE_PERM = 4; /** diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java index 6ea02ef0967e..f6ac2cf5ab43 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ExecutionStatus.java @@ -14,16 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.common.enums; +import java.util.HashMap; import com.baomidou.mybatisplus.annotation.EnumValue; -import java.util.HashMap; - /** * running status for workflow and task nodes - * */ public enum ExecutionStatus { @@ -41,6 +40,7 @@ public enum ExecutionStatus { * 9 kill * 10 waiting thread * 11 waiting depend node complete + * 12 delay execution */ SUBMITTED_SUCCESS(0, "submit success"), RUNNING_EXECUTION(1, "running"), @@ -53,9 +53,10 @@ public enum ExecutionStatus { NEED_FAULT_TOLERANCE(8, "need fault tolerance"), KILL(9, "kill"), WAITTING_THREAD(10, "waiting thread"), - WAITTING_DEPEND(11, "waiting depend node complete"); + WAITTING_DEPEND(11, "waiting depend node complete"), + DELAY_EXECUTION(12, "delay execution"); - ExecutionStatus(int code, String descp){ + ExecutionStatus(int code, String descp) { this.code = code; this.descp = descp; } @@ -64,77 +65,85 @@ public enum ExecutionStatus { private final int code; private final String descp; - private static HashMap EXECUTION_STATUS_MAP=new HashMap<>(); + private static HashMap EXECUTION_STATUS_MAP = new HashMap<>(); static { - for (ExecutionStatus executionStatus:ExecutionStatus.values()){ - EXECUTION_STATUS_MAP.put(executionStatus.code,executionStatus); - } + for (ExecutionStatus executionStatus : ExecutionStatus.values()) { + EXECUTION_STATUS_MAP.put(executionStatus.code, executionStatus); + } } - /** - * status is success - * @return status - */ - public boolean typeIsSuccess(){ - return this == SUCCESS; - } - - /** - * status is failure - * @return status - */ - public boolean typeIsFailure(){ - return this == FAILURE || this == NEED_FAULT_TOLERANCE || this == KILL; - } - - /** - * status is finished - * @return status - */ - public boolean typeIsFinished(){ - - return typeIsSuccess() || typeIsFailure() || typeIsCancel() || typeIsPause() - || typeIsStop(); - } + /** + * status is success + * + * @return status + */ + public boolean typeIsSuccess() { + return this == SUCCESS; + } + + /** + * status is failure + * + * @return status + */ + public boolean typeIsFailure() { + return this == FAILURE || this == NEED_FAULT_TOLERANCE || this == KILL; + } + + /** + * status is finished + * + * @return status + */ + public boolean typeIsFinished() { + return typeIsSuccess() || typeIsFailure() || typeIsCancel() || typeIsPause() + || typeIsStop(); + } /** * status is waiting thread + * * @return status */ - public boolean typeIsWaitingThread(){ - return this == WAITTING_THREAD; - } + public boolean typeIsWaitingThread() { + return this == WAITTING_THREAD; + } /** * status is pause + * * @return status */ - public boolean typeIsPause(){ - return this == PAUSE; - } + public boolean typeIsPause() { + return this == PAUSE; + } + /** * status is pause + * * @return status */ - public boolean typeIsStop(){ + public boolean typeIsStop() { return this == STOP; } /** * status is running + * * @return status */ - public boolean typeIsRunning(){ - return this == RUNNING_EXECUTION || this == WAITTING_DEPEND; - } + public boolean typeIsRunning() { + return this == RUNNING_EXECUTION || this == WAITTING_DEPEND || this == DELAY_EXECUTION; + } /** * status is cancel + * * @return status */ - public boolean typeIsCancel(){ - return this == KILL || this == STOP ; + public boolean typeIsCancel() { + return this == KILL || this == STOP; } public int getCode() { @@ -145,10 +154,10 @@ public String getDescp() { return descp; } - public static ExecutionStatus of(int status){ - if(EXECUTION_STATUS_MAP.containsKey(status)){ - return EXECUTION_STATUS_MAP.get(status); - } + public static ExecutionStatus of(int status) { + if (EXECUTION_STATUS_MAP.containsKey(status)) { + return EXECUTION_STATUS_MAP.get(status); + } throw new IllegalArgumentException("invalid status : " + status); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java index 11ab8560b7ec..36766a7f4d04 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskStateType.java @@ -31,12 +31,13 @@ public enum TaskStateType { /** * convert task state to execute status integer array ; + * * @param taskStateType task state type * @return result of execution status */ - public static int[] convert2ExecutStatusIntArray(TaskStateType taskStateType){ + public static int[] convert2ExecutStatusIntArray(TaskStateType taskStateType) { - switch (taskStateType){ + switch (taskStateType) { case SUCCESS: return new int[]{ExecutionStatus.SUCCESS.ordinal()}; case FAILED: @@ -51,14 +52,15 @@ public static int[] convert2ExecutStatusIntArray(TaskStateType taskStateType){ case RUNNING: return new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), ExecutionStatus.RUNNING_EXECUTION.ordinal(), + ExecutionStatus.DELAY_EXECUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), ExecutionStatus.READY_STOP.ordinal()}; case WAITTING: return new int[]{ ExecutionStatus.SUBMITTED_SUCCESS.ordinal() }; - default: - break; + default: + break; } return new int[0]; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index f79439645735..cd3e573b16dc 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -136,6 +136,11 @@ public class TaskNode { @JsonSerialize(using = JSONUtils.JsonDataSerializer.class) private String timeout; + /** + * delay execution time. + */ + private int delayTime; + public String getId() { return id; } @@ -310,24 +315,25 @@ public boolean isConditionsTask(){ @Override public String toString() { - return "TaskNode{" + - "id='" + id + '\'' + - ", name='" + name + '\'' + - ", desc='" + desc + '\'' + - ", type='" + type + '\'' + - ", runFlag='" + runFlag + '\'' + - ", loc='" + loc + '\'' + - ", maxRetryTimes=" + maxRetryTimes + - ", retryInterval=" + retryInterval + - ", params='" + params + '\'' + - ", preTasks='" + preTasks + '\'' + - ", extras='" + extras + '\'' + - ", depList=" + depList + - ", dependence='" + dependence + '\'' + - ", taskInstancePriority=" + taskInstancePriority + - ", timeout='" + timeout + '\'' + - ", workerGroup='" + workerGroup + '\'' + - '}'; + return "TaskNode{" + + "id='" + id + '\'' + + ", name='" + name + '\'' + + ", desc='" + desc + '\'' + + ", type='" + type + '\'' + + ", runFlag='" + runFlag + '\'' + + ", loc='" + loc + '\'' + + ", maxRetryTimes=" + maxRetryTimes + + ", retryInterval=" + retryInterval + + ", params='" + params + '\'' + + ", preTasks='" + preTasks + '\'' + + ", extras='" + extras + '\'' + + ", depList=" + depList + + ", dependence='" + dependence + '\'' + + ", taskInstancePriority=" + taskInstancePriority + + ", timeout='" + timeout + '\'' + + ", workerGroup='" + workerGroup + '\'' + + ", delayTime=" + delayTime + + '}'; } public String getWorkerGroup() { @@ -353,4 +359,12 @@ public Integer getWorkerGroupId() { public void setWorkerGroupId(Integer workerGroupId) { this.workerGroupId = workerGroupId; } + + public int getDelayTime() { + return delayTime; + } + + public void setDelayTime(int delayTime) { + this.delayTime = delayTime; + } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java index 1033816e8e87..6cd1d5867e76 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/DateUtils.java @@ -1 +1,443 @@ -/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.Calendar; import java.util.Date; /** * date utils */ public class DateUtils { private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); /** * date to local datetime * * @param date date * @return local datetime */ private static LocalDateTime date2LocalDateTime(Date date) { return LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); } /** * local datetime to date * * @param localDateTime local datetime * @return date */ private static Date localDateTime2Date(LocalDateTime localDateTime) { Instant instant = localDateTime.atZone(ZoneId.systemDefault()).toInstant(); return Date.from(instant); } /** * get current date str * * @return date string */ public static String getCurrentTime() { return getCurrentTime(Constants.YYYY_MM_DD_HH_MM_SS); } /** * get the date string in the specified format of the current time * * @param format date format * @return date string */ public static String getCurrentTime(String format) { return LocalDateTime.now().format(DateTimeFormatter.ofPattern(format)); } /** * get the formatted date string * * @param date date * @param format e.g. yyyy-MM-dd HH:mm:ss * @return date string */ public static String format(Date date, String format) { return format(date2LocalDateTime(date), format); } /** * get the formatted date string * * @param localDateTime local data time * @param format yyyy-MM-dd HH:mm:ss * @return date string */ public static String format(LocalDateTime localDateTime, String format) { return localDateTime.format(DateTimeFormatter.ofPattern(format)); } /** * convert time to yyyy-MM-dd HH:mm:ss format * * @param date date * @return date string */ public static String dateToString(Date date) { return format(date, Constants.YYYY_MM_DD_HH_MM_SS); } /** * convert string to date and time * * @param date date * @param format format * @return date */ public static Date parse(String date, String format) { try { LocalDateTime ldt = LocalDateTime.parse(date, DateTimeFormatter.ofPattern(format)); return localDateTime2Date(ldt); } catch (Exception e) { logger.error("error while parse date:" + date, e); } return null; } /** * convert date str to yyyy-MM-dd HH:mm:ss format * * @param str date string * @return yyyy-MM-dd HH:mm:ss format */ public static Date stringToDate(String str) { return parse(str, Constants.YYYY_MM_DD_HH_MM_SS); } /** * get seconds between two dates * * @param d1 date1 * @param d2 date2 * @return differ seconds */ public static long differSec(Date d1, Date d2) { if(d1 == null || d2 == null){ return 0; } return (long) Math.ceil(differMs(d1, d2) / 1000.0); } /** * get ms between two dates * * @param d1 date1 * @param d2 date2 * @return differ ms */ public static long differMs(Date d1, Date d2) { return Math.abs(d1.getTime() - d2.getTime()); } /** * get hours between two dates * * @param d1 date1 * @param d2 date2 * @return differ hours */ public static long diffHours(Date d1, Date d2) { return (long) Math.ceil(diffMin(d1, d2) / 60.0); } /** * get minutes between two dates * * @param d1 date1 * @param d2 date2 * @return differ minutes */ public static long diffMin(Date d1, Date d2) { return (long) Math.ceil(differSec(d1, d2) / 60.0); } /** * get the date of the specified date in the days before and after * * @param date date * @param day day * @return the date of the specified date in the days before and after */ public static Date getSomeDay(Date date, int day) { Calendar calendar = Calendar.getInstance(); calendar.setTime(date); calendar.add(Calendar.DATE, day); return calendar.getTime(); } /** * get the hour of day. * * @param date date * @return hour of day */ public static int getHourIndex(Date date) { Calendar calendar = Calendar.getInstance(); calendar.setTime(date); return calendar.get(Calendar.HOUR_OF_DAY); } /** * compare two dates * * @param future future date * @param old old date * @return true if future time greater than old time */ public static boolean compare(Date future, Date old) { return future.getTime() > old.getTime(); } /** * convert schedule string to date * * @param schedule schedule * @return convert schedule string to date */ public static Date getScheduleDate(String schedule) { return stringToDate(schedule); } /** * format time to readable * * @param ms ms * @return format time */ public static String format2Readable(long ms) { long days = ms / (1000 * 60 * 60 * 24); long hours = (ms % (1000 * 60 * 60 * 24)) / (1000 * 60 * 60); long minutes = (ms % (1000 * 60 * 60)) / (1000 * 60); long seconds = (ms % (1000 * 60)) / 1000; return String.format("%02d %02d:%02d:%02d", days, hours, minutes, seconds); } /** * get monday * * note: Set the first day of the week to Monday, the default is Sunday * @param date date * @return get monday */ public static Date getMonday(Date date) { Calendar cal = Calendar.getInstance(); cal.setTime(date); cal.setFirstDayOfWeek(Calendar.MONDAY); cal.set(Calendar.DAY_OF_WEEK, Calendar.MONDAY); return cal.getTime(); } /** * get sunday * * note: Set the first day of the week to Monday, the default is Sunday * @param date date * @return get sunday */ public static Date getSunday(Date date) { Calendar cal = Calendar.getInstance(); cal.setTime(date); cal.setFirstDayOfWeek(Calendar.MONDAY); cal.set(Calendar.DAY_OF_WEEK, Calendar.SUNDAY); return cal.getTime(); } /** * get first day of month * * @param date date * @return first day of month * */ public static Date getFirstDayOfMonth(Date date) { Calendar cal = Calendar.getInstance(); cal.setTime(date); cal.set(Calendar.DAY_OF_MONTH, 1); return cal.getTime(); } /** * get some hour of day * * @param date date * @param offsetHour hours * @return some hour of day * */ public static Date getSomeHourOfDay(Date date, int offsetHour) { Calendar cal = Calendar.getInstance(); cal.setTime(date); cal.set(Calendar.HOUR_OF_DAY, cal.get(Calendar.HOUR_OF_DAY) + offsetHour); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); return cal.getTime(); } /** * get last day of month * * @param date date * @return get last day of month */ public static Date getLastDayOfMonth(Date date) { Calendar cal = Calendar.getInstance(); cal.setTime(date); cal.add(Calendar.MONTH, 1); cal.set(Calendar.DAY_OF_MONTH, 1); cal.add(Calendar.DAY_OF_MONTH, -1); return cal.getTime(); } /** * return YYYY-MM-DD 00:00:00 * * @param inputDay date * @return start day */ public static Date getStartOfDay(Date inputDay) { Calendar cal = Calendar.getInstance(); cal.setTime(inputDay); cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); return cal.getTime(); } /** * return YYYY-MM-DD 23:59:59 * * @param inputDay day * @return end of day */ public static Date getEndOfDay(Date inputDay) { Calendar cal = Calendar.getInstance(); cal.setTime(inputDay); cal.set(Calendar.HOUR_OF_DAY, 23); cal.set(Calendar.MINUTE, 59); cal.set(Calendar.SECOND, 59); cal.set(Calendar.MILLISECOND, 999); return cal.getTime(); } /** * return YYYY-MM-DD 00:00:00 * * @param inputDay day * @return start of hour */ public static Date getStartOfHour(Date inputDay) { Calendar cal = Calendar.getInstance(); cal.setTime(inputDay); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); return cal.getTime(); } /** * return YYYY-MM-DD 23:59:59 * * @param inputDay day * @return end of hour */ public static Date getEndOfHour(Date inputDay) { Calendar cal = Calendar.getInstance(); cal.setTime(inputDay); cal.set(Calendar.MINUTE, 59); cal.set(Calendar.SECOND, 59); cal.set(Calendar.MILLISECOND, 999); return cal.getTime(); } /** * get current date * @return current date */ public static Date getCurrentDate() { return DateUtils.parse(DateUtils.getCurrentTime(), Constants.YYYY_MM_DD_HH_MM_SS); } /** * get date * @param date date * @param calendarField calendarField * @param amount amount * @return date */ public static Date add(final Date date, final int calendarField, final int amount) { if (date == null) { throw new IllegalArgumentException("The date must not be null"); } final Calendar c = Calendar.getInstance(); c.setTime(date); c.add(calendarField, amount); return c.getTime(); } } \ No newline at end of file +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.utils; + +import org.apache.dolphinscheduler.common.Constants; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Calendar; +import java.util.Date; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * date utils + */ +public class DateUtils { + + private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); + + /** + * date to local datetime + * + * @param date date + * @return local datetime + */ + private static LocalDateTime date2LocalDateTime(Date date) { + return LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); + } + + /** + * local datetime to date + * + * @param localDateTime local datetime + * @return date + */ + private static Date localDateTime2Date(LocalDateTime localDateTime) { + Instant instant = localDateTime.atZone(ZoneId.systemDefault()).toInstant(); + return Date.from(instant); + } + + /** + * get current date str + * + * @return date string + */ + public static String getCurrentTime() { + return getCurrentTime(Constants.YYYY_MM_DD_HH_MM_SS); + } + + /** + * get the date string in the specified format of the current time + * + * @param format date format + * @return date string + */ + public static String getCurrentTime(String format) { + return LocalDateTime.now().format(DateTimeFormatter.ofPattern(format)); + } + + /** + * get the formatted date string + * + * @param date date + * @param format e.g. yyyy-MM-dd HH:mm:ss + * @return date string + */ + public static String format(Date date, String format) { + return format(date2LocalDateTime(date), format); + } + + /** + * get the formatted date string + * + * @param localDateTime local data time + * @param format yyyy-MM-dd HH:mm:ss + * @return date string + */ + public static String format(LocalDateTime localDateTime, String format) { + return localDateTime.format(DateTimeFormatter.ofPattern(format)); + } + + /** + * convert time to yyyy-MM-dd HH:mm:ss format + * + * @param date date + * @return date string + */ + public static String dateToString(Date date) { + return format(date, Constants.YYYY_MM_DD_HH_MM_SS); + } + + /** + * convert string to date and time + * + * @param date date + * @param format format + * @return date + */ + public static Date parse(String date, String format) { + try { + LocalDateTime ldt = LocalDateTime.parse(date, DateTimeFormatter.ofPattern(format)); + return localDateTime2Date(ldt); + } catch (Exception e) { + logger.error("error while parse date:" + date, e); + } + return null; + } + + /** + * convert date str to yyyy-MM-dd HH:mm:ss format + * + * @param str date string + * @return yyyy-MM-dd HH:mm:ss format + */ + public static Date stringToDate(String str) { + return parse(str, Constants.YYYY_MM_DD_HH_MM_SS); + } + + /** + * get seconds between two dates + * + * @param d1 date1 + * @param d2 date2 + * @return differ seconds + */ + public static long differSec(Date d1, Date d2) { + if (d1 == null || d2 == null) { + return 0; + } + return (long) Math.ceil(differMs(d1, d2) / 1000.0); + } + + /** + * get ms between two dates + * + * @param d1 date1 + * @param d2 date2 + * @return differ ms + */ + public static long differMs(Date d1, Date d2) { + return Math.abs(d1.getTime() - d2.getTime()); + } + + /** + * get hours between two dates + * + * @param d1 date1 + * @param d2 date2 + * @return differ hours + */ + public static long diffHours(Date d1, Date d2) { + return (long) Math.ceil(diffMin(d1, d2) / 60.0); + } + + /** + * get minutes between two dates + * + * @param d1 date1 + * @param d2 date2 + * @return differ minutes + */ + public static long diffMin(Date d1, Date d2) { + return (long) Math.ceil(differSec(d1, d2) / 60.0); + } + + /** + * get the date of the specified date in the days before and after + * + * @param date date + * @param day day + * @return the date of the specified date in the days before and after + */ + public static Date getSomeDay(Date date, int day) { + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + calendar.add(Calendar.DATE, day); + return calendar.getTime(); + } + + /** + * get the hour of day. + * + * @param date date + * @return hour of day + */ + public static int getHourIndex(Date date) { + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + return calendar.get(Calendar.HOUR_OF_DAY); + } + + /** + * compare two dates + * + * @param future future date + * @param old old date + * @return true if future time greater than old time + */ + public static boolean compare(Date future, Date old) { + return future.getTime() > old.getTime(); + } + + /** + * convert schedule string to date + * + * @param schedule schedule + * @return convert schedule string to date + */ + public static Date getScheduleDate(String schedule) { + return stringToDate(schedule); + } + + /** + * format time to readable + * + * @param ms ms + * @return format time + */ + public static String format2Readable(long ms) { + + long days = ms / (1000 * 60 * 60 * 24); + long hours = (ms % (1000 * 60 * 60 * 24)) / (1000 * 60 * 60); + long minutes = (ms % (1000 * 60 * 60)) / (1000 * 60); + long seconds = (ms % (1000 * 60)) / 1000; + + return String.format("%02d %02d:%02d:%02d", days, hours, minutes, seconds); + + } + + /** + * get monday + *

+ * note: Set the first day of the week to Monday, the default is Sunday + * + * @param date date + * @return get monday + */ + public static Date getMonday(Date date) { + Calendar cal = Calendar.getInstance(); + + cal.setTime(date); + + cal.setFirstDayOfWeek(Calendar.MONDAY); + cal.set(Calendar.DAY_OF_WEEK, Calendar.MONDAY); + + return cal.getTime(); + } + + /** + * get sunday + *

+ * note: Set the first day of the week to Monday, the default is Sunday + * + * @param date date + * @return get sunday + */ + public static Date getSunday(Date date) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + + cal.setFirstDayOfWeek(Calendar.MONDAY); + cal.set(Calendar.DAY_OF_WEEK, Calendar.SUNDAY); + + return cal.getTime(); + } + + /** + * get first day of month + * + * @param date date + * @return first day of month + */ + public static Date getFirstDayOfMonth(Date date) { + Calendar cal = Calendar.getInstance(); + + cal.setTime(date); + cal.set(Calendar.DAY_OF_MONTH, 1); + + return cal.getTime(); + } + + /** + * get some hour of day + * + * @param date date + * @param offsetHour hours + * @return some hour of day + */ + public static Date getSomeHourOfDay(Date date, int offsetHour) { + Calendar cal = Calendar.getInstance(); + + cal.setTime(date); + cal.set(Calendar.HOUR_OF_DAY, cal.get(Calendar.HOUR_OF_DAY) + offsetHour); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + return cal.getTime(); + } + + /** + * get last day of month + * + * @param date date + * @return get last day of month + */ + public static Date getLastDayOfMonth(Date date) { + Calendar cal = Calendar.getInstance(); + + cal.setTime(date); + + cal.add(Calendar.MONTH, 1); + cal.set(Calendar.DAY_OF_MONTH, 1); + cal.add(Calendar.DAY_OF_MONTH, -1); + + return cal.getTime(); + } + + /** + * return YYYY-MM-DD 00:00:00 + * + * @param inputDay date + * @return start day + */ + public static Date getStartOfDay(Date inputDay) { + Calendar cal = Calendar.getInstance(); + cal.setTime(inputDay); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTime(); + } + + /** + * return YYYY-MM-DD 23:59:59 + * + * @param inputDay day + * @return end of day + */ + public static Date getEndOfDay(Date inputDay) { + Calendar cal = Calendar.getInstance(); + cal.setTime(inputDay); + cal.set(Calendar.HOUR_OF_DAY, 23); + cal.set(Calendar.MINUTE, 59); + cal.set(Calendar.SECOND, 59); + cal.set(Calendar.MILLISECOND, 999); + return cal.getTime(); + } + + /** + * return YYYY-MM-DD 00:00:00 + * + * @param inputDay day + * @return start of hour + */ + public static Date getStartOfHour(Date inputDay) { + Calendar cal = Calendar.getInstance(); + cal.setTime(inputDay); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + return cal.getTime(); + } + + /** + * return YYYY-MM-DD 23:59:59 + * + * @param inputDay day + * @return end of hour + */ + public static Date getEndOfHour(Date inputDay) { + Calendar cal = Calendar.getInstance(); + cal.setTime(inputDay); + cal.set(Calendar.MINUTE, 59); + cal.set(Calendar.SECOND, 59); + cal.set(Calendar.MILLISECOND, 999); + return cal.getTime(); + } + + /** + * get current date + * + * @return current date + */ + public static Date getCurrentDate() { + return DateUtils.parse(DateUtils.getCurrentTime(), + Constants.YYYY_MM_DD_HH_MM_SS); + } + + /** + * get date + * + * @param date date + * @param calendarField calendarField + * @param amount amount + * @return date + */ + public static Date add(final Date date, final int calendarField, final int amount) { + if (date == null) { + throw new IllegalArgumentException("The date must not be null"); + } + final Calendar c = Calendar.getInstance(); + c.setTime(date); + c.add(calendarField, amount); + return c.getTime(); + } + + /** + * starting from the current time, get how many seconds are left before the target time. + * targetTime = baseTime + intervalSeconds + * + * @param baseTime base time + * @param intervalSeconds a period of time + * @return the number of seconds + */ + public static long getRemainTime(Date baseTime, long intervalSeconds) { + if (baseTime == null) { + return 0; + } + long usedTime = (System.currentTimeMillis() - baseTime.getTime()) / 1000; + return intervalSeconds - usedTime; + } +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/enums/ExecutionStatusTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/enums/ExecutionStatusTest.java new file mode 100644 index 000000000000..6d4be78aefae --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/enums/ExecutionStatusTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.enums; + +import junit.framework.TestCase; + +/** + * execution status test. + */ +public class ExecutionStatusTest extends TestCase { + + public void testTypeIsRunning() { + assertTrue(ExecutionStatus.RUNNING_EXECUTION.typeIsRunning()); + assertTrue(ExecutionStatus.WAITTING_DEPEND.typeIsRunning()); + assertTrue(ExecutionStatus.DELAY_EXECUTION.typeIsRunning()); + } +} \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index b82da62b0229..9688200b2c3b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -42,7 +42,7 @@ public class TaskInstance implements Serializable { /** * id */ - @TableId(value="id", type=IdType.AUTO) + @TableId(value = "id", type = IdType.AUTO) private int id; /** @@ -51,7 +51,6 @@ public class TaskInstance implements Serializable { private String name; - /** * task type */ @@ -83,22 +82,28 @@ public class TaskInstance implements Serializable { */ private ExecutionStatus state; + /** + * task first submit time. + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date firstSubmitTime; + /** * task submit time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date submitTime; /** * task start time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date startTime; /** * task end time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date endTime; /** @@ -214,11 +219,14 @@ public class TaskInstance implements Serializable { @TableField(exist = false) - private Map resources; - + private Map resources; + /** + * delay execution time. + */ + private int delayTime; - public void init(String host,Date startTime,String executePath){ + public void init(String host, Date startTime, String executePath) { this.host = host; this.startTime = startTime; this.executePath = executePath; @@ -297,6 +305,14 @@ public void setState(ExecutionStatus state) { this.state = state; } + public Date getFirstSubmitTime() { + return firstSubmitTime; + } + + public void setFirstSubmitTime(Date firstSubmitTime) { + this.firstSubmitTime = firstSubmitTime; + } + public Date getSubmitTime() { return submitTime; } @@ -361,7 +377,7 @@ public void setRetryTimes(int retryTimes) { this.retryTimes = retryTimes; } - public Boolean isTaskSuccess(){ + public Boolean isTaskSuccess() { return this.state == ExecutionStatus.SUCCESS; } @@ -400,6 +416,7 @@ public Flag getFlag() { public void setFlag(Flag flag) { this.flag = flag; } + public String getProcessInstanceName() { return processInstanceName; } @@ -464,33 +481,33 @@ public void setResources(Map resources) { this.resources = resources; } - public boolean isSubProcess(){ + public boolean isSubProcess() { return TaskType.SUB_PROCESS.equals(TaskType.valueOf(this.taskType)); } - public boolean isDependTask(){ + public boolean isDependTask() { return TaskType.DEPENDENT.equals(TaskType.valueOf(this.taskType)); } - public boolean isConditionsTask(){ + public boolean isConditionsTask() { return TaskType.CONDITIONS.equals(TaskType.valueOf(this.taskType)); } - /** * determine if you can try again + * * @return can try result */ public boolean taskCanRetry() { - if(this.isSubProcess()){ + if (this.isSubProcess()) { return false; } - if(this.getState() == ExecutionStatus.NEED_FAULT_TOLERANCE){ + if (this.getState() == ExecutionStatus.NEED_FAULT_TOLERANCE) { return true; - }else { + } else { return (this.getState().typeIsFailure() - && this.getRetryTimes() < this.getMaxRetryTimes()); + && this.getRetryTimes() < this.getMaxRetryTimes()); } } @@ -526,40 +543,50 @@ public void setDependentResult(String dependentResult) { this.dependentResult = dependentResult; } + public int getDelayTime() { + return delayTime; + } + + public void setDelayTime(int delayTime) { + this.delayTime = delayTime; + } + @Override public String toString() { - return "TaskInstance{" + - "id=" + id + - ", name='" + name + '\'' + - ", taskType='" + taskType + '\'' + - ", processDefinitionId=" + processDefinitionId + - ", processInstanceId=" + processInstanceId + - ", processInstanceName='" + processInstanceName + '\'' + - ", taskJson='" + taskJson + '\'' + - ", state=" + state + - ", submitTime=" + submitTime + - ", startTime=" + startTime + - ", endTime=" + endTime + - ", host='" + host + '\'' + - ", executePath='" + executePath + '\'' + - ", logPath='" + logPath + '\'' + - ", retryTimes=" + retryTimes + - ", alertFlag=" + alertFlag + - ", processInstance=" + processInstance + - ", processDefine=" + processDefine + - ", pid=" + pid + - ", appLink='" + appLink + '\'' + - ", flag=" + flag + - ", dependency='" + dependency + '\'' + - ", duration=" + duration + - ", maxRetryTimes=" + maxRetryTimes + - ", retryInterval=" + retryInterval + - ", taskInstancePriority=" + taskInstancePriority + - ", processInstancePriority=" + processInstancePriority + - ", dependentResult='" + dependentResult + '\'' + - ", workerGroup='" + workerGroup + '\'' + - ", executorId=" + executorId + - ", executorName='" + executorName + '\'' + - '}'; + return "TaskInstance{" + + "id=" + id + + ", name='" + name + '\'' + + ", taskType='" + taskType + '\'' + + ", processDefinitionId=" + processDefinitionId + + ", processInstanceId=" + processInstanceId + + ", processInstanceName='" + processInstanceName + '\'' + + ", taskJson='" + taskJson + '\'' + + ", state=" + state + + ", firstSubmitTime=" + firstSubmitTime + + ", submitTime=" + submitTime + + ", startTime=" + startTime + + ", endTime=" + endTime + + ", host='" + host + '\'' + + ", executePath='" + executePath + '\'' + + ", logPath='" + logPath + '\'' + + ", retryTimes=" + retryTimes + + ", alertFlag=" + alertFlag + + ", processInstance=" + processInstance + + ", processDefine=" + processDefine + + ", pid=" + pid + + ", appLink='" + appLink + '\'' + + ", flag=" + flag + + ", dependency='" + dependency + '\'' + + ", duration=" + duration + + ", maxRetryTimes=" + maxRetryTimes + + ", retryInterval=" + retryInterval + + ", taskInstancePriority=" + taskInstancePriority + + ", processInstancePriority=" + processInstancePriority + + ", dependentResult='" + dependentResult + '\'' + + ", workerGroup='" + workerGroup + '\'' + + ", executorId=" + executorId + + ", executorName='" + executorName + '\'' + + ", delayTime=" + delayTime + + '}'; } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java index a69022214de4..179ae1bef8a5 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java @@ -75,6 +75,7 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t out.add(packet); // checkpoint(State.MAGIC); + break; default: logger.warn("unknown decoder state {}", state()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java index 535c274989f8..74b06351450e 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/builder/TaskExecutionContextBuilder.java @@ -40,6 +40,7 @@ public static TaskExecutionContextBuilder get(){ public TaskExecutionContextBuilder buildTaskInstanceRelatedInfo(TaskInstance taskInstance){ taskExecutionContext.setTaskInstanceId(taskInstance.getId()); taskExecutionContext.setTaskName(taskInstance.getName()); + taskExecutionContext.setFirstSubmitTime(taskInstance.getFirstSubmitTime()); taskExecutionContext.setStartTime(taskInstance.getStartTime()); taskExecutionContext.setTaskType(taskInstance.getTaskType()); taskExecutionContext.setLogPath(taskInstance.getLogPath()); @@ -48,6 +49,7 @@ public TaskExecutionContextBuilder buildTaskInstanceRelatedInfo(TaskInstance tas taskExecutionContext.setWorkerGroup(taskInstance.getWorkerGroup()); taskExecutionContext.setHost(taskInstance.getHost()); taskExecutionContext.setResources(taskInstance.getResources()); + taskExecutionContext.setDelayTime(taskInstance.getDelayTime()); return this; } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java index 81488fb134ef..1589c365c2f2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/entity/TaskExecutionContext.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.server.entity; -import com.fasterxml.jackson.annotation.JsonFormat; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand; import org.apache.dolphinscheduler.remote.utils.JsonSerializer; @@ -26,30 +26,38 @@ import java.util.Date; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonFormat; + /** - * master/worker task transport + * master/worker task transport */ -public class TaskExecutionContext implements Serializable{ +public class TaskExecutionContext implements Serializable { /** - * task id + * task id */ private int taskInstanceId; /** - * task name + * task name */ private String taskName; /** - * task start time + * task first submit time. */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + private Date firstSubmitTime; + + /** + * task start time + */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date startTime; /** - * task type + * task type */ private String taskType; @@ -57,9 +65,9 @@ public class TaskExecutionContext implements Serializable{ * host */ private String host; - + /** - * task execute path + * task execute path */ private String executePath; @@ -69,7 +77,7 @@ public class TaskExecutionContext implements Serializable{ private String logPath; /** - * task json + * task json */ private String taskJson; @@ -84,53 +92,53 @@ public class TaskExecutionContext implements Serializable{ private String appIds; /** - * process instance id + * process instance id */ private int processInstanceId; /** - * process instance schedule time + * process instance schedule time */ - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date scheduleTime; /** - * process instance global parameters + * process instance global parameters */ private String globalParams; /** - * execute user id + * execute user id */ private int executorId; /** - * command type if complement + * command type if complement */ private int cmdTypeIfComplement; /** - * tenant code + * tenant code */ private String tenantCode; /** - * task queue + * task queue */ private String queue; /** - * process define id + * process define id */ private int processDefineId; /** - * project id + * project id */ private int projectId; @@ -140,12 +148,12 @@ public class TaskExecutionContext implements Serializable{ private String taskParams; /** - * envFile + * envFile */ private String envFile; /** - * definedParams + * definedParams */ private Map definedParams; @@ -155,7 +163,7 @@ public class TaskExecutionContext implements Serializable{ private String taskAppId; /** - * task timeout strategy + * task timeout strategy */ private int taskTimeoutStrategy; @@ -169,18 +177,28 @@ public class TaskExecutionContext implements Serializable{ */ private String workerGroup; + /** + * delay execution time. + */ + private int delayTime; + + /** + * current execution status + */ + private ExecutionStatus currentExecutionStatus; + /** * resources full name and tenant code */ - private Map resources; + private Map resources; /** - * sql TaskExecutionContext + * sql TaskExecutionContext */ private SQLTaskExecutionContext sqlTaskExecutionContext; /** - * datax TaskExecutionContext + * datax TaskExecutionContext */ private DataxTaskExecutionContext dataxTaskExecutionContext; @@ -195,7 +213,7 @@ public class TaskExecutionContext implements Serializable{ private SqoopTaskExecutionContext sqoopTaskExecutionContext; /** - * procedure TaskExecutionContext + * procedure TaskExecutionContext */ private ProcedureTaskExecutionContext procedureTaskExecutionContext; @@ -215,6 +233,14 @@ public void setTaskName(String taskName) { this.taskName = taskName; } + public Date getFirstSubmitTime() { + return firstSubmitTime; + } + + public void setFirstSubmitTime(Date firstSubmitTime) { + this.firstSubmitTime = firstSubmitTime; + } + public Date getStartTime() { return startTime; } @@ -407,6 +433,22 @@ public void setWorkerGroup(String workerGroup) { this.workerGroup = workerGroup; } + public int getDelayTime() { + return delayTime; + } + + public void setDelayTime(int delayTime) { + this.delayTime = delayTime; + } + + public ExecutionStatus getCurrentExecutionStatus() { + return currentExecutionStatus; + } + + public void setCurrentExecutionStatus(ExecutionStatus currentExecutionStatus) { + this.currentExecutionStatus = currentExecutionStatus; + } + public SQLTaskExecutionContext getSqlTaskExecutionContext() { return sqlTaskExecutionContext; } @@ -431,7 +473,7 @@ public void setProcedureTaskExecutionContext(ProcedureTaskExecutionContext proce this.procedureTaskExecutionContext = procedureTaskExecutionContext; } - public Command toCommand(){ + public Command toCommand() { TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand(); requestCommand.setTaskExecutionContext(JsonSerializer.serializeToString(this)); return requestCommand.convert2Command(); @@ -463,39 +505,42 @@ public void setSqoopTaskExecutionContext(SqoopTaskExecutionContext sqoopTaskExec @Override public String toString() { - return "TaskExecutionContext{" + - "taskInstanceId=" + taskInstanceId + - ", taskName='" + taskName + '\'' + - ", startTime=" + startTime + - ", taskType='" + taskType + '\'' + - ", host='" + host + '\'' + - ", executePath='" + executePath + '\'' + - ", logPath='" + logPath + '\'' + - ", taskJson='" + taskJson + '\'' + - ", processId=" + processId + - ", appIds='" + appIds + '\'' + - ", processInstanceId=" + processInstanceId + - ", scheduleTime=" + scheduleTime + - ", globalParams='" + globalParams + '\'' + - ", executorId=" + executorId + - ", cmdTypeIfComplement=" + cmdTypeIfComplement + - ", tenantCode='" + tenantCode + '\'' + - ", queue='" + queue + '\'' + - ", processDefineId=" + processDefineId + - ", projectId=" + projectId + - ", taskParams='" + taskParams + '\'' + - ", envFile='" + envFile + '\'' + - ", definedParams=" + definedParams + - ", taskAppId='" + taskAppId + '\'' + - ", taskTimeoutStrategy=" + taskTimeoutStrategy + - ", taskTimeout=" + taskTimeout + - ", workerGroup='" + workerGroup + '\'' + - ", resources=" + resources + - ", sqlTaskExecutionContext=" + sqlTaskExecutionContext + - ", dataxTaskExecutionContext=" + dataxTaskExecutionContext + - ", dependenceTaskExecutionContext=" + dependenceTaskExecutionContext + - ", sqoopTaskExecutionContext=" + sqoopTaskExecutionContext + - ", procedureTaskExecutionContext=" + procedureTaskExecutionContext + - '}'; + return "TaskExecutionContext{" + + "taskInstanceId=" + taskInstanceId + + ", taskName='" + taskName + '\'' + + ", currentExecutionStatus=" + currentExecutionStatus + + ", firstSubmitTime=" + firstSubmitTime + + ", startTime=" + startTime + + ", taskType='" + taskType + '\'' + + ", host='" + host + '\'' + + ", executePath='" + executePath + '\'' + + ", logPath='" + logPath + '\'' + + ", taskJson='" + taskJson + '\'' + + ", processId=" + processId + + ", appIds='" + appIds + '\'' + + ", processInstanceId=" + processInstanceId + + ", scheduleTime=" + scheduleTime + + ", globalParams='" + globalParams + '\'' + + ", executorId=" + executorId + + ", cmdTypeIfComplement=" + cmdTypeIfComplement + + ", tenantCode='" + tenantCode + '\'' + + ", queue='" + queue + '\'' + + ", processDefineId=" + processDefineId + + ", projectId=" + projectId + + ", taskParams='" + taskParams + '\'' + + ", envFile='" + envFile + '\'' + + ", definedParams=" + definedParams + + ", taskAppId='" + taskAppId + '\'' + + ", taskTimeoutStrategy=" + taskTimeoutStrategy + + ", taskTimeout=" + taskTimeout + + ", workerGroup='" + workerGroup + '\'' + + ", delayTime=" + delayTime + + ", resources=" + resources + + ", sqlTaskExecutionContext=" + sqlTaskExecutionContext + + ", dataxTaskExecutionContext=" + dataxTaskExecutionContext + + ", dependenceTaskExecutionContext=" + dependenceTaskExecutionContext + + ", sqoopTaskExecutionContext=" + sqoopTaskExecutionContext + + ", procedureTaskExecutionContext=" + procedureTaskExecutionContext + + '}'; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java index a410f99fc156..6878d5a8b1f4 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java @@ -60,6 +60,7 @@ public class ConditionsTaskExecThread extends MasterBaseTaskExecThread { */ public ConditionsTaskExecThread(TaskInstance taskInstance) { super(taskInstance); + taskInstance.setStartTime(new Date()); } @Override diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java index 183f1aac4223..cfe885cbfaff 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java @@ -64,6 +64,7 @@ public class DependentTaskExecThread extends MasterBaseTaskExecThread { */ public DependentTaskExecThread(TaskInstance taskInstance) { super(taskInstance); + taskInstance.setStartTime(new Date()); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java index aac201c403c9..cf5359d57957 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterBaseTaskExecThread.java @@ -16,11 +16,11 @@ */ package org.apache.dolphinscheduler.server.master.runner; -import ch.qos.logback.classic.LoggerContext; -import ch.qos.logback.classic.sift.SiftingAppender; +import static org.apache.dolphinscheduler.common.Constants.UNDERLINE; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -30,11 +30,14 @@ import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue; import org.apache.dolphinscheduler.service.queue.TaskPriorityQueueImpl; + +import java.util.concurrent.Callable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.dolphinscheduler.common.Constants.*; -import java.util.concurrent.Callable; +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.sift.SiftingAppender; /** @@ -171,9 +174,10 @@ public Boolean dispatchTask(TaskInstance taskInstance) { logger.info(String.format("submit task , but task [%s] state [%s] is already finished. ", taskInstance.getName(), taskInstance.getState().toString())); return true; } - // task cannot submit when running - if(taskInstance.getState() == ExecutionStatus.RUNNING_EXECUTION){ - logger.info(String.format("submit to task, but task [%s] state already be running. ", taskInstance.getName())); + // task cannot be submitted because its execution state is RUNNING or DELAY. + if (taskInstance.getState() == ExecutionStatus.RUNNING_EXECUTION + || taskInstance.getState() == ExecutionStatus.DELAY_EXECUTION) { + logger.info("submit task, but the status of the task {} is already running or delayed.", taskInstance.getName()); return true; } logger.info("task ready to submit: {}", taskInstance); @@ -272,5 +276,4 @@ public String getTaskLogPath(TaskInstance task) { return logPath; } - } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 5be6050f8790..788b30638e10 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -16,10 +16,21 @@ */ package org.apache.dolphinscheduler.server.master.runner; -import com.google.common.collect.Lists; -import org.apache.commons.io.FileUtils; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_RECOVERY_START_NODE_STRING; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_START_NODE_NAMES; +import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP; +import static org.apache.dolphinscheduler.common.Constants.SEC_2_MINUTES_TIME_UNIT; + import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.DependResult; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.Priority; +import org.apache.dolphinscheduler.common.enums.TaskDependType; import org.apache.dolphinscheduler.common.graph.DAG; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; @@ -27,7 +38,13 @@ import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -37,17 +54,26 @@ import org.apache.dolphinscheduler.server.utils.AlertManager; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + +import org.apache.commons.io.FileUtils; import java.io.File; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import static org.apache.dolphinscheduler.common.Constants.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; /** * master exec thread,split dag @@ -470,9 +496,6 @@ private TaskInstance createTaskInstance(ProcessInstance processInstance, String // task instance whether alert taskInstance.setAlertFlag(Flag.NO); - // task instance start time - taskInstance.setStartTime(new Date()); - // task instance flag taskInstance.setFlag(Flag.YES); @@ -501,6 +524,8 @@ private TaskInstance createTaskInstance(ProcessInstance processInstance, String taskInstance.setWorkerGroup(taskWorkerGroup); } + // delay execution time + taskInstance.setDelayTime(taskNode.getDelayTime()); } return taskInstance; } @@ -719,9 +744,10 @@ private List getCompleteTaskByState(ExecutionStatus state){ * @return ExecutionStatus */ private ExecutionStatus runningState(ExecutionStatus state){ - if(state == ExecutionStatus.READY_STOP || - state == ExecutionStatus.READY_PAUSE || - state == ExecutionStatus.WAITTING_THREAD){ + if (state == ExecutionStatus.READY_STOP + || state == ExecutionStatus.READY_PAUSE + || state == ExecutionStatus.WAITTING_THREAD + || state == ExecutionStatus.DELAY_EXECUTION) { // if the running task is not completed, the state remains unchanged return state; }else{ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java index a2dd7cea8dfc..72ee0fcb89df 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecThread.java @@ -24,6 +24,8 @@ import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -39,7 +41,6 @@ import java.util.Date; import java.util.Set; -import org.apache.dolphinscheduler.common.utils.*; /** @@ -150,7 +151,7 @@ public Boolean waitTaskQuit(){ break; } if(checkTimeout){ - long remainTime = getRemaintime(taskTimeoutParameter.getInterval() * 60L); + long remainTime = DateUtils.getRemainTime(taskInstance.getStartTime(), taskTimeoutParameter.getInterval() * 60L); if (remainTime < 0) { logger.warn("task id: {} execution time out",taskInstance.getId()); // process define @@ -256,16 +257,4 @@ private TaskTimeoutParameter getTaskTimeoutParameter(){ TaskNode taskNode = JSONUtils.parseObject(taskJson, TaskNode.class); return taskNode.getTaskTimeoutParameter(); } - - - /** - * get remain time?s? - * - * @return remain time - */ - private long getRemaintime(long timeoutSeconds) { - Date startTime = taskInstance.getStartTime(); - long usedTime = (System.currentTimeMillis() - startTime.getTime()) / 1000; - return timeoutSeconds - usedTime; - } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index 2ce451527951..2dc9bb2ce9f7 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -18,15 +18,17 @@ package org.apache.dolphinscheduler.server.worker.processor; -import ch.qos.logback.classic.LoggerContext; -import ch.qos.logback.classic.sift.SiftingAppender; -import com.github.rholder.retry.RetryException; -import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; +import org.apache.dolphinscheduler.common.utils.Preconditions; +import org.apache.dolphinscheduler.common.utils.RetryerUtils; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; @@ -38,14 +40,21 @@ import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Date; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.github.rholder.retry.RetryException; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.sift.SiftingAppender; +import io.netty.channel.Channel; + /** * worker request processor */ @@ -97,7 +106,7 @@ public void process(Channel channel, Command command) { return; } - taskExecutionContext.setHost(NetUtils.getHost() + ":" + workerConfig.getListenPort()); + taskExecutionContext.setHost(NetUtils.getHost() + ":" + workerConfig.getListenPort()); // custom logger Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, @@ -122,7 +131,15 @@ public void process(Channel channel, Command command) { taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(), new NettyRemoteChannel(channel, command.getOpaque())); - // tell master that task is in executing + if (DateUtils.getRemainTime(taskExecutionContext.getFirstSubmitTime(), taskExecutionContext.getDelayTime() * 60L) > 0) { + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.DELAY_EXECUTION); + taskExecutionContext.setStartTime(null); + } else { + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); + taskExecutionContext.setStartTime(new Date()); + } + + // tell master the status of this task (RUNNING_EXECUTION or DELAY_EXECUTION) final Command ackCommand = buildAckCommand(taskExecutionContext).convert2Command(); try { @@ -167,10 +184,10 @@ private String getTaskLogPath(TaskExecutionContext taskExecutionContext) { private TaskExecuteAckCommand buildAckCommand(TaskExecutionContext taskExecutionContext) { TaskExecuteAckCommand ackCommand = new TaskExecuteAckCommand(); ackCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); - ackCommand.setStatus(ExecutionStatus.RUNNING_EXECUTION.getCode()); + ackCommand.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode()); ackCommand.setLogPath(getTaskLogPath(taskExecutionContext)); ackCommand.setHost(taskExecutionContext.getHost()); - ackCommand.setStartTime(new Date()); + ackCommand.setStartTime(taskExecutionContext.getStartTime()); if(taskExecutionContext.getTaskType().equals(TaskType.SQL.name()) || taskExecutionContext.getTaskType().equals(TaskType.PROCEDURE.name())){ ackCommand.setExecutePath(null); }else{ diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java index 26494bc77bcf..3ba49451c7fa 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java @@ -16,25 +16,20 @@ */ package org.apache.dolphinscheduler.server.worker.runner; -import java.io.File; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.Set; - -import org.apache.commons.collections.MapUtils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.RetryerUtils; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager; @@ -43,9 +38,23 @@ import org.apache.dolphinscheduler.server.worker.task.AbstractTask; import org.apache.dolphinscheduler.server.worker.task.TaskManager; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; + +import org.apache.commons.collections.MapUtils; + +import java.io.File; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.github.rholder.retry.RetryException; + /** * task scheduler thread @@ -105,6 +114,15 @@ public void run() { // task node TaskNode taskNode = JSONUtils.parseObject(taskExecutionContext.getTaskJson(), TaskNode.class); + delayExecutionIfNeeded(); + if (taskExecutionContext.getStartTime() == null) { + taskExecutionContext.setStartTime(new Date()); + } + if (taskExecutionContext.getCurrentExecutionStatus() != ExecutionStatus.RUNNING_EXECUTION) { + changeTaskExecutionStatusToRunning(); + } + logger.info("the task begins to execute. task instance id: {}", taskExecutionContext.getTaskInstanceId()); + // copy hdfs/minio file to local downloadResource(taskExecutionContext.getExecutePath(), taskExecutionContext.getResources(), @@ -138,7 +156,7 @@ public void run() { responseCommand.setProcessId(task.getProcessId()); responseCommand.setAppIds(task.getAppIds()); logger.info("task instance id : {},task final status : {}", taskExecutionContext.getTaskInstanceId(), task.getExitStatus()); - }catch (Exception e){ + } catch (Exception e) { logger.error("task scheduler failure", e); kill(); responseCommand.setStatus(ExecutionStatus.FAILURE.getCode()); @@ -147,9 +165,10 @@ public void run() { responseCommand.setAppIds(task.getAppIds()); } finally { try { + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.of(responseCommand.getStatus())); taskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId()); taskCallbackService.sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command()); - }catch (Exception e){ + } catch (Exception e) { ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS); taskCallbackService.sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand.convert2Command()); } @@ -256,4 +275,59 @@ private void downloadResource(String execLocalPath, } } } + + /** + * delay execution if needed. + */ + private void delayExecutionIfNeeded() { + long remainTime = DateUtils.getRemainTime(taskExecutionContext.getFirstSubmitTime(), + taskExecutionContext.getDelayTime() * 60L); + logger.info("delay execution time: {} s", remainTime < 0 ? 0 : remainTime); + if (remainTime > 0) { + try { + Thread.sleep(remainTime * Constants.SLEEP_TIME_MILLIS); + } catch (Exception e) { + logger.error("delay task execution failure, the task will be executed directly. process instance id:{}, task instance id:{}", + taskExecutionContext.getProcessInstanceId(), + taskExecutionContext.getTaskInstanceId()); + } + } + } + + /** + * send an ack to change the status of the task. + */ + private void changeTaskExecutionStatusToRunning() { + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); + Command ackCommand = buildAckCommand().convert2Command(); + try { + RetryerUtils.retryCall(() -> { + taskCallbackService.sendAck(taskExecutionContext.getTaskInstanceId(), ackCommand); + return Boolean.TRUE; + }); + } catch (ExecutionException | RetryException e) { + logger.error(e.getMessage(), e); + } + } + + /** + * build ack command. + * + * @return TaskExecuteAckCommand + */ + private TaskExecuteAckCommand buildAckCommand() { + TaskExecuteAckCommand ackCommand = new TaskExecuteAckCommand(); + ackCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId()); + ackCommand.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode()); + ackCommand.setStartTime(taskExecutionContext.getStartTime()); + ackCommand.setLogPath(taskExecutionContext.getLogPath()); + ackCommand.setHost(taskExecutionContext.getHost()); + if (taskExecutionContext.getTaskType().equals(TaskType.SQL.name()) + || taskExecutionContext.getTaskType().equals(TaskType.PROCEDURE.name())) { + ackCommand.setExecutePath(null); + } else { + ackCommand.setExecutePath(taskExecutionContext.getExecutePath()); + } + return ackCommand; + } } \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java index 3459e35b72b3..a6f0f1a29ee2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/AbstractCommandExecutor.java @@ -16,12 +16,14 @@ */ package org.apache.dolphinscheduler.server.worker.task; -import com.sun.jna.platform.win32.Kernel32; -import com.sun.jna.platform.win32.WinNT; +import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_FAILURE; +import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_SUCCESS; + import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.HadoopUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; @@ -32,9 +34,12 @@ import org.apache.dolphinscheduler.server.worker.cache.TaskExecutionContextCacheManager; import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; -import org.slf4j.Logger; -import java.io.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -47,8 +52,10 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_FAILURE; -import static org.apache.dolphinscheduler.common.Constants.EXIT_CODE_SUCCESS; +import org.slf4j.Logger; + +import com.sun.jna.platform.win32.Kernel32; +import com.sun.jna.platform.win32.WinNT; /** * abstract command executor @@ -474,8 +481,7 @@ private String findAppId(String line) { * @return remain time */ private long getRemaintime() { - long usedTime = (System.currentTimeMillis() - taskExecutionContext.getStartTime().getTime()) / 1000; - long remainTime = taskExecutionContext.getTaskTimeout() - usedTime; + long remainTime = DateUtils.getRemainTime(taskExecutionContext.getStartTime(), taskExecutionContext.getTaskTimeout()); if (remainTime < 0) { throw new RuntimeException("task execution time out"); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java index 997c129573e5..32881b56813c 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumerTest.java @@ -55,7 +55,8 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, CuratorZookeeperClient.class, NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, TaskPriorityQueueConsumer.class, - ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, MasterConfig.class}) + ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, MasterConfig.class, + CuratorZookeeperClient.class}) public class TaskPriorityQueueConsumerTest { diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java index 98231bee06a0..de1f0517bddf 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java @@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry; import org.apache.dolphinscheduler.server.zk.SpringZKServer; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; import org.junit.Test; @@ -46,7 +47,7 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes={DependencyConfig.class, SpringApplicationContext.class, SpringZKServer.class, WorkerRegistry.class, NettyExecutorManager.class, ExecutorDispatcher.class, ZookeeperRegistryCenter.class, WorkerConfig.class, - ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class}) + ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, CuratorZookeeperClient.class}) public class ExecutorDispatcherTest { @Autowired diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java index a1c6b71437d1..f7d98baed13c 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java @@ -37,6 +37,7 @@ import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry; import org.apache.dolphinscheduler.server.zk.SpringZKServer; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; +import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; import org.junit.Assert; @@ -52,7 +53,7 @@ */ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes={DependencyConfig.class, SpringZKServer.class, WorkerRegistry.class, - ZookeeperNodeManager.class, ZookeeperRegistryCenter.class, WorkerConfig.class, + ZookeeperNodeManager.class, ZookeeperRegistryCenter.class, WorkerConfig.class, CuratorZookeeperClient.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, SpringApplicationContext.class, NettyExecutorManager.class}) public class NettyExecutorManagerTest { diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java index c38c9d4ae8a5..21780635acd2 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java @@ -28,6 +28,7 @@ import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistry; import org.apache.dolphinscheduler.server.zk.SpringZKServer; +import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; import org.junit.Assert; @@ -43,7 +44,7 @@ */ @RunWith(SpringRunner.class) @ContextConfiguration(classes={DependencyConfig.class, SpringZKServer.class, WorkerRegistry.class, ZookeeperRegistryCenter.class, WorkerConfig.class, - ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class}) + ZookeeperNodeManager.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, CuratorZookeeperClient.class}) public class RoundRobinHostManagerTest { diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java index f19e2b4b6409..a2b1b4ecc28a 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java @@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.server.registry.ZookeeperNodeManager; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.server.zk.SpringZKServer; +import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; import org.junit.Assert; @@ -35,7 +36,8 @@ @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes={DependencyConfig.class, SpringZKServer.class, TaskResponseService.class, ZookeeperRegistryCenter.class, - ZookeeperCachedOperator.class, ZookeeperConfig.class, ZookeeperNodeManager.class, TaskResponseService.class}) + ZookeeperCachedOperator.class, ZookeeperConfig.class, ZookeeperNodeManager.class, TaskResponseService.class, + CuratorZookeeperClient.class}) public class TaskResponseServiceTest { @Autowired diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java index 18c8b496d761..ea822cbfb11a 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryTest.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.server.zk.SpringZKServer; +import org.apache.dolphinscheduler.service.zk.CuratorZookeeperClient; import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator; import org.apache.dolphinscheduler.service.zk.ZookeeperConfig; import org.junit.Assert; @@ -38,7 +39,8 @@ * master registry test */ @RunWith(SpringRunner.class) -@ContextConfiguration(classes={SpringZKServer.class, MasterRegistry.class,ZookeeperRegistryCenter.class, MasterConfig.class, ZookeeperCachedOperator.class, ZookeeperConfig.class}) +@ContextConfiguration(classes = {SpringZKServer.class, MasterRegistry.class, ZookeeperRegistryCenter.class, + MasterConfig.class, ZookeeperCachedOperator.class, ZookeeperConfig.class, CuratorZookeeperClient.class}) public class MasterRegistryTest { @Autowired diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java new file mode 100644 index 000000000000..2e7e531f30cc --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.worker.runner; + +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.model.TaskNode; +import org.apache.dolphinscheduler.common.task.AbstractParameters; +import org.apache.dolphinscheduler.common.utils.CommonUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; +import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand; +import org.apache.dolphinscheduler.server.entity.TaskExecutionContext; +import org.apache.dolphinscheduler.server.worker.cache.impl.TaskExecutionContextCacheManagerImpl; +import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService; +import org.apache.dolphinscheduler.server.worker.task.AbstractTask; +import org.apache.dolphinscheduler.server.worker.task.TaskManager; +import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; + +import java.util.Date; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * test task execute thread. + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest({TaskManager.class, JSONUtils.class, CommonUtils.class, SpringApplicationContext.class}) +public class TaskExecuteThreadTest { + + private TaskExecutionContext taskExecutionContext; + + private TaskCallbackService taskCallbackService; + + private Command ackCommand; + + private Command responseCommand; + + private Logger taskLogger; + + private TaskExecutionContextCacheManagerImpl taskExecutionContextCacheManager; + + @Before + public void before() { + // init task execution context, logger + taskExecutionContext = new TaskExecutionContext(); + taskExecutionContext.setProcessId(12345); + taskExecutionContext.setProcessDefineId(1); + taskExecutionContext.setProcessInstanceId(1); + taskExecutionContext.setTaskInstanceId(1); + taskExecutionContext.setTaskType(""); + taskExecutionContext.setFirstSubmitTime(new Date()); + taskExecutionContext.setDelayTime(0); + taskExecutionContext.setLogPath("/tmp/test.log"); + taskExecutionContext.setHost("localhost"); + taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/exec/process/1/2/3/4"); + + ackCommand = new TaskExecuteAckCommand().convert2Command(); + responseCommand = new TaskExecuteResponseCommand(taskExecutionContext.getTaskInstanceId()).convert2Command(); + + taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId( + LoggerUtils.TASK_LOGGER_INFO_PREFIX, + taskExecutionContext.getProcessDefineId(), + taskExecutionContext.getProcessInstanceId(), + taskExecutionContext.getTaskInstanceId() + )); + + taskExecutionContextCacheManager = new TaskExecutionContextCacheManagerImpl(); + taskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext); + + taskCallbackService = PowerMockito.mock(TaskCallbackService.class); + PowerMockito.doNothing().when(taskCallbackService).sendAck(taskExecutionContext.getTaskInstanceId(), ackCommand); + PowerMockito.doNothing().when(taskCallbackService).sendResult(taskExecutionContext.getTaskInstanceId(), responseCommand); + + PowerMockito.mockStatic(SpringApplicationContext.class); + PowerMockito.when(SpringApplicationContext.getBean(TaskExecutionContextCacheManagerImpl.class)) + .thenReturn(taskExecutionContextCacheManager); + + PowerMockito.mockStatic(TaskManager.class); + PowerMockito.when(TaskManager.newTask(taskExecutionContext, taskLogger)) + .thenReturn(new SimpleTask(taskExecutionContext, taskLogger)); + + PowerMockito.mockStatic(JSONUtils.class); + PowerMockito.when(JSONUtils.parseObject(taskExecutionContext.getTaskJson(), TaskNode.class)) + .thenReturn(new TaskNode()); + + PowerMockito.mockStatic(CommonUtils.class); + PowerMockito.when(CommonUtils.getSystemEnvPath()).thenReturn("/user_home/.bash_profile"); + } + + @Test + public void testNormalExecution() { + taskExecutionContext.setTaskType("SQL"); + taskExecutionContext.setStartTime(new Date()); + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.RUNNING_EXECUTION); + TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger); + taskExecuteThread.run(); + + Assert.assertEquals(ExecutionStatus.SUCCESS, taskExecutionContext.getCurrentExecutionStatus()); + } + + @Test + public void testDelayExecution() { + taskExecutionContext.setTaskType("PYTHON"); + taskExecutionContext.setStartTime(null); + taskExecutionContext.setDelayTime(1); + taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.DELAY_EXECUTION); + TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext, taskCallbackService, taskLogger); + taskExecuteThread.run(); + + Assert.assertEquals(ExecutionStatus.SUCCESS, taskExecutionContext.getCurrentExecutionStatus()); + } + + private class SimpleTask extends AbstractTask { + + protected SimpleTask(TaskExecutionContext taskExecutionContext, Logger logger) { + super(taskExecutionContext, logger); + // pid + this.processId = taskExecutionContext.getProcessId(); + } + + @Override + public AbstractParameters getParameters() { + return null; + } + + @Override + public void init() { + + } + + @Override + public void handle() throws Exception { + + } + + @Override + public void after() { + + } + + @Override + public ExecutionStatus getExitStatus() { + return ExecutionStatus.SUCCESS; + } + } +} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index 7df56c6cdd56..6f642672cd6e 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -16,31 +16,89 @@ */ package org.apache.dolphinscheduler.service.process; -import com.cronutils.model.Cron; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.lang.ArrayUtils; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_EMPTY_SUB_PROCESS; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_RECOVER_PROCESS_ID_STRING; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_SUB_PROCESS; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_SUB_PROCESS_DEFINE_ID; +import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_SUB_PROCESS_PARENT_INSTANCE_ID; +import static org.apache.dolphinscheduler.common.Constants.YYYY_MM_DD_HH_MM_SS; + +import static java.util.stream.Collectors.toSet; + import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.enums.*; +import org.apache.dolphinscheduler.common.enums.AuthorizationType; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.CycleEnum; +import org.apache.dolphinscheduler.common.enums.ExecutionStatus; +import org.apache.dolphinscheduler.common.enums.FailureStrategy; +import org.apache.dolphinscheduler.common.enums.Flag; +import org.apache.dolphinscheduler.common.enums.ResourceType; +import org.apache.dolphinscheduler.common.enums.TaskDependType; +import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.model.DateInterval; import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; -import org.apache.dolphinscheduler.common.utils.*; -import org.apache.dolphinscheduler.dao.entity.*; -import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.DateUtils; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.ParameterUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.dao.entity.Command; +import org.apache.dolphinscheduler.dao.entity.CycleDependency; +import org.apache.dolphinscheduler.dao.entity.DataSource; +import org.apache.dolphinscheduler.dao.entity.ErrorCommand; +import org.apache.dolphinscheduler.dao.entity.ProcessData; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.Schedule; +import org.apache.dolphinscheduler.dao.entity.TaskInstance; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.CommandMapper; +import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.TenantMapper; +import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.service.log.LogClientService; import org.apache.dolphinscheduler.service.quartz.cron.CronUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + import org.quartz.CronExpression; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; -import java.util.*; -import java.util.stream.Collectors; -import static java.util.stream.Collectors.toSet; -import static org.apache.dolphinscheduler.common.Constants.*; + +import com.cronutils.model.Cron; +import com.fasterxml.jackson.databind.node.ObjectNode; /** * process relative dao that some mappers in this. @@ -52,6 +110,7 @@ public class ProcessService { private final int[] stateArray = new int[]{ExecutionStatus.SUBMITTED_SUCCESS.ordinal(), ExecutionStatus.RUNNING_EXECUTION.ordinal(), + ExecutionStatus.DELAY_EXECUTION.ordinal(), ExecutionStatus.READY_PAUSE.ordinal(), ExecutionStatus.READY_STOP.ordinal()}; @@ -1001,8 +1060,9 @@ public TaskInstance submitTaskInstanceToDB(TaskInstance taskInstance, ProcessIns if(taskInstance.getState() != ExecutionStatus.NEED_FAULT_TOLERANCE){ taskInstance.setRetryTimes(taskInstance.getRetryTimes() + 1 ); } + taskInstance.setSubmitTime(null); + taskInstance.setStartTime(null); taskInstance.setEndTime(null); - taskInstance.setStartTime(new Date()); taskInstance.setFlag(Flag.YES); taskInstance.setHost(null); taskInstance.setId(0); @@ -1012,7 +1072,12 @@ public TaskInstance submitTaskInstanceToDB(TaskInstance taskInstance, ProcessIns taskInstance.setExecutorId(processInstance.getExecutorId()); taskInstance.setProcessInstancePriority(processInstance.getProcessInstancePriority()); taskInstance.setState(getSubmitTaskState(taskInstance, processInstanceState)); - taskInstance.setSubmitTime(new Date()); + if (taskInstance.getSubmitTime() == null) { + taskInstance.setSubmitTime(new Date()); + } + if (taskInstance.getFirstSubmitTime() == null) { + taskInstance.setFirstSubmitTime(taskInstance.getSubmitTime()); + } boolean saveResult = saveTaskInstance(taskInstance); if(!saveResult){ return null; @@ -1062,10 +1127,11 @@ public String taskZkInfo(TaskInstance taskInstance) { public ExecutionStatus getSubmitTaskState(TaskInstance taskInstance, ExecutionStatus processInstanceState){ ExecutionStatus state = taskInstance.getState(); if( - // running or killed + // running, delayed or killed // the task already exists in task queue // return state state == ExecutionStatus.RUNNING_EXECUTION + || state == ExecutionStatus.DELAY_EXECUTION || state == ExecutionStatus.KILL || checkTaskExistsInTaskQueue(taskInstance) ){ @@ -1588,7 +1654,7 @@ public CycleDependency getCycleDependency(int masterId, int processDefinitionId, */ public List getCycleDependencies(int masterId,int[] ids,Date scheduledFireTime) throws Exception { List cycleDependencyList = new ArrayList(); - if(ArrayUtils.isEmpty(ids)){ + if (ids == null || ids.length == 0) { logger.warn("ids[] is empty!is invalid!"); return cycleDependencyList; } @@ -1772,7 +1838,7 @@ public List getProjectIdListHavePerm(int userId){ public List listUnauthorized(int userId,T[] needChecks,AuthorizationType authorizationType){ List resultList = new ArrayList(); - if (!ArrayUtils.isEmpty(needChecks)) { + if (Objects.nonNull(needChecks) && needChecks.length > 0) { Set originResSet = new HashSet(Arrays.asList(needChecks)); switch (authorizationType){ diff --git a/pom.xml b/pom.xml index 2937a814fc1e..49e00ba1b0ce 100644 --- a/pom.xml +++ b/pom.xml @@ -792,6 +792,7 @@ **/common/utils/RetryerUtilsTest.java **/common/plugin/FilePluginManagerTest **/common/plugin/PluginClassLoaderTest + **/common/enums/ExecutionStatusTest **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java **/dao/mapper/CommandMapperTest.java @@ -842,6 +843,7 @@ **/server/worker/task/sqoop/SqoopTaskTest.java **/server/worker/EnvFileTest.java + **/server/worker/runner/TaskExecuteThreadTest.java **/service/quartz/cron/CronUtilsTest.java **/service/zk/DefaultEnsembleProviderTest.java **/service/zk/ZKServerTest.java diff --git a/sql/dolphinscheduler-postgre.sql b/sql/dolphinscheduler-postgre.sql index 4a947d134d3f..3a0b1843beb2 100644 --- a/sql/dolphinscheduler-postgre.sql +++ b/sql/dolphinscheduler-postgre.sql @@ -566,8 +566,10 @@ CREATE TABLE t_ds_task_instance ( retry_interval int DEFAULT NULL , max_retry_times int DEFAULT NULL , task_instance_priority int DEFAULT NULL , - worker_group varchar(64), + worker_group varchar(64), executor_id int DEFAULT NULL , + first_submit_time timestamp DEFAULT NULL , + delay_time int DEFAULT '0' , PRIMARY KEY (id) ) ; diff --git a/sql/dolphinscheduler_mysql.sql b/sql/dolphinscheduler_mysql.sql index f4ad9a3cb86a..bb3dbb095a26 100644 --- a/sql/dolphinscheduler_mysql.sql +++ b/sql/dolphinscheduler_mysql.sql @@ -707,6 +707,8 @@ CREATE TABLE `t_ds_task_instance` ( `task_instance_priority` int(11) DEFAULT NULL COMMENT 'task instance priority:0 Highest,1 High,2 Medium,3 Low,4 Lowest', `worker_group` varchar(64) DEFAULT NULL COMMENT 'worker group id', `executor_id` int(11) DEFAULT NULL, + `first_submit_time` datetime DEFAULT NULL COMMENT 'task first submit time', + `delay_time` int(4) DEFAULT '0' COMMENT 'task delay execution time', PRIMARY KEY (`id`), KEY `process_instance_id` (`process_instance_id`) USING BTREE, KEY `task_instance_index` (`process_definition_id`,`process_instance_id`) USING BTREE, diff --git a/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_ddl.sql index 40e7a3f4dc03..a24c9bc92866 100644 --- a/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_ddl.sql +++ b/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_ddl.sql @@ -297,4 +297,3 @@ delimiter ; CALL ac_dolphin_T_t_ds_user_A_state; DROP PROCEDURE ac_dolphin_T_t_ds_user_A_state; - diff --git a/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 000000000000..9eaf3f8d5037 --- /dev/null +++ b/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); +-- uc_dolphin_T_t_ds_task_instance_A_first_submit_time +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_instance_A_first_submit_time; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_task_instance_A_first_submit_time() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='first_submit_time') + THEN + ALTER TABLE t_ds_task_instance ADD `first_submit_time` datetime DEFAULT NULL COMMENT 'task first submit time'; + END IF; + END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_task_instance_A_first_submit_time(); +DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_A_first_submit_time; + +-- uc_dolphin_T_t_ds_task_instance_A_delay_time +drop PROCEDURE if EXISTS uc_dolphin_T_t_ds_task_instance_A_delay_time; +delimiter d// +CREATE PROCEDURE uc_dolphin_T_t_ds_task_instance_A_delay_time() + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND TABLE_SCHEMA=(SELECT DATABASE()) + AND COLUMN_NAME ='delay_time') + THEN + ALTER TABLE t_ds_task_instance ADD `delay_time` int(4) DEFAULT '0' COMMENT 'task delay execution time'; + END IF; + END; + +d// + +delimiter ; +CALL uc_dolphin_T_t_ds_task_instance_A_delay_time(); +DROP PROCEDURE uc_dolphin_T_t_ds_task_instance_A_delay_time; + diff --git a/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 000000000000..4a14f326b985 --- /dev/null +++ b/sql/upgrade/1.3.3_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ diff --git a/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 000000000000..9a65824238f9 --- /dev/null +++ b/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +-- uc_dolphin_T_t_ds_task_instance_A_first_submit_time +delimiter d// +CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_task_instance_A_first_submit_time() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND COLUMN_NAME ='first_submit_time') + THEN + ALTER TABLE t_ds_task_instance ADD COLUMN first_submit_time timestamp DEFAULT NULL; + END IF; +END; +$$ LANGUAGE plpgsql; +d// + +delimiter ; +SELECT uc_dolphin_T_t_ds_task_instance_A_first_submit_time(); +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_A_first_submit_time(); + + +-- uc_dolphin_T_t_ds_task_instance_A_delay_time +delimiter d// +CREATE OR REPLACE FUNCTION uc_dolphin_T_t_ds_task_instance_A_delay_time() RETURNS void AS $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.COLUMNS + WHERE TABLE_NAME='t_ds_task_instance' + AND COLUMN_NAME ='delay_time') + THEN + ALTER TABLE t_ds_task_instance ADD COLUMN delay_time int DEFAULT '0'; + END IF; +END; +$$ LANGUAGE plpgsql; +d// + +delimiter ; +SELECT uc_dolphin_T_t_ds_task_instance_A_delay_time(); +DROP FUNCTION IF EXISTS uc_dolphin_T_t_ds_task_instance_A_delay_time(); \ No newline at end of file diff --git a/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 000000000000..4a14f326b985 --- /dev/null +++ b/sql/upgrade/1.3.3_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ From 056168fc326e2d935a56d22216c9c1abf628c157 Mon Sep 17 00:00:00 2001 From: yinancx Date: Wed, 19 Aug 2020 11:46:40 +0800 Subject: [PATCH 07/53] [Feature][ui] Delay execution and typo fix (#3508) --- .../js/conf/home/pages/dag/_source/config.js | 7 + .../pages/dag/_source/formModel/formModel.vue | 21 ++- .../_source/instanceConditions/common.js | 3 + .../pages/index/_source/taskStatusCount.vue | 147 ++++++++++++++++++ .../home/pages/projects/pages/index/index.vue | 8 +- .../js/conf/home/store/projects/actions.js | 2 +- .../src/js/module/i18n/locale/en_US.js | 2 + .../src/js/module/i18n/locale/zh_CN.js | 2 + 8 files changed, 186 insertions(+), 6 deletions(-) create mode 100644 dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js index a4960f7ac581..2e609295776a 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/config.js @@ -229,6 +229,13 @@ const tasksState = { color: '#5101be', icoUnicode: 'ans-icon-dependence', isSpin: false + }, + DELAY_EXECUTION: { + id: 12, + desc: `${i18n.$t('Delay execution')}`, + color: '#5102ce', + icoUnicode: 'ans-icon-coin', + isSpin: false } } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue index 6f07f97f02af..8444863aea29 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/formModel.vue @@ -109,6 +109,20 @@ ({{$t('Minute')}}) + + +
+
+ {{$t('Delay execution time')}} +
+
+ + + ({{$t('Minute')}}) +
+
+ +
{{$t('State')}} @@ -127,7 +141,6 @@
-
{{$t('State')}} @@ -339,6 +352,8 @@ maxRetryTimes: '0', // Failure retry interval retryInterval: '1', + // Delay execution time + delayTime: '0', // Task timeout alarm timeout: {}, // Task priority @@ -466,6 +481,7 @@ dependence: this.cacheDependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, + delayTime: this.delayTime, timeout: this.timeout, taskInstancePriority: this.taskInstancePriority, workerGroup: this.workerGroup, @@ -544,6 +560,7 @@ dependence: this.dependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, + delayTime: this.delayTime, timeout: this.timeout, taskInstancePriority: this.taskInstancePriority, workerGroup: this.workerGroup, @@ -634,6 +651,7 @@ this.description = o.description this.maxRetryTimes = o.maxRetryTimes this.retryInterval = o.retryInterval + this.delayTime = o.delayTime if(o.conditionResult) { this.successBranch = o.conditionResult.successNode[0] this.failedBranch = o.conditionResult.failedNode[0] @@ -699,6 +717,7 @@ dependence: this.cacheDependence, maxRetryTimes: this.maxRetryTimes, retryInterval: this.retryInterval, + delayTime: this.delayTime, timeout: this.timeout, taskInstancePriority: this.taskInstancePriority, workerGroup: this.workerGroup, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js index 694d04748ca3..8a13aeacb46a 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/_source/instanceConditions/common.js @@ -60,6 +60,9 @@ const stateType = [ }, { code: 'WAITTING_DEPEND', label: `${i18n.$t('Waiting for dependency to complete')}` + }, { + code: 'DELAY_EXECUTION', + label: `${i18n.$t('Delay execution')}` } ] diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue new file mode 100644 index 000000000000..90ae53f4c916 --- /dev/null +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/taskStatusCount.vue @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue index 2b5cbbc017bd..7ca6e3a0f65c 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/index.vue @@ -35,8 +35,8 @@ {{$t('Task status statistics')}}
- - + +
@@ -68,7 +68,7 @@ import dayjs from 'dayjs' import mDefineUserCount from './_source/defineUserCount' import mCommandStateCount from './_source/commandStateCount' - import mTaskCtatusCount from './_source/taskCtatusCount' + import mTaskStatusCount from './_source/taskStatusCount' import mProcessStateCount from './_source/processStateCount' import mQueueCount from './_source/queueCount' import localStore from '@/module/util/localStorage' @@ -105,7 +105,7 @@ mListConstruction, mDefineUserCount, mCommandStateCount, - mTaskCtatusCount, + mTaskStatusCount, mProcessStateCount, mQueueCount } diff --git a/dolphinscheduler-ui/src/js/conf/home/store/projects/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/projects/actions.js index 43273de9e272..6a18fdaf9cdd 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/projects/actions.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/projects/actions.js @@ -69,7 +69,7 @@ export default { /** * Task status statistics */ - getTaskCtatusCount ({ state }, payload) { + getTaskStatusCount ({ state }, payload) { return new Promise((resolve, reject) => { io.get('projects/analysis/task-state-count', payload, res => { resolve(res) diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 0ef53404882f..e5e96131f516 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -42,6 +42,8 @@ export default { Times: 'Times', 'Failed retry interval': 'Failed retry interval', Minute: 'Minute', + 'Delay execution time': 'Delay execution time', + 'Delay execution': 'Delay execution', Cancel: 'Cancel', 'Confirm add': 'Confirm add', 'The newly created sub-Process has not yet been executed and cannot enter the sub-Process': 'The newly created sub-Process has not yet been executed and cannot enter the sub-Process', diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index a352183fcaea..26326d3fe2ea 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -43,6 +43,7 @@ export default { Times: '次', 'Failed retry interval': '失败重试间隔', Minute: '分', + 'Delay execution time': '延时执行时间', Cancel: '取消', 'Confirm add': '确认添加', 'The newly created sub-Process has not yet been executed and cannot enter the sub-Process': '新创建子工作流还未执行,不能进入子工作流', @@ -425,6 +426,7 @@ export default { hour: '时', Running: '正在运行', 'Waiting for dependency to complete': '等待依赖完成', + 'Delay execution': '延时执行', Selected: '已选', CurrentHour: '当前小时', Last1Hour: '前1小时', From a1ce54b8e6b36f915fe73e02a8aa43923e665476 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Wed, 19 Aug 2020 15:15:28 +0800 Subject: [PATCH 08/53] [Improvement][server] Fix some sonar analyzed bugs (#3415) * [FIX_BUG][*]fix sonar bugs fix assert error(assertions always pass) * fix assert error(assertions always pass) insert data should judging the success status * fix assert error(assertions always pass) * use AtomicReference to replace unsafe volatile Object * use AtomicReference to replace unsafe volatile Object * lazy initialization holder class Singleton... * reformat code judge database operations * reformat code judge database operations * revert * use ThreadLocalRandom to build random numbers * reform code * fix dao test error * judge database operations --- .../dao/mapper/AccessTokenMapperTest.java | 75 ++++++++++------ .../dao/mapper/ResourceMapperTest.java | 85 +++++++++++++------ .../server/master/future/TaskFuture.java | 21 ++--- .../processor/queue/TaskResponseService.java | 1 + .../service/quartz/QuartzExecutors.java | 23 ++--- 5 files changed, 123 insertions(+), 82 deletions(-) diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java index 0c1dc20ac9fd..30c8cdc7b979 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/AccessTokenMapperTest.java @@ -34,6 +34,7 @@ import javax.annotation.Resource; import java.text.SimpleDateFormat; import java.util.*; +import java.util.concurrent.ThreadLocalRandom; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThan; @@ -45,7 +46,7 @@ @RunWith(SpringRunner.class) @SpringBootTest @Transactional -@Rollback(true) +@Rollback public class AccessTokenMapperTest { @Autowired @@ -56,10 +57,11 @@ public class AccessTokenMapperTest { /** * test insert + * * @throws Exception */ @Test - public void testInsert() throws Exception{ + public void testInsert() throws Exception { Integer userId = 1; AccessToken accessToken = createAccessToken(userId); @@ -69,10 +71,11 @@ public void testInsert() throws Exception{ /** * test select by id + * * @throws Exception */ @Test - public void testSelectById() throws Exception{ + public void testSelectById() throws Exception { Integer userId = 1; AccessToken accessToken = createAccessToken(userId); AccessToken resultAccessToken = accessTokenMapper.selectById(accessToken.getId()); @@ -81,6 +84,7 @@ public void testSelectById() throws Exception{ /** * test hashCode method + * * @throws Exception */ @Test @@ -94,6 +98,7 @@ public void testHashCodeMethod() throws Exception { /** * test equals method + * * @throws Exception */ @Test @@ -108,7 +113,7 @@ public void testEqualsMethod() throws Exception { * test page */ @Test - public void testSelectAccessTokenPage() throws Exception{ + public void testSelectAccessTokenPage() throws Exception { Integer count = 4; String userName = "zhangsan"; @@ -120,11 +125,11 @@ public void testSelectAccessTokenPage() throws Exception{ Page page = new Page(offset, size); IPage accessTokenPage = accessTokenMapper.selectAccessTokenPage(page, userName, 0); - assertEquals(Integer.valueOf(accessTokenPage.getRecords().size()),size); + assertEquals(Integer.valueOf(accessTokenPage.getRecords().size()), size); - for (AccessToken accessToken : accessTokenPage.getRecords()){ + for (AccessToken accessToken : accessTokenPage.getRecords()) { AccessToken resultAccessToken = accessTokenMap.get(accessToken.getId()); - assertEquals(accessToken,resultAccessToken); + assertEquals(accessToken, resultAccessToken); } } @@ -133,14 +138,17 @@ public void testSelectAccessTokenPage() throws Exception{ * test update */ @Test - public void testUpdate() throws Exception{ + public void testUpdate() throws Exception { Integer userId = 1; AccessToken accessToken = createAccessToken(userId); //update accessToken.setToken("56789"); accessToken.setExpireTime(DateUtils.getCurrentDate()); accessToken.setUpdateTime(DateUtils.getCurrentDate()); - accessTokenMapper.updateById(accessToken); + int status = accessTokenMapper.updateById(accessToken); + if (status != 1) { + Assert.fail("update access token fail"); + } AccessToken resultAccessToken = accessTokenMapper.selectById(accessToken.getId()); assertEquals(accessToken, resultAccessToken); } @@ -149,11 +157,14 @@ public void testUpdate() throws Exception{ * test delete */ @Test - public void testDelete() throws Exception{ + public void testDelete() throws Exception { Integer userId = 1; AccessToken accessToken = createAccessToken(userId); - accessTokenMapper.deleteById(accessToken.getId()); + int status = accessTokenMapper.deleteById(accessToken.getId()); + if (status != 1) { + Assert.fail("delete access token data fail"); + } AccessToken resultAccessToken = accessTokenMapper.selectById(accessToken.getId()); @@ -163,21 +174,22 @@ public void testDelete() throws Exception{ /** * create accessTokens - * @param count create accessToken count + * + * @param count create accessToken count * @param userName username * @return accessToken map * @throws Exception */ - private Map createAccessTokens( - Integer count,String userName) throws Exception{ + private Map createAccessTokens( + Integer count, String userName) throws Exception { User user = createUser(userName); - Map accessTokenMap = new HashMap<>(); - for (int i = 1 ; i<= count ; i++){ - AccessToken accessToken = createAccessToken(user.getId(),userName); + Map accessTokenMap = new HashMap<>(); + for (int i = 1; i <= count; i++) { + AccessToken accessToken = createAccessToken(user.getId(), userName); - accessTokenMap.put(accessToken.getId(),accessToken); + accessTokenMap.put(accessToken.getId(), accessToken); } return accessTokenMap; @@ -185,11 +197,12 @@ private Map createAccessTokens( /** * create user + * * @param userName userName * @return user * @throws Exception */ - private User createUser(String userName) throws Exception{ + private User createUser(String userName) throws Exception { User user = new User(); user.setUserName(userName); user.setUserPassword("123"); @@ -201,42 +214,50 @@ private User createUser(String userName) throws Exception{ user.setUpdateTime(DateUtils.getCurrentDate()); user.setQueue("default"); - userMapper.insert(user); + int status = userMapper.insert(user); + + if (status != 1) { + Assert.fail("insert user data error"); + } return user; } /** * create access token - * @param userId userId + * + * @param userId userId * @param userName userName * @return accessToken * @throws Exception */ - private AccessToken createAccessToken(Integer userId,String userName)throws Exception{ - Random random = new Random(); + private AccessToken createAccessToken(Integer userId, String userName) throws Exception { //insertOne AccessToken accessToken = new AccessToken(); accessToken.setUserName(userName); accessToken.setUserId(userId); - accessToken.setToken(String.valueOf(random.nextLong())); + accessToken.setToken(String.valueOf(ThreadLocalRandom.current().nextLong())); accessToken.setCreateTime(DateUtils.getCurrentDate()); accessToken.setUpdateTime(DateUtils.getCurrentDate()); accessToken.setExpireTime(DateUtils.getCurrentDate()); - accessTokenMapper.insert(accessToken); + int status = accessTokenMapper.insert(accessToken); + if (status != 1) { + Assert.fail("insert data error"); + } return accessToken; } /** * create access token + * * @param userId userId * @return accessToken * @throws Exception */ - private AccessToken createAccessToken(Integer userId)throws Exception{ - return createAccessToken(userId,null); + private AccessToken createAccessToken(Integer userId) throws Exception { + return createAccessToken(userId, null); } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java index 818f88fb493b..76741a7db9e3 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java @@ -65,9 +65,10 @@ public class ResourceMapperTest { /** * insert + * * @return Resource */ - private Resource insertOne(){ + private Resource insertOne() { //insertOne Resource resource = new Resource(); resource.setAlias("ut-resource"); @@ -76,16 +77,20 @@ private Resource insertOne(){ resource.setDirectory(false); resource.setType(ResourceType.FILE); resource.setUserId(111); - resourceMapper.insert(resource); + int status = resourceMapper.insert(resource); + if (status != 1) { + Assert.fail("insert data error"); + } return resource; } /** * create resource by user + * * @param user user * @return Resource */ - private Resource createResource(User user,boolean isDirectory,ResourceType resourceType,int pid,String alias,String fullName){ + private Resource createResource(User user, boolean isDirectory, ResourceType resourceType, int pid, String alias, String fullName) { //insertOne Resource resource = new Resource(); resource.setDirectory(isDirectory); @@ -93,19 +98,23 @@ private Resource createResource(User user,boolean isDirectory,ResourceType resou resource.setAlias(alias); resource.setFullName(fullName); resource.setUserId(user.getId()); - resourceMapper.insert(resource); + int status = resourceMapper.insert(resource); + if (status != 1) { + Assert.fail("insert data error"); + } return resource; } /** * create resource by user + * * @param user user * @return Resource */ - private Resource createResource(User user){ + private Resource createResource(User user) { //insertOne - String alias = String.format("ut-resource-%s",user.getUserName()); - String fullName = String.format("/%s",alias); + String alias = String.format("ut-resource-%s", user.getUserName()); + String fullName = String.format("/%s", alias); Resource resource = createResource(user, false, ResourceType.FILE, -1, alias, fullName); return resource; @@ -113,9 +122,10 @@ private Resource createResource(User user){ /** * create user + * * @return User */ - private User createGeneralUser(String userName){ + private User createGeneralUser(String userName) { User user = new User(); user.setUserName(userName); user.setUserPassword("1"); @@ -124,15 +134,20 @@ private User createGeneralUser(String userName){ user.setCreateTime(new Date()); user.setTenantId(1); user.setUpdateTime(new Date()); - userMapper.insert(user); + int status = userMapper.insert(user); + + if (status != 1) { + Assert.fail("insert data error"); + } return user; } /** * create resource user + * * @return ResourcesUser */ - private ResourcesUser createResourcesUser(Resource resource,User user){ + private ResourcesUser createResourcesUser(Resource resource, User user) { //insertOne ResourcesUser resourcesUser = new ResourcesUser(); resourcesUser.setCreateTime(new Date()); @@ -145,16 +160,17 @@ private ResourcesUser createResourcesUser(Resource resource,User user){ } @Test - public void testInsert(){ + public void testInsert() { Resource resource = insertOne(); assertNotNull(resource.getId()); - assertThat(resource.getId(),greaterThan(0)); + assertThat(resource.getId(), greaterThan(0)); } + /** * test update */ @Test - public void testUpdate(){ + public void testUpdate() { //insertOne Resource resource = insertOne(); resource.setCreateTime(new Date()); @@ -167,7 +183,7 @@ public void testUpdate(){ * test delete */ @Test - public void testDelete(){ + public void testDelete() { Resource resourceMap = insertOne(); int delete = resourceMapper.deleteById(resourceMap.getId()); Assert.assertEquals(1, delete); @@ -294,19 +310,31 @@ public void testQueryTenantCodeByResourceName() { Tenant tenant = new Tenant(); tenant.setTenantName("ut tenant "); tenant.setTenantCode("ut tenant code for resource"); - tenantMapper.insert(tenant); + int tenantInsertStatus = tenantMapper.insert(tenant); + + if (tenantInsertStatus != 1) { + Assert.fail("insert tenant data error"); + } User user = new User(); user.setTenantId(tenant.getId()); user.setUserName("ut user"); - userMapper.insert(user); + int userInsertStatus = userMapper.insert(user); + + if (userInsertStatus != 1) { + Assert.fail("insert user data error"); + } + Resource resource = insertOne(); resource.setUserId(user.getId()); - resourceMapper.updateById(resource); + int userUpdateStatus = resourceMapper.updateById(resource); + if (userUpdateStatus != 1) { + Assert.fail("update user data error"); + } String resource1 = resourceMapper.queryTenantCodeByResourceName( - resource.getFullName(),ResourceType.FILE.ordinal() + resource.getFullName(), ResourceType.FILE.ordinal() ); @@ -315,7 +343,7 @@ public void testQueryTenantCodeByResourceName() { } @Test - public void testListAuthorizedResource(){ + public void testListAuthorizedResource() { // create a general user User generalUser1 = createGeneralUser("user1"); User generalUser2 = createGeneralUser("user2"); @@ -328,20 +356,19 @@ public void testListAuthorizedResource(){ List resources = resourceMapper.listAuthorizedResource(generalUser2.getId(), resNames); - Assert.assertEquals(generalUser2.getId(),resource.getUserId()); + Assert.assertEquals(generalUser2.getId(), resource.getUserId()); Assert.assertFalse(resources.stream().map(t -> t.getFullName()).collect(toList()).containsAll(Arrays.asList(resNames))); - // authorize object unauthorizedResource to generalUser - createResourcesUser(unauthorizedResource,generalUser2); + createResourcesUser(unauthorizedResource, generalUser2); List authorizedResources = resourceMapper.listAuthorizedResource(generalUser2.getId(), resNames); Assert.assertTrue(authorizedResources.stream().map(t -> t.getFullName()).collect(toList()).containsAll(Arrays.asList(resNames))); } @Test - public void deleteIdsTest(){ + public void deleteIdsTest() { // create a general user User generalUser1 = createGeneralUser("user1"); @@ -352,11 +379,11 @@ public void deleteIdsTest(){ resourceList.add(resource.getId()); resourceList.add(resource1.getId()); int result = resourceMapper.deleteIds(resourceList.toArray(new Integer[resourceList.size()])); - Assert.assertEquals(result,2); + Assert.assertEquals(result, 2); } @Test - public void queryResourceListAuthoredTest(){ + public void queryResourceListAuthoredTest() { // create a general user User generalUser1 = createGeneralUser("user1"); User generalUser2 = createGeneralUser("user2"); @@ -372,16 +399,18 @@ public void queryResourceListAuthoredTest(){ } @Test - public void batchUpdateResourceTest(){ + public void batchUpdateResourceTest() { // create a general user User generalUser1 = createGeneralUser("user1"); // create resource Resource resource = createResource(generalUser1); - resource.setFullName(String.format("%s-update",resource.getFullName())); + resource.setFullName(String.format("%s-update", resource.getFullName())); resource.setUpdateTime(new Date()); List resourceList = new ArrayList<>(); resourceList.add(resource); int result = resourceMapper.batchUpdateResource(resourceList); - Assert.assertTrue(result>0); + if (result != resourceList.size()) { + Assert.fail("batch update resource data error"); + } } } \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/future/TaskFuture.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/future/TaskFuture.java index 918ed6764bfc..e7d89240710a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/future/TaskFuture.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/future/TaskFuture.java @@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; /** * task fulture @@ -55,11 +56,11 @@ public class TaskFuture { /** * response command */ - private volatile Command responseCommand; + private AtomicReference responseCommandReference = new AtomicReference<>(); private volatile boolean sendOk = true; - private volatile Throwable cause; + private AtomicReference causeReference; public TaskFuture(long opaque, long timeoutMillis) { this.opaque = opaque; @@ -74,7 +75,7 @@ public TaskFuture(long opaque, long timeoutMillis) { */ public Command waitResponse() throws InterruptedException { this.latch.await(timeoutMillis, TimeUnit.MILLISECONDS); - return this.responseCommand; + return this.responseCommandReference.get(); } /** @@ -83,7 +84,7 @@ public Command waitResponse() throws InterruptedException { * @param responseCommand responseCommand */ public void putResponse(final Command responseCommand) { - this.responseCommand = responseCommand; + responseCommandReference.set(responseCommand); this.latch.countDown(); FUTURE_TABLE.remove(opaque); } @@ -114,11 +115,11 @@ public void setSendOk(boolean sendOk) { } public void setCause(Throwable cause) { - this.cause = cause; + causeReference.set(cause); } public Throwable getCause() { - return cause; + return causeReference.get(); } public long getOpaque() { @@ -134,11 +135,11 @@ public long getBeginTimestamp() { } public Command getResponseCommand() { - return responseCommand; + return responseCommandReference.get(); } public void setResponseCommand(Command responseCommand) { - this.responseCommand = responseCommand; + responseCommandReference.set(responseCommand); } @@ -166,9 +167,9 @@ public String toString() { ", timeoutMillis=" + timeoutMillis + ", latch=" + latch + ", beginTimestamp=" + beginTimestamp + - ", responseCommand=" + responseCommand + + ", responseCommand=" + responseCommandReference.get() + ", sendOk=" + sendOk + - ", cause=" + cause + + ", cause=" + causeReference.get() + '}'; } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java index 7abb31b31c44..ba07313a9acc 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseService.java @@ -108,6 +108,7 @@ public void run() { TaskResponseEvent taskResponseEvent = eventQueue.take(); persist(taskResponseEvent); } catch (InterruptedException e){ + Thread.currentThread().interrupt(); break; } catch (Exception e){ logger.error("persist task error",e); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java index 01d7b2f1e595..3b15810e057b 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java @@ -61,20 +61,20 @@ public class QuartzExecutors { */ private static Scheduler scheduler; - /** - * instance of QuartzExecutors - */ - private static volatile QuartzExecutors INSTANCE = null; - /** * load conf */ private static Configuration conf; + private static final class Holder { + private static final QuartzExecutors instance = new QuartzExecutors(); + } + private QuartzExecutors() { try { conf = new PropertiesConfiguration(QUARTZ_PROPERTIES_PATH); + init(); }catch (ConfigurationException e){ logger.warn("not loaded quartz configuration file, will used default value",e); } @@ -85,18 +85,7 @@ private QuartzExecutors() { * @return instance of Quartz Executors */ public static QuartzExecutors getInstance() { - if (INSTANCE == null) { - synchronized (QuartzExecutors.class) { - // when more than two threads run into the first null check same time, to avoid instanced more than one time, it needs to be checked again. - if (INSTANCE == null) { - QuartzExecutors quartzExecutors = new QuartzExecutors(); - //finish QuartzExecutors init - quartzExecutors.init(); - INSTANCE = quartzExecutors; - } - } - } - return INSTANCE; + return Holder.instance; } From d30bc7bcf3dda45672845130e04212dfec9f0133 Mon Sep 17 00:00:00 2001 From: JinyLeeChina <42576980+JinyLeeChina@users.noreply.github.com> Date: Wed, 19 Aug 2020 16:24:47 +0800 Subject: [PATCH 09/53] '3549' (#3550) --- .../apache/dolphinscheduler/server/worker/task/sql/SqlTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java index 939426b3bf1f..f28f5804b021 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java @@ -262,7 +262,7 @@ private void resultProcess(ResultSet resultSet) throws Exception{ while (rowCount < LIMIT && resultSet.next()) { ObjectNode mapOfColValues = JSONUtils.createObjectNode(); for (int i = 1; i <= num; i++) { - mapOfColValues.set(md.getColumnName(i), JSONUtils.toJsonNode(resultSet.getObject(i))); + mapOfColValues.set(md.getColumnLabel(i), JSONUtils.toJsonNode(resultSet.getObject(i))); } resultJSONArray.add(mapOfColValues); rowCount++; From 2f0102580268ff045e6042c3a691f1dee4c49962 Mon Sep 17 00:00:00 2001 From: "felix.wang" <59079269+felix-thinkingdata@users.noreply.github.com> Date: Wed, 19 Aug 2020 16:27:39 +0800 Subject: [PATCH 10/53] [Fix-3299][dao&server] Fix 3299,when Json string parsing problem caused by non-standard json format. (#3552) * #3299 Json string parsing problem caused by non-standard json format. * #3299 Json string parsing problem caused by non-standard json format. * #3299 Json string parsing problem caused by non-standard json format. fix code style * #3299 Json string parsing problem caused by non-standard json format. fix code style Co-authored-by: wangjianda --- .../apache/dolphinscheduler/dao/AlertDao.java | 81 +++++++++---- .../server/utils/AlertManager.java | 109 ++++++++---------- 2 files changed, 104 insertions(+), 86 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java index 49b8c01ecee7..685d72c1e867 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/AlertDao.java @@ -14,15 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.dao; +package org.apache.dolphinscheduler.dao; -import org.apache.dolphinscheduler.common.utils.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; import org.apache.dolphinscheduler.common.enums.AlertStatus; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.ShowType; +import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; @@ -30,13 +29,17 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.AlertMapper; import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.util.ArrayList; import java.util.Date; +import java.util.LinkedHashMap; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + @Component public class AlertDao extends AbstractBaseDao { @@ -56,21 +59,23 @@ protected void init() { /** * insert alert + * * @param alert alert * @return add alert result */ - public int addAlert(Alert alert){ + public int addAlert(Alert alert) { return alertMapper.insert(alert); } /** * update alert + * * @param alertStatus alertStatus * @param log log * @param id id * @return update alert result */ - public int updateAlert(AlertStatus alertStatus,String log,int id){ + public int updateAlert(AlertStatus alertStatus, String log, int id) { Alert alert = alertMapper.selectById(id); alert.setAlertStatus(alertStatus); alert.setUpdateTime(new Date()); @@ -80,46 +85,61 @@ public int updateAlert(AlertStatus alertStatus,String log,int id){ /** * query user list by alert group id + * * @param alerGroupId alerGroupId * @return user list */ - public List queryUserByAlertGroupId(int alerGroupId){ + public List queryUserByAlertGroupId(int alerGroupId) { return userAlertGroupMapper.listUserByAlertgroupId(alerGroupId); } /** * MasterServer or WorkerServer stoped + * * @param alertgroupId alertgroupId * @param host host * @param serverType serverType */ - public void sendServerStopedAlert(int alertgroupId,String host,String serverType){ + public void sendServerStopedAlert(int alertgroupId, String host, String serverType) { Alert alert = new Alert(); - String content = String.format("[{'type':'%s','host':'%s','event':'server down','warning level':'serious'}]", - serverType, host); + List serverStopList = new ArrayList<>(1); + LinkedHashMap serverStopedMap = new LinkedHashMap(); + serverStopedMap.put("type", serverType); + serverStopedMap.put("host", host); + serverStopedMap.put("event", "server down"); + serverStopedMap.put("warning level", "serious"); + serverStopList.add(serverStopedMap); + String content = JSONUtils.toJsonString(serverStopList); alert.setTitle("Fault tolerance warning"); saveTaskTimeoutAlert(alert, content, alertgroupId, null, null); } /** * process time out alert + * * @param processInstance processInstance * @param processDefinition processDefinition */ - public void sendProcessTimeoutAlert(ProcessInstance processInstance, ProcessDefinition processDefinition){ + public void sendProcessTimeoutAlert(ProcessInstance processInstance, ProcessDefinition processDefinition) { int alertgroupId = processInstance.getWarningGroupId(); String receivers = processDefinition.getReceivers(); String receiversCc = processDefinition.getReceiversCc(); Alert alert = new Alert(); - String content = String.format("[{'id':'%d','name':'%s','event':'timeout','warnLevel':'middle'}]", - processInstance.getId(), processInstance.getName()); + List processTimeoutList = new ArrayList<>(1); + LinkedHashMap processTimeoutMap = new LinkedHashMap(); + processTimeoutMap.put("id", String.valueOf(processInstance.getId())); + processTimeoutMap.put("name", processInstance.getName()); + processTimeoutMap.put("event", "timeout"); + processTimeoutMap.put("warnLevel", "middle"); + processTimeoutList.add(processTimeoutMap); + String content = JSONUtils.toJsonString(processTimeoutList); alert.setTitle("Process Timeout Warn"); saveTaskTimeoutAlert(alert, content, alertgroupId, receivers, receiversCc); } - private void saveTaskTimeoutAlert(Alert alert, String content, int alertgroupId, - String receivers, String receiversCc){ + private void saveTaskTimeoutAlert(Alert alert, String content, int alertgroupId, + String receivers, String receiversCc) { alert.setShowType(ShowType.TABLE); alert.setContent(content); alert.setAlertType(AlertType.EMAIL); @@ -135,9 +155,9 @@ private void saveTaskTimeoutAlert(Alert alert, String content, int alertgroupId alertMapper.insert(alert); } - /** * task timeout warn + * * @param alertgroupId alertgroupId * @param receivers receivers * @param receiversCc receiversCc @@ -146,34 +166,45 @@ private void saveTaskTimeoutAlert(Alert alert, String content, int alertgroupId * @param taskId taskId * @param taskName taskName */ - public void sendTaskTimeoutAlert(int alertgroupId,String receivers,String receiversCc, int processInstanceId, - String processInstanceName, int taskId,String taskName){ + public void sendTaskTimeoutAlert(int alertgroupId, String receivers, String receiversCc, int processInstanceId, + String processInstanceName, int taskId, String taskName) { Alert alert = new Alert(); - String content = String.format("[{'process instance id':'%d','task name':'%s','task id':'%d','task name':'%s'," + - "'event':'timeout','warnLevel':'middle'}]", processInstanceId, processInstanceName, taskId, taskName); + List taskTimeoutList = new ArrayList<>(1); + LinkedHashMap taskTimeoutMap = new LinkedHashMap(); + taskTimeoutMap.put("process instance id", String.valueOf(processInstanceId)); + taskTimeoutMap.put("process name", processInstanceName); + taskTimeoutMap.put("task id", String.valueOf(taskId)); + taskTimeoutMap.put("task name", taskName); + taskTimeoutMap.put("event", "timeout"); + taskTimeoutMap.put("warnLevel", "middle"); + taskTimeoutList.add(taskTimeoutMap); + String content = JSONUtils.toJsonString(taskTimeoutList); alert.setTitle("Task Timeout Warn"); saveTaskTimeoutAlert(alert, content, alertgroupId, receivers, receiversCc); } /** * list the alert information of waiting to be executed + * * @return alert list */ - public List listWaitExecutionAlert(){ + public List listWaitExecutionAlert() { return alertMapper.listAlertByStatus(AlertStatus.WAIT_EXECUTION); } /** * list user information by alert group id + * * @param alertgroupId alertgroupId * @return user list */ - public List listUserByAlertgroupId(int alertgroupId){ + public List listUserByAlertgroupId(int alertgroupId) { return userAlertGroupMapper.listUserByAlertgroupId(alertgroupId); } /** * for test + * * @return AlertMapper */ public AlertMapper getAlertMapper() { diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java index 49ec9d3fdd21..08c602251913 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/AlertManager.java @@ -14,29 +14,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.dolphinscheduler.server.utils; +package org.apache.dolphinscheduler.server.utils; import org.apache.dolphinscheduler.common.enums.AlertType; import org.apache.dolphinscheduler.common.enums.CommandType; import org.apache.dolphinscheduler.common.enums.ShowType; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.*; +import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; import org.apache.dolphinscheduler.dao.entity.Alert; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * alert manager */ @@ -50,8 +51,7 @@ public class AlertManager { /** * alert dao */ - private AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class); - + private final AlertDao alertDao = DaoFactory.getDaoInstance(AlertDao.class); /** * command type convert chinese @@ -86,50 +86,37 @@ private String getCommandCnName(CommandType commandType) { } } - /** - * process instance format - */ - private static final String PROCESS_INSTANCE_FORMAT = - "\"id:%d\"," + - "\"name:%s\"," + - "\"job type: %s\"," + - "\"state: %s\"," + - "\"recovery:%s\"," + - "\"run time: %d\"," + - "\"start time: %s\"," + - "\"end time: %s\"," + - "\"host: %s\"" ; - /** * get process instance content - * @param processInstance process instance - * @param taskInstances task instance list + * + * @param processInstance process instance + * @param taskInstances task instance list * @return process instance format content */ public String getContentProcessInstance(ProcessInstance processInstance, - List taskInstances){ + List taskInstances) { String res = ""; - if(processInstance.getState().typeIsSuccess()){ - res = String.format(PROCESS_INSTANCE_FORMAT, - processInstance.getId(), - processInstance.getName(), - getCommandCnName(processInstance.getCommandType()), - processInstance.getState().toString(), - processInstance.getRecovery().toString(), - processInstance.getRunTimes(), - DateUtils.dateToString(processInstance.getStartTime()), - DateUtils.dateToString(processInstance.getEndTime()), - processInstance.getHost() - - ); - res = "[" + res + "]"; - }else if(processInstance.getState().typeIsFailure()){ + if (processInstance.getState().typeIsSuccess()) { + List successTaskList = new ArrayList<>(1); + LinkedHashMap successTaskMap = new LinkedHashMap(); + successTaskMap.put("id", String.valueOf(processInstance.getId())); + successTaskMap.put("name", processInstance.getName()); + successTaskMap.put("job type", getCommandCnName(processInstance.getCommandType())); + successTaskMap.put("state", processInstance.getState().toString()); + successTaskMap.put("recovery", processInstance.getRecovery().toString()); + successTaskMap.put("run time", String.valueOf(processInstance.getRunTimes())); + successTaskMap.put("start time", DateUtils.dateToString(processInstance.getStartTime())); + successTaskMap.put("end time", DateUtils.dateToString(processInstance.getEndTime())); + successTaskMap.put("host", processInstance.getHost()); + successTaskList.add(successTaskMap); + res = JSONUtils.toJsonString(successTaskList); + } else if (processInstance.getState().typeIsFailure()) { List failedTaskList = new ArrayList<>(); - for(TaskInstance task : taskInstances){ - if(task.getState().typeIsSuccess()){ + for (TaskInstance task : taskInstances) { + if (task.getState().typeIsSuccess()) { continue; } LinkedHashMap failedTaskMap = new LinkedHashMap(); @@ -154,15 +141,15 @@ public String getContentProcessInstance(ProcessInstance processInstance, /** * getting worker fault tolerant content * - * @param processInstance process instance + * @param processInstance process instance * @param toleranceTaskList tolerance task list * @return worker tolerance content */ - private String getWorkerToleranceContent(ProcessInstance processInstance, List toleranceTaskList){ + private String getWorkerToleranceContent(ProcessInstance processInstance, List toleranceTaskList) { - List> toleranceTaskInstanceList = new ArrayList<>(); + List> toleranceTaskInstanceList = new ArrayList<>(); - for(TaskInstance taskInstance: toleranceTaskList){ + for (TaskInstance taskInstance : toleranceTaskList) { LinkedHashMap toleranceWorkerContentMap = new LinkedHashMap(); toleranceWorkerContentMap.put("process name", processInstance.getName()); toleranceWorkerContentMap.put("task name", taskInstance.getName()); @@ -176,11 +163,11 @@ private String getWorkerToleranceContent(ProcessInstance processInstance, List toleranceTaskList){ - try{ + public void sendAlertWorkerToleranceFault(ProcessInstance processInstance, List toleranceTaskList) { + try { Alert alert = new Alert(); alert.setTitle("worker fault tolerance"); alert.setShowType(ShowType.TABLE); @@ -188,13 +175,13 @@ public void sendAlertWorkerToleranceFault(ProcessInstance processInstance, List< alert.setContent(content); alert.setAlertType(AlertType.EMAIL); alert.setCreateTime(new Date()); - alert.setAlertGroupId(processInstance.getWarningGroupId() == null ? 1:processInstance.getWarningGroupId()); + alert.setAlertGroupId(processInstance.getWarningGroupId() == null ? 1 : processInstance.getWarningGroupId()); alert.setReceivers(processInstance.getProcessDefinition().getReceivers()); alert.setReceiversCc(processInstance.getProcessDefinition().getReceiversCc()); alertDao.addAlert(alert); logger.info("add alert to db , alert : {}", alert.toString()); - }catch (Exception e){ + } catch (Exception e) { logger.error("send alert failed:{} ", e.getMessage()); } @@ -202,40 +189,40 @@ public void sendAlertWorkerToleranceFault(ProcessInstance processInstance, List< /** * send process instance alert - * @param processInstance process instance - * @param taskInstances task instance list + * + * @param processInstance process instance + * @param taskInstances task instance list */ public void sendAlertProcessInstance(ProcessInstance processInstance, - List taskInstances){ + List taskInstances) { boolean sendWarnning = false; WarningType warningType = processInstance.getWarningType(); - switch (warningType){ + switch (warningType) { case ALL: - if(processInstance.getState().typeIsFinished()){ + if (processInstance.getState().typeIsFinished()) { sendWarnning = true; } break; case SUCCESS: - if(processInstance.getState().typeIsSuccess()){ + if (processInstance.getState().typeIsSuccess()) { sendWarnning = true; } break; case FAILURE: - if(processInstance.getState().typeIsFailure()){ + if (processInstance.getState().typeIsFailure()) { sendWarnning = true; } break; - default: + default: } - if(!sendWarnning){ + if (!sendWarnning) { return; } Alert alert = new Alert(); - String cmdName = getCommandCnName(processInstance.getCommandType()); - String success = processInstance.getState().typeIsSuccess() ? "success" :"failed"; + String success = processInstance.getState().typeIsSuccess() ? "success" : "failed"; alert.setTitle(cmdName + " " + success); ShowType showType = processInstance.getState().typeIsSuccess() ? ShowType.TEXT : ShowType.TABLE; alert.setShowType(showType); @@ -254,7 +241,7 @@ public void sendAlertProcessInstance(ProcessInstance processInstance, /** * send process timeout alert * - * @param processInstance process instance + * @param processInstance process instance * @param processDefinition process definition */ public void sendProcessTimeoutAlert(ProcessInstance processInstance, ProcessDefinition processDefinition) { From 68db6a75b4db590a40826352353dfd1d27d86a8b Mon Sep 17 00:00:00 2001 From: "felix.wang" <59079269+felix-thinkingdata@users.noreply.github.com> Date: Wed, 19 Aug 2020 16:28:47 +0800 Subject: [PATCH 11/53] #3542 Optimized mail configuration (#3544) * #3542 Optimized mail configuration * #3542 Optimized mail configuration Co-authored-by: wangjianda --- .../dolphinscheduler/alert/utils/MailUtils.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java index 6f6746277134..888c9dbb261d 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/MailUtils.java @@ -59,7 +59,7 @@ public class MailUtils { public static final String STARTTLS_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_STARTTLS_ENABLE); - public static final String SSL_ENABLE = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_ENABLE); + public static final Boolean SSL_ENABLE = PropertyUtils.getBoolean(Constants.MAIL_SMTP_SSL_ENABLE); public static final String SSL_TRUST = PropertyUtils.getString(Constants.MAIL_SMTP_SSL_TRUST); @@ -213,6 +213,7 @@ private static MimeMessage getMimeMessage(Collection receivers) throws M /** * get session + * * @return the new Session */ private static Session getSession() { @@ -222,8 +223,10 @@ private static Session getSession() { props.setProperty(Constants.MAIL_SMTP_AUTH, Constants.STRING_TRUE); props.setProperty(Constants.MAIL_TRANSPORT_PROTOCOL, MAIL_PROTOCOL); props.setProperty(Constants.MAIL_SMTP_STARTTLS_ENABLE, STARTTLS_ENABLE); - props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, SSL_ENABLE); - props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST); + if (SSL_ENABLE) { + props.setProperty(Constants.MAIL_SMTP_SSL_ENABLE, "true"); + props.setProperty(Constants.MAIL_SMTP_SSL_TRUST, SSL_TRUST); + } Authenticator auth = new Authenticator() { @Override @@ -345,5 +348,5 @@ private static void handleException(Collection receivers, Map Date: Wed, 19 Aug 2020 16:31:00 +0800 Subject: [PATCH 12/53] [Improvement][code style] Add sign next line automatic assigner (#3522) --- style/intellij-java-code-style.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/style/intellij-java-code-style.xml b/style/intellij-java-code-style.xml index 8e0cd2f7203b..42286025b01f 100644 --- a/style/intellij-java-code-style.xml +++ b/style/intellij-java-code-style.xml @@ -95,4 +95,7 @@