diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java index c37b439b84..35948b606c 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java @@ -385,7 +385,10 @@ Response> listStepExecutionHistory( String scopeId, @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) @PathVariable("stepInstanceId") - Long stepInstanceId + Long stepInstanceId, + @ApiParam(value = "滚动批次,非滚动步骤不需要传入", name = "batch") + @RequestParam(value = "batch", required = false) + Integer batch ); } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java index 1e21128e72..513f54392f 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java @@ -47,6 +47,8 @@ public class TaskOperationLogVO { private Long stepInstanceId; @ApiModelProperty("步骤执行次数") private Integer retry; + @ApiModelProperty("滚动执行批次") + private Integer batch; @ApiModelProperty("步骤名称") private String stepName; @ApiModelProperty("操作时间") diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java index eff05e13e7..331e0cf16c 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java @@ -84,7 +84,8 @@ public void testBatchSaveAgentTasks() { List agentTaskList = new ArrayList<>(); AgentTaskDTO agentTask1 = new AgentTaskDTO(); agentTask1.setStepInstanceId(100L); - agentTask1.setExecuteCount(0); + agentTask1.setExecuteCount(1); + agentTask1.setActualExecuteCount(1); agentTask1.setBatch(1); agentTask1.setFileTaskMode(FileTaskModeEnum.UPLOAD); agentTask1.setHostId(101L); @@ -99,7 +100,8 @@ public void testBatchSaveAgentTasks() { AgentTaskDTO agentTask2 = new AgentTaskDTO(); agentTask2.setStepInstanceId(100L); - agentTask2.setExecuteCount(0); + agentTask2.setExecuteCount(1); + agentTask2.setActualExecuteCount(1); agentTask2.setBatch(1); agentTask2.setFileTaskMode(FileTaskModeEnum.DOWNLOAD); agentTask2.setHostId(102L); @@ -115,10 +117,11 @@ public void testBatchSaveAgentTasks() { fileAgentTaskDAO.batchSaveAgentTasks(agentTaskList); - AgentTaskDTO agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 0, 1, + AgentTaskDTO agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, FileTaskModeEnum.UPLOAD, 101L); assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(100L); - assertThat(agentTask1Return.getExecuteCount()).isEqualTo(0L); + assertThat(agentTask1Return.getExecuteCount()).isEqualTo(1L); + assertThat(agentTask1Return.getActualExecuteCount()).isEqualTo(1L); assertThat(agentTask1Return.getBatch()).isEqualTo(1); assertThat(agentTask1Return.getFileTaskMode()).isEqualTo(FileTaskModeEnum.UPLOAD); assertThat(agentTask1Return.getHostId()).isEqualTo(101L); @@ -131,10 +134,11 @@ public void testBatchSaveAgentTasks() { assertThat(agentTask1Return.getStatus()).isEqualTo(1); - AgentTaskDTO agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 0, 1, + AgentTaskDTO agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, FileTaskModeEnum.DOWNLOAD, 102L); assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(100L); - assertThat(agentTask2Return.getExecuteCount()).isEqualTo(0L); + assertThat(agentTask2Return.getExecuteCount()).isEqualTo(1L); + assertThat(agentTask2Return.getActualExecuteCount()).isEqualTo(1L); assertThat(agentTask2Return.getBatch()).isEqualTo(1); assertThat(agentTask2Return.getFileTaskMode()).isEqualTo(FileTaskModeEnum.DOWNLOAD); assertThat(agentTask2Return.getHostId()).isEqualTo(102L); diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java index 10413ca2f4..5d8ba6143a 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java @@ -86,7 +86,8 @@ public void testBatchSaveAgentTasks() { List agentTaskList = new ArrayList<>(); AgentTaskDTO agentTask1 = new AgentTaskDTO(); agentTask1.setStepInstanceId(100L); - agentTask1.setExecuteCount(0); + agentTask1.setExecuteCount(1); + agentTask1.setActualExecuteCount(1); agentTask1.setBatch(1); agentTask1.setHostId(101L); agentTask1.setAgentId("0:127.0.0.1"); @@ -102,7 +103,8 @@ public void testBatchSaveAgentTasks() { AgentTaskDTO agentTask2 = new AgentTaskDTO(); agentTask2.setStepInstanceId(100L); - agentTask2.setExecuteCount(0); + agentTask2.setExecuteCount(1); + agentTask2.setActualExecuteCount(1); agentTask2.setBatch(1); agentTask2.setHostId(102L); agentTask2.setAgentId("0:127.0.0.2"); @@ -120,9 +122,10 @@ public void testBatchSaveAgentTasks() { scriptAgentTaskDAO.batchSaveAgentTasks(agentTaskList); - AgentTaskDTO agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 0, 1, 101L); + AgentTaskDTO agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, 101L); assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(100L); - assertThat(agentTask1Return.getExecuteCount()).isEqualTo(0L); + assertThat(agentTask1Return.getExecuteCount()).isEqualTo(1L); + assertThat(agentTask1Return.getActualExecuteCount()).isEqualTo(1L); assertThat(agentTask1Return.getBatch()).isEqualTo(1); assertThat(agentTask1Return.getHostId()).isEqualTo(101L); assertThat(agentTask1Return.getAgentId()).isEqualTo("0:127.0.0.1"); @@ -136,9 +139,10 @@ public void testBatchSaveAgentTasks() { assertThat(agentTask1Return.getExitCode()).isEqualTo(1); - AgentTaskDTO agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 0, 1, 102L); + AgentTaskDTO agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, 102L); assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(100L); - assertThat(agentTask2Return.getExecuteCount()).isEqualTo(0L); + assertThat(agentTask2Return.getExecuteCount()).isEqualTo(1L); + assertThat(agentTask2Return.getActualExecuteCount()).isEqualTo(1L); assertThat(agentTask2Return.getBatch()).isEqualTo(1); assertThat(agentTask2Return.getHostId()).isEqualTo(102L); assertThat(agentTask2Return.getGseTaskId()).isEqualTo(1001L); diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java index c72fac7d69..fe5bc7f5d4 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java @@ -70,7 +70,7 @@ void queryRollingTask() { @Test @DisplayName("根据步骤实例ID查询步骤滚动任务") void listRollingTasks() { - List rollingTasks = stepInstanceRollingTaskDAO.listRollingTasks(1L); + List rollingTasks = stepInstanceRollingTaskDAO.listRollingTasks(1L, null, null); assertThat(rollingTasks).hasSize(2); StepInstanceRollingTaskDTO rollingTask1 = rollingTasks.get(0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql index 386ec711fb..a659b2740e 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql @@ -149,15 +149,15 @@ CREATE TABLE IF NOT EXISTS `step_instance_confirm` CREATE TABLE IF NOT EXISTS `gse_task_log` ( - `step_instance_id` bigint(20) NOT NULL DEFAULT '0', - `execute_count` int(11) NOT NULL DEFAULT '0', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(11) DEFAULT NULL, - `status` tinyint(4) DEFAULT '1', - `gse_task_id` varchar(64) DEFAULT NULL, - `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `step_instance_id` bigint(20) NOT NULL DEFAULT '0', + `execute_count` int(11) NOT NULL DEFAULT '0', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(11) DEFAULT NULL, + `status` tinyint(4) DEFAULT '1', + `gse_task_id` varchar(64) DEFAULT NULL, + `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`step_instance_id`, `execute_count`) ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; @@ -188,66 +188,71 @@ CREATE TABLE IF NOT EXISTS `gse_task_ip_log` DEFAULT CHARSET = utf8mb4; CREATE TABLE IF NOT EXISTS `gse_task` - ( +( `id` bigint(20) NOT NULL AUTO_INCREMENT, `step_instance_id` bigint(20) NOT NULL DEFAULT '0', - `execute_count` int(11) NOT NULL DEFAULT '0', + `execute_count` smallint(6) NOT NULL DEFAULT '0', `batch` smallint(6) NOT NULL DEFAULT '0', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(11) DEFAULT NULL, - `status` tinyint(4) DEFAULT '1', - `gse_task_id` varchar(64) DEFAULT NULL, + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(11) DEFAULT NULL, + `status` tinyint(4) DEFAULT '1', + `gse_task_id` varchar(64) DEFAULT NULL, `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), - UNIQUE KEY (`step_instance_id`,`execute_count`,`batch`), + UNIQUE KEY (`step_instance_id`, `execute_count`, `batch`), KEY (`gse_task_id`) - ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; CREATE TABLE IF NOT EXISTS `gse_script_agent_task` - ( - `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, - `step_instance_id` bigint(20) NOT NULL, - `execute_count` int(11) NOT NULL DEFAULT '0', - `batch` smallint(6) NOT NULL DEFAULT '0', - `host_id` bigint(20) NOT NULL DEFAULT '0', - `agent_id` varchar(64) NOT NULL DEFAULT '', - `gse_task_id` bigint(20) NOT NULL DEFAULT '0', - `status` int(11) DEFAULT '1', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(20) DEFAULT NULL, - `error_code` int(11) DEFAULT '0', - `exit_code` int(11) DEFAULT NULL, - `tag` varchar(256) DEFAULT '', - `log_offset` int(11) NOT NULL DEFAULT '0', - `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - UNIQUE KEY (`step_instance_id`,`execute_count`,`batch`,`host_id`), - KEY (`step_instance_id`,`host_id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +( + `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, + `step_instance_id` bigint(20) NOT NULL, + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) DEFAULT NULL, + `batch` smallint(6) NOT NULL DEFAULT '0', + `host_id` bigint(20) NOT NULL DEFAULT '0', + `agent_id` varchar(64) NOT NULL DEFAULT '', + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', + `status` int(11) DEFAULT '1', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `error_code` int(11) DEFAULT '0', + `exit_code` int(11) DEFAULT NULL, + `tag` varchar(256) DEFAULT '', + `log_offset` int(11) NOT NULL DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY (`step_instance_id`, `execute_count`, `batch`, `host_id`), + KEY (`step_instance_id`, `host_id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; CREATE TABLE IF NOT EXISTS `gse_file_agent_task` - ( - `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, - `step_instance_id` bigint(20) NOT NULL, - `execute_count` int(11) NOT NULL DEFAULT '0', - `batch` smallint(6) NOT NULL DEFAULT '0', - `host_id` bigint(20) NOT NULL DEFAULT '0', - `agent_id` varchar(64) NOT NULL DEFAULT '', - `mode` tinyint(1) NOT NULL, - `gse_task_id` bigint(20) NOT NULL DEFAULT '0', - `status` int(11) DEFAULT '1', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(20) DEFAULT NULL, - `error_code` int(11) DEFAULT '0', - `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - UNIQUE KEY (`step_instance_id`,`execute_count`,`batch`,`mode`,`host_id`), - KEY (`step_instance_id`,`host_id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +( + `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, + `step_instance_id` bigint(20) NOT NULL, + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) DEFAULT NULL, + `batch` smallint(6) NOT NULL DEFAULT '0', + `host_id` bigint(20) NOT NULL DEFAULT '0', + `agent_id` varchar(64) NOT NULL DEFAULT '', + `mode` tinyint(1) NOT NULL, + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', + `status` int(11) DEFAULT '1', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `error_code` int(11) DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY (`step_instance_id`, `execute_count`, `batch`, `mode`, `host_id`), + KEY (`step_instance_id`, `host_id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; CREATE TABLE IF NOT EXISTS `operation_log` ( @@ -339,16 +344,16 @@ CREATE TABLE IF NOT EXISTS `rolling_config` CREATE TABLE IF NOT EXISTS `step_instance_rolling_task` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `step_instance_id` bigint(20) NOT NULL DEFAULT '0', - `batch` smallint(6) NOT NULL DEFAULT '0', - `execute_count` tinyint(4) NOT NULL DEFAULT '0', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(11) DEFAULT NULL, - `status` tinyint(4) NOT NULL DEFAULT '1', - `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `step_instance_id` bigint(20) NOT NULL DEFAULT '0', + `batch` smallint(6) NOT NULL DEFAULT '0', + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(11) DEFAULT NULL, + `status` tinyint(4) NOT NULL DEFAULT '1', + `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY (`step_instance_id`, `batch`, `execute_count`) ) ENGINE = InnoDB diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java index de5171536a..55ca673d2a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java @@ -909,10 +909,11 @@ public Response> listStepExecutionHistory(String use AppResourceScope appResourceScope, String scopeType, String scopeId, - Long stepInstanceId) { + Long stepInstanceId, + Integer batch) { List stepExecutionRecords = taskResultService.listStepExecutionHistory(username, - appResourceScope.getAppId(), stepInstanceId); + appResourceScope.getAppId(), stepInstanceId, batch); return Response.buildSuccessResp(stepExecutionRecords.stream().map(stepExecutionRecord -> { StepExecutionRecordVO vo = new StepExecutionRecordVO(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java index 6b892849c8..ce448fe37f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java @@ -414,6 +414,7 @@ public Response> getTaskInstanceOperationLog(String use vo.setStepInstanceId(detail.getStepInstanceId()); vo.setStepName(detail.getStepName()); vo.setRetry(detail.getExecuteCount()); + vo.setBatch(detail.getBatch()); vo.setDetail(buildDetail(operationLog.getOperationEnum(), operationLog.getDetail())); vos.add(vo); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java index 92d3acb959..51bdc7914f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java @@ -139,21 +139,19 @@ AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Int FileTaskModeEnum mode, long hostId); /** - * 获取Agent任务实际执行成功的executeCount值(重试场景) + * 判断步骤实例的Agent Task 记录是否存在 * * @param stepInstanceId 步骤实例ID - * @param batch 滚动执行批次;如果传入null或者0,忽略该参数 - * @param mode 文件分发任务模式 - * @param hostId 主机ID - * @return Agent任务实际执行成功的executeCount值 */ - int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, FileTaskModeEnum mode, long hostId); + boolean isStepInstanceRecordExist(long stepInstanceId); /** - * 判断步骤实例的Agent Task 记录是否存在 + * 更新Agent任务实际执行的步骤重试次数 * - * @param stepInstanceId 步骤实例ID + * @param stepInstanceId 步骤实例ID + * @param batch 滚动执行批次;传入null将忽略该条件 + * @param actualExecuteCount Agent任务实际执行的步骤重试次数 */ - boolean isStepInstanceRecordExist(long stepInstanceId); + void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java index b0e9e424ad..cbe7a7e1fa 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java @@ -138,21 +138,22 @@ List listAgentTasks(Long stepInstanceId, AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, long hostId); /** - * 获取Agent任务实际执行成功的executeCount值(重试场景) + * 判断步骤实例的Agent Task 记录是否存在 * * @param stepInstanceId 步骤实例ID - * @param batch 滚动执行批次;如果传入null或者0,忽略该参数 - * @param hostId 主机ID - * @return Agent任务实际执行成功的executeCount值 */ - int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, long hostId); + boolean isStepInstanceRecordExist(long stepInstanceId); /** - * 判断步骤实例的Agent Task 记录是否存在 + * 更新Agent任务实际执行的步骤重试次数 * - * @param stepInstanceId 步骤实例ID + * @param stepInstanceId 步骤实例ID + * @param batch 滚动执行批次;传入null将忽略该条件 + * @param actualExecuteCount Agent任务实际执行的步骤重试次数 */ - boolean isStepInstanceRecordExist(long stepInstanceId); + void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount); + + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java index 7c3c229fac..d7a0f9f3e6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java @@ -50,9 +50,13 @@ StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, * 查询步骤滚动任务 * * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数;如果不为null,会根据executeCount过滤 + * @param batch 滚动执行批次;如果不为null,会根据batch过滤 * @return 步骤滚动任务 */ - List listRollingTasks(long stepInstanceId); + List listRollingTasks(long stepInstanceId, + Integer executeCount, + Integer batch); /** * 保存步骤滚动任务 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java index 6856c428cc..d46ed8def2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java @@ -41,7 +41,9 @@ import org.jooq.SelectLimitPercentStep; import org.jooq.SelectSeekStep1; import org.jooq.TableField; +import org.jooq.UpdateConditionStep; import org.jooq.generated.tables.GseFileAgentTask; +import org.jooq.generated.tables.records.GseFileAgentTaskRecord; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; @@ -60,6 +62,7 @@ public class FileAgentTaskDAOImpl implements FileAgentTaskDAO { private static final TableField[] ALL_FIELDS = { T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID, T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT, + T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT, T_GSE_FILE_AGENT_TASK.BATCH, T_GSE_FILE_AGENT_TASK.MODE, T_GSE_FILE_AGENT_TASK.HOST_ID, @@ -81,25 +84,26 @@ public FileAgentTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX @Override public void batchSaveAgentTasks(Collection agentTasks) { - String sql = "insert into gse_file_agent_task (step_instance_id, execute_count, batch, mode, host_id, agent_id" - + " ,gse_task_id,status, start_time, end_time, total_time, error_code)" - + " values (?,?,?,?,?,?,?,?,?,?,?,?)"; - Object[][] params = new Object[agentTasks.size()][12]; + String sql = "insert into gse_file_agent_task (step_instance_id, execute_count, actual_execute_count, batch," + + "mode, host_id, agent_id ,gse_task_id,status, start_time, end_time, total_time, error_code)" + + " values (?,?,?,?,?,?,?,?,?,?,?,?,?)"; + Object[][] params = new Object[agentTasks.size()][13]; int batchCount = 0; for (AgentTaskDTO agentTask : agentTasks) { - Object[] param = new Object[12]; + Object[] param = new Object[13]; param[0] = agentTask.getStepInstanceId(); param[1] = agentTask.getExecuteCount(); - param[2] = agentTask.getBatch(); - param[3] = agentTask.getFileTaskMode().getValue(); - param[4] = agentTask.getHostId(); - param[5] = agentTask.getAgentId(); - param[6] = agentTask.getGseTaskId(); - param[7] = agentTask.getStatus(); - param[8] = agentTask.getStartTime(); - param[9] = agentTask.getEndTime(); - param[10] = agentTask.getTotalTime(); - param[11] = agentTask.getErrorCode(); + param[2] = agentTask.getActualExecuteCount(); + param[3] = agentTask.getBatch(); + param[4] = agentTask.getFileTaskMode().getValue(); + param[5] = agentTask.getHostId(); + param[6] = agentTask.getAgentId(); + param[7] = agentTask.getGseTaskId(); + param[8] = agentTask.getStatus(); + param[9] = agentTask.getStartTime(); + param[10] = agentTask.getEndTime(); + param[11] = agentTask.getTotalTime(); + param[12] = agentTask.getErrorCode(); params[batchCount++] = param; } CTX.batch(sql, params).execute(); @@ -137,9 +141,10 @@ public void batchUpdateAgentTasks(Collection agentTasks) { public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { Integer count = CTX.selectCount() .from(T_GSE_FILE_AGENT_TASK) - .where(T_GSE_FILE_AGENT_TASK.STATUS.in(AgentTaskStatus.LAST_SUCCESS.getValue(), AgentTaskStatus.SUCCESS.getValue())) + .where(T_GSE_FILE_AGENT_TASK.STATUS.in(AgentTaskStatus.LAST_SUCCESS.getValue(), + AgentTaskStatus.SUCCESS.getValue())) .and(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())) .fetchOne(0, Integer.class); return count == null ? 0 : count; @@ -151,7 +156,7 @@ public List listResultGroups(long stepInstanceId, i CTX.select(T_GSE_FILE_AGENT_TASK.STATUS, count().as("ip_count")) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); @@ -181,7 +186,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_FILE_AGENT_TASK.STATUS.eq(status)) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); if (batch != null && batch > 0) { @@ -206,7 +211,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, Order order) { List conditions = new ArrayList<>(); conditions.add(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); - conditions.add(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)); + conditions.add(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); conditions.add(T_GSE_FILE_AGENT_TASK.STATUS.eq(status)); conditions.add(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); @@ -271,7 +276,7 @@ public List listAgentTasks(Long stepInstanceId, SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)); + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -296,6 +301,8 @@ private AgentTaskDTO extract(Record record) { AgentTaskDTO agentTask = new AgentTaskDTO(); agentTask.setStepInstanceId(record.get(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID)); agentTask.setExecuteCount(record.get(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT)); + Short actualExecuteCount = record.get(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT); + agentTask.setActualExecuteCount(actualExecuteCount != null ? actualExecuteCount.intValue() : null); agentTask.setBatch(record.get(T_GSE_FILE_AGENT_TASK.BATCH)); agentTask.setFileTaskMode(FileTaskModeEnum.getFileTaskMode(record.get(T_GSE_FILE_AGENT_TASK.MODE).intValue())); agentTask.setHostId(record.get(T_GSE_FILE_AGENT_TASK.HOST_ID)); @@ -329,7 +336,7 @@ public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCou Record record = CTX.select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch == null ? 0 : batch.shortValue())) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(mode.getValue().byteValue())) .and(T_GSE_FILE_AGENT_TASK.HOST_ID.eq(hostId)) @@ -338,26 +345,19 @@ public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCou } @Override - public int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, FileTaskModeEnum mode, long hostId) { - Record record = CTX.select(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT) - .from(T_GSE_FILE_AGENT_TASK) - .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch == null ? 0 : batch.shortValue())) - .and(T_GSE_FILE_AGENT_TASK.HOST_ID.eq(hostId)) - .and(T_GSE_FILE_AGENT_TASK.STATUS.eq(AgentTaskStatus.SUCCESS.getValue())) - .and(T_GSE_FILE_AGENT_TASK.MODE.eq(mode.getValue().byteValue())) - .orderBy(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.desc()) - .limit(1) - .fetchOne(); - if (record != null && record.size() > 0) { - return record.getValue(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT); - } else { - return 0; - } + public boolean isStepInstanceRecordExist(long stepInstanceId) { + return CTX.fetchExists(T_GSE_FILE_AGENT_TASK, T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); } @Override - public boolean isStepInstanceRecordExist(long stepInstanceId) { - return CTX.fetchExists(T_GSE_FILE_AGENT_TASK, T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); + public void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount) { + UpdateConditionStep updateConditionStep = + CTX.update(T_GSE_FILE_AGENT_TASK) + .set(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT, (short) actualExecuteCount) + .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); + if (batch != null) { + updateConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + updateConditionStep.execute(); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java index a29179a7fb..a5856695ca 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java @@ -57,7 +57,7 @@ private GseTaskDTO extractInfo(Record record) { gseTaskDTO.setId(record.get(TABLE.ID)); gseTaskDTO.setStepInstanceId(record.get(TABLE.STEP_INSTANCE_ID)); - gseTaskDTO.setExecuteCount(record.get(TABLE.EXECUTE_COUNT)); + gseTaskDTO.setExecuteCount(record.get(TABLE.EXECUTE_COUNT).intValue()); gseTaskDTO.setBatch(record.get(TABLE.BATCH)); gseTaskDTO.setStartTime(record.get(TABLE.START_TIME)); gseTaskDTO.setEndTime(record.get(TABLE.END_TIME)); @@ -81,7 +81,7 @@ public long saveGseTask(GseTaskDTO gseTask) { TABLE.GSE_TASK_ID) .values( gseTask.getStepInstanceId(), - gseTask.getExecuteCount(), + gseTask.getExecuteCount().shortValue(), (short) gseTask.getBatch(), gseTask.getStartTime(), gseTask.getEndTime(), @@ -111,7 +111,7 @@ public long saveGseTask(GseTaskDTO gseTask) { public GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch) { Record record = dslContext.select(ALL_FIELDS).from(TABLE) .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(TABLE.EXECUTE_COUNT.eq(executeCount)) + .and(TABLE.EXECUTE_COUNT.eq((short)executeCount)) .and(TABLE.BATCH.eq(batch == null ? 0 : batch.shortValue())) .fetchOne(); return extractInfo(record); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java index 626ca49549..1fe6ec3931 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java @@ -41,7 +41,9 @@ import org.jooq.SelectLimitPercentStep; import org.jooq.SelectSeekStep1; import org.jooq.TableField; +import org.jooq.UpdateConditionStep; import org.jooq.generated.tables.GseScriptAgentTask; +import org.jooq.generated.tables.records.GseScriptAgentTaskRecord; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; @@ -60,6 +62,7 @@ public class ScriptAgentTaskDAOImpl implements ScriptAgentTaskDAO { private static final TableField[] ALL_FIELDS = { T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID, T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT, + T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT, T_GSE_SCRIPT_AGENT_TASK.BATCH, T_GSE_SCRIPT_AGENT_TASK.HOST_ID, T_GSE_SCRIPT_AGENT_TASK.AGENT_ID, @@ -83,27 +86,29 @@ public ScriptAgentTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext C @Override public void batchSaveAgentTasks(Collection agentTasks) { - String sql = "insert into gse_script_agent_task (step_instance_id, execute_count, batch, host_id, agent_id, " - + "gse_task_id, status, start_time, end_time, total_time, error_code, exit_code, tag, log_offset)" - + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - Object[][] params = new Object[agentTasks.size()][14]; + String sql = "insert into gse_script_agent_task (step_instance_id, execute_count, actual_execute_count, batch," + + " host_id, agent_id, gse_task_id, status, start_time, end_time, total_time, error_code, exit_code, tag," + + " log_offset)" + + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + Object[][] params = new Object[agentTasks.size()][15]; int batchCount = 0; for (AgentTaskDTO agentTask : agentTasks) { - Object[] param = new Object[14]; + Object[] param = new Object[15]; param[0] = agentTask.getStepInstanceId(); param[1] = agentTask.getExecuteCount(); - param[2] = agentTask.getBatch(); - param[3] = agentTask.getHostId(); - param[4] = agentTask.getAgentId(); - param[5] = agentTask.getGseTaskId(); - param[6] = agentTask.getStatus(); - param[7] = agentTask.getStartTime(); - param[8] = agentTask.getEndTime(); - param[9] = agentTask.getTotalTime(); - param[10] = agentTask.getErrorCode(); - param[11] = agentTask.getExitCode(); - param[12] = StringUtils.truncate(agentTask.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); - param[13] = agentTask.getScriptLogOffset(); + param[2] = agentTask.getActualExecuteCount(); + param[3] = agentTask.getBatch(); + param[4] = agentTask.getHostId(); + param[5] = agentTask.getAgentId(); + param[6] = agentTask.getGseTaskId(); + param[7] = agentTask.getStatus(); + param[8] = agentTask.getStartTime(); + param[9] = agentTask.getEndTime(); + param[10] = agentTask.getTotalTime(); + param[11] = agentTask.getErrorCode(); + param[12] = agentTask.getExitCode(); + param[13] = StringUtils.truncate(agentTask.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); + param[14] = agentTask.getScriptLogOffset(); params[batchCount++] = param; } CTX.batch(sql, params).execute(); @@ -146,7 +151,7 @@ public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { .where(T_GSE_SCRIPT_AGENT_TASK.STATUS.in(AgentTaskStatus.LAST_SUCCESS.getValue(), AgentTaskStatus.SUCCESS.getValue())) .and(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) .fetchOne(0, Integer.class); return count == null ? 0 : count; } @@ -157,7 +162,7 @@ public List listResultGroups(long stepInstanceId, i CTX.select(T_GSE_SCRIPT_AGENT_TASK.STATUS, T_GSE_SCRIPT_AGENT_TASK.TAG, count().as("ip_count")) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)); + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -187,7 +192,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_SCRIPT_AGENT_TASK.STATUS.eq(status)) .and(T_GSE_SCRIPT_AGENT_TASK.TAG.eq(tag == null ? "" : tag)); if (batch != null && batch > 0) { @@ -213,7 +218,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, Order order) { List conditions = new ArrayList<>(); conditions.add(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); - conditions.add(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)); + conditions.add(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); conditions.add(T_GSE_SCRIPT_AGENT_TASK.STATUS.eq(status)); conditions.add(T_GSE_SCRIPT_AGENT_TASK.TAG.eq(tag == null ? "" : tag)); @@ -283,7 +288,7 @@ public List listAgentTasks(Long stepInstanceId, SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)); + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -305,6 +310,8 @@ private AgentTaskDTO extract(Record record) { AgentTaskDTO agentTask = new AgentTaskDTO(); agentTask.setStepInstanceId(record.get(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID)); agentTask.setExecuteCount(record.get(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT)); + Short actualExecuteCount = record.get(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT); + agentTask.setActualExecuteCount(actualExecuteCount != null ? actualExecuteCount.intValue() : null); agentTask.setBatch(record.get(T_GSE_SCRIPT_AGENT_TASK.BATCH)); agentTask.setHostId(record.get(T_GSE_SCRIPT_AGENT_TASK.HOST_ID)); agentTask.setAgentId(record.get(T_GSE_SCRIPT_AGENT_TASK.AGENT_ID)); @@ -339,7 +346,7 @@ public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCou Record record = CTX.select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch == null ? 0 : batch.shortValue())) .and(T_GSE_SCRIPT_AGENT_TASK.HOST_ID.eq(hostId)) .fetchOne(); @@ -347,25 +354,19 @@ public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCou } @Override - public int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, long hostId) { - Record record = CTX.select(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT) - .from(T_GSE_SCRIPT_AGENT_TASK) - .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch == null ? 0 : batch.shortValue())) - .and(T_GSE_SCRIPT_AGENT_TASK.HOST_ID.eq(hostId)) - .and(T_GSE_SCRIPT_AGENT_TASK.STATUS.eq(AgentTaskStatus.SUCCESS.getValue())) - .orderBy(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.desc()) - .limit(1) - .fetchOne(); - if (record != null && record.size() > 0) { - return record.getValue(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT); - } else { - return 0; - } + public boolean isStepInstanceRecordExist(long stepInstanceId) { + return CTX.fetchExists(T_GSE_SCRIPT_AGENT_TASK, T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); } @Override - public boolean isStepInstanceRecordExist(long stepInstanceId) { - return CTX.fetchExists(T_GSE_SCRIPT_AGENT_TASK, T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); + public void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount) { + UpdateConditionStep updateConditionStep = + CTX.update(T_GSE_SCRIPT_AGENT_TASK) + .set(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT, (short) actualExecuteCount) + .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); + if (batch != null) { + updateConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + updateConditionStep.execute(); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java index 7683d3f749..e446fb3b5c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java @@ -32,6 +32,7 @@ import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; +import org.jooq.SelectConditionStep; import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; import org.jooq.generated.tables.StepInstanceRollingTask; @@ -70,7 +71,7 @@ public StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, int exec Record record = CTX.select(ALL_FIELDS) .from(TABLE) .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toByte(executeCount))) + .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toShort(executeCount))) .and(TABLE.BATCH.eq(JooqDataTypeUtil.toShort(batch))) .fetchOne(); return extract(record); @@ -93,15 +94,24 @@ private StepInstanceRollingTaskDTO extract(Record record) { } @Override - public List listRollingTasks(long stepInstanceId) { - Result result = CTX.select(ALL_FIELDS) + public List listRollingTasks(long stepInstanceId, + Integer executeCount, + Integer batch) { + SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(TABLE) - .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) - .orderBy(TABLE.BATCH.asc()) - .fetch(); + .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)); + if (executeCount != null) { + selectConditionStep.and(TABLE.EXECUTE_COUNT.eq(executeCount.shortValue())); + } + if (batch != null && batch > 0) { + selectConditionStep.and(TABLE.BATCH.eq(batch.shortValue())); + } + + Result result = selectConditionStep.orderBy(TABLE.BATCH.asc()).fetch(); + List stepInstanceRollingTasks = new ArrayList<>(); if (result.size() > 0) { - result.into(record -> stepInstanceRollingTasks.add(extract(record))); + stepInstanceRollingTasks = result.map(this::extract); } return stepInstanceRollingTasks; } @@ -119,7 +129,7 @@ public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { TABLE.TOTAL_TIME) .values( rollingTask.getStepInstanceId(), - JooqDataTypeUtil.toByte(rollingTask.getExecuteCount()), + JooqDataTypeUtil.toShort(rollingTask.getExecuteCount()), JooqDataTypeUtil.toShort(rollingTask.getBatch()), JooqDataTypeUtil.toByte(rollingTask.getStatus()), rollingTask.getStartTime(), @@ -169,7 +179,7 @@ public void updateRollingTask(long stepInstanceId, return; } updateSetMoreStep.where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toByte(executeCount))) + .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toByte(executeCount).shortValue())) .and(TABLE.BATCH.eq(JooqDataTypeUtil.toShort(batch))) .execute(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java index 4a1a1e9d33..6432f424a5 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java @@ -24,7 +24,9 @@ package com.tencent.bk.job.execute.engine.listener; +import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.RollingModeEnum; +import com.tencent.bk.job.common.exception.NotImplementedException; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; @@ -186,7 +188,7 @@ private void startStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { } Long gseTaskId = saveInitialGseTask(stepInstance); - saveInitialGseAgentTasks(gseTaskId, stepInstance, rollingConfig); + saveGseAgentTasksForStartStep(gseTaskId, stepInstance, rollingConfig); taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.RUNNING, stepInstance.getStartTime() == null ? DateUtils.currentTimeMillis() : null, null, null); @@ -244,57 +246,82 @@ private Long saveInitialGseTask(StepInstanceDTO stepInstance) { } /** - * 初始化的GSE Agent 任务 + * 启动步骤的时候保存 GSE Agent 任务 * * @param gseTaskId GSE任务ID * @param stepInstance 步骤实例 * @param rollingConfig 滚动配置 */ - private void saveInitialGseAgentTasks(Long gseTaskId, - StepInstanceDTO stepInstance, - RollingConfigDTO rollingConfig) { - List agentTasks = new ArrayList<>(); - + private void saveGseAgentTasksForStartStep(Long gseTaskId, + StepInstanceDTO stepInstance, + RollingConfigDTO rollingConfig) { long stepInstanceId = stepInstance.getId(); int executeCount = stepInstance.getExecuteCount(); int batch = stepInstance.getBatch(); - if (stepInstance.isRollingStep()) { - if (rollingConfig.isBatchRollingStep(stepInstanceId) && stepInstance.isFirstRollingBatch()) { - List serverBatchList = rollingConfig.getConfigDetail().getServerBatchList(); - serverBatchList.forEach(serverBatch -> agentTasks.addAll(buildGseAgentTasks(stepInstanceId, - executeCount, serverBatch.getBatch(), gseTaskId, serverBatch.getServers(), - AgentTaskStatus.WAITING))); - } else if (rollingConfig.isAllRollingStep(stepInstanceId)) { - agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, batch, + if (needInitAgentTasksWhenStartStep(stepInstance)) { + List agentTasks = new ArrayList<>(); + + if (stepInstance.isRollingStep() && stepInstance.isFirstRollingBatch()) { + // 如果是第一批次的执行,需要初始化所有批次的agent任务(查询需要) + if (rollingConfig.isBatchRollingStep(stepInstanceId)) { + List serverBatchList = + rollingConfig.getConfigDetail().getServerBatchList(); + serverBatchList.forEach(serverBatch -> { + Integer actualExecuteCount = serverBatch.getBatch() == 1 ? executeCount : null; + agentTasks.addAll(buildGseAgentTasks(stepInstanceId, + executeCount, actualExecuteCount, serverBatch.getBatch(), gseTaskId, + serverBatch.getServers(), AgentTaskStatus.WAITING)); + }); + } else if (rollingConfig.isAllRollingStep(stepInstanceId)) { + // 暂时不支持,滚动执行二期需求 + log.warn("All rolling step is not supported!"); + throw new NotImplementedException("All rolling step is not supported", + ErrorCode.NOT_SUPPORT_FEATURE); + } + } else { + agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, executeCount, batch, gseTaskId, stepInstance.getTargetServers().getIpList(), AgentTaskStatus.WAITING)); } + + // 无效主机 + if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getInvalidIpList())) { + agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, executeCount, batch, + 0L, stepInstance.getTargetServers().getInvalidIpList(), AgentTaskStatus.HOST_NOT_EXIST)); + } + saveAgentTasks(stepInstance, agentTasks); } else { - agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, batch, - gseTaskId, stepInstance.getTargetServers().getIpList(), AgentTaskStatus.WAITING)); + // 仅更新 AgentTask 的 actualExecuteCount + if (stepInstance.isScriptStep()) { + scriptAgentTaskService.updateActualExecuteCount(stepInstanceId, batch, executeCount); + } else if (stepInstance.isFileStep()) { + fileAgentTaskService.updateActualExecuteCount(stepInstanceId, batch, executeCount); + } } - // 无效主机 - if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getInvalidIpList())) { - agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, batch, - 0L, stepInstance.getTargetServers().getInvalidIpList(), AgentTaskStatus.HOST_NOT_EXIST)); - } - saveAgentTasks(stepInstance, agentTasks); + } + + private boolean needInitAgentTasksWhenStartStep(StepInstanceBaseDTO stepInstance) { + // 步骤启动阶段(非滚动步骤启动、滚动步骤的第一批次启动),需要初始化所有AgentTask + return !stepInstance.isRollingStep() || stepInstance.isFirstRollingBatch(); } private List buildGseAgentTasks(long stepInstanceId, int executeCount, + Integer actualExecuteCount, int batch, Long gseTaskId, List hosts, AgentTaskStatus status) { return hosts.stream() - .map(host -> buildGseAgentTask(stepInstanceId, executeCount, batch, gseTaskId, host, status)) + .map(host -> buildGseAgentTask(stepInstanceId, executeCount, actualExecuteCount, + batch, gseTaskId, host, status)) .collect(Collectors.toList()); } protected AgentTaskDTO buildGseAgentTask(long stepInstanceId, int executeCount, + Integer actualExecuteCount, int batch, Long gseTaskId, HostDTO host, @@ -302,6 +329,7 @@ protected AgentTaskDTO buildGseAgentTask(long stepInstanceId, AgentTaskDTO agentTask = new AgentTaskDTO(); agentTask.setStepInstanceId(stepInstanceId); agentTask.setExecuteCount(executeCount); + agentTask.setActualExecuteCount(actualExecuteCount); agentTask.setBatch(batch); agentTask.setGseTaskId(gseTaskId); agentTask.setStatus(status.getValue()); @@ -380,11 +408,8 @@ private void ignoreError(StepInstanceDTO stepInstance) { stepInstanceRollingTaskService.queryRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch()); if (stepInstanceRollingTask != null) { - long endTime = stepInstanceRollingTask.getEndTime() != null ? - stepInstanceRollingTask.getEndTime() : System.currentTimeMillis(); - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR, null, endTime, - TaskCostCalculator.calculate(stepInstanceRollingTask.getStartTime(), endTime, null)); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.IGNORE_ERROR); } } @@ -506,7 +531,7 @@ private boolean isStepSupportRetry(RunStatusEnum stepStatus) { private void saveAgentTasksForRetryFail(StepInstanceBaseDTO stepInstance, int executeCount, Integer batch, Long gseTaskId) { - List latestAgentTasks = listAgentTasks(stepInstance, executeCount - 1, batch); + List latestAgentTasks = listAgentTasks(stepInstance, executeCount - 1); for (AgentTaskDTO latestAgentTask : latestAgentTasks) { latestAgentTask.setExecuteCount(executeCount); @@ -514,6 +539,7 @@ private void saveAgentTasksForRetryFail(StepInstanceBaseDTO stepInstance, int ex if (batch != null && latestAgentTask.getBatch() != batch) { continue; } + latestAgentTask.setActualExecuteCount(executeCount); latestAgentTask.resetTaskInitialStatus(); latestAgentTask.setGseTaskId(gseTaskId); } @@ -525,13 +551,14 @@ private void saveAgentTasksForRetryFail(StepInstanceBaseDTO stepInstance, int ex private void saveAgentTasksForRetryAll(StepInstanceBaseDTO stepInstance, int executeCount, Integer batch, Long gseTaskId) { - List latestAgentTasks = listAgentTasks(stepInstance, executeCount - 1, batch); + List latestAgentTasks = listAgentTasks(stepInstance, executeCount - 1); for (AgentTaskDTO latestAgentTask : latestAgentTasks) { latestAgentTask.setExecuteCount(executeCount); if (batch != null && latestAgentTask.getBatch() != batch) { continue; } + latestAgentTask.setActualExecuteCount(executeCount); latestAgentTask.resetTaskInitialStatus(); latestAgentTask.setGseTaskId(gseTaskId); } @@ -539,12 +566,12 @@ private void saveAgentTasksForRetryAll(StepInstanceBaseDTO stepInstance, int exe saveAgentTasks(stepInstance, latestAgentTasks); } - private List listAgentTasks(StepInstanceBaseDTO stepInstance, int executeCount, Integer batch) { + private List listAgentTasks(StepInstanceBaseDTO stepInstance, int executeCount) { List agentTasks = Collections.emptyList(); if (stepInstance.isScriptStep()) { - agentTasks = scriptAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, batch); + agentTasks = scriptAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, null); } else if (stepInstance.isFileStep()) { - agentTasks = fileAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, batch); + agentTasks = fileAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, null); } return agentTasks; } @@ -680,16 +707,13 @@ private void onSuccess(StepInstanceDTO stepInstance) { long totalTime = endTime - startTime; if (stepInstance.isRollingStep()) { - log.info("rolling step"); RollingConfigDTO rollingConfig = rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.SUCCESS, startTime, endTime, totalTime); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.SUCCESS); int totalBatch = rollingConfig.getConfigDetail().getTotalBatch(); - log.info("rollingConfig: {}, stepInstance: {}", rollingConfig, stepInstance); boolean isLastBatch = totalBatch == stepInstance.getBatch(); if (isLastBatch) { - log.info("updateStepExecutionInfo-SUCCESS"); taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, startTime, endTime, totalTime); // 步骤执行成功后清理产生的临时文件 @@ -697,10 +721,8 @@ private void onSuccess(StepInstanceDTO stepInstance) { } else { taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.ROLLING_WAITING, startTime, endTime, totalTime); - log.info("updateStepExecutionInfo-ROLLING_WAITING"); } } else { - log.info("not rolling step"); taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, startTime, endTime, totalTime); // 步骤执行成功后清理产生的临时文件 @@ -718,8 +740,8 @@ private void onFail(StepInstanceDTO stepInstance) { taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); if (stepInstance.isRollingStep()) { - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.IGNORE_ERROR); } return; } @@ -731,15 +753,15 @@ private void onFail(StepInstanceDTO stepInstance) { switch (rollingMode) { case IGNORE_ERROR: log.info("Ignore error for rolling step, rollingMode: {}", rollingMode); - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.IGNORE_ERROR); taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); break; case PAUSE_IF_FAIL: case MANUAL: - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.FAIL, startTime, endTime, totalTime); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.FAIL); taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.FAIL, startTime, endTime, totalTime); break; @@ -763,8 +785,8 @@ private void onStopSuccess(StepInstanceDTO stepInstance) { taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.STOP_SUCCESS, startTime, endTime, totalTime); if (stepInstance.isRollingStep()) { - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.STOP_SUCCESS, startTime, endTime, totalTime); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.STOP_SUCCESS); } } else { log.error("Refresh step fail, stepInstanceId: {}, stepStatus: {}, gseTaskStatus: {}", @@ -773,14 +795,10 @@ private void onStopSuccess(StepInstanceDTO stepInstance) { } private void onAbnormalState(StepInstanceDTO stepInstance) { - long stepInstanceId = stepInstance.getId(); - long endTime = System.currentTimeMillis(); - long startTime = stepInstance.getStartTime(); - long totalTime = endTime - startTime; setAbnormalStatusForStep(stepInstance); if (stepInstance.isRollingStep()) { - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.ABNORMAL_STATE, startTime, endTime, totalTime); + finishRollingTask(stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.ABNORMAL_STATE); } } @@ -804,4 +822,19 @@ private void setAbnormalStatusForStep(StepInstanceDTO stepInstance) { ); } } + + private void finishRollingTask(long stepInstanceId, int executeCount, int batch, RunStatusEnum status) { + StepInstanceRollingTaskDTO rollingTask = + stepInstanceRollingTaskService.queryRollingTask(stepInstanceId, executeCount, batch); + if (rollingTask == null) { + log.error("Rolling task is not exist, skip update! stepInstanceId: {}, executeCount: {}, batch: {}", + stepInstanceId, executeCount, batch); + return; + } + long now = System.currentTimeMillis(); + long startTime = rollingTask.getStartTime() != null ? rollingTask.getStartTime() : now; + + stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, executeCount, + batch, status, startTime, now, now - startTime); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java index 6f6cbd88e5..8876cfacf9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java @@ -204,10 +204,8 @@ private void refreshJob(TaskInstanceDTO taskInstance) { // 验证作业状态,只有'正在执行'、'强制终止中'的作业可以刷新状态进入下一步或者结束 if (RunStatusEnum.STOPPING.getValue() == taskStatus) { - if (RunStatusEnum.STOP_SUCCESS.getValue() == stepStatus - || RunStatusEnum.SUCCESS.getValue() == stepStatus - || RunStatusEnum.FAIL.getValue() == stepStatus) { - + // 非正在执行的步骤可以直接终止 + if (RunStatusEnum.RUNNING.getValue() != stepStatus) { finishJob(taskInstance, currentStepInstance, RunStatusEnum.STOP_SUCCESS); } else { log.error("Unsupported task instance run status for refresh task, taskInstanceId={}, taskStatus={}, " + diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java index 02592a3ada..fe1118cfaf 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java @@ -50,6 +50,10 @@ public class AgentTaskDTO { * 步骤执行次数 */ private int executeCount; + /** + * Agent 任务对应的实际的步骤执行次数(重试场景,可能Agent任务并没有实际被执行) + */ + private Integer actualExecuteCount; /** * 滚动执行批次 */ @@ -139,6 +143,7 @@ public AgentTaskDTO(long stepInstanceId, int executeCount, int batch, FileTaskMo public AgentTaskDTO(AgentTaskDTO agentTask) { this.stepInstanceId = agentTask.getStepInstanceId(); this.executeCount = agentTask.getExecuteCount(); + this.actualExecuteCount = agentTask.getActualExecuteCount(); this.batch = agentTask.getBatch(); this.fileTaskMode = agentTask.getFileTaskMode(); this.hostId = agentTask.getHostId(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java index e3d597639e..c280311af6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.model; import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.execute.constants.UserOperationEnum; import lombok.Data; @@ -43,10 +44,12 @@ public class OperationLogDTO { */ @Data @JsonInclude(JsonInclude.Include.NON_NULL) + @PersistenceObject public static class OperationDetail { private Long taskInstanceId; private Long stepInstanceId; private Integer executeCount; + private Integer batch; private String stepName; private Integer startupMode; private String appCode; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java index 8d380fc0dc..5bffb4de6d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.service; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.execute.model.AgentTaskDTO; import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; @@ -143,5 +144,24 @@ List listAgentTaskDetail(StepInstanceBaseDTO stepInstance, Integer executeCount, Integer batch); + /** + * 获取Agent任务实际执行成功的executeCount值(重试场景,兼容历史数据) + * + * @param stepInstanceId 步骤实例ID + * @param cloudIp 云区域+ip + * @return Agent任务实际执行成功的executeCount值 + */ + @CompatibleImplementation(name = "rolling_execution", explain = "兼容历史数据", version = "3.7.x") + int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp); + + /** + * 更新Agent任务实际执行的步骤重试次数 + * + * @param stepInstanceId 步骤实例ID + * @param batch 滚动执行批次;传入null将忽略该条件 + * @param actualExecuteCount Agent任务实际执行的步骤重试次数 + */ + void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount); + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java index 8baeba64ba..5d2c7c6bd2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java @@ -63,16 +63,4 @@ AgentTaskDTO getAgentTaskByHost(StepInstanceDTO stepInstance, Integer batch, FileTaskModeEnum fileTaskMode, HostDTO host); - - - /** - * 获取Agent任务实际执行成功的executeCount值(重试场景) - * - * @param stepInstanceId 步骤实例ID - * @param batch 滚动执行批次;如果传入null或者0,忽略该参数 - * @param mode 文件分发任务模式 - * @param host 主机 - * @return Agent任务实际执行成功的executeCount值 - */ - int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, FileTaskModeEnum mode, HostDTO host); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java index 7571016d0c..67cb8cfa34 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java @@ -45,14 +45,4 @@ AgentTaskDTO getAgentTaskByHost(StepInstanceBaseDTO stepInstance, Integer executeCount, Integer batch, HostDTO host); - - /** - * 获取Agent任务实际执行成功的executeCount值(重试场景) - * - * @param stepInstanceId 步骤实例ID - * @param batch 滚动执行批次;如果传入null或者0,忽略该参数 - * @param host 主机 - * @return Agent任务实际执行成功的executeCount值 - */ - int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, HostDTO host); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java index 3562b641f8..3bdce59ac7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java @@ -49,12 +49,21 @@ StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, /** * 查询最新的步骤滚动任务 * - * @param stepInstanceId 步骤实例ID - * @param executeCount 步骤执行次数 + * @param stepInstanceId 步骤实例ID + * @param executeCount 步骤执行次数 * @return 步骤滚动任务 */ List listLatestRollingTasks(long stepInstanceId, int executeCount); + /** + * 根据批次查询滚动任务 + * + * @param stepInstanceId 步骤实例ID + * @param batch 滚动批次 + * @return 步骤滚动任务 + */ + List listRollingTasksByBatch(long stepInstanceId, Integer batch); + /** * 保存步骤滚动任务 * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java index 7e7713db1b..fae4360820 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java @@ -35,14 +35,14 @@ public interface TaskOperationLogService { /** * 保存用户操作日志 * - * @param operationLog + * @param operationLog 操作日志 */ long saveOperationLog(OperationLogDTO operationLog); /** * 获取用户操作日志 * - * @param taskInstanceId + * @param taskInstanceId 作业实例ID */ List listOperationLog(long taskInstanceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java index ed7a10c2e5..92ba34f03b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java @@ -125,8 +125,12 @@ List getHostsByResultType(String username, * @param username 用户名 * @param appId 业务ID * @param stepInstanceId 步骤实例ID + * @param batch 滚动执行批次;如果传入,则会按照batch过滤步骤执行历史 * @return 执行历史 */ - List listStepExecutionHistory(String username, Long appId, Long stepInstanceId); + List listStepExecutionHistory(String username, + Long appId, + Long stepInstanceId, + Integer batch); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java index 954fbbce6d..bcaafdb676 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java @@ -230,13 +230,9 @@ public List listAgentTasks(Long stepInstanceId, Integer executeCou } @Override - public int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, FileTaskModeEnum mode, HostDTO host) { - if (isStepInstanceRecordExist(stepInstanceId)) { - return fileAgentTaskDAO.getActualSuccessExecuteCount(stepInstanceId, batch, mode, host.getHostId()); - } else { - // 兼容历史数据 - return gseTaskIpLogDAO.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); - } + public int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp) { + // 兼容历史数据 + return gseTaskIpLogDAO.getActualSuccessExecuteCount(stepInstanceId, cloudIp); } @Override @@ -250,4 +246,9 @@ public List listAgentTaskDetail(StepInstanceBaseDTO stepInst private boolean isStepInstanceRecordExist(long stepInstanceId) { return fileAgentTaskDAO.isStepInstanceRecordExist(stepInstanceId); } + + @Override + public void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount) { + fileAgentTaskDAO.updateActualExecuteCount(stepInstanceId, batch, actualExecuteCount); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java index 3bf488b760..7772f35638 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java @@ -146,26 +146,27 @@ public ScriptHostLogContent getScriptHostLogContent(long stepInstanceId, int exe // 如果存在重试,那么该ip可能是之前已经执行过的,查询日志的时候需要获取到对应的executeCount int actualExecuteCount = executeCount; AgentTaskDTO agentTask = scriptAgentTaskService.getAgentTaskByHost(stepInstance, executeCount, batch, host); - log.info("Get agent task by host, host: {}, agentTask:{}", host, agentTask); if (agentTask == null) { return null; } - if (agentTask.getStatus() == AgentTaskStatus.LAST_SUCCESS.getValue()) { - actualExecuteCount = scriptAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, - agentTask.getBatch(), host); + + if (executeCount > 0 && agentTask.getActualExecuteCount() != null) { + actualExecuteCount = agentTask.getActualExecuteCount(); + } else if (agentTask.getStatus() == AgentTaskStatus.LAST_SUCCESS.getValue()) { + // 兼容历史数据 + actualExecuteCount = scriptAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); } + String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC")); InternalResponse resp; if (agentTask.getHostId() != null) { resp = logServiceResourceClient.getScriptHostLogByHostId(taskCreateDateStr, stepInstanceId, actualExecuteCount, agentTask.getHostId(), batch); - log.info("Get log by hostId, resp: {}", resp); } else { // 兼容ip查询 resp = logServiceResourceClient.getScriptHostLogByIp(taskCreateDateStr, stepInstanceId, actualExecuteCount, agentTask.getCloudIp(), batch); - log.info("Get log by ip, resp: {}", resp); } if (!resp.isSuccess()) { log.error("Get script log content by host error, stepInstanceId={}, executeCount={}, batch={}, host={}", @@ -260,9 +261,11 @@ public FileIpLogContent getFileIpLogContent(long stepInstanceId, int executeCoun if (agentTask == null) { return null; } - if (agentTask.getStatus() == AgentTaskStatus.LAST_SUCCESS.getValue()) { - actualExecuteCount = fileAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, - agentTask.getBatch(), agentTask.getFileTaskMode(), host); + if (executeCount > 0 && agentTask.getActualExecuteCount() != null) { + actualExecuteCount = agentTask.getActualExecuteCount(); + } else if (agentTask.getStatus() == AgentTaskStatus.LAST_SUCCESS.getValue()) { + // 兼容历史数据 + actualExecuteCount = scriptAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); } String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java index 8f79d6f00f..24f8315bbf 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java @@ -138,13 +138,9 @@ public AgentTaskDTO getAgentTaskByHost(StepInstanceBaseDTO stepInstance, } @Override - public int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, HostDTO host) { - if (isStepInstanceRecordExist(stepInstanceId)) { - return scriptAgentTaskDAO.getActualSuccessExecuteCount(stepInstanceId, batch, host.getHostId()); - } else { - // 兼容历史数据 - return gseTaskIpLogDAO.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); - } + public int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp) { + // 兼容历史数据 + return gseTaskIpLogDAO.getActualSuccessExecuteCount(stepInstanceId, cloudIp); } @Override @@ -209,4 +205,9 @@ public List listAgentTaskDetail(StepInstanceBaseDTO stepInst private boolean isStepInstanceRecordExist(long stepInstanceId) { return scriptAgentTaskDAO.isStepInstanceRecordExist(stepInstanceId); } + + @Override + public void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount) { + scriptAgentTaskDAO.updateActualExecuteCount(stepInstanceId, batch, actualExecuteCount); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java index 760ada5959..d2cd2c3e56 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java @@ -60,7 +60,7 @@ public StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, @Override public List listLatestRollingTasks(long stepInstanceId, int executeCount) { List stepInstanceRollingTasks = - stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId); + stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId, null, null); if (CollectionUtils.isEmpty(stepInstanceRollingTasks)) { return stepInstanceRollingTasks; } @@ -91,6 +91,11 @@ public List listLatestRollingTasks(long stepInstance .collect(Collectors.toList()); } + @Override + public List listRollingTasksByBatch(long stepInstanceId, Integer batch) { + return stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId, null, batch); + } + @Override public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { return stepInstanceRollingTaskDAO.saveRollingTask(rollingTask); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index 6ba6c96c83..f1132121a3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -703,14 +703,35 @@ private void fillStepHostDetail(List stepInstanceList, ServiceL if (stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue())) { continue; } + // 目标主机设置主机详情 + fillTargetHostDetail(stepInstance, hostMap); + // 文件源设置主机详情 + fillFileSourceHostDetail(stepInstance, hostMap); + } + } + + private void fillTargetHostDetail(StepInstanceDTO stepInstance, Map hostMap) { + if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getStaticIpList())) { + stepInstance.getTargetServers().getStaticIpList() + .forEach(host -> fillHostDetail(host, hostMap)); + } + if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getIpList())) { stepInstance.getTargetServers().getIpList() .forEach(host -> fillHostDetail(host, hostMap)); - if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { - List fileSourceList = stepInstance.getFileSourceList(); - if (fileSourceList != null) { - for (FileSourceDTO fileSource : fileSourceList) { - ServersDTO servers = fileSource.getServers(); - if (servers != null && servers.getIpList() != null) { + } + } + + private void fillFileSourceHostDetail(StepInstanceDTO stepInstance, Map hostMap) { + if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { + List fileSourceList = stepInstance.getFileSourceList(); + if (fileSourceList != null) { + for (FileSourceDTO fileSource : fileSourceList) { + ServersDTO servers = fileSource.getServers(); + if (servers != null) { + if (CollectionUtils.isNotEmpty(servers.getStaticIpList())) { + servers.getStaticIpList().forEach(host -> fillHostDetail(host, hostMap)); + } + if (CollectionUtils.isNotEmpty(servers.getIpList())) { servers.getIpList().forEach(host -> fillHostDetail(host, hostMap)); } } @@ -1093,7 +1114,6 @@ private void authExecuteJobPlan(String username, long appId, ServiceTaskPlanDTO } } - AuthResult accountAuthResult = executeAuthService.batchAuthAccountExecutable( username, new AppResourceScope(appId), accountIds); @@ -1103,16 +1123,14 @@ private void authExecuteJobPlan(String username, long appId, ServiceTaskPlanDTO // 主机为空,无需对主机鉴权 authResult = accountAuthResult; } else { - AuthResult serverAuthResult = null; + AuthResult serverAuthResult; if (isDebugTask) { - // 鉴权调试 - + // 鉴权模板调试 serverAuthResult = executeAuthService.authDebugTemplate( username, new AppResourceScope(appId), plan.getTaskTemplateId(), authServers); } else { - // 鉴权执行方案 - + // 鉴权执行方案的执行 serverAuthResult = executeAuthService.authExecutePlan( username, new AppResourceScope(appId), plan.getTaskTemplateId(), plan.getId(), plan.getName(), authServers); @@ -1157,9 +1175,6 @@ private void authRedoJob(String username, long appId, TaskInstanceDTO taskInstan private StepExecuteTypeEnum getExecuteTypeFromTaskStepType(ServiceTaskStepDTO step) throws ServiceException { StepExecuteTypeEnum executeType = null; TaskStepTypeEnum stepType = TaskStepTypeEnum.valueOf(step.getType()); - if (stepType == null) { - throw new InternalException(ErrorCode.INTERNAL_ERROR); - } switch (stepType) { case SCRIPT: ScriptTypeEnum scriptType = ScriptTypeEnum.valueOf(step.getScriptStepInfo().getType()); @@ -2081,6 +2096,7 @@ private OperationLogDTO buildCommonStepOperationLog(StepInstanceDTO stepInstance taskDetail.setStepInstanceId(stepInstance.getId()); taskDetail.setStepName(stepInstance.getName()); taskDetail.setExecuteCount(stepInstance.getExecuteCount()); + taskDetail.setBatch(stepInstance.getBatch()); operationLog.setDetail(taskDetail); return operationLog; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java index 49590b2bfb..725ca06f66 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java @@ -963,30 +963,44 @@ public List getHostsByResultType(String username, @Override - public List listStepExecutionHistory(String username, Long appId, Long stepInstanceId) { + public List listStepExecutionHistory(String username, + Long appId, + Long stepInstanceId, + Integer batch) { StepInstanceBaseDTO stepInstance = checkGetStepExecutionDetail(username, appId, stepInstanceId); - int latestExecuteCount = stepInstance.getExecuteCount(); - if (latestExecuteCount == 0) { + + // 步骤没有重试执行过 + if (stepInstance.getExecuteCount() == 0) { StepExecutionRecordDTO record = new StepExecutionRecordDTO(); record.setStepInstanceId(stepInstanceId); - record.setRetryCount(latestExecuteCount); + record.setRetryCount(0); record.setCreateTime(stepInstance.getCreateTime()); return Collections.singletonList(record); } - List operationLogs = operationLogService.listOperationLog(stepInstance.getTaskInstanceId()); - List records = new ArrayList<>(); - if (CollectionUtils.isEmpty(operationLogs)) { - for (int executeCount = latestExecuteCount; executeCount >= 0; executeCount--) { - StepExecutionRecordDTO record = new StepExecutionRecordDTO(); - record.setStepInstanceId(stepInstanceId); - record.setRetryCount(executeCount); - record.setCreateTime(stepInstance.getCreateTime()); - records.add(record); - } - return records; + List records; + if (batch == null) { + // 获取步骤维度的重试记录 + records = queryStepRetryRecords(stepInstance); + } else { + // 获取滚动任务维度的重试记录 + records = queryStepRollingTaskRetryRecords(stepInstanceId, batch); } + records.sort(Comparator.comparingInt(StepExecutionRecordDTO::getRetryCount).reversed()); + + return records; + } + + private List queryStepRetryRecords(StepInstanceBaseDTO stepInstance) { + Long stepInstanceId = stepInstance.getId(); + // 步骤最新的重试次数 + int latestExecuteCount = stepInstance.getExecuteCount(); + List operationLogs = + operationLogService.listOperationLog(stepInstance.getTaskInstanceId()); + + + List records = new ArrayList<>(); Map executeCountAndCreateTimeMap = new HashMap<>(); operationLogs.forEach(opLog -> { UserOperationEnum operation = opLog.getOperationEnum(); @@ -1006,6 +1020,21 @@ public List listStepExecutionHistory(String username, Lo record.setCreateTime(executeCountAndCreateTimeMap.get(executeCount)); records.add(record); } + + return records; + } + + private List queryStepRollingTaskRetryRecords(long stepInstanceId, int batch) { + List records = new ArrayList<>(); + List rollingTasks = + stepInstanceRollingTaskService.listRollingTasksByBatch(stepInstanceId, batch); + rollingTasks.forEach(rollingTask -> { + StepExecutionRecordDTO record = new StepExecutionRecordDTO(); + record.setStepInstanceId(stepInstanceId); + record.setRetryCount(rollingTask.getExecuteCount()); + record.setCreateTime(rollingTask.getStartTime()); + records.add(record); + }); return records; } } diff --git a/support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.5.0.0_mysql.sql b/support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.6.0.0_mysql.sql similarity index 63% rename from support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.5.0.0_mysql.sql rename to support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.6.0.0_mysql.sql index 95b70c3a5f..b274fa9677 100644 --- a/support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.5.0.0_mysql.sql +++ b/support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.6.0.0_mysql.sql @@ -17,7 +17,7 @@ BEGIN ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `step_instance_id` bigint(20) NOT NULL DEFAULT '0', - `execute_count` int(11) NOT NULL DEFAULT '0', + `execute_count` smallint(6) NOT NULL DEFAULT '0', `batch` smallint(6) NOT NULL DEFAULT '0', `start_time` bigint(20) DEFAULT NULL, `end_time` bigint(20) DEFAULT NULL, @@ -35,23 +35,24 @@ BEGIN CREATE TABLE IF NOT EXISTS `gse_script_agent_task` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `step_instance_id` bigint(20) NOT NULL, - `execute_count` int(11) NOT NULL DEFAULT '0', - `batch` smallint(6) NOT NULL DEFAULT '0', - `host_id` bigint(20) NOT NULL DEFAULT '0', - `agent_id` varchar(64) NOT NULL DEFAULT '', - `gse_task_id` bigint(20) NOT NULL DEFAULT '0', - `status` int(11) DEFAULT '1', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(20) DEFAULT NULL, - `error_code` int(11) DEFAULT '0', - `exit_code` int(11) DEFAULT NULL, - `tag` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT '', - `log_offset` int(11) NOT NULL DEFAULT '0', - `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `step_instance_id` bigint(20) NOT NULL, + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) DEFAULT NULL, + `batch` smallint(6) NOT NULL DEFAULT '0', + `host_id` bigint(20) NOT NULL DEFAULT '0', + `agent_id` varchar(64) NOT NULL DEFAULT '', + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', + `status` int(11) DEFAULT '1', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `error_code` int(11) DEFAULT '0', + `exit_code` int(11) DEFAULT NULL, + `tag` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT '', + `log_offset` int(11) NOT NULL DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `uk_step_id_execute_count_batch_host_id` (`step_instance_id`,`execute_count`,`batch`,`host_id`), KEY `idx_step_id_host_id` (`step_instance_id`,`host_id`), @@ -62,21 +63,22 @@ BEGIN CREATE TABLE IF NOT EXISTS `gse_file_agent_task` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `step_instance_id` bigint(20) NOT NULL, - `execute_count` int(11) NOT NULL DEFAULT '0', - `batch` smallint(6) NOT NULL DEFAULT '0', - `host_id` bigint(20) NOT NULL DEFAULT '0', - `agent_id` varchar(64) NOT NULL DEFAULT '', - `mode` tinyint(1) NOT NULL, - `gse_task_id` bigint(20) NOT NULL DEFAULT '0', - `status` int(11) DEFAULT '1', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(20) DEFAULT NULL, - `error_code` int(11) DEFAULT '0', - `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `step_instance_id` bigint(20) NOT NULL, + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) DEFAULT NULL, + `batch` smallint(6) NOT NULL DEFAULT '0', + `host_id` bigint(20) NOT NULL DEFAULT '0', + `agent_id` varchar(64) NOT NULL DEFAULT '', + `mode` tinyint(1) NOT NULL, + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', + `status` int(11) DEFAULT '1', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `error_code` int(11) DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `uk_step_id_execute_count_batch_mode_host_id` (`step_instance_id`,`execute_count`,`batch`,`mode`,`host_id`), KEY `idx_step_id_host_id` (`step_instance_id`,`host_id`), @@ -102,7 +104,7 @@ BEGIN ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `step_instance_id` bigint(20) NOT NULL DEFAULT '0', - `execute_count` tinyint(4) NOT NULL DEFAULT '0', + `execute_count` smallint(6) NOT NULL DEFAULT '0', `batch` smallint(6) NOT NULL DEFAULT '0', `start_time` bigint(20) DEFAULT NULL, `end_time` bigint(20) DEFAULT NULL,