From 569dde45bb943c3fdf0d3cb490542770930f2574 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Thu, 7 Jul 2022 23:09:16 +0800 Subject: [PATCH 01/10] =?UTF-8?q?feature:=20=E5=8E=9F=E5=AD=90=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E8=83=BD=E5=8A=9B=E6=94=AF=E6=8C=81=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=20#446?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复步骤重试记录获取不正确导致执行详情显示错误的问题 --- .../web/WebTaskExecutionResultResource.java | 5 +- ...anceRollingTaskDAOImplIntegrationTest.java | 2 +- .../WebTaskExecutionResultResourceImpl.java | 5 +- .../dao/StepInstanceRollingTaskDAO.java | 6 +- .../impl/StepInstanceRollingTaskDAOImpl.java | 22 +++-- .../engine/listener/GseStepEventHandler.java | 10 +- .../bk/job/execute/model/OperationLogDTO.java | 3 + .../StepInstanceRollingTaskService.java | 13 ++- .../service/TaskOperationLogService.java | 4 +- .../execute/service/TaskResultService.java | 6 +- .../StepInstanceRollingTaskServiceImpl.java | 7 +- .../service/impl/TaskExecuteServiceImpl.java | 1 + .../service/impl/TaskResultServiceImpl.java | 93 +++++++++++++------ 13 files changed, 125 insertions(+), 52 deletions(-) diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java index c37b439b84..35948b606c 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java @@ -385,7 +385,10 @@ Response> listStepExecutionHistory( String scopeId, @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) @PathVariable("stepInstanceId") - Long stepInstanceId + Long stepInstanceId, + @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") + @RequestParam(value = "batch", required = false) + Integer batch ); } diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java index c72fac7d69..fe5bc7f5d4 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java @@ -70,7 +70,7 @@ void queryRollingTask() { @Test @DisplayName("根据步骤实例ID查询步骤滚动任务") void listRollingTasks() { - List rollingTasks = stepInstanceRollingTaskDAO.listRollingTasks(1L); + List rollingTasks = stepInstanceRollingTaskDAO.listRollingTasks(1L, null, null); assertThat(rollingTasks).hasSize(2); StepInstanceRollingTaskDTO rollingTask1 = rollingTasks.get(0); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java index de5171536a..55ca673d2a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java @@ -909,10 +909,11 @@ public Response> listStepExecutionHistory(String use AppResourceScope appResourceScope, String scopeType, String scopeId, - Long stepInstanceId) { + Long stepInstanceId, + Integer batch) { List stepExecutionRecords = taskResultService.listStepExecutionHistory(username, - appResourceScope.getAppId(), stepInstanceId); + appResourceScope.getAppId(), stepInstanceId, batch); return Response.buildSuccessResp(stepExecutionRecords.stream().map(stepExecutionRecord -> { StepExecutionRecordVO vo = new StepExecutionRecordVO(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java index 7c3c229fac..d7a0f9f3e6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java @@ -50,9 +50,13 @@ StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, * 查询步骤滚动任务 * * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数;如果不为null，会根据executeCount过滤 + * @param batch 滚动执行批次;如果不为null，会根据batch过滤 * @return 步骤滚动任务 */ - List listRollingTasks(long stepInstanceId); + List listRollingTasks(long stepInstanceId, + Integer executeCount, + Integer batch); /** * 保存步骤滚动任务 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java index 7683d3f749..26c585530a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java @@ -32,6 +32,7 @@ import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; +import org.jooq.SelectConditionStep; import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; import org.jooq.generated.tables.StepInstanceRollingTask; @@ -93,15 +94,24 @@ private StepInstanceRollingTaskDTO extract(Record record) { } @Override - public List listRollingTasks(long stepInstanceId) { - Result result = CTX.select(ALL_FIELDS) + public List listRollingTasks(long stepInstanceId, + Integer executeCount, + Integer batch) { + SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(TABLE) - .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) - .orderBy(TABLE.BATCH.asc()) - .fetch(); + .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)); + if (executeCount != null) { + selectConditionStep.and(TABLE.EXECUTE_COUNT.eq(executeCount.byteValue())); + } + if (batch != null && batch > 0) { + selectConditionStep.and(TABLE.BATCH.eq(batch.shortValue())); + } + + Result result = selectConditionStep.orderBy(TABLE.BATCH.asc()).fetch(); + List stepInstanceRollingTasks = new ArrayList<>(); if (result.size() > 0) { - result.into(record -> stepInstanceRollingTasks.add(extract(record))); + stepInstanceRollingTasks = result.map(this::extract); } return stepInstanceRollingTasks; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java index 4a1a1e9d33..5177afd7c8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java @@ -506,7 +506,7 @@ private boolean isStepSupportRetry(RunStatusEnum stepStatus) { private void saveAgentTasksForRetryFail(StepInstanceBaseDTO stepInstance, int executeCount, Integer batch, Long gseTaskId) { - List latestAgentTasks = listAgentTasks(stepInstance, executeCount - 1, batch); + List latestAgentTasks = listAgentTasks(stepInstance, executeCount - 1); for (AgentTaskDTO latestAgentTask : latestAgentTasks) { latestAgentTask.setExecuteCount(executeCount); @@ -525,7 +525,7 @@ private void saveAgentTasksForRetryFail(StepInstanceBaseDTO stepInstance, int ex private void saveAgentTasksForRetryAll(StepInstanceBaseDTO stepInstance, int executeCount, Integer batch, Long gseTaskId) { - List latestAgentTasks = listAgentTasks(stepInstance, executeCount - 1, batch); + List latestAgentTasks = listAgentTasks(stepInstance, executeCount - 1); for (AgentTaskDTO latestAgentTask : latestAgentTasks) { latestAgentTask.setExecuteCount(executeCount); @@ -539,12 +539,12 @@ private void saveAgentTasksForRetryAll(StepInstanceBaseDTO stepInstance, int exe saveAgentTasks(stepInstance, latestAgentTasks); } - private List listAgentTasks(StepInstanceBaseDTO stepInstance, int executeCount, Integer batch) { + private List listAgentTasks(StepInstanceBaseDTO stepInstance, int executeCount) { List agentTasks = Collections.emptyList(); if (stepInstance.isScriptStep()) { - agentTasks = scriptAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, batch); + agentTasks = scriptAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, null); } else if (stepInstance.isFileStep()) { - agentTasks = fileAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, batch); + agentTasks = fileAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, null); } return agentTasks; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java index e3d597639e..c280311af6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.model; import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.execute.constants.UserOperationEnum; import lombok.Data; @@ -43,10 +44,12 @@ public class OperationLogDTO { */ @Data @JsonInclude(JsonInclude.Include.NON_NULL) + @PersistenceObject public static class OperationDetail { private Long taskInstanceId; private Long stepInstanceId; private Integer executeCount; + private Integer batch; private String stepName; private Integer startupMode; private String appCode; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java index 3562b641f8..3bdce59ac7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java @@ -49,12 +49,21 @@ StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, /** * 查询最新的步骤滚动任务 * - * @param stepInstanceId 步骤实例ID - * @param executeCount 步骤执行次数 + * @param stepInstanceId 步骤实例ID + * @param executeCount 步骤执行次数 * @return 步骤滚动任务 */ List listLatestRollingTasks(long stepInstanceId, int executeCount); + /** + * 根据批次查询滚动任务 + * + * @param stepInstanceId 步骤实例ID + * @param batch 滚动批次 + * @return 步骤滚动任务 + */ + List listRollingTasksByBatch(long stepInstanceId, Integer batch); + /** * 保存步骤滚动任务 * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java index 7e7713db1b..fae4360820 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java @@ -35,14 +35,14 @@ public interface TaskOperationLogService { /** * 保存用户操作日志 * - * @param operationLog + * @param operationLog 操作日志 */ long saveOperationLog(OperationLogDTO operationLog); /** * 获取用户操作日志 * - * @param taskInstanceId + * @param taskInstanceId 作业实例ID */ List listOperationLog(long taskInstanceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java index ed7a10c2e5..92ba34f03b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java @@ -125,8 +125,12 @@ List getHostsByResultType(String username, * @param username 用户名 * @param appId 业务ID * @param stepInstanceId 步骤实例ID + * @param batch 滚动执行批次;如果传入，则会按照batch过滤步骤执行历史 * @return 执行历史 */ - List listStepExecutionHistory(String username, Long appId, Long stepInstanceId); + List listStepExecutionHistory(String username, + Long appId, + Long stepInstanceId, + Integer batch); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java index 760ada5959..d2cd2c3e56 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java @@ -60,7 +60,7 @@ public StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, @Override public List listLatestRollingTasks(long stepInstanceId, int executeCount) { List stepInstanceRollingTasks = - stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId); + stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId, null, null); if (CollectionUtils.isEmpty(stepInstanceRollingTasks)) { return stepInstanceRollingTasks; } @@ -91,6 +91,11 @@ public List listLatestRollingTasks(long stepInstance .collect(Collectors.toList()); } + @Override + public List listRollingTasksByBatch(long stepInstanceId, Integer batch) { + return stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId, null, batch); + } + @Override public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { return stepInstanceRollingTaskDAO.saveRollingTask(rollingTask); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index 6ba6c96c83..62649acd8a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -2081,6 +2081,7 @@ private OperationLogDTO buildCommonStepOperationLog(StepInstanceDTO stepInstance taskDetail.setStepInstanceId(stepInstance.getId()); taskDetail.setStepName(stepInstance.getName()); taskDetail.setExecuteCount(stepInstance.getExecuteCount()); + taskDetail.setBatch(stepInstance.getBatch()); operationLog.setDetail(taskDetail); return operationLog; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java index 49590b2bfb..dae8d99b82 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java @@ -963,49 +963,82 @@ public List getHostsByResultType(String username, @Override - public List listStepExecutionHistory(String username, Long appId, Long stepInstanceId) { + public List listStepExecutionHistory(String username, + Long appId, + Long stepInstanceId, + Integer batch) { StepInstanceBaseDTO stepInstance = checkGetStepExecutionDetail(username, appId, stepInstanceId); - int latestExecuteCount = stepInstance.getExecuteCount(); - if (latestExecuteCount == 0) { + + // 步骤没有重试执行过 + if (stepInstance.getExecuteCount() == 0) { StepExecutionRecordDTO record = new StepExecutionRecordDTO(); record.setStepInstanceId(stepInstanceId); - record.setRetryCount(latestExecuteCount); + record.setRetryCount(0); record.setCreateTime(stepInstance.getCreateTime()); return Collections.singletonList(record); } + List records; + if (batch == null) { + // 获取步骤维度的重试记录 + records = queryStepRetryRecords(stepInstance); + } else { + // 获取滚动任务维护的重试记录 + records = queryStepRollingTaskRetryRecords(stepInstanceId, batch); + } + + records.sort(Comparator.comparingInt(StepExecutionRecordDTO::getRetryCount).reversed()); + + return records; + } + + private List queryStepRetryRecords(StepInstanceBaseDTO stepInstance) { + Long stepInstanceId = stepInstance.getId(); List operationLogs = operationLogService.listOperationLog(stepInstance.getTaskInstanceId()); + // 重试操作 + List retryOperationLogs = operationLogs.stream().filter(operationLog -> { + UserOperationEnum operation = operationLog.getOperationEnum(); + OperationLogDTO.OperationDetail operationDetail = operationLog.getDetail(); + return ((UserOperationEnum.RETRY_STEP_ALL == operation || UserOperationEnum.RETRY_STEP_FAIL == operation) + && (operationDetail != null && stepInstanceId.equals(operationDetail.getStepInstanceId()))); + }).collect(Collectors.toList()); + + List records = new ArrayList<>(); - if (CollectionUtils.isEmpty(operationLogs)) { - for (int executeCount = latestExecuteCount; executeCount >= 0; executeCount--) { - StepExecutionRecordDTO record = new StepExecutionRecordDTO(); - record.setStepInstanceId(stepInstanceId); - record.setRetryCount(executeCount); - record.setCreateTime(stepInstance.getCreateTime()); - records.add(record); - } - return records; - } - - Map executeCountAndCreateTimeMap = new HashMap<>(); - operationLogs.forEach(opLog -> { - UserOperationEnum operation = opLog.getOperationEnum(); - if (UserOperationEnum.START == operation) { - executeCountAndCreateTimeMap.put(0, opLog.getCreateTime()); - } else if ((UserOperationEnum.RETRY_STEP_ALL == operation || UserOperationEnum.RETRY_STEP_FAIL == operation) - && (opLog.getDetail() != null && stepInstanceId.equals(opLog.getDetail().getStepInstanceId()))) { - // 操作记录保存的是重试前的任务信息，所以executeCount需要+1 - executeCountAndCreateTimeMap.put(opLog.getDetail().getExecuteCount() + 1, opLog.getCreateTime()); - } - }); + // 重试记录 + List retryRecords = retryOperationLogs.stream().map(operationLog -> { + StepExecutionRecordDTO record = new StepExecutionRecordDTO(); + record.setStepInstanceId(stepInstanceId); + record.setRetryCount(operationLog.getDetail().getExecuteCount()); + record.setCreateTime(operationLog.getCreateTime()); + return record; + }).collect(Collectors.toList()); + + if (CollectionUtils.isNotEmpty(retryRecords)) { + records.addAll(retryRecords); + } + + // 当前正在执行的记录 + StepExecutionRecordDTO currentRecord = new StepExecutionRecordDTO(); + currentRecord.setStepInstanceId(stepInstanceId); + currentRecord.setRetryCount(stepInstance.getExecuteCount()); + currentRecord.setCreateTime(stepInstance.getCreateTime()); + records.add(currentRecord); - for (int executeCount = latestExecuteCount; executeCount >= 0; executeCount--) { + return records; + } + + private List queryStepRollingTaskRetryRecords(long stepInstanceId, int batch) { + List records = new ArrayList<>(); + List rollingTasks = + stepInstanceRollingTaskService.listRollingTasksByBatch(stepInstanceId, batch); + rollingTasks.forEach(rollingTask -> { StepExecutionRecordDTO record = new StepExecutionRecordDTO(); record.setStepInstanceId(stepInstanceId); - record.setRetryCount(executeCount); - record.setCreateTime(executeCountAndCreateTimeMap.get(executeCount)); + record.setRetryCount(rollingTask.getExecuteCount()); + record.setCreateTime(rollingTask.getStartTime()); records.add(record); - } + }); return records; } } From af732279616e387e3342c8dc18ec37493a257304 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Fri, 8 Jul 2022 14:27:48 +0800 Subject: [PATCH 02/10] =?UTF-8?q?feature:=20=E5=8E=9F=E5=AD=90=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E8=83=BD=E5=8A=9B=E6=94=AF=E6=8C=81=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=20#446?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit agent_task 新增 actual_execute_count字段用于表示实际重试的次数 --- .../model/web/vo/TaskOperationLogVO.java | 2 + .../FileAgentTaskDAOImplIntegrationTest.java | 16 +- ...ScriptAgentTaskDAOImplIntegrationTest.java | 16 +- .../src/test/resources/init_schema.sql | 141 +++++++++--------- .../web/impl/WebTaskInstanceResourceImpl.java | 1 + .../dao/impl/FileAgentTaskDAOImpl.java | 45 +++--- .../job/execute/dao/impl/GseTaskDAOImpl.java | 6 +- .../dao/impl/ScriptAgentTaskDAOImpl.java | 50 ++++--- .../engine/listener/GseStepEventHandler.java | 16 +- .../bk/job/execute/model/AgentTaskDTO.java | 7 + .../service/impl/TaskResultServiceImpl.java | 46 +++--- ...b_execute_20220117-1000_V3.5.0.0_mysql.sql | 70 ++++----- 12 files changed, 225 insertions(+), 191 deletions(-) diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java index 1e21128e72..513f54392f 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java @@ -47,6 +47,8 @@ public class TaskOperationLogVO { private Long stepInstanceId; @ApiModelProperty("步骤执行次数") private Integer retry; + @ApiModelProperty("滚动执行批次") + private Integer batch; @ApiModelProperty("步骤名称") private String stepName; @ApiModelProperty("操作时间") diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java index eff05e13e7..331e0cf16c 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java @@ -84,7 +84,8 @@ public void testBatchSaveAgentTasks() { List agentTaskList = new ArrayList<>(); AgentTaskDTO agentTask1 = new AgentTaskDTO(); agentTask1.setStepInstanceId(100L); - agentTask1.setExecuteCount(0); + agentTask1.setExecuteCount(1); + agentTask1.setActualExecuteCount(1); agentTask1.setBatch(1); agentTask1.setFileTaskMode(FileTaskModeEnum.UPLOAD); agentTask1.setHostId(101L); @@ -99,7 +100,8 @@ public void testBatchSaveAgentTasks() { AgentTaskDTO agentTask2 = new AgentTaskDTO(); agentTask2.setStepInstanceId(100L); - agentTask2.setExecuteCount(0); + agentTask2.setExecuteCount(1); + agentTask2.setActualExecuteCount(1); agentTask2.setBatch(1); agentTask2.setFileTaskMode(FileTaskModeEnum.DOWNLOAD); agentTask2.setHostId(102L); @@ -115,10 +117,11 @@ public void testBatchSaveAgentTasks() { fileAgentTaskDAO.batchSaveAgentTasks(agentTaskList); - AgentTaskDTO agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 0, 1, + AgentTaskDTO agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, FileTaskModeEnum.UPLOAD, 101L); assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(100L); - assertThat(agentTask1Return.getExecuteCount()).isEqualTo(0L); + assertThat(agentTask1Return.getExecuteCount()).isEqualTo(1L); + assertThat(agentTask1Return.getActualExecuteCount()).isEqualTo(1L); assertThat(agentTask1Return.getBatch()).isEqualTo(1); assertThat(agentTask1Return.getFileTaskMode()).isEqualTo(FileTaskModeEnum.UPLOAD); assertThat(agentTask1Return.getHostId()).isEqualTo(101L); @@ -131,10 +134,11 @@ public void testBatchSaveAgentTasks() { assertThat(agentTask1Return.getStatus()).isEqualTo(1); - AgentTaskDTO agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 0, 1, + AgentTaskDTO agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, FileTaskModeEnum.DOWNLOAD, 102L); assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(100L); - assertThat(agentTask2Return.getExecuteCount()).isEqualTo(0L); + assertThat(agentTask2Return.getExecuteCount()).isEqualTo(1L); + assertThat(agentTask2Return.getActualExecuteCount()).isEqualTo(1L); assertThat(agentTask2Return.getBatch()).isEqualTo(1); assertThat(agentTask2Return.getFileTaskMode()).isEqualTo(FileTaskModeEnum.DOWNLOAD); assertThat(agentTask2Return.getHostId()).isEqualTo(102L); diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java index 10413ca2f4..5d8ba6143a 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java @@ -86,7 +86,8 @@ public void testBatchSaveAgentTasks() { List agentTaskList = new ArrayList<>(); AgentTaskDTO agentTask1 = new AgentTaskDTO(); agentTask1.setStepInstanceId(100L); - agentTask1.setExecuteCount(0); + agentTask1.setExecuteCount(1); + agentTask1.setActualExecuteCount(1); agentTask1.setBatch(1); agentTask1.setHostId(101L); agentTask1.setAgentId("0:127.0.0.1"); @@ -102,7 +103,8 @@ public void testBatchSaveAgentTasks() { AgentTaskDTO agentTask2 = new AgentTaskDTO(); agentTask2.setStepInstanceId(100L); - agentTask2.setExecuteCount(0); + agentTask2.setExecuteCount(1); + agentTask2.setActualExecuteCount(1); agentTask2.setBatch(1); agentTask2.setHostId(102L); agentTask2.setAgentId("0:127.0.0.2"); @@ -120,9 +122,10 @@ public void testBatchSaveAgentTasks() { scriptAgentTaskDAO.batchSaveAgentTasks(agentTaskList); - AgentTaskDTO agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 0, 1, 101L); + AgentTaskDTO agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, 101L); assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(100L); - assertThat(agentTask1Return.getExecuteCount()).isEqualTo(0L); + assertThat(agentTask1Return.getExecuteCount()).isEqualTo(1L); + assertThat(agentTask1Return.getActualExecuteCount()).isEqualTo(1L); assertThat(agentTask1Return.getBatch()).isEqualTo(1); assertThat(agentTask1Return.getHostId()).isEqualTo(101L); assertThat(agentTask1Return.getAgentId()).isEqualTo("0:127.0.0.1"); @@ -136,9 +139,10 @@ public void testBatchSaveAgentTasks() { assertThat(agentTask1Return.getExitCode()).isEqualTo(1); - AgentTaskDTO agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 0, 1, 102L); + AgentTaskDTO agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, 102L); assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(100L); - assertThat(agentTask2Return.getExecuteCount()).isEqualTo(0L); + assertThat(agentTask2Return.getExecuteCount()).isEqualTo(1L); + assertThat(agentTask2Return.getActualExecuteCount()).isEqualTo(1L); assertThat(agentTask2Return.getBatch()).isEqualTo(1); assertThat(agentTask2Return.getHostId()).isEqualTo(102L); assertThat(agentTask2Return.getGseTaskId()).isEqualTo(1001L); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql index 386ec711fb..ef70ee4bf3 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql @@ -149,15 +149,15 @@ CREATE TABLE IF NOT EXISTS `step_instance_confirm` CREATE TABLE IF NOT EXISTS `gse_task_log` ( - `step_instance_id` bigint(20) NOT NULL DEFAULT '0', - `execute_count` int(11) NOT NULL DEFAULT '0', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(11) DEFAULT NULL, - `status` tinyint(4) DEFAULT '1', - `gse_task_id` varchar(64) DEFAULT NULL, - `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `step_instance_id` bigint(20) NOT NULL DEFAULT '0', + `execute_count` int(11) NOT NULL DEFAULT '0', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(11) DEFAULT NULL, + `status` tinyint(4) DEFAULT '1', + `gse_task_id` varchar(64) DEFAULT NULL, + `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`step_instance_id`, `execute_count`) ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; @@ -188,66 +188,71 @@ CREATE TABLE IF NOT EXISTS `gse_task_ip_log` DEFAULT CHARSET = utf8mb4; CREATE TABLE IF NOT EXISTS `gse_task` - ( +( `id` bigint(20) NOT NULL AUTO_INCREMENT, `step_instance_id` bigint(20) NOT NULL DEFAULT '0', - `execute_count` int(11) NOT NULL DEFAULT '0', + `execute_count` smallint(6) NOT NULL DEFAULT '0', `batch` smallint(6) NOT NULL DEFAULT '0', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(11) DEFAULT NULL, - `status` tinyint(4) DEFAULT '1', - `gse_task_id` varchar(64) DEFAULT NULL, + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(11) DEFAULT NULL, + `status` tinyint(4) DEFAULT '1', + `gse_task_id` varchar(64) DEFAULT NULL, `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), - UNIQUE KEY (`step_instance_id`,`execute_count`,`batch`), + UNIQUE KEY (`step_instance_id`, `execute_count`, `batch`), KEY (`gse_task_id`) - ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; CREATE TABLE IF NOT EXISTS `gse_script_agent_task` - ( - `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, - `step_instance_id` bigint(20) NOT NULL, - `execute_count` int(11) NOT NULL DEFAULT '0', - `batch` smallint(6) NOT NULL DEFAULT '0', - `host_id` bigint(20) NOT NULL DEFAULT '0', - `agent_id` varchar(64) NOT NULL DEFAULT '', - `gse_task_id` bigint(20) NOT NULL DEFAULT '0', - `status` int(11) DEFAULT '1', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(20) DEFAULT NULL, - `error_code` int(11) DEFAULT '0', - `exit_code` int(11) DEFAULT NULL, - `tag` varchar(256) DEFAULT '', - `log_offset` int(11) NOT NULL DEFAULT '0', - `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - UNIQUE KEY (`step_instance_id`,`execute_count`,`batch`,`host_id`), - KEY (`step_instance_id`,`host_id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +( + `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, + `step_instance_id` bigint(20) NOT NULL, + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) NOT NULL DEFAULT '0', + `batch` smallint(6) NOT NULL DEFAULT '0', + `host_id` bigint(20) NOT NULL DEFAULT '0', + `agent_id` varchar(64) NOT NULL DEFAULT '', + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', + `status` int(11) DEFAULT '1', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `error_code` int(11) DEFAULT '0', + `exit_code` int(11) DEFAULT NULL, + `tag` varchar(256) DEFAULT '', + `log_offset` int(11) NOT NULL DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY (`step_instance_id`, `execute_count`, `batch`, `host_id`), + KEY (`step_instance_id`, `host_id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; CREATE TABLE IF NOT EXISTS `gse_file_agent_task` - ( - `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, - `step_instance_id` bigint(20) NOT NULL, - `execute_count` int(11) NOT NULL DEFAULT '0', - `batch` smallint(6) NOT NULL DEFAULT '0', - `host_id` bigint(20) NOT NULL DEFAULT '0', - `agent_id` varchar(64) NOT NULL DEFAULT '', - `mode` tinyint(1) NOT NULL, - `gse_task_id` bigint(20) NOT NULL DEFAULT '0', - `status` int(11) DEFAULT '1', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(20) DEFAULT NULL, - `error_code` int(11) DEFAULT '0', - `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - UNIQUE KEY (`step_instance_id`,`execute_count`,`batch`,`mode`,`host_id`), - KEY (`step_instance_id`,`host_id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +( + `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, + `step_instance_id` bigint(20) NOT NULL, + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) NOT NULL DEFAULT '0', + `batch` smallint(6) NOT NULL DEFAULT '0', + `host_id` bigint(20) NOT NULL DEFAULT '0', + `agent_id` varchar(64) NOT NULL DEFAULT '', + `mode` tinyint(1) NOT NULL, + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', + `status` int(11) DEFAULT '1', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `error_code` int(11) DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY (`step_instance_id`, `execute_count`, `batch`, `mode`, `host_id`), + KEY (`step_instance_id`, `host_id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; CREATE TABLE IF NOT EXISTS `operation_log` ( @@ -339,16 +344,16 @@ CREATE TABLE IF NOT EXISTS `rolling_config` CREATE TABLE IF NOT EXISTS `step_instance_rolling_task` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `step_instance_id` bigint(20) NOT NULL DEFAULT '0', - `batch` smallint(6) NOT NULL DEFAULT '0', - `execute_count` tinyint(4) NOT NULL DEFAULT '0', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(11) DEFAULT NULL, - `status` tinyint(4) NOT NULL DEFAULT '1', - `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `step_instance_id` bigint(20) NOT NULL DEFAULT '0', + `batch` smallint(6) NOT NULL DEFAULT '0', + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(11) DEFAULT NULL, + `status` tinyint(4) NOT NULL DEFAULT '1', + `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY (`step_instance_id`, `batch`, `execute_count`) ) ENGINE = InnoDB diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java index 6b892849c8..ce448fe37f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java @@ -414,6 +414,7 @@ public Response> getTaskInstanceOperationLog(String use vo.setStepInstanceId(detail.getStepInstanceId()); vo.setStepName(detail.getStepName()); vo.setRetry(detail.getExecuteCount()); + vo.setBatch(detail.getBatch()); vo.setDetail(buildDetail(operationLog.getOperationEnum(), operationLog.getDetail())); vos.add(vo); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java index 6856c428cc..d255b36f16 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java @@ -60,6 +60,7 @@ public class FileAgentTaskDAOImpl implements FileAgentTaskDAO { private static final TableField[] ALL_FIELDS = { T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID, T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT, + T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT, T_GSE_FILE_AGENT_TASK.BATCH, T_GSE_FILE_AGENT_TASK.MODE, T_GSE_FILE_AGENT_TASK.HOST_ID, @@ -81,25 +82,26 @@ public FileAgentTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX @Override public void batchSaveAgentTasks(Collection agentTasks) { - String sql = "insert into gse_file_agent_task (step_instance_id, execute_count, batch, mode, host_id, agent_id" - + " ,gse_task_id,status, start_time, end_time, total_time, error_code)" - + " values (?,?,?,?,?,?,?,?,?,?,?,?)"; - Object[][] params = new Object[agentTasks.size()][12]; + String sql = "insert into gse_file_agent_task (step_instance_id, execute_count, actual_execute_count, batch," + + "mode, host_id, agent_id ,gse_task_id,status, start_time, end_time, total_time, error_code)" + + " values (?,?,?,?,?,?,?,?,?,?,?,?,?)"; + Object[][] params = new Object[agentTasks.size()][13]; int batchCount = 0; for (AgentTaskDTO agentTask : agentTasks) { - Object[] param = new Object[12]; + Object[] param = new Object[13]; param[0] = agentTask.getStepInstanceId(); param[1] = agentTask.getExecuteCount(); - param[2] = agentTask.getBatch(); - param[3] = agentTask.getFileTaskMode().getValue(); - param[4] = agentTask.getHostId(); - param[5] = agentTask.getAgentId(); - param[6] = agentTask.getGseTaskId(); - param[7] = agentTask.getStatus(); - param[8] = agentTask.getStartTime(); - param[9] = agentTask.getEndTime(); - param[10] = agentTask.getTotalTime(); - param[11] = agentTask.getErrorCode(); + param[2] = agentTask.getActualExecuteCount(); + param[3] = agentTask.getBatch(); + param[4] = agentTask.getFileTaskMode().getValue(); + param[5] = agentTask.getHostId(); + param[6] = agentTask.getAgentId(); + param[7] = agentTask.getGseTaskId(); + param[8] = agentTask.getStatus(); + param[9] = agentTask.getStartTime(); + param[10] = agentTask.getEndTime(); + param[11] = agentTask.getTotalTime(); + param[12] = agentTask.getErrorCode(); params[batchCount++] = param; } CTX.batch(sql, params).execute(); @@ -139,7 +141,7 @@ public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STATUS.in(AgentTaskStatus.LAST_SUCCESS.getValue(), AgentTaskStatus.SUCCESS.getValue())) .and(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short)executeCount)) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())) .fetchOne(0, Integer.class); return count == null ? 0 : count; @@ -151,7 +153,7 @@ public List listResultGroups(long stepInstanceId, i CTX.select(T_GSE_FILE_AGENT_TASK.STATUS, count().as("ip_count")) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short)executeCount)) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); @@ -181,7 +183,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_FILE_AGENT_TASK.STATUS.eq(status)) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); if (batch != null && batch > 0) { @@ -206,7 +208,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, Order order) { List conditions = new ArrayList<>(); conditions.add(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); - conditions.add(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)); + conditions.add(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); conditions.add(T_GSE_FILE_AGENT_TASK.STATUS.eq(status)); conditions.add(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); @@ -271,7 +273,7 @@ public List listAgentTasks(Long stepInstanceId, SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)); + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -296,6 +298,7 @@ private AgentTaskDTO extract(Record record) { AgentTaskDTO agentTask = new AgentTaskDTO(); agentTask.setStepInstanceId(record.get(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID)); agentTask.setExecuteCount(record.get(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT)); + agentTask.setActualExecuteCount(record.get(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT)); agentTask.setBatch(record.get(T_GSE_FILE_AGENT_TASK.BATCH)); agentTask.setFileTaskMode(FileTaskModeEnum.getFileTaskMode(record.get(T_GSE_FILE_AGENT_TASK.MODE).intValue())); agentTask.setHostId(record.get(T_GSE_FILE_AGENT_TASK.HOST_ID)); @@ -329,7 +332,7 @@ public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCou Record record = CTX.select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch == null ? 0 : batch.shortValue())) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(mode.getValue().byteValue())) .and(T_GSE_FILE_AGENT_TASK.HOST_ID.eq(hostId)) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java index a29179a7fb..a5856695ca 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java @@ -57,7 +57,7 @@ private GseTaskDTO extractInfo(Record record) { gseTaskDTO.setId(record.get(TABLE.ID)); gseTaskDTO.setStepInstanceId(record.get(TABLE.STEP_INSTANCE_ID)); - gseTaskDTO.setExecuteCount(record.get(TABLE.EXECUTE_COUNT)); + gseTaskDTO.setExecuteCount(record.get(TABLE.EXECUTE_COUNT).intValue()); gseTaskDTO.setBatch(record.get(TABLE.BATCH)); gseTaskDTO.setStartTime(record.get(TABLE.START_TIME)); gseTaskDTO.setEndTime(record.get(TABLE.END_TIME)); @@ -81,7 +81,7 @@ public long saveGseTask(GseTaskDTO gseTask) { TABLE.GSE_TASK_ID) .values( gseTask.getStepInstanceId(), - gseTask.getExecuteCount(), + gseTask.getExecuteCount().shortValue(), (short) gseTask.getBatch(), gseTask.getStartTime(), gseTask.getEndTime(), @@ -111,7 +111,7 @@ public long saveGseTask(GseTaskDTO gseTask) { public GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch) { Record record = dslContext.select(ALL_FIELDS).from(TABLE) .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(TABLE.EXECUTE_COUNT.eq(executeCount)) + .and(TABLE.EXECUTE_COUNT.eq((short)executeCount)) .and(TABLE.BATCH.eq(batch == null ? 0 : batch.shortValue())) .fetchOne(); return extractInfo(record); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java index 626ca49549..26ba809b32 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java @@ -60,6 +60,7 @@ public class ScriptAgentTaskDAOImpl implements ScriptAgentTaskDAO { private static final TableField[] ALL_FIELDS = { T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID, T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT, + T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT, T_GSE_SCRIPT_AGENT_TASK.BATCH, T_GSE_SCRIPT_AGENT_TASK.HOST_ID, T_GSE_SCRIPT_AGENT_TASK.AGENT_ID, @@ -83,27 +84,29 @@ public ScriptAgentTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext C @Override public void batchSaveAgentTasks(Collection agentTasks) { - String sql = "insert into gse_script_agent_task (step_instance_id, execute_count, batch, host_id, agent_id, " - + "gse_task_id, status, start_time, end_time, total_time, error_code, exit_code, tag, log_offset)" - + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - Object[][] params = new Object[agentTasks.size()][14]; + String sql = "insert into gse_script_agent_task (step_instance_id, execute_count, actual_execute_count, batch," + + " host_id, agent_id, gse_task_id, status, start_time, end_time, total_time, error_code, exit_code, tag," + + " log_offset)" + + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + Object[][] params = new Object[agentTasks.size()][15]; int batchCount = 0; for (AgentTaskDTO agentTask : agentTasks) { - Object[] param = new Object[14]; + Object[] param = new Object[15]; param[0] = agentTask.getStepInstanceId(); param[1] = agentTask.getExecuteCount(); - param[2] = agentTask.getBatch(); - param[3] = agentTask.getHostId(); - param[4] = agentTask.getAgentId(); - param[5] = agentTask.getGseTaskId(); - param[6] = agentTask.getStatus(); - param[7] = agentTask.getStartTime(); - param[8] = agentTask.getEndTime(); - param[9] = agentTask.getTotalTime(); - param[10] = agentTask.getErrorCode(); - param[11] = agentTask.getExitCode(); - param[12] = StringUtils.truncate(agentTask.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); - param[13] = agentTask.getScriptLogOffset(); + param[2] = agentTask.getActualExecuteCount(); + param[3] = agentTask.getBatch(); + param[4] = agentTask.getHostId(); + param[5] = agentTask.getAgentId(); + param[6] = agentTask.getGseTaskId(); + param[7] = agentTask.getStatus(); + param[8] = agentTask.getStartTime(); + param[9] = agentTask.getEndTime(); + param[10] = agentTask.getTotalTime(); + param[11] = agentTask.getErrorCode(); + param[12] = agentTask.getExitCode(); + param[13] = StringUtils.truncate(agentTask.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); + param[14] = agentTask.getScriptLogOffset(); params[batchCount++] = param; } CTX.batch(sql, params).execute(); @@ -146,7 +149,7 @@ public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { .where(T_GSE_SCRIPT_AGENT_TASK.STATUS.in(AgentTaskStatus.LAST_SUCCESS.getValue(), AgentTaskStatus.SUCCESS.getValue())) .and(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short)executeCount)) .fetchOne(0, Integer.class); return count == null ? 0 : count; } @@ -157,7 +160,7 @@ public List listResultGroups(long stepInstanceId, i CTX.select(T_GSE_SCRIPT_AGENT_TASK.STATUS, T_GSE_SCRIPT_AGENT_TASK.TAG, count().as("ip_count")) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)); + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -187,7 +190,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_SCRIPT_AGENT_TASK.STATUS.eq(status)) .and(T_GSE_SCRIPT_AGENT_TASK.TAG.eq(tag == null ? "" : tag)); if (batch != null && batch > 0) { @@ -213,7 +216,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, Order order) { List conditions = new ArrayList<>(); conditions.add(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); - conditions.add(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)); + conditions.add(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); conditions.add(T_GSE_SCRIPT_AGENT_TASK.STATUS.eq(status)); conditions.add(T_GSE_SCRIPT_AGENT_TASK.TAG.eq(tag == null ? "" : tag)); @@ -283,7 +286,7 @@ public List listAgentTasks(Long stepInstanceId, SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)); + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -305,6 +308,7 @@ private AgentTaskDTO extract(Record record) { AgentTaskDTO agentTask = new AgentTaskDTO(); agentTask.setStepInstanceId(record.get(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID)); agentTask.setExecuteCount(record.get(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT)); + agentTask.setActualExecuteCount(record.get(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT)); agentTask.setBatch(record.get(T_GSE_SCRIPT_AGENT_TASK.BATCH)); agentTask.setHostId(record.get(T_GSE_SCRIPT_AGENT_TASK.HOST_ID)); agentTask.setAgentId(record.get(T_GSE_SCRIPT_AGENT_TASK.AGENT_ID)); @@ -339,7 +343,7 @@ public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCou Record record = CTX.select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch == null ? 0 : batch.shortValue())) .and(T_GSE_SCRIPT_AGENT_TASK.HOST_ID.eq(hostId)) .fetchOne(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java index 5177afd7c8..f48cf0c769 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java @@ -263,20 +263,20 @@ private void saveInitialGseAgentTasks(Long gseTaskId, if (rollingConfig.isBatchRollingStep(stepInstanceId) && stepInstance.isFirstRollingBatch()) { List serverBatchList = rollingConfig.getConfigDetail().getServerBatchList(); serverBatchList.forEach(serverBatch -> agentTasks.addAll(buildGseAgentTasks(stepInstanceId, - executeCount, serverBatch.getBatch(), gseTaskId, serverBatch.getServers(), + executeCount, executeCount, serverBatch.getBatch(), gseTaskId, serverBatch.getServers(), AgentTaskStatus.WAITING))); } else if (rollingConfig.isAllRollingStep(stepInstanceId)) { - agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, batch, + agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, executeCount, batch, gseTaskId, stepInstance.getTargetServers().getIpList(), AgentTaskStatus.WAITING)); } } else { - agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, batch, + agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, executeCount, batch, gseTaskId, stepInstance.getTargetServers().getIpList(), AgentTaskStatus.WAITING)); } // 无效主机 if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getInvalidIpList())) { - agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, batch, + agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, executeCount, batch, 0L, stepInstance.getTargetServers().getInvalidIpList(), AgentTaskStatus.HOST_NOT_EXIST)); } saveAgentTasks(stepInstance, agentTasks); @@ -284,17 +284,20 @@ private void saveInitialGseAgentTasks(Long gseTaskId, private List buildGseAgentTasks(long stepInstanceId, int executeCount, + int actualExecuteCount, int batch, Long gseTaskId, List hosts, AgentTaskStatus status) { return hosts.stream() - .map(host -> buildGseAgentTask(stepInstanceId, executeCount, batch, gseTaskId, host, status)) + .map(host -> buildGseAgentTask(stepInstanceId, executeCount, actualExecuteCount, + batch, gseTaskId, host, status)) .collect(Collectors.toList()); } protected AgentTaskDTO buildGseAgentTask(long stepInstanceId, int executeCount, + int actualExecuteCount, int batch, Long gseTaskId, HostDTO host, @@ -302,6 +305,7 @@ protected AgentTaskDTO buildGseAgentTask(long stepInstanceId, AgentTaskDTO agentTask = new AgentTaskDTO(); agentTask.setStepInstanceId(stepInstanceId); agentTask.setExecuteCount(executeCount); + agentTask.setActualExecuteCount(actualExecuteCount); agentTask.setBatch(batch); agentTask.setGseTaskId(gseTaskId); agentTask.setStatus(status.getValue()); @@ -514,6 +518,7 @@ private void saveAgentTasksForRetryFail(StepInstanceBaseDTO stepInstance, int ex if (batch != null && latestAgentTask.getBatch() != batch) { continue; } + latestAgentTask.setActualExecuteCount(executeCount); latestAgentTask.resetTaskInitialStatus(); latestAgentTask.setGseTaskId(gseTaskId); } @@ -532,6 +537,7 @@ private void saveAgentTasksForRetryAll(StepInstanceBaseDTO stepInstance, int exe if (batch != null && latestAgentTask.getBatch() != batch) { continue; } + latestAgentTask.setActualExecuteCount(executeCount); latestAgentTask.resetTaskInitialStatus(); latestAgentTask.setGseTaskId(gseTaskId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java index 02592a3ada..44cdc595cd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java @@ -50,6 +50,10 @@ public class AgentTaskDTO { * 步骤执行次数 */ private int executeCount; + /** + * Agent 任务对应的实际的步骤执行次数（重试场景，可能Agent任务并没有实际被执行) + */ + private int actualExecuteCount; /** * 滚动执行批次 */ @@ -121,6 +125,7 @@ public class AgentTaskDTO { public AgentTaskDTO(long stepInstanceId, int executeCount, int batch, Long hostId, String agentId) { this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; + this.actualExecuteCount = executeCount; this.batch = batch; this.hostId = hostId; this.agentId = agentId; @@ -130,6 +135,7 @@ public AgentTaskDTO(long stepInstanceId, int executeCount, int batch, FileTaskMo Long hostId, String agentId) { this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; + this.actualExecuteCount = executeCount; this.batch = batch; this.fileTaskMode = fileTaskMode; this.hostId = hostId; @@ -139,6 +145,7 @@ public AgentTaskDTO(long stepInstanceId, int executeCount, int batch, FileTaskMo public AgentTaskDTO(AgentTaskDTO agentTask) { this.stepInstanceId = agentTask.getStepInstanceId(); this.executeCount = agentTask.getExecuteCount(); + this.actualExecuteCount = agentTask.getActualExecuteCount(); this.batch = agentTask.getBatch(); this.fileTaskMode = agentTask.getFileTaskMode(); this.hostId = agentTask.getHostId(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java index dae8d99b82..725ca06f66 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java @@ -983,7 +983,7 @@ public List listStepExecutionHistory(String username, // 获取步骤维度的重试记录 records = queryStepRetryRecords(stepInstance); } else { - // 获取滚动任务维护的重试记录 + // 获取滚动任务维度的重试记录 records = queryStepRollingTaskRetryRecords(stepInstanceId, batch); } @@ -994,37 +994,33 @@ public List listStepExecutionHistory(String username, private List queryStepRetryRecords(StepInstanceBaseDTO stepInstance) { Long stepInstanceId = stepInstance.getId(); - List operationLogs = operationLogService.listOperationLog(stepInstance.getTaskInstanceId()); - // 重试操作 - List retryOperationLogs = operationLogs.stream().filter(operationLog -> { - UserOperationEnum operation = operationLog.getOperationEnum(); - OperationLogDTO.OperationDetail operationDetail = operationLog.getDetail(); - return ((UserOperationEnum.RETRY_STEP_ALL == operation || UserOperationEnum.RETRY_STEP_FAIL == operation) - && (operationDetail != null && stepInstanceId.equals(operationDetail.getStepInstanceId()))); - }).collect(Collectors.toList()); + // 步骤最新的重试次数 + int latestExecuteCount = stepInstance.getExecuteCount(); + List operationLogs = + operationLogService.listOperationLog(stepInstance.getTaskInstanceId()); List records = new ArrayList<>(); - // 重试记录 - List retryRecords = retryOperationLogs.stream().map(operationLog -> { + Map executeCountAndCreateTimeMap = new HashMap<>(); + operationLogs.forEach(opLog -> { + UserOperationEnum operation = opLog.getOperationEnum(); + if (UserOperationEnum.START == operation) { + executeCountAndCreateTimeMap.put(0, opLog.getCreateTime()); + } else if ((UserOperationEnum.RETRY_STEP_ALL == operation || UserOperationEnum.RETRY_STEP_FAIL == operation) + && (opLog.getDetail() != null && stepInstanceId.equals(opLog.getDetail().getStepInstanceId()))) { + // 操作记录保存的是重试前的任务信息，所以executeCount需要+1 + executeCountAndCreateTimeMap.put(opLog.getDetail().getExecuteCount() + 1, opLog.getCreateTime()); + } + }); + + for (int executeCount = latestExecuteCount; executeCount >= 0; executeCount--) { StepExecutionRecordDTO record = new StepExecutionRecordDTO(); record.setStepInstanceId(stepInstanceId); - record.setRetryCount(operationLog.getDetail().getExecuteCount()); - record.setCreateTime(operationLog.getCreateTime()); - return record; - }).collect(Collectors.toList()); - - if (CollectionUtils.isNotEmpty(retryRecords)) { - records.addAll(retryRecords); + record.setRetryCount(executeCount); + record.setCreateTime(executeCountAndCreateTimeMap.get(executeCount)); + records.add(record); } - // 当前正在执行的记录 - StepExecutionRecordDTO currentRecord = new StepExecutionRecordDTO(); - currentRecord.setStepInstanceId(stepInstanceId); - currentRecord.setRetryCount(stepInstance.getExecuteCount()); - currentRecord.setCreateTime(stepInstance.getCreateTime()); - records.add(currentRecord); - return records; } diff --git a/support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.5.0.0_mysql.sql b/support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.5.0.0_mysql.sql index 95b70c3a5f..9e10162e7e 100644 --- a/support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.5.0.0_mysql.sql +++ b/support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.5.0.0_mysql.sql @@ -17,7 +17,7 @@ BEGIN ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `step_instance_id` bigint(20) NOT NULL DEFAULT '0', - `execute_count` int(11) NOT NULL DEFAULT '0', + `execute_count` smallint(6) NOT NULL DEFAULT '0', `batch` smallint(6) NOT NULL DEFAULT '0', `start_time` bigint(20) DEFAULT NULL, `end_time` bigint(20) DEFAULT NULL, @@ -35,23 +35,24 @@ BEGIN CREATE TABLE IF NOT EXISTS `gse_script_agent_task` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `step_instance_id` bigint(20) NOT NULL, - `execute_count` int(11) NOT NULL DEFAULT '0', - `batch` smallint(6) NOT NULL DEFAULT '0', - `host_id` bigint(20) NOT NULL DEFAULT '0', - `agent_id` varchar(64) NOT NULL DEFAULT '', - `gse_task_id` bigint(20) NOT NULL DEFAULT '0', - `status` int(11) DEFAULT '1', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(20) DEFAULT NULL, - `error_code` int(11) DEFAULT '0', - `exit_code` int(11) DEFAULT NULL, - `tag` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT '', - `log_offset` int(11) NOT NULL DEFAULT '0', - `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `step_instance_id` bigint(20) NOT NULL, + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) NOT NULL DEFAULT '0', + `batch` smallint(6) NOT NULL DEFAULT '0', + `host_id` bigint(20) NOT NULL DEFAULT '0', + `agent_id` varchar(64) NOT NULL DEFAULT '', + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', + `status` int(11) DEFAULT '1', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `error_code` int(11) DEFAULT '0', + `exit_code` int(11) DEFAULT NULL, + `tag` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT '', + `log_offset` int(11) NOT NULL DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `uk_step_id_execute_count_batch_host_id` (`step_instance_id`,`execute_count`,`batch`,`host_id`), KEY `idx_step_id_host_id` (`step_instance_id`,`host_id`), @@ -62,21 +63,22 @@ BEGIN CREATE TABLE IF NOT EXISTS `gse_file_agent_task` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `step_instance_id` bigint(20) NOT NULL, - `execute_count` int(11) NOT NULL DEFAULT '0', - `batch` smallint(6) NOT NULL DEFAULT '0', - `host_id` bigint(20) NOT NULL DEFAULT '0', - `agent_id` varchar(64) NOT NULL DEFAULT '', - `mode` tinyint(1) NOT NULL, - `gse_task_id` bigint(20) NOT NULL DEFAULT '0', - `status` int(11) DEFAULT '1', - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `total_time` bigint(20) DEFAULT NULL, - `error_code` int(11) DEFAULT '0', - `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `step_instance_id` bigint(20) NOT NULL, + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) NOT NULL DEFAULT '0', + `batch` smallint(6) NOT NULL DEFAULT '0', + `host_id` bigint(20) NOT NULL DEFAULT '0', + `agent_id` varchar(64) NOT NULL DEFAULT '', + `mode` tinyint(1) NOT NULL, + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', + `status` int(11) DEFAULT '1', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `error_code` int(11) DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `uk_step_id_execute_count_batch_mode_host_id` (`step_instance_id`,`execute_count`,`batch`,`mode`,`host_id`), KEY `idx_step_id_host_id` (`step_instance_id`,`host_id`), @@ -102,7 +104,7 @@ BEGIN ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `step_instance_id` bigint(20) NOT NULL DEFAULT '0', - `execute_count` tinyint(4) NOT NULL DEFAULT '0', + `execute_count` smallint(6) NOT NULL DEFAULT '0', `batch` smallint(6) NOT NULL DEFAULT '0', `start_time` bigint(20) DEFAULT NULL, `end_time` bigint(20) DEFAULT NULL, From a938a1b441bab41eda9abe1af715eb66f7cc009a Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Fri, 8 Jul 2022 14:58:03 +0800 Subject: [PATCH 03/10] =?UTF-8?q?feature:=20=E5=8E=9F=E5=AD=90=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E8=83=BD=E5=8A=9B=E6=94=AF=E6=8C=81=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=20#446?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit agent_task 新增 actual_execute_count字段用于表示实际重试的次数 --- .../bk/job/execute/dao/FileAgentTaskDAO.java | 11 ---------- .../job/execute/dao/ScriptAgentTaskDAO.java | 10 --------- .../dao/impl/FileAgentTaskDAOImpl.java | 21 +------------------ .../dao/impl/ScriptAgentTaskDAOImpl.java | 20 +----------------- .../bk/job/execute/model/AgentTaskDTO.java | 2 +- .../job/execute/service/AgentTaskService.java | 11 ++++++++++ .../execute/service/FileAgentTaskService.java | 12 ----------- .../service/ScriptAgentTaskService.java | 10 --------- .../impl/FileAgentTaskServiceImpl.java | 10 +++------ .../execute/service/impl/LogServiceImpl.java | 18 ++++++++++------ .../impl/ScriptAgentTaskServiceImpl.java | 10 +++------ 11 files changed, 32 insertions(+), 103 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java index 92d3acb959..19b1fb52fc 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java @@ -138,17 +138,6 @@ List listAgentTasks(Long stepInstanceId, AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, FileTaskModeEnum mode, long hostId); - /** - * 获取Agent任务实际执行成功的executeCount值(重试场景) - * - * @param stepInstanceId 步骤实例ID - * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 - * @param mode 文件分发任务模式 - * @param hostId 主机ID - * @return Agent任务实际执行成功的executeCount值 - */ - int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, FileTaskModeEnum mode, long hostId); - /** * 判断步骤实例的Agent Task 记录是否存在 * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java index b0e9e424ad..443db724e6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java @@ -137,16 +137,6 @@ List listAgentTasks(Long stepInstanceId, */ AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, long hostId); - /** - * 获取Agent任务实际执行成功的executeCount值(重试场景) - * - * @param stepInstanceId 步骤实例ID - * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 - * @param hostId 主机ID - * @return Agent任务实际执行成功的executeCount值 - */ - int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, long hostId); - /** * 判断步骤实例的Agent Task 记录是否存在 * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java index d255b36f16..58fc990a1a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java @@ -298,7 +298,7 @@ private AgentTaskDTO extract(Record record) { AgentTaskDTO agentTask = new AgentTaskDTO(); agentTask.setStepInstanceId(record.get(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID)); agentTask.setExecuteCount(record.get(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT)); - agentTask.setActualExecuteCount(record.get(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT)); + agentTask.setActualExecuteCount(record.get(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT).intValue()); agentTask.setBatch(record.get(T_GSE_FILE_AGENT_TASK.BATCH)); agentTask.setFileTaskMode(FileTaskModeEnum.getFileTaskMode(record.get(T_GSE_FILE_AGENT_TASK.MODE).intValue())); agentTask.setHostId(record.get(T_GSE_FILE_AGENT_TASK.HOST_ID)); @@ -340,25 +340,6 @@ public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCou return extract(record); } - @Override - public int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, FileTaskModeEnum mode, long hostId) { - Record record = CTX.select(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT) - .from(T_GSE_FILE_AGENT_TASK) - .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch == null ? 0 : batch.shortValue())) - .and(T_GSE_FILE_AGENT_TASK.HOST_ID.eq(hostId)) - .and(T_GSE_FILE_AGENT_TASK.STATUS.eq(AgentTaskStatus.SUCCESS.getValue())) - .and(T_GSE_FILE_AGENT_TASK.MODE.eq(mode.getValue().byteValue())) - .orderBy(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.desc()) - .limit(1) - .fetchOne(); - if (record != null && record.size() > 0) { - return record.getValue(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT); - } else { - return 0; - } - } - @Override public boolean isStepInstanceRecordExist(long stepInstanceId) { return CTX.fetchExists(T_GSE_FILE_AGENT_TASK, T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java index 26ba809b32..df412118b2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java @@ -308,7 +308,7 @@ private AgentTaskDTO extract(Record record) { AgentTaskDTO agentTask = new AgentTaskDTO(); agentTask.setStepInstanceId(record.get(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID)); agentTask.setExecuteCount(record.get(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT)); - agentTask.setActualExecuteCount(record.get(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT)); + agentTask.setActualExecuteCount(record.get(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT).intValue()); agentTask.setBatch(record.get(T_GSE_SCRIPT_AGENT_TASK.BATCH)); agentTask.setHostId(record.get(T_GSE_SCRIPT_AGENT_TASK.HOST_ID)); agentTask.setAgentId(record.get(T_GSE_SCRIPT_AGENT_TASK.AGENT_ID)); @@ -350,24 +350,6 @@ public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCou return extract(record); } - @Override - public int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, long hostId) { - Record record = CTX.select(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT) - .from(T_GSE_SCRIPT_AGENT_TASK) - .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch == null ? 0 : batch.shortValue())) - .and(T_GSE_SCRIPT_AGENT_TASK.HOST_ID.eq(hostId)) - .and(T_GSE_SCRIPT_AGENT_TASK.STATUS.eq(AgentTaskStatus.SUCCESS.getValue())) - .orderBy(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.desc()) - .limit(1) - .fetchOne(); - if (record != null && record.size() > 0) { - return record.getValue(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT); - } else { - return 0; - } - } - @Override public boolean isStepInstanceRecordExist(long stepInstanceId) { return CTX.fetchExists(T_GSE_SCRIPT_AGENT_TASK, T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java index 44cdc595cd..4a8d7865b5 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java @@ -53,7 +53,7 @@ public class AgentTaskDTO { /** * Agent 任务对应的实际的步骤执行次数（重试场景，可能Agent任务并没有实际被执行) */ - private int actualExecuteCount; + private Integer actualExecuteCount; /** * 滚动执行批次 */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java index 8d380fc0dc..9e04392a21 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.service; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.execute.model.AgentTaskDTO; import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; @@ -143,5 +144,15 @@ List listAgentTaskDetail(StepInstanceBaseDTO stepInstance, Integer executeCount, Integer batch); + /** + * 获取Agent任务实际执行成功的executeCount值(重试场景,兼容历史数据) + * + * @param stepInstanceId 步骤实例ID + * @param cloudIp 云区域+ip + * @return Agent任务实际执行成功的executeCount值 + */ + @CompatibleImplementation(name = "rolling_execution", explain = "兼容历史数据", version = "3.7.x") + int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp); + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java index 8baeba64ba..5d2c7c6bd2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java @@ -63,16 +63,4 @@ AgentTaskDTO getAgentTaskByHost(StepInstanceDTO stepInstance, Integer batch, FileTaskModeEnum fileTaskMode, HostDTO host); - - - /** - * 获取Agent任务实际执行成功的executeCount值(重试场景) - * - * @param stepInstanceId 步骤实例ID - * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 - * @param mode 文件分发任务模式 - * @param host 主机 - * @return Agent任务实际执行成功的executeCount值 - */ - int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, FileTaskModeEnum mode, HostDTO host); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java index 7571016d0c..67cb8cfa34 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java @@ -45,14 +45,4 @@ AgentTaskDTO getAgentTaskByHost(StepInstanceBaseDTO stepInstance, Integer executeCount, Integer batch, HostDTO host); - - /** - * 获取Agent任务实际执行成功的executeCount值(重试场景) - * - * @param stepInstanceId 步骤实例ID - * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 - * @param host 主机 - * @return Agent任务实际执行成功的executeCount值 - */ - int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, HostDTO host); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java index 954fbbce6d..edde63c69c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java @@ -230,13 +230,9 @@ public List listAgentTasks(Long stepInstanceId, Integer executeCou } @Override - public int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, FileTaskModeEnum mode, HostDTO host) { - if (isStepInstanceRecordExist(stepInstanceId)) { - return fileAgentTaskDAO.getActualSuccessExecuteCount(stepInstanceId, batch, mode, host.getHostId()); - } else { - // 兼容历史数据 - return gseTaskIpLogDAO.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); - } + public int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp) { + // 兼容历史数据 + return gseTaskIpLogDAO.getActualSuccessExecuteCount(stepInstanceId, cloudIp); } @Override diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java index 3bf488b760..47d5ef594a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java @@ -150,10 +150,14 @@ public ScriptHostLogContent getScriptHostLogContent(long stepInstanceId, int exe if (agentTask == null) { return null; } - if (agentTask.getStatus() == AgentTaskStatus.LAST_SUCCESS.getValue()) { - actualExecuteCount = scriptAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, - agentTask.getBatch(), host); + + if (executeCount > 0 && agentTask.getActualExecuteCount() != null) { + actualExecuteCount = agentTask.getActualExecuteCount(); + } else if (agentTask.getStatus() == AgentTaskStatus.LAST_SUCCESS.getValue()) { + // 兼容历史数据 + actualExecuteCount = scriptAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); } + String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC")); InternalResponse resp; @@ -260,9 +264,11 @@ public FileIpLogContent getFileIpLogContent(long stepInstanceId, int executeCoun if (agentTask == null) { return null; } - if (agentTask.getStatus() == AgentTaskStatus.LAST_SUCCESS.getValue()) { - actualExecuteCount = fileAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, - agentTask.getBatch(), agentTask.getFileTaskMode(), host); + if (executeCount > 0 && agentTask.getActualExecuteCount() != null) { + actualExecuteCount = agentTask.getActualExecuteCount(); + } else if (agentTask.getStatus() == AgentTaskStatus.LAST_SUCCESS.getValue()) { + // 兼容历史数据 + actualExecuteCount = scriptAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); } String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java index 8f79d6f00f..4dbd8aa92a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java @@ -138,13 +138,9 @@ public AgentTaskDTO getAgentTaskByHost(StepInstanceBaseDTO stepInstance, } @Override - public int getActualSuccessExecuteCount(long stepInstanceId, Integer batch, HostDTO host) { - if (isStepInstanceRecordExist(stepInstanceId)) { - return scriptAgentTaskDAO.getActualSuccessExecuteCount(stepInstanceId, batch, host.getHostId()); - } else { - // 兼容历史数据 - return gseTaskIpLogDAO.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); - } + public int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp) { + // 兼容历史数据 + return gseTaskIpLogDAO.getActualSuccessExecuteCount(stepInstanceId, cloudIp); } @Override From bd491b3e5e8606ec38a178e07d654719758db229 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Fri, 8 Jul 2022 15:05:17 +0800 Subject: [PATCH 04/10] =?UTF-8?q?feature:=20=E5=8E=9F=E5=AD=90=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E8=83=BD=E5=8A=9B=E6=94=AF=E6=8C=81=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=20#446?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit agent_task 新增 actual_execute_count字段用于表示实际重试的次数 --- .../execute/dao/impl/StepInstanceRollingTaskDAOImpl.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java index 26c585530a..e446fb3b5c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java @@ -71,7 +71,7 @@ public StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, int exec Record record = CTX.select(ALL_FIELDS) .from(TABLE) .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toByte(executeCount))) + .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toShort(executeCount))) .and(TABLE.BATCH.eq(JooqDataTypeUtil.toShort(batch))) .fetchOne(); return extract(record); @@ -101,7 +101,7 @@ public List listRollingTasks(long stepInstanceId, .from(TABLE) .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)); if (executeCount != null) { - selectConditionStep.and(TABLE.EXECUTE_COUNT.eq(executeCount.byteValue())); + selectConditionStep.and(TABLE.EXECUTE_COUNT.eq(executeCount.shortValue())); } if (batch != null && batch > 0) { selectConditionStep.and(TABLE.BATCH.eq(batch.shortValue())); @@ -129,7 +129,7 @@ public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { TABLE.TOTAL_TIME) .values( rollingTask.getStepInstanceId(), - JooqDataTypeUtil.toByte(rollingTask.getExecuteCount()), + JooqDataTypeUtil.toShort(rollingTask.getExecuteCount()), JooqDataTypeUtil.toShort(rollingTask.getBatch()), JooqDataTypeUtil.toByte(rollingTask.getStatus()), rollingTask.getStartTime(), @@ -179,7 +179,7 @@ public void updateRollingTask(long stepInstanceId, return; } updateSetMoreStep.where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toByte(executeCount))) + .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toByte(executeCount).shortValue())) .and(TABLE.BATCH.eq(JooqDataTypeUtil.toShort(batch))) .execute(); From 42b8e3101bffbc5c67dffc4b44a853a334635c88 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Sat, 9 Jul 2022 10:17:10 +0800 Subject: [PATCH 05/10] =?UTF-8?q?feature:=20=E5=8E=9F=E5=AD=90=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E8=83=BD=E5=8A=9B=E6=94=AF=E6=8C=81=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=20#446?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit agent_task 新增 actual_execute_count字段用于表示实际重试的次数 --- .../engine/listener/GseStepEventHandler.java | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java index f48cf0c769..09afee4401 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java @@ -384,11 +384,8 @@ private void ignoreError(StepInstanceDTO stepInstance) { stepInstanceRollingTaskService.queryRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch()); if (stepInstanceRollingTask != null) { - long endTime = stepInstanceRollingTask.getEndTime() != null ? - stepInstanceRollingTask.getEndTime() : System.currentTimeMillis(); - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR, null, endTime, - TaskCostCalculator.calculate(stepInstanceRollingTask.getStartTime(), endTime, null)); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.IGNORE_ERROR); } } @@ -686,16 +683,13 @@ private void onSuccess(StepInstanceDTO stepInstance) { long totalTime = endTime - startTime; if (stepInstance.isRollingStep()) { - log.info("rolling step"); RollingConfigDTO rollingConfig = rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.SUCCESS, startTime, endTime, totalTime); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.SUCCESS); int totalBatch = rollingConfig.getConfigDetail().getTotalBatch(); - log.info("rollingConfig: {}, stepInstance: {}", rollingConfig, stepInstance); boolean isLastBatch = totalBatch == stepInstance.getBatch(); if (isLastBatch) { - log.info("updateStepExecutionInfo-SUCCESS"); taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, startTime, endTime, totalTime); // 步骤执行成功后清理产生的临时文件 @@ -703,10 +697,8 @@ private void onSuccess(StepInstanceDTO stepInstance) { } else { taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.ROLLING_WAITING, startTime, endTime, totalTime); - log.info("updateStepExecutionInfo-ROLLING_WAITING"); } } else { - log.info("not rolling step"); taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, startTime, endTime, totalTime); // 步骤执行成功后清理产生的临时文件 @@ -724,8 +716,8 @@ private void onFail(StepInstanceDTO stepInstance) { taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); if (stepInstance.isRollingStep()) { - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.IGNORE_ERROR); } return; } @@ -737,15 +729,15 @@ private void onFail(StepInstanceDTO stepInstance) { switch (rollingMode) { case IGNORE_ERROR: log.info("Ignore error for rolling step, rollingMode: {}", rollingMode); - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.IGNORE_ERROR); taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); break; case PAUSE_IF_FAIL: case MANUAL: - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.FAIL, startTime, endTime, totalTime); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.FAIL); taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.FAIL, startTime, endTime, totalTime); break; @@ -769,8 +761,8 @@ private void onStopSuccess(StepInstanceDTO stepInstance) { taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.STOP_SUCCESS, startTime, endTime, totalTime); if (stepInstance.isRollingStep()) { - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.STOP_SUCCESS, startTime, endTime, totalTime); + finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.STOP_SUCCESS); } } else { log.error("Refresh step fail, stepInstanceId: {}, stepStatus: {}, gseTaskStatus: {}", @@ -779,14 +771,10 @@ private void onStopSuccess(StepInstanceDTO stepInstance) { } private void onAbnormalState(StepInstanceDTO stepInstance) { - long stepInstanceId = stepInstance.getId(); - long endTime = System.currentTimeMillis(); - long startTime = stepInstance.getStartTime(); - long totalTime = endTime - startTime; setAbnormalStatusForStep(stepInstance); if (stepInstance.isRollingStep()) { - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.ABNORMAL_STATE, startTime, endTime, totalTime); + finishRollingTask(stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), + RunStatusEnum.ABNORMAL_STATE); } } @@ -810,4 +798,19 @@ private void setAbnormalStatusForStep(StepInstanceDTO stepInstance) { ); } } + + private void finishRollingTask(long stepInstanceId, int executeCount, int batch, RunStatusEnum status) { + StepInstanceRollingTaskDTO rollingTask = + stepInstanceRollingTaskService.queryRollingTask(stepInstanceId, executeCount, batch); + if (rollingTask == null) { + log.error("Rolling task is not exist, skip update! stepInstanceId: {}, executeCount: {}, batch: {}", + stepInstanceId, executeCount, batch); + return; + } + long now = System.currentTimeMillis(); + long startTime = rollingTask.getStartTime() != null ? rollingTask.getStartTime() : now; + + stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, executeCount, + batch, status, startTime, now, now - startTime); + } } From eadb6039480d9f1b72dd2d40daf11b59b1c3f674 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Sat, 9 Jul 2022 11:25:34 +0800 Subject: [PATCH 06/10] =?UTF-8?q?feature:=20=E5=8E=9F=E5=AD=90=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E8=83=BD=E5=8A=9B=E6=94=AF=E6=8C=81=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=20#446?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 解决执行方案运行发生500报错的问题 --- .../service/impl/TaskExecuteServiceImpl.java | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index 62649acd8a..aea7d1f5c9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -703,15 +703,26 @@ private void fillStepHostDetail(List stepInstanceList, ServiceL if (stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue())) { continue; } - stepInstance.getTargetServers().getIpList() - .forEach(host -> fillHostDetail(host, hostMap)); + if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getStaticIpList())) { + stepInstance.getTargetServers().getStaticIpList() + .forEach(host -> fillHostDetail(host, hostMap)); + } + if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getIpList())) { + stepInstance.getTargetServers().getIpList() + .forEach(host -> fillHostDetail(host, hostMap)); + } if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { List fileSourceList = stepInstance.getFileSourceList(); if (fileSourceList != null) { for (FileSourceDTO fileSource : fileSourceList) { ServersDTO servers = fileSource.getServers(); - if (servers != null && servers.getIpList() != null) { - servers.getIpList().forEach(host -> fillHostDetail(host, hostMap)); + if (servers != null) { + if (CollectionUtils.isNotEmpty(servers.getStaticIpList())) { + servers.getStaticIpList().forEach(host -> fillHostDetail(host, hostMap)); + } + if (CollectionUtils.isNotEmpty(servers.getIpList())) { + servers.getIpList().forEach(host -> fillHostDetail(host, hostMap)); + } } } } @@ -1093,7 +1104,6 @@ private void authExecuteJobPlan(String username, long appId, ServiceTaskPlanDTO } } - AuthResult accountAuthResult = executeAuthService.batchAuthAccountExecutable( username, new AppResourceScope(appId), accountIds); @@ -1103,16 +1113,14 @@ private void authExecuteJobPlan(String username, long appId, ServiceTaskPlanDTO // 主机为空，无需对主机鉴权 authResult = accountAuthResult; } else { - AuthResult serverAuthResult = null; + AuthResult serverAuthResult; if (isDebugTask) { - // 鉴权调试 - + // 鉴权模板调试 serverAuthResult = executeAuthService.authDebugTemplate( username, new AppResourceScope(appId), plan.getTaskTemplateId(), authServers); } else { - // 鉴权执行方案 - + // 鉴权执行方案的执行 serverAuthResult = executeAuthService.authExecutePlan( username, new AppResourceScope(appId), plan.getTaskTemplateId(), plan.getId(), plan.getName(), authServers); @@ -1157,9 +1165,6 @@ private void authRedoJob(String username, long appId, TaskInstanceDTO taskInstan private StepExecuteTypeEnum getExecuteTypeFromTaskStepType(ServiceTaskStepDTO step) throws ServiceException { StepExecuteTypeEnum executeType = null; TaskStepTypeEnum stepType = TaskStepTypeEnum.valueOf(step.getType()); - if (stepType == null) { - throw new InternalException(ErrorCode.INTERNAL_ERROR); - } switch (stepType) { case SCRIPT: ScriptTypeEnum scriptType = ScriptTypeEnum.valueOf(step.getScriptStepInfo().getType()); From 51f31adbf497f8598ee6e280eb7266abd42ef8d2 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Mon, 11 Jul 2022 20:26:34 +0800 Subject: [PATCH 07/10] =?UTF-8?q?feature:=20=E5=8E=9F=E5=AD=90=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E8=83=BD=E5=8A=9B=E6=94=AF=E6=8C=81=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=20#446?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 解决重试之后，查询不到执行日志的问题 --- .../src/test/resources/init_schema.sql | 4 +- .../bk/job/execute/dao/FileAgentTaskDAO.java | 9 +++ .../job/execute/dao/ScriptAgentTaskDAO.java | 11 +++ .../dao/impl/FileAgentTaskDAOImpl.java | 21 +++++- .../dao/impl/ScriptAgentTaskDAOImpl.java | 16 ++++- .../engine/listener/GseStepEventHandler.java | 72 ++++++++++++------- .../bk/job/execute/model/AgentTaskDTO.java | 2 - .../job/execute/service/AgentTaskService.java | 9 +++ .../impl/FileAgentTaskServiceImpl.java | 5 ++ .../execute/service/impl/LogServiceImpl.java | 3 - .../impl/ScriptAgentTaskServiceImpl.java | 5 ++ 11 files changed, 122 insertions(+), 35 deletions(-) diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql index ef70ee4bf3..a659b2740e 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql @@ -211,7 +211,7 @@ CREATE TABLE IF NOT EXISTS `gse_script_agent_task` `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, `step_instance_id` bigint(20) NOT NULL, `execute_count` smallint(6) NOT NULL DEFAULT '0', - `actual_execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) DEFAULT NULL, `batch` smallint(6) NOT NULL DEFAULT '0', `host_id` bigint(20) NOT NULL DEFAULT '0', `agent_id` varchar(64) NOT NULL DEFAULT '', @@ -236,7 +236,7 @@ CREATE TABLE IF NOT EXISTS `gse_file_agent_task` `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, `step_instance_id` bigint(20) NOT NULL, `execute_count` smallint(6) NOT NULL DEFAULT '0', - `actual_execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) DEFAULT NULL, `batch` smallint(6) NOT NULL DEFAULT '0', `host_id` bigint(20) NOT NULL DEFAULT '0', `agent_id` varchar(64) NOT NULL DEFAULT '', diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java index 19b1fb52fc..51bdc7914f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java @@ -145,4 +145,13 @@ AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Int */ boolean isStepInstanceRecordExist(long stepInstanceId); + /** + * 更新Agent任务实际执行的步骤重试次数 + * + * @param stepInstanceId 步骤实例ID + * @param batch 滚动执行批次；传入null将忽略该条件 + * @param actualExecuteCount Agent任务实际执行的步骤重试次数 + */ + void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount); + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java index 443db724e6..cbe7a7e1fa 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java @@ -144,5 +144,16 @@ List listAgentTasks(Long stepInstanceId, */ boolean isStepInstanceRecordExist(long stepInstanceId); + /** + * 更新Agent任务实际执行的步骤重试次数 + * + * @param stepInstanceId 步骤实例ID + * @param batch 滚动执行批次；传入null将忽略该条件 + * @param actualExecuteCount Agent任务实际执行的步骤重试次数 + */ + void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount); + + + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java index 58fc990a1a..b4ced4e7f0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java @@ -41,7 +41,9 @@ import org.jooq.SelectLimitPercentStep; import org.jooq.SelectSeekStep1; import org.jooq.TableField; +import org.jooq.UpdateConditionStep; import org.jooq.generated.tables.GseFileAgentTask; +import org.jooq.generated.tables.records.GseFileAgentTaskRecord; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; @@ -139,9 +141,10 @@ public void batchUpdateAgentTasks(Collection agentTasks) { public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { Integer count = CTX.selectCount() .from(T_GSE_FILE_AGENT_TASK) - .where(T_GSE_FILE_AGENT_TASK.STATUS.in(AgentTaskStatus.LAST_SUCCESS.getValue(), AgentTaskStatus.SUCCESS.getValue())) + .where(T_GSE_FILE_AGENT_TASK.STATUS.in(AgentTaskStatus.LAST_SUCCESS.getValue(), + AgentTaskStatus.SUCCESS.getValue())) .and(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short)executeCount)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())) .fetchOne(0, Integer.class); return count == null ? 0 : count; @@ -153,7 +156,7 @@ public List listResultGroups(long stepInstanceId, i CTX.select(T_GSE_FILE_AGENT_TASK.STATUS, count().as("ip_count")) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short)executeCount)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); @@ -344,4 +347,16 @@ public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCou public boolean isStepInstanceRecordExist(long stepInstanceId) { return CTX.fetchExists(T_GSE_FILE_AGENT_TASK, T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); } + + @Override + public void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount) { + UpdateConditionStep updateConditionStep = + CTX.update(T_GSE_FILE_AGENT_TASK) + .set(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT, (short) actualExecuteCount) + .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); + if (batch != null) { + updateConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + updateConditionStep.execute(); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java index df412118b2..32f5041d62 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java @@ -41,7 +41,9 @@ import org.jooq.SelectLimitPercentStep; import org.jooq.SelectSeekStep1; import org.jooq.TableField; +import org.jooq.UpdateConditionStep; import org.jooq.generated.tables.GseScriptAgentTask; +import org.jooq.generated.tables.records.GseScriptAgentTaskRecord; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; @@ -149,7 +151,7 @@ public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { .where(T_GSE_SCRIPT_AGENT_TASK.STATUS.in(AgentTaskStatus.LAST_SUCCESS.getValue(), AgentTaskStatus.SUCCESS.getValue())) .and(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short)executeCount)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) .fetchOne(0, Integer.class); return count == null ? 0 : count; } @@ -354,4 +356,16 @@ public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCou public boolean isStepInstanceRecordExist(long stepInstanceId) { return CTX.fetchExists(T_GSE_SCRIPT_AGENT_TASK, T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); } + + @Override + public void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount) { + UpdateConditionStep updateConditionStep = + CTX.update(T_GSE_SCRIPT_AGENT_TASK) + .set(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT, (short) actualExecuteCount) + .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); + if (batch != null) { + updateConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + updateConditionStep.execute(); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java index 09afee4401..6432f424a5 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java @@ -24,7 +24,9 @@ package com.tencent.bk.job.execute.engine.listener; +import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.RollingModeEnum; +import com.tencent.bk.job.common.exception.NotImplementedException; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; @@ -186,7 +188,7 @@ private void startStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { } Long gseTaskId = saveInitialGseTask(stepInstance); - saveInitialGseAgentTasks(gseTaskId, stepInstance, rollingConfig); + saveGseAgentTasksForStartStep(gseTaskId, stepInstance, rollingConfig); taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.RUNNING, stepInstance.getStartTime() == null ? DateUtils.currentTimeMillis() : null, null, null); @@ -244,47 +246,69 @@ private Long saveInitialGseTask(StepInstanceDTO stepInstance) { } /** - * 初始化的GSE Agent 任务 + * 启动步骤的时候保存 GSE Agent 任务 * * @param gseTaskId GSE任务ID * @param stepInstance 步骤实例 * @param rollingConfig 滚动配置 */ - private void saveInitialGseAgentTasks(Long gseTaskId, - StepInstanceDTO stepInstance, - RollingConfigDTO rollingConfig) { - List agentTasks = new ArrayList<>(); - + private void saveGseAgentTasksForStartStep(Long gseTaskId, + StepInstanceDTO stepInstance, + RollingConfigDTO rollingConfig) { long stepInstanceId = stepInstance.getId(); int executeCount = stepInstance.getExecuteCount(); int batch = stepInstance.getBatch(); - if (stepInstance.isRollingStep()) { - if (rollingConfig.isBatchRollingStep(stepInstanceId) && stepInstance.isFirstRollingBatch()) { - List serverBatchList = rollingConfig.getConfigDetail().getServerBatchList(); - serverBatchList.forEach(serverBatch -> agentTasks.addAll(buildGseAgentTasks(stepInstanceId, - executeCount, executeCount, serverBatch.getBatch(), gseTaskId, serverBatch.getServers(), - AgentTaskStatus.WAITING))); - } else if (rollingConfig.isAllRollingStep(stepInstanceId)) { + if (needInitAgentTasksWhenStartStep(stepInstance)) { + List agentTasks = new ArrayList<>(); + + if (stepInstance.isRollingStep() && stepInstance.isFirstRollingBatch()) { + // 如果是第一批次的执行，需要初始化所有批次的agent任务（查询需要) + if (rollingConfig.isBatchRollingStep(stepInstanceId)) { + List serverBatchList = + rollingConfig.getConfigDetail().getServerBatchList(); + serverBatchList.forEach(serverBatch -> { + Integer actualExecuteCount = serverBatch.getBatch() == 1 ? executeCount : null; + agentTasks.addAll(buildGseAgentTasks(stepInstanceId, + executeCount, actualExecuteCount, serverBatch.getBatch(), gseTaskId, + serverBatch.getServers(), AgentTaskStatus.WAITING)); + }); + } else if (rollingConfig.isAllRollingStep(stepInstanceId)) { + // 暂时不支持，滚动执行二期需求 + log.warn("All rolling step is not supported!"); + throw new NotImplementedException("All rolling step is not supported", + ErrorCode.NOT_SUPPORT_FEATURE); + } + } else { agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, executeCount, batch, gseTaskId, stepInstance.getTargetServers().getIpList(), AgentTaskStatus.WAITING)); } + + // 无效主机 + if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getInvalidIpList())) { + agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, executeCount, batch, + 0L, stepInstance.getTargetServers().getInvalidIpList(), AgentTaskStatus.HOST_NOT_EXIST)); + } + saveAgentTasks(stepInstance, agentTasks); } else { - agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, executeCount, batch, - gseTaskId, stepInstance.getTargetServers().getIpList(), AgentTaskStatus.WAITING)); + // 仅更新 AgentTask 的 actualExecuteCount + if (stepInstance.isScriptStep()) { + scriptAgentTaskService.updateActualExecuteCount(stepInstanceId, batch, executeCount); + } else if (stepInstance.isFileStep()) { + fileAgentTaskService.updateActualExecuteCount(stepInstanceId, batch, executeCount); + } } - // 无效主机 - if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getInvalidIpList())) { - agentTasks.addAll(buildGseAgentTasks(stepInstanceId, executeCount, executeCount, batch, - 0L, stepInstance.getTargetServers().getInvalidIpList(), AgentTaskStatus.HOST_NOT_EXIST)); - } - saveAgentTasks(stepInstance, agentTasks); + } + + private boolean needInitAgentTasksWhenStartStep(StepInstanceBaseDTO stepInstance) { + // 步骤启动阶段(非滚动步骤启动、滚动步骤的第一批次启动)，需要初始化所有AgentTask + return !stepInstance.isRollingStep() || stepInstance.isFirstRollingBatch(); } private List buildGseAgentTasks(long stepInstanceId, int executeCount, - int actualExecuteCount, + Integer actualExecuteCount, int batch, Long gseTaskId, List hosts, @@ -297,7 +321,7 @@ private List buildGseAgentTasks(long stepInstanceId, protected AgentTaskDTO buildGseAgentTask(long stepInstanceId, int executeCount, - int actualExecuteCount, + Integer actualExecuteCount, int batch, Long gseTaskId, HostDTO host, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java index 4a8d7865b5..fe1118cfaf 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java @@ -125,7 +125,6 @@ public class AgentTaskDTO { public AgentTaskDTO(long stepInstanceId, int executeCount, int batch, Long hostId, String agentId) { this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; - this.actualExecuteCount = executeCount; this.batch = batch; this.hostId = hostId; this.agentId = agentId; @@ -135,7 +134,6 @@ public AgentTaskDTO(long stepInstanceId, int executeCount, int batch, FileTaskMo Long hostId, String agentId) { this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; - this.actualExecuteCount = executeCount; this.batch = batch; this.fileTaskMode = fileTaskMode; this.hostId = hostId; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java index 9e04392a21..5bffb4de6d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java @@ -154,5 +154,14 @@ List listAgentTaskDetail(StepInstanceBaseDTO stepInstance, @CompatibleImplementation(name = "rolling_execution", explain = "兼容历史数据", version = "3.7.x") int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp); + /** + * 更新Agent任务实际执行的步骤重试次数 + * + * @param stepInstanceId 步骤实例ID + * @param batch 滚动执行批次；传入null将忽略该条件 + * @param actualExecuteCount Agent任务实际执行的步骤重试次数 + */ + void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount); + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java index edde63c69c..bcaafdb676 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java @@ -246,4 +246,9 @@ public List listAgentTaskDetail(StepInstanceBaseDTO stepInst private boolean isStepInstanceRecordExist(long stepInstanceId) { return fileAgentTaskDAO.isStepInstanceRecordExist(stepInstanceId); } + + @Override + public void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount) { + fileAgentTaskDAO.updateActualExecuteCount(stepInstanceId, batch, actualExecuteCount); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java index 47d5ef594a..7772f35638 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java @@ -146,7 +146,6 @@ public ScriptHostLogContent getScriptHostLogContent(long stepInstanceId, int exe // 如果存在重试，那么该ip可能是之前已经执行过的，查询日志的时候需要获取到对应的executeCount int actualExecuteCount = executeCount; AgentTaskDTO agentTask = scriptAgentTaskService.getAgentTaskByHost(stepInstance, executeCount, batch, host); - log.info("Get agent task by host, host: {}, agentTask:{}", host, agentTask); if (agentTask == null) { return null; } @@ -164,12 +163,10 @@ public ScriptHostLogContent getScriptHostLogContent(long stepInstanceId, int exe if (agentTask.getHostId() != null) { resp = logServiceResourceClient.getScriptHostLogByHostId(taskCreateDateStr, stepInstanceId, actualExecuteCount, agentTask.getHostId(), batch); - log.info("Get log by hostId, resp: {}", resp); } else { // 兼容ip查询 resp = logServiceResourceClient.getScriptHostLogByIp(taskCreateDateStr, stepInstanceId, actualExecuteCount, agentTask.getCloudIp(), batch); - log.info("Get log by ip, resp: {}", resp); } if (!resp.isSuccess()) { log.error("Get script log content by host error, stepInstanceId={}, executeCount={}, batch={}, host={}", diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java index 4dbd8aa92a..24f8315bbf 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java @@ -205,4 +205,9 @@ public List listAgentTaskDetail(StepInstanceBaseDTO stepInst private boolean isStepInstanceRecordExist(long stepInstanceId) { return scriptAgentTaskDAO.isStepInstanceRecordExist(stepInstanceId); } + + @Override + public void updateActualExecuteCount(long stepInstanceId, Integer batch, int actualExecuteCount) { + scriptAgentTaskDAO.updateActualExecuteCount(stepInstanceId, batch, actualExecuteCount); + } } From 005d2b863f134a1e2aca5e6e16f89e71a8dfc619 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Mon, 11 Jul 2022 20:38:29 +0800 Subject: [PATCH 08/10] =?UTF-8?q?feature:=20=E5=8E=9F=E5=AD=90=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E8=83=BD=E5=8A=9B=E6=94=AF=E6=8C=81=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=20#446?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 解决重试之后，查询不到执行日志的问题 --- .../tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java | 3 ++- .../bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java | 3 ++- ....sql => 0018_job_execute_20220117-1000_V3.6.0.0_mysql.sql} | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) rename support-files/sql/job-execute/{0018_job_execute_20220117-1000_V3.5.0.0_mysql.sql => 0018_job_execute_20220117-1000_V3.6.0.0_mysql.sql} (97%) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java index b4ced4e7f0..d46ed8def2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java @@ -301,7 +301,8 @@ private AgentTaskDTO extract(Record record) { AgentTaskDTO agentTask = new AgentTaskDTO(); agentTask.setStepInstanceId(record.get(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID)); agentTask.setExecuteCount(record.get(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT)); - agentTask.setActualExecuteCount(record.get(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT).intValue()); + Short actualExecuteCount = record.get(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT); + agentTask.setActualExecuteCount(actualExecuteCount != null ? actualExecuteCount.intValue() : null); agentTask.setBatch(record.get(T_GSE_FILE_AGENT_TASK.BATCH)); agentTask.setFileTaskMode(FileTaskModeEnum.getFileTaskMode(record.get(T_GSE_FILE_AGENT_TASK.MODE).intValue())); agentTask.setHostId(record.get(T_GSE_FILE_AGENT_TASK.HOST_ID)); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java index 32f5041d62..1fe6ec3931 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java @@ -310,7 +310,8 @@ private AgentTaskDTO extract(Record record) { AgentTaskDTO agentTask = new AgentTaskDTO(); agentTask.setStepInstanceId(record.get(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID)); agentTask.setExecuteCount(record.get(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT)); - agentTask.setActualExecuteCount(record.get(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT).intValue()); + Short actualExecuteCount = record.get(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT); + agentTask.setActualExecuteCount(actualExecuteCount != null ? actualExecuteCount.intValue() : null); agentTask.setBatch(record.get(T_GSE_SCRIPT_AGENT_TASK.BATCH)); agentTask.setHostId(record.get(T_GSE_SCRIPT_AGENT_TASK.HOST_ID)); agentTask.setAgentId(record.get(T_GSE_SCRIPT_AGENT_TASK.AGENT_ID)); diff --git a/support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.5.0.0_mysql.sql b/support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.6.0.0_mysql.sql similarity index 97% rename from support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.5.0.0_mysql.sql rename to support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.6.0.0_mysql.sql index 9e10162e7e..b274fa9677 100644 --- a/support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.5.0.0_mysql.sql +++ b/support-files/sql/job-execute/0018_job_execute_20220117-1000_V3.6.0.0_mysql.sql @@ -38,7 +38,7 @@ BEGIN `id` bigint(20) NOT NULL AUTO_INCREMENT, `step_instance_id` bigint(20) NOT NULL, `execute_count` smallint(6) NOT NULL DEFAULT '0', - `actual_execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) DEFAULT NULL, `batch` smallint(6) NOT NULL DEFAULT '0', `host_id` bigint(20) NOT NULL DEFAULT '0', `agent_id` varchar(64) NOT NULL DEFAULT '', @@ -66,7 +66,7 @@ BEGIN `id` bigint(20) NOT NULL AUTO_INCREMENT, `step_instance_id` bigint(20) NOT NULL, `execute_count` smallint(6) NOT NULL DEFAULT '0', - `actual_execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) DEFAULT NULL, `batch` smallint(6) NOT NULL DEFAULT '0', `host_id` bigint(20) NOT NULL DEFAULT '0', `agent_id` varchar(64) NOT NULL DEFAULT '', From 364c039f9f9a79a9660810b75c82a5fefe3f6f94 Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Mon, 11 Jul 2022 21:36:51 +0800 Subject: [PATCH 09/10] =?UTF-8?q?feature:=20=E5=8E=9F=E5=AD=90=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E8=83=BD=E5=8A=9B=E6=94=AF=E6=8C=81=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=20#446?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 解决滚动等待状态的步骤，无法被终止的问题 --- .../tencent/bk/job/execute/engine/listener/JobListener.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java index 6f6cbd88e5..8876cfacf9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java @@ -204,10 +204,8 @@ private void refreshJob(TaskInstanceDTO taskInstance) { // 验证作业状态，只有'正在执行'、'强制终止中'的作业可以刷新状态进入下一步或者结束 if (RunStatusEnum.STOPPING.getValue() == taskStatus) { - if (RunStatusEnum.STOP_SUCCESS.getValue() == stepStatus - || RunStatusEnum.SUCCESS.getValue() == stepStatus - || RunStatusEnum.FAIL.getValue() == stepStatus) { - + // 非正在执行的步骤可以直接终止 + if (RunStatusEnum.RUNNING.getValue() != stepStatus) { finishJob(taskInstance, currentStepInstance, RunStatusEnum.STOP_SUCCESS); } else { log.error("Unsupported task instance run status for refresh task, taskInstanceId={}, taskStatus={}, " + From 4e02122994d90a5ec5a12046785163dab48ec2ac Mon Sep 17 00:00:00 2001 From: wangyu096 Date: Tue, 12 Jul 2022 10:31:25 +0800 Subject: [PATCH 10/10] =?UTF-8?q?feature:=20=E5=8E=9F=E5=AD=90=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E8=83=BD=E5=8A=9B=E6=94=AF=E6=8C=81=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=20#446?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复CodeReview问题 --- .../service/impl/TaskExecuteServiceImpl.java | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index aea7d1f5c9..f1132121a3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -703,26 +703,36 @@ private void fillStepHostDetail(List stepInstanceList, ServiceL if (stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue())) { continue; } - if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getStaticIpList())) { - stepInstance.getTargetServers().getStaticIpList() - .forEach(host -> fillHostDetail(host, hostMap)); - } - if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getIpList())) { - stepInstance.getTargetServers().getIpList() - .forEach(host -> fillHostDetail(host, hostMap)); - } - if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { - List fileSourceList = stepInstance.getFileSourceList(); - if (fileSourceList != null) { - for (FileSourceDTO fileSource : fileSourceList) { - ServersDTO servers = fileSource.getServers(); - if (servers != null) { - if (CollectionUtils.isNotEmpty(servers.getStaticIpList())) { - servers.getStaticIpList().forEach(host -> fillHostDetail(host, hostMap)); - } - if (CollectionUtils.isNotEmpty(servers.getIpList())) { - servers.getIpList().forEach(host -> fillHostDetail(host, hostMap)); - } + // 目标主机设置主机详情 + fillTargetHostDetail(stepInstance, hostMap); + // 文件源设置主机详情 + fillFileSourceHostDetail(stepInstance, hostMap); + } + } + + private void fillTargetHostDetail(StepInstanceDTO stepInstance, Map hostMap) { + if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getStaticIpList())) { + stepInstance.getTargetServers().getStaticIpList() + .forEach(host -> fillHostDetail(host, hostMap)); + } + if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getIpList())) { + stepInstance.getTargetServers().getIpList() + .forEach(host -> fillHostDetail(host, hostMap)); + } + } + + private void fillFileSourceHostDetail(StepInstanceDTO stepInstance, Map hostMap) { + if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { + List fileSourceList = stepInstance.getFileSourceList(); + if (fileSourceList != null) { + for (FileSourceDTO fileSource : fileSourceList) { + ServersDTO servers = fileSource.getServers(); + if (servers != null) { + if (CollectionUtils.isNotEmpty(servers.getStaticIpList())) { + servers.getStaticIpList().forEach(host -> fillHostDetail(host, hostMap)); + } + if (CollectionUtils.isNotEmpty(servers.getIpList())) { + servers.getIpList().forEach(host -> fillHostDetail(host, hostMap)); } } }