Skip to content

Commit

Permalink
feature: 异常通知的收敛/抑制能力 TencentBlueKing#205
Browse files Browse the repository at this point in the history
  • Loading branch information
wadema committed Sep 11, 2021
1 parent a7c115c commit 7154c56
Show file tree
Hide file tree
Showing 8 changed files with 272 additions and 124 deletions.
12 changes: 12 additions & 0 deletions ...job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobVO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,18 @@ public class CronJobVO {
@ApiModelProperty("上次执行结果 0 - 未执行 1 - 成功 2 - 失败")
private Integer lastExecuteStatus;

/**
* 上次执行错误码
*/
@ApiModelProperty("上次执行错误码")
private Long lastExecuteErrorCode;

/**
* 上次执行错误次数
*/
@ApiModelProperty("上次执行错误次数")
private Integer lastExecuteErrorCount;

/**
* 是否启用
*/
Expand Down
18 changes: 18 additions & 0 deletions ...-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/CronJobDAO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,15 @@ PageData<CronJobInfoDTO> listPageCronJobsByCondition(CronJobInfoDTO cronJobCondi
*/
CronJobInfoDTO getCronJobById(long appId, long cronJobId);

/**
* 根据定时任务 ID 查询定时任务简单信息，上次执行状态，错误码，错误次数
*
* @param appId 业务 ID
* @param cronJobId 定时任务 ID
* @return 定时任务信息
*/
CronJobInfoDTO getCronJobSimpleById(long appId, long cronJobId);

/**
* 新增定时任务信息
*
Expand All @@ -86,6 +95,14 @@ PageData<CronJobInfoDTO> listPageCronJobsByCondition(CronJobInfoDTO cronJobCondi
*/
boolean updateCronJobById(CronJobInfoDTO cronJob);

/**
* 根据 ID 和定时任务 ID 更新定时任务
*
* @param cronJobSimpleInfo 定时任务简单信息
* @return 是否更新成功
*/
boolean updateCronJobSimpleById(CronJobInfoDTO cronJobSimpleInfo);

/**
* 根据定时任务 ID 删除定时任务
*
Expand Down Expand Up @@ -148,4 +165,5 @@ PageData<CronJobInfoDTO> listPageCronJobsByCondition(CronJobInfoDTO cronJobCondi
boolean isExistAnyAppCronJob(Long appId);

Integer countCronJob(Long appId, Boolean active, Boolean cron);

}
184 changes: 117 additions & 67 deletions ...service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/CronJobDAOImpl.java
View file

Large diffs are not rendered by default.

12 changes: 12 additions & 0 deletions ...ervice-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobInfoDTO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,16 @@ public class CronJobInfoDTO {
*/
private Integer lastExecuteStatus;

/**
* 上次执行错误码
*/
private Long lastExecuteErrorCode;

/**
* 上次执行错误次数
*/
private Integer lastExecuteErrorCount;

/**
* 是否启用
*/
Expand Down Expand Up @@ -191,6 +201,8 @@ public static CronJobVO toVO(CronJobInfoDTO cronJobInfo) {
cronJobVO.setVariableValue(Collections.emptyList());
}
cronJobVO.setLastExecuteStatus(cronJobInfo.getLastExecuteStatus());
cronJobVO.setLastExecuteErrorCode(cronJobInfo.getLastExecuteErrorCode());
cronJobVO.setLastExecuteErrorCount(cronJobInfo.getLastExecuteErrorCount());
cronJobVO.setEnable(cronJobInfo.getEnable());
cronJobVO.setLastModifyUser(cronJobInfo.getLastModifyUser());
cronJobVO.setLastModifyTime(cronJobInfo.getLastModifyTime());
Expand Down
18 changes: 18 additions & 0 deletions .../service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,23 @@ PageData<CronJobInfoDTO> listPageCronJobInfos(CronJobInfoDTO cronJobCondition,
*/
CronJobInfoDTO getCronJobInfoById(Long appId, Long cronJobId);

/**
* 根据 ID 查询定时任务简单信息，上次执行状态，错误码，错误次数
*
* @param appId 业务 ID
* @param cronJobId 定时任务 ID
* @return 定时任务信息
*/
CronJobInfoDTO getCronJobSimpleInfoById(Long appId, Long cronJobId);

/**
* 根据 ID 和定时任务 ID 更新定时任务上次执行状态，错误码，错误次数简单信息
*
* @param cronJobSimpleInfo 定时任务简单信息
* @return 是否更新成功
*/
boolean updateCronJobSimpleById(CronJobInfoDTO cronJobSimpleInfo);

/**
* 新增、保存定时任务信息
*
Expand Down Expand Up @@ -228,4 +245,5 @@ PageData<CronJobInfoDTO> listPageCronJobInfos(CronJobInfoDTO cronJobCondition,
boolean isExistAnyAppCronJob(Long appId);

Integer countCronJob(Long appId, Boolean active, Boolean cron);

}
108 changes: 59 additions & 49 deletions ...job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobServiceImpl.java
View file

Large diffs are not rendered by default.

38 changes: 30 additions & 8 deletions ...ob-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/SimpleJobExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ protected void executeInternalInternal(JobExecutionContext context) throws JobEx
long scheduledFireTime = getScheduledFireTime(context).toEpochMilli();

CronJobHistoryDTO cronJobHistory =
cronJobHistoryService.getHistoryByIdAndTime(appId, cronJobId, scheduledFireTime);
cronJobHistoryService.getHistoryByIdAndTime(appId, cronJobId, scheduledFireTime);
if (cronJobHistory != null) {
log.warn("Job already running!|{}", cronJobHistory);
return;
Expand All @@ -109,50 +109,72 @@ protected void executeInternalInternal(JobExecutionContext context) throws JobEx
log.debug("Get cronjob info return|{}", cronJobInfo);
}

CronJobInfoDTO cronJobSimpleInfo = cronJobService.getCronJobSimpleInfoById(appId, cronJobId);
if (log.isDebugEnabled()) {
log.debug("Get cronjob simple info return|{}", cronJobSimpleInfo);
}
Integer lastExecuteStatus = cronJobSimpleInfo.getLastExecuteStatus();
Long lastExecuteErrorCode = cronJobSimpleInfo.getLastExecuteErrorCode();
Integer lastExecuteErrorCount = cronJobSimpleInfo.getLastExecuteErrorCount();

List<CronJobVariableDTO> variables = cronJobInfo.getVariableValue();
List<ServiceTaskVariable> taskVariables = null;
if (CollectionUtils.isNotEmpty(variables)) {
taskVariables =
variables.parallelStream().map(CronJobVariableDTO::toServiceTaskVariable).collect(Collectors.toList());
variables.parallelStream().map(CronJobVariableDTO::toServiceTaskVariable).collect(Collectors.toList());
}

boolean executeFailed = false;
Integer errorCode = null;
String errorMessage = null;
cronJobHistoryService.fillExecutor(historyId, cronJobInfo.getLastModifyUser());
ServiceResponse<ServiceTaskExecuteResult> executeResult = executeTaskService.executeTask(appId,
cronJobInfo.getTaskPlanId(),
cronJobInfo.getId(), cronJobInfo.getName(), taskVariables, cronJobInfo.getLastModifyUser());
cronJobInfo.getTaskPlanId(),
cronJobInfo.getId(), cronJobInfo.getName(), taskVariables, cronJobInfo.getLastModifyUser());
if (log.isDebugEnabled()) {
log.debug("Execute result|{}", executeResult);
}
if (executeResult != null && executeResult.getData() != null
&& executeResult.getData().getTaskInstanceId() > 0) {
&& executeResult.getData().getTaskInstanceId() > 0) {
if (log.isDebugEnabled()) {
log.debug("Execute success! Task instance id {}", executeResult.getData().getTaskInstanceId());
}
cronJobHistoryService.updateStatusByIdAndTime(appId, cronJobId, scheduledFireTime,
ExecuteStatusEnum.RUNNING);
ExecuteStatusEnum.RUNNING);
cronJobSimpleInfo.setLastExecuteStatus(1);
cronJobSimpleInfo.setLastExecuteErrorCode(null);
cronJobSimpleInfo.setLastExecuteErrorCount(0);
} else {
log.error("Execute task failed!|{}|{}|{}|{}", appId, cronJobId, scheduledFireTime, executeResult);
cronJobHistoryService.updateStatusByIdAndTime(appId, cronJobId, scheduledFireTime, ExecuteStatusEnum.FAIL);
cronJobSimpleInfo.setLastExecuteStatus(2);
executeFailed = true;
if (executeResult != null) {
errorCode = executeResult.getCode();
errorMessage = executeResult.getErrorMsg();
if (errorCode != null) {
cronJobHistoryService.fillErrorInfo(historyId, errorCode.longValue(), errorMessage);
cronJobSimpleInfo.setLastExecuteErrorCode(errorCode.longValue());
if (errorCode.longValue() == lastExecuteErrorCode) {
cronJobSimpleInfo.setLastExecuteErrorCount(lastExecuteErrorCount + 1);
} else {
cronJobSimpleInfo.setLastExecuteErrorCount(1);
}
}
}
}

cronJobService.updateCronJobSimpleById(cronJobSimpleInfo);

if (context.getNextFireTime() == null) {
cronJobService.disableExpiredCronJob(appId, cronJobId, cronJobInfo.getLastModifyUser(),
cronJobInfo.getLastModifyTime());
cronJobInfo.getLastModifyTime());
}

if (executeFailed) {
notifyService.sendCronJobFailedNotification(errorCode, errorMessage, cronJobInfo);
if (cronJobSimpleInfo.getLastExecuteErrorCount() % 5 == 1 || !lastExecuteStatus.equals(cronJobSimpleInfo.getLastExecuteStatus())) {
notifyService.sendCronJobFailedNotification(errorCode, errorMessage, cronJobInfo);
}
}
}
}
6 changes: 6 additions & 0 deletions support-files/sql/job-crontab/0003_job_crontab_20210909-1000_V3.4.0.0_mysql.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
SET NAMES utf8mb4;
USE job_crontab;

ALTER TABLE `job_crontab`.`cron_job` ADD COLUMN `last_execute_error_code` bigint(20) UNSIGNED NULL DEFAULT NULL AFTER `last_execute_status`;

ALTER TABLE `job_crontab`.`cron_job` ADD COLUMN `last_execute_error_count` int(11) UNSIGNED NOT NULL DEFAULT '0' AFTER `last_execute_error_code`;

0 comments on commit 7154c56

Please sign in to comment.