Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: 当业务下有大量的执行作业，加载 web 页面首页的时候会触发DB慢查询 #2228 #2236

Merged
merged 1 commit into from
Jul 18, 2023
Merged

fix: 当业务下有大量的执行作业，加载 web 页面首页的时候会触发DB慢查询 #2228 #2236

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions ...end/commons/common/src/main/java/com/tencent/bk/job/common/model/BaseSearchCondition.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@

package com.tencent.bk.job.common.model;

import io.swagger.annotations.ApiModelProperty;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
Expand All @@ -41,24 +40,25 @@ public class BaseSearchCondition implements Cloneable {
/**
* 分页起始
*/
@ApiModelProperty(value = "分页起始", required = false)
private Integer start;
/**
* 分页大小
*/
@ApiModelProperty(value = "分页大小", required = false)
private Integer length;

/**
* 分页-是否计算总数；默认计算。分页计算总数可能会影响 API 性能, 必要场景才可使用
*/
private boolean countPageTotal = true;

/**
* 排序 0：降序 1：升序
*/
@ApiModelProperty(value = "排序 0：降序 1：升序", required = false)
private Integer order;

/**
* 排序的字段
*/
@ApiModelProperty(value = "排序的字段", required = false)
private String orderField;


Expand Down Expand Up @@ -94,6 +94,15 @@ public static BaseSearchCondition pageCondition(Integer start, Integer length) {
BaseSearchCondition searchCondition = new BaseSearchCondition();
searchCondition.setStart(start);
searchCondition.setLength(length);
searchCondition.setCountPageTotal(true);
return searchCondition;
}

public static BaseSearchCondition pageCondition(Integer start, Integer length, boolean countPageTotal) {
BaseSearchCondition searchCondition = new BaseSearchCondition();
searchCondition.setStart(start);
searchCondition.setLength(length);
searchCondition.setCountPageTotal(countPageTotal);
return searchCondition;
}

Expand Down
7 changes: 5 additions & 2 deletions ...cute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,12 +106,15 @@ Response<PageData<TaskInstanceVO>> getTaskHistoryList(
@ApiParam(value = "耗时类型", name = "totalTimeType")
@RequestParam(value = "totalTimeType", required = false)
TaskTotalTimeTypeEnum totalTimeType,
@ApiParam(value = "分页-开始")
@ApiParam(value = "分页-开始，默认值：0")
@RequestParam(value = "start", required = false)
Integer start,
@ApiParam(value = "分页-每页大小")
@ApiParam(value = "分页-每页大小，默认值：10")
@RequestParam(value = "pageSize", required = false)
Integer pageSize,
@ApiParam(value = "分页-是否计算总数；默认值：true。计算总数可能会影响 API 性能, 必要场景才可使用")
@RequestParam(value = "countPageTotal", required = false)
Boolean countPageTotal,
@ApiParam(value = "定时任务ID")
@RequestParam(value = "cronTaskId", required = false)
Long cronTaskId,
Expand Down
91 changes: 51 additions & 40 deletions ...main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
import com.tencent.bk.job.common.constant.ErrorCode;
import com.tencent.bk.job.common.constant.Order;
import com.tencent.bk.job.common.constant.TaskVariableTypeEnum;
import com.tencent.bk.job.common.exception.FailedPreconditionException;
import com.tencent.bk.job.common.exception.InvalidParamException;
import com.tencent.bk.job.common.exception.NotFoundException;
import com.tencent.bk.job.common.gse.constants.FileDistModeEnum;
import com.tencent.bk.job.common.i18n.service.MessageI18nService;
Expand All @@ -41,7 +43,6 @@
import com.tencent.bk.job.common.model.InternalResponse;
import com.tencent.bk.job.common.model.PageData;
import com.tencent.bk.job.common.model.Response;
import com.tencent.bk.job.common.model.ValidateResult;
import com.tencent.bk.job.common.model.dto.AppResourceScope;
import com.tencent.bk.job.common.model.dto.HostDTO;
import com.tencent.bk.job.common.util.CustomCollectionUtils;
Expand Down Expand Up @@ -212,19 +213,55 @@ public Response<PageData<TaskInstanceVO>> getTaskHistoryList(String username,
TaskTotalTimeTypeEnum totalTimeType,
Integer start,
Integer pageSize,
Boolean countPageTotal,
Long cronTaskId,
String startupModes,
String ip) {
TaskInstanceQuery taskQuery = buildListTaskInstanceQuery(appResourceScope, taskName, taskInstanceId,
status, operator, taskType, startTime, endTime, timeRange, totalTimeType, cronTaskId, startupModes, ip);
BaseSearchCondition baseSearchCondition = BaseSearchCondition.pageCondition(start, pageSize,
countPageTotal == null ? true : countPageTotal);

PageData<TaskInstanceDTO> pageData = taskResultService.listPageTaskInstance(taskQuery, baseSearchCondition);
if (pageData == null) {
return Response.buildSuccessResp(PageData.emptyPageData(start, pageSize));
}

PageData<TaskInstanceVO> pageDataVO = new PageData<>();
pageDataVO.setTotal(pageData.getTotal());
pageDataVO.setStart(pageData.getStart());
pageDataVO.setPageSize(pageData.getPageSize());

List<TaskInstanceVO> taskInstanceVOS = new ArrayList<>();
if (pageData.getData() != null) {
pageData.getData().forEach(taskInstanceDTO -> taskInstanceVOS.add(TaskInstanceConverter
.convertToTaskInstanceVO(taskInstanceDTO)));
}
pageDataVO.setData(taskInstanceVOS);
batchSetPermissionsForTaskInstance(username, appResourceScope, taskInstanceVOS);
return Response.buildSuccessResp(pageDataVO);
}

private TaskInstanceQuery buildListTaskInstanceQuery(AppResourceScope appResourceScope,
String taskName,
Long taskInstanceId,
Integer status,
String operator,
Integer taskType,
String startTime,
String endTime,
Integer timeRange,
TaskTotalTimeTypeEnum totalTimeType,
Long cronTaskId,
String startupModes,
String ip) {
TaskInstanceQuery taskQuery = new TaskInstanceQuery();
taskQuery.setTaskInstanceId(taskInstanceId);
taskQuery.setAppId(appResourceScope.getAppId());
taskQuery.setTaskName(taskName);
taskQuery.setCronTaskId(cronTaskId);

ValidateResult validateResult = validateAndSetQueryTimeRange(taskQuery, startTime, endTime, timeRange);
if (!validateResult.isPass()) {
return Response.buildValidateFailResp(validateResult);
}
validateAndSetQueryTimeRange(taskQuery, startTime, endTime, timeRange);

setTotalTimeCondition(taskQuery, totalTimeType);
taskQuery.setOperator(operator);
Expand All @@ -245,49 +282,24 @@ public Response<PageData<TaskInstanceVO>> getTaskHistoryList(String username,
}
}
taskQuery.setIp(ip);
BaseSearchCondition baseSearchCondition = new BaseSearchCondition();
baseSearchCondition.setStart(start);
baseSearchCondition.setLength(pageSize);


PageData<TaskInstanceDTO> pageData;
if (timeRange == null) {
pageData = taskResultService.listPageTaskInstance(taskQuery, baseSearchCondition);
} else {
// 临时优化代码，使用 timeRange 的 web 页面会触发全表扫描的问题。后续跟前端一块优化
pageData = taskResultService.listPageTaskInstanceWithoutCount(taskQuery, baseSearchCondition);
}
if (pageData == null) {
return Response.buildSuccessResp(PageData.emptyPageData(start, pageSize));
}

PageData<TaskInstanceVO> pageDataVO = new PageData<>();
pageDataVO.setTotal(pageData.getTotal());
pageDataVO.setStart(pageData.getStart());
pageDataVO.setPageSize(pageData.getPageSize());

List<TaskInstanceVO> taskInstanceVOS = new ArrayList<>();
if (pageData.getData() != null) {
pageData.getData().forEach(taskInstanceDTO -> taskInstanceVOS.add(TaskInstanceConverter
.convertToTaskInstanceVO(taskInstanceDTO)));
}
pageDataVO.setData(taskInstanceVOS);
batchSetPermissionsForTaskInstance(username, appResourceScope, taskInstanceVOS);
return Response.buildSuccessResp(pageDataVO);
return taskQuery;
}

private ValidateResult validateAndSetQueryTimeRange(TaskInstanceQuery taskInstanceQuery, String startTime,
String endTime, Integer timeRange) {
private void validateAndSetQueryTimeRange(TaskInstanceQuery taskInstanceQuery,
String startTime,
String endTime,
Integer timeRange) {
Long start = null;
Long end = null;
if (timeRange != null) {
if (timeRange < 1) {
log.warn("Param timeRange should greater than 0");
return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM);
throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM);
}
if (timeRange > 30) {
log.warn("Param timeRange should less then 30");
return ValidateResult.fail(ErrorCode.TASK_INSTANCE_QUERY_TIME_SPAN_MORE_THAN_30_DAYS);
throw new FailedPreconditionException(ErrorCode.TASK_INSTANCE_QUERY_TIME_SPAN_MORE_THAN_30_DAYS);
}
// 当天结束时间 - 往前的天数
start = DateUtils.getUTCCurrentDayEndTimestamp() - timeRange * 24 * 3600 * 1000L;
Expand All @@ -303,20 +315,19 @@ private ValidateResult validateAndSetQueryTimeRange(TaskInstanceQuery taskInstan

if (start == null) {
log.info("StartTime should not be empty!");
return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM);
throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM);
}
if (end == null) {
end = System.currentTimeMillis();
}
if (end - start > MAX_SEARCH_TASK_HISTORY_RANGE_MILLS) {
log.info("Query task instance history time span must be less than 30 days");
return ValidateResult.fail(ErrorCode.TASK_INSTANCE_QUERY_TIME_SPAN_MORE_THAN_30_DAYS);
throw new FailedPreconditionException(ErrorCode.TASK_INSTANCE_QUERY_TIME_SPAN_MORE_THAN_30_DAYS);
}
}

taskInstanceQuery.setStartTime(start);
taskInstanceQuery.setEndTime(end);
return ValidateResult.pass();
}

private void setTotalTimeCondition(TaskInstanceQuery taskQuery, TaskTotalTimeTypeEnum totalTimeType) {
Expand Down
9 changes: 0 additions & 9 deletions ...ute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceDAO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,6 @@ public interface TaskInstanceDAO {
PageData<TaskInstanceDTO> listPageTaskInstance(TaskInstanceQuery taskQuery,
BaseSearchCondition baseSearchCondition);

/**
* 分页查询作业执行实例（不包含总数）
*
* @param taskQuery 查询
* @param baseSearchCondition 基础查询条件
* @return 分页结果
*/
PageData<TaskInstanceDTO> listPageTaskInstanceWithoutCount(TaskInstanceQuery taskQuery,
BaseSearchCondition baseSearchCondition);

void addCallbackUrl(long taskInstanceId, String callBackUrl);

Expand Down
25 changes: 6 additions & 19 deletions ...ce-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,27 +235,20 @@ public void updateTaskTotalTime(Long taskInstanceId, Long totalTime) {
@Override
public PageData<TaskInstanceDTO> listPageTaskInstance(TaskInstanceQuery taskQuery,
BaseSearchCondition baseSearchCondition) {
return listPageTaskInstanceInternal(taskQuery, baseSearchCondition, true);
}

private PageData<TaskInstanceDTO> listPageTaskInstanceInternal(TaskInstanceQuery taskQuery,
BaseSearchCondition baseSearchCondition,
boolean countTotalRecords) {
if (StringUtils.isNotEmpty(taskQuery.getIp()) || StringUtils.isNotEmpty(taskQuery.getIpv6())) {
return listPageTaskInstanceByIp(taskQuery, baseSearchCondition, countTotalRecords);
return listPageTaskInstanceByIp(taskQuery, baseSearchCondition);
} else {
return listPageTaskInstanceByBasicInfo(taskQuery, baseSearchCondition, countTotalRecords);
return listPageTaskInstanceByBasicInfo(taskQuery, baseSearchCondition);
}
}

private PageData<TaskInstanceDTO> listPageTaskInstanceByBasicInfo(TaskInstanceQuery taskQuery,
BaseSearchCondition baseSearchCondition,
boolean countTotalRecords) {
BaseSearchCondition baseSearchCondition) {
int start = baseSearchCondition.getStartOrDefault(0);
int length = baseSearchCondition.getLengthOrDefault(10);

int count = 0;
if (countTotalRecords) {
if (baseSearchCondition.isCountPageTotal()) {
count = getPageTaskInstanceCount(taskQuery);
if (count == 0) {
return PageData.emptyPageData(start, length);
Expand All @@ -281,8 +274,7 @@ private PageData<TaskInstanceDTO> listPageTaskInstanceByBasicInfo(TaskInstanceQu
}

private PageData<TaskInstanceDTO> listPageTaskInstanceByIp(TaskInstanceQuery taskQuery,
BaseSearchCondition baseSearchCondition,
boolean countTotalRecords) {
BaseSearchCondition baseSearchCondition) {
List<Condition> conditions = buildSearchCondition(taskQuery);
if (StringUtils.isNotEmpty(taskQuery.getIp())) {
conditions.add(TASK_INSTANCE_HOST.IP.eq(taskQuery.getIp()));
Expand All @@ -292,7 +284,7 @@ private PageData<TaskInstanceDTO> listPageTaskInstanceByIp(TaskInstanceQuery tas
int start = baseSearchCondition.getStartOrDefault(0);
int length = baseSearchCondition.getLengthOrDefault(10);
int count = 0;
if (countTotalRecords) {
if (baseSearchCondition.isCountPageTotal()) {
count = ctx.selectCount().from(TaskInstance.TASK_INSTANCE)
.leftJoin(TASK_INSTANCE_HOST).on(TaskInstance.TASK_INSTANCE.ID.eq(TASK_INSTANCE_HOST.TASK_INSTANCE_ID))
.where(conditions)
Expand Down Expand Up @@ -384,11 +376,6 @@ private List<Condition> buildSearchCondition(TaskInstanceQuery taskQuery) {
return conditions;
}

public PageData<TaskInstanceDTO> listPageTaskInstanceWithoutCount(TaskInstanceQuery taskQuery,
BaseSearchCondition baseSearchCondition) {
return listPageTaskInstanceInternal(taskQuery, baseSearchCondition, false);
}

@Override
public void addCallbackUrl(long taskInstanceId, String callBackUrl) {
ctx.update(TASK_INSTANCE).set(TASK_INSTANCE.CALLBACK_URL, callBackUrl)
Expand Down
9 changes: 0 additions & 9 deletions ...rvice-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,6 @@ public interface TaskResultService {
PageData<TaskInstanceDTO> listPageTaskInstance(TaskInstanceQuery taskQuery,
BaseSearchCondition baseSearchCondition);

/**
* 分页查询作业执行实例（不包含总数）
*
* @param taskQuery 查询
* @param baseSearchCondition 基础查询条件
* @return 分页结果
*/
PageData<TaskInstanceDTO> listPageTaskInstanceWithoutCount(TaskInstanceQuery taskQuery,
BaseSearchCondition baseSearchCondition);

/**
* 获取作业执行结果
Expand Down
9 changes: 0 additions & 9 deletions ...-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,15 +163,6 @@ private void computeTotalTime(List<TaskInstanceDTO> pageData) {
}
}

@Override
public PageData<TaskInstanceDTO> listPageTaskInstanceWithoutCount(TaskInstanceQuery taskQuery,
BaseSearchCondition baseSearchCondition) {
PageData<TaskInstanceDTO> pageData = taskInstanceDAO.listPageTaskInstanceWithoutCount(
taskQuery, baseSearchCondition);
computeTotalTime(pageData.getData());
return pageData;
}

@Override
public TaskExecuteResultDTO getTaskExecutionResult(String username, Long appId,
Long taskInstanceId) throws ServiceException {
Expand Down