Merge pull request #2229 from wangyu096/issue_2228

fix: 当业务下有大量的执行作业，加载 web 页面首页的时候会触发DB慢查询 #2228
TencentBlueKing · Jul 14, 2023 · 19e87f5 · 19e87f5
2 parents c92dae5 + 56fc950
commit 19e87f5
Show file tree

Hide file tree

Showing 12 changed files with 73 additions and 310 deletions.
diff --git a/...end/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java b/...end/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java
@@ -34,6 +34,7 @@
 import java.text.SimpleDateFormat;
 import java.time.Duration;
 import java.time.Instant;
+import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
@@ -353,4 +354,23 @@ public static Long calcDaysBetween(String startDateStr, String endDateStr, Strin
             return null;
         }
     }
+
+    /**
+     * 计算当天（UTC 时区)的截止时间戳
+     *
+     * @return 当天（UTC 时区)的截止时间戳，单位毫秒
+     */
+    public static long getUTCCurrentDayEndTimestamp() {
+        return getUTCDayEndTimestamp(LocalDate.now(ZoneId.of("UTC")));
+    }
+
+    /**
+     * 根据日期（UTC 时区)计算这一天的截止时间戳
+     *
+     * @return 日期（UTC 时区)对应当天截止时间戳，单位毫秒
+     */
+    public static long getUTCDayEndTimestamp(LocalDate localDateUTC) {
+
+        return 1000 * (localDateUTC.atStartOfDay().toEpochSecond(ZoneOffset.UTC) + 86400);
+    }
 }
diff --git a/...commons/common-utils/src/test/java/com/tencent/bk/job/common/util/date/DateUtilsTest.java b/...commons/common-utils/src/test/java/com/tencent/bk/job/common/util/date/DateUtilsTest.java
@@ -26,6 +26,10 @@
 
 import org.junit.jupiter.api.Test;
 
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class DateUtilsTest {
@@ -42,4 +46,32 @@ public void testGetPreviousDate() {
         assertThat(DateUtils.calcDaysBetween("2021-2-28", "2021-3-1")).isEqualTo(1);
         assertThat(DateUtils.calcDaysBetween("2020-2-28", "2020-3-1")).isEqualTo(2);
     }
+
+    @Test
+    void getUTCDayEndTimestamp() {
+        ZonedDateTime dateTime = LocalDateTime.of(2023, 7, 14, 15, 14, 23, 12312)
+            .atZone(ZoneId.of("UTC"));
+        long endTime = DateUtils.getUTCDayEndTimestamp(dateTime.toLocalDate());
+        // 2023-07-15 00:00:00 UTC
+        assertThat(endTime).isEqualTo(1689379200000L);
+
+
+        ZonedDateTime dateTime2 = LocalDateTime.of(2023, 7, 14, 7, 15, 22, 123)
+            .atZone(ZoneId.of("UTC"));
+        long endTime2 = DateUtils.getUTCDayEndTimestamp(dateTime2.toLocalDate());
+        // 2023-07-15 00:00:00 UTC
+        assertThat(endTime2).isEqualTo(1689379200000L);
+
+        ZonedDateTime dateTime3 = LocalDateTime.of(2023, 7, 13, 0, 0, 0, 0)
+            .atZone(ZoneId.of("UTC"));
+        long endTime3 = DateUtils.getUTCDayEndTimestamp(dateTime3.toLocalDate());
+        // 2023-07-14 00:00:00 UTC
+        assertThat(endTime3).isEqualTo(1689292800000L);
+
+        ZonedDateTime dateTime4 = LocalDateTime.of(2023, 7, 13, 23, 59, 59, 99999999)
+            .atZone(ZoneId.of("UTC"));
+        long endTime4 = DateUtils.getUTCDayEndTimestamp(dateTime4.toLocalDate());
+        // 2023-07-14 00:00:00 UTC
+        assertThat(endTime4).isEqualTo(1689292800000L);
+    }
 }
diff --git a/...mon-web/src/main/java/com/tencent/bk/job/common/web/interceptor/EsbApiLogInterceptor.java b/...mon-web/src/main/java/com/tencent/bk/job/common/web/interceptor/EsbApiLogInterceptor.java
@@ -68,6 +68,7 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons
             request.setAttribute(ATTR_REQUEST_START, System.currentTimeMillis());
             apiName = getAPIName(wrapperRequest.getRequestURI());
             request.setAttribute(ATTR_API_NAME, apiName);
+            desensitizedQueryParams = desensitizeQueryParams(request.getQueryString());
             if (request.getMethod().equals(HttpMethod.POST.name())
                 || request.getMethod().equals(HttpMethod.PUT.name())) {
                 if (StringUtils.isNotBlank(wrapperRequest.getBody())) {
@@ -85,17 +86,15 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons
             } else if (request.getMethod().equals(HttpMethod.GET.name())) {
                 username = request.getParameter("bk_username");
                 appCode = request.getParameter("bk_app_code");
-                desensitizedQueryParams = desensitizeQueryParams(request.getQueryString());
-
                 request.setAttribute(ATTR_USERNAME, username);
                 request.setAttribute(ATTR_APP_CODE, appCode);
             }
         } catch (Throwable e) {
             return true;
         } finally {
-            log.info("request-id:{}|lang:{}|API:{}|uri:{}|appCode:{}|username:{}|body:{}|queryParams:{}", requestId,
-                lang, apiName,
-                request.getRequestURI(), appCode, username, desensitizedBody, desensitizedQueryParams);
+            log.info("request-id: {}|lang: {}|API: {}|uri: {}|appCode: {}|username: {}|body: {}|queryParams: {}",
+                requestId, lang, apiName, request.getRequestURI(), appCode, username, desensitizedBody,
+                desensitizedQueryParams);
         }
         return true;
     }
@@ -143,7 +142,7 @@ public void afterCompletion(HttpServletRequest request, HttpServletResponse resp
             String requestId = request.getHeader(JobCommonHeaders.BK_GATEWAY_REQUEST_ID);
             int respStatus = response.getStatus();
             long cost = System.currentTimeMillis() - startTimeInMills;
-            log.info("request-id:{}|API:{}|uri:{}|appCode:{}|username:{}|status:{}|resp:{}|cost:{}",
+            log.info("request-id: {}|API: {}|uri: {}|appCode: {}|username: {}|status: {}|resp: {}|cost: {}",
                 requestId,
                 apiName,
                 request.getRequestURI(),

diff --git a/...ob-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceMetricsResource.java b/...ob-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceMetricsResource.java
@@ -27,10 +27,6 @@
 import com.tencent.bk.job.common.annotation.InternalAPI;
 import com.tencent.bk.job.common.model.InternalResponse;
 import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO;
-import com.tencent.bk.job.execute.common.constants.RunStatusEnum;
-import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum;
-import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum;
-import com.tencent.bk.job.execute.common.constants.TaskTypeEnum;
 import com.tencent.bk.job.execute.model.inner.request.ServiceTriggerStatisticsRequest;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
@@ -65,75 +61,6 @@ InternalResponse<Boolean> hasExecuteHistory(
         @RequestParam(value = "toTime", required = false) Long toTime
     );
 
-    /**
-     * 大数据量下容易导致慢查询影响全局，非特殊情况不使用
-     */
-    @Deprecated
-    @ApiOperation(value = "快速文件分发统计", produces = "application/json")
-    @GetMapping("/service/metrics/fastPushFile/count")
-    InternalResponse<Integer> countFastPushFile(
-        @ApiParam(value = "业务Id", required = false)
-        @RequestParam(value = "appId", required = false) Long appId,
-        @ApiParam(value = "传输模式", required = false)
-        @RequestParam(value = "transferMode", required = false) Integer transferMode,
-        @ApiParam(value = "文件源是否为本地文件", required = false)
-        @RequestParam(value = "localUpload", required = false) Boolean localUpload,
-        @ApiParam(value = "步骤状态", required = false)
-        @RequestParam(value = "runStatus", required = false) RunStatusEnum runStatus,
-        @ApiParam(value = "统计的起始时间", required = false)
-        @RequestParam(value = "fromTime", required = false) Long fromTime,
-        @ApiParam(value = "统计的截止时间", required = false)
-        @RequestParam(value = "toTime", required = false) Long toTime
-    );
-
-    /**
-     * 大数据量下容易导致慢查询影响全局，非特殊情况不使用
-     */
-    @Deprecated
-    @ApiOperation(value = "步骤执行统计", produces = "application/json")
-    @GetMapping("/service/metrics/stepInstances/count")
-    InternalResponse<Integer> countStepInstances(
-        @ApiParam(value = "业务Id", required = false)
-        @RequestParam(value = "appId", required = false) Long appId,
-        @ApiParam(value = "步骤对应的StepId列表", required = false)
-        @RequestParam(value = "stepIdList", required = false) List<Long> stepIdList,
-        @ApiParam(value = "步骤类型", required = false)
-        @RequestParam(value = "stepExecuteType", required = false) StepExecuteTypeEnum stepExecuteType,
-        @ApiParam(value = "脚本类型", required = false)
-        @RequestParam(value = "scriptType", required = false) Integer scriptType,
-        @ApiParam(value = "步骤状态", required = false)
-        @RequestParam(value = "runStatus", required = false) RunStatusEnum runStatus,
-        @ApiParam(value = "统计的起始时间", required = false)
-        @RequestParam(value = "fromTime", required = false) Long fromTime,
-        @ApiParam(value = "统计的截止时间", required = false)
-        @RequestParam(value = "toTime", required = false) Long toTime
-    );
-
-    /**
-     * 大数据量下容易导致慢查询影响全局，非特殊情况不使用
-     */
-    @Deprecated
-    @ApiOperation(value = "任务（含快速/作业）执行统计", produces = "application/json")
-    @GetMapping("/service/metrics/taskInstances/count")
-    InternalResponse<Integer> countTaskInstances(
-        @ApiParam(value = "业务Id", required = false)
-        @RequestParam(value = "appId", required = false) Long appId,
-        @ApiParam(value = "最小执行用时(单位：秒)", required = false)
-        @RequestParam(value = "minTotalTime", required = false) Long minTotalTime,
-        @ApiParam(value = "最大执行用时(单位：秒)", required = false)
-        @RequestParam(value = "maxTotalTime", required = false) Long maxTotalTime,
-        @ApiParam(value = "触发方式", required = false)
-        @RequestParam(value = "taskStartupMode", required = false) TaskStartupModeEnum taskStartupMode,
-        @ApiParam(value = "任务类型", required = false)
-        @RequestParam(value = "taskType", required = false) TaskTypeEnum taskType,
-        @ApiParam(value = "任务状态列表", required = false)
-        @RequestParam(value = "runStatusList", required = false) List<Byte> runStatusList,
-        @ApiParam(value = "统计的起始时间(ms)", required = false)
-        @RequestParam(value = "fromTime", required = false) Long fromTime,
-        @ApiParam(value = "统计的截止时间(ms)", required = false)
-        @RequestParam(value = "toTime", required = false) Long toTime
-    );
-
     @ApiOperation(value = "获取统计数据", produces = "application/json")
     @GetMapping("/service/metrics/statistics")
     InternalResponse<StatisticsDTO> getStatistics(
@@ -149,26 +76,10 @@ InternalResponse<StatisticsDTO> getStatistics(
         @RequestParam(value = "dateStr", required = true) String dateStr
     );
 
-    @ApiOperation(value = "获取统计数据", produces = "application/json")
-    @GetMapping("/service/metrics/statistics/list")
-    InternalResponse<List<StatisticsDTO>> listStatistics(
-        @ApiParam(value = "业务Id", required = false)
-        @RequestParam(value = "appId", required = false) Long appId,
-        @ApiParam(value = "资源类型", required = false)
-        @RequestParam(value = "resource", required = false) String resource,
-        @ApiParam(value = "资源维度", required = false)
-        @RequestParam(value = "dimension", required = false) String dimension,
-        @ApiParam(value = "资源维度取值", required = false)
-        @RequestParam(value = "dimensionValue", required = false) String dimensionValue,
-        @ApiParam(value = "统计日期(yyyy-MM-dd)", required = false)
-        @RequestParam(value = "dateStr", required = false) String dateStr
-    );
-
     @ApiOperation(value = "触发指定时间的数据统计", produces = "application/json")
     @PostMapping("/service/metrics/statistics/trigger")
     InternalResponse<Boolean> triggerStatistics(
         @ApiParam(value = "统计日期(yyyy-MM-dd)", required = false)
         @RequestBody ServiceTriggerStatisticsRequest request
     );
-
 }
diff --git a/.../main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3ResourceImpl.java b/.../main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3ResourceImpl.java
@@ -147,8 +147,13 @@ private ValidateResult checkRequest(EsbGetJobInstanceListV3Request request) {
             return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "create_time_end");
         }
         long period = request.getCreateTimeEnd() - request.getCreateTimeStart();
-        if (period > Consts.MAX_SEARCH_TASK_HISTORY_RANGE_MILLS) {
-            log.warn("Search time range greater than 30 days!");
+        if (period <= 0) {
+            log.warn("CreateStartTime is greater or equal to createTimeEnd, getCreateTimeStart={}, createTimeEnd={}",
+                request.getCreateTimeStart(), request.getCreateTimeEnd());
+            return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME,
+                "create_time_start|create_time_end");
+        } else if (period > Consts.MAX_SEARCH_TASK_HISTORY_RANGE_MILLS) {
+            log.warn("Search time range greater than {} days!", Consts.MAX_SEARCH_TASK_HISTORY_RANGE_MILLS);
             return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM, "create_time_start|create_time_end");
         }
         if (request.getTaskType() != null && TaskTypeEnum.valueOf(request.getTaskType()) == null) {

diff --git a/...xecute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceMetricsResourceImpl.java b/...xecute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceMetricsResourceImpl.java
@@ -26,14 +26,9 @@
 
 import com.tencent.bk.job.common.model.InternalResponse;
 import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO;
-import com.tencent.bk.job.execute.common.constants.RunStatusEnum;
-import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum;
-import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum;
-import com.tencent.bk.job.execute.common.constants.TaskTypeEnum;
 import com.tencent.bk.job.execute.model.inner.request.ServiceTriggerStatisticsRequest;
 import com.tencent.bk.job.execute.service.TaskInstanceService;
 import com.tencent.bk.job.execute.statistics.StatisticsService;
-import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum;
 import io.micrometer.core.annotation.Timed;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.web.bind.annotation.RestController;
@@ -67,46 +62,13 @@ public InternalResponse<Boolean> hasExecuteHistory(Long appId, Long cronTaskId,
             toTime));
     }
 
-    @Override
-    @Timed(extraTags = {IGNORE_TAG, BOOLEAN_TRUE_TAG_VALUE})
-    public InternalResponse<Integer> countFastPushFile(Long appId, Integer transferMode, Boolean localUpload,
-                                                  RunStatusEnum runStatus, Long fromTime, Long toTime) {
-        return InternalResponse.buildSuccessResp(taskInstanceService.countFastPushFile(appId, transferMode,
-            localUpload, runStatus, fromTime, toTime));
-    }
-
-    @Override
-    @Timed(extraTags = {IGNORE_TAG, BOOLEAN_TRUE_TAG_VALUE})
-    public InternalResponse<Integer> countStepInstances(Long appId, List<Long> stepIdList,
-                                                   StepExecuteTypeEnum stepExecuteType, Integer scriptType,
-                                                   RunStatusEnum runStatus, Long fromTime, Long toTime) {
-        return InternalResponse.buildSuccessResp(taskInstanceService.countStepInstances(appId, stepIdList,
-            stepExecuteType, ScriptTypeEnum.valueOf(scriptType), runStatus, fromTime, toTime));
-    }
-
-    @Override
-    @Timed(extraTags = {IGNORE_TAG, BOOLEAN_TRUE_TAG_VALUE})
-    public InternalResponse<Integer> countTaskInstances(Long appId, Long minTotalTime, Long maxTotalTime,
-                                                   TaskStartupModeEnum taskStartupMode, TaskTypeEnum taskType,
-                                                   List<Byte> runStatusList, Long fromTime, Long toTime) {
-        return InternalResponse.buildSuccessResp(taskInstanceService.countTaskInstances(appId, minTotalTime,
-            maxTotalTime, taskStartupMode, taskType, runStatusList, fromTime, toTime));
-    }
-
     @Override
     public InternalResponse<StatisticsDTO> getStatistics(Long appId, String resource, String dimension,
-                                                    String dimensionValue, String dateStr) {
+                                                         String dimensionValue, String dateStr) {
         return InternalResponse.buildSuccessResp(statisticsService.getStatistics(appId, resource, dimension,
             dimensionValue, dateStr));
     }
 
-    @Override
-    public InternalResponse<List<StatisticsDTO>> listStatistics(Long appId, String resource, String dimension,
-                                                           String dimensionValue, String dateStr) {
-        return InternalResponse.buildSuccessResp(statisticsService.listStatistics(appId, resource, dimension,
-            dimensionValue, dateStr));
-    }
-
     @Override
     public InternalResponse<Boolean> triggerStatistics(ServiceTriggerStatisticsRequest request) {
         return InternalResponse.buildSuccessResp(statisticsService.triggerStatistics(request.getDateList()));

diff --git a/...main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java b/...main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java
@@ -108,9 +108,6 @@
 import org.springframework.web.bind.annotation.RestController;
 
 import java.nio.charset.StandardCharsets;
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.LocalTime;
 import java.time.ZoneId;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
@@ -286,9 +283,8 @@ private ValidateResult validateAndSetQueryTimeRange(TaskInstanceQuery taskInstan
                 log.warn("Param timeRange should less then 30");
                 return ValidateResult.fail(ErrorCode.TASK_INSTANCE_QUERY_TIME_SPAN_MORE_THAN_30_DAYS);
             }
-            // 当天结束时间
-            long todayMaxMills = LocalDateTime.of(LocalDate.now(), LocalTime.MAX).getSecond() * 1000L;
-            start = todayMaxMills - 30 * 24 * 3600 * 1000L;
+            // 当天结束时间 - 往前的天数
+            start = DateUtils.getUTCCurrentDayEndTimestamp() - timeRange * 24 * 3600 * 1000L;
         } else {
             if (StringUtils.isNotBlank(startTime)) {
                 start = DateUtils.convertUnixTimestampFromDateTimeStr(startTime, "yyyy-MM-dd HH:mm:ss",