From c0fe67e93e42471cc3137f5a442c44e46db6ccd0 Mon Sep 17 00:00:00 2001 From: Black Pan Date: Tue, 10 Sep 2024 10:06:51 +0800 Subject: [PATCH] refactor: improve and refine the usage scope of interfaces and classes (#429) Co-authored-by: gaoxh <32359336+gaoxh@users.noreply.github.com> --- .../porcessor/AgentConfigProcessor.java | 13 +- .../server/service/AgentConfigAcquirer.java | 34 ---- .../service/DefaultAgentConfigAcquirer.java | 7 +- .../agent/service/ServiceRegistryService.java | 113 ++----------- .../impl/ServiceRegistryServiceImpl.java | 147 +++++++++++++++++ .../log/api/enums/LogStorageTypeEnum.java | 7 +- .../controller/MilogStatisticController.java | 25 +-- .../service/HeralogHomePageService.java | 145 ++++------------- .../manager/service/StatisticsService.java | 70 ++++++++ .../impl/HeralogHomePageServiceImpl.java | 152 ++++++++++++++++++ .../{ => impl}/StatisticsServiceImpl.java | 11 +- 11 files changed, 453 insertions(+), 271 deletions(-) delete mode 100644 ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/AgentConfigAcquirer.java create mode 100644 ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/impl/ServiceRegistryServiceImpl.java create mode 100644 ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StatisticsService.java create mode 100644 ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeralogHomePageServiceImpl.java rename ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/{ => impl}/StatisticsServiceImpl.java (98%) diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/AgentConfigProcessor.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/AgentConfigProcessor.java index 1c0992af8..6b4e41d69 100644 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/AgentConfigProcessor.java +++ b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/porcessor/AgentConfigProcessor.java @@ -13,13 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.xiaomi.mone.log.server.porcessor; import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor; import com.xiaomi.data.push.rpc.protocol.RemotingCommand; import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; +import com.xiaomi.mone.log.api.service.AgentConfigService; import com.xiaomi.mone.log.common.Constant; -import com.xiaomi.mone.log.server.service.AgentConfigAcquirer; import com.xiaomi.mone.log.server.service.DefaultAgentConfigAcquirer; import com.xiaomi.youpin.docean.Ioc; import io.netty.channel.ChannelHandlerContext; @@ -35,22 +36,22 @@ */ @Slf4j public class AgentConfigProcessor implements NettyRequestProcessor { - + @Override public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws Exception { RemotingCommand response = RemotingCommand.createResponseCommand(Constant.RPCCMD_AGENT_CONFIG_CODE); String ip = new String(request.getBody()); log.info("agent start get metadata configļ¼Œagent ip:{}", ip); - - AgentConfigAcquirer agentConfigService = Ioc.ins().getBean(DefaultAgentConfigAcquirer.class); - + + AgentConfigService agentConfigService = Ioc.ins().getBean(DefaultAgentConfigAcquirer.class); + LogCollectMeta logCollectMeta = agentConfigService.getLogCollectMetaFromManager(ip); String responseInfo = GSON.toJson(logCollectMeta); log.info("agent start get metadata config info:{}", responseInfo); response.setBody(responseInfo.getBytes()); return response; } - + @Override public boolean rejectRequest() { return false; diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/AgentConfigAcquirer.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/AgentConfigAcquirer.java deleted file mode 100644 index 6f85ad997..000000000 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/AgentConfigAcquirer.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2020 Xiaomi Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.xiaomi.mone.log.server.service; - -import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; - -/** - * @author wtt - * @version 1.0 - * @description - * @date 2022/12/6 14:29 - */ -public interface AgentConfigAcquirer { - - /** - * Get log configuration information - * @param ip - * @return - */ - LogCollectMeta getLogCollectMetaFromManager(String ip); -} diff --git a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultAgentConfigAcquirer.java b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultAgentConfigAcquirer.java index f990c1179..cde3e56aa 100644 --- a/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultAgentConfigAcquirer.java +++ b/ozhera-log/log-agent-server/src/main/java/com/xiaomi/mone/log/server/service/DefaultAgentConfigAcquirer.java @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.xiaomi.mone.log.server.service; import com.xiaomi.mone.log.api.model.meta.LogCollectMeta; @@ -29,11 +30,11 @@ */ @Component @Slf4j -public class DefaultAgentConfigAcquirer implements AgentConfigAcquirer { - +public class DefaultAgentConfigAcquirer implements AgentConfigService { + @Reference(interfaceClass = AgentConfigService.class, group = "$dubbo.group", check = false, timeout = 10000) private AgentConfigService agentConfigService; - + @Override public LogCollectMeta getLogCollectMetaFromManager(String ip) { LogCollectMeta logCollectMeta = new LogCollectMeta(); diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/ServiceRegistryService.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/ServiceRegistryService.java index a1d3e8e48..c72d68a1f 100644 --- a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/ServiceRegistryService.java +++ b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/ServiceRegistryService.java @@ -13,29 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.xiaomi.mone.log.agent.service; -import com.alibaba.nacos.api.exception.NacosException; import com.alibaba.nacos.api.naming.pojo.Instance; -import com.google.common.collect.Maps; -import com.xiaomi.data.push.common.SafeRun; import com.xiaomi.data.push.nacos.NacosNaming; -import com.xiaomi.data.push.rpc.RpcClient; -import com.xiaomi.mone.log.common.Config; -import com.xiaomi.mone.log.utils.NetUtil; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import javax.annotation.Resource; -import java.net.InetAddress; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; -import java.util.Objects; - -import static com.xiaomi.mone.log.common.Constant.STREAM_CONTAINER_POD_NAME_KEY; -import static com.xiaomi.mone.log.utils.ConfigUtils.getDataHashKey; /** * @author wtt @@ -43,87 +25,16 @@ * @description * @date 2023/12/19 14:15 */ -@Service -@Slf4j -public class ServiceRegistryService { - - @Resource - private RpcClient rpcClient; - private final static String SERVER_PREFIX = "prometheus_server"; - private final static String APP_NAME_LABEL = "app_name"; - private final static String APP_ID_LABEL = "app_id"; - private final static String ENV_ID_LABEL = "env_id"; - private final static String ENV_NAME_LABEL = "env_name"; - private static final String DEFAULT_TIME_DATE_FORMAT = "yyyy-MM-dd hh:mm:ss"; - public static final String STREAM_VERSION = "hera-log-agent:1.0.0:2023-12-20"; - - private String appName; - private String appId; - private String envId; - private String envName; - private Integer port; - private String ip; - - public void init() { - this.initializeEnvironmentParameters(); - String registrationInitiationFlag = Config.ins().get("registration_initiation_flag", "false"); - if (Objects.equals("true", registrationInitiationFlag)) { - this.registerServiceInstance(); - } - } - - private void registerServiceInstance() { - NacosNaming nacosNaming = rpcClient.getNacosNaming(); - int appIndex = getDataHashKey(ip, Integer.parseInt(Config.ins().get("app_max_index", "30"))); - String serviceName = String.format("%s_%s_%s_%s", SERVER_PREFIX, appId, appName, appIndex); - - try { - nacosNaming.registerInstance(serviceName, buildInstance(serviceName)); - addShutdownHook(nacosNaming, serviceName); - } catch (NacosException e) { - log.error("registerService error,serviceName:{}", serviceName, e); - } - } - - private void addShutdownHook(NacosNaming nacosNaming, String serviceName) { - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - log.info("agent unregisters the instance and the service name:{}", serviceName); - nacosNaming.deregisterInstance(serviceName, ip, port); - } catch (NacosException e) { - log.error("agent unregisters the instance error,service name:{}", serviceName, e); - } - })); - } - - private Instance buildInstance(String serviceName) { - Instance instance = new Instance(); - instance.setEnabled(true); - instance.setHealthy(true); - instance.setIp(ip); - instance.setPort(port); - instance.setServiceName(serviceName); - - Map metaData = Maps.newHashMap(); - metaData.put("ctime", new SimpleDateFormat(DEFAULT_TIME_DATE_FORMAT).format(new Date())); - metaData.put("version", STREAM_VERSION); - metaData.put(STREAM_CONTAINER_POD_NAME_KEY, System.getenv(STREAM_CONTAINER_POD_NAME_KEY)); - metaData.put(ENV_ID_LABEL, envId); - metaData.put(ENV_NAME_LABEL, envName); - - SafeRun.run(() -> metaData.put("hostname", InetAddress.getLocalHost().getHostName())); - instance.setMetadata(metaData); - - return instance; - } - - public void initializeEnvironmentParameters() { - appName = StringUtils.isNotBlank(System.getenv(APP_NAME_LABEL)) ? System.getenv(APP_NAME_LABEL) : "log_agent"; - appId = StringUtils.isNotBlank(System.getenv(APP_ID_LABEL)) ? System.getenv(APP_ID_LABEL) : "10010"; - envName = StringUtils.isNotBlank(System.getenv(ENV_NAME_LABEL)) ? System.getenv(ENV_NAME_LABEL) : "default_env"; - envId = StringUtils.isNotBlank(System.getenv(ENV_ID_LABEL)) ? System.getenv(ENV_ID_LABEL) : "1"; - port = Integer.parseInt(Config.ins().get("port", "9799")); - ip = NetUtil.getLocalIp(); - } +public interface ServiceRegistryService { + + void init(); + + void registerServiceInstance(); + + void addShutdownHook(NacosNaming nacosNaming, String serviceName); + + Instance buildInstance(String serviceName); + + void initializeEnvironmentParameters(); } diff --git a/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/impl/ServiceRegistryServiceImpl.java b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/impl/ServiceRegistryServiceImpl.java new file mode 100644 index 000000000..681c0a96e --- /dev/null +++ b/ozhera-log/log-agent/src/main/java/com/xiaomi/mone/log/agent/service/impl/ServiceRegistryServiceImpl.java @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2020 Xiaomi Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.xiaomi.mone.log.agent.service.impl; + +import com.alibaba.nacos.api.exception.NacosException; +import com.alibaba.nacos.api.naming.pojo.Instance; +import com.google.common.collect.Maps; +import com.xiaomi.data.push.common.SafeRun; +import com.xiaomi.data.push.nacos.NacosNaming; +import com.xiaomi.data.push.rpc.RpcClient; +import com.xiaomi.mone.log.agent.service.ServiceRegistryService; +import com.xiaomi.mone.log.common.Config; +import com.xiaomi.mone.log.utils.NetUtil; +import com.xiaomi.youpin.docean.anno.Service; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import javax.annotation.Resource; +import java.net.InetAddress; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; +import java.util.Objects; + +import static com.xiaomi.mone.log.common.Constant.STREAM_CONTAINER_POD_NAME_KEY; +import static com.xiaomi.mone.log.utils.ConfigUtils.getDataHashKey; + +/** + * @author wtt + * @version 1.0 + * @description + * @date 2023/12/19 14:15 + */ +@Service +@Slf4j +public class ServiceRegistryServiceImpl implements ServiceRegistryService { + + @Resource + private RpcClient rpcClient; + + private final static String SERVER_PREFIX = "prometheus_server"; + + private final static String APP_NAME_LABEL = "app_name"; + + private final static String APP_ID_LABEL = "app_id"; + + private final static String ENV_ID_LABEL = "env_id"; + + private final static String ENV_NAME_LABEL = "env_name"; + + private static final String DEFAULT_TIME_DATE_FORMAT = "yyyy-MM-dd hh:mm:ss"; + + public static final String STREAM_VERSION = "hera-log-agent:1.0.0:2023-12-20"; + + private String appName; + + private String appId; + + private String envId; + + private String envName; + + private Integer port; + + private String ip; + + @Override + public void init() { + this.initializeEnvironmentParameters(); + String registrationInitiationFlag = Config.ins().get("registration_initiation_flag", "false"); + if (Objects.equals("true", registrationInitiationFlag)) { + this.registerServiceInstance(); + } + } + + @Override + public void registerServiceInstance() { + NacosNaming nacosNaming = rpcClient.getNacosNaming(); + int appIndex = getDataHashKey(ip, Integer.parseInt(Config.ins().get("app_max_index", "30"))); + String serviceName = String.format("%s_%s_%s_%s", SERVER_PREFIX, appId, appName, appIndex); + + try { + nacosNaming.registerInstance(serviceName, buildInstance(serviceName)); + addShutdownHook(nacosNaming, serviceName); + } catch (NacosException e) { + log.error("registerService error,serviceName:{}", serviceName, e); + } + } + + @Override + public void addShutdownHook(NacosNaming nacosNaming, String serviceName) { + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + log.info("agent unregisters the instance and the service name:{}", serviceName); + nacosNaming.deregisterInstance(serviceName, ip, port); + } catch (NacosException e) { + log.error("agent unregisters the instance error,service name:{}", serviceName, e); + } + })); + } + + @Override + public Instance buildInstance(String serviceName) { + Instance instance = new Instance(); + instance.setEnabled(true); + instance.setHealthy(true); + instance.setIp(ip); + instance.setPort(port); + instance.setServiceName(serviceName); + + Map metaData = Maps.newHashMap(); + metaData.put("ctime", new SimpleDateFormat(DEFAULT_TIME_DATE_FORMAT).format(new Date())); + metaData.put("version", STREAM_VERSION); + metaData.put(STREAM_CONTAINER_POD_NAME_KEY, System.getenv(STREAM_CONTAINER_POD_NAME_KEY)); + metaData.put(ENV_ID_LABEL, envId); + metaData.put(ENV_NAME_LABEL, envName); + + SafeRun.run(() -> metaData.put("hostname", InetAddress.getLocalHost().getHostName())); + instance.setMetadata(metaData); + + return instance; + } + + @Override + public void initializeEnvironmentParameters() { + appName = StringUtils.isNotBlank(System.getenv(APP_NAME_LABEL)) ? System.getenv(APP_NAME_LABEL) : "log_agent"; + appId = StringUtils.isNotBlank(System.getenv(APP_ID_LABEL)) ? System.getenv(APP_ID_LABEL) : "10010"; + envName = StringUtils.isNotBlank(System.getenv(ENV_NAME_LABEL)) ? System.getenv(ENV_NAME_LABEL) : "default_env"; + envId = StringUtils.isNotBlank(System.getenv(ENV_ID_LABEL)) ? System.getenv(ENV_ID_LABEL) : "1"; + port = Integer.parseInt(Config.ins().get("port", "9799")); + ip = NetUtil.getLocalIp(); + } +} diff --git a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogStorageTypeEnum.java b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogStorageTypeEnum.java index 478d7195d..c184a32a6 100644 --- a/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogStorageTypeEnum.java +++ b/ozhera-log/log-api/src/main/java/com/xiaomi/mone/log/api/enums/LogStorageTypeEnum.java @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.xiaomi.mone.log.api.enums; /** @@ -22,11 +23,11 @@ * @date 2023/11/10 14:31 */ public enum LogStorageTypeEnum { - + ELASTICSEARCH, DORIS, - CLICKEHOUSE; - + CLICKHOUSE; + public static LogStorageTypeEnum queryByName(String name) { if (null == name || name.isEmpty()) { return null; diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStatisticController.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStatisticController.java index 4440f2c4a..c30376875 100644 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStatisticController.java +++ b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/controller/MilogStatisticController.java @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.xiaomi.mone.log.manager.controller; import com.xiaomi.mone.log.common.Result; @@ -20,7 +21,7 @@ import com.xiaomi.mone.log.manager.model.dto.EsStatisticResult; import com.xiaomi.mone.log.manager.model.dto.EsStatisticsKeyWord; import com.xiaomi.mone.log.manager.model.vo.LogQuery; -import com.xiaomi.mone.log.manager.service.StatisticsServiceImpl; +import com.xiaomi.mone.log.manager.service.StatisticsService; import com.xiaomi.mone.log.manager.service.impl.EsDataServiceImpl; import com.xiaomi.youpin.docean.anno.Controller; import com.xiaomi.youpin.docean.anno.RequestMapping; @@ -35,37 +36,39 @@ @Controller public class MilogStatisticController { + private static final Logger LOGGER = LoggerFactory.getLogger(MilogStatisticController.class); - + @Resource - private StatisticsServiceImpl statisticsService; - + private StatisticsService statisticsService; + @Resource private EsDataServiceImpl esDataService; - + @RequestMapping(path = "/milog/statistic/es") public Result statisticEs(@RequestParam("param") LogQuery param) throws Exception { return esDataService.EsStatistic(param); } - + @RequestMapping(path = "/log/queryTailStatisticsByHour") public Result> queryTailStatisticsByHour(StatisticsQuery statisticsQuery) throws IOException { return statisticsService.queryTailStatisticsByHour(statisticsQuery); } - + @RequestMapping(path = "/log/queryStoreTopByDay") - public Result> queryStoreTopTailStatisticsByDay(StatisticsQuery statisticsQuery) throws IOException { + public Result> queryStoreTopTailStatisticsByDay(StatisticsQuery statisticsQuery) + throws IOException { return statisticsService.queryStoreTopTailStatisticsByDay(statisticsQuery); } - + @RequestMapping(path = "/log/querySpaceTopStore") public Result> querySpaceTopStore(StatisticsQuery statisticsQuery) throws IOException { return statisticsService.querySpaceTopStoreByDay(statisticsQuery); } - + @RequestMapping(path = "/log/store/index/field/ration") public Result> queryEsStatisticsRation(LogQuery param) { return statisticsService.queryEsStatisticsRation(param); } - + } diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeralogHomePageService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeralogHomePageService.java index a5f237782..5fd344edf 100644 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeralogHomePageService.java +++ b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/HeralogHomePageService.java @@ -13,130 +13,55 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.xiaomi.mone.log.manager.service; -import com.google.common.collect.Lists; -import com.xiaomi.mone.app.api.response.AppBaseInfo; -import com.xiaomi.mone.log.common.Config; import com.xiaomi.mone.log.common.Result; -import com.xiaomi.mone.log.exception.CommonError; -import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; -import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; -import com.xiaomi.mone.log.manager.dao.MilogStoreSpaceAuthDao; -import com.xiaomi.mone.log.manager.model.dto.MapDTO; import com.xiaomi.mone.log.manager.model.dto.MilogSpaceTreeDTO; import com.xiaomi.mone.log.manager.model.dto.UnAccessAppDTO; import com.xiaomi.mone.log.manager.model.dto.ValueDTO; import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; -import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; -import com.xiaomi.mone.log.manager.model.pojo.MilogStoreSpaceAuth; -import com.xiaomi.mone.log.manager.service.impl.HeraAppServiceImpl; -import com.xiaomi.youpin.docean.anno.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; -import org.jetbrains.annotations.Nullable; - -import javax.annotation.Resource; -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -@Service -public class HeralogHomePageService { - @Resource - private MilogLogTailDao milogLogtailDao; - - @Resource - private HeraAppServiceImpl heraAppService; - - @Resource - private MilogLogstoreDao milogLogstoreDao; - - @Resource - private MilogStoreSpaceAuthDao milogStoreSpaceAuthDao; - - private List> milogpattern; - { - String pattern = Config.ins().get("milogpattern", ""); - String[] split = pattern.split(","); - ArrayList> valueDTOS = new ArrayList<>(); - for (String s : split) { - valueDTOS.add(new ValueDTO<>(s)); - } - milogpattern = valueDTOS; - } +import java.util.List; +import java.util.Map; - public Result> milogAccess() { - Long total = heraAppService.getAppCount(); - int access = milogLogtailDao.appCount(); - HashMap map = new HashMap<>(); - map.put("total", total); - map.put("access", access); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), map); - } - - public Result> unAccessAppList() { - List appBaseInfos = heraAppService.queryAllExistsApp(); - Map appMap = appBaseInfos.stream().collect(Collectors.toMap(AppBaseInfo::getId, AppBaseInfo::getAppName)); - List hasAccessAppId = milogLogtailDao.queryAllAppId(); - ArrayList list = new ArrayList<>(); - for (Map.Entry app : appMap.entrySet()) { - if (!hasAccessAppId.contains(app.getKey())) { - list.add(new UnAccessAppDTO(app.getKey().longValue(), app.getValue())); - } - } - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), list); - } - - public Result> getMilogSpaceTree(Long spaceId) { - List stores = getMilogLogStoreDOS(spaceId); - List spaceTreeDTOS = stores.stream().map(milogLogstoreDO -> { - Long logstoreDOId = milogLogstoreDO.getId(); - MilogSpaceTreeDTO milogSpaceTreeDTO = new MilogSpaceTreeDTO(); - milogSpaceTreeDTO.setLabel(milogLogstoreDO.getLogstoreName()); - milogSpaceTreeDTO.setValue(logstoreDOId); - List logTailDos = milogLogtailDao.getMilogLogtailByStoreId(logstoreDOId); - if (CollectionUtils.isNotEmpty(logTailDos)) { - List> collect = logTailDos.stream() - .map(logTailDo -> { - MapDTO mapDTO = new MapDTO(); - mapDTO.setValue(logTailDo.getId()); - mapDTO.setLabel(logTailDo.getTail()); - return mapDTO; - }).collect(Collectors.toList()); - milogSpaceTreeDTO.setChildren(collect); - } - return milogSpaceTreeDTO; - }).collect(Collectors.toList()); - return Result.success(spaceTreeDTOS); - } +public interface HeralogHomePageService { + + /** + * Get all apps info + * + * @return + */ + Result> milogAccess(); + + /** + * Package and return all application details. + * + * @return + */ + Result> unAccessAppList(); + + /** + * Get all logStoreInfo by spaceId and return in a tree structure. + * + * @param spaceId + * @return + */ + Result> getMilogSpaceTree(Long spaceId); + /** * Query the store that originally belonged to the space, and query the authorized store * * @param spaceId * @return */ - @Nullable - private List getMilogLogStoreDOS(Long spaceId) { - List storeDOS = Lists.newArrayList(); - List stores = milogLogstoreDao.getMilogLogstoreBySpaceId(spaceId); - List storeSpaceAuths = milogStoreSpaceAuthDao.queryStoreIdsBySpaceId(spaceId); - if (CollectionUtils.isNotEmpty(stores)) { - storeDOS = stores; - } - if (CollectionUtils.isNotEmpty(storeSpaceAuths)) { - List collect = storeSpaceAuths.stream() - .map(storeSpaceAuth -> milogLogstoreDao.queryById(storeSpaceAuth.getStoreId())) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - storeDOS.addAll(collect); - } - return storeDOS; - } - - public Result>> getMiloglogAccessPattern() { - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), milogpattern); - } + List getMilogLogStoreDOS(Long spaceId); + + /** + * Get log pattern from properties + * + * @return + */ + Result>> getMiloglogAccessPattern(); } diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StatisticsService.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StatisticsService.java new file mode 100644 index 000000000..e9f867469 --- /dev/null +++ b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StatisticsService.java @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2020 Xiaomi Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.xiaomi.mone.log.manager.service; + +import com.xiaomi.mone.log.common.Result; +import com.xiaomi.mone.log.manager.model.StatisticsQuery; +import com.xiaomi.mone.log.manager.model.dto.EsStatisticsKeyWord; +import com.xiaomi.mone.log.manager.model.vo.LogQuery; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + + +public interface StatisticsService { + + + /** + * Hourly data volume statistics for a single tail + * + * @param statisticsQuery + * @return + * @throws IOException + */ + Result> queryTailStatisticsByHour(StatisticsQuery statisticsQuery) throws IOException; + + + /** + * Top 5 in the data volume of all tails in a single store on the day + * + * @param statisticsQuery + * @return + * @throws IOException + */ + Result> queryStoreTopTailStatisticsByDay(StatisticsQuery statisticsQuery) throws IOException; + + + /** + * Top 5 in the volume of data of all stores in a single space + * + * @param statisticsQuery + * @return + * @throws IOException + */ + Result> querySpaceTopStoreByDay(StatisticsQuery statisticsQuery) throws IOException; + + /** + * Query and calculate the statistical percentages of log keywords in Elasticsearch. + * + * @param logQuery + * @return + */ + Result> queryEsStatisticsRation(LogQuery logQuery); + + +} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeralogHomePageServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeralogHomePageServiceImpl.java new file mode 100644 index 000000000..f1139b7ca --- /dev/null +++ b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/HeralogHomePageServiceImpl.java @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2020 Xiaomi Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.xiaomi.mone.log.manager.service.impl; + +import com.google.common.collect.Lists; +import com.xiaomi.mone.app.api.response.AppBaseInfo; +import com.xiaomi.mone.log.common.Config; +import com.xiaomi.mone.log.common.Result; +import com.xiaomi.mone.log.exception.CommonError; +import com.xiaomi.mone.log.manager.dao.MilogLogTailDao; +import com.xiaomi.mone.log.manager.dao.MilogLogstoreDao; +import com.xiaomi.mone.log.manager.dao.MilogStoreSpaceAuthDao; +import com.xiaomi.mone.log.manager.model.dto.MapDTO; +import com.xiaomi.mone.log.manager.model.dto.MilogSpaceTreeDTO; +import com.xiaomi.mone.log.manager.model.dto.UnAccessAppDTO; +import com.xiaomi.mone.log.manager.model.dto.ValueDTO; +import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; +import com.xiaomi.mone.log.manager.model.pojo.MilogLogTailDo; +import com.xiaomi.mone.log.manager.model.pojo.MilogStoreSpaceAuth; +import com.xiaomi.mone.log.manager.service.HeralogHomePageService; +import com.xiaomi.youpin.docean.anno.Service; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.jetbrains.annotations.Nullable; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +@Slf4j +@Service +public class HeralogHomePageServiceImpl implements HeralogHomePageService { + + @Resource + private MilogLogTailDao milogLogtailDao; + + @Resource + private HeraAppServiceImpl heraAppService; + + @Resource + private MilogLogstoreDao milogLogstoreDao; + + @Resource + private MilogStoreSpaceAuthDao milogStoreSpaceAuthDao; + + private List> milogpattern; + + { + String pattern = Config.ins().get("milogpattern", ""); + String[] split = pattern.split(","); + ArrayList> valueDTOS = new ArrayList<>(); + for (String s : split) { + valueDTOS.add(new ValueDTO<>(s)); + } + milogpattern = valueDTOS; + } + + @Override + public Result> milogAccess() { + Long total = heraAppService.getAppCount(); + int access = milogLogtailDao.appCount(); + HashMap map = new HashMap<>(); + map.put("total", total); + map.put("access", access); + return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), map); + } + + @Override + public Result> unAccessAppList() { + List appBaseInfos = heraAppService.queryAllExistsApp(); + Map appMap = appBaseInfos.stream() + .collect(Collectors.toMap(AppBaseInfo::getId, AppBaseInfo::getAppName)); + List hasAccessAppId = milogLogtailDao.queryAllAppId(); + ArrayList list = new ArrayList<>(); + for (Map.Entry app : appMap.entrySet()) { + if (!hasAccessAppId.contains(app.getKey())) { + list.add(new UnAccessAppDTO(app.getKey().longValue(), app.getValue())); + } + } + return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), list); + } + + @Override + public Result> getMilogSpaceTree(Long spaceId) { + List stores = getMilogLogStoreDOS(spaceId); + List spaceTreeDTOS = stores.stream().map(milogLogstoreDO -> { + Long logstoreDOId = milogLogstoreDO.getId(); + MilogSpaceTreeDTO milogSpaceTreeDTO = new MilogSpaceTreeDTO(); + milogSpaceTreeDTO.setLabel(milogLogstoreDO.getLogstoreName()); + milogSpaceTreeDTO.setValue(logstoreDOId); + List logTailDos = milogLogtailDao.getMilogLogtailByStoreId(logstoreDOId); + if (CollectionUtils.isNotEmpty(logTailDos)) { + List> collect = logTailDos.stream().map(logTailDo -> { + MapDTO mapDTO = new MapDTO(); + mapDTO.setValue(logTailDo.getId()); + mapDTO.setLabel(logTailDo.getTail()); + return mapDTO; + }).collect(Collectors.toList()); + milogSpaceTreeDTO.setChildren(collect); + } + return milogSpaceTreeDTO; + }).collect(Collectors.toList()); + return Result.success(spaceTreeDTOS); + } + + /** + * Query the store that originally belonged to the space, and query the authorized store + * + * @param spaceId + * @return + */ + @Override + @Nullable + public List getMilogLogStoreDOS(Long spaceId) { + List storeDOS = Lists.newArrayList(); + List stores = milogLogstoreDao.getMilogLogstoreBySpaceId(spaceId); + List storeSpaceAuths = milogStoreSpaceAuthDao.queryStoreIdsBySpaceId(spaceId); + if (CollectionUtils.isNotEmpty(stores)) { + storeDOS = stores; + } + if (CollectionUtils.isNotEmpty(storeSpaceAuths)) { + List collect = storeSpaceAuths.stream() + .map(storeSpaceAuth -> milogLogstoreDao.queryById(storeSpaceAuth.getStoreId())) + .filter(Objects::nonNull).collect(Collectors.toList()); + storeDOS.addAll(collect); + } + return storeDOS; + } + + @Override + public Result>> getMiloglogAccessPattern() { + return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage(), milogpattern); + } +} diff --git a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StatisticsServiceImpl.java b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/StatisticsServiceImpl.java similarity index 98% rename from ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StatisticsServiceImpl.java rename to ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/StatisticsServiceImpl.java index 4bd076fe9..207212c91 100644 --- a/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/StatisticsServiceImpl.java +++ b/ozhera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/StatisticsServiceImpl.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.xiaomi.mone.log.manager.service; +package com.xiaomi.mone.log.manager.service.impl; import cn.hutool.core.util.NumberUtil; import com.google.common.collect.Lists; @@ -28,6 +28,7 @@ import com.xiaomi.mone.log.manager.model.dto.EsStatisticsKeyWord; import com.xiaomi.mone.log.manager.model.pojo.MilogLogStoreDO; import com.xiaomi.mone.log.manager.model.vo.LogQuery; +import com.xiaomi.mone.log.manager.service.StatisticsService; import com.xiaomi.youpin.docean.anno.Service; import com.xiaomi.youpin.docean.plugin.es.EsService; import lombok.extern.slf4j.Slf4j; @@ -62,7 +63,7 @@ @Service @Slf4j -public class StatisticsServiceImpl { +public class StatisticsServiceImpl implements StatisticsService { @Resource @@ -79,6 +80,7 @@ public class StatisticsServiceImpl { /** * Hourly data volume statistics for a single tail */ + @Override public Result> queryTailStatisticsByHour(StatisticsQuery statisticsQuery) throws IOException { Map result = new LinkedHashMap<>(); @@ -123,6 +125,7 @@ public Result> queryTailStatisticsByHour(StatisticsQuery stati /** * Top 5 in the data volume of all tails in a single store on the day */ + @Override public Result> queryStoreTopTailStatisticsByDay(StatisticsQuery statisticsQuery) throws IOException { if (statisticsQuery.getStartTime() == null || statisticsQuery.getStartTime() == 0) { @@ -161,6 +164,7 @@ public Result> queryStoreTopTailStatisticsByDay(StatisticsQuer /** * Top 5 in the volume of data of all stores in a single space */ + @Override public Result> querySpaceTopStoreByDay(StatisticsQuery statisticsQuery) throws IOException { Map result = new LinkedHashMap<>(); @@ -196,7 +200,8 @@ public Result> querySpaceTopStoreByDay(StatisticsQuery statist } return Result.success(result); } - + + @Override public Result> queryEsStatisticsRation(LogQuery logQuery) { List results = Lists.newArrayList(); if (null == logQuery.getStoreId()) {