From 9a364cec661521beb7cb8124fdd6e4fe746a88b8 Mon Sep 17 00:00:00 2001 From: trydofor Date: Wed, 28 Aug 2024 19:10:47 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=B8=20LogView=20ignore=20preview=20if?= =?UTF-8?q?=20no=20keywords=20#241?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../slardar/monitor/metric/LogMetric.java | 47 ++++++++++++------- .../slardar/monitor/viewer/LogViewer.java | 27 +++++++++-- .../spring/prop/SlardarMonitorProp.java | 12 +++++ .../slardar/monitor/viewer/LogViewerTest.java | 30 ++++++++++-- .../warlock/controller/api/WebLogViewer.java | 2 +- 5 files changed, 93 insertions(+), 25 deletions(-) diff --git a/wings/slardar/src/main/java/pro/fessional/wings/slardar/monitor/metric/LogMetric.java b/wings/slardar/src/main/java/pro/fessional/wings/slardar/monitor/metric/LogMetric.java index 924f5223..69805c9e 100644 --- a/wings/slardar/src/main/java/pro/fessional/wings/slardar/monitor/metric/LogMetric.java +++ b/wings/slardar/src/main/java/pro/fessional/wings/slardar/monitor/metric/LogMetric.java @@ -62,7 +62,7 @@ public String getKey() { } final long from = readLastForm(); - final LogStat.Stat stat = LogStat.stat(rule.file, from, rule.getPreview(), rule.getSection(), rule.getRuntimeKeys()); + final LogStat.Stat stat = LogStat.stat(rule.file, from, rule.getPreview(), rule.getSection(), rule.genStatKey()); log.debug("LogStat-{}, stat={}", key, stat); writeLastFrom(stat); @@ -226,6 +226,7 @@ public static class Rule { /** * section size of intended lines + * * @see #Key$section */ private int section = 50; @@ -280,26 +281,38 @@ public String maskKey(String kw) { * Auto remove a pair of quotes, construct bytes by charset */ @SneakyThrows - public List getRuntimeKeys() { + public List genStatKey() { List rst = new ArrayList<>(); - if (level != null) { - for (String s : level) { - String kw = trimKey(s, false); - if (kw.isEmpty()) continue; - LogStat.Word wd = new LogStat.Word(); - wd.range2 = bound; - wd.bytes = kw.getBytes(charset); - rst.add(wd); + for (String kw : genRuleKey()) { + LogStat.Word wd = new LogStat.Word(); + wd.range2 = bound; + wd.bytes = kw.getBytes(charset); + rst.add(wd); + } + return rst; + } + + /** + * Auto remove a pair of quotes, merge level and its keyword + */ + public List genRuleKey() { + List rst = new ArrayList<>(); + if (level == null) return rst; + + for (String s : level) { + String kw = trimKey(s, false); + if (!kw.isEmpty()) { + rst.add(kw); } - for (String s : keyword) { - String kw = trimKey(s, false); - if (kw.isEmpty()) continue; - LogStat.Word wd = new LogStat.Word(); - wd.range1 = bound; - wd.bytes = kw.getBytes(charset); - rst.add(wd); + } + + for (String s : keyword) { + String kw = trimKey(s, false); + if (kw.isEmpty()) { + rst.add(kw); } } + return rst; } } diff --git a/wings/slardar/src/main/java/pro/fessional/wings/slardar/monitor/viewer/LogViewer.java b/wings/slardar/src/main/java/pro/fessional/wings/slardar/monitor/viewer/LogViewer.java index 91220a28..5a9300e7 100644 --- a/wings/slardar/src/main/java/pro/fessional/wings/slardar/monitor/viewer/LogViewer.java +++ b/wings/slardar/src/main/java/pro/fessional/wings/slardar/monitor/viewer/LogViewer.java @@ -4,6 +4,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.cache2k.Cache; +import org.jetbrains.annotations.NotNull; import pro.fessional.mirana.id.Ulid; import pro.fessional.wings.slardar.cache.cache2k.WingsCache2k; import pro.fessional.wings.slardar.monitor.WarnFilter; @@ -17,9 +18,11 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.regex.Pattern; /** @@ -31,11 +34,13 @@ public class LogViewer implements WarnFilter { @Getter private final LogConf conf; + private final Set keys = new HashSet<>(); private final Cache cache; - public LogViewer(LogConf conf) { + public LogViewer(@NotNull LogConf conf, @NotNull Collection keys) { this.conf = conf; this.cache = WingsCache2k.builder(LogViewer.class, "cache", 2_000, conf.getAlive(), null, String.class, String.class).build(); + this.keys.addAll(keys); } public void view(String id, OutputStream output) throws IOException { @@ -101,15 +106,15 @@ protected boolean canIgnoreHead(String out) { final File file = new File(out); if (file.length() > max || !file.canRead()) return false; - final Pattern head = conf.getHeader(); try (BufferedReader reader = new BufferedReader(new FileReader(file))) { String line; int tol = 0; int cnt = 0; + final Pattern head = conf.getHeader(); while ((line = reader.readLine()) != null && max > 0) { max -= line.length() + 1; // loose calculation - if (line.isEmpty() || (head != null && !head.matcher(line).find())) { + if (ignoreLine(line, head)) { continue; } @@ -125,7 +130,23 @@ protected boolean canIgnoreHead(String out) { return tol == cnt; } catch (Exception e) { + return false; + } + } + + private boolean ignoreLine(String line, Pattern head) { + if (line.isEmpty()) return true; + + if (head != null && !head.matcher(line).find()) { return true; } + + for (String key : keys) { + if (line.contains(key)) { + return false; + } + } + + return true; } } diff --git a/wings/slardar/src/main/java/pro/fessional/wings/slardar/spring/prop/SlardarMonitorProp.java b/wings/slardar/src/main/java/pro/fessional/wings/slardar/spring/prop/SlardarMonitorProp.java index 09a504b9..cc3bfe3e 100644 --- a/wings/slardar/src/main/java/pro/fessional/wings/slardar/spring/prop/SlardarMonitorProp.java +++ b/wings/slardar/src/main/java/pro/fessional/wings/slardar/spring/prop/SlardarMonitorProp.java @@ -7,6 +7,7 @@ import pro.fessional.wings.slardar.monitor.viewer.LogConf; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.Map; /** @@ -70,4 +71,15 @@ public class SlardarMonitorProp { */ private String dingNotice = "monitor"; public static final String Key$dingNotice = Key + ".ding-notice"; + + /** + * generate all log's rule keys + */ + public LinkedHashSet genRuleKey() { + LinkedHashSet rst = new LinkedHashSet<>(); + for (LogMetric.Rule rl : log.values()) { + rst.addAll(rl.genRuleKey()); + } + return rst; + } } diff --git a/wings/slardar/src/test/java/pro/fessional/wings/slardar/monitor/viewer/LogViewerTest.java b/wings/slardar/src/test/java/pro/fessional/wings/slardar/monitor/viewer/LogViewerTest.java index 5c9a501b..7ae9e713 100644 --- a/wings/slardar/src/test/java/pro/fessional/wings/slardar/monitor/viewer/LogViewerTest.java +++ b/wings/slardar/src/test/java/pro/fessional/wings/slardar/monitor/viewer/LogViewerTest.java @@ -24,15 +24,37 @@ class LogViewerTest { @TmsLink("13128") @Test void canIgnoreHead() throws Exception { - final LogViewer lv = new LogViewer(slardarMonitorProp.getView()); + final LogViewer lv = new LogViewer(slardarMonitorProp.getView(), slardarMonitorProp.genRuleKey()); final Path tmp0 = Files.createTempFile("test-", null); tmp0.toFile().deleteOnExit(); Files.writeString(tmp0, """ - 2024-07-21 22:05:22.957 ERROR 10884 --- [kite-front] [XNIO-1 I/O-4] io.undertow.request : UT005071: Undertow request failed HttpServerExchange{ CONNECT api.ipify.org:443} - + ######### #1 KEYWORD: ERROR ######### + 2024-08-28 07:01:54.441 ERROR 3306238 --- [admin-test] [XNIO-1 I/O-2] io.undertow.request : UT005071: Undertow request failed HttpServerExchange{ CONNECT eth0.me:443} + java.lang.IllegalArgumentException: UT000068: Servlet path match failed - at io.undertow.servlet.handlers.ServletPathMatchesData.getServletHandlerByPath(ServletPathMatchesData.java:83) ~[undertow-servlet-2.3.10.Final.jar!/:2.3.10.Final] + at io.undertow.servlet.handlers.ServletPathMatchesData.getServletHandlerByPath(ServletPathMatchesData.java:83) ~[undertow-servlet-2.3.13.Final.jar!/:2.3.13.Final] + at io.undertow.servlet.handlers.ServletPathMatches.getServletHandlerByPath(ServletPathMatches.java:133) ~[undertow-servlet-2.3.13.Final.jar!/:2.3.13.Final] + at io.undertow.servlet.handlers.ServletInitialHandler.handleRequest(ServletInitialHandler.java:148) ~[undertow-servlet-2.3.13.Final.jar!/:2.3.13.Final] + at io.undertow.server.handlers.HttpContinueReadHandler.handleRequest(HttpContinueReadHandler.java:69) ~[undertow-core-2.3.13.Final.jar!/:2.3.13.Final] + at org.springframework.boot.web.embedded.undertow.DeploymentManagerHttpHandlerFactory$DeploymentManagerHandler.handleRequest(DeploymentManagerHttpHandlerFactory.java:74) ~[spring-boot-3.2.8.jar!/:3.2.8] + at io.undertow.server.handlers.GracefulShutdownHandler.handleRequest(GracefulShutdownHandler.java:94) ~[undertow-core-2.3.13.Final.jar!/:2.3.13.Final] + at io.undertow.server.Connectors.executeRootHandler(Connectors.java:393) ~[undertow-core-2.3.13.Final.jar!/:2.3.13.Final] + at io.undertow.server.protocol.http.HttpReadListener.handleEventWithNoRunningRequest(HttpReadListener.java:265) ~[undertow-core-2.3.13.Final.jar!/:2.3.13.Final] + at io.undertow.server.protocol.http.HttpReadListener.handleEvent(HttpReadListener.java:136) ~[undertow-core-2.3.13.Final.jar!/:2.3.13.Final] + at io.undertow.server.protocol.http.HttpReadListener.handleEvent(HttpReadListener.java:59) ~[undertow-core-2.3.13.Final.jar!/:2.3.13.Final] + at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:92) ~[xnio-api-3.8.8.Final.jar!/:3.8.8.Final] + at org.xnio.conduits.ReadReadyHandler$ChannelListenerHandler.readReady(ReadReadyHandler.java:66) ~[xnio-api-3.8.8.Final.jar!/:3.8.8.Final] + at org.xnio.nio.NioSocketConduit.handleReady(NioSocketConduit.java:89) ~[xnio-nio-3.8.8.Final.jar!/:3.8.8.Final] + at org.xnio.nio.WorkerThread.run(WorkerThread.java:591) ~[xnio-nio-3.8.8.Final.jar!/:3.8.8.Final] + + 2024-08-28 07:02:13.000 DEBUG 3306238 --- [admin-test] [task-7] org.jooq.tools.LoggerListener : Executing query : update `win_task_define` set `next_lock` = (`next_lock` + ?), `last_exec` = ? where (`id` = ? and `next_lock` = ?) + 2024-08-28 07:02:13.000 DEBUG 3306238 --- [admin-test] [task-7] org.jooq.tools.LoggerListener : -> with bind values : update `win_task_define` set `next_lock` = (`next_lock` + 1), `last_exec` = {ts '2024-08-28 07:02:13.0'} where (`id` = 1107 and `next_lock` = 89552) + 2024-08-28 07:02:13.005 DEBUG 3306238 --- [admin-test] [task-7] org.jooq.tools.LoggerListener : Affected row(s) : 1 + 2024-08-28 07:02:13.005 INFO 3306238 --- [admin-test] [task-7] p.f.w.t.t.s.i.TinyTaskExecServiceImpl : tiny-task exec, id=1107, prop=bill-expire + 2024-08-28 07:02:13.006 INFO 3306238 --- [admin-test] [task-7] p.f.w.t.t.s.i.TinyTaskExecServiceImpl : tiny-task done, id=1107, prop=bill-expire + 2024-08-28 07:02:13.007 DEBUG 3306238 --- [admin-test] [light-id-buffered-provider-2] o.s.jdbc.core.JdbcTemplate : Executing prepared SQL query + 2024-08-28 07:02:13.007 DEBUG 3306238 --- [admin-test] [light-id-buffered-provider-2] o.s.jdbc.core.JdbcTemplate : Executing prepared SQL statement [SELECT next_val, step_val FROM sys_light_sequence WHERE block_id=? AND seq_name=? FOR UPDATE] """.stripIndent()); diff --git a/wings/warlock-shadow/src/main/java/pro/fessional/wings/warlock/controller/api/WebLogViewer.java b/wings/warlock-shadow/src/main/java/pro/fessional/wings/warlock/controller/api/WebLogViewer.java index 89e90d2d..33cabd83 100644 --- a/wings/warlock-shadow/src/main/java/pro/fessional/wings/warlock/controller/api/WebLogViewer.java +++ b/wings/warlock-shadow/src/main/java/pro/fessional/wings/warlock/controller/api/WebLogViewer.java @@ -25,7 +25,7 @@ public class WebLogViewer extends LogViewer { @Autowired public WebLogViewer(SlardarMonitorProp prop) { - super(prop.getView()); + super(prop.getView(), prop.genRuleKey()); } @Operation(summary = "Alarm logs can be viewed in conjunction with alarm notifications when self-monitoring is enabled.", description = """