From 8bdccdc91f43288583c0d447c3232dc55b7d4d9a Mon Sep 17 00:00:00 2001 From: Benito Palacios Sanchez Date: Fri, 11 Nov 2016 12:47:50 +0100 Subject: [PATCH 1/2] Create markdown formatter with the current output --- src/devices/formatdevice.py | 54 +++++ src/devices/markdownformatdevice.py | 234 ++++++++++++++++++++ src/rtilogparser.py | 326 +++++++--------------------- 3 files changed, 366 insertions(+), 248 deletions(-) create mode 100644 src/devices/formatdevice.py create mode 100644 src/devices/markdownformatdevice.py diff --git a/src/devices/formatdevice.py b/src/devices/formatdevice.py new file mode 100644 index 0000000..47314f2 --- /dev/null +++ b/src/devices/formatdevice.py @@ -0,0 +1,54 @@ +# Log Parser for RTI Connext. +# +# Copyright 2016 Real-Time Innovations, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Device for the output format. + +The module contains the abstract class representation for a formatter of the +parsed logs. + +Classes: + + FormatDevice: Abstract base class for format device implementations. +""" + + +class FormatDevice(object): + """Abstract base class for format device implementations. + + You will need to implement the following methods: + + write_header: write the header if any. + + write_configurations: write the configuration messages. + + write_warnings: write the warning messages. + + write_errors: write the error messages. + """ + + def write_header(self, state): + """Write the header if any.""" + raise NotImplementedError("write_header not implemented") + + def write_message(self, state): + """Write the message.""" + raise NotImplementedError("write_message not implemented") + + def write_configurations(self, state): + """Write the configuration messages.""" + raise NotImplementedError("write_configurations not implemented") + + def write_warnings(self, state): + """Write the warning messages.""" + raise NotImplementedError("write_warnings not implemented") + + def write_errors(self, state): + """Write the error messages.""" + raise NotImplementedError("write_errors not implemented") diff --git a/src/devices/markdownformatdevice.py b/src/devices/markdownformatdevice.py new file mode 100644 index 0000000..4e9367c --- /dev/null +++ b/src/devices/markdownformatdevice.py @@ -0,0 +1,234 @@ +# Log Parser for RTI Connext. +# +# Copyright 2016 Real-Time Innovations, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Format device to show the output as Markdown. + +Classes: + + MarkdownFormatDevice: Format device for Markdown. +""" +from devices.formatdevice import FormatDevice +from devices.logger import COLORS +from rtilogparser import __version__ + + +class MarkdownFormatDevice(FormatDevice): + """Format device for Markdown. + + Functions: + + write_header: write the header. + + write_message: write the message. + + write_warnings: write the warning messages. + + write_errors: write the warning messages. + + write_configurations: write the configuration messages. + + write_countset: write a generic log message list. + + write_locators: write the locators if any. + + write_host_summary: write the host summary. + + write_statistics_bandwidth: write the bandwidth statistics. + + write_throughput: write the throughput information. + + write_statistics_packets: write the packet statistics. + + bytes_to_string: convert a byte unit value into string. + """ + + def __init__(self, state): + """Initialize the device.""" + self.write = state['output_device'].write + + def write_header(self, state): + """Write the header.""" + self.write("# Log Parser for RTI Connext ~ " + __version__) + self.write() + self.write("## Legend:") + self.write("* ---> or <--- for output or input packet.") + self.write("* An asterisk in remote address if inside initial_peers") + self.write("* Remote Address format: 'HostID AppID ObjID' or IP:Port") + self.write("* Port format is 'Domain.Index Kind' where kind:") + self.write(" * MeMu: Meta-traffic over Multicast") + self.write(" * MeUn: Meta-traffic over Unicast") + self.write(" * UsMu: User-traffic over Multicast") + self.write(" * UsUn: User-traffic over Unicast") + self.write("* H3.A2.P3 means 3rd participant from 2nd app of 3rd host") + self.write(" At the end there is a summary with the assigned IP") + self.write("* Reader and writer identifiers are: ID_TsK where:") + self.write(" * ID: identifier number of the entity.") + self.write(" * T: entity kind. 'W' for writers, 'R' for readers.") + self.write(" * sK: if the entity is keyed (+K) or unkeyed (-K).") + self.write() + self.write() + + self.write("## Network Data Flow and Application Events") + header = " In/Out | Remote Address | Local Entity | Message" + headln = "---------|:----------------------:|:--------------:|--------" + if not state['no_timestamp']: + header = "Timestamp".ljust(28) + "|" + header + headln = "----------------------------|" + headln + if state['show_lines']: + header = " Log/Parser |" + header + headln = "------------|" + headln + self.write(header) + self.write(headln) + + def write_message(self, state): + """Write the message.""" + raise NotImplementedError("write_message not implemented") + + def write_warnings(self, state): + """Write the warning messages.""" + title = "Warnings" + if not state['no_colors']: + title = COLORS['WARNING'] + title + COLORS['ENDC'] + self.write_countset(state['warnings'], title) + + def write_errors(self, state): + """Write the warning messages.""" + title = "Errors" + if not state['no_colors']: + title = COLORS['FAIL'] + title + COLORS['ENDC'] + self.write_countset(state['errors'], title) + + def write_configurations(self, state): + """Write the configuration messages.""" + self.write("----------------------") + if 'locators' in state: + self.write_locators(state) + if 'names' in state and 'name_table' in state: + self.write_host_summary(state) + if 'statistics' in state and not state['no_stats']: + self.write_statistics_bandwidth(state) + if 'statistics_packet' in state and not state['no_stats']: + self.write_statistics_packets(state) + self.write_countset(state['config'], 'Config') + + def write_countset(self, items, title): + """Write a generic log message list.""" + self.write("----------------------") + self.write("## %s:" % title) + for i, msg in enumerate(sorted(items.keys(), key=lambda m: m[0])): + self.write("%2d. %dx %s" % (i, items[msg][1], msg)) + self.write() + + def write_locators(self, state): + """Write the locators if any.""" + self.write("### Locators:") + for part in state['locators']: + self.write("* Participant: " + part) + self.write(" * Send locators:") + for loc in state['locators'][part]['send']: + self.write(" * " + loc) + self.write(" * Receive locators:") + for loc in state['locators'][part]['receive']: + self.write(" * " + loc) + self.write() + + def write_host_summary(self, state): + """Write the host summary.""" + self.write("### Assigned names:") + + apps_num = 0 + part_num = 0 + table = state['name_table'] + names = state['names'] + for host in table: + # Print host + if host in names: + self.write("* Host %s: %s" % (names[host], host)) + else: + self.write("* Host %s" % host) + + # For each application. + for app in table[host]: + apps_num += 1 + addr = host + " " + app + if addr in names: + self.write(" * App %s: %s" % (names[addr], app)) + else: + self.write(" * App %s" % app) + + # For each participant of the application + for part in table[host][app]: + part_num += 1 + part_guid = addr + " " + part + if part_guid in names: + self.write(" * Participant %s: %s" % + (names[part_guid], part)) + else: + self.write(" * Participant %s" % part) + + # Final stats + self.write() + self.write("Number of hosts: %d " % len(table)) # Trailing SP for MD + self.write("Number of apps: %d" % apps_num) + self.write("Number of participants: %d" % part_num) + self.write() + + def write_statistics_bandwidth(self, state): + """Write the bandwidth statistics.""" + self.write("### Bandwidth statistics:") + + stats = state['statistics'] + for addr in stats: + self.write("* Address: %s" % addr) + for typ in stats[addr]: + # If this is a port with dictionary of statistics types + if isinstance(stats[addr][typ], dict): + # Show statistics per port with verbosity >= 1 + if state['verbosity'] < 1: + continue + port = typ + self.write(" * Port %s" % port) + for typ in stats[addr][port]: + info = stats[addr][port][typ] + self.write_throughput(" * %s: " % typ, info) + # If this is the host counter + else: + info = stats[addr][typ] + self.write_throughput(" * %s: " % typ, info) + self.write() + + def write_throughput(self, prefix, info): + """Write the throughput information.""" + time_diff = info[1] - info[0] + qty = self.bytes_to_string(info[2]) + if time_diff > 0: + throughput = self.bytes_to_string(info[2] / time_diff) + self.write("%s%s (%s/s)" % (prefix, qty, throughput)) + else: + self.write("%s%s" % (prefix, qty)) + + def write_statistics_packets(self, state): + """Write the packet statistics.""" + self.write("### Packet statistics:") + stats = state['statistics_packet'] + for guid in stats: + self.write("* GUID: %s" % guid) + for typ in stats[guid]: + total = float(stats[guid][typ]['ALL']) + self.write(" * %s: %d packets" % (typ, total)) + for packet in stats[guid][typ]: + if packet == "ALL": + continue + qty = stats[guid][typ][packet] + self.write(" * %s: %d (%.1f%%)" % + (packet, qty, qty / total * 100)) + self.write() + + @staticmethod + def bytes_to_string(qty): + """Convert a byte unit value into string.""" + typ = ["GB", "MB", "KB", "B"] + for i in range(len(typ) - 1, 0, -1): + rang = float(2 ** (10 * i)) + if qty > rang: + return "%.2f %s" % (qty / rang, typ[i]) + return str(int(qty)) + " B" diff --git a/src/rtilogparser.py b/src/rtilogparser.py index 037a816..4cd006a 100644 --- a/src/rtilogparser.py +++ b/src/rtilogparser.py @@ -50,197 +50,62 @@ from traceback import extract_tb from devices.inputdevices import InputConsoleDevice, InputFileDevice -from devices.logger import COLORS, log_error, log_warning +from devices.logger import log_error, log_warning +from devices.markdownformatdevice import MarkdownFormatDevice from devices.outputdevices import OutputConsoleDevice, OutputFileDevice from logs import create_regex_list from utils import compare_times -__version__ = "1.2a1" +__version__ = "1.2a2" DATE_REGEX = re.compile(r'\[(\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}.\d{6})\]' + r'\[(\d{10}.\d{6})\]') SINGLE_DATE_REGEX = re.compile(r'\[(\d{10}).(\d{6})\]') -def print_countset(items, typ, state, color=None): - """Print a generic log message list.""" - write = state['output_device'].write - if not state['no_colors'] and color: - typ = color + typ + COLORS['ENDC'] - - write("----------------------") - write("## %s:" % typ) - for i, msg in enumerate(sorted(items.keys(), key=lambda m: m[0])): - write("%2d. %dx %s" % (i, items[msg][1], msg)) - write() - - -def print_config(state): - """Print the configuration logs.""" - state['output_device'].write("----------------------") - if 'locators' in state: - print_locators(state) - if 'names' in state and 'name_table' in state: - print_host_summary(state) - if 'statistics' in state and not state['no_stats']: - print_statistics_bandwidth(state) - if 'statistics_packet' in state and not state['no_stats']: - print_statistics_packets(state) - print_countset(state['config'], 'Config', state) - - -def print_locators(state): - """Print the locators if any.""" - write = state['output_device'].write - write("### Locators:") - for part in state['locators']: - write("* Participant: " + part) - write(" * Send locators:") - for loc in state['locators'][part]['send']: - write(" * " + loc) - write(" * Receive locators:") - for loc in state['locators'][part]['receive']: - write(" * " + loc) - write() - - -def print_host_summary(state): - """Print the host summary.""" - write = state['output_device'].write - write("### Assigned names:") - - apps_num = 0 - table = state['name_table'] - for host in table: - # Print host - if host in state['names']: - write("* Host %s: %s" % (state['names'][host], host)) - else: - write("* Host %s" % host) - - # For each application. - for app in table[host]: - addr = host + " " + app - apps_num += 1 - if addr in state['names']: - write(" * App %s: %s" % (state['names'][addr], app)) - else: - write(" * App %s" % app) - - # For each participant of the application - for part in table[host][app]: - part_guid = addr + " " + part - if part_guid in state['names']: - write(" * Participant %s: %s" % - (state['names'][part_guid], part)) - - # Final stats - write() - write("Number of hosts: %d " % len(table)) # Trailing space for markdown - write("Number of apps: %d" % apps_num) - write() - - -def print_statistics_bandwidth(state): - """Print the bandwidth statistics.""" - write = state['output_device'].write - stats = state['statistics'] - - write("### Bandwidth statistics:") - for addr in stats: - write("* Address: %s" % addr) - for typ in stats[addr]: - # If this is a port with dictionary of statistics types - if isinstance(stats[addr][typ], dict): - # Show statistics per port with verbosity >= 1 - if state['verbosity'] < 1: - continue - port = typ - write(" * Port %s" % port) - for typ in stats[addr][port]: - info = stats[addr][port][typ] - print_throughput_info(" * %s: " % typ, info, state) - # If this is the host counter - else: - info = stats[addr][typ] - print_throughput_info(" * %s: " % typ, info, state) - write() - - -def print_throughput_info(prefix, info, state): - """Print the throughput information.""" - write = state['output_device'].write - - time_diff = info[1] - info[0] - qty = bytes_to_string(info[2]) - if time_diff > 0: - throughput = bytes_to_string(info[2] / time_diff) - write("%s%s (%s/s)" % (prefix, qty, throughput)) - else: - write("%s%s" % (prefix, qty)) - - -def print_statistics_packets(state): - """Print the packet statistics.""" - write = state['output_device'].write - write("### Packet statistics:") - stats = state['statistics_packet'] - for guid in stats: - write("* GUID: %s" % guid) - for typ in stats[guid]: - total = float(stats[guid][typ]['ALL']) - write(" * %s: %d packets" % (typ, total)) - for packet in stats[guid][typ]: - if packet == "ALL": - continue - qty = stats[guid][typ][packet] - write(" * %s: %d (%.1f%%)" % - (packet, qty, qty / total * 100)) - write() - - -def bytes_to_string(qty): - """Convert a byte unit value into string.""" - typ = ["GB", "MB", "KB", "B"] - for i in range(len(typ) - 1, 0, -1): - rang = float(2 ** (10 * i)) - if qty > rang: - return "%.2f %s" % (qty / rang, typ[i]) - return str(int(qty)) + " B" +def check_time_distance(new_clocks, old_clocks, state): + """Check that the distance between logs it's not large.""" + MAX_TIME_SEC = 60 + result = compare_times(old_clocks[1], new_clocks[1], + timedelta(seconds=MAX_TIME_SEC)) + if result: + log_warning("System clock went %s by %s." % + (result[0], result[1]), state) + + if new_clocks[0]: + result = compare_times(old_clocks[0], new_clocks[0], MAX_TIME_SEC) + if result: + log_warning("Monotonic clock went %s by %.3f." % + (result[0], result[1]), state) def match_date(line, state): """Try to match the log date.""" + # Try to match the two clock format. clocks = DATE_REGEX.search(line) - er_52 = True # If both clocks are set this is the engineering build 52. + two_clocks = clocks is not None + + # If it doesn't match, try with the single clock format. if not clocks: clocks = SINGLE_DATE_REGEX.search(line) - er_52 = False - - if clocks: - # Get clocks - if er_52: - system = datetime.strptime(clocks.group(1), "%m/%d/%Y %H:%M:%S.%f") - monotonic = float(clocks.group(2)) - else: - system = datetime.utcfromtimestamp(int(clocks.group(1))) - system += timedelta(microseconds=int(clocks.group(2))) - monotonic = None - - # Check that the distance between logs it's not so long (60 sec) - if 'clocks' in state: - result = compare_times(state['clocks'][1], system, - timedelta(seconds=60)) - if result: - log_warning("System clock went %s by %s." % - (result[0], result[1]), state) - - if monotonic: - result = compare_times(state['clocks'][0], monotonic, 60) - if result: - log_warning("Monotonic clock went %s by %.3f." % - (result[0], result[1]), state) - - state['clocks'] = (monotonic, system) if monotonic else (None, system) + + # If we don't match the default clock either, nothing to do + if not clocks: + return + + # Get clocks + if two_clocks: + system = datetime.strptime(clocks.group(1), "%m/%d/%Y %H:%M:%S.%f") + monotonic = float(clocks.group(2)) + else: + system = datetime.utcfromtimestamp(int(clocks.group(1))) + system += timedelta(microseconds=int(clocks.group(2))) + monotonic = None + + new_clocks = (monotonic, system) + if 'clocks' in state: + check_time_distance(new_clocks, state['clocks'], state) + + state['clocks'] = new_clocks def match_line(line, expressions, state): @@ -253,6 +118,39 @@ def match_line(line, expressions, state): break +def parse_log(expressions, state): + """Parse a log.""" + device = state['input_device'] + + if state['write_original']: + originalOutput = OutputFileDevice(state, state['write_original'], True) + + # While there is a new line, parse it. + line = True # For the first condition. + while line: + # If the line contains non-UTF8 chars it could raise an exception. + state['input_line'] += 1 + line = device.read_line().rstrip("\r\n") + + # If EOF or the line is empty, continue. + if not line or line == "": + continue + + # Write original log if needed + if state['write_original']: + originalOutput.write(line) + + # We can get exceptions if the file contains output from two + # different applications since the logs are messed up. + try: + match_line(line, expressions, state) + except Exception as ex: # pylint: disable=W0703 + exc_traceback = exc_info()[2] + stacktraces = extract_tb(exc_traceback) + log_error("[ScriptError] %s %s" % (str(stacktraces[-1]), ex), + state) + + def get_urandom(): """Get a cryptographic random value.""" rnd = urandom(32) @@ -352,78 +250,10 @@ def initialize_state(args): state['input_device'] = InputFileDevice(args.input, state) else: state['input_device'] = InputConsoleDevice(state) + state['format_device'] = MarkdownFormatDevice(state) return state -def parse_log(expressions, state): - """Parse a log file.""" - device = state['input_device'] - - if state['write_original']: - originalOutput = OutputFileDevice(state, state['write_original'], True) - - # While there is a new line, parse it. - line = True # For the first condition. - while line: - # If the line contains non-UTF8 chars it could raise an exception. - state['input_line'] += 1 - line = device.read_line().rstrip("\r\n") - - # If EOF or the line is empty, continue. - if not line or line == "": - continue - - # Write original log if needed - if state['write_original']: - originalOutput.write(line) - - # We can get exceptions if the file contains output from two - # different applications since the logs are messed up. - try: - match_line(line, expressions, state) - except Exception as ex: # pylint: disable=W0703 - exc_traceback = exc_info()[2] - stacktraces = extract_tb(exc_traceback) - log_error("[ScriptError] %s %s" % (str(stacktraces[-1]), ex), - state) - - -def print_header(state): - """Print the header information.""" - write = state['output_device'].write - write("# Log Parser for RTI Connext ~ " + __version__) - write() - write("## Legend:") - write(" * ---> or <--- denotes if it's an output or input packet.") - write(" * An asterisk in remote address means 'inside initial_peers'") - write(" * Remote Address format is 'HostId AppId ObjId' or 'Ip:Port'") - write(" * Port format for out messages is 'Domain.Idx kind' where kind:") - write(" * MeMu: Meta-traffic over Multicast") - write(" * MeUn: Meta-traffic over Unicast") - write(" * UsMu: User-traffic over Multicast") - write(" * UsUn: User-traffic over Unicast") - write(" * H3.A2.P3 is third participant from second app of third host. ") - write(" At the end of the log there is a summary with the assigned IP") - write(" * Reader and writer identifiers are: ID_TsK where:") - write(" * ID is the identifier number of the entity.") - write(" * T is the entity kind: 'W' for writers and 'R' for readers.") - write(" * sK determines if the entity is keyed (+K) or unkeyed (-K).") - write() - write() - - write("## Network Data Flow and Application Events") - header = " In/Out | Remote Address | Local Entity | Description" - headln = "---------|:----------------------:|:--------------:|------------" - if not state['no_timestamp']: - header = "Timestamp".ljust(28) + "|" + header - headln = "----------------------------|" + headln - if state['show_lines']: - header = " Log/Parser |" + header - headln = "------------|" + headln - write(header) - write(headln) - - def main(): """Main application entry.""" args = read_arguments() @@ -431,7 +261,7 @@ def main(): expressions = create_regex_list(state) # Read log file and parse - print_header(state) + state['format_device'].write_header(state) try: parse_log(expressions, state) except KeyboardInterrupt: @@ -448,9 +278,9 @@ def main(): log_warning("Catched SIGINT", state) # Print result of config, errors and warnings. - print_config(state) - print_countset(state['warnings'], 'Warnings', state, COLORS['WARNING']) - print_countset(state['errors'], 'Errors', state, COLORS['FAIL']) + state['format_device'].write_configurations(state) + state['format_device'].write_warnings(state) + state['format_device'].write_errors(state) if __name__ == "__main__": From f83d6e1194c6300483abc02aa71477b45df09214 Mon Sep 17 00:00:00 2001 From: Benito Palacios Sanchez Date: Fri, 11 Nov 2016 18:18:10 +0100 Subject: [PATCH 2/2] Move logger logic into Markdown formatter --- src/__init__.py | 20 +++++ src/devices/formatdevice.py | 15 +++- src/devices/logger.py | 130 +++++++++++++++++----------- src/devices/markdownformatdevice.py | 50 +++++++---- src/rtilogparser.py | 3 +- 5 files changed, 149 insertions(+), 69 deletions(-) create mode 100644 src/__init__.py diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..3853b50 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,20 @@ +# Log Parser for RTI Connext. +# +# Copyright 2016 Real-Time Innovations, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Package LogParser.""" + +__version__ = "1.2a2" +__license__ = "Apache" +__copyright__ = "Copyright 2016 Real-Time Innovations, Inc." diff --git a/src/devices/formatdevice.py b/src/devices/formatdevice.py index 47314f2..f561fbf 100644 --- a/src/devices/formatdevice.py +++ b/src/devices/formatdevice.py @@ -37,8 +37,19 @@ def write_header(self, state): """Write the header if any.""" raise NotImplementedError("write_header not implemented") - def write_message(self, state): - """Write the message.""" + def write_message(self, content, state): + """Write the message. + + The content argument is a dictionary with at least 'description' item. + The optional items are: + + kind: the kind or remark for the message. + + timestamp: the timestamp of the message. + + input_line: the current input line. + + output_line: the current output line. + + inout: [packets-only] 'in' if it's input packet, 'out' otherwise. + + remote: [packets-only] the remote address of the sender/receiver. + + entity: [packets-only] the local entity sending/receiving. + """ raise NotImplementedError("write_message not implemented") def write_configurations(self, state): diff --git a/src/devices/logger.py b/src/devices/logger.py index 1ce3e95..87897a0 100644 --- a/src/devices/logger.py +++ b/src/devices/logger.py @@ -16,82 +16,104 @@ """Logger functions. The module contains functions to log the new messages. +The 'content' dictionary is defined in the FormatDevice class. Functions: - + log: Log the given message. - + log_recv: Log a received packet. - + log_send: Log a sent packet. - + log_process: Log a processed packet. - + log_cfg: Log a configuration message. - + log_warning: Log a warning message. - + log_error: Log an error. - + countset_add_element: Add an element to the countset. + + log: log the given message. + + log_recv: log a received packet. + + log_send: log a sent packet. + + log_process: log a processed packet. + + log_cfg: log a configuration message. + + log_warning: log a warning message. + + log_error: log an error. + + countset_add_element: add an element to the countset. + + dict_regex_search: apply the regex over all the fields of the dictionary. Constants: - + COLORS: Colors to use in log messages. + + COLORS: ANSI colors to use in log messages. + + KIND_TO_COLOR: conversion between log kind and ANSI color """ -COLORS = {'HEADER': '\033[95m', 'BLUE': '\033[94m', 'OK': '\033[92m', - 'WARNING': '\033[93m', 'FAIL': '\033[91m', 'ENDC': '\033[0m', - 'BOLD': '\033[1m', 'UNDERLINE': '\033[4m'} +COLORS = { + 'RED': '\033[91m', + 'GREEN': '\033[92m', + 'YELLOW': '\033[93m', + 'BLUE': '\033[94m', + 'MAGENTA': '\033[95m', + 'BOLD': '\033[1m', + 'FAINT': '\033[2m', + 'ITALIC': '\033[3m', + 'UNDERLINE': '\033[4m', + 'END': '\033[0m', +} +KIND_TO_COLOR = { + 'WARNING': 'YELLOW|ITALIC', + 'ERROR': 'RED|BOLD', + 'IMPORTANT': 'BOLD' +} -def log(msg, level, state, color=None): + +def log(content, level, state): """Log the given message.""" if state['verbosity'] < level: return - # Add the clock if so - if not state['no_timestamp']: - if 'clocks' in state: - clock = " %s " % state['clocks'][1].isoformat() - else: - clock = "".ljust(28) - msg = clock + "|" + msg + # Add the clock if available + if 'clocks' in state and state['clocks'][1]: + content['timestamp'] = " %s " % state['clocks'][1].isoformat() - # Add the current line if so - if state['show_lines']: - msg = " %05d/%04d |%s" % ( - state['input_line'], state['output_line'] + 1, msg) + # Add the current line + content['input_line'] = state['input_line'] + content['output_line'] = state['output_line'] + 1 # This message count - if 'onlyIf' in state and not state['onlyIf'].search(msg): + # Apply the filter + if 'onlyIf' in state and not dict_regex_search(content, state['onlyIf']): return - # Highlight the message if so - if 'highlight' in state and state['highlight'].search(msg): - color = (color or "") + COLORS['BOLD'] + # Highlight the message if match + if 'highlight' in state and dict_regex_search(content, state['highlight']): + content['kind'] = content.get('kind', "") + "|IMPORTANT" # Apply color if specified - if color and not state['no_colors']: - msg = color + msg + COLORS['ENDC'] + if not state['no_colors']: + color = "" + for kind in filter(None, content.get('kind', '').split("|")): + for subkind in KIND_TO_COLOR[kind].split("|"): + color += COLORS[subkind] + if len(color) > 0: + content['description'] = color + content['description'] + \ + COLORS['END'] # Write the message - state['output_device'].write(msg) + state['format_device'].write_message(content, state) def log_recv(addr, entity, text, state, level=0): """Log a received packet.""" - if not state['ignore_packets']: - log("%s|%s|%s| %s" % - ("---> ".rjust(9), addr.center(24), entity.center(16), text), - level, state) + if state['ignore_packets']: + return + content = {'description': text, 'remote': addr, 'entity': entity, + 'inout': 'in'} + log(content, level, state) def log_send(addr, entity, text, state, level=0): """Log a sent packet.""" - if not state['ignore_packets']: - log("%s|%s|%s| %s" % - (" <---".ljust(9), addr.center(24), entity.center(16), text), - level, state) + if state['ignore_packets']: + return + content = {'description': text, 'remote': addr, 'entity': entity, + 'inout': 'out'} + log(content, level, state) def log_process(addr, entity, text, state, level=0): """Log a processed packet.""" - if not state['ignore_packets']: - log("%s|%s|%s| %s" % - ("".ljust(9), addr.center(24), entity.center(16), text), - level, state) + if state['ignore_packets']: + return + content = {'description': text, 'remote': addr, 'entity': entity} + log(content, level, state) def log_cfg(text, state, level=0): @@ -103,8 +125,8 @@ def log_cfg(text, state, level=0): def log_event(text, state, level=0): """Log an application event.""" - log("%s|%s|%s| %s" % ("".ljust(9), "".ljust(24), "".ljust(16), text), - level, state) + content = {'description': text} + log(content, level, state) def log_warning(text, state, level=0): @@ -114,9 +136,8 @@ def log_warning(text, state, level=0): countset_add_element(state['warnings'], text) if state['inline']: - text = "%s|%s|%s| *Warning: %s*" % ( - "".ljust(9), "".ljust(24), "".ljust(16), text) - log(text, level, state, COLORS['WARNING']) + content = {'description': "Warning: " + text, 'kind': 'WARNING'} + log(content, level, state) def log_error(text, state, level=0): @@ -126,9 +147,8 @@ def log_error(text, state, level=0): countset_add_element(state['errors'], text) if state['inline']: - text = "%s|%s|%s| **Error: %s**" % ( - "".ljust(9), "".ljust(24), "".ljust(16), text) - log(text, level, state, COLORS['FAIL']) + content = {'description': "Error: " + text, 'kind': 'ERROR'} + log(content, level, state) def countset_add_element(countset, el): @@ -136,3 +156,11 @@ def countset_add_element(countset, el): if el not in countset: countset[el] = [len(countset), 0] countset[el][1] += 1 + + +def dict_regex_search(content, regex): + """Apply the regex over all the fields of the dictionary.""" + match = False + for field in content: + match = match if match else regex.search(field) + return match diff --git a/src/devices/markdownformatdevice.py b/src/devices/markdownformatdevice.py index 4e9367c..ddb1d33 100644 --- a/src/devices/markdownformatdevice.py +++ b/src/devices/markdownformatdevice.py @@ -18,9 +18,9 @@ Classes: + MarkdownFormatDevice: Format device for Markdown. """ +from __future__ import absolute_import +from __init__ import __version__ from devices.formatdevice import FormatDevice -from devices.logger import COLORS -from rtilogparser import __version__ class MarkdownFormatDevice(FormatDevice): @@ -44,6 +44,8 @@ class MarkdownFormatDevice(FormatDevice): def __init__(self, state): """Initialize the device.""" self.write = state['output_device'].write + self.show_timestamp = not state['no_timestamp'] + self.show_lines = state['show_lines'] def write_header(self, state): """Write the header.""" @@ -70,32 +72,50 @@ def write_header(self, state): self.write("## Network Data Flow and Application Events") header = " In/Out | Remote Address | Local Entity | Message" headln = "---------|:----------------------:|:--------------:|--------" - if not state['no_timestamp']: + if self.show_timestamp: header = "Timestamp".ljust(28) + "|" + header - headln = "----------------------------|" + headln - if state['show_lines']: + headln = "-".ljust(28, "-") + "|" + headln + if self.show_lines: header = " Log/Parser |" + header headln = "------------|" + headln self.write(header) self.write(headln) - def write_message(self, state): + def write_message(self, content, state): """Write the message.""" - raise NotImplementedError("write_message not implemented") + # Create the standard message + if 'inout' not in content: + inout = "".ljust(9) + elif content['inout'] == 'in': + inout = "---> ".center(9) + else: + inout = " <---".center(9) + description = content['description'] + if content.get('kind') in ['ERROR', 'IMPORTANT']: + description = "**" + description + "**" + elif content.get('kind') == 'WARNING': + description = "*" + description + "*" + remote = content.get('remote', '').center(24) + entity = content.get('entity', '').center(16) + msg = "%s|%s|%s| %s" % (inout, remote, entity, description) + + # Add the optional columns + if self.show_timestamp: + timestamp = content.get('timestamp', '').center(28) + msg = timestamp + "|" + msg + if self.show_lines: + msg = " %05d/%04d |%s" % (content['input_line'], + content['output_line'], msg) + + self.write(msg) def write_warnings(self, state): """Write the warning messages.""" - title = "Warnings" - if not state['no_colors']: - title = COLORS['WARNING'] + title + COLORS['ENDC'] - self.write_countset(state['warnings'], title) + self.write_countset(state['warnings'], "Warnings") def write_errors(self, state): """Write the warning messages.""" - title = "Errors" - if not state['no_colors']: - title = COLORS['FAIL'] + title + COLORS['ENDC'] - self.write_countset(state['errors'], title) + self.write_countset(state['errors'], "Errors") def write_configurations(self, state): """Write the configuration messages.""" diff --git a/src/rtilogparser.py b/src/rtilogparser.py index 4cd006a..3bbdc91 100644 --- a/src/rtilogparser.py +++ b/src/rtilogparser.py @@ -49,6 +49,7 @@ from sys import exc_info from traceback import extract_tb +from __init__ import __version__ from devices.inputdevices import InputConsoleDevice, InputFileDevice from devices.logger import log_error, log_warning from devices.markdownformatdevice import MarkdownFormatDevice @@ -56,7 +57,7 @@ from logs import create_regex_list from utils import compare_times -__version__ = "1.2a2" + DATE_REGEX = re.compile(r'\[(\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}.\d{6})\]' + r'\[(\d{10}.\d{6})\]') SINGLE_DATE_REGEX = re.compile(r'\[(\d{10}).(\d{6})\]')