From 431a0eb076d1960eec14ed50eead3c6ba5078b25 Mon Sep 17 00:00:00 2001 From: Yuriy Volynets Date: Fri, 5 Jul 2019 19:12:01 +0300 Subject: [PATCH 1/3] Added approaches to move loganalyzer to pytest. Signed-off-by: Yuriy Volynets --- doc/loganalyzer_to_pytest.md | 108 +++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 doc/loganalyzer_to_pytest.md diff --git a/doc/loganalyzer_to_pytest.md b/doc/loganalyzer_to_pytest.md new file mode 100644 index 0000000000..b9521ab45a --- /dev/null +++ b/doc/loganalyzer_to_pytest.md @@ -0,0 +1,108 @@ +#### Approach 1 + +#### Requirements +Move existed loganalyzer to be supported by pytest framework. + +#### Overview +Current loganalyzer implementation consists of the following parts: +Init - copy loganalyzer.py to the DUT. Add start marker to the syslog. + +Analyze - copy loganalyzer.py and all match,expect,ignore files to the DUT. Extract logs from syslog. Add end marker. Analyze logs based on regexp described in match,expect,ignore files. +Generate summary and result files. + +End - Check if "TOTAL MATCHES != 0" then generate system dump. Store system dump, result and summary files to the ansible host. + +In proposed approach script copies loganalyzer.py and files with regexp to the DUT and runs syslog analyze on the DUT. + +Current loganalyzer implementation description: +https://github.com/Azure/SONiC/wiki/LogAnalyzer + +**Development** +**Implement:** + Module "loganalyzer.py" with class "Loganalyzer". + "Loganalyzer" class - executes shell commands on DUT. + **"Loganalyzer" class interface:** +- __init__(match: list, expect: list, ignore: list) +- exec_cmd(cmd) - execute shell command on the DUT +- extract_syslog() - calls "extract_syslog" from "log_processor.py" +- upload_search_files(dest) - upload match, expect, ignore files to the DUT +- parse_summary_file() - convert generated on DUT "result.loganalysis.{TEST_NAME}..." file to json. +- parse_result_file() - convert generated on DUT "summary.loganalysis.{TEST_NAME}" file to json. +- init() - copy log_processor.py to the DUT. Add start marker to the DUT syslog. +- analyze() - calls extract_syslog(), upload_search_files(dest). Copy log_processor.py and all match, expect, ignore files to the DUT. Calls "log_processor.py" on the DUT with "--action analyze", which - extract syslog logs by start/end markers. Analyze logs based on regexp described in match, expect, ignore files. Generate summary and result files. Calls parse_summary_file(), parse_result_file(). Return converted to dictionary "result.loganalysis.{TEST_NAME}..." and "summary.loganalysis.{TEST_NAME}" files. + +**Implement**: +Module "log_processor.py" with class "LogProcessor". +"log_processor.py" - python module which is running on the DUT. Module perform some actions based on script input parameters. It will be based on the legacy loganalyzer.py. +Supported actions: init, analyze, extract_syslog +- init and analyze are described above (legacy). +- extract_syslog - Extract all syslog entries since the latest start marker (new). + +"LogProcessor" class interface: +- Includes legacy "LogAnalyzer" class from loganalyzer.py module. +- New: +- extract_syslog(dst="/tmp/syslog") - Extract all syslog entries since the latest start marker and store them to "/tmp/syslog" folder on the DUT. + +Usage example: + + def test(localhost, ansible_adhoc, testbed): + loganalyzer = Loganalyzer(match="{PATH}/loganalyzer_common_match.txt", expect="{PATH}/loganalyzer_common_expect.txt", ignore="{PATH}/loganalyzer_common_ignore.txt") + loganalyzer.init() + # Perform test steps + result = loganalyzer.analyze() + if result["summary"]["TOTAL MATCHES"] != 0: + loganalyzer.fetch_fail_artifacts(dst="test/{}".format(loganalyzer.unic_test_name)) + pytest.fail("SOME_MESSAGE\n{}\n{}".format(result["summary"], result["match_messages"])) + +#### Approach 2 +#### Requirements +Move existed loganalyzer to be supported by pytest framework. + +#### Overview +At this approach all the work can be done on the Ansible host if script will download extracted syslog file to the Ansible host. + +Current loganalyzer implementation description: +https://github.com/Azure/SONiC/wiki/LogAnalyzer + +#### Development +Module "loganalyzer.py" with class "Loganalyzer". + +"Loganalyzer" class interface: +- load_common_config() - Load regexps from common configuration files: match, expect and ignore which are located in some configuration directory or with current module. +- get_search_regexp() - returns dictionary with defined regular expressions which are used for logs processing. +- update_match(action="pop/add", type="match/expect/ignore", *args) - add or remove regexp for syslog processing. +Supported parameters values: +action: "pop", "add" +type: "match", "expect", "ignore" +args: *args +- init() - Add start marker to the DUT syslog. +- analyze() - Extract syslog and copy it to ansible host. Analyze extracted syslog files localy. Return python dictionary object. +Return example: + {"summary": + "counters": {"match": 1, "expected_match": 0, "expected_missing_match": 0}, + "match_files": {"/tmp/syslog": {"match": 0, "expected_match": 32}, + "/tmp/syslog1": {"match": 0, "expected_match": 15}}, + "match_messages": {"/tmp/syslog1": ["Message 1", "Message 2", "Message n"], + "/tmp/syslog2": ["Message 1", "Message 2", "Message n"]} + } +- run_cmd(callable) - call function and analyze DUT logs, return the same result as "analyze" function + +Usage example: + + from loganalyzer import Loganalyzer + def test(localhost, ansible_adhoc, testbed): + loganalyzer = Loganalyzer() + + # If it is a need to load common search regular expressions. It will load common: match, expect, ignore regular expressions. + loganalyzer.load_common_config() + + # Add start marker to DUT syslog + loganalyzer.init() + + # Add search marker specific for None MAC test + loganalyzer.update_match(action="add", type="match", "Runtime error: can't parse mac address 'None'") + + # Execute test steps here... + + result = loganalyzer.analyze() + assert result["summary"]["counters"]["match"] == 0, "Failure message\n{}\n{}".format(result["summary"], result["match_messages"]) From e7436e5001bb7ada06a5c166473d6b6ec00649a6 Mon Sep 17 00:00:00 2001 From: Yuriy Volynets Date: Mon, 8 Jul 2019 16:39:35 +0300 Subject: [PATCH 2/3] Added updates, comments fixes. Signed-off-by: Yuriy Volynets --- doc/loganalyzer_to_pytest.md | 120 +++++++++++++---------------------- 1 file changed, 43 insertions(+), 77 deletions(-) diff --git a/doc/loganalyzer_to_pytest.md b/doc/loganalyzer_to_pytest.md index b9521ab45a..9b33ac77fd 100644 --- a/doc/loganalyzer_to_pytest.md +++ b/doc/loganalyzer_to_pytest.md @@ -1,60 +1,3 @@ -#### Approach 1 - -#### Requirements -Move existed loganalyzer to be supported by pytest framework. - -#### Overview -Current loganalyzer implementation consists of the following parts: -Init - copy loganalyzer.py to the DUT. Add start marker to the syslog. - -Analyze - copy loganalyzer.py and all match,expect,ignore files to the DUT. Extract logs from syslog. Add end marker. Analyze logs based on regexp described in match,expect,ignore files. -Generate summary and result files. - -End - Check if "TOTAL MATCHES != 0" then generate system dump. Store system dump, result and summary files to the ansible host. - -In proposed approach script copies loganalyzer.py and files with regexp to the DUT and runs syslog analyze on the DUT. - -Current loganalyzer implementation description: -https://github.com/Azure/SONiC/wiki/LogAnalyzer - -**Development** -**Implement:** - Module "loganalyzer.py" with class "Loganalyzer". - "Loganalyzer" class - executes shell commands on DUT. - **"Loganalyzer" class interface:** -- __init__(match: list, expect: list, ignore: list) -- exec_cmd(cmd) - execute shell command on the DUT -- extract_syslog() - calls "extract_syslog" from "log_processor.py" -- upload_search_files(dest) - upload match, expect, ignore files to the DUT -- parse_summary_file() - convert generated on DUT "result.loganalysis.{TEST_NAME}..." file to json. -- parse_result_file() - convert generated on DUT "summary.loganalysis.{TEST_NAME}" file to json. -- init() - copy log_processor.py to the DUT. Add start marker to the DUT syslog. -- analyze() - calls extract_syslog(), upload_search_files(dest). Copy log_processor.py and all match, expect, ignore files to the DUT. Calls "log_processor.py" on the DUT with "--action analyze", which - extract syslog logs by start/end markers. Analyze logs based on regexp described in match, expect, ignore files. Generate summary and result files. Calls parse_summary_file(), parse_result_file(). Return converted to dictionary "result.loganalysis.{TEST_NAME}..." and "summary.loganalysis.{TEST_NAME}" files. - -**Implement**: -Module "log_processor.py" with class "LogProcessor". -"log_processor.py" - python module which is running on the DUT. Module perform some actions based on script input parameters. It will be based on the legacy loganalyzer.py. -Supported actions: init, analyze, extract_syslog -- init and analyze are described above (legacy). -- extract_syslog - Extract all syslog entries since the latest start marker (new). - -"LogProcessor" class interface: -- Includes legacy "LogAnalyzer" class from loganalyzer.py module. -- New: -- extract_syslog(dst="/tmp/syslog") - Extract all syslog entries since the latest start marker and store them to "/tmp/syslog" folder on the DUT. - -Usage example: - - def test(localhost, ansible_adhoc, testbed): - loganalyzer = Loganalyzer(match="{PATH}/loganalyzer_common_match.txt", expect="{PATH}/loganalyzer_common_expect.txt", ignore="{PATH}/loganalyzer_common_ignore.txt") - loganalyzer.init() - # Perform test steps - result = loganalyzer.analyze() - if result["summary"]["TOTAL MATCHES"] != 0: - loganalyzer.fetch_fail_artifacts(dst="test/{}".format(loganalyzer.unic_test_name)) - pytest.fail("SOME_MESSAGE\n{}\n{}".format(result["summary"], result["match_messages"])) - -#### Approach 2 #### Requirements Move existed loganalyzer to be supported by pytest framework. @@ -65,33 +8,46 @@ Current loganalyzer implementation description: https://github.com/Azure/SONiC/wiki/LogAnalyzer #### Development -Module "loganalyzer.py" with class "Loganalyzer". +Module "loganalyzer.py" with class "Loganalyzer". Class is intendent to do: +- Add start marker to the DUT syslog +- Extract DUT syslog based on the markers. Download extracted logs to the Ansible host +- Perform analize of downloaded syslog and return the result + +- Configure regular expressions which will be used to found matches in the logs "Loganalyzer" class interface: -- load_common_config() - Load regexps from common configuration files: match, expect and ignore which are located in some configuration directory or with current module. -- get_search_regexp() - returns dictionary with defined regular expressions which are used for logs processing. -- update_match(action="pop/add", type="match/expect/ignore", *args) - add or remove regexp for syslog processing. -Supported parameters values: -action: "pop", "add" -type: "match", "expect", "ignore" -args: *args +- __init__(ansible_host: ansible_host) +- load_common_config() - Load regular expressions from common configuration files: match, expect and ignore which are located in some configuration directory or with current module. Clear previous configured match, expect and ignore. Save loaded configuration. +- parse_regexp_file(file_path) - read and parse regular expressions from specified file. Return list of strings of defined regular expressions. +- run_cmd(callable, *args, **kwargs) - call function and analyze DUT logs, return the same result as "analyze" function - init() - Add start marker to the DUT syslog. - analyze() - Extract syslog and copy it to ansible host. Analyze extracted syslog files localy. Return python dictionary object. Return example: - {"summary": - "counters": {"match": 1, "expected_match": 0, "expected_missing_match": 0}, - "match_files": {"/tmp/syslog": {"match": 0, "expected_match": 32}, - "/tmp/syslog1": {"match": 0, "expected_match": 15}}, - "match_messages": {"/tmp/syslog1": ["Message 1", "Message 2", "Message n"], - "/tmp/syslog2": ["Message 1", "Message 2", "Message n"]} - } -- run_cmd(callable) - call function and analyze DUT logs, return the same result as "analyze" function +{"counters": {"match": 1, "expected_match": 0, "expected_missing_match": 0}, + "match_files": {"/tmp/syslog": {"match": 0, "expected_match": 32}, + "/tmp/syslog1": {"match": 0, "expected_match": 15} + }, + "match_messages": {"/tmp/syslog1": ["Message 1", "Message 2", "Message n"], + "/tmp/syslog2": ["Message 1", "Message 2", "Message n"] + } +} +- save_full_log(file_path) - save downloaded DUT syslog logs to the Ansible host folder specified in 'path' input parameter. + +Python properties with defined setter and getter: +- match - list of regular expression strings to match +- expect - list of regular expression strings to expect +- ignore - list of regular expression strings to ignore Usage example: + from ansible_host import ansible_host from loganalyzer import Loganalyzer + def test(localhost, ansible_adhoc, testbed): - loganalyzer = Loganalyzer() + hostname = testbed['dut'] + ans_host = ansible_host(ansible_adhoc, hostname) + + loganalyzer = Loganalyzer(ans_host) # If it is a need to load common search regular expressions. It will load common: match, expect, ignore regular expressions. loganalyzer.load_common_config() @@ -99,10 +55,20 @@ Usage example: # Add start marker to DUT syslog loganalyzer.init() - # Add search marker specific for None MAC test - loganalyzer.update_match(action="add", type="match", "Runtime error: can't parse mac address 'None'") + # Example: If test need specific search marker + # Add search marker + loganalyzer.match.append("Runtime error: can't parse mac address 'None'") + + # Example: Get current match regular expressions + loganalyzer.match + + # Example: Remove specific match regular expression + loganalyzer.match.remove("Runtime error: can't parse mac address 'None'") + # Example: read test specific match file and add read strings to the existed match list + loganalyzer.match.extend(loganalyzer.parse_regexp_file("PATH_TO_THE_FILE/FILE.txt")) + # Execute test steps here... result = loganalyzer.analyze() - assert result["summary"]["counters"]["match"] == 0, "Failure message\n{}\n{}".format(result["summary"], result["match_messages"]) + assert result["counters"]["match"] == 0, "Failure message\n{}\n{}".format(result["counters"], result["match_messages"]) From b0bb3ae8876d8a070496aea49e3b4c4ba2cbaa25 Mon Sep 17 00:00:00 2001 From: Yuriy Volynets Date: Thu, 11 Jul 2019 13:45:07 +0300 Subject: [PATCH 3/3] Loganalyzer to pytest High level design Signed-off-by: Yuriy Volynets --- doc/loganalyzer_to_pytest.md | 47 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/doc/loganalyzer_to_pytest.md b/doc/loganalyzer_to_pytest.md index 9b33ac77fd..fdc38a0c4b 100644 --- a/doc/loganalyzer_to_pytest.md +++ b/doc/loganalyzer_to_pytest.md @@ -2,26 +2,26 @@ Move existed loganalyzer to be supported by pytest framework. #### Overview -At this approach all the work can be done on the Ansible host if script will download extracted syslog file to the Ansible host. +At this approach all the work will be done on the Ansible host in comparing with current implementation, so script will download extracted syslog file to the Ansible host. +Functionality will give possibility to: +- Add start marker to the DUT syslog. Start Marker will be automatically generated based on the prefix + date. Start marker format will be kept the same as in current implementation +- Configure regular expressions which will be used to found matches in the syslog logs +- Extract DUT syslog based on the markers. Download extracted logs to the Ansible host +- Perform analysis of downloaded syslog and return the result Current loganalyzer implementation description: https://github.com/Azure/SONiC/wiki/LogAnalyzer #### Development -Module "loganalyzer.py" with class "Loganalyzer". Class is intendent to do: -- Add start marker to the DUT syslog -- Extract DUT syslog based on the markers. Download extracted logs to the Ansible host -- Perform analize of downloaded syslog and return the result - -- Configure regular expressions which will be used to found matches in the logs +Module "loganalyzer.py" with class "Loganalyzer". "Loganalyzer" class interface: -- __init__(ansible_host: ansible_host) -- load_common_config() - Load regular expressions from common configuration files: match, expect and ignore which are located in some configuration directory or with current module. Clear previous configured match, expect and ignore. Save loaded configuration. -- parse_regexp_file(file_path) - read and parse regular expressions from specified file. Return list of strings of defined regular expressions. -- run_cmd(callable, *args, **kwargs) - call function and analyze DUT logs, return the same result as "analyze" function -- init() - Add start marker to the DUT syslog. -- analyze() - Extract syslog and copy it to ansible host. Analyze extracted syslog files localy. Return python dictionary object. +- __init__(ansible_host: ansible_host, run_dir="/tmp") +- load_common_config() - Clear previous configured match, expect and ignore. Load regular expressions from common configuration files: match, expect and ignore which are located in some configuration directory or with current module. Save loaded configuration to the self.match_regex, self.expect_regex and self.ignore_regex attributes. +- parse_regexp_file(file_path) - Read and parse regular expressions from specified file. Return list of strings of defined regular expressions. +- run_cmd(callable, *args, **kwargs) - Call function and analyze DUT syslog during function execution. Return the same result as "analyze" function. +- init(marker_prefix: string) - Add start marker to the DUT syslog. Generated marker format: marker_prefix + "%Y-%m-%d-%H:%M:%S" +- analyze() - Extract syslog based on the start marker and copy it to ansible host. Analyze extracted syslog file localy. Return python dictionary object. Return example: {"counters": {"match": 1, "expected_match": 0, "expected_missing_match": 0}, "match_files": {"/tmp/syslog": {"match": 0, "expected_match": 32}, @@ -31,12 +31,12 @@ Return example: "/tmp/syslog2": ["Message 1", "Message 2", "Message n"] } } -- save_full_log(file_path) - save downloaded DUT syslog logs to the Ansible host folder specified in 'path' input parameter. +- save_full_log(dest_path) - Download extracted DUT syslog (/tmp/syslog) to the Ansible host folder specified in 'dest_path' input parameter. -Python properties with defined setter and getter: -- match - list of regular expression strings to match -- expect - list of regular expression strings to expect -- ignore - list of regular expression strings to ignore +Attributes: +- match_regex - list of regular expression strings to match +- expect_regex - list of regular expression strings to expect +- ignore_regex - list of regular expression strings to ignore Usage example: @@ -49,24 +49,23 @@ Usage example: loganalyzer = Loganalyzer(ans_host) - # If it is a need to load common search regular expressions. It will load common: match, expect, ignore regular expressions. + # If it is a need to load common search regular expressions. It will load regexp from common files and store values in the match_regex, expect_regex and ignore_regex attributes. loganalyzer.load_common_config() # Add start marker to DUT syslog loganalyzer.init() - # Example: If test need specific search marker - # Add search marker + # Example: If test need specific search marker it can be added loganalyzer.match.append("Runtime error: can't parse mac address 'None'") # Example: Get current match regular expressions - loganalyzer.match + print(loganalyzer.match_regex) # Example: Remove specific match regular expression - loganalyzer.match.remove("Runtime error: can't parse mac address 'None'") + loganalyzer.match_regex.remove("Runtime error: can't parse mac address 'None'") # Example: read test specific match file and add read strings to the existed match list - loganalyzer.match.extend(loganalyzer.parse_regexp_file("PATH_TO_THE_FILE/FILE.txt")) + loganalyzer.match_regex.extend(loganalyzer.parse_regexp_file("PATH_TO_THE_FILE/FILE.txt")) # Execute test steps here...