From 7ceb876e0d91bde7e68edada14503bcaec86abba Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Mon, 8 Apr 2019 13:32:01 -0500 Subject: [PATCH 1/3] Correct all linting errors --- tools/memap.py | 142 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 93 insertions(+), 49 deletions(-) diff --git a/tools/memap.py b/tools/memap.py index ad8230c278c..962e4cc7381 100644 --- a/tools/memap.py +++ b/tools/memap.py @@ -24,11 +24,6 @@ from os import sep from os.path import (basename, dirname, join, relpath, abspath, commonprefix, splitext) - -# Be sure that the tools directory is in the search path -ROOT = abspath(join(dirname(__file__), "..")) -path.insert(0, ROOT) - import re import csv import json @@ -39,8 +34,16 @@ from jinja2 import FileSystemLoader, StrictUndefined from jinja2.environment import Environment -from tools.utils import (argparse_filestring_type, argparse_lowercase_hyphen_type, - argparse_uppercase_type) + +# Be sure that the tools directory is in the search path +ROOT = abspath(join(dirname(__file__), "..")) +path.insert(0, ROOT) + +from tools.utils import ( + argparse_filestring_type, + argparse_lowercase_hyphen_type, + argparse_uppercase_type +) # noqa: E402 class _Parser(object): @@ -105,13 +108,19 @@ def parse_mapfile(self, mapfile): class _GccParser(_Parser): RE_OBJECT_FILE = re.compile(r'^(.+\/.+\.o(bj)?)$') - RE_LIBRARY_OBJECT = re.compile(r'^.+' + r''.format(sep) + r'lib((.+\.a)\((.+\.o(bj)?)\))$') + RE_LIBRARY_OBJECT = re.compile( + r'^.+' + r''.format(sep) + r'lib((.+\.a)\((.+\.o(bj)?)\))$' + ) RE_STD_SECTION = re.compile(r'^\s+.*0x(\w{8,16})\s+0x(\w+)\s(.+)$') RE_FILL_SECTION = re.compile(r'^\s*\*fill\*\s+0x(\w{8,16})\s+0x(\w+).*$') OBJECT_EXTENSIONS = (".o", ".obj") - ALL_SECTIONS = _Parser.SECTIONS + _Parser.OTHER_SECTIONS + \ - _Parser.MISC_FLASH_SECTIONS + ('unknown', 'OUTPUT') + ALL_SECTIONS = ( + _Parser.SECTIONS + + _Parser.OTHER_SECTIONS + + _Parser.MISC_FLASH_SECTIONS + + ('unknown', 'OUTPUT') + ) def check_new_section(self, line): """ Check whether a new section in a map file has been detected @@ -119,19 +128,16 @@ def check_new_section(self, line): Positional arguments: line - the line to check for a new section - return value - A section name, if a new section was found, False + return value - A section name, if a new section was found, None otherwise """ for i in self.ALL_SECTIONS: if line.startswith(i): - # should name of the section (assuming it's a known one) return i - if line.startswith('.'): - return 'unknown' # all others are classified are unknown + return 'unknown' else: - return False # everything else, means no change in section - + return None def parse_object_name(self, line): """ Parse a path to object file @@ -158,8 +164,10 @@ def parse_object_name(self, line): return join('[lib]', test_re_obj_name.group(2), test_re_obj_name.group(3)) else: - if (not line.startswith("LONG") and - not line.startswith("linker stubs")): + if ( + not line.startswith("LONG") and + not line.startswith("linker stubs") + ): print("Unknown object name found in GCC map file: %s" % line) return '[misc]' @@ -168,8 +176,8 @@ def parse_section(self, line): """ Parse data from a section of gcc map file examples: - 0x00004308 0x7c ./BUILD/K64F/GCC_ARM/mbed-os/hal/targets/hal/TARGET_Freescale/TARGET_KPSDK_MCUS/spi_api.o - .text 0x00000608 0x198 ./BUILD/K64F/GCC_ARM/mbed-os/core/mbed-rtos/rtx/TARGET_CORTEX_M/TARGET_RTOS_M4_M7/TOOLCHAIN/HAL_CM4.o + 0x00004308 0x7c ./BUILD/K64F/GCC_ARM/spi_api.o + .text 0x00000608 0x198 ./BUILD/K64F/HAL_CM4.o Positional arguments: line - the line to parse a section from @@ -215,7 +223,11 @@ def parse_mapfile(self, file_desc): self.module_add(object_name, object_size, current_section) common_prefix = dirname(commonprefix([ - o for o in self.modules.keys() if (o.endswith(self.OBJECT_EXTENSIONS) and not o.startswith("[lib]"))])) + o for o in self.modules.keys() + if ( + o.endswith(self.OBJECT_EXTENSIONS) + and not o.startswith("[lib]") + )])) new_modules = {} for name, stats in self.modules.items(): if name.startswith("[lib]"): @@ -245,9 +257,13 @@ def parse_object_name(self, line): else: is_obj = re.match(self.RE_OBJECT, line) if is_obj: - return join('[lib]', basename(is_obj.group(1)), is_obj.group(3)) + return join( + '[lib]', basename(is_obj.group(1)), is_obj.group(3) + ) else: - print("Malformed input found when parsing ARMCC map: %s" % line) + print( + "Malformed input found when parsing ARMCC map: %s" % line + ) return '[misc]' def parse_section(self, line): @@ -260,7 +276,7 @@ def parse_section(self, line): Positional arguments: line - the line to parse the section data from - """ + """ # noqa: E501 test_re = re.match(self.RE, line) if test_re: @@ -276,8 +292,10 @@ def parse_section(self, line): elif test_re.group(3) == 'Code': section = '.text' else: - print("Malformed input found when parsing armcc map: %s, %r" - % (line, test_re.groups())) + print( + "Malformed input found when parsing armcc map: %s, %r" + % (line, test_re.groups()) + ) return ["", 0, ""] @@ -307,10 +325,20 @@ def parse_mapfile(self, file_desc): self.module_add(*self.parse_section(line)) common_prefix = dirname(commonprefix([ - o for o in self.modules.keys() if (o.endswith(self.OBJECT_EXTENSIONS) and o != "anon$$obj.o" and o != "anon$$obj.obj" and not o.startswith("[lib]"))])) + o for o in self.modules.keys() + if ( + o.endswith(self.OBJECT_EXTENSIONS) + and o != "anon$$obj.o" + and o != "anon$$obj.obj" + and not o.startswith("[lib]") + )])) new_modules = {} for name, stats in self.modules.items(): - if name == "anon$$obj.o" or name == "anon$$obj.obj" or name.startswith("[lib]"): + if ( + name == "anon$$obj.o" + or name == "anon$$obj.obj" + or name.startswith("[lib]") + ): new_modules[name] = stats elif name.endswith(self.OBJECT_EXTENSIONS): new_modules[relpath(name, common_prefix)] = stats @@ -365,11 +393,13 @@ def parse_section(self, line): Positional_arguments: line - the line to parse section data from - """ + """ # noqa: E501 test_re = re.match(self.RE, line) if test_re: - if (test_re.group(2) == 'const' or - test_re.group(2) == 'ro code'): + if ( + test_re.group(2) == 'const' or + test_re.group(2) == 'ro code' + ): section = '.text' elif (test_re.group(2) == 'zero' or test_re.group(2) == 'uninit'): @@ -378,7 +408,7 @@ def parse_section(self, line): elif test_re.group(1)[0:6] == 'CSTACK': section = '.stack' else: - section = '.bss' # default section + section = '.bss' # default section elif test_re.group(2) == 'inited': section = '.data' @@ -409,7 +439,8 @@ def check_new_library(self, line): def check_new_object_lib(self, line): """ - Searches for objects within a library section and returns name. Example: + Searches for objects within a library section and returns name. + Example: rt7M_tl.a: [44] ABImemclr4.o 6 ABImemcpy_unaligned.o 118 @@ -435,7 +466,10 @@ def parse_command_line(self, lines): break for arg in line.split(" "): arg = arg.rstrip(" \n") - if (not arg.startswith("-")) and arg.endswith(self.OBJECT_EXTENSIONS): + if ( + not arg.startswith("-") + and arg.endswith(self.OBJECT_EXTENSIONS) + ): self.cmd_modules[basename(arg)] = arg common_prefix = dirname(commonprefix(list(self.cmd_modules.values()))) @@ -458,7 +492,7 @@ def parse_mapfile(self, file_desc): for line in infile: self.module_add(*self.parse_section(line)) - if line.startswith('*** MODULE SUMMARY'): # finish section + if line.startswith('*** MODULE SUMMARY'): # finish section break current_library = "" @@ -484,7 +518,6 @@ class MemapParser(object): print_sections = ('.text', '.data', '.bss') delta_sections = ('.text-delta', '.data-delta', '.bss-delta') - # sections to print info (generic for all toolchains) sections = _Parser.SECTIONS misc_flash_sections = _Parser.MISC_FLASH_SECTIONS @@ -498,7 +531,6 @@ def __init__(self): # short version with specific depth self.short_modules = dict() - # Memory report (sections + summary) self.mem_report = [] @@ -528,7 +560,7 @@ def reduce_depth(self, depth): mbed-os/drivers """ - if depth == 0 or depth == None: + if depth == 0 or depth is None: self.short_modules = deepcopy(self.modules) else: self.short_modules = dict() @@ -539,8 +571,9 @@ def reduce_depth(self, depth): new_name = join(*split_name[:depth]) self.short_modules.setdefault(new_name, defaultdict(int)) for section_idx, value in v.items(): - self.short_modules[new_name][section_idx] += self.modules[module_name][section_idx] - self.short_modules[new_name][section_idx + '-delta'] += self.modules[module_name][section_idx] + self.short_modules[new_name][section_idx] += value + delta_name = section_idx + '-delta' + self.short_modules[new_name][delta_name] += value if self.old_modules: for module_name, v in self.old_modules.items(): split_name = module_name.split(sep) @@ -549,7 +582,8 @@ def reduce_depth(self, depth): new_name = join(*split_name[:depth]) self.short_modules.setdefault(new_name, defaultdict(int)) for section_idx, value in v.items(): - self.short_modules[new_name][section_idx + '-delta'] -= self.old_modules[module_name][section_idx] + delta_name = section_idx + '-delta' + self.short_modules[new_name][delta_name] -= value export_formats = ["json", "csv-ci", "html", "table"] @@ -657,7 +691,10 @@ def generate_html(self, file_desc): if not modules: break next_module = modules.pop(0) - if not any(cld['name'] == next_module for cld in cur_text['children']): + if not any( + cld['name'] == next_module + for cld in cur_text['children'] + ): break cur_text = self._move_up_tree(cur_text, next_module) cur_data = self._move_up_tree(cur_data, next_module) @@ -759,8 +796,10 @@ def generate_table(self, file_desc): row = [i] for k in self.print_sections: - row.append("{}({:+})".format(self.short_modules[i][k], - self.short_modules[i][k + "-delta"])) + row.append("{}({:+})".format( + self.short_modules[i][k], + self.short_modules[i][k + "-delta"] + )) table.add_row(row) @@ -815,7 +854,7 @@ def compute_report(self): for name, sizes in sorted(self.short_modules.items()): self.mem_report.append({ "module": name, - "size":{ + "size": { k: sizes.get(k, 0) for k in (self.print_sections + self.delta_sections) } @@ -855,6 +894,7 @@ def parse(self, mapfile, toolchain): print("I/O error({0}): {1}".format(error.errno, error.strerror)) return False + def main(): """Entry Point""" version = '0.4.0' @@ -912,10 +952,13 @@ def main(): returned_string = None # Write output in file - if args.output != None: - returned_string = memap.generate_output(args.export, \ - depth, args.output) - else: # Write output in screen + if args.output is not None: + returned_string = memap.generate_output( + args.export, + depth, + args.output + ) + else: # Write output in screen returned_string = memap.generate_output(args.export, depth) if args.export == 'table' and returned_string: @@ -923,5 +966,6 @@ def main(): exit(0) + if __name__ == "__main__": main() From 76226b2f249f373fc25d9c8dffe9ce8fbcc8fa89 Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Mon, 8 Apr 2019 13:32:44 -0500 Subject: [PATCH 2/3] Parse section starts beginning with whitespace --- tools/memap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/memap.py b/tools/memap.py index 962e4cc7381..f692aeb3c20 100644 --- a/tools/memap.py +++ b/tools/memap.py @@ -131,6 +131,7 @@ def check_new_section(self, line): return value - A section name, if a new section was found, None otherwise """ + line = line.strip() for i in self.ALL_SECTIONS: if line.startswith(i): return i From 002a2b17c03132696f3767f4217a7379f8e2f6f9 Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Tue, 9 Apr 2019 09:19:41 -0500 Subject: [PATCH 3/3] Fix the bug that travis caught --- tools/memap.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/memap.py b/tools/memap.py index f692aeb3c20..a8fde8e4692 100644 --- a/tools/memap.py +++ b/tools/memap.py @@ -131,9 +131,9 @@ def check_new_section(self, line): return value - A section name, if a new section was found, None otherwise """ - line = line.strip() + line_s = line.strip() for i in self.ALL_SECTIONS: - if line.startswith(i): + if line_s.startswith(i): return i if line.startswith('.'): return 'unknown'