diff --git a/targets/TARGET_NXP/TARGET_LPC176X/device/TOOLCHAIN_GCC_ARM/startup_LPC17xx.S b/targets/TARGET_NXP/TARGET_LPC176X/device/TOOLCHAIN_GCC_ARM/startup_LPC17xx.S index 4a8e973a338..fffc8954ee1 100644 --- a/targets/TARGET_NXP/TARGET_LPC176X/device/TOOLCHAIN_GCC_ARM/startup_LPC17xx.S +++ b/targets/TARGET_NXP/TARGET_LPC176X/device/TOOLCHAIN_GCC_ARM/startup_LPC17xx.S @@ -30,20 +30,6 @@ aborting compilation, it is not the run time limit: Heap_Size + Stack_Size = 0x80 + 0x80 = 0x100 */ - .section .stack - .align 3 -#ifdef __STACK_SIZE - .equ Stack_Size, __STACK_SIZE -#else - .equ Stack_Size, 0xc00 -#endif - .globl __StackTop - .globl __StackLimit -__StackLimit: - .space Stack_Size - .size __StackLimit, . - __StackLimit -__StackTop: - .size __StackTop, . - __StackTop .section .heap .align 3 diff --git a/tools/cmake/mbed_target_functions.cmake b/tools/cmake/mbed_target_functions.cmake index 11a8e203b98..f5dcbec1a3f 100644 --- a/tools/cmake/mbed_target_functions.cmake +++ b/tools/cmake/mbed_target_functions.cmake @@ -48,6 +48,9 @@ function(mbed_generate_map_file target) set(MBED_MEMAP_CREATE_JSON FALSE CACHE BOOL "create report in json file") set(MBED_MEMAP_CREATE_HTML FALSE CACHE BOOL "create report in html file") + # Config process saves the JSON file here + set(MEMORY_BANKS_JSON_PATH ${CMAKE_BINARY_DIR}/memory_banks.json) + # generate table for screen add_custom_command( TARGET @@ -55,7 +58,7 @@ function(mbed_generate_map_file target) POST_BUILD COMMAND ${Python3_EXECUTABLE} -m memap.memap -t ${MBED_TOOLCHAIN} ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.map - --depth ${MBED_MEMAP_DEPTH} + --depth ${MBED_MEMAP_DEPTH} --memory-banks-json ${MEMORY_BANKS_JSON_PATH} WORKING_DIRECTORY ${mbed-os_SOURCE_DIR}/tools/python ) @@ -71,6 +74,7 @@ function(mbed_generate_map_file target) --depth ${MBED_MEMAP_DEPTH} -e json -o ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.memmap.json + --memory-banks-json ${MEMORY_BANKS_JSON_PATH} WORKING_DIRECTORY ${mbed-os_SOURCE_DIR}/tools/python ) @@ -87,6 +91,7 @@ function(mbed_generate_map_file target) --depth ${MBED_MEMAP_DEPTH} -e html -o ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.memmap.html + --memory-banks-json ${MEMORY_BANKS_JSON_PATH} WORKING_DIRECTORY ${mbed-os_SOURCE_DIR}/tools/python ) diff --git a/tools/python/memap/memap.py b/tools/python/memap/memap.py index 2e2ddbc9911..af781051223 100644 --- a/tools/python/memap/memap.py +++ b/tools/python/memap/memap.py @@ -17,9 +17,39 @@ See the License for the specific language governing permissions and limitations under the License. """ -from __future__ import print_function, division, absolute_import +from __future__ import annotations -from abc import abstractmethod, ABCMeta +""" + memap term glossary, for code reviewers and for developers working on this script + -------------------------------------------------------------------------------------------- + + - Module: In this script, a module refers to the code library (i.e. the .o file) where an object came from. + - Symbol: Any entity declared in the program that has a global address. Generally this means any global + variables and all functions. Note that symbol names have to be alphanumeric, so C++ implemented + "mangling" to encode class and function names as valid symbol names. This means that C++ symbols will look + like "_ZN4mbed8AnalogIn6_mutexE" to the linker. You can use the "c++filt" tool to convert those back + into a human readable name like "mbed::AnalogIn::_mutex." + - Section: A logical region of an elf of object file. Each section has a specific name and occupies contiguous memory. + It's a vague term. + - Input section: The section in the object (.o/.obj) file that a symbol comes from. It generally has a specific name, + e.g. a function could be from the .text.my_function input section. + - Output section: The section in the linked application file (.elf) that a symbol has been put into. The output + section *might* match the input section, but not always! A linker script can happily put stuff from + any input section into any output section if so desired. + - VMA (Virtual Memory Address): The address that an output section will have when the application runs. + Note that this name is something of a misnomer as it is inherited from desktop Linux. There is no virtual + memory on microcontrollers! + - LMA (Load Memory Address): The address that an output section is loaded from in flash when the program boots. + - .bss: Output section for global variables which have zero values at boot. This region of RAM is zeroed at boot. + - .data: Output section for global variables with nonzero default values. This region is copied, as a single block + of data, from the LMA to the VMA at boot. + - .text: Output section for code and constant data (e.g. the values of constant arrays). This region + is mapped directly into flash and does not need to be copied at runtime. +""" + +import dataclasses +from typing import Optional, TextIO +from abc import abstractmethod, ABC from sys import stdout, exit, argv, path from os import sep from os.path import (basename, dirname, join, relpath, abspath, commonprefix, @@ -47,7 +77,28 @@ ) # noqa: E402 -class _Parser(with_metaclass(ABCMeta, object)): +@dataclasses.dataclass +class MemoryBankInfo: + name: str + """Name of the bank, from cmsis_mcu_descriptions.json""" + + start_addr: int + """Start address of memory bank""" + + total_size: int + """Total size of the memory bank in bytes""" + + used_size: int = 0 + """Size used in the memory bank in bytes (sum of the sizes of all symbols)""" + + def contains_addr(self, addr: int) -> bool: + """ + :return: True if the given address is contained inside this memory bank + """ + return addr >= self.start_addr and addr < self.start_addr + self.total_size + + +class _Parser(ABC): """Internal interface for parsing""" SECTIONS = ('.text', '.data', '.bss', '.heap', '.stack') MISC_FLASH_SECTIONS = ('.interrupts', '.flash_config') @@ -57,19 +108,62 @@ class _Parser(with_metaclass(ABCMeta, object)): '.stabstr', '.ARM.exidx', '.ARM') def __init__(self): - self.modules = dict() + self.modules: dict[str, dict[str, int]] = {} + """Dict of object name to {section name, size}""" + + self.memory_banks: dict[str, list[MemoryBankInfo]] = {"RAM": [], "ROM": []} + """Memory bank info, by type (RAM/ROM)""" + + def _add_symbol_to_memory_banks(self, symbol_name: str, symbol_start_addr: int, size: int) -> None: + """ + Update the memory banks structure to add the space used by a symbol. + """ + + if len(self.memory_banks["RAM"]) == 0 and len(self.memory_banks["ROM"]) == 0: + # No memory banks loaded, skip + return + + end_addr = symbol_start_addr + size + for banks in self.memory_banks.values(): + for bank_info in banks: + if bank_info.contains_addr(symbol_start_addr): + if bank_info.contains_addr(end_addr): + # Symbol fully inside this memory bank + bank_info.used_size += size - def module_add(self, object_name, size, section): - """ Adds a module or section to the list + # Uncomment to show debug info about each symbol + # print(f"Symbol {symbol_name} uses {size} bytes in {bank_info.name}") + + return + print(f"Warning: Symbol {symbol_name} is only partially contained by memory bank {bank_info.name}") + first_addr_after_bank = bank_info.start_addr + bank_info.total_size + bank_info.used_size += first_addr_after_bank - symbol_start_addr + + print(f"Warning: Symbol {symbol_name} (at address 0x{symbol_start_addr:x}, size {size}) is not inside a " + f"defined memory bank for this target.") + + def add_symbol(self, symbol_name: str, object_name: str, start_addr: int, size: int, section: str, vma_lma_offset: int) -> None: + """ Adds information about a symbol (e.g. a function or global variable) to the data structures. Positional arguments: - object_name - name of the entry to add - size - the size of the module being added - section - the section the module contributes to + symbol_name - Descriptive name of the symbol, e.g. ".text.some_function" + object_name - name of the object file containing the symbol + start addr - start address of symbol + size - the size of the symbol being added + section - Name of the output section, e.g. ".text". Can also be "unknown". + vma_lma_offset - Offset from where the output section exists in memory to where it's loaded from. If nonzero, + the initializer for this section will be considered too """ - if not object_name or not size or not section: + if not object_name or not size: return + # Don't count the heap output section for memory bank size tracking, because the linker scripts (almost always?) + # configure that section to expand to fill the remaining amount of space + if section not in {".heap"}: + self._add_symbol_to_memory_banks(symbol_name, start_addr, size) + if vma_lma_offset != 0: + self._add_symbol_to_memory_banks(f"", start_addr + vma_lma_offset, size) + if object_name in self.modules: self.modules[object_name].setdefault(section, 0) self.modules[object_name][section] += size @@ -82,19 +176,25 @@ def module_add(self, object_name, size, section): contents[section] += size return - new_module = defaultdict(int) - new_module[section] = size - self.modules[object_name] = new_module + new_symbol = defaultdict(int) + new_symbol[section] = size + self.modules[object_name] = new_symbol - def module_replace(self, old_object, new_object): - """ Replaces an object name with a new one + def load_memory_banks_info(self, memory_banks_json_file: TextIO) -> None: + """ + Load the memory bank information from a memory_banks.json file """ - if old_object in self.modules: - self.modules[new_object] = self.modules[old_object] - del self.modules[old_object] + memory_banks_json = json.load(memory_banks_json_file) + for bank_type, banks in memory_banks_json["configured_memory_banks"].items(): + for bank_name, bank_data in banks.items(): + self.memory_banks[bank_type].append(MemoryBankInfo( + name=bank_name, + start_addr=bank_data["start"], + total_size=bank_data["size"] + )) @abstractmethod - def parse_mapfile(self, mapfile): + def parse_mapfile(self, file_desc: TextIO) -> dict[str, dict[str, int]]: """Parse a given file object pointing to a map file Positional arguments: @@ -116,32 +216,84 @@ class _GccParser(_Parser): RE_TRANS_FILE = re.compile(r'^(.+\/|.+\.ltrans.o(bj)?)$') OBJECT_EXTENSIONS = (".o", ".obj") + # Parses a line beginning a new output section in the map file that has a load address + # Groups: + # 1 = section name, including dot + # 2 = in-memory address, hex, no 0x + # 3 = section size + # 4 = load address, i.e. where is the data for this section stored in flash + RE_OUTPUT_SECTION_WITH_LOAD_ADDRESS = re.compile(r'^(.\w+) +0x([0-9a-f]+) +0x([0-9a-f]+) +load address +0x([0-9a-f]+)') + + # Parses a line beginning a new output section in the map file does not have a load address + # Groups: + # 1 = section name, including dot + # 2 = in-memory address, hex, no 0x + # 3 = section size + # 4 = load address, i.e. where is the data for this section stored in flash + RE_OUTPUT_SECTION_NO_LOAD_ADDRESS = re.compile(r'^(.\w+) +0x([0-9a-f]+) +0x([0-9a-f]+)') + + # Gets the input section name from the line, if it exists. + # Input section names are always indented 1 space. + RE_INPUT_SECTION_NAME = re.compile(r'^ (\.\w+\.?\w*\.?\w*)') # Note: This allows up to 3 dots... hopefully that's enough... + ALL_SECTIONS = ( _Parser.SECTIONS + _Parser.OTHER_SECTIONS + _Parser.MISC_FLASH_SECTIONS - + ('unknown', 'OUTPUT') + + ('unknown', ) ) - def check_new_section(self, line): - """ Check whether a new section in a map file has been detected + def check_new_output_section(self, line: str) -> tuple[str, int] | None: + """ Check whether a new output section in a map file has been detected Positional arguments: line - the line to check for a new section - return value - A section name, if a new section was found, None + return value - Tuple of (name, vma to lma offset), if a new section was found, None otherwise + The vma to lma offset is the offset to be added to a memory address to get the + address where it's loaded from. If this is zero, the section is not loaded from flash to RAM at startup. """ - line_s = line.strip() - for i in self.ALL_SECTIONS: - if line_s.startswith(i): - return i - if line.startswith('.'): - return 'unknown' + + match = re.match(self.RE_OUTPUT_SECTION_WITH_LOAD_ADDRESS, line) + if match: + section_name = match.group(1) + memory_address = int(match.group(2), 16) + load_address = int(match.group(4), 16) + load_addr_offset = load_address - memory_address else: + match = re.match(self.RE_OUTPUT_SECTION_NO_LOAD_ADDRESS, line) + if not match: + return None + section_name = match.group(1) + load_addr_offset = 0 + + # Ensure that this is a known section name, remove if not + if section_name not in self.ALL_SECTIONS: + section_name = "unknown" + + # Strangely, GCC still generates load address info for sections that are not loaded, such as .bss. + # For now, suppress this for all sections other than .data. + if section_name != ".data": + load_addr_offset = 0 + + return section_name, load_addr_offset + + def check_input_section(self, line) -> Optional[str]: + """ Check whether a new input section in a map file has been detected. + + Positional arguments: + line - the line to check for a new section + + return value - Input section name if found, None otherwise + """ + match = re.match(self.RE_INPUT_SECTION_NAME, line) + if not match: return None - def parse_object_name(self, line): + return match.group(1) + + def parse_object_name(self, line: str) -> str: """ Parse a path to object file Positional arguments: @@ -177,8 +329,8 @@ def parse_object_name(self, line): % line) return '[misc]' - def parse_section(self, line): - """ Parse data from a section of gcc map file + def parse_section(self, line: str) -> tuple[str, int, int]: + """ Parse data from a section of gcc map file describing one symbol in the code. examples: 0x00004308 0x7c ./BUILD/K64F/GCC_ARM/spi_api.o @@ -186,167 +338,74 @@ def parse_section(self, line): Positional arguments: line - the line to parse a section from + + Returns tuple of (name, start addr, size) """ is_fill = re.match(self.RE_FILL_SECTION, line) if is_fill: o_name = '[fill]' + o_start_addr = int(is_fill.group(1), 16) o_size = int(is_fill.group(2), 16) - return [o_name, o_size] + return o_name, o_start_addr, o_size is_section = re.match(self.RE_STD_SECTION, line) if is_section: + o_start_addr = int(is_section.group(1), 16) o_size = int(is_section.group(2), 16) if o_size: o_name = self.parse_object_name(is_section.group(3)) - return [o_name, o_size] + return o_name, o_start_addr, o_size - return ["", 0] + return "", 0, 0 - def parse_mapfile(self, file_desc): + def parse_mapfile(self, file_desc: TextIO) -> dict[str, dict[str, int]]: """ Main logic to decode gcc map files Positional arguments: file_desc - a stream object to parse as a gcc map file """ - current_section = 'unknown' + + # GCC can put the section/symbol info on its own line or on the same line as the size and address. + # So since this is a line oriented parser, we have to remember the most recently seen input & output + # section name for later. + current_output_section = 'unknown' + current_output_section_addr_offset = 0 + current_input_section = 'unknown' with file_desc as infile: for line in infile: if line.startswith('Linker script and memory map'): - current_section = "unknown" break for line in infile: - next_section = self.check_new_section(line) - - if next_section == "OUTPUT": + if line.startswith("OUTPUT("): + # Done with memory map part of the map file break - elif next_section: - current_section = next_section - - object_name, object_size = self.parse_section(line) - self.module_add(object_name, object_size, current_section) - - common_prefix = dirname(commonprefix([ - o for o in self.modules.keys() - if ( - o.endswith(self.OBJECT_EXTENSIONS) - and not o.startswith("[lib]") - )])) - new_modules = {} - for name, stats in self.modules.items(): - if name.startswith("[lib]"): - new_modules[name] = stats - elif name.endswith(self.OBJECT_EXTENSIONS): - new_modules[relpath(name, common_prefix)] = stats - else: - new_modules[name] = stats - return new_modules + next_section = self.check_new_output_section(line) + if next_section is not None: + current_output_section, current_output_section_addr_offset = next_section -class _ArmccParser(_Parser): - RE = re.compile( - r'^\s+0x(\w{8})\s+0x(\w{8})\s+(\w+)\s+(\w+)\s+(\d+)\s+[*]?.+\s+(.+)$') - RE_OBJECT = re.compile(r'(.+\.(l|a|ar))\((.+\.o(bj)?)\)') - OBJECT_EXTENSIONS = (".o", ".obj") - - def parse_object_name(self, line): - """ Parse object file - - Positional arguments: - line - the line containing the object or library - """ - if line.endswith(self.OBJECT_EXTENSIONS): - return line - - else: - is_obj = re.match(self.RE_OBJECT, line) - if is_obj: - return join( - '[lib]', basename(is_obj.group(1)), is_obj.group(3) - ) - else: - print( - "Malformed input found when parsing ARMCC map: %s" % line - ) - return '[misc]' - - def parse_section(self, line): - """ Parse data from an armcc map file + next_input_section = self.check_input_section(line) + if next_input_section is not None: + current_input_section = next_input_section - Examples of armcc map file: - Base_Addr Size Type Attr Idx E Section Name Object - 0x00000000 0x00000400 Data RO 11222 self.RESET startup_MK64F12.o - 0x00000410 0x00000008 Code RO 49364 * !!!main c_w.l(__main.o) - - Positional arguments: - line - the line to parse the section data from - """ # noqa: E501 - test_re = re.match(self.RE, line) - - if ( - test_re - and "ARM_LIB_HEAP" not in line - ): - size = int(test_re.group(2), 16) - - if test_re.group(4) == 'RO': - section = '.text' - else: - if test_re.group(3) == 'Data': - section = '.data' - elif test_re.group(3) == 'Zero': - section = '.bss' - elif test_re.group(3) == 'Code': - section = '.text' - else: - print( - "Malformed input found when parsing armcc map: %s, %r" - % (line, test_re.groups()) - ) - - return ["", 0, ""] - - # check name of object or library - object_name = self.parse_object_name( - test_re.group(6)) - - return [object_name, size, section] - - else: - return ["", 0, ""] - - def parse_mapfile(self, file_desc): - """ Main logic to decode armc5 map files - - Positional arguments: - file_desc - a file like object to parse as an armc5 map file - """ - with file_desc as infile: - # Search area to parse - for line in infile: - if line.startswith(' Base Addr Size'): - break + symbol_name, symbol_start_addr, symbol_size = self.parse_section(line) - # Start decoding the map file - for line in infile: - self.module_add(*self.parse_section(line)) + # With GCC at least, the closest we can get to a descriptive symbol name is the input section + # name. Thanks to the -ffunction-sections and -fdata-sections options, the section names should + # be unique for each symbol. + self.add_symbol(current_input_section, symbol_name, symbol_start_addr, symbol_size, current_output_section, current_output_section_addr_offset) common_prefix = dirname(commonprefix([ o for o in self.modules.keys() if ( - o.endswith(self.OBJECT_EXTENSIONS) - and o != "anon$$obj.o" - and o != "anon$$obj.obj" - and not o.startswith("[lib]") + o.endswith(self.OBJECT_EXTENSIONS) + and not o.startswith("[lib]") )])) new_modules = {} for name, stats in self.modules.items(): - if ( - name == "anon$$obj.o" - or name == "anon$$obj.obj" - or name.startswith("[lib]") - ): + if name.startswith("[lib]"): new_modules[name] = stats elif name.endswith(self.OBJECT_EXTENSIONS): new_modules[relpath(name, common_prefix)] = stats @@ -355,169 +414,6 @@ def parse_mapfile(self, file_desc): return new_modules -class _IarParser(_Parser): - RE = re.compile( - r'^\s+(.+)\s+(zero|const|ro code|inited|uninit)\s' - r'+0x([\'\w]+)\s+0x(\w+)\s+(.+)\s.+$') - - RE_CMDLINE_FILE = re.compile(r'^#\s+(.+\.o(bj)?)') - RE_LIBRARY = re.compile(r'^(.+\.a)\:.+$') - RE_OBJECT_LIBRARY = re.compile(r'^\s+(.+\.o(bj)?)\s.*') - OBJECT_EXTENSIONS = (".o", ".obj") - - def __init__(self): - _Parser.__init__(self) - # Modules passed to the linker on the command line - # this is a dict because modules are looked up by their basename - self.cmd_modules = {} - - def parse_object_name(self, object_name): - """ Parse object file - - Positional arguments: - line - the line containing the object or library - """ - if object_name.endswith(self.OBJECT_EXTENSIONS): - try: - return self.cmd_modules[object_name] - except KeyError: - return object_name - else: - return '[misc]' - - def parse_section(self, line): - """ Parse data from an IAR map file - - Examples of IAR map file: - Section Kind Address Size Object - .intvec ro code 0x00000000 0x198 startup_MK64F12.o [15] - .rodata const 0x00000198 0x0 zero_init3.o [133] - .iar.init_table const 0x00008384 0x2c - Linker created - - Initializer bytes const 0x00000198 0xb2 - .data inited 0x20000000 0xd4 driverAtmelRFInterface.o [70] - .bss zero 0x20000598 0x318 RTX_Conf_CM.o [4] - .iar.dynexit uninit 0x20001448 0x204 - HEAP uninit 0x20001650 0x10000 - - Positional_arguments: - line - the line to parse section data from - """ # noqa: E501 - test_re = re.match(self.RE, line) - if test_re: - if ( - test_re.group(2) == 'const' or - test_re.group(2) == 'ro code' - ): - section = '.text' - elif (test_re.group(2) == 'zero' or - test_re.group(2) == 'uninit'): - if test_re.group(1)[0:4] == 'HEAP': - section = '.heap' - elif test_re.group(1)[0:6] == 'CSTACK': - section = '.stack' - else: - section = '.bss' # default section - - elif test_re.group(2) == 'inited': - section = '.data' - else: - print("Malformed input found when parsing IAR map: %s" % line) - return ["", 0, ""] - - # lookup object in dictionary and return module name - object_name = self.parse_object_name(test_re.group(5)) - - size = int(test_re.group(4), 16) - return [object_name, size, section] - - else: - return ["", 0, ""] - - def check_new_library(self, line): - """ - Searches for libraries and returns name. Example: - m7M_tls.a: [43] - - """ - test_address_line = re.match(self.RE_LIBRARY, line) - if test_address_line: - return test_address_line.group(1) - else: - return "" - - def check_new_object_lib(self, line): - """ - Searches for objects within a library section and returns name. - Example: - rt7M_tl.a: [44] - ABImemclr4.o 6 - ABImemcpy_unaligned.o 118 - ABImemset48.o 50 - I64DivMod.o 238 - I64DivZer.o 2 - - """ - test_address_line = re.match(self.RE_OBJECT_LIBRARY, line) - if test_address_line: - return test_address_line.group(1) - else: - return "" - - def parse_command_line(self, lines): - """Parse the files passed on the command line to the iar linker - - Positional arguments: - lines -- an iterator over the lines within a file - """ - for line in lines: - if line.startswith("*"): - break - for arg in line.split(" "): - arg = arg.rstrip(" \n") - if ( - not arg.startswith("-") - and arg.endswith(self.OBJECT_EXTENSIONS) - ): - self.cmd_modules[basename(arg)] = arg - - common_prefix = dirname(commonprefix(list(self.cmd_modules.values()))) - self.cmd_modules = {s: relpath(f, common_prefix) - for s, f in self.cmd_modules.items()} - - def parse_mapfile(self, file_desc): - """ Main logic to decode IAR map files - - Positional arguments: - file_desc - a file like object to parse as an IAR map file - """ - with file_desc as infile: - self.parse_command_line(infile) - - for line in infile: - if line.startswith(' Section '): - break - - for line in infile: - self.module_add(*self.parse_section(line)) - - if line.startswith('*** MODULE SUMMARY'): # finish section - break - - current_library = "" - for line in infile: - library = self.check_new_library(line) - - if library: - current_library = library - - object_name = self.check_new_object_lib(line) - - if object_name and current_library: - temp = join('[lib]', current_library, object_name) - self.module_replace(object_name, temp) - return self.modules - - class MemapParser(object): """An object that represents parsed results, parses the memory map files, and writes out different file types of memory results @@ -757,7 +653,7 @@ def generate_json(self, file_desc): "Total Flash memory (text + data): {}({:+}) bytes\n" ) - def generate_csv(self, file_desc): + def generate_csv(self, file_desc: TextIO) -> None: """Generate a CSV file from a memoy map Positional arguments: @@ -830,12 +726,22 @@ def generate_table(self, file_desc): self.mem_summary['total_flash_delta'] ) + output += '\n' + for bank_type, banks in self.memory_banks.items(): + for bank_info in banks: + this_bank_deltas = self.memory_bank_summary[bank_type][bank_info.name] + output += (f"{bank_type} Bank {bank_info.name}: {bank_info.used_size}({this_bank_deltas['delta_bytes_used']:+})/" + f"{bank_info.total_size} bytes used, " + f"{this_bank_deltas['percent_used']:.01f}% ({this_bank_deltas['delta_percent_used']:+.01f}%) used\n") + return output toolchains = ["ARM", "ARM_STD", "ARM_MICRO", "GCC_ARM", "IAR"] def compute_report(self): - """ Generates summary of memory usage for main areas + """ + Generates summary of memory usage for main areas. Result is put into the 'self.mem_report' + dict, which is processed by tests and also dumped as JSON for the JSON output format. """ self.subtotal = defaultdict(int) @@ -857,22 +763,49 @@ def compute_report(self): self.subtotal['.text-delta'] + self.subtotal['.data-delta'], } - self.mem_report = [] + self.mem_report = {} + modules = [] if self.short_modules: for name, sizes in sorted(self.short_modules.items()): - self.mem_report.append({ + modules.append({ "module": name, "size": { k: sizes.get(k, 0) for k in (self.print_sections + self.delta_sections) } }) + self.mem_report["modules"] = modules + + self.mem_report["summary"] = self.mem_summary - self.mem_report.append({ - 'summary': self.mem_summary - }) + # Calculate the delta sizes for each memory bank in a couple different formats + self.memory_bank_summary: dict[str, dict[str, dict[str, float|int]]] = {} + for bank_type, banks in self.memory_banks.items(): + self.memory_bank_summary[bank_type] = {} + for bank_info in banks: - def parse(self, mapfile, toolchain): + this_bank_info = {} + + # Find matching memory bank in old memory banks. Compare by name as it would be possible + # for the indices to change between builds if someone edited the memory bank definition + old_bank_info = None + if self.old_memory_banks is not None and bank_type in self.old_memory_banks: + for curr_old_bank_info in self.old_memory_banks[bank_type]: + if curr_old_bank_info.name == bank_info.name: + old_bank_info = curr_old_bank_info + break + + this_bank_info["bytes_used"] = bank_info.used_size + this_bank_info["total_size"] = bank_info.total_size + this_bank_info["delta_bytes_used"] = 0 if old_bank_info is None else bank_info.used_size - old_bank_info.used_size + this_bank_info["percent_used"] = 100 * bank_info.used_size/bank_info.total_size + this_bank_info["delta_percent_used"] = 100 * this_bank_info["delta_bytes_used"]/bank_info.total_size + + self.memory_bank_summary[bank_type][bank_info.name] = this_bank_info + + self.mem_report["memory_bank_usage"] = self.memory_bank_summary + + def parse(self, mapfile: str, toolchain: str, memory_banks_json_path: str | None) -> bool: """ Parse and decode map file depending on the toolchain Positional arguments: @@ -880,22 +813,28 @@ def parse(self, mapfile, toolchain): toolchain - the toolchain used to create the file """ self.tc_name = toolchain.title() - if toolchain in ("ARM", "ARM_STD", "ARM_MICRO", "ARMC6"): - parser = _ArmccParser - elif toolchain == "GCC_ARM": - parser = _GccParser - elif toolchain == "IAR": - parser = _IarParser + if toolchain == "GCC_ARM": + parser_class = _GccParser else: return False + parser = parser_class() + old_map_parser = parser_class() + + if memory_banks_json_path is not None: + with open(memory_banks_json_path, 'r') as memory_banks_json_file: + parser.load_memory_banks_info(memory_banks_json_file) + try: with open(mapfile, 'r') as file_input: - self.modules = parser().parse_mapfile(file_input) + self.modules = parser.parse_mapfile(file_input) + self.memory_banks = parser.memory_banks try: with open("%s.old" % mapfile, 'r') as old_input: - self.old_modules = parser().parse_mapfile(old_input) + self.old_modules = old_map_parser.parse_mapfile(old_input) + self.old_memory_banks = old_map_parser.memory_banks except IOError: self.old_modules = None + self.old_memory_banks = None return True except IOError as error: @@ -938,6 +877,11 @@ def main(): parser.add_argument('-v', '--version', action='version', version=version) + parser.add_argument( + '-m', '--memory-banks-json', + type=argparse_filestring_type, + help='Path to memory bank JSON file. If passed, memap will track the used space in each memory bank.') + # Parse/run command if len(argv) <= 1: parser.print_help() @@ -950,7 +894,7 @@ def main(): # Parse and decode a map file if args.file and args.toolchain: - if memap.parse(args.file, args.toolchain) is False: + if memap.parse(args.file, args.toolchain, args.memory_banks_json) is False: exit(0) if args.depth is None: diff --git a/tools/python/memap/utils.py b/tools/python/memap/utils.py index d2aae7ff73f..378b15b699c 100644 --- a/tools/python/memap/utils.py +++ b/tools/python/memap/utils.py @@ -33,10 +33,6 @@ from intelhex import IntelHex import io -try: - unicode -except NameError: - unicode = str def remove_if_in(lst, thing): if thing in lst: @@ -427,7 +423,7 @@ def parse_type(string): the string, or the hyphens/underscores do not match the expected style of the argument. """ - if not isinstance(string, unicode): + if not isinstance(string, str): string = string.decode() if prefer_hyphen: newstring = casedness(string).replace("_", "-") @@ -447,10 +443,10 @@ def parse_type(string): return middle # short cuts for the argparse_type versions -argparse_uppercase_type = argparse_type(unicode.upper, False) -argparse_lowercase_type = argparse_type(unicode.lower, False) -argparse_uppercase_hyphen_type = argparse_type(unicode.upper, True) -argparse_lowercase_hyphen_type = argparse_type(unicode.lower, True) +argparse_uppercase_type = argparse_type(str.upper, False) +argparse_lowercase_type = argparse_type(str.lower, False) +argparse_uppercase_hyphen_type = argparse_type(str.upper, True) +argparse_lowercase_hyphen_type = argparse_type(str.lower, True) def argparse_force_type(case): """ validate that an argument passed in (as string) is a member of the list @@ -458,11 +454,11 @@ def argparse_force_type(case): """ def middle(lst, type_name): """ The parser type generator""" - if not isinstance(lst[0], unicode): + if not isinstance(lst[0], str): lst = [o.decode() for o in lst] def parse_type(string): """ The parser type""" - if not isinstance(string, unicode): + if not isinstance(string, str): string = string.decode() for option in lst: if case(string) == case(option): @@ -474,8 +470,8 @@ def parse_type(string): return middle # these two types convert the case of their arguments _before_ validation -argparse_force_uppercase_type = argparse_force_type(unicode.upper) -argparse_force_lowercase_type = argparse_force_type(unicode.lower) +argparse_force_uppercase_type = argparse_force_type(str.upper) +argparse_force_lowercase_type = argparse_force_type(str.lower) def argparse_many(func): """ An argument parser combinator that takes in an argument parser and diff --git a/tools/python/python_tests/memap/arm.map b/tools/python/python_tests/memap/arm.map deleted file mode 100644 index 0ecc1a5bdc0..00000000000 --- a/tools/python/python_tests/memap/arm.map +++ /dev/null @@ -1,47 +0,0 @@ -Component: ARM Compiler 5.06 update 5 (build 528) Tool: armlink [4d35e2] - -============================================================================== - -Memory Map of the image - - Image Entry point : 0x0001b0c1 - - Load Region LR_IROM1 (Base: 0x0001b000, Size: 0x0000ed04, Max: 0x00025000, ABSOLUTE, COMPRESSED[0x0000e23c]) - - Execution Region ER_IROM1 (Base: 0x0001b000, Size: 0x0000e1c4, Max: 0x00025000, ABSOLUTE) - - Base Addr Size Type Attr Idx E Section Name Object - - 0x0001b000 0x000000c0 Data RO 7002 RESET /common/path/startup/startup.o - 0x0001b0c0 0x00000008 Code RO 8820 * !!!main /installed/libs/../lib/armlib/c_p.l(__main.o) - 0x0001b26c 0x00000098 Code RO 6076 .text /common/path/irqs/irqs.o - 0x000206a0 0x00000036 Code RO 27 i._Z9time_funcPN4mbed5TimerEi /common/path/main.o - 0x200039b4 0x00000018 Data RW 8092 .data /common/path/data/data.o - 0x20003af8 0x00000198 Zero RW 57 .bss /common/path/data/data.o - -============================================================================== - -Image component sizes - - - Code (inc. data) RO Data RW Data ZI Data Debug - - 344 368 0 24 408 36188 Object Totals - 8 0 0 0 0 7596 Library Totals - -============================================================================== - - - Code (inc. data) RO Data RW Data ZI Data Debug - - 352 376 0 24 408 17208 Grand Totals - 352 376 0 24 408 17208 ELF Image Totals (compressed) - 352 376 0 24 0 0 ROM Totals - -============================================================================== - - Total RO Size (Code + RO Data) 352 ( 0.35kB) - Total RW Size (RW Data + ZI Data) 432 ( 0.43kB) - Total ROM Size (Code + RO Data + RW Data) 376 ( 0.37kB) - -============================================================================== diff --git a/tools/python/python_tests/memap/iar.map b/tools/python/python_tests/memap/iar.map deleted file mode 100644 index ba3112a12fe..00000000000 --- a/tools/python/python_tests/memap/iar.map +++ /dev/null @@ -1,83 +0,0 @@ -############################################################################### -# -# IAR ELF Linker V7.80.1.28/LNX for ARM 18/Sep/2017 14:26:09 -# Copyright 2007-2016 IAR Systems AB. -# -# Output file = -# /common/path/project.elf -# Map file = -# /common/path/project.map -# Command line = -# -f -# /common/path/.link_files.txt -# (-o /common/path/project.elf -# --map=/common/path/project.map /common/path/main.o -# /common/path/startup/startup.o /common/path/irqs/irqs.o -# /common/path/data/data.o -# -############################################################################### - -******************************************************************************* -*** RUNTIME MODEL ATTRIBUTES -*** - -CppFlavor = * -__CPP_Exceptions = Disabled -__CPP_Language = C++ -__Heap_Handler = DLMalloc -__SystemLibrary = DLib -__dlib_dynamic_initialization = postponed -__dlib_has_iterator_debugging = 0 -__dlib_jmp_buf_num_elements = 8 - - -******************************************************************************* -*** PLACEMENT SUMMARY -*** - -"A0": place at 0x0001b000 { ro section .intvec }; -"P1": place in [from 0x0001b0c0 to 0x0003ffff] { ro }; -"P2": place in [from 0x20002ef8 to 0x20007fff] { rw, block HEAP, block CSTACK }; -do not initialize { section .noinit }; -initialize by copy { rw }; - { section .intvec }; - - Section Kind Address Size Object - ------- ---- ------- ---- ------ -"A0": 0xc0 - .intvec ro code 0x0001b000 0xc0 startup.o [4] - - 0x0001b0c0 0xc0 - -"P1": 0x - .text ro code 0x0001c753 0x36 main.o [3] - .text ro code 0x0001cfff 0x98 irqs.o [5] - .text ro code 0x0001c778 0x8 __main.o [67] - -"P2", part 1 of 2: 0x18 - P2-1 0x20002ef8 0x18 - .data inited 0x20002fa8 0x18 data.o [6] - -"P2", part 2 of 2: 0x198 - P2-2 0x20005388 0x198 - .bss zero 0x20002fa8 0x198 data.o [6] - -******************************************************************************* -*** INIT TABLE -*** - -******************************************************************************* -*** MODULE SUMMARY -*** - -d16M_tlf.a: [67] - __main.o 8 - ------------------------------------------------ - Total: 8 - - Linker created ---------------------------------------------------- - Grand Total: - -******************************************************************************* -*** ENTRY LIST -*** \ No newline at end of file diff --git a/tools/python/python_tests/memap/memap_test.py b/tools/python/python_tests/memap/memap_test.py index 2570d0a4194..37af53d2122 100644 --- a/tools/python/python_tests/memap/memap_test.py +++ b/tools/python/python_tests/memap/memap_test.py @@ -22,7 +22,7 @@ import pytest from memap import memap -from memap.memap import MemapParser +from memap.memap import MemapParser, MemoryBankInfo from copy import deepcopy """ @@ -38,7 +38,7 @@ def memap_parser(): """ memap_parser = MemapParser() - memap_parser.modules = { + memap_parser.symbols = { "mbed-os/targets/TARGET/TARGET_MCUS/api/pinmap.o": { ".text": 1, ".data": 2, @@ -136,6 +136,24 @@ def memap_parser(): "OUTPUT":0, }, } + + memap_parser.memory_banks = { + "RAM": [ + MemoryBankInfo(name="IRAM1", start_addr=0x20000000, total_size=32768, used_size=2000) + ], + "ROM": [ + MemoryBankInfo(name="IROM1", start_addr=0x20000000, total_size=65536, used_size=10000) + ] + } + memap_parser.old_memory_banks = { + "RAM": [ + MemoryBankInfo(name="IRAM1", start_addr=0x20000000, total_size=32768, used_size=2014) + ], + "ROM": [ + MemoryBankInfo(name="IROM1", start_addr=0x20000000, total_size=65536, used_size=9000) + ] + } + return memap_parser @@ -217,3 +235,21 @@ def test_generate_output_csv_ci(memap_parser, tmpdir, depth, sep): file_name = str(tmpdir.join('output.csv').realpath()) generate_test_helper(memap_parser, 'csv-ci', depth, sep, file_name) assert isfile(file_name), "Failed to create csv-ci file" + + +def test_memory_bank_summary(memap_parser: MemapParser): + """ + Test that the memory bank summary has the expected information in it. + """ + + memap_parser.generate_output('table', 1, "/") + + assert memap_parser.memory_bank_summary["RAM"].keys() == {"IRAM1"} + assert memap_parser.memory_bank_summary["ROM"].keys() == {"IROM1"} + + # Check details of the ROM bank more closely + assert memap_parser.memory_bank_summary["ROM"]["IROM1"]["bytes_used"] == 10000 + assert memap_parser.memory_bank_summary["ROM"]["IROM1"]["total_size"] == 65536 + assert memap_parser.memory_bank_summary["ROM"]["IROM1"]["delta_bytes_used"] == 1000 + assert memap_parser.memory_bank_summary["ROM"]["IROM1"]["percent_used"] == pytest.approx(15.3, abs=0.1) + assert memap_parser.memory_bank_summary["ROM"]["IROM1"]["delta_percent_used"] == pytest.approx(1.5, abs=0.1) \ No newline at end of file diff --git a/tools/python/python_tests/memap/parse_test.py b/tools/python/python_tests/memap/parse_test.py index 656f51c4663..71b6debccf1 100644 --- a/tools/python/python_tests/memap/parse_test.py +++ b/tools/python/python_tests/memap/parse_test.py @@ -20,84 +20,55 @@ from io import open from os import sep from os.path import isfile, join, dirname -import json +from collections import defaultdict import pytest -from memap.memap import MemapParser, _ArmccParser +from memap.memap import MemapParser, _GccParser from copy import deepcopy -PARSED_ARM_DATA = { - "startup/startup.o": {".text": 0xc0}, - "[lib]/c_p.l/__main.o": {".text": 8}, - "irqs/irqs.o": {".text": 0x98}, - "data/data.o": {".data": 0x18, ".bss": 0x198}, - "main.o": {".text": 0x36}, -} - -def test_parse_armcc(): - memap = MemapParser() - memap.parse(join(dirname(__file__), "arm.map"), "ARM") - - parsed_data_os_agnostic = dict() - for k in PARSED_ARM_DATA: - parsed_data_os_agnostic[k.replace('/', sep)] = PARSED_ARM_DATA[k] - - assert memap.modules == parsed_data_os_agnostic - -PARSED_IAR_DATA = { - "startup/startup.o": {".text": 0xc0}, - "[lib]/d16M_tlf.a/__main.o": {".text": 8}, - "irqs/irqs.o": {".text": 0x98}, - "data/data.o": {".data": 0x18, ".bss": 0x198}, - "main.o": {".text": 0x36}, -} - -def test_parse_iar(): - memap = MemapParser() - memap.parse(join(dirname(__file__), "iar.map"), "IAR") - - parsed_data_os_agnostic = dict() - for k in PARSED_IAR_DATA: - parsed_data_os_agnostic[k.replace('/', sep)] = PARSED_IAR_DATA[k] - - assert memap.modules == parsed_data_os_agnostic - PARSED_GCC_DATA = { - "startup/startup.o": {".text": 0xc0}, - "[lib]/d16M_tlf.a/__main.o": {".text": 8}, - "[lib]/misc/foo.o": {".text": 8}, - "irqs/irqs.o": {".text": 0x98}, - "data/data.o": {".data": 0x18, ".bss": 0x198}, - "main.o": {".text": 0x36}, + "startup/startup.o": defaultdict(int, {".text": 0xc0}), + "[lib]/d16M_tlf.a/__main.o": defaultdict(int, {".text": 8}), + "[lib]/misc/foo.o": defaultdict(int, {".text": 8}), + "irqs/irqs.o": defaultdict(int, {".text": 0x98}), + "data/data.o":defaultdict(int, {".data": 0x18, ".bss": 0x198}), + "main.o": defaultdict(int, {".text": 0x36}), } def test_parse_gcc(): memap = MemapParser() - memap.parse(join(dirname(__file__), "gcc.map"), "GCC_ARM") + + this_script_dir = dirname(__file__) + memap.parse(join(this_script_dir, "gcc.map"), "GCC_ARM", join(this_script_dir, "test_memory_banks.json")) parsed_data_os_agnostic = dict() for k in PARSED_GCC_DATA: parsed_data_os_agnostic[k.replace('/', sep)] = PARSED_GCC_DATA[k] + # Sum of everything in .text and .data + assert memap.memory_banks["ROM"][0].used_size == 0x1B6 + + # Sum of everything in .bss and .data + assert memap.memory_banks["RAM"][0].used_size == 0x1B0 + assert memap.modules == parsed_data_os_agnostic -def test_add_empty_module(): - memap = _ArmccParser() - old_modules = deepcopy(memap.modules) - memap.module_add("", 8, ".data") - assert(old_modules == memap.modules) - memap.module_add("main.o", 0, ".text") - assert(old_modules == memap.modules) - memap.module_add("main.o", 8, "") - assert(old_modules == memap.modules) +def test_add_symbol_missing_info(): + memap = _GccParser() + old_symbols = deepcopy(memap.modules) + memap.add_symbol(".data.some_func", "", 8, 10, ".data", 1000) + assert(old_symbols == memap.modules) + memap.add_symbol(".data.some_func", "foo.o", 8, 0, ".data", 1000) + assert(old_symbols == memap.modules) + def test_add_full_module(): - memap = _ArmccParser() + memap = _GccParser() old_modules = deepcopy(memap.modules) - memap.module_add("main.o", 8, ".data") + memap.add_symbol(".data.foo", "main.o", 5, 8, ".data", 1000) assert(old_modules != memap.modules) assert("main.o" in memap.modules) assert(".data" in memap.modules["main.o"]) diff --git a/tools/python/python_tests/memap/test_memory_banks.json b/tools/python/python_tests/memap/test_memory_banks.json new file mode 100644 index 00000000000..88cc44837fc --- /dev/null +++ b/tools/python/python_tests/memap/test_memory_banks.json @@ -0,0 +1,38 @@ +{ + "configured_memory_banks": { + "ROM": { + "IROM1": { + "access": { + "execute": true, + "non_secure": false, + "non_secure_callable": false, + "peripheral": false, + "read": true, + "secure": false, + "write": false + }, + "default": true, + "size": 524288, + "start": 0, + "startup": true + } + }, + "RAM": { + "IRAM1": { + "access": { + "execute": false, + "non_secure": false, + "non_secure_callable": false, + "peripheral": false, + "read": true, + "secure": false, + "write": true + }, + "default": true, + "size": 32768, + "start": 536870912, + "startup": false + } + } + } +} \ No newline at end of file