Supply memory bank information from the configuration script

tools/python/mbed_tools/build/_internal/memory_banks.py

@@ -3,7 +3,12 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-from typing import Dict, Any, Set
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from typing import Dict, Any, Set, TypedDict, NotRequired
+
 import pathlib
 import copy
 import json
@@ -18,6 +23,27 @@
 logger = logging.getLogger(__name__)
 
 
+if TYPE_CHECKING:
+    # Type hints for memory bank config
+    class MemoryBankInfo(TypedDict):
+        """
+        Info about one memory bank
+        """
+        size: int
+        start: int
+        default: NotRequired[bool]
+        startup: NotRequired[bool]
+        access: Dict[str, bool]
+
+
+    class BanksByType(TypedDict):
+        """
+        Info about all memory banks, ROM and RAM
+        """
+        ROM: Dict[str, MemoryBankInfo]
+        RAM: Dict[str, MemoryBankInfo]
+
+
 # Deprecated memory configuration properties from old (Mbed CLI 1) configuration system
 DEPRECATED_MEM_CONFIG_PROPERTIES = {
     "mbed_rom_start",
@@ -65,58 +91,21 @@ def incorporate_memory_bank_data_from_cmsis(target_attributes: Dict[str, Any],
     target_attributes["memory_banks"] = target_memory_banks_section
 
 
-def _pretty_print_size(size: int):
-    """
-    Pretty-print a memory size as MiB/KiB/B
-    """
-    if size >= 1024*1024 and (size % (1024*1024)) == 0:
-        return f"{size//(1024*1024)} MiB"
-    elif size >= 1024 and (size % 1024) == 0:
-        return f"{size//1024} kiB"
-    else:
-        return f"{size} B"
-
+def _apply_configured_overrides(banks_by_type: BanksByType, bank_config: Dict[str, Dict[str, int]]) -> BanksByType:
 
-def process_memory_banks(config: Config, mem_banks_json_file: pathlib.Path) -> None:
     """
-    Process memory bank information in the config.  Reads the 'memory_banks' and
-    'memory_bank_config' sections and adds the memory_bank_macros section accordingly.
-
-    :param config: Config structure containing merged data from every JSON file (app, lib, and targets)
-    :param mem_banks_json_file: Memory banks JSON file is written here
+    Apply overrides from configuration to the physical memory bank information, producing the configured
+    memory bank information.
+    :param bank_config: memory_bank_config element from target JSON
+    :param banks_by_type: Physical memory bank information
     """
 
-    memory_banks = config.get("memory_banks", {})
-
-    # Check for deprecated properties
-    for property_name in DEPRECATED_MEM_CONFIG_PROPERTIES:
-        if property_name in config:
-            logger.warning(f"Configuration uses old-style memory bank configuration property '{property_name}'.  "
-                           f"This is deprecated and is not processed anymore, replace it with a "
-                           f"'memory_bank_config' section.  See here for more: "
-                           f"https://github.com/mbed-ce/mbed-os/wiki/Mbed-Memory-Bank-Information")
-
-    # Check attributes, sort into rom and ram
-    banks_by_type: Dict[str, Dict[str, Dict[str, Any]]] = {"ROM": {}, "RAM": {}}
-    for bank_name, bank_data in memory_banks.items():
-        if "access" not in bank_data or "start" not in bank_data or "size" not in bank_data:
-            raise MbedBuildError(f"Memory bank '{bank_name}' must contain 'access', 'size', and 'start' elements")
-        if not isinstance(bank_data["size"], int) or not isinstance(bank_data["start"], int):
-            raise MbedBuildError(f"Memory bank '{bank_name}': start and size must be integers")
-
-        if bank_data["access"]["read"] and bank_data["access"]["write"]:
-            banks_by_type["RAM"][bank_name] = bank_data
-        elif bank_data["access"]["read"] and bank_data["access"]["execute"]:
-            banks_by_type["ROM"][bank_name] = bank_data
-
-    # Create configured memory bank structure
-    memory_bank_config = config.get("memory_bank_config", {})
     configured_memory_banks = copy.deepcopy(banks_by_type)
 
-    for bank_name, bank_data in memory_bank_config.items():
+    for bank_name, bank_data in bank_config.items():
 
         if bank_name not in configured_memory_banks["RAM"] and bank_name not in configured_memory_banks["ROM"]:
-            raise MbedBuildError(f"Attempt to configure memory bank {bank_name} which does not exist for this device.""")
+            raise MbedBuildError(f"Attempt to configure memory bank {bank_name} which does not exist for this device.")
         bank_type = "RAM" if bank_name in configured_memory_banks["RAM"] else "ROM"
 
         if len(set(bank_data.keys()) - {"size", "start"}):
@@ -129,7 +118,29 @@ def process_memory_banks(config: Config, mem_banks_json_file: pathlib.Path) -> N
 
             configured_memory_banks[bank_type][bank_name][property_name] = property_value
 
-    # Print summary
+    return configured_memory_banks
+
+
+def _pretty_print_size(size: int) -> str:
+    """
+    Pretty-print a memory size as MiB/KiB/B
+    """
+    if size >= 1024*1024 and (size % (1024*1024)) == 0:
+        return f"{size//(1024*1024)} MiB"
+    elif size >= 1024 and (size % 1024) == 0:
+        return f"{size//1024} kiB"
+    else:
+        return f"{size} B"
+
+
+def _print_mem_bank_summary(banks_by_type: BanksByType, configured_banks_by_type: BanksByType) -> None:
+
+    """
+    Print a summary of the memory banks to the console
+    :param banks_by_type: Physical memory bank information
+    :param configured_banks_by_type: Configured memory bank information
+    """
+
     print("Summary of available memory banks:")
     for bank_type, banks in banks_by_type.items():
 
@@ -141,14 +152,13 @@ def process_memory_banks(config: Config, mem_banks_json_file: pathlib.Path) -> N
 
         print(f"Target {bank_type} banks: -----------------------------------------------------------")
 
-        bank_index = 0
-        for bank_name, bank_data in banks.items():
+        for bank_index, (bank_name, bank_data) in enumerate(banks.items()):
 
             bank_size = bank_data["size"]
             bank_start = bank_data["start"]
 
-            configured_size = configured_memory_banks[bank_type][bank_name]["size"]
-            configured_start_addr = configured_memory_banks[bank_type][bank_name]["start"]
+            configured_size = configured_banks_by_type[bank_type][bank_name]["size"]
+            configured_start_addr = configured_banks_by_type[bank_type][bank_name]["start"]
 
             # If the configured sizes are different, add info to the summary
             configured_size_str = ""
@@ -158,23 +168,30 @@ def process_memory_banks(config: Config, mem_banks_json_file: pathlib.Path) -> N
             if configured_start_addr != bank_start:
                 configured_start_addr_str = f" (configured to 0x{configured_start_addr:08x})"
 
-            print(f"{bank_index}. {bank_name}, start addr 0x{bank_start:08x}{configured_start_addr_str}, size {_pretty_print_size(bank_size)}{configured_size_str}")
+            print(f"{bank_index}. {bank_name}, "
+                  f"start addr 0x{bank_start:08x}{configured_start_addr_str}, "
+                  f"size {_pretty_print_size(bank_size)}{configured_size_str}")
+
+        print()
 
-            bank_index += 1
 
-        print("")
+def _generate_macros_for_memory_banks(banks_by_type: BanksByType,
+                            configured_banks_by_type: BanksByType) -> Set[str]:
 
-    # Define macros
+    """
+    Generate a set of macros to define to pass the memory bank information into Mbed.
+    :param banks_by_type: Physical memory bank information
+    :param configured_banks_by_type: Configured memory bank information
+    """
     all_macros: Set[str] = set()
 
     for bank_type, banks in banks_by_type.items():
 
-        bank_index = 0
-        for bank_name, bank_data in banks.items():
+        for bank_index, (bank_name, bank_data) in enumerate(banks.items()):
 
             bank_number_str = "" if bank_index == 0 else str(bank_index)
 
-            configured_bank_data = configured_memory_banks[bank_type][bank_name]
+            configured_bank_data = configured_banks_by_type[bank_type][bank_name]
 
             # Legacy numbered definitions
             all_macros.add(f"MBED_{bank_type}{bank_number_str}_START=0x{bank_data['start']:x}")
@@ -190,15 +207,54 @@ def process_memory_banks(config: Config, mem_banks_json_file: pathlib.Path) -> N
             all_macros.add(f"MBED_CONFIGURED_{bank_type}_BANK_{bank_name}_START=0x{configured_bank_data['start']:x}")
             all_macros.add(f"MBED_CONFIGURED_{bank_type}_BANK_{bank_name}_SIZE=0x{configured_bank_data['size']:x}")
 
-            bank_index += 1
+    return all_macros
+
+
+def process_memory_banks(config: Config) -> Dict[str, BanksByType]:
+    """
+    Process memory bank information in the config.  Reads the 'memory_banks' and
+    'memory_bank_config' sections and adds the memory_bank_macros section accordingly.
+
+    :param config: Config structure containing merged data from every JSON file (app, lib, and targets)
+    :return: Memory bank information structure that shall be written to memory_banks.json
+    """
+
+    memory_banks = config.get("memory_banks", {})
+
+    # Check for deprecated properties
+    for property_name in DEPRECATED_MEM_CONFIG_PROPERTIES:
+        if property_name in config:
+            logger.warning("Configuration uses old-style memory bank configuration property %s. "
+                           "This is deprecated and is not processed anymore, replace it with a "
+                           "'memory_bank_config' section.  See here for more: "
+                           "https://github.com/mbed-ce/mbed-os/wiki/Mbed-Memory-Bank-Information", property_name)
 
-    # Save macros into configuration
-    config["memory_bank_macros"] = all_macros
+    # Check attributes, sort into rom and ram
+    banks_by_type: BanksByType = {"ROM": {}, "RAM": {}}
+    for bank_name, bank_data in memory_banks.items():
+        if "access" not in bank_data or "start" not in bank_data or "size" not in bank_data:
+            raise MbedBuildError(f"Memory bank '{bank_name}' must contain 'access', 'size', and 'start' elements")
+        if not isinstance(bank_data["size"], int) or not isinstance(bank_data["start"], int):
+            raise MbedBuildError(f"Memory bank '{bank_name}': start and size must be integers")
+
+        if bank_data["access"]["read"] and bank_data["access"]["write"]:
+            banks_by_type["RAM"][bank_name] = bank_data
+        elif bank_data["access"]["read"] and bank_data["access"]["execute"]:
+            banks_by_type["ROM"][bank_name] = bank_data
+
+    # Create configured memory bank structure
+    memory_bank_config = config.get("memory_bank_config", {})
+    configured_banks_by_type = _apply_configured_overrides(banks_by_type, memory_bank_config)
+
+    # Print summary
+    _print_mem_bank_summary(banks_by_type, configured_banks_by_type)
+
+    # Generate define macros
+    config["memory_bank_macros"] = _generate_macros_for_memory_banks(banks_by_type, configured_banks_by_type)
 
     # Write out JSON file
-    memory_banks_json_content = {
+    return {
         "memory_banks": banks_by_type,
-        "configured_memory_banks": configured_memory_banks
+        "configured_memory_banks": configured_banks_by_type
     }
-    mem_banks_json_file.parent.mkdir(parents=True, exist_ok=True)
-    mem_banks_json_file.write_text(json.dumps(memory_banks_json_content, indent=4))
+

tools/python/mbed_tools/build/config.py

@@ -6,6 +6,7 @@
 import pathlib
 
 from typing import Any, Tuple
+import json
 
 from mbed_tools.lib.json_helpers import decode_json_file
 from mbed_tools.project import MbedProgram
@@ -39,7 +40,12 @@ def generate_config(target_name: str, toolchain: str, program: MbedProgram) -> T
     config = assemble_config(
         target_build_attributes, [program.root, program.mbed_os.root], program.files.app_config_file
     )
-    process_memory_banks(config, program.files.cmake_build_dir / MEMORY_BANKS_JSON_FILE)
+
+    # Process memory banks and save JSON data for other tools (e.g. memap) to use
+    memory_banks_json_content = process_memory_banks(config)
+    program.files.cmake_build_dir.mkdir(parents=True, exist_ok=True)
+    (program.files.cmake_build_dir / MEMORY_BANKS_JSON_FILE).write_text(json.dumps(memory_banks_json_content, indent=4))
+
     cmake_file_contents = render_mbed_config_cmake_template(
         target_name=target_name, config=config, toolchain_name=toolchain,
     )

tools/python/mbed_tools/lib/json_helpers.py

@@ -25,7 +25,7 @@ def decode_json_file(path: Path) -> Any:
     elif path.suffix == '.json5':
         try:
             logger.debug(f"Loading JSON file {path}")
-            with open(path, "r") as json_file:
+            with path.open() as json_file:
                 return pyjson5.decode_io(json_file)
         except ValueError:
             logger.error(f"Failed to decode JSON data in the file located at '{path}'")

tools/python/python_tests/mbed_tools/build/_internal/test_memory_banks.py

@@ -3,14 +3,11 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-import pytest
-import pathlib
-
 from mbed_tools.build._internal.memory_banks import process_memory_banks
 
 
 class TestMemoryBankProcessing:
-    def test_simple_memory_layout(self, tmp_path: pathlib.Path):
+    def test_simple_memory_layout(self):
         """
         Test a simple memory layout to ensure we process it correctly.
         """
@@ -53,8 +50,7 @@ def test_simple_memory_layout(self, tmp_path: pathlib.Path):
             }
         }
 
-        memory_banks_json_path = tmp_path / "memory_banks.json"
-        process_memory_banks(config, memory_banks_json_path)
+        print(repr(process_memory_banks(config)))
 
         assert config["memory_bank_macros"] == {
             # New style definitions (ROM)
@@ -82,7 +78,7 @@ def test_simple_memory_layout(self, tmp_path: pathlib.Path):
             'MBED_CONFIGURED_RAM_SIZE=0x100000',
             }
 
-    def test_memory_configuration(self, tmp_path: pathlib.Path):
+    def test_memory_configuration(self):
         """
         Test configuring the size and address of a memory bank
         """
@@ -136,8 +132,7 @@ def test_memory_configuration(self, tmp_path: pathlib.Path):
             }
         }
 
-        memory_banks_json_path = tmp_path / "memory_banks.json"
-        process_memory_banks(config, memory_banks_json_path)
+        process_memory_banks(config)
 
         assert config["memory_bank_macros"] == {
             # New style definitions (ROM)
@@ -165,7 +160,7 @@ def test_memory_configuration(self, tmp_path: pathlib.Path):
             'MBED_CONFIGURED_RAM_SIZE=0xa0000',
             }
 
-    def test_two_ram_banks(self, tmp_path: pathlib.Path):
+    def test_two_ram_banks(self):
         """
         Test to see if two RAM banks are handled correctly.
         """
@@ -222,8 +217,7 @@ def test_two_ram_banks(self, tmp_path: pathlib.Path):
             }
         }
 
-        memory_banks_json_path = tmp_path / "memory_banks.json"
-        process_memory_banks(config, memory_banks_json_path)
+        process_memory_banks(config)
 
         # Note: IRAM2 should become MBED_RAM1 because it is listed second
         # in the dictionary