diff --git a/README.md b/README.md index 45074332..85680d27 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,12 @@ Command line building requires: > [!TIP] > If you have set up Simplicity Studio on macOS, everything will be automatically -> detected with the exception of `slc-cli`. This is the only tool you need to download. +> detected with the exception of `slc`. This is the only tool you need to download. + +> [!WARNING] +> M1 users should set `JAVA_HOME=$(/usr/libexec/java_home -a x86_64)` when running the +> build command to make sure the correct Java version is picked by slc-cli. It currently +> is not compatible with ARM Java. `slc-cli` maintains its own SDK and extension trust store so you first must trust all SDK extensions for every SDK you plan to use: diff --git a/manifests/skyconnect_bootloader-uart-xmodem.yaml b/manifests/nabucasa/skyconnect_bootloader-uart-xmodem.yaml similarity index 100% rename from manifests/skyconnect_bootloader-uart-xmodem.yaml rename to manifests/nabucasa/skyconnect_bootloader-uart-xmodem.yaml diff --git a/manifests/skyconnect_firmware-eraser.yaml b/manifests/nabucasa/skyconnect_firmware-eraser.yaml similarity index 100% rename from manifests/skyconnect_firmware-eraser.yaml rename to manifests/nabucasa/skyconnect_firmware-eraser.yaml diff --git a/manifests/skyconnect_ncp-uart-hw.yaml b/manifests/nabucasa/skyconnect_ncp-uart-hw.yaml similarity index 100% rename from manifests/skyconnect_ncp-uart-hw.yaml rename to manifests/nabucasa/skyconnect_ncp-uart-hw.yaml diff --git a/manifests/skyconnect_ot-rcp.yaml b/manifests/nabucasa/skyconnect_ot-rcp.yaml similarity index 100% rename from manifests/skyconnect_ot-rcp.yaml rename to manifests/nabucasa/skyconnect_ot-rcp.yaml diff --git a/manifests/skyconnect_rcp-uart-802154.yaml b/manifests/nabucasa/skyconnect_rcp-uart-802154.yaml similarity index 100% rename from manifests/skyconnect_rcp-uart-802154.yaml rename to manifests/nabucasa/skyconnect_rcp-uart-802154.yaml diff --git a/manifests/slusb001a-lr2_zwave_ncp_serial_api_controller.yaml b/manifests/nabucasa/slusb001a-lr2_zwave_ncp_serial_api_controller.yaml similarity index 100% rename from manifests/slusb001a-lr2_zwave_ncp_serial_api_controller.yaml rename to manifests/nabucasa/slusb001a-lr2_zwave_ncp_serial_api_controller.yaml diff --git a/manifests/slusb001a-lr_zwave_ncp_serial_api_controller.yaml b/manifests/nabucasa/slusb001a-lr_zwave_ncp_serial_api_controller.yaml similarity index 100% rename from manifests/slusb001a-lr_zwave_ncp_serial_api_controller.yaml rename to manifests/nabucasa/slusb001a-lr_zwave_ncp_serial_api_controller.yaml diff --git a/manifests/slusb001a_zwave_ncp_serial_api_controller.yaml b/manifests/nabucasa/slusb001a_zwave_ncp_serial_api_controller.yaml similarity index 100% rename from manifests/slusb001a_zwave_ncp_serial_api_controller.yaml rename to manifests/nabucasa/slusb001a_zwave_ncp_serial_api_controller.yaml diff --git a/manifests/yellow_bootloader-uart-xmodem.yaml b/manifests/nabucasa/yellow_bootloader-uart-xmodem.yaml similarity index 100% rename from manifests/yellow_bootloader-uart-xmodem.yaml rename to manifests/nabucasa/yellow_bootloader-uart-xmodem.yaml diff --git a/manifests/yellow_ncp-uart-hw.yaml b/manifests/nabucasa/yellow_ncp-uart-hw.yaml similarity index 100% rename from manifests/yellow_ncp-uart-hw.yaml rename to manifests/nabucasa/yellow_ncp-uart-hw.yaml diff --git a/manifests/yellow_ot-rcp.yaml b/manifests/nabucasa/yellow_ot-rcp.yaml similarity index 100% rename from manifests/yellow_ot-rcp.yaml rename to manifests/nabucasa/yellow_ot-rcp.yaml diff --git a/manifests/yellow_rcp-uart-802154.yaml b/manifests/nabucasa/yellow_rcp-uart-802154.yaml similarity index 100% rename from manifests/yellow_rcp-uart-802154.yaml rename to manifests/nabucasa/yellow_rcp-uart-802154.yaml diff --git a/tools/build_project.py b/tools/build_project.py index ab489bdc..5e72582d 100755 --- a/tools/build_project.py +++ b/tools/build_project.py @@ -3,12 +3,12 @@ from __future__ import annotations -import os import re import ast import sys import copy import json +import time import shutil import typing import hashlib @@ -24,30 +24,22 @@ # Matches all known chip and board names. Some varied examples: # MGM210PB32JIA EFM32GG890F512 MGM12P22F1024GA EZR32WG330F128R69 EFR32FG14P231F256GM32 -CHIP_SPECIFIC_REGEX = re.compile( - r""" - ^ - (?: - # Chips - (?:[a-z]+\d+){2,5}[a-z]* - | - # Boards and board-specific config - brd\d+[a-z](?:_.+)? - ) - $ -""", - flags=re.IGNORECASE | re.VERBOSE, -) - +CHIP_REGEX = re.compile(r"^(?:[a-z]+\d+){2,5}[a-z]*$", flags=re.IGNORECASE) +BOARD_REGEX = re.compile(r"^brd\d+[a-z](?:_.+)?$", flags=re.IGNORECASE) GRAPH_COMPONENT_REGEX = re.compile(r"^(\w+)|- (\w+)", flags=re.MULTILINE) + CHIP_SPECIFIC_COMPONENTS = { "sl_system": "sl_system", "sl_memory": "sl_memory", # There are only a few components whose filename doesn't match the component name "freertos": "freertos_kernel", } + +SLC = ["slc", "--daemon", "--daemon-timeout", "1"] + LOGGER = logging.getLogger(__name__) + yaml = YAML(typ="safe") @@ -149,11 +141,18 @@ def git(*args: str) -> str: def determine_chip_specific_config_filenames( - slcp_path: pathlib.Path, sdk: pathlib.Path, slc: str + slcp_path: pathlib.Path, sdk: pathlib.Path ) -> list[str]: """Determine the chip-specific config files to remove.""" proc = subprocess.run( - [slc, "graph", "-p", str(slcp_path.absolute()), "--sdk", str(sdk.absolute())], + SLC + + [ + "graph", + "--project-file", + str(slcp_path.absolute()), + "--sdk", + str(sdk.absolute()), + ], cwd=str(slcp_path.parent), check=True, text=True, @@ -169,7 +168,7 @@ def determine_chip_specific_config_filenames( slcc_paths = {slcc.stem.lower(): slcc for slcc in sdk.glob("**/*.slcc")} for component_id in all_components: - if CHIP_SPECIFIC_REGEX.match(component_id): + if CHIP_REGEX.match(component_id) or BOARD_REGEX.match(component_id): pass elif component_id in CHIP_SPECIFIC_COMPONENTS: component_id = CHIP_SPECIFIC_COMPONENTS[component_id] @@ -220,13 +219,13 @@ def main(): parser.add_argument( "--build-dir", type=pathlib.Path, - required=True, - help="Temporary build directory", + default=None, + help="Temporary build directory, generated based on the manifest by default", ) parser.add_argument( "--build-system", choices=["cmake", "makefile"], - default="cmake", + default="makefile", help="Build system", ) parser.add_argument( @@ -262,9 +261,25 @@ def main(): default=[], help="Override config key with JSON.", ) + parser.add_argument( + "--keep-slc-daemon", + action="store_true", + dest="keep_slc_daemon", + default=False, + help="Do not shut down the SLC daemon after the build", + ) args = parser.parse_args() + if args.build_system != "makefile": + LOGGER.warning("Only the `makefile` build system is currently supported") + args.build_system = "makefile" + + if args.build_dir is None: + args.build_dir = pathlib.Path(f"build/{time.time():.0f}_{args.manifest.stem}") + + LOGGER.info("Building in %s", args.build_dir.resolve()) + # argparse defaults should be replaced, not extended if args.sdks != get_sdk_default_paths(): args.sdks = args.sdks[len(get_sdk_default_paths()) :] @@ -278,7 +293,7 @@ def main(): manifest[key] = override # First, load the base project - projects_root = args.manifest.parent.parent + projects_root = pathlib.Path(__file__).parent.parent base_project_path = projects_root / manifest["base_project"] assert base_project_path.is_relative_to(projects_root) (base_project_slcp,) = base_project_path.glob("*.slcp") @@ -289,7 +304,9 @@ def main(): # Strip chip- and board-specific components to modify the base device type output_project["component"] = [ - c for c in output_project["component"] if not CHIP_SPECIFIC_REGEX.match(c["id"]) + c + for c in output_project["component"] + if not CHIP_REGEX.match(c["id"]) and not BOARD_REGEX.match(c["id"]) ] output_project["component"].append({"id": manifest["device"]}) @@ -335,7 +352,7 @@ def main(): with contextlib.suppress(OSError): shutil.rmtree(args.build_dir) - args.build_dir.mkdir(exist_ok=True) + args.build_dir.mkdir(exist_ok=True, parents=True) shutil.copytree( base_project_path, @@ -362,50 +379,42 @@ def main(): yaml.dump(output_project, f) # Create a GBL metadata file - with pathlib.Path(args.build_dir / "gbl_metadata.yaml").open("w") as f: + with (args.build_dir / "gbl_metadata.yaml").open("w") as f: yaml.dump(manifest["gbl"], f) # Generate a build directory - args.build_dir = args.build_dir cmake_build_root = args.build_dir / f"{base_project_name}_cmake" shutil.rmtree(cmake_build_root, ignore_errors=True) - # On macOS `slc` doesn't execute properly - slc = shutil.which("slc-cli") or shutil.which("slc") - - if not slc: - print("`slc` and/or `slc-cli` not found in PATH") - sys.exit(1) - # Find the SDK version required by the project for sdk in args.sdks: try: sdk_meta = yaml.load((sdk / "gecko_sdk.slcs").read_text()) except FileNotFoundError: - print(f"SDK {sdk} is not valid, skipping") + LOGGER.warning("SDK %s is not valid, skipping", sdk) continue assert base_project["sdk"]["id"] == "gecko_sdk" - print(f"SDK {sdk} has version {sdk_meta['sdk_version']}") + LOGGER.info("SDK %s has version %s", sdk, sdk_meta["sdk_version"]) if base_project["sdk"]["version"] == sdk_meta["sdk_version"]: - print(f"Version is correct, picking {sdk}") + LOGGER.info("Version is correct, picking %s", sdk) break else: - print(f"Project SDK version {base_project['sdk']['version']} not found") + LOGGER.error("Project SDK version %s not found", base_project["sdk"]["version"]) sys.exit(1) - print("Building component graph and identifying board-specific files") + LOGGER.info("Building component graph and identifying board-specific files") for name in determine_chip_specific_config_filenames( - slcp_path=base_project_slcp, sdk=sdk, slc=slc + slcp_path=base_project_slcp, sdk=sdk ): try: (args.build_dir / f"config/{name}").unlink() except FileNotFoundError: pass else: - print(f"Deleted device-specific config: {name}") + LOGGER.info("Deleted device-specific config: %s", name) # Find the toolchain required by the project slps_path = (args.build_dir / base_project_name).with_suffix(".slps") @@ -433,13 +442,13 @@ def main(): toolchain_id = version_info["basever"] + "." + version_info["datestamp"] - print(f"Toolchain {toolchain} has version {toolchain_id}") + LOGGER.info("Toolchain %s has version %s", toolchain, toolchain_id) if toolchain_id == slps_toolchain_id: - print(f"Version is correct, picking {toolchain}") + LOGGER.info("Version is correct, picking %s", toolchain) break else: - print(f"Project toolchain version {slps_toolchain_id} not found") + LOGGER.error("Project toolchain version %s not found", slps_toolchain_id) sys.exit(1) # Make sure all extensions are valid @@ -447,12 +456,12 @@ def main(): expected_dir = sdk / f"extension/{sdk_extension['id']}_extension" if not expected_dir.is_dir(): - print(f"Referenced extension not present in SDK: {expected_dir}") + LOGGER.error("Referenced extension not present in SDK: %s", expected_dir) sys.exit(1) subprocess.run( - [ - slc, + SLC + + [ "generate", "--project-file", (args.build_dir / f"{base_project_name}.slcp").resolve(), @@ -517,7 +526,7 @@ def main(): written_config[define] = value if define not in unused_defines: - print(f"Define {define!r} used twice!") + LOGGER.error("Define %r used twice!", define) sys.exit(1) unused_defines.remove(define) @@ -526,11 +535,11 @@ def main(): new_config_h_lines.append(line) if written_config: - print(f"Patching {config_f} with {written_config}") + LOGGER.info("Patching %s with %s", config_f, written_config) config_f.write_text("\n".join(new_config_h_lines)) if unused_defines: - print(f"Defines were unused, aborting: {unused_defines}") + LOGGER.error("Defines were unused, aborting: %s", unused_defines) sys.exit(1) # Remove absolute paths from the build for reproducibility @@ -543,119 +552,48 @@ def main(): }.items() ] - if args.build_system == "makefile": - output_artifact = ( - args.build_dir / "build/debug" / base_project_name - ).with_suffix(".gbl") - - makefile = args.build_dir / f"{base_project_name}.Makefile" - makefile_contents = makefile.read_text() - - # Inject a postbuild step into the makefile - makefile_contents += "\n" - makefile_contents += "post-build:\n" - makefile_contents += ( - f"\t-{args.postbuild}" - f' postbuild "{(args.build_dir / base_project_name).resolve()}.slpb"' - f' --parameter build_dir:"{output_artifact.parent.resolve()}"' - f' --parameter sdk_dir:"{sdk}"' - "\n" - ) - makefile_contents += "\t-@echo ' '" - - for flag in ("C_FLAGS", "CXX_FLAGS"): - line = f"{flag:<17} = \n" - suffix = " ".join([f'"{m}"' for m in extra_compiler_flags]) + "\n" - makefile_contents = makefile_contents.replace( - line, f"{line.rstrip()} {suffix}\n" - ) + output_artifact = (args.build_dir / "build/debug" / base_project_name).with_suffix( + ".gbl" + ) - makefile.write_text(makefile_contents) - - subprocess.run( - [ - "make", - "-C", - args.build_dir, - "-f", - f"{base_project_name}.Makefile", - f"-j{multiprocessing.cpu_count()}", - f"ARM_GCC_DIR={toolchain}", - f"POST_BUILD_EXE={args.postbuild}", - ], - check=True, - ) - elif args.build_system == "cmake": - cmake_build_root = args.build_dir / f"{base_project_name}_cmake" - cmake_config = cmake_build_root / f"{base_project_name}.cmake" - fixed_cmake = [] - - fixed_cmake.append("add_compile_options(") - - for flag in extra_compiler_flags: - assert '"' not in flag # TODO: fix this - fixed_cmake.append(f' "{flag}"') - - fixed_cmake.append(")") - - # Fix CMake config quoting bug - for line in cmake_config.read_text().split("\n"): - if ">:SHELL:" not in line and (":-imacros " in line or ":-x " in line): - line = ' "' + line.replace(">:", ">:SHELL:").strip() + '"' - - fixed_cmake.append(line) - - cmake_config.write_text("\n".join(fixed_cmake)) - - # Insert our postbuild step - cmakelists_txt = cmake_build_root / "CMakeLists.txt" - cmakelists = cmakelists_txt.read_text() - s37_line = next(line for line in cmakelists.split("\n") if "-O srec" in line) - s37_output_file = s37_line.split(" ")[-1] - s37_build_folder = s37_output_file.split("/", 1)[0] + '"' - - cmakelists_txt.write_text( - cmakelists.replace( - s37_line, - ( - f" COMMAND {args.postbuild} postbuild" - f' "{(args.build_dir / base_project_name).resolve()}.slpb"' - f" --parameter build_dir:{s37_build_folder}" - f' --parameter sdk_dir:"{sdk}"\n' - ) - + s37_line, - ) - ) + makefile = args.build_dir / f"{base_project_name}.Makefile" + makefile_contents = makefile.read_text() + + # Inject a postbuild step into the makefile + makefile_contents += "\n" + makefile_contents += "post-build:\n" + makefile_contents += ( + f"\t-{args.postbuild}" + f' postbuild "{(args.build_dir / base_project_name).resolve()}.slpb"' + f' --parameter build_dir:"{output_artifact.parent.resolve()}"' + f' --parameter sdk_dir:"{sdk}"' + "\n" + ) + makefile_contents += "\t-@echo ' '" - # Generate the build system - subprocess.run( - [ - "cmake", - "-G", - "Ninja", - "-DCMAKE_TOOLCHAIN_FILE=toolchain.cmake", - ".", - ], - cwd=cmake_build_root, - env={ - **os.environ, - "ARM_GCC_DIR": toolchain, - "POST_BUILD_EXE": args.postbuild, - }, - check=True, + for flag in ("C_FLAGS", "CXX_FLAGS"): + line = f"{flag:<17} = \n" + suffix = " ".join([f'"{m}"' for m in extra_compiler_flags]) + "\n" + makefile_contents = makefile_contents.replace( + line, f"{line.rstrip()} {suffix}\n" ) - # Build it! - subprocess.run( - [ - "ninja", - "-C", - cmake_build_root, - ], - check=True, - ) + makefile.write_text(makefile_contents) - output_artifact = (cmake_build_root / base_project_name).with_suffix(".gbl") + subprocess.run( + [ + "make", + "-C", + args.build_dir, + "-f", + f"{base_project_name}.Makefile", + f"-j{multiprocessing.cpu_count()}", + f"ARM_GCC_DIR={toolchain}", + f"POST_BUILD_EXE={args.postbuild}", + "VERBOSE=1", + ], + check=True, + ) # Read the metadata extracted from the source and build trees extracted_gbl_metadata = json.loads( @@ -682,6 +620,10 @@ def main(): with contextlib.suppress(OSError): shutil.rmtree(args.build_dir) + if not args.keep_slc_daemon: + subprocess.run(SLC + ["daemon-shutdown"], check=True) + if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) main() diff --git a/tools/create_gbl.py b/tools/create_gbl.py index accafca5..282d3b06 100755 --- a/tools/create_gbl.py +++ b/tools/create_gbl.py @@ -12,7 +12,6 @@ import subprocess from ruamel.yaml import YAML -from xml.etree import ElementTree def parse_c_header_defines(file_content: str) -> dict[str, str]: @@ -126,7 +125,6 @@ def main(): ) project_root = slcp_path.parent - slps_path = (project_root / project_name).with_suffix(".slps") if "sdk_dir" in args.parameters: gsdk_path = pathlib.Path(args.parameters["sdk_dir"]) @@ -143,17 +141,6 @@ def main(): # Parse the main Simplicity Studio project config slcp = YAML(typ="safe").load(slcp_path.read_text()) - # Extract the chip ID from the SLPS file, `commander` needs it - slps_xml = ElementTree.parse(slps_path) - device_part_id = ( - slps_xml.getroot() - .find(".//properties[@key='projectCommon.partId']") - .attrib["value"] - .split(".")[-1] - .upper() - ) - print("Detected device part ID:", device_part_id, flush=True) - gbl_metadata = YAML(typ="safe").load( (project_root / "gbl_metadata.yaml").read_text() ) @@ -257,8 +244,6 @@ def main(): else "--bootloader" ), out_file, - "--device", - device_part_id, ] + ( [ "--metadata",