From b8a5fecd6468ce0b97b861064dba53701086d6f7 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Fri, 3 May 2024 12:14:41 -0400 Subject: [PATCH 1/6] Include version strings in generated firmware names --- .../skyconnect_bootloader-uart-xmodem.yaml | 1 + manifests/skyconnect_firmware-eraser.yaml | 1 + manifests/skyconnect_ncp-uart-hw.yaml | 1 + manifests/skyconnect_ot-rcp.yaml | 1 + manifests/yellow_bootloader-uart-xmodem.yaml | 1 + manifests/yellow_ncp-uart-hw.yaml | 1 + manifests/yellow_ot-rcp.yaml | 1 + tools/build_project.py | 60 +++++++++++++++---- 8 files changed, 55 insertions(+), 12 deletions(-) diff --git a/manifests/skyconnect_bootloader-uart-xmodem.yaml b/manifests/skyconnect_bootloader-uart-xmodem.yaml index ea2953d4..015dee64 100644 --- a/manifests/skyconnect_bootloader-uart-xmodem.yaml +++ b/manifests/skyconnect_bootloader-uart-xmodem.yaml @@ -1,6 +1,7 @@ name: SkyConnect Bootloader device: EFR32MG21A020F512IM32 base_project: src/bootloader-uart-xmodem +filename: "{manifest_name}_{gecko_bootloader_version}" gbl: fw_type: gecko-bootloader diff --git a/manifests/skyconnect_firmware-eraser.yaml b/manifests/skyconnect_firmware-eraser.yaml index 4950a00d..9feb8a14 100644 --- a/manifests/skyconnect_firmware-eraser.yaml +++ b/manifests/skyconnect_firmware-eraser.yaml @@ -1,5 +1,6 @@ name: SkyConnect Firmware Eraser device: EFR32MG21A020F512IM32 base_project: misc/firmware-eraser +filename: "{manifest_name}_gsdk_{sdk_version}" gbl: {} diff --git a/manifests/skyconnect_ncp-uart-hw.yaml b/manifests/skyconnect_ncp-uart-hw.yaml index cb6d8c60..fc0e2890 100644 --- a/manifests/skyconnect_ncp-uart-hw.yaml +++ b/manifests/skyconnect_ncp-uart-hw.yaml @@ -1,6 +1,7 @@ name: SkyConnect Zigbee device: EFR32MG21A020F512IM32 base_project: src/ncp-uart-hw +filename: "{manifest_name}_{ezsp_version}" gbl: fw_type: ncp-uart-hw diff --git a/manifests/skyconnect_ot-rcp.yaml b/manifests/skyconnect_ot-rcp.yaml index b3889686..168c8d09 100644 --- a/manifests/skyconnect_ot-rcp.yaml +++ b/manifests/skyconnect_ot-rcp.yaml @@ -1,6 +1,7 @@ name: SkyConnect OpenThread RCP device: EFR32MG21A020F512IM32 base_project: src/ot-rcp +filename: "{manifest_name}_{ot_rcp_version.split('/')[-1]}_gsdk_{sdk_version}" gbl: fw_type: ot-rcp diff --git a/manifests/yellow_bootloader-uart-xmodem.yaml b/manifests/yellow_bootloader-uart-xmodem.yaml index 91c98bee..9b263385 100644 --- a/manifests/yellow_bootloader-uart-xmodem.yaml +++ b/manifests/yellow_bootloader-uart-xmodem.yaml @@ -1,6 +1,7 @@ name: Yellow Bootloader device: MGM210PA32JIA base_project: src/bootloader-uart-xmodem +filename: "{manifest_name}_{gecko_bootloader_version}" gbl: fw_type: gecko-bootloader diff --git a/manifests/yellow_ncp-uart-hw.yaml b/manifests/yellow_ncp-uart-hw.yaml index de4df9c2..d2fda810 100644 --- a/manifests/yellow_ncp-uart-hw.yaml +++ b/manifests/yellow_ncp-uart-hw.yaml @@ -1,6 +1,7 @@ name: Yellow Zigbee device: MGM210PA32JIA base_project: src/ncp-uart-hw +filename: "{manifest_name}_{ezsp_version}" gbl: fw_type: ncp-uart-hw diff --git a/manifests/yellow_ot-rcp.yaml b/manifests/yellow_ot-rcp.yaml index 45fca086..c0a60b6a 100644 --- a/manifests/yellow_ot-rcp.yaml +++ b/manifests/yellow_ot-rcp.yaml @@ -1,6 +1,7 @@ name: Yellow OpenThread RCP device: MGM210PA32JIA base_project: src/ot-rcp +filename: "{manifest_name}_{ot_rcp_version.split('/')[-1]}_gsdk_{sdk_version}" gbl: fw_type: ot-rcp diff --git a/tools/build_project.py b/tools/build_project.py index d677cb27..bf2b68e1 100755 --- a/tools/build_project.py +++ b/tools/build_project.py @@ -44,6 +44,14 @@ yaml = YAML(typ="safe") +def evaulate_f_string(f_string: str, variables: dict[str, typing.Any]) -> str: + """ + Evaluates an `f`-string with the given locals. + """ + + return eval("f" + repr(f_string), variables) + + def ensure_folder(path: str | pathlib.Path) -> pathlib.Path: """Ensure that the path exists and is a folder.""" path = pathlib.Path(path) @@ -89,15 +97,21 @@ def parse_override(override: str) -> tuple[str, dict | list]: raise argparse.ArgumentTypeError(f"Invalid JSON: {exc}") -def parse_prefixed_output(output: str) -> tuple[str, pathlib.Path]: +def parse_prefixed_output(output: str) -> tuple[str, pathlib.Path | None]: """Parse a prefixed output parameter.""" - if ":" not in output: + if ":" in output: + prefix, _, path = output.partition(":") + path = pathlib.Path(path) + else: + prefix = output + path = None + + if prefix not in ("gbl", "hex", "out"): raise argparse.ArgumentTypeError( - "Override must be of the form (e.g.) `gbl=output.gbl`" + "Output format is of the form `gbl:overridden_filename.gbl` or just `gbl`" ) - prefix, _, path = output.partition(":") - return prefix, pathlib.Path(path) + return prefix, path def get_git_commit_id(repo: pathlib.Path) -> str: @@ -145,6 +159,13 @@ def main(): required=True, help="Output file prefixed with its file type", ) + parser.add_argument( + "--output-dir", + dest="output_dir", + type=pathlib.Path, + default=pathlib.Path("."), + help="Output directory for artifacts, will be created if it does not exist", + ) parser.add_argument( "--no-clean-build-dir", action="store_false", @@ -207,11 +228,6 @@ def main(): if args.toolchains != get_toolchain_default_paths(): args.toolchains = args.toolchains[len(get_toolchain_default_paths()) :] - # Template variables for C defines - value_template_env = { - "git_repo_hash": get_git_commit_id(repo=pathlib.Path(__file__).parent.parent), - } - manifest = yaml.load(args.manifest.read_text()) for key, override in args.overrides: @@ -264,6 +280,12 @@ def main(): # Otherwise, append it output_config.append({"name": name, "value": value}) + # Template variables for C defines + value_template_env = { + "git_repo_hash": get_git_commit_id(repo=pathlib.Path(__file__).parent.parent), + "manifest_name": args.manifest.stem, + } + # Copy the base project into the output directory if args.clean_build_dir: with contextlib.suppress(OSError): @@ -581,11 +603,25 @@ def main(): output_artifact = (cmake_build_root / base_project_name).with_suffix(".gbl") + # Read the metadata extracted from the source and build trees + extracted_gbl_metadata = json.loads( + (output_artifact.parent / "gbl_metadata.json").read_text() + ) + base_filename = evaulate_f_string( + manifest.get("filename", "{manifest_name}"), + {**value_template_env, **extracted_gbl_metadata}, + ) + + args.output_dir.mkdir(exist_ok=True) + # Copy the output artifacts - for extension, path in args.outputs: + for extension, output_path in args.outputs: + if output_path is None: + output_path = f"{base_filename}.{extension}" + shutil.copy( src=output_artifact.with_suffix(f".{extension}"), - dst=path, + dst=args.output_dir / output_path, ) if args.clean_build_dir: From 99d24fcc7e489214899eec709d590cff08da5263 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Fri, 3 May 2024 12:28:39 -0400 Subject: [PATCH 2/6] Unconditionally write the GBL metadata JSON --- tools/create_gbl.py | 144 ++++++++++++++++++++++---------------------- 1 file changed, 71 insertions(+), 73 deletions(-) diff --git a/tools/create_gbl.py b/tools/create_gbl.py index ca603117..accafca5 100755 --- a/tools/create_gbl.py +++ b/tools/create_gbl.py @@ -158,88 +158,86 @@ def main(): (project_root / "gbl_metadata.yaml").read_text() ) - # Only create GBL metadata JSON if there is something to create - if "fw_type" in gbl_metadata or "baudrate" in gbl_metadata: - # Prepare the GBL metadata - metadata = { - "metadata_version": 1, - "sdk_version": slcp["sdk"]["version"], - "fw_type": gbl_metadata["fw_type"], - "baudrate": gbl_metadata["baudrate"], - } - - # Compute the dynamic metadata - gbl_dynamic = gbl_metadata.get("dynamic", []) - - if "ezsp_version" in gbl_dynamic: - gbl_dynamic.remove("ezsp_version") - zigbee_esf_props = parse_properties_file( - (gsdk_path / "protocol/zigbee/esf.properties").read_text() - ) - metadata["ezsp_version"] = zigbee_esf_props["version"][0] + # Prepare the GBL metadata + metadata = { + "metadata_version": 1, + "sdk_version": slcp["sdk"]["version"], + "fw_type": gbl_metadata.get("fw_type"), + "baudrate": gbl_metadata.get("baudrate"), + } + + # Compute the dynamic metadata + gbl_dynamic = gbl_metadata.get("dynamic", []) + + if "ezsp_version" in gbl_dynamic: + gbl_dynamic.remove("ezsp_version") + zigbee_esf_props = parse_properties_file( + (gsdk_path / "protocol/zigbee/esf.properties").read_text() + ) + metadata["ezsp_version"] = zigbee_esf_props["version"][0] - if "cpc_version" in gbl_dynamic: - gbl_dynamic.remove("cpc_version") - sl_gsdk_version_h = parse_c_header_defines( - (gsdk_path / "platform/common/inc/sl_gsdk_version.h").read_text() - ) - metadata["cpc_version"] = ".".join( - [ - str(sl_gsdk_version_h["SL_GSDK_MAJOR_VERSION"]), - str(sl_gsdk_version_h["SL_GSDK_MINOR_VERSION"]), - str(sl_gsdk_version_h["SL_GSDK_PATCH_VERSION"]), - ] - ) + if "cpc_version" in gbl_dynamic: + gbl_dynamic.remove("cpc_version") + sl_gsdk_version_h = parse_c_header_defines( + (gsdk_path / "platform/common/inc/sl_gsdk_version.h").read_text() + ) + metadata["cpc_version"] = ".".join( + [ + str(sl_gsdk_version_h["SL_GSDK_MAJOR_VERSION"]), + str(sl_gsdk_version_h["SL_GSDK_MINOR_VERSION"]), + str(sl_gsdk_version_h["SL_GSDK_PATCH_VERSION"]), + ] + ) - try: - internal_app_config_h = parse_c_header_defines( - (project_root / "config/internal_app_config.h").read_text() - ) - except FileNotFoundError: - internal_app_config_h = {} - - if "CPC_SECONDARY_APP_VERSION_SUFFIX" in internal_app_config_h: - metadata["cpc_version"] += internal_app_config_h[ - "CPC_SECONDARY_APP_VERSION_SUFFIX" - ] - - if "zwave_version" in gbl_dynamic: - gbl_dynamic.remove("zwave_version") - zwave_esf_props = parse_properties_file( - (gsdk_path / "protocol/z-wave/esf.properties").read_text() + try: + internal_app_config_h = parse_c_header_defines( + (project_root / "config/internal_app_config.h").read_text() ) - metadata["zwave_version"] = zwave_esf_props["version"][0] + except FileNotFoundError: + internal_app_config_h = {} + + if "CPC_SECONDARY_APP_VERSION_SUFFIX" in internal_app_config_h: + metadata["cpc_version"] += internal_app_config_h[ + "CPC_SECONDARY_APP_VERSION_SUFFIX" + ] + + if "zwave_version" in gbl_dynamic: + gbl_dynamic.remove("zwave_version") + zwave_esf_props = parse_properties_file( + (gsdk_path / "protocol/z-wave/esf.properties").read_text() + ) + metadata["zwave_version"] = zwave_esf_props["version"][0] - if "ot_rcp_version" in gbl_dynamic: - gbl_dynamic.remove("ot_rcp_version") - openthread_config_h = parse_c_header_defines( - (project_root / "config/sl_openthread_generic_config.h").read_text() - ) - metadata["ot_rcp_version"] = openthread_config_h["PACKAGE_STRING"] + if "ot_rcp_version" in gbl_dynamic: + gbl_dynamic.remove("ot_rcp_version") + openthread_config_h = parse_c_header_defines( + (project_root / "config/sl_openthread_generic_config.h").read_text() + ) + metadata["ot_rcp_version"] = openthread_config_h["PACKAGE_STRING"] - if "gecko_bootloader_version" in gbl_dynamic: - gbl_dynamic.remove("gecko_bootloader_version") - btl_config_h = parse_c_header_defines( - (gsdk_path / "platform/bootloader/config/btl_config.h").read_text() - ) + if "gecko_bootloader_version" in gbl_dynamic: + gbl_dynamic.remove("gecko_bootloader_version") + btl_config_h = parse_c_header_defines( + (gsdk_path / "platform/bootloader/config/btl_config.h").read_text() + ) - metadata["gecko_bootloader_version"] = ".".join( - [ - str(btl_config_h["BOOTLOADER_VERSION_MAIN_MAJOR"]), - str(btl_config_h["BOOTLOADER_VERSION_MAIN_MINOR"]), - str(btl_config_h["BOOTLOADER_VERSION_MAIN_CUSTOMER"]), - ] - ) + metadata["gecko_bootloader_version"] = ".".join( + [ + str(btl_config_h["BOOTLOADER_VERSION_MAIN_MAJOR"]), + str(btl_config_h["BOOTLOADER_VERSION_MAIN_MINOR"]), + str(btl_config_h["BOOTLOADER_VERSION_MAIN_CUSTOMER"]), + ] + ) - if gbl_dynamic: - raise ValueError(f"Unknown dynamic metadata: {gbl_dynamic}") + if gbl_dynamic: + raise ValueError(f"Unknown dynamic metadata: {gbl_dynamic}") - print("Generated GBL metadata:", metadata, flush=True) + print("Generated GBL metadata:", metadata, flush=True) - # Write it to a file for `commander` to read - (artifact_root / "gbl_metadata.json").write_text( - json.dumps(metadata, sort_keys=True) - ) + # Write it to a file for `commander` to read + (artifact_root / "gbl_metadata.json").write_text( + json.dumps(metadata, sort_keys=True) + ) # Make sure the Commander binary is included in the PATH on macOS if sys.platform == "darwin": From bb1d40ffbca2163385eb3954856887857adf57fb Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Fri, 3 May 2024 12:36:56 -0400 Subject: [PATCH 3/6] Use the output name as the archive name --- .github/workflows/build.yaml | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1e17b875..4e88cbd2 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -109,20 +109,6 @@ jobs: steps: - uses: actions/checkout@v4.1.4 - - name: Parse firmware manifest - id: read_manifest_yaml - run: | - yq -r ' - to_entries - | .[] - | select(.value | type == "string") - | .key + "=" + .value - ' "${{ matrix.manifest }}" >> $GITHUB_OUTPUT - - manifest_filename=$(basename "${{ matrix.manifest }}") - manifest_base="${manifest_filename%%.*}" - echo "manifest_base=$manifest_base" >> $GITHUB_OUTPUT - - name: Install SDK extensions run: | # XXX: slc-cli does not actually work when the extensions aren't in the SDK! @@ -137,6 +123,7 @@ jobs: done - name: Build firmware + id: build-firmware run: | # Fix `fatal: detected dubious ownership in repository at` git config --global --add safe.directory "$GITHUB_WORKSPACE" @@ -154,18 +141,20 @@ jobs: done # Build it - mkdir outputs - filename="${{ steps.read_manifest_yaml.outputs['manifest_base'] }}" - python3 tools/build_project.py \ $sdk_args \ $toolchain_args \ --manifest "${{ matrix.manifest }}" \ --build-dir build \ --build-system makefile \ - --output "gbl:outputs/$filename.gbl" \ - --output "hex:outputs/$filename.hex" \ - --output "out:outputs/$filename.out" + --output-dir outputs \ + --output gbl" \ + --output hex" \ + --output out" + + # Get the basename of the GBL in `outputs` + output_basename=$(basename -- $(basename -- $(ls -1 artifacts/*.gbl | head -n 1)) .gbl) + echo "output_basename=$output_basename" >> $GITHUB_OUTPUT - name: Install node within container (act) if: ${{ env.ACT }} @@ -176,7 +165,7 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v4.3.3 with: - name: ${{ steps.read_manifest_yaml.outputs['manifest_base'] }} + name: ${{ steps.build-firmware.outputs.output_basename }} path: outputs/* compression-level: 9 if-no-files-found: error From d250f8325b749779f0d0ba7492ea304df52f5e3a Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Fri, 3 May 2024 12:40:16 -0400 Subject: [PATCH 4/6] Fix quoting --- .github/workflows/build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4e88cbd2..5b89cdee 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -148,9 +148,9 @@ jobs: --build-dir build \ --build-system makefile \ --output-dir outputs \ - --output gbl" \ - --output hex" \ - --output out" + --output gbl \ + --output hex \ + --output out # Get the basename of the GBL in `outputs` output_basename=$(basename -- $(basename -- $(ls -1 artifacts/*.gbl | head -n 1)) .gbl) From 9d179f488c27514f5e69f2f32cf3942a23765347 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Fri, 3 May 2024 12:45:15 -0400 Subject: [PATCH 5/6] Add a filename format to multi-PAN as well --- manifests/skyconnect_rcp-uart-802154.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/skyconnect_rcp-uart-802154.yaml b/manifests/skyconnect_rcp-uart-802154.yaml index 0ab42d5b..c6ab5c9d 100644 --- a/manifests/skyconnect_rcp-uart-802154.yaml +++ b/manifests/skyconnect_rcp-uart-802154.yaml @@ -1,6 +1,7 @@ name: SkyConnect Multi-PAN device: EFR32MG21A020F512IM32 base_project: src/rcp-uart-802154 +filename: "{manifest_name}_gsdk_{sdk_version}" gbl: fw_type: rcp-uart-802154 From c6f3ddbe2dc22e0fc57b5649a9ad3e2b98d74849 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Fri, 3 May 2024 12:47:20 -0400 Subject: [PATCH 6/6] Use the correct folder name --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5b89cdee..692f83c3 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -153,7 +153,7 @@ jobs: --output out # Get the basename of the GBL in `outputs` - output_basename=$(basename -- $(basename -- $(ls -1 artifacts/*.gbl | head -n 1)) .gbl) + output_basename=$(basename -- $(basename -- $(ls -1 outputs/*.gbl | head -n 1)) .gbl) echo "output_basename=$output_basename" >> $GITHUB_OUTPUT - name: Install node within container (act)