Initial E2E robot tests

.github/workflows/e2e_test.yaml

@@ -0,0 +1,114 @@
+name: E2E Tests
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/robot-tests.yml'
+      - 'test/**'
+      - '!test/README.md'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-latest, macos-13]
+        arch: [amd64, arm64]
+        include:
+          - os: windows-latest
+            target: win
+          - os: ubuntu-latest
+            target: lin
+          - os: macos-13
+            target: mac
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install pip dependencies
+        run: |
+            pip install --upgrade pip
+            pip install -r test/requirements.txt
+
+      - name: Setup vcpkg environment
+        uses: soumeh01/actions/vcpkg@testwindows
+        with:
+          config: "./test/vcpkg-configuration.json"
+          vcpkg-root: "${{ github.workspace }}/.vcpkg"
+          cache: "-"
+
+      - name: Activate Arm tool license
+        run: |
+          armlm activate --server https://mdk-preview.keil.arm.com --product KEMDK-COM0
+        working-directory: ./test
+
+      - name: Run Test
+        run: |
+          python -m robot --outputdir reports-${{ matrix.target }}-${{ matrix.arch }} --settag ${{ matrix.target }}-${{ matrix.arch }} --name ${{ matrix.target }}-${{ matrix.arch }} ./test
+
+      - name: Archieve test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: reports-${{ matrix.target }}-${{ matrix.arch }}
+          path: reports-${{ matrix.target }}-${{ matrix.arch }}
+
+  report:
+    runs-on: ubuntu-latest
+    if: always()
+    needs: test
+    permissions: write-all
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+
+    - name: Install pip dependencies
+      run: |
+          pip install --upgrade pip
+          pip install -r test/requirements.txt
+
+    - name: Download reports
+      uses: actions/download-artifact@v4
+      with:
+        path: artifacts
+        pattern: reports-*
+
+    - name: Consolidate robot test results
+      working-directory: artifacts
+      run: |
+        python -m robot.rebot --name Collective_Robot_Results --outputdir collective_robot_results --output output.xml \
+          ./reports-win-amd64/output.xml ./reports-win-arm64/output.xml \
+          ./reports-lin-amd64/output.xml ./reports-lin-arm64/output.xml \
+          ./reports-mac-amd64/output.xml ./reports-mac-arm64/output.xml
+
+    - name: Generate Summary report
+      if: always()
+      run: |
+        python ./test/lib/execution_summary.py artifacts/collective_robot_results/output.xml
+
+    - name: Print E2E Report
+      if: always()
+      run: cat summary_report.md >> $GITHUB_STEP_SUMMARY
+
+    - name: Archieve consolidated test results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: consolidated-reports
+        path: artifacts/collective_robot_results

.gitignore

@@ -1 +1,7 @@
-build
+# Directories
+/build*/
+**/run
+**/__pycache__
+
+# Files
+**/stdout.txt

ADR System test framework.md

@@ -0,0 +1,49 @@
+# System test framework
+
+The objective of this document is to conduct a thorough evaluation of various system test frameworks while taking into account our specific use cases. Our aim is to gain insight into the advantages and disadvantages of the frameworks under consideration, thereby facilitating a fair comparison.
+
+## Contributors
+
+- Sourabh Mehta
+
+## Reviewer
+
+- Daniel Brondani
+- Jonatan Antony
+- Samuel Pelegrinello Caipers
+- Evgueni Driouk
+- Thorsten de Buhr
+
+## Context
+
+The end-to-end (E2E) tests developed using decided framework aim to validate the fully integrated applications, assessing the interactions among different utilities within the entire system. The overarching goal is to simulate user interactions and ensure that the desired build output is achieved through comprehensive system testing.
+
+## Basic expectation from framework
+
+- Access to system env variables
+- Flexibility to run EXEs, capture return code, stdout, stderr
+- Golden file comparison
+- Generates HTML report
+
+## Comparison
+
+| Sr. No |  key points  |GoogleTest | PyTest | RobotTest |
+| :----- |:-- |:--------- | :----- | :-------- |
+|1|Language Support|C++|python|keyword/macro based (backed by python)|
+|2|Debug|yes|yes|debug with prints or logs|
+|3|Dependencies|external dep (need to maintain as submodule in repo)| external dep need to be installed|external dep need to be installed|
+|4|HTML Report|Generates XML, 3rd party lib is needed to generate HTML|HTML reports with pytest-html plugin|Inbuilt [HTML report](https://robotframework.org/RobotDemo/report.html) generation|
+|5|Ease of use|yes|yes|yes (Needs learning)|
+|6|Flexibility and Customization|yes|yes|yes|
+|7|Integration with CI/CD|yes|yes|yes|
+|8|Feedback Mechanisms & diagnosis|easy to diagnose|easy to diagnose|use [debug logs](https://robotframework.org/RobotDemo/log.html) for diagnosis|
+
+## Considerations
+
+- cbuildgen has toochain related tests (AC6, GCC, IAR) 
+  - Are we running all toolchains tests on cmsis-toolbox?
+- cpackget test should be migrated
+
+## Design Decision
+
+It was determined that the `robot test` would be the optimal choice. This option requires no language-specific learning and offers a more user-friendly reporting mechanism. Additionally, the robot library boasts extensive support and numerous prebuilt keywords.

test/RobotTests.md

@@ -0,0 +1,96 @@
+# Installation and Running Robot Framework Tests
+
+This guide will walk you through the installation process and running of Robot Framework tests.
+
+## Prerequisites
+
+Before running Robot Framework tests, ensure you have the following prerequisites installed on your system:
+
+- Python (minimum recommended version **3.11**)
+- pip (python package manager)
+- [vcpkg](https://github.com/Open-CMSIS-Pack/cmsis-toolbox/blob/main/docs/installation.md#vcpkg---setup-using-cli)
+
+## Install Test Environment
+
+### Install Robot Framework
+
+Install Robot Framework and its dependencies using pip:
+
+```bash
+cd <root_dir>/test
+pip install --upgrade pip
+pip install -r requirements.txt
+```
+
+### Install Toolchains using vcpkg
+
+These commands will install all the required toolchains listed [here](./vcpkg-configuration.json):
+
+```bash
+cd <root_dir>/test
+vcpkg x-update-registry --all
+vcpkg activate
+```
+
+```txt
+👉 Important:
+    The AC6 toolchain installation includes the "fromelf" utility, which can be found in the AC6 toolchain installation
+    directory. Ensure that "fromelf" utlity is in PATH.
+```
+
+## Running Tests
+
+### Run all tests
+
+This command will run all tests located in the `test` directory and place the test reports and logs under specified directory.
+
+```bash
+robot -d <output_directory> <path_to_tests>
+```
+
+for e.g.
+
+```bash
+vcpkg activate
+robot -d results ./test
+```
+
+### Running Specific Tests
+
+To run specific tests, use the `--test` options:
+
+```bash
+robot --test <test_name> <path_to_tests>
+```
+
+for e.g.
+
+```bash
+vcpkg activate
+robot --test "Validate build-c Example"  test/test.robot
+```
+
+## Adding Tests
+
+The test cases in [test.robot](./test.robot) are implemented in a data-driven style, where each test case utilizes a single higher-level keyword to encapsulate the test workflow. To incorporate a new example for validation, follow the steps outlined below.
+
+- Add Example under [data](./data/) directory.
+- Add test details under **Test Cases** section following below conventions
+
+  ```robot
+  # <Name of the Test>
+  #    <Path to the input <project>.csolution.yml file>    <Expected build status>    <Example root directory name>
+  ```
+
+  for e.g.
+
+    ```robot
+    Validate USB Example
+        ${TEST_DATA_DIR}${/}${USB}${/}solution.csolution.yml    ${0}    ${USB}
+    ```
+
+```txt
+☑️ Note:
+    All options in the tests should be separated by "TABs".
+    For more information on robot test follow https://docs.robotframework.org/docs/testcase_styles/datadriven
+```

test/__init__.robot

@@ -0,0 +1,3 @@
+*** Settings ***
+Documentation       Initialization for the robot tests in this directory
+Test Timeout        10 minutes

test/data/build-asm/project/AC6/AsmArm.s

@@ -0,0 +1,16 @@
+
+     AREA   DATA
+
+     IF     HEXADECIMAL_TEST != 11259375 ; 0xABCDEF
+     INFO   1, "HEXADECIMAL_TEST failed!"
+     ENDIF
+
+     IF     DECIMAL_TEST != 1234567890
+     INFO   1, "DECIMAL_TEST failed!"
+     ENDIF
+
+     IF     STRING_TEST != "String0"
+     INFO   1, "STRING_TEST failed!"
+     ENDIF
+
+     END

test/data/build-asm/project/AC6/Auto.s

@@ -0,0 +1,9 @@
+
+     AREA   DATA
+
+     IF     :LNOT::DEF:AUTO_DEF
+     INFO   1, "AUTO_DEF is not defined!"
+     ENDIF
+
+     END
+

test/data/build-asm/project/AC6/GnuSyntax.s

@@ -0,0 +1,10 @@
+
+   .syntax unified
+   .arch armv7-m
+
+   .ifndef GAS_DEF
+   .error "GAS_DEF is not defined!"
+   .endif
+
+   .end
+

test/data/build-asm/project/AC6/PreProcessed.S

@@ -0,0 +1,9 @@
+
+   .syntax unified
+   .arch armv7-m
+
+#ifndef PRE_PROCESSED_DEF
+   .error "PRE_PROCESSED_DEF is not defined!"
+#endif
+
+   .end

test/data/build-asm/project/GCC/GAS.s

@@ -0,0 +1,9 @@
+
+   .syntax unified
+   .arch armv7-m
+
+   .ifndef GAS_DEF
+   .error "GAS_DEF is not defined!"
+   .endif
+
+   .end

test/data/build-asm/project/GCC/NonPreProcessed.s

@@ -0,0 +1,13 @@
+
+   .syntax unified
+   .arch armv7-m
+
+#ifndef PRE_PROCESSED_DEF
+   .equ SET_ERR_DEF,1
+#endif
+
+   .ifndef SET_ERR_DEF
+   .error "SET_ERR_DEF is not defined! It seems this file was preprocessed but it shouldn't!"
+   .endif
+
+   .end

test/data/build-asm/project/GCC/PreProcessed.S

@@ -0,0 +1,9 @@
+
+   .syntax unified
+   .arch armv7-m
+
+#ifndef PRE_PROCESSED_DEF
+   .error "PRE_PROCESSED_DEF is not defined!"
+#endif
+
+   .end

test/data/build-asm/project/IAR/Asm.s

@@ -0,0 +1,15 @@
+                MODULE   ?Asm
+
+#ifndef IAR_ASM_DEF
+#error "IAR_ASM_DEF is not defined!"
+#endif
+
+                EXTERN   Reset_Handler_C
+
+                PUBWEAK  Reset_Handler
+                SECTION  .text:CODE:REORDER:NOROOT(2)
+Reset_Handler
+                LDR      R0, =Reset_Handler_C
+                BX       R0
+
+                END

test/data/build-asm/project/main.c

@@ -0,0 +1,6 @@
+#include "RTE_Components.h"
+#include CMSIS_device_header
+
+int main(void) {
+  return 0;
+}