[Tizen] Run matter unit tests on QEMU (#26563)

* Run unit tests on Tizen platform under QEMU

* Run all generated unit test executables

* Move failing tests to XFAIL group

* Manual for debugging tests on QEMU

* Add "Run test (Linux x64)" to VSCode debugger

* Print failed tests

* Drop XFAIL since now all tests pass on Tizen

* Do not duplicate example builds with QEMU tests

* Do not add rootshell to the word list

* Revert defines reordering

* Add TODO for GN test assets

---------

Co-authored-by: Andrei Litvin <andy314@gmail.com>

.github/workflows/examples-tizen.yaml

@@ -25,7 +25,7 @@ concurrency:
 
 env:
  CHIP_NO_LOG_TIMESTAMPS: true
- 
+
 jobs:
  tizen:
  name: Tizen
@@ -54,10 +54,8 @@ jobs:
  "./scripts/build/build_examples.py \
  --enable-flashbundle \
  --target tizen-arm-all-clusters \
- --target tizen-arm-all-clusters-minimal-no-wifi \
  --target tizen-arm-chip-tool-ubsan \
  --target tizen-arm-light \
- --target tizen-arm-light-no-ble-no-wifi \
  build \
  --copy-artifacts-to out/artifacts \
  "

.vscode/launch.json

@@ -20,12 +20,20 @@
  },
  {
  "name": "Run application (Linux x64)",
- "type": "lldb",
+ "type": "cppdbg",
  "request": "launch",
  "program": "${workspaceFolder}/out/${input:outAppLinux}",
  "args": [],
  "cwd": "${workspaceFolder}"
  },
+ {
+ "name": "Run test (Linux x64)",
+ "type": "cppdbg",
+ "request": "launch",
+ "program": "${workspaceFolder}/out/${input:outTestLinux}",
+ "args": [],
+ "cwd": "${workspaceFolder}"
+ },
  {
  "name": "QRCode Tests",
  "type": "cppdbg",
@@ -486,6 +494,15 @@
  "description": "Select the application to run"
  }
  },
+ {
+ "type": "command",
+ "id": "outTestLinux",
+ "command": "shellCommand.execute",
+ "args": {
+ "command": "find ${workspaceFolder}/out/linux-x64-*/tests -type f -executable |sort |sed 's$${workspaceFolder}/out/$$'",
+ "description": "Select the test to run"
+ }
+ },
  {
  "type": "pickString",
  "id": "mbedDebugProfile",

build/chip/chip_test.gni

@@ -41,21 +41,23 @@ if (chip_link_tests) {
  group(_test_name + "_lib") {
  }
 
- pw_python_action(_test_name + "_run") {
- deps = [ ":${_test_name}" ]
- inputs = [ pw_unit_test_AUTOMATIC_RUNNER ]
- module = "pw_unit_test.test_runner"
- python_deps = [
- "$dir_pw_cli/py",
- "$dir_pw_unit_test/py",
- ]
- args = [
- "--runner",
- rebase_path(pw_unit_test_AUTOMATIC_RUNNER, root_build_dir),
- "--test",
- rebase_path("$_test_output_dir/$_test_name", root_build_dir),
- ]
- stamp = true
+ if (chip_pw_run_tests) {
+ pw_python_action(_test_name + "_run") {
+ deps = [ ":${_test_name}" ]
+ inputs = [ pw_unit_test_AUTOMATIC_RUNNER ]
+ module = "pw_unit_test.test_runner"
+ python_deps = [
+ "$dir_pw_cli/py",
+ "$dir_pw_unit_test/py",
+ ]
+ args = [
+ "--runner",
+ rebase_path(pw_unit_test_AUTOMATIC_RUNNER, root_build_dir),
+ "--test",
+ rebase_path("$_test_output_dir/$_test_name", root_build_dir),
+ ]
+ stamp = true
+ }
  }
  }
 } else {

build/chip/chip_test_group.gni

@@ -60,11 +60,13 @@ template("chip_test_group") {
  }
  }
 
- group("${_test_group_name}_run") {
- if (chip_link_tests) {
- deps = []
- foreach(_test, invoker.deps) {
- deps += [ get_label_info(_test, "label_no_toolchain") + "_run" ]
+ if (chip_pw_run_tests) {
+ group("${_test_group_name}_run") {
+ if (chip_link_tests) {
+ deps = []
+ foreach(_test, invoker.deps) {
+ deps += [ get_label_info(_test, "label_no_toolchain") + "_run" ]
+ }
  }
  }
  }

build/chip/chip_test_suite.gni

@@ -151,10 +151,12 @@ template("chip_test_suite") {
  }
  }
 
- group("${_suite_name}_run") {
- deps = []
- foreach(_test, tests) {
- deps += [ ":${_test}_run" ]
+ if (chip_pw_run_tests) {
+ group("${_suite_name}_run") {
+ deps = []
+ foreach(_test, tests) {
+ deps += [ ":${_test}_run" ]
+ }
  }
  }
  } else {

build/chip/tests.gni

@@ -24,10 +24,17 @@ declare_args() {
 
 declare_args() {
  # Build executables for running individual tests.
- chip_link_tests =
- chip_build_tests && (current_os == "linux" || current_os == "mac") &&
- current_cpu == target_cpu
+ chip_link_tests = chip_build_tests &&
+ (current_os == "linux" || current_os == "mac" ||
+ current_os == "tizen") && current_cpu == target_cpu
+}
 
+declare_args() {
+ # Run tests with pigweed test runner.
+ chip_pw_run_tests = chip_link_tests && current_os != "tizen"
+}
+
+declare_args() {
  # Use source_set instead of static_lib for tests.
  chip_build_test_static_libraries = chip_device_platform != "efr32"
 }

scripts/build/builders/tizen.py

@@ -113,6 +113,9 @@ def __init__(self,
  except FileNotFoundError:
  pass
 
+ if app == TizenApp.TESTS:
+ self.extra_gn_options.append('chip_build_tests=true')
+
  if not enable_ble:
  self.extra_gn_options.append('chip_config_network_layer_ble=false')
  if not enable_thread:

src/test_driver/tizen/BUILD.gn

@@ -14,5 +14,8 @@
 
 group("check") {
  testonly = true
- deps = [ "integration_tests/lighting-app:check" ]
+ deps = [
+ "chip_tests:check",
+ "integration_tests/lighting-app:check",
+ ]
 }

src/test_driver/tizen/README.md

@@ -55,3 +55,20 @@ gn gen --check --fail-on-unused-args \
 # Run Tizen QEMU-based tests
 ninja -C out/tizen-check check
 ```
+
+## Debugging Tests on QEMU
+
+When running tests with `ninja -C out/tizen-check check`, the test driver prints
+the command to run the test on QEMU. Please search for the following line in the
+output of the `ninja` command: `INFO:root:run: qemu-system-arm`. Make sure that
+your terminal does not limit the number of lines printed, or simply redirect the
+output to a file as follows:
+
+```sh
+ninja -C out/tizen-check check > ninja-tizen-check.log 2>&1
+```
+
+Then, use the run command and add the `rootshell` keyword to kernel arguments
+passed to QEMU (the string after the `-append` option). This will run QEMU, but
+instead of running the test, it will drop you to the shell. From there, you can
+run the test manually by typing `/mnt/chip/runner.sh`.

src/test_driver/tizen/chip_tests/BUILD.gn

@@ -0,0 +1,49 @@
+# Copyright (c) 2020 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import("//build_overrides/chip.gni")
+import("//build_overrides/tizen.gni")
+
+import("${tizen_sdk_build_root}/tizen_sdk.gni")
+
+tizen_qemu_mkisofs("chip-tests-runner") {
+ runner = "runner.sh"
+
+ # Build CHIP unit tests.
+ deps = [ "${chip_root}/src:tests" ]
+
+ # Bundle all created unit tests.
+ # TODO (arkq): Collect all unit tests in a way that we could add all targets
+ # to the assets list. Right now QEMU runner dependencies are not
+ # tracked properly. Changes to the unit tests will not trigger
+ # rebuild of the ISO image, so the test will be run with old
+ # binaries.
+ assets_non_tracked = [ rebase_path("${root_build_dir}/tests") ]
+}
+
+tizen_qemu_run("chip-tests") {
+ # Enable network support, so Tizen can obtain current date/time from the
+ # network. Correct date/time is required for the commissioning process -
+ # attestation will fail otherwise.
+ virtio_net = true
+
+ deps = [ ":chip-tests-runner" ]
+ mkisofs_outputs = get_target_outputs(":chip-tests-runner")
+ iso_image = rebase_path(mkisofs_outputs[0])
+}
+
+group("check") {
+ deps = [ ":chip-tests" ]
+ testonly = true
+}

src/test_driver/tizen/chip_tests/runner.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+#
+# Copyright (c) 2021 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+# Print CHIP logs on stdout
+dlogutil CHIP &
+
+FAILED=()
+STATUS=0
+
+# Run all executables in the /mnt/chip directory except the runner.sh script
+while IFS= read -r TEST; do
+
+ NAME=$(basename "$TEST")
+
+ echo
+ echo "RUN: $NAME"
+
+ RV=0
+ "$TEST" || RV=$?
+
+ if [ "$RV" -eq 0 ]; then
+ echo -e "DONE: \e[32mSUCCESS\e[0m"
+ else
+ FAILED+=("$NAME")
+ STATUS=$((STATUS + 1))
+ echo -e "DONE: \e[31mFAIL\e[0m"
+ fi
+
+done < <(find /mnt/chip -type f -executable ! -name runner.sh)
+
+if [ ! "$STATUS" -eq 0 ]; then
+ echo
+ echo "### FAILED: ${FAILED[*]}"
+fi
+
+exit "$STATUS"

src/test_driver/tizen/integration_tests/lighting-app/BUILD.gn

@@ -17,7 +17,7 @@ import("//build_overrides/tizen.gni")
 
 import("${tizen_sdk_build_root}/tizen_sdk.gni")
 
-tizen_qemu_mkisofs("test-runner") {
+tizen_qemu_mkisofs("lighting-app-runner") {
  runner = "runner.sh"
 
  # Build applications used in the test.
@@ -34,13 +34,18 @@ tizen_qemu_mkisofs("test-runner") {
  ]
 }
 
-tizen_qemu_run("check") {
+tizen_qemu_run("lighting-app") {
  # Enable network support, so Tizen can obtain current date/time from the
  # network. Correct date/time is required for the commissioning process -
  # attestation will fail otherwise.
  virtio_net = true
 
- deps = [ ":test-runner" ]
- mkisofs_outputs = get_target_outputs(":test-runner")
+ deps = [ ":lighting-app-runner" ]
+ mkisofs_outputs = get_target_outputs(":lighting-app-runner")
  iso_image = rebase_path(mkisofs_outputs[0])
 }
+
+group("check") {
+ deps = [ ":lighting-app" ]
+ testonly = true
+}

third_party/tizen/tizen_sdk.gni

@@ -198,6 +198,9 @@ template("tizen_qemu_mkisofs") {
  args += invoker.assets
  inputs += invoker.assets
  }
+ if (defined(invoker.assets_non_tracked)) {
+ args += invoker.assets_non_tracked
+ }
 
  script = "${build_root}/gn_run_binary.py"
  }