From 74586a0f97659849adafa8b132031a4fa870362e Mon Sep 17 00:00:00 2001 From: Anas Nashif Date: Thu, 21 May 2020 09:11:40 -0400 Subject: [PATCH 1/6] sanitycheck: cleanup fixture processing Cleanup fixture processing and allow ztest testcases to support harness_config with fixture definition. Signed-off-by: Anas Nashif --- scripts/sanity_chk/harness.py | 3 +++ scripts/sanity_chk/sanitylib.py | 16 ++++++++-------- scripts/sanitycheck | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/scripts/sanity_chk/harness.py b/scripts/sanity_chk/harness.py index 0063c8521aac..33e18c72aa8e 100644 --- a/scripts/sanity_chk/harness.py +++ b/scripts/sanity_chk/harness.py @@ -149,3 +149,6 @@ def handle(self, line): self.capture_coverage = False self.process_test(line) + +class Ztest(Test): + pass diff --git a/scripts/sanity_chk/sanitylib.py b/scripts/sanity_chk/sanitylib.py index de71132d46d0..6f6fd5420b9b 100644 --- a/scripts/sanity_chk/sanitylib.py +++ b/scripts/sanity_chk/sanitylib.py @@ -1533,7 +1533,7 @@ def __lt__(self, other): return self.name < other.name # Global testsuite parameters - def check_build_or_run(self, build_only=False, enable_slow=False, device_testing=False, fixture=[]): + def check_build_or_run(self, build_only=False, enable_slow=False, device_testing=False, fixtures=[]): # right now we only support building on windows. running is still work # in progress. @@ -1571,13 +1571,13 @@ def check_build_or_run(self, build_only=False, enable_slow=False, device_testing runnable = False # console harness allows us to run the test and capture data. - if self.testcase.harness == 'console': + if self.testcase.harness in [ 'console', 'ztest']: # if we have a fixture that is also being supplied on the # command-line, then we need to run the test, not just build it. - if "fixture" in self.testcase.harness_config: - fixture_cfg = self.testcase.harness_config['fixture'] - if fixture_cfg in fixture: + fixture = self.testcase.harness_config.get('fixture') + if fixture: + if fixture in fixtures: _build_only = False else: _build_only = True @@ -2234,7 +2234,7 @@ def __init__(self, board_root_list=[], testcase_roots=[], outdir=None): self.cleanup = False self.enable_slow = False self.device_testing = False - self.fixture = [] + self.fixtures = [] self.enable_coverage = False self.enable_lsan = False self.enable_asan = False @@ -2580,7 +2580,7 @@ def load_from_file(self, file, filter_status=[]): self.build_only, self.enable_slow, self.device_testing, - self.fixture + self.fixtures ) instance.create_overlay(platform, self.enable_asan, self.enable_coverage, self.coverage_platform) instance_list.append(instance) @@ -2641,7 +2641,7 @@ def apply_filters(self, **kwargs): self.build_only, self.enable_slow, self.device_testing, - self.fixture + self.fixtures ) if not force_platform and plat.name in exclude_platform: discards[instance] = "Platform is excluded on command line." diff --git a/scripts/sanitycheck b/scripts/sanitycheck index a1e6050a8cc6..ea180410bcc3 100755 --- a/scripts/sanitycheck +++ b/scripts/sanitycheck @@ -774,7 +774,7 @@ def main(): suite.test_only = options.test_only suite.enable_slow = options.enable_slow suite.device_testing = options.device_testing - suite.fixture = options.fixture + suite.fixtures = options.fixture suite.enable_asan = options.enable_asan suite.enable_lsan = options.enable_lsan suite.enable_coverage = options.enable_coverage From a05907eeb90250c4d9b9f8bca9521cea4ae8e8ba Mon Sep 17 00:00:00 2001 From: Anas Nashif Date: Thu, 21 May 2020 09:12:08 -0400 Subject: [PATCH 2/6] tests: maxim_ds3231_api: use fixture This test depends on additional hardware being connected to the board, add a fixture and cleanup whitelist. Fixes #25177 Signed-off-by: Anas Nashif --- tests/drivers/counter/maxim_ds3231_api/testcase.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/drivers/counter/maxim_ds3231_api/testcase.yaml b/tests/drivers/counter/maxim_ds3231_api/testcase.yaml index 24ac39d4f604..be0bd2aac621 100644 --- a/tests/drivers/counter/maxim_ds3231_api/testcase.yaml +++ b/tests/drivers/counter/maxim_ds3231_api/testcase.yaml @@ -1,8 +1,10 @@ tests: drivers.counter.maxim_ds3231: tags: drivers - depends_on: counter + depends_on: counter i2c min_ram: 16 timeout: 400 filter: dt_compat_enabled("maxim,ds3231") - platform_whitelist: efr32mg_sltb004a frdm_k64f nrf51dk_nrf51422 nrf52840dk_nrf52840 nucleo_l476rg particle_xenon + harness: ztest + harness_config: + fixture: maxim_ds3231 From 33b208608b849be46fde8a58f4f1eadb7e0d2f0b Mon Sep 17 00:00:00 2001 From: Anas Nashif Date: Thu, 21 May 2020 09:15:01 -0400 Subject: [PATCH 3/6] sanitycheck: support fixtures Add fixtures to hardware map schema. Signed-off-by: Anas Nashif --- scripts/sanity_chk/hwmap-schema.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/sanity_chk/hwmap-schema.yaml b/scripts/sanity_chk/hwmap-schema.yaml index a3143c5f6697..135021adaeee 100644 --- a/scripts/sanity_chk/hwmap-schema.yaml +++ b/scripts/sanity_chk/hwmap-schema.yaml @@ -33,3 +33,8 @@ sequence: "pre_script": type: str required: false + "fixtures": + type: seq + required: no + sequence: + - type: str From aa39dc6013b9e22291ac9809e1c24d5746fbc3d3 Mon Sep 17 00:00:00 2001 From: Anas Nashif Date: Thu, 21 May 2020 10:35:33 -0400 Subject: [PATCH 4/6] sanitycheck: add fixture support to hardware map It is now possible to add a list of fixture a platform supports which is matched to testcases requesting fixtures to be avaiable to be able to run. For example: - available: true connected: true id: 0240000026334e450015400f5e0e000b4eb1000097969900 platform: frdm_k64f product: DAPLink CMSIS-DAP runner: pyocd serial: /dev/ttyACM9 fixtures: - gpio_loopback Fixes #24943 Signed-off-by: Anas Nashif --- scripts/sanity_chk/sanitylib.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/scripts/sanity_chk/sanitylib.py b/scripts/sanity_chk/sanitylib.py index 6f6fd5420b9b..a2b243253c4d 100644 --- a/scripts/sanity_chk/sanitylib.py +++ b/scripts/sanity_chk/sanitylib.py @@ -504,14 +504,19 @@ def monitor_serial(self, ser, halt_fileno, harness): log_out_fp.close() - def device_is_available(self, device): + def device_is_available(self, instance): + device = instance.platform.name + fixture = instance.testcase.harness_config.get("fixture") for i in self.suite.connected_hardware: + if fixture and fixture not in i.get('fixtures', []): + continue if i['platform'] == device and i['available'] and i['serial']: return True return False - def get_available_device(self, device): + def get_available_device(self, instance): + device = instance.platform.name for i in self.suite.connected_hardware: if i['platform'] == device and i['available'] and i['serial']: i['available'] = False @@ -559,13 +564,14 @@ def handle(self): else: command = [self.generator_cmd, "-C", self.build_dir, "flash"] - while not self.device_is_available(self.instance.platform.name): + while not self.device_is_available(self.instance): logger.debug("Waiting for device {} to become available".format(self.instance.platform.name)) time.sleep(1) - hardware = self.get_available_device(self.instance.platform.name) + hardware = self.get_available_device(self.instance) - runner = hardware.get('runner', None) + if hardware: + runner = hardware.get('runner', None) if runner: board_id = hardware.get("probe_id", hardware.get("id", None)) product = hardware.get("product", None) @@ -1583,6 +1589,7 @@ def check_build_or_run(self, build_only=False, enable_slow=False, device_testing _build_only = True else: _build_only = False + elif self.testcase.harness: _build_only = True else: @@ -2643,6 +2650,14 @@ def apply_filters(self, **kwargs): self.device_testing, self.fixtures ) + + if device_testing_filter: + for h in self.connected_hardware: + if h['platform'] == plat.name: + if tc.harness_config.get('fixture') in h.get('fixtures', []): + instance.build_only = False + instance.run = True + if not force_platform and plat.name in exclude_platform: discards[instance] = "Platform is excluded on command line." continue From 9f8a7d4d9d46881293b468420fa66a99df5dae1d Mon Sep 17 00:00:00 2001 From: Anas Nashif Date: Thu, 21 May 2020 10:35:57 -0400 Subject: [PATCH 5/6] tests: gpio_basic: this test requires a fixture This test requires a fixture to be installed, in this case a wire connecting two GPIO pins. Signed-off-by: Anas Nashif --- tests/drivers/gpio/gpio_basic_api/testcase.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/drivers/gpio/gpio_basic_api/testcase.yaml b/tests/drivers/gpio/gpio_basic_api/testcase.yaml index d9e07999fa40..4144055f62a9 100644 --- a/tests/drivers/gpio/gpio_basic_api/testcase.yaml +++ b/tests/drivers/gpio/gpio_basic_api/testcase.yaml @@ -4,3 +4,6 @@ tests: depends_on: gpio min_flash: 34 filter: dt_compat_enabled("test,gpio_basic_api") + harness: ztest + harness_config: + fixture: gpio_loopback From 66199af04b74f6848134d99eae67ce65efdf47f3 Mon Sep 17 00:00:00 2001 From: Anas Nashif Date: Fri, 22 May 2020 06:53:27 -0400 Subject: [PATCH 6/6] doc: sanitycheck: fixture documentation Add documentation about fixtures and how to use them in the hardware map. Signed-off-by: Anas Nashif --- doc/guides/test/fixtures.svg | 3 +++ doc/guides/test/sanitycheck.rst | 37 +++++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 doc/guides/test/fixtures.svg diff --git a/doc/guides/test/fixtures.svg b/doc/guides/test/fixtures.svg new file mode 100644 index 000000000000..a3f950f1b233 --- /dev/null +++ b/doc/guides/test/fixtures.svg @@ -0,0 +1,3 @@ + + +
Board
Board
sensor XYZ
sensor XYZ
Testcase
Testcase
harness: console
harness_config:
     fixture: sensor_xyz
harness: console...
Hardware Map

...
- available: true
  connected: true
  fixtures:
    - sensor_xyz
  id: 123456
  platform: frdm_k64f
  product: DAPLink CMSIS-DAP
  runner: pyocd
  serial: /dev/ttyACM9

....
Hardware Map...Viewer does not support full SVG 1.1 \ No newline at end of file diff --git a/doc/guides/test/sanitycheck.rst b/doc/guides/test/sanitycheck.rst index 28a7dbd678db..573e44ac94d8 100644 --- a/doc/guides/test/sanitycheck.rst +++ b/doc/guides/test/sanitycheck.rst @@ -354,8 +354,9 @@ harness_config: specific test setup and board selection logic to pick the particular board(s) out of multiple boards that fulfill the dependency in an automation setup based on "fixture" keyword. Some sample fixture names - are fixture_i2c_hts221, fixture_i2c_bme280, fixture_i2c_FRAM, - fixture_ble_fw and fixture_gpio_loop. + are i2c_hts221, i2c_bme280, i2c_FRAM, ble_fw and gpio_loop. + + Only one fixture can be defined per testcase. The following is an example yaml file with a few harness_config options. @@ -372,7 +373,7 @@ harness_config: regex: - "Temperature:(.*)C" - "Relative Humidity:(.*)%" - fixture: fixture_i2c_hts221 + fixture: i2c_hts221 tests: test: tags: sensors @@ -545,6 +546,30 @@ on those platforms. with the hardware map features. Boards that require other runners to flash the Zephyr binary are still work in progress. -To produce test reports, use the ``--detailed-report FILENAME`` option which will -generate an XML file using the JUNIT syntax. This file can be used to generate -other reports, for example using ``junit2html`` which can be installed via PIP. +Fixtures ++++++++++ + +Some tests require additional setup or special wiring specific to the test. +Running the tests without this setup or test fixture may fail. A testcase can +specify the fixture it needs which can then be matched with hardware capability +of a board and the fixtures it supports via the command line or using the hardware +map file. + +Fixtures are defined in the hardware map file as a list:: + + - available: true + connected: true + fixtures: + - gpio_loopback + id: 0240000026334e450015400f5e0e000b4eb1000097969900 + platform: frdm_k64f + product: DAPLink CMSIS-DAP + runner: pyocd + serial: /dev/ttyACM9 + +When running `sanitycheck` with ``--device-testing``, the configured fixture +in the hardware map file will be matched to testcases requesting the same fixtures +and these tests will be executed on the boards that provide this fixture. + +.. figure:: fixtures.svg + :figclass: align-center