sanitycheck: improve fixture support and fix some bugs...

doc/guides/test/sanitycheck.rst

@@ -354,8 +354,9 @@ harness_config: <harness configuration options>
         specific test setup and board selection logic to pick the particular
         board(s) out of multiple boards that fulfill the dependency in an
         automation setup based on "fixture" keyword. Some sample fixture names
-        are fixture_i2c_hts221, fixture_i2c_bme280, fixture_i2c_FRAM,
-        fixture_ble_fw and fixture_gpio_loop.
+        are i2c_hts221, i2c_bme280, i2c_FRAM, ble_fw and gpio_loop.
+
+        Only one fixture can be defined per testcase.
 
     The following is an example yaml file with a few harness_config options.
 
@@ -372,7 +373,7 @@ harness_config: <harness configuration options>
              regex:
                - "Temperature:(.*)C"
                - "Relative Humidity:(.*)%"
-             fixture: fixture_i2c_hts221
+             fixture: i2c_hts221
          tests:
            test:
              tags: sensors
@@ -545,6 +546,30 @@ on those platforms.
   with the hardware map features. Boards that require other runners to flash the
   Zephyr binary are still work in progress.
 
-To produce test reports, use the ``--detailed-report FILENAME`` option which will
-generate an XML file using the JUNIT syntax. This file can be used to generate
-other reports, for example using ``junit2html`` which can be installed via PIP.
+Fixtures
++++++++++
+
+Some tests require additional setup or special wiring specific to the test.
+Running the tests without this setup or test fixture may fail. A testcase can
+specify the fixture it needs which can then be matched with hardware capability
+of a board and the fixtures it supports via the command line or using the hardware
+map file.
+
+Fixtures are defined in the hardware map file as a list::
+
+      - available: true
+        connected: true
+        fixtures:
+          - gpio_loopback
+        id: 0240000026334e450015400f5e0e000b4eb1000097969900
+        platform: frdm_k64f
+        product: DAPLink CMSIS-DAP
+        runner: pyocd
+        serial: /dev/ttyACM9
+
+When running `sanitycheck` with ``--device-testing``, the configured fixture
+in the hardware map file will be matched to testcases requesting the same fixtures
+and these tests will be executed on the boards that provide this fixture.
+
+.. figure:: fixtures.svg
+   :figclass: align-center

scripts/sanity_chk/harness.py

@@ -149,3 +149,6 @@ def handle(self, line):
             self.capture_coverage = False
 
         self.process_test(line)
+
+class Ztest(Test):
+    pass

scripts/sanity_chk/hwmap-schema.yaml

@@ -33,3 +33,8 @@ sequence:
       "pre_script":
         type: str
         required: false
+      "fixtures":
+        type: seq
+        required: no
+        sequence:
+          - type: str

scripts/sanity_chk/sanitylib.py

@@ -504,14 +504,19 @@ def monitor_serial(self, ser, halt_fileno, harness):
 
         log_out_fp.close()
 
-    def device_is_available(self, device):
+    def device_is_available(self, instance):
+        device = instance.platform.name
+        fixture = instance.testcase.harness_config.get("fixture")
         for i in self.suite.connected_hardware:
+            if fixture and fixture not in i.get('fixtures', []):
+                continue
             if i['platform'] == device and i['available'] and i['serial']:
                 return True
 
         return False
 
-    def get_available_device(self, device):
+    def get_available_device(self, instance):
+        device = instance.platform.name
         for i in self.suite.connected_hardware:
             if i['platform'] == device and i['available'] and i['serial']:
                 i['available'] = False
@@ -559,13 +564,14 @@ def handle(self):
         else:
             command = [self.generator_cmd, "-C", self.build_dir, "flash"]
 
-        while not self.device_is_available(self.instance.platform.name):
+        while not self.device_is_available(self.instance):
             logger.debug("Waiting for device {} to become available".format(self.instance.platform.name))
             time.sleep(1)
 
-        hardware = self.get_available_device(self.instance.platform.name)
+        hardware = self.get_available_device(self.instance)
 
-        runner = hardware.get('runner', None)
+        if hardware:
+            runner = hardware.get('runner', None)
         if runner:
             board_id = hardware.get("probe_id", hardware.get("id", None))
             product = hardware.get("product", None)
@@ -1533,7 +1539,7 @@ def __lt__(self, other):
         return self.name < other.name
 
     # Global testsuite parameters
-    def check_build_or_run(self, build_only=False, enable_slow=False, device_testing=False, fixture=[]):
+    def check_build_or_run(self, build_only=False, enable_slow=False, device_testing=False, fixtures=[]):
 
         # right now we only support building on windows. running is still work
         # in progress.
@@ -1571,18 +1577,19 @@ def check_build_or_run(self, build_only=False, enable_slow=False, device_testing
                 runnable = False
 
         # console harness allows us to run the test and capture data.
-        if self.testcase.harness == 'console':
+        if self.testcase.harness in [ 'console', 'ztest']:
 
             # if we have a fixture that is also being supplied on the
             # command-line, then we need to run the test, not just build it.
-            if "fixture" in self.testcase.harness_config:
-                fixture_cfg = self.testcase.harness_config['fixture']
-                if fixture_cfg in fixture:
+            fixture = self.testcase.harness_config.get('fixture')
+            if fixture:
+                if fixture in fixtures:
                     _build_only = False
                 else:
                     _build_only = True
             else:
                 _build_only = False
+
         elif self.testcase.harness:
             _build_only = True
         else:
@@ -2234,7 +2241,7 @@ def __init__(self, board_root_list=[], testcase_roots=[], outdir=None):
         self.cleanup = False
         self.enable_slow = False
         self.device_testing = False
-        self.fixture = []
+        self.fixtures = []
         self.enable_coverage = False
         self.enable_lsan = False
         self.enable_asan = False
@@ -2580,7 +2587,7 @@ def load_from_file(self, file, filter_status=[]):
                         self.build_only,
                         self.enable_slow,
                         self.device_testing,
-                        self.fixture
+                        self.fixtures
                     )
                     instance.create_overlay(platform, self.enable_asan, self.enable_coverage, self.coverage_platform)
                     instance_list.append(instance)
@@ -2641,8 +2648,16 @@ def apply_filters(self, **kwargs):
                     self.build_only,
                     self.enable_slow,
                     self.device_testing,
-                    self.fixture
+                    self.fixtures
                 )
+
+                if device_testing_filter:
+                    for h in self.connected_hardware:
+                        if h['platform'] == plat.name:
+                            if tc.harness_config.get('fixture') in h.get('fixtures', []):
+                                instance.build_only = False
+                                instance.run = True
+
                 if not force_platform and plat.name in exclude_platform:
                     discards[instance] = "Platform is excluded on command line."
                     continue

scripts/sanitycheck

@@ -774,7 +774,7 @@ def main():
     suite.test_only = options.test_only
     suite.enable_slow = options.enable_slow
     suite.device_testing = options.device_testing
-    suite.fixture = options.fixture
+    suite.fixtures = options.fixture
     suite.enable_asan = options.enable_asan
     suite.enable_lsan = options.enable_lsan
     suite.enable_coverage = options.enable_coverage

tests/drivers/counter/maxim_ds3231_api/testcase.yaml

@@ -1,8 +1,10 @@
 tests:
   drivers.counter.maxim_ds3231:
     tags: drivers
-    depends_on: counter
+    depends_on: counter i2c
     min_ram: 16
     timeout: 400
     filter: dt_compat_enabled("maxim,ds3231")
-    platform_whitelist: efr32mg_sltb004a frdm_k64f nrf51dk_nrf51422 nrf52840dk_nrf52840 nucleo_l476rg particle_xenon
+    harness: ztest
+    harness_config:
+      fixture: maxim_ds3231

tests/drivers/gpio/gpio_basic_api/testcase.yaml

@@ -4,3 +4,6 @@ tests:
     depends_on: gpio
     min_flash: 34
     filter: dt_compat_enabled("test,gpio_basic_api")
+    harness: ztest
+    harness_config:
+      fixture: gpio_loopback