diff --git a/CHANGELOG.md b/CHANGELOG.md index c71310b34..cab4d55af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,41 @@ This Changelog can be updated by calling: invoke changelog ``` -## Version 0.18.0 (2022-06-07) +## Version Unreleased (2022-07-11) #### New Features +* agnostic tags +* make DuT DT-aux capable ([#77](https://github.com/eclipse/kiso-testing/issues/77)) +* Implement double threaded auxiliary interface ([#74](https://github.com/eclipse/kiso-testing/issues/74)) +* detect test collection errors and abort execution +* add show tag script for test information analysis +#### Fixes + +* multiple yaml logging +#### Refactorings + +* adapt acroname aux to dt interface ([#84](https://github.com/eclipse/kiso-testing/issues/84)) +* migrate recorder to DT-aux interface ([#80](https://github.com/eclipse/kiso-testing/issues/80)) +#### Docs + +* add tools section and remove tools from coverage +#### Others + +* rename show-tag CLI to pykiso-tags +* fix docstring +* fix issues induced by tmp_path fixture +* try to fix get_yaml_files test +* increase coverage +* add tests for show-tag + +Full set of changes: [`0.18.0...0ee4fb2`](https://github.com/eclipse/kiso-testing/compare/0.18.0...0ee4fb2) + +## Version 0.18.0 (2022-06-08) + +#### New Features + +* make release 0.18.0 ([#72](https://github.com/eclipse/kiso-testing/issues/72)) * create base class for test ([#61](https://github.com/eclipse/kiso-testing/issues/61)) * make the proxy agnostic of transitioning messages * add pykiso to pytest tool ([#62](https://github.com/eclipse/kiso-testing/issues/62)) diff --git a/README.md b/README.md index 644726c4a..76a025abf 100644 --- a/README.md +++ b/README.md @@ -114,9 +114,10 @@ Once installed the application is bound to `pykiso`, it can be called with the f ```bash Usage: pykiso [OPTIONS] [PATTERN] - Embedded Integration Test Framework - CLI Entry Point. + TAG Filters: Just add pairs of arguments to the pykiso call. Multiple values + for a key must be seperate via a comma. - PATTERN: overwrite the test filter pattern from the YAML file (optional) + For example: pykiso -c your_config.yaml --branch dev,master --variant delta Options: -c, --test-configuration-file FILE @@ -130,10 +131,6 @@ Options: set the verbosity of the logging --version Show the version and exit. -h, --help Show this message and exit. - --variant allow the user to execute a subset of tests - based on variants - --branch-level allow the user to execute a subset of tests - based on branch levels --junit enables the generation of a junit report --text default, test results are only displayed in the console diff --git a/docs/getting_started/config_file.rst b/docs/getting_started/config_file.rst index 77b465672..7fc2f3ec8 100644 --- a/docs/getting_started/config_file.rst +++ b/docs/getting_started/config_file.rst @@ -85,11 +85,11 @@ The test suite definition is a list of key-value pairs. Each test suite consists of a `test_suite_id`, a `suite_dir` and a `test_filter_pattern`. For fast test development, the `test_filter_pattern` can be overwritten from the command line -in order to e.g. execute a single test file inside the `suite_dir` using the CLI argument `PATTERN`: +in order to e.g. execute a single test file inside the `suite_dir` using the CLI argument `-p` or `--pattern`: ..code:: bash - pykiso -c dummy.yaml test_suite_1.py + pykiso -c dummy.yaml -p test_suite_1.py Requirements specification diff --git a/docs/introduction/integration_test.rst b/docs/introduction/integration_test.rst index 8e5147177..dc7c07896 100644 --- a/docs/introduction/integration_test.rst +++ b/docs/introduction/integration_test.rst @@ -184,7 +184,7 @@ an additional reference can be added into the test_run decorator: In order to run only a subset of tests, an additional reference can be added to the test_run decorator: -- variant : optional test variants can be defined like: +- tag : [optional] the variant can be defined like: .. code:: python @@ -193,12 +193,27 @@ In order to run only a subset of tests, an additional reference can be added to Both parameters (variant/branch_level), will play the role of filter to fine tune the test collection and at the end ensure the execution of very specific tests subset. -.. note:: branch_level parameter is also part of the CLI and both (variant/branch_level) - accept multiple values. +.. note:: cli tags must be given in pairs. If one key has multiple values seperate them with a comma .. code:: bash - pykiso -c configuration_file --variant var1 --variant var2 --branch-level daily --branch-level nightly + pykiso -c configuration_file --variant var1,var2 --branch-level daily,nightly + +.. table:: Exectuion table for test case tags and cli tag arguments + :widths: auto + + ======================================================= =============================== ======== + test case tags cli tags executed + ======================================================= =============================== ======== + "branch_level": ["daily","nightly"] branch_level nightly 🗸 + "branch_level": ["daily","nightly"] branch_level nightly,daily 🗸 + "branch_level": ["daily","nightly"] branch_level master ✗ + "branch_level": ["daily","nightly"],"variant":["var1"] branch_level nightly 🗸 + "branch_level": ["daily","nightly"],"variant":["var1"] variant var1 🗸 + "branch_level": ["daily","nightly"],"variant":["var1"] variant var2 ✗ + "branch_level": ["daily","nightly"],"variant":["var1"] branch_level daily variant var1 ✗ + ======================================================= =============================== ======== + In order to utilise the SetUp/TearDown test-suite feature, users have to define a class inheriting from :py:class:`~pykiso.test_coordinator.test_suite.BasicTestSuiteSetup` or diff --git a/docs/whats_new/version_ongoing.rst b/docs/whats_new/version_ongoing.rst index 216892332..c92d202ef 100644 --- a/docs/whats_new/version_ongoing.rst +++ b/docs/whats_new/version_ongoing.rst @@ -52,3 +52,23 @@ The collected messages by the Communication auxiliary can still be cleared with :py:meth`~pykiso.lib.auxiliaries.communication_auxiliary.CommunicationAuxiliary.clear_buffer` See :ref:`communication_auxiliary` + +DUT Auxiliary adaption +^^^^^^^^^^^^^^^^^^^^^^ +refactor/redesign of the device under test auxiliary to fit with the brand new double +threaded auxiliary interface + +Record Auxiliary adaption +^^^^^^^^^^^^^^^^^^^^^^^^^ +adapt the record auxiliary to fit with the brand new double threaded auxiliary interface + +Acroname Auxiliary adaption +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +adapt the acroname auxiliary to fit with the brand new double threaded auxiliary interface + +Agnostic tag call +^^^^^^^^^^^^^^^^^ +Instead of having only the 2 tags "variant" and "branch_level" to select tests, users +can now set any tagname. + +See: :ref:`define_test_information` for more details. diff --git a/pyproject.toml b/pyproject.toml index 4717b1fe3..2c73f0ad4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,7 +99,7 @@ omit = [ ] [tool.coverage.report] -exclude_lines = ["pass", "def __repr__"] +exclude_lines = ["pass", "def __repr__" ,"pragma: no cover","if TYPE_CHECKING:"] [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/src/pykiso/cli.py b/src/pykiso/cli.py index 166667e3a..927227fe8 100644 --- a/src/pykiso/cli.py +++ b/src/pykiso/cli.py @@ -20,16 +20,17 @@ """ import collections -import itertools import logging import pprint import sys import time from pathlib import Path -from typing import NamedTuple, Optional, Tuple +from typing import Dict, List, NamedTuple, Optional, Tuple import click +from pykiso.test_coordinator.test_execution import ExitCode + from . import __version__ from .config_parser import parse_config from .global_config import Grabber @@ -39,6 +40,8 @@ LogOptions = collections.namedtuple("LogOptions", "log_path log_level report_type") +click.UsageError.exit_code = ExitCode.BAD_CLI_USAGE + # use to store the selected logging options log_options: Optional[NamedTuple] = None @@ -119,7 +122,47 @@ def get_logging_options() -> LogOptions: return log_options -@click.command(context_settings={"help_option_names": ["-h", "--help"]}) +def eval_user_tags(click_context: click.Context) -> Dict[str, List[str]]: + """Evaluate commandline args for user tags and raise exceptions for invalid + arguments. + + :param click_context: click context + :raises click.NoSuchOption: if key doesnt start with "--" or has an invalid + character like "_" + :raises click.BadOptionUsage: no value specfied for user tag + :return: user tags with values + """ + user_tags = {} + user_args = click_context.args.copy() + if not user_args: + return user_tags + while user_args: + try: + key = user_args.pop(0) + if not key.startswith("--") or "_" in key: + correct_key = ( + f'{"" if key.startswith("--") else "--" }{key.replace("_","-")}' + ) + raise click.NoSuchOption(option_name=key, possibilities=[correct_key]) + + value = user_args.pop(0) + user_tags[key[2:]] = value.split(",") + except IndexError: + raise click.BadOptionUsage( + option_name=key, + message=f"No value specified for tag {key}", + ctx=click_context, + ) + return user_tags + + +@click.command( + context_settings={ + "help_option_names": ["-h", "--help"], + "ignore_unknown_options": True, + "allow_extra_args": True, + } +) @click.option( "-c", "--test-configuration-file", @@ -161,43 +204,39 @@ def get_logging_options() -> LogOptions: default=True, help="default, test results are only displayed in the console", ) -@click.option( - "--variant", - multiple=True, - type=str, - default=None, - help="allow the user to execute a subset of tests based on variants", -) -@click.option( - "--branch-level", - multiple=True, - type=str, - default=None, - help="allow the user to execute a subset of tests based on branch levels", -) @click.option( "--failfast", is_flag=True, help="stop the test run on the first error or failure", ) -@click.argument("pattern", required=False) +@click.option( + "-p", + "--pattern", + type=click.STRING, + required=False, + help="test filter pattern, e.g. 'test_suite_1.py' or 'test_*.py'. It will be applied to all defined test suites.", +) @click.version_option(__version__) @Grabber.grab_cli_config +@click.pass_context def main( + click_context: click.Context, test_configuration_file: Tuple[PathType], log_path: PathType = None, log_level: str = "INFO", report_type: str = "text", - variant: Optional[tuple] = None, - branch_level: Optional[tuple] = None, pattern: Optional[str] = None, failfast: bool = False, ): """Embedded Integration Test Framework - CLI Entry Point. - PATTERN: overwrite the test filter pattern from the YAML file (optional) + TAG Filters: any additional option to be passed to the test as tag through + the pykiso call. Multiple values must be separated with a comma. + + For example: pykiso -c your_config.yaml --branch-level dev,master --variant delta \f + :param click_context: click context :param test_configuration_file: path to the YAML config file :param log_path: path to an existing directory or file to write logs to :param log_level: any of DEBUG, INFO, WARNING, ERROR @@ -218,8 +257,10 @@ def main( ConfigRegistry.register_aux_con(cfg_dict) + user_tags = eval_user_tags(click_context) + exit_code = test_execution.execute( - cfg_dict, report_type, variant, branch_level, pattern, failfast + cfg_dict, report_type, user_tags, pattern, failfast ) ConfigRegistry.delete_aux_con() for handler in logging.getLogger().handlers: diff --git a/src/pykiso/test_coordinator/test_execution.py b/src/pykiso/test_coordinator/test_execution.py index 4c3c267d4..e6636748e 100644 --- a/src/pykiso/test_coordinator/test_execution.py +++ b/src/pykiso/test_coordinator/test_execution.py @@ -23,8 +23,14 @@ 3. Loop per suite 4. Gather result """ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .test_case import BasicTest + import enum -import itertools import logging import time import unittest @@ -50,6 +56,7 @@ class ExitCode(enum.IntEnum): ONE_OR_MORE_TESTS_RAISED_UNEXPECTED_EXCEPTION = 2 ONE_OR_MORE_TESTS_FAILED_AND_RAISED_UNEXPECTED_EXCEPTION = 3 AUXILIARY_CREATION_FAILED = 4 + BAD_CLI_USAGE = 5 def create_test_suite( @@ -69,81 +76,51 @@ def create_test_suite( ) -def apply_variant_filter( - all_tests_to_run: dict, variants: tuple, branch_levels: tuple +def apply_tag_filter( + all_tests_to_run: unittest.TestSuite, usr_tags: Dict[str, List[str]] ) -> None: - """Filter the test cases based on the variant string. + """Filter the test cases based on user tags. :param all_tests_to_run: a dict containing all testsuites and testcases - :param variants: encapsulate user's variant choices - :param branch_levels: encapsulate user's branch level choices + :param usr_tags: encapsulate user's variant choices """ - def tag_present(tag_list: tuple, tc_tags: list) -> bool: - """Determine if at least branch_level or variant tag is present - at test fixture decorator level. - - :param tag_list: encapsulate user's variant choices - :param tc_tags: encapsulate all available tags on test case - decorator level + def is_skip_test(test_case: BasicTest) -> bool: + """Check if test shall be skipped by evaluating the test case tag + attribute - :return: True if a variant or branch_level is found - otherwise False + :param test_case: test_case to check + :return: True if test shall be skipped else False """ - for tag in tag_list: - if tag in tc_tags: + for tag_id, tag_value in usr_tags.items(): + if tag_id in test_case.tag.keys(): + items = tag_value if isinstance(tag_value, list) else [tag_value] + for item in items: + if item in test_case.tag[tag_id]: + continue + else: + return True + else: return True - else: - return False - - def both_present(variants: tuple, branches: tuple, tc_tags: list) -> bool: - """Determine if the couple branch_level and variant is present - at test fixture decorator level. + return False - :param variants: encapsulate user's variant choices - :param branches: encapsulate user's branch level choices - :param tc_tags: encapsulate all available tags on test case - decorator level + def set_skipped(test_case: BasicTest) -> None: + """Set testcase to skipped - :return: True if a couple variant/branch_level is found - otherwise False + :param test_case: testcase to be skipped """ - for variant, branch in itertools.product(variants, branches): - if variant in tc_tags and branch in tc_tags: - return True - else: - return False - - base_suite = test_suite.flatten(all_tests_to_run) - both_given = variants and branch_levels - - for tc in base_suite: - # if variant at decorator level is not given just run it - if tc.tag is not None: - # extract variant and branch from test fixture - tc_variants = tc.tag.get("variant", list()) - tc_branches = tc.tag.get("branch_level", list()) - tc_tags = list(itertools.chain(tc_variants, tc_branches)) - - # if user gives both branch and variant filter using couple - # (variant, branch_level) - if both_given and both_present(variants, branch_levels, tc_tags): - continue - # user gives only variant param, filter using only variant - elif not both_given and tag_present(variants, tc_tags): - continue - # user gives only branch param, filter using only branch - elif not both_given and tag_present(branch_levels, tc_tags): - continue - # the test is not intended to be run skip it - else: - tc.setUp = lambda: "setup_skipped" - setattr( - tc, - tc._testMethodName, - lambda: tc.skipTest("skipped due to non-matching variant value"), - ) - tc.tearDown = lambda: "tearDown_skipped" + test_case.setUp = lambda: "setup_skipped" + setattr( + test_case, + test_case._testMethodName, + lambda: test_case.skipTest("skipped due to non-matching variant value"), + ) + test_case.tearDown = lambda: "tearDown_skipped" + log.info(f"Skip test case: {test_case}") + + base_suite: List[BasicTest] = test_suite.flatten(all_tests_to_run) + + list(map(set_skipped, filter(is_skip_test, base_suite))) def failure_and_error_handling(result: unittest.TestResult) -> int: @@ -196,8 +173,7 @@ def collect_test_suites( def execute( config: Dict[str, Any], report_type: str = "text", - variants: Optional[tuple] = None, - branch_levels: Optional[tuple] = None, + user_tags: Optional[Dict[str, List[str]]] = None, pattern_inject: Optional[str] = None, failfast: bool = False, ) -> int: @@ -206,8 +182,7 @@ def execute( :param config: dict from converted YAML config file :param report_type: str to set the type of report wanted, i.e. test or junit - :param variants: encapsulate user's variant choices. - :param branch_levels: encapsulate user's branch level choices. + :param user_tags: test case tags to execute :param pattern_inject: optional pattern that will override test_filter_pattern for all suites. Used in test development to run specific tests. @@ -221,8 +196,8 @@ def execute( # Group all the collected test suites in one global test suite all_tests_to_run = unittest.TestSuite(test_suites) # filter test cases based on variant and branch-level options - if variants or branch_levels: - apply_variant_filter(all_tests_to_run, variants, branch_levels) + if user_tags: + apply_tag_filter(all_tests_to_run, user_tags) # TestRunner selection: generate or not a junit report. Start the tests and publish the results if report_type == "junit": junit_report_name = time.strftime("TEST-pykiso-%Y-%m-%d_%H-%M-%S.xml") diff --git a/tests/test_cli.py b/tests/test_cli.py index 030d7bddd..2ee805a1d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,9 +9,9 @@ import logging import os -import pathlib from pathlib import Path +import click import pytest from click.testing import CliRunner @@ -73,3 +73,60 @@ def test_main(runner): "examples/acroname.yaml", ], ) + + +@pytest.mark.parametrize( + "user_tags,expected_results", + [ + (["--branch-level", "dev"], {"branch-level": ["dev"]}), + (["--branch-level", "dev,master"], {"branch-level": ["dev", "master"]}), + ( + ["--branch-level", "dev", "--variant", "delta"], + {"branch-level": ["dev"], "variant": ["delta"]}, + ), + ], +) +def test_eval_user_tags(user_tags, expected_results, mocker): + click_context_mock = mocker.MagicMock() + click_context_mock.args = user_tags + user_tags = cli.eval_user_tags(click_context_mock) + assert user_tags == expected_results + + +def test_eval_user_tags_empty(mocker): + click_context_mock = mocker.MagicMock() + click_context_mock.args = [] + user_tags = cli.eval_user_tags(click_context_mock) + assert user_tags == {} + + +@pytest.mark.parametrize( + "user_tags,expected_message", + [ + ( + ["branch-level", "dev"], + "no such option: branch-level Did you mean --branch-level", + ), + ( + ["--forbidden_underscore", "dev"], + "no such option: --forbidden_underscore Did you mean --forbidden-underscore", + ), + ], +) +def test_eval_user_tags_exception_no_such_option(user_tags, expected_message, mocker): + # user_tags = ["branch-level", "dev"] + click_context_mock = mocker.MagicMock() + click_context_mock.args = user_tags + with pytest.raises(click.NoSuchOption) as exec_info: + eval_user_tags = cli.eval_user_tags(click_context_mock) + assert expected_message in exec_info.value.format_message() + + +def test_eval_user_tags_exception_bad_option_usage(mocker): + click_context_mock = mocker.MagicMock() + click_context_mock.args = ["--branch-level"] + with pytest.raises(click.BadOptionUsage) as exec_info: + eval_user_tags = cli.eval_user_tags(click_context_mock) + assert ( + "No value specified for tag --branch-level" in exec_info.value.format_message() + ) diff --git a/tests/test_config_registry_and_test_execution.py b/tests/test_config_registry_and_test_execution.py index 76d91f608..2f51549cc 100644 --- a/tests/test_config_registry_and_test_execution.py +++ b/tests/test_config_registry_and_test_execution.py @@ -61,6 +61,26 @@ def test_config_registry_and_test_execution_with_pattern(tmp_test, capsys): assert "Ran 0 tests" in output.err +@pytest.mark.parametrize("tmp_test", [("aux_1", "aux_2", False)], indirect=True) +def test_config_registry_and_test_execution_with_user_tags(tmp_test, mocker): + """Call execute function from test_execution using + configuration data coming from parse_config method + with additional user tags. + + Validation criteria: + - apply_filter called once + """ + apply_filter_mock = mocker.patch( + "pykiso.test_coordinator.test_execution.apply_tag_filter" + ) + user_tags = {"variant": ["delta"]} + cfg = parse_config(tmp_test) + ConfigRegistry.register_aux_con(cfg) + exit_code = test_execution.execute(cfg, user_tags=user_tags) + ConfigRegistry.delete_aux_con() + apply_filter_mock.assert_called_once() + + @pytest.mark.parametrize( "tmp_test", [("collector_error_aux1", "collector_error_aux_2", False)], @@ -91,6 +111,32 @@ def test_config_registry_test_collection_error(tmp_test, capsys, mocker, caplog) assert "Ran 0 tests" not in output.err +@pytest.mark.parametrize("tmp_test", [("aux1", "aux2", False)], indirect=True) +def test_config_registry_and_test_execution_collect_error(tmp_test, capsys, mocker): + """Call execute function from test_execution using + configuration data coming from parse_config method + by specifying a pattern + + Validation criteria: + - run is executed without error + """ + mocker.patch( + "pykiso.test_coordinator.test_suite.tc_sort_key", side_effect=Exception + ) + + cfg = parse_config(tmp_test) + ConfigRegistry.register_aux_con(cfg) + + with pytest.raises(pykiso.TestCollectionError): + test_execution.collect_test_suites(cfg["test_suite_list"]) + + ConfigRegistry.delete_aux_con() + + output = capsys.readouterr() + assert "FAIL" not in output.err + assert "Ran 0 tests" not in output.err + + @pytest.mark.parametrize( "tmp_test", [("collector_error_2_aux1", "collector_error_2_aux", False)], @@ -240,20 +286,20 @@ def test_config_registry_and_test_execution_failure_and_error_handling(): @pytest.mark.parametrize( - "m_tag ,tests_to_run, variants, branch_levels, expected_success", + "tc_tags ,cli_tags, is_test_running", [ - (None, {}, (), (), True), + (None, {}, True), ( { "variant": [ "omicron", ], "branch_level": [], - }, - {}, - ("omicron",), - (), - True, + }, # tc_tags + { + "variant": "omicron", + }, # cli_tags + True, # is_test_running ), ( { @@ -263,11 +309,11 @@ def test_config_registry_and_test_execution_failure_and_error_handling(): "branch_level": [ "leaf", ], - }, - {}, - ("omicron",), - ("leaf",), - True, + }, # tc_tags + { + "branch_level": "leaf", + }, # cli_tags + True, # is_test_running ), ( { @@ -277,113 +323,109 @@ def test_config_registry_and_test_execution_failure_and_error_handling(): "branch_level": [ "leaf", ], - }, - {}, - (), - ("leaf",), - True, + }, # tc_tags + { + "branch_level": "bud", + }, # cli_tags + False, # is_test_running ), ( { - "variant": [ - "omicron", - ], - "branch_level": [ - "leaf", - ], - }, - {}, - ("omicron",), - (), - True, + "k1": ["v1", "v3"], + "k2": ["v2", "v5", "v6"], + }, # tc_tags + { + "k1": "v1", + "k2": "v2", + }, # cli_tags + True, # is_test_running ), ( { - "variant": [], - "branch_level": [ - "leaf", - ], - }, - {}, - (), - ("leaf",), - True, + "k1": ["v1", "v7"], + "k2": ["v2", "v5", "v6"], + }, # tc_tags + { + "k1": "v1", + "k2": "v2", + }, # cli_tags + True, # is_test_running ), ( { - "variant": [ - "omicron", - ], - "branch_level": [], - }, - {}, - (), - ("leaf",), - False, + "k1": ["v1", "v7"], + "k2": ["v2", "v5", "v6"], + }, # tc_tags + { + "k1": "v1", + "k2": "v10", + }, # cli_tags + False, # is_test_running ), ( { - "variant": [ - "omicron", - ], - "branch_level": [ - "leaf", - ], - }, - {}, - ("delta",), - ("feuille",), - False, + "k1": ["v1", "v3"], + "k2": ["v2", "v5", "v6"], + }, # tc_tags + { + "k1": "v1", + }, # cli_tags + True, # is_test_running ), ( { - "variant": [ - "omicron", - ], - "branch_level": [ - "leaf", - ], - }, - {}, - (), - ("feuille",), - False, + "k1": ["v1", "v7"], + "k2": ["v2", "v5", "v6"], + }, # tc_tags + { + "k1": "v1", + }, # cli_tags + True, # is_test_running ), ( { - "variant": [ - "omicron", - ], - "branch_level": [ - "leaf", - ], - }, - {}, - ("delta",), - (), - False, + "k1": ["v1", "v3"], + "k2": ["v2", "v5", "v6"], + }, # tc_tags + {"k1": ["v1", "v3"]}, # cli_tags + True, # is_test_running + ), + ( + { + "k1": ["v1", "v3"], + "k2": ["v2", "v5", "v6"], + }, # tc_tags + {"k1": ["v1", "v3"], "k2": ["v2", "v5"]}, # cli_tags + True, # is_test_running + ), + ( + { + "k1": ["v1", "v3"], + }, # tc_tags + {"k2": ["v2", "v5"]}, # cli_tags + False, # is_test_running ), ], ) def test_config_registry_and_test_execution_apply_variant_filter( - m_tag, tests_to_run, variants, branch_levels, expected_success, mocker + tc_tags, cli_tags, is_test_running, mocker ): mock_test_case = mocker.Mock() mock_test_case.setUp = None mock_test_case.tearDown = None mock_test_case.testMethodName = None mock_test_case.skipTest = lambda x: x # return input - mock_test_case.tag = m_tag + mock_test_case.tag = tc_tags mock_test_case._testMethodName = "testMethodName" mock = mocker.patch( "pykiso.test_coordinator.test_execution.test_suite.flatten", return_value=[mock_test_case], ) - test_execution.apply_variant_filter(tests_to_run, variants, branch_levels) + test_execution.apply_tag_filter({}, cli_tags) mock.assert_called_once() - if expected_success: + if is_test_running: assert mock_test_case.setUp is None assert mock_test_case.tearDown is None assert mock_test_case.testMethodName is None diff --git a/tests/test_message.py b/tests/test_message.py index b23ad62ad..1d67a51a9 100644 --- a/tests/test_message.py +++ b/tests/test_message.py @@ -176,6 +176,20 @@ def test_ack_message_gen_and_match(self): message_sent.check_if_ack_message_is_matching(ack_message_received) is True ) + def test_ack_message_gen_and_match_wrong_type(self): + # Create the messages + message_sent = Message( + msg_type=MessageType.COMMAND, + sub_type=MessageCommandType.TEST_CASE_SETUP, + test_suite=2, + test_case=3, + ) + ack_message_received = message_sent.generate_ack_message( + TlvKnownTags.TEST_REPORT + ) + # Parse and compare + self.assertIsNone(ack_message_received) + def test_get_crc(self): crc = Message.get_crc(b"@\x01\x00\x00\x00UU\x00", 2) assert b"\xc5\n" == struct.pack("H", crc)