Start of python YAML test parsers and executer (#23533)

* Start of python YAML test parsers and executer

Tested inside chip-repl with the following commands (with all-cluster
app running on separate terminal already commissioned):
  import chip.yaml
  foo = chip.yaml.Yaml.YamlTestParser("src/app/tests/suites/TestCluster.yaml")
  foo.ExecuteTests(devCtrl)

Co-authored-by: Jerry Johns <johnsj@google.com>

* Address PR comments

* Reduce line length to 100

* Address PR comments

* Address PR comments

* Fix minor nit docstring issue

* Address more PR comments, rename module names to adhere to guidelines

* Fix style

* Apparently cluster is not always set in 'config'

* Fix up chip-repl CI failure

Co-authored-by: Jerry Johns <johnsj@google.com>

src/controller/python/BUILD.gn

@@ -226,6 +226,11 @@ chip_python_wheel_action("chip-core") {
  "chip/storage/__init__.py",
  "chip/utils/CommissioningBuildingBlocks.py",
  "chip/utils/__init__.py",
+ "chip/yaml/__init__.py",
+ "chip/yaml/data_model_lookup.py",
+ "chip/yaml/errors.py",
+ "chip/yaml/format_converter.py",
+ "chip/yaml/parser.py",
  ]
 
  if (chip_controller) {
@@ -269,6 +274,7 @@ chip_python_wheel_action("chip-core") {
  "chip.internal",
  "chip.interaction_model",
  "chip.logging",
+ "chip.yaml",
  "chip.native",
  "chip.clusters",
  "chip.setup_payload",

src/controller/python/chip/yaml/__init__.py

@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2022 Project CHIP Authors
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# @file
+# Provides Python APIs for Matter.
+
+"""Provides yaml parser Python APIs for Matter."""
+from . import parser

src/controller/python/chip/yaml/data_model_lookup.py

@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2022 Project CHIP Authors
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from abc import ABC, abstractmethod
+import chip.clusters as Clusters
+
+
+class DataModelLookup(ABC):
+ @abstractmethod
+ def get_cluster(self, cluster: str):
+ pass
+
+ @abstractmethod
+ def get_command(self, cluster: str, command: str):
+ pass
+
+ @abstractmethod
+ def get_attribute(self, cluster: str, attribute: str):
+ pass
+
+
+class PreDefinedDataModelLookup(DataModelLookup):
+ def get_cluster(self, cluster: str):
+ try:
+ return getattr(Clusters, cluster, None)
+ except AttributeError:
+ return None
+
+ def get_command(self, cluster: str, command: str):
+ try:
+ commands = getattr(Clusters, cluster, None).Commands
+ return getattr(commands, command, None)
+ except AttributeError:
+ return None
+
+ def get_attribute(self, cluster: str, attribute: str):
+ try:
+ attributes = getattr(Clusters, cluster, None).Attributes
+ return getattr(attributes, attribute, None)
+ except AttributeError:
+ return None

src/controller/python/chip/yaml/errors.py

@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2022 Project CHIP Authors
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+class ParsingError(ValueError):
+ def __init__(self, message):
+ super().__init__(message)
+
+
+class UnexpectedParsingError(ParsingError):
+ def __init__(self, message):
+ super().__init__(message)
+
+
+class ValidationError(Exception):
+ def __init__(self, message):
+ super().__init__(message)

src/controller/python/chip/yaml/format_converter.py

@@ -0,0 +1,126 @@
+#
+# Copyright (c) 2022 Project CHIP Authors
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import typing
+from chip.clusters.Types import Nullable, NullValue
+from chip.tlv import uint, float32
+import enum
+from chip.yaml.errors import ValidationError
+
+
+_HEX_PREFIX = 'hex:'
+
+
+def convert_name_value_pair_to_dict(arg_values):
+ ''' Fix yaml command arguments.
+
+ For some reason, instead of treating the entire data payload of a
+ command as a singular struct, the top-level args are specified as 'name'
+ and 'value' pairs, while the payload of each argument is itself
+ correctly encapsulated. This fixes up this oddity to create a new
+ key/value pair with the key being the value of the 'name' field, and
+ the value being 'value' field.
+ '''
+ ret_value = {}
+
+ for item in arg_values:
+ ret_value[item['name']] = item['value']
+
+ return ret_value
+
+
+def convert_yaml_type(field_value, field_type, use_from_dict=False):
+ ''' Converts yaml value to expected type.
+
+ The YAML representation when converted to a Python dictionary does not
+ quite line up in terms of type (see each of the specific if branches
+ below for the rationale for the necessary fix-ups). This function does
+ a fix-up given a field value (as present in the YAML) and its matching
+ cluster object type and returns it.
+ '''
+ origin = typing.get_origin(field_type)
+
+ if field_value is None:
+ field_value = NullValue
+
+ if (origin == typing.Union or origin == typing.Optional or origin == Nullable):
+ underlying_field_type = None
+
+ if field_value is NullValue:
+ for t in typing.get_args(field_type):
+ if t == Nullable:
+ return field_value
+
+ for t in typing.get_args(field_type):
+ # Comparison below explicitly not using 'isinstance' as that doesn't do what we want.
+ if t != Nullable and t != type(None):
+ underlying_field_type = t
+ break
+
+ if (underlying_field_type is None):
+ raise ValueError(f"Can't find the underling type for {field_type}")
+
+ field_type = underlying_field_type
+
+ # Dictionary represents a data model struct.
+ if (type(field_value) is dict):
+ return_field_value = {}
+ field_descriptors = field_type.descriptor
+ for item in field_value:
+ try:
+ # We search for a matching item in the list of field descriptors
+ # for this struct and ensure we can find a field with a matching
+ # label.
+ field_descriptor = next(
+ x for x in field_descriptors.Fields if x.Label.lower() ==
+ item.lower())
+ except StopIteration as exc:
+ raise ValidationError(
+ f'Did not find field "{item}" in {str(field_type)}') from None
+
+ return_field_value[field_descriptor.Label] = convert_yaml_type(
+ field_value[item], field_descriptor.Type, use_from_dict)
+ if use_from_dict:
+ return field_type.FromDict(return_field_value)
+ return return_field_value
+ elif(type(field_value) is float):
+ return float32(field_value)
+ # list represents a data model list
+ elif(type(field_value) is list):
+ list_element_type = typing.get_args(field_type)[0]
+
+ # The field type passed in is the type of the list element and not list[T].
+ for idx, item in enumerate(field_value):
+ field_value[idx] = convert_yaml_type(item, list_element_type, use_from_dict)
+ return field_value
+ # YAML conversion treats all numbers as ints. Convert to a uint type if the schema
+ # type indicates so.
+ elif (field_type == uint):
+ # Longer number are stored as strings. Need to make this conversion first.
+ value = int(field_value)
+ return field_type(value)
+ # YAML treats enums as ints. Convert to the typed enum class.
+ elif (issubclass(field_type, enum.Enum)):
+ return field_type(field_value)
+ # YAML treats bytes as strings. Convert to a byte string.
+ elif (field_type == bytes and type(field_value) != bytes):
+ if isinstance(field_value, str) and field_value.startswith(_HEX_PREFIX):
+ return bytes.fromhex(field_value[len(_HEX_PREFIX):])
+ return str.encode(field_value)
+ # By default, just return the field_value casted to field_type.
+ else:
+ return field_type(field_value)