Add a parser for CSA XML data models (project-chip#29888)

* Move xml to say zapxml since that is the format

* Start defining a data_model_xml parser (no functionality for now)

* Start adding some basic support for data model xml parsing

* make the DM parser  executable

* Start having the ability to parse clusters

* More updates, we seem to have parsing for features

* Hard-code global attributes

* Remove some comments

* Add enumeration handling

* Add bitmap handling

* Restyle

* Parse structs

* Re-organize parsing a bit

* Make a linter happy

* Another linter fix

* Handling of events

* More handling and logic on events

* Add support for access privilege parsing

* XMLs have maybe invalid enum entries. Handle them gracefully

* More attribute handling updates

* restyle

* Support deprecate constraint

* Support constraint decoding, apply on attributes for now

* Restyle

* Restyle

* Field handling

* Field handling

* Some bug fixing

* Start adding command handling

* Restyle

* Name normalization and more parsing updates

* Name normalization and more parsing updates

* Better messaging and fix constraint types

* Restyle

* Start creating a IDL codegen so we can self-test parsed output

* Start with listing clusters

* Enum listing

* A lot more things supported

* Attribute rendering

* Support for string and octet string sizes

* Timed command support

* Restyle

* Add descriptions to clusters

* Attempt to fix up alignment of things

* Alignment looks slightly better

* Better command separation

* Align comments

* Align and output descriptions including clusters

* More work regarding loop structures

* Apply hex formatting to bitmaps. output now seems identical except one whitespace change

* Identical output for now

* Support API maturity. Notice that doccomments are lost on maturity :(

* Fix doxygen parsing for api maturity at the cluster level

* Restyle

* Support endpoints, although that is not 1:1 as hex encoding and ordering for events is lost

* Restyle

* Add todo note that default value does not string escaping

* Default rendering and add to files

* More updates on file dependencies

* Unit test IDL generator

* Add the IDL unit test as a standard unit test

* Update for python compatibility

* Fix unit testing of builds when GSDK root is defined

* Added a readme file

* Restyle

* Make xml parser use the idl codegen

* Restyle

* look to fix misspell warnings

* Undo repo update

* Fix linter errors

* Codegen as idl for data_model_xml_parser.py

* Parsing closer to matter idl content

* more normalization and type processing

* More mandatory conformance logic

* Fix mandatory conditionals

* Make unit test pass

* Fix tests a bit more and make parsers better

* Restyle

* Ignore min/max values while parsing xmls, even though raw data internally contains them

* Restyle

* Fix space after click annotations

* Compare support for human reviews

* Restyle

* Fix slash

* Undo submodule change

* fix xml to zapxml naming changes

* Restyle

* Update dates from 2022 to 2023

* Add note about the complex test input

* Remove unused imports

* Restyle

* Add some commends based on code review

* Add heuristic for setting enum and bitmap sizes, to make output from XML much more readable

* Add support for timed and fabric scoped commands

* Add missing import

---------

Co-authored-by: Andrei Litvin <andreilitvin@google.com>

.github/workflows/tests.yaml

@@ -85,7 +85,7 @@ jobs:
  #
  run: |
  ./scripts/run_in_build_env.sh \
- "./scripts/py_matter_idl/matter_idl/xml_parser.py \
+ "./scripts/py_matter_idl/matter_idl/zapxml_parser.py \
  --no-print \
  --log-level info \
  src/app/zap-templates/zcl/data-model/chip/global-attributes.xml \

scripts/py_matter_idl/BUILD.gn

@@ -67,7 +67,7 @@ pw_python_package("matter_idl") {
  "matter_idl/test_matter_idl_parser.py",
  "matter_idl/test_generators.py",
  "matter_idl/test_idl_generator.py",
- "matter_idl/test_xml_parser.py",
+ "matter_idl/test_zapxml.py",
  ]
 
  # TODO: at a future time consider enabling all (* or missing) here to get

scripts/py_matter_idl/files.gni

@@ -23,6 +23,12 @@ matter_idl_generator_templates = [
 matter_idl_generator_sources = [
  "${chip_root}/scripts/py_matter_idl/matter_idl/__init__.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/backwards_compatibility.py",
+ "${chip_root}/scripts/py_matter_idl/matter_idl/data_model_xml/__init__.py",
+ "${chip_root}/scripts/py_matter_idl/matter_idl/data_model_xml/handlers/__init__.py",
+ "${chip_root}/scripts/py_matter_idl/matter_idl/data_model_xml/handlers/base.py",
+ "${chip_root}/scripts/py_matter_idl/matter_idl/data_model_xml/handlers/context.py",
+ "${chip_root}/scripts/py_matter_idl/matter_idl/data_model_xml/handlers/handlers.py",
+ "${chip_root}/scripts/py_matter_idl/matter_idl/data_model_xml/handlers/parsing.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/__init__.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/__init__.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/generators/cpp/application/__init__.py",
@@ -38,16 +44,17 @@ matter_idl_generator_sources = [
  "${chip_root}/scripts/py_matter_idl/matter_idl/matter_idl_parser.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/matter_idl_types.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/test_backwards_compatibility.py",
+ "${chip_root}/scripts/py_matter_idl/matter_idl/test_data_model_xml.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/test_generators.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/test_matter_idl_parser.py",
- "${chip_root}/scripts/py_matter_idl/matter_idl/test_xml_parser.py",
- "${chip_root}/scripts/py_matter_idl/matter_idl/xml_parser.py",
+ "${chip_root}/scripts/py_matter_idl/matter_idl/test_zapxml.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/__init__.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/handlers/__init__.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/handlers/base.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/handlers/context.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/handlers/handlers.py",
  "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml/handlers/parsing.py",
+ "${chip_root}/scripts/py_matter_idl/matter_idl/zapxml_parser.py",
 ]
 
 # All the files that the matter idl infrastructure will use

scripts/py_matter_idl/matter_idl/data_model_xml/__init__.py

@@ -0,0 +1,128 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import typing
+import xml.sax.handler
+from dataclasses import dataclass
+from typing import List, Optional, Union
+
+from matter_idl.data_model_xml.handlers import Context, DataModelXmlHandler
+from matter_idl.matter_idl_types import Idl
+
+
+class ParseHandler(xml.sax.handler.ContentHandler):
+ """A parser for data model XML data definitions.
+
+ Defers its processing to DataModelXmlHandler and keeps track of:
+ - an internal context for all handlers
+ - the parsed Idl structure that is incrementally built
+ - sets up parsing location within the context
+ - keeps track of ParsePath
+
+ Overall converts a python SAX handler into matter_idl.zapxml.handlers
+ """
+
+ def __init__(self, include_meta_data=True):
+ super().__init__()
+ self._idl = Idl()
+ self._processing_stack = []
+ # Context persists across all
+ self._context = Context()
+ self._include_meta_data = include_meta_data
+ self._locator = None
+
+ def PrepareParsing(self, filename):
+ # This is a bit ugly: filename keeps changing during parse
+ # IDL meta is not prepared for this (as source is XML and .matter is
+ # single file)
+ if self._include_meta_data:
+ self._idl.parse_file_name = filename
+
+ self._context.file_name = filename
+
+ def Finish(self) -> Idl:
+ self._context.PostProcess(self._idl)
+ return self._idl
+
+ def startDocument(self):
+ if self._include_meta_data and self._locator:
+ self._context.locator = self._locator
+ self._processing_stack = [
+ DataModelXmlHandler(self._context, self._idl)]
+
+ def endDocument(self):
+ if len(self._processing_stack) != 1:
+ raise Exception("Unexpected nesting!")
+
+ def startElement(self, name: str, attrs):
+ logging.debug("ELEMENT START: %r / %r" % (name, attrs))
+ self._context.path.push(name)
+ self._processing_stack.append(
+ self._processing_stack[-1].GetNextProcessor(name, attrs))
+
+ def endElement(self, name: str):
+ logging.debug("ELEMENT END: %r" % name)
+
+ last = self._processing_stack.pop()
+ last.EndProcessing()
+
+ # important to pop AFTER processing end to allow processing
+ # end to access the current context
+ self._context.path.pop()
+
+ def characters(self, content):
+ self._processing_stack[-1].HandleContent(content)
+
+
+@dataclass
+class ParseSource:
+ """Represents an input sopurce for ParseXmls.
+
+ Allows for named data sources to be parsed.
+ """
+ source: Union[str, typing.IO] # filename or stream
+ # actual filename to use, None if the source is a filename already
+ name: Optional[str] = None
+
+ @ property
+ def source_file_name(self):
+ if self.name:
+ return self.name
+ return self.source # assume string
+
+
+def ParseXmls(sources: List[ParseSource], include_meta_data=True) -> Idl:
+ """Parse one or more XML inputs and return the resulting Idl data.
+
+ Params:
+ sources - what to parse
+ include_meta_data - if parsing location data should be included in the Idl
+ """
+ handler = ParseHandler(include_meta_data=include_meta_data)
+
+ for source in sources:
+ logging.info('Parsing %s...' % source.source_file_name)
+ handler.PrepareParsing(source.source_file_name)
+
+ parser = xml.sax.make_parser()
+ parser.setContentHandler(handler)
+ try:
+ parser.parse(source.source)
+ except AssertionError as e:
+ logging.error("AssertionError %s at %r", e,
+ handler._context.GetCurrentLocationMeta())
+ raise
+
+ return handler.Finish()

scripts/py_matter_idl/matter_idl/data_model_xml/handlers/__init__.py

@@ -0,0 +1,34 @@
+# Copyright (c) 2023 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from matter_idl.matter_idl_types import Idl
+
+from .base import BaseHandler
+from .context import Context
+from .handlers import ClusterHandler
+
+
+class DataModelXmlHandler(BaseHandler):
+ """Handles the top level (/) of a data model xml file
+ """
+
+ def __init__(self, context: Context, idl: Idl):
+ super().__init__(context)
+ self._idl = idl
+
+ def GetNextProcessor(self, name, attrs):
+ if name.lower() == 'cluster':
+ return ClusterHandler(self.context, self._idl, attrs)
+ else:
+ return BaseHandler(self.context)

scripts/py_matter_idl/matter_idl/data_model_xml/handlers/base.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2023 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import enum
+
+from .context import Context
+
+
+class HandledDepth:
+ """Defines how deep a XML element has been handled."""
+ NOT_HANDLED = enum.auto() # Unknown/parsed element
+ ENTIRE_TREE = enum.auto() # Entire tree can be ignored
+ SINGLE_TAG = enum.auto() # Single tag processed, but not sub-items
+
+
+class BaseHandler:
+ """A generic element handler.
+
+ XML processing is done in the form of depth-first processing:
+ - Tree is descended into using `GetNextProcessor`
+ - Processors are expected to extend `BaseHandler` and allow for:
+ - GetNextProcessor to recurse
+ - HandleContent in case the text content is relevant
+ - EndProcessing once the entire tree has been walked (when xml element ends)
+
+ BaseHandler keeps track if it has been handled or ot by its `_handled` setting and
+ init parameter. Non-handled elements will be tagged within the context, resulting
+ in logs. This is to detect if unknown/new tags appear in XML files.
+ """
+
+ def __init__(self, context: Context, handled=HandledDepth.NOT_HANDLED):
+ self.context = context
+ self._handled = handled
+
+ def GetNextProcessor(self, name, attrs):
+ """Get the next processor to use for the given name"""
+
+ if self._handled == HandledDepth.SINGLE_TAG:
+ handled = HandledDepth.NOT_HANDLED
+ else:
+ handled = self._handled
+
+ return BaseHandler(context=self.context, handled=handled)
+
+ def HandleContent(self, content):
+ """Processes some content"""
+ pass
+
+ def EndProcessing(self):
+ """Finalizes the processing of the current element"""
+ if self._handled == HandledDepth.NOT_HANDLED:
+ self.context.MarkTagNotHandled()