Allow IDLs to define storage info (callbacks and default value) (#17787)

* Add a README.md entry for desirable storage access info

* Start adding unit tests (they will fail)

* Grammar update: put semicolor outside of struct field so we can inject attributes

* Grammar update: support traits

* Unit tests pass

* Restyle

* Add accessor method for isCallback

* Pretty print by default in the idl parser - much more viewable data

* Some more keywords can be case insensitive just in case

* remove strange comment tag - I do not believe this was ever used

* Add `persist` support

* Fix typo

* Restyle

* Update scripts/idl/README.md

Co-authored-by: Tennessee Carmel-Veilleux <tennessee.carmelveilleux@gmail.com>

Co-authored-by: Tennessee Carmel-Veilleux <tennessee.carmelveilleux@gmail.com>

scripts/idl/README.md

@@ -97,6 +97,24 @@ server cluster AccessControl = 31 {
  // attributes may be read-only as well
  readonly attribute int16u clusterRevision = 65533;
 
+ // Code-gen specific information for storage can be added as a keyword list
+ // after the attribute number. Specifically these are supported
+ // - `callback` is the equivalent of EXTERNAL in ember/zap, which means
+ // the value does not have a RAM backing store (use callbacks for get/set)
+ // - `persist` is the equivalent of NVM in ember/zap, which means
+ // the value will be persisted to storage when written (boot-time restores
+ // any set value)
+ // - `default` is supported to set a default value in RAM-based attribute store
+ //
+ // Not all combination of values are compatible. In particular:
+ // - `callback` is incompatible with `default` or `persist` as all value
+ // computation is deferred to the app.
+ readonly attribute int16u usingExternalAccess = 10 [callback];
+ readonly attribute int16u isPersisted = 10 [persist];
+ readonly attribute int16u hasDefaultValue = 11 [default=123];
+ readonly attribute char_string<16> defaultStringValue = 12 [default="abc"];
+
+
  // Commands have spec-defined numbers which are used for over-the-wire
  // invocation.
  //

scripts/idl/matter_grammar.lark

@@ -1,4 +1,4 @@
-struct: "struct"i id "{" struct_field* "}"
+struct: "struct"i id "{" (struct_field ";")* "}"
 enum: "enum"i id ":" type "{" constant_entry* "}"
 bitmap: "bitmap"i id ":" type "{" constant_entry* "}"
 
@@ -14,34 +14,42 @@ event_access: "access" "(" ("read" ":" access_privilege)? ")"
 
 event_with_access: "event" event_access? id
 
-event: event_priority event_with_access "=" number "{" struct_field* "}"
+event: event_priority event_with_access "=" number "{" (struct_field ";")* "}"
 
 ?event_priority: "critical"i -> critical_priority
  | "info"i -> info_priority
  | "debug"i -> debug_priority
 
 attribute_access_entry: attribute_access_type ":" access_privilege
 
-attribute_access: "access" "(" (attribute_access_entry ("," attribute_access_entry)* )? ")"
+default_value: "default"i "=" (number | ESCAPED_STRING)
+
+attribute_is_callback: "callback"i
+attribute_is_persist: "persist"i
+?single_attribute_trait: attribute_is_callback | attribute_is_persist | default_value
+
+attribute_traits: "[" single_attribute_trait ("," single_attribute_trait)* "]"
+
+attribute_access: "access"i "(" (attribute_access_entry ("," attribute_access_entry)* )? ")"
 
 attribute_with_access: attribute_access? struct_field
 
-attribute: attribute_tag* "attribute"i attribute_with_access
+attribute: attribute_tag* "attribute"i attribute_with_access attribute_traits? ";"
 attribute_tag: "readonly"i -> attr_readonly
  | "nosubscribe"i -> attr_nosubscribe
 
 request_struct: "request"i struct
 
 // Response structures must have a response id
-response_struct: "response"i "struct"i id "=" number "{" struct_field * "}"
+response_struct: "response"i "struct"i id "=" number "{" (struct_field ";") * "}"
 
 command_attribute: "timed"i -> timed_command
 command_attributes: command_attribute*
 
 
-command_access: "access" "(" ("invoke" ":" access_privilege)? ")"
+command_access: "access"i "(" ("invoke"i ":" access_privilege)? ")"
 
-command_with_access: "command" command_access? id
+command_with_access: "command"i command_access? id
 
 command: command_attributes command_with_access "(" id? ")" ":" id "=" number ";"
 
@@ -63,23 +71,21 @@ struct_field: member_attribute* field
 member_attribute: "optional"i -> optional
  | "nullable"i -> nullable
 
-field: data_type id list_marker? "=" number ";"
+field: data_type id list_marker? "=" number
 list_marker: "[" "]"
 
 data_type: type ("<" number ">")?
 
 id: ID
 type: ID
 
-COMMENT: "{" /(.|\n)+/ "}"
- | "//" /.*/
-
 POSITIVE_INTEGER: /\d+/ 
 HEX_INTEGER: /0x[A-Fa-f0-9]+/ 
 ID: /[a-zA-Z_][a-zA-Z0-9_]*/
 
 idl: (struct|enum|cluster|endpoint)*
 
+%import common.ESCAPED_STRING
 %import common.WS
 %import common.C_COMMENT
 %import common.CPP_COMMENT

scripts/idl/matter_idl_parser.py

@@ -15,6 +15,30 @@
  from matter_idl_types import *
 
 
+class AttributeTransformDefaultValue:
+ def __init__(self, value):
+ self.value = value
+
+ def __call__(self, attr):
+ attr.default = self.value
+
+
+class AttributeTransformSetCallback:
+ def __init__(self):
+ pass
+
+ def __call__(self, attr):
+ attr.tags.add(AttributeTag.CALLBACK)
+
+
+class AttributeTransformSetPersisted:
+ def __init__(self):
+ pass
+
+ def __call__(self, attr):
+ attr.tags.add(AttributeTag.PERSIST)
+
+
 class MatterIdlTransformer(Transformer):
  """
  A transformer capable to transform data parsed by Lark according to 
@@ -227,12 +251,39 @@ def attribute_with_access(self, args):
 
  return (args[-1], acl)
 
+ def ESCAPED_STRING(self, s):
+ # handle escapes, skip the start and end quotes
+ return s.value[1:-1].encode('utf-8').decode('unicode-escape')
+
+ @v_args(inline=True)
+ def default_value(self, value):
+ return AttributeTransformDefaultValue(value)
+
+ @v_args(inline=True)
+ def attribute_is_callback(self):
+ return AttributeTransformSetCallback()
+
+ @v_args(inline=True)
+ def attribute_is_persist(self):
+ return AttributeTransformSetPersisted()
+
+ def attribute_traits(self, traits):
+ # traits as is as a list
+ return traits
+
  def attribute(self, args):
  # Input arguments are:
- # - tags (0 or more)
- # - attribute_with_access (i.e. pair of definition and acl arguments)
- tags = set(args[:-1])
- (definition, acl) = args[-1]
+ # - tags (0 or more)
+ # - attribute_with_access (i.e. pair of definition and acl arguments)
+ # - attribute traits (last element)
+ if type(args[-1]) is tuple:
+ tags = set(args[:-1])
+ (definition, acl) = args[-1]
+ extra_attrs = []
+ else:
+ tags = set(args[:-2])
+ (definition, acl) = args[-2]
+ extra_attrs = args[-1]
 
  # until we support write only (and need a bit of a reshuffle)
  # if the 'attr_readonly == READABLE' is not in the list, we make things
@@ -241,7 +292,11 @@ def attribute(self, args):
  tags.add(AttributeTag.READABLE)
  tags.add(AttributeTag.WRITABLE)
 
- return Attribute(definition=definition, tags=tags, **acl)
+ attr = Attribute(definition=definition, tags=tags, **acl)
+ for f in extra_attrs:
+ f(attr)
+
+ return attr
 
  @v_args(inline=True)
  def struct(self, id, *fields):
@@ -326,6 +381,7 @@ def CreateParser():
  # The ability to run is for debug and to print out the parsed AST.
  import click
  import coloredlogs
+ import pprint
 
  # Supported log levels, mapping string values required for argument
  # parsing into logging constants
@@ -352,6 +408,6 @@ def main(log_level, filename=None):
  logging.info("Parse completed")
 
  logging.info("Data:")
- print(data)
+ pprint.pp(data)
 
  main()

scripts/idl/matter_idl_types.py

@@ -1,7 +1,7 @@
 import enum
 
 from dataclasses import dataclass, field
-from typing import List, Set, Optional
+from typing import List, Set, Optional, Union
 
 
 class FieldAttribute(enum.Enum):
@@ -17,6 +17,8 @@ class AttributeTag(enum.Enum):
  READABLE = enum.auto()
  WRITABLE = enum.auto()
  NOSUBSCRIBE = enum.auto()
+ CALLBACK = enum.auto()
+ PERSIST = enum.auto()
 
 
 class EventPriority(enum.Enum):
@@ -83,6 +85,7 @@ class Attribute:
  tags: Set[AttributeTag] = field(default_factory=set)
  readacl: AccessPrivilege = AccessPrivilege.VIEW
  writeacl: AccessPrivilege = AccessPrivilege.OPERATE
+ default: Optional[Union[str, int]] = None
 
  @property
  def is_readable(self):
@@ -96,6 +99,10 @@ def is_writable(self):
  def is_subscribable(self):
  return AttributeTag.NOSUBSCRIBE not in self.tags
 
+ @property
+ def is_callback(self):
+ return AttributeTag.CALLBACK in self.tags
+
 
 @dataclass
 class Struct:

scripts/idl/test_matter_idl_parser.py

@@ -180,6 +180,42 @@ def test_attribute_access(self):
  )])
  self.assertEqual(actual, expected)
 
+ def test_attribute_storage_info(self):
+ actual = parseText("""
+ server cluster MyCluster = 1 {
+ attribute char_string<11> attr1 = 1 [callback];
+ attribute char_string<33> attr2 = 2 [default="abc\\n with escapes: \\""];
+ attribute int32u withDefault = 3 [default=11];
+ attribute int32u intWithCallback = 4 [callback];
+ attribute int32u persisted = 5 [persist];
+ attribute int32u persisted_with_default = 6 [persist, default=55];
+ readonly attribute int32u readonlyDefault = 7 [default=321];
+ }
+ """)
+
+ expected = Idl(clusters=[
+ Cluster(side=ClusterSide.SERVER,
+ name="MyCluster",
+ code=1,
+ attributes=[
+ Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE, AttributeTag.CALLBACK]), definition=Field(
+ data_type=DataType(name="char_string", max_length=11), code=1, name="attr1")),
+ Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE]), definition=Field(
+ data_type=DataType(name="char_string", max_length=33), code=2, name="attr2"), default='abc\n with escapes: "'),
+ Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE]), definition=Field(
+ data_type=DataType(name="int32u"), code=3, name="withDefault"), default=11),
+ Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE, AttributeTag.CALLBACK]), definition=Field(
+ data_type=DataType(name="int32u"), code=4, name="intWithCallback")),
+ Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE, AttributeTag.PERSIST]), definition=Field(
+ data_type=DataType(name="int32u"), code=5, name="persisted")),
+ Attribute(tags=set([AttributeTag.READABLE, AttributeTag.WRITABLE, AttributeTag.PERSIST]), definition=Field(
+ data_type=DataType(name="int32u"), code=6, name="persisted_with_default"), default=55),
+ Attribute(tags=set([AttributeTag.READABLE]), definition=Field(
+ data_type=DataType(name="int32u"), code=7, name="readonlyDefault"), default=321),
+ ]
+ )])
+ self.assertEqual(actual, expected)
+
  def test_cluster_commands(self):
  actual = parseText("""
  server cluster WithCommands = 1 {