Allow code pre-generation to accept files outside of chip-root (#25970)

* Start supporting separate root pregeneration

* Update documentation

* Restyle

* Fix typos

* Fix pregen logic

* Update path handling to support full and relative paths, codepregen of external roots now works

* Restyle

* Update docs/code_generation.md

Co-authored-by: Boris Zbarsky <bzbarsky@apple.com>

* Update docs/code_generation.md

Co-authored-by: Boris Zbarsky <bzbarsky@apple.com>

* Update docs/code_generation.md

Co-authored-by: Boris Zbarsky <bzbarsky@apple.com>

* Slight rephrase

* Another rephrase

* Restyle

* Normalize paths to not be off by one on generation path lenghts

---------

Co-authored-by: Andrei Litvin <andreilitvin@google.com>
Co-authored-by: Boris Zbarsky <bzbarsky@apple.com>

docs/code_generation.md

@@ -234,9 +234,48 @@ scripts/codepregen.py ${OUTPUT_DIRECTORY:-./zzz_pregenerated/}
 
 # To generate a single output you can use `--input-glob`:
 
-scripts/codepregen.py --input-glob "*all-clusters*" ${OUTPUT_DIRECTORY:-./zzz_pregenerated/}
+scripts/codepregen.py --input-glob "*all-clusters*" --input-glob "*controller*" ${OUTPUT_DIRECTORY:-./zzz_pregenerated/}
 ```
 
+### External applications/zap files
+
+#### Ensure you have a `.matter` file
+
+Code generation generally will use both `.zap` or `.matter` files. If you only
+have a `.zap` file, you can create the corresponding `.matter` file via:
+
+```bash
+scripts/tools/zap/generate.py ${ZAP_FILE_PATH}
+```
+
+The above will use the template `src/app/zap-templates/matter-idl.json` to
+generate a `.matter` file corresponding to the input `.zap` file.
+
+`.matter` files are designed to be human readable. It is recommended to take a
+look at the generated file and see if it contains what is expected and also lint
+it. If anything seems wrong, the `.zap` file should be fixed (`.matter`
+represents the content of `.zap`). To lint use:
+
+```bash
+scripts/idl_lint.py ${MATTER_FILE_PATH}
+```
+
+#### Running pre-generation
+
+If you have zap files outside the CHIP repository (i.e. not in `src` or
+`examples`) you should provide the root of your application source.
+
+```bash
+scripts/codepregen.py --external-root ${PATH_TO_SOURCE_ROOT} ${OUTPUT_DIRECTORY:-./zzz_pregenerated/}
+```
+
+NOTE: `$PATH_TO_SOURCE_ROOT` should be a top-level directory containing
+zap/matter files as the code pre-generation will generate files based on the
+path inside the root:
+
+- if files are `$PATH_TO_SOURCE_ROOT/some/path/foo.zap` this will generate
+ files into `$OUTPUT_DIRECTORY/some/path/foo/...`
+
 ### Using pre-generated code
 
 Instead of generating code at compile time, the chip build system accepts usage

scripts/codepregen.py

@@ -85,8 +85,13 @@ def _ParallelGenerateOne(arg):
  '--sdk-root',
  default=None,
  help='Path to the SDK root (where .zap/.matter files exist).')
+@click.option(
+ '--external-root',
+ default=None,
+ multiple=True,
+ help='Path to an external app root (where .zap/.matter files exist).')
 @click.argument('output_dir')
-def main(log_level, parallel, dry_run, generator, input_glob, sdk_root, output_dir):
+def main(log_level, parallel, dry_run, generator, input_glob, sdk_root, external_root, output_dir):
  if _has_coloredlogs:
  coloredlogs.install(level=__LOG_LEVELS__[
  log_level], fmt='%(asctime)s %(levelname)-7s %(message)s')
@@ -122,7 +127,7 @@ def main(log_level, parallel, dry_run, generator, input_glob, sdk_root, output_d
  elif generator == 'codegen':
  filter.file_type = IdlFileType.MATTER
 
- targets = FindPregenerationTargets(sdk_root, filter, runner)
+ targets = FindPregenerationTargets(sdk_root, external_root, filter, runner)
 
  runner.ensure_directory_exists(output_dir)
  if parallel:

scripts/pregenerate/__init__.py

@@ -25,7 +25,20 @@
 from .using_zap import ZapApplicationPregenerator
 
 
-def FindAllIdls(sdk_root: str) -> Iterator[InputIdlFile]:
+def _IdlsInDirectory(top_directory_name: str, truncate_length: int):
+ for root, dirs, files in os.walk(top_directory_name):
+ for file in files:
+ if file.endswith('.zap'):
+ yield InputIdlFile(file_type=IdlFileType.ZAP,
+ full_path=os.path.join(root, file),
+ relative_path=os.path.join(root[truncate_length:], file))
+ if file.endswith('.matter'):
+ yield InputIdlFile(file_type=IdlFileType.MATTER,
+ full_path=os.path.join(root, file),
+ relative_path=os.path.join(root[truncate_length:], file))
+
+
+def _FindAllIdls(sdk_root: str, external_roots: Optional[List[str]]) -> Iterator[InputIdlFile]:
  relevant_subdirs = [
  'examples', # all example apps
  'src', # realistically only controller/data_model
@@ -35,17 +48,19 @@ def FindAllIdls(sdk_root: str) -> Iterator[InputIdlFile]:
  sdk_root = sdk_root[:-1]
  sdk_root_length = len(sdk_root)
 
+ # first go over SDK items
  for subdir_name in relevant_subdirs:
  top_directory_name = os.path.join(sdk_root, subdir_name)
  logging.debug(f"Searching {top_directory_name}")
- for root, dirs, files in os.walk(top_directory_name):
- for file in files:
- if file.endswith('.zap'):
- yield InputIdlFile(file_type=IdlFileType.ZAP,
- relative_path=os.path.join(root[sdk_root_length+1:], file))
- if file.endswith('.matter'):
- yield InputIdlFile(file_type=IdlFileType.MATTER,
- relative_path=os.path.join(root[sdk_root_length+1:], file))
+ for idl in _IdlsInDirectory(top_directory_name, sdk_root_length+1):
+ yield idl
+
+ # next external roots
+ if external_roots:
+ for root in external_roots:
+ root = os.path.normpath(root)
+ for idl in _IdlsInDirectory(root, len(root) + 1):
+ yield idl
 
 
 @dataclass
@@ -67,7 +82,7 @@ def matches(self, s: str):
  return fnmatch.fnmatch(s, self.pattern)
 
 
-def FindPregenerationTargets(sdk_root: str, filter: TargetFilter, runner):
+def FindPregenerationTargets(sdk_root: str, external_roots: Optional[List[str]], filter: TargetFilter, runner):
  """Finds all relevand pre-generation targets in the given
  SDK root.
 
@@ -88,7 +103,7 @@ def FindPregenerationTargets(sdk_root: str, filter: TargetFilter, runner):
 
  path_matchers = [GlobMatcher(pattern) for pattern in filter.path_glob]
 
- for idl in FindAllIdls(sdk_root):
+ for idl in _FindAllIdls(sdk_root, external_roots):
  if filter.file_type is not None:
  if idl.file_type != filter.file_type:
  logging.debug(f"Will not process file of type {idl.file_type}: {idl.relative_path}")

scripts/pregenerate/types.py

@@ -26,6 +26,7 @@ class IdlFileType(Enum):
 class InputIdlFile:
  file_type: IdlFileType
  relative_path: str
+ full_path: str
 
  @property
  def pregen_subdir(self):

scripts/pregenerate/using_codegen.py

@@ -44,14 +44,14 @@ def Generate(self, output_root: str):
  output_root, self.idl.pregen_subdir, self.generator)
 
  logging.info(
- f"Generating: {self.generator}:{self.idl.relative_path} into {output_dir}")
+ f"Generating: {self.generator}:{self.idl.full_path} into {output_dir}")
 
  cmd = [
  CODEGEN_PY_PATH,
  '--log-level', 'fatal',
  '--generator', self.generator,
  '--output-dir', output_dir,
- os.path.join(self.sdk_root, self.idl.relative_path)
+ self.idl.full_path
  ]
 
  logging.debug(f"Executing {cmd}")

scripts/pregenerate/using_zap.py

@@ -56,16 +56,21 @@ def Generate(self, output_root: str):
 
  output_dir = os.path.join(output_root, self.idl.pregen_subdir, self.generation_type.subdir)
 
- logging.info(f"Generating: {self.generation_type}:{self.idl.relative_path} into {output_dir}")
+ logging.info(f"Generating: {self.generation_type}:{self.idl.full_path} into {output_dir}")
 
  self.runner.ensure_directory_exists(output_dir)
 
+ if self.idl.full_path.startswith(self.sdk_root):
+ idl_path = self.idl.relative_path
+ else:
+ idl_path = self.idl.full_path
+
  cmd = [
  ZAP_GENERATE_PATH,
  '--templates', self.generation_type.generation_template,
  '--output-dir', output_dir,
  '--parallel',
- self.idl.relative_path
+ idl_path
  ]
  logging.debug(f"Executing {cmd}")
  self.runner.run(cmd, cwd=self.sdk_root)