ESP32 - fix parallel build configuration (sdkconfig) and add ipv6 var…

…iant (#10494)

* Initial attempt to get esp32 to compile without IPV4

* Auto remove sdkconfig file

* Fix build logic

* Make ESP32 rebuild be able to regen

* Undo debug only assert in IPAddress

* Restyle fixes

* Update unit tests and make unit tests easier to update. Get rid of cwd directory

* Restyle fixes

* Fix unit test

* Also add devkitc ipv6 variant

* Fix unit test

config/esp32/components/chip/CMakeLists.txt

@@ -70,6 +70,10 @@ else()
 chip_gn_arg_append("chip_config_network_layer_ble"           "false")
 endif()
 
+if(CONFIG_DISABLE_IPV4)
+    chip_gn_arg_append("chip_inet_config_enable_ipv4"        "false")
+endif()
+
 if(CONFIG_ENABLE_PW_RPC)
     string(APPEND chip_gn_args "import(\"//build_overrides/pigweed.gni\")\n")
     chip_gn_arg_append("remove_default_configs"             "[\"//third_party/connectedhomeip/third_party/pigweed/repo/pw_build:cpp17\"]")

config/esp32/components/chip/Kconfig

@@ -97,6 +97,12 @@ menu "CHIP Core"
             help
                 Link the application against CHIP interactive shell.
 
+        config DISABLE_IPV4
+            bool "Disable IPv4 functionality in the CHIP stack"
+            default "n"
+            help
+                Matter spec is based on IPv6 communication only. Enabling this option may save some flash/ram.
+
         # TODO: add log level selection
 
     endmenu # "General Options"

config/esp32/components/chip/component.mk

@@ -147,6 +147,9 @@ endif
 	if [[ "$(CONFIG_USE_MINIMAL_MDNS)" = "n" ]]; then \
 	  echo "chip_mdns = platform" >> $(OUTPUT_DIR)/args.gn ;\
 	fi
+	if [[ "$(CONFIG_DISABLE_IPV4)" = "y" ]]; then \
+	  echo "chip_inet_config_enable_ipv4 = false" >> $(OUTPUT_DIR)/args.gn ;\
+	fi
 	echo "Written file $(OUTPUT_DIR)/args.gn"
 	cd $(CHIP_ROOT) && PW_ENVSETUP_QUIET=1 . scripts/activate.sh && cd $(COMPONENT_PATH) && gn gen --check --fail-on-unused-args $(OUTPUT_DIR)
 	cd $(COMPONENT_PATH); ninja $(subst 1,-v,$(filter 1,$(V))) -C $(OUTPUT_DIR) esp32

examples/all-clusters-app/esp32/main/Kconfig.projbuild

@@ -101,7 +101,6 @@ menu "Demo"
         help
             To reduce wear and heat the M5Stack's Display is automatically switched off after a few seconds
 
-
 endmenu
 
 menu "PW RPC Debug channel"

scripts/build/build/__init__.py

@@ -43,6 +43,8 @@ def Generate(self):
         if BuildSteps.GENERATED in self.completed_steps:
             return
 
+        self.runner.StartCommandExecution()
+
         for builder in self.builders:
             logging.info('Generating %s', builder.output_dir)
             builder.generate()

scripts/build/build/targets.py

@@ -88,20 +88,15 @@ def HostTargets():
 def Esp32Targets():
     esp32_target = Target('esp32', Esp32Builder)
 
+    yield esp32_target.Extend('m5stack-all-clusters', board=Esp32Board.M5Stack, app=Esp32App.ALL_CLUSTERS)
+    yield esp32_target.Extend('m5stack-all-clusters-ipv6only', board=Esp32Board.M5Stack, app=Esp32App.ALL_CLUSTERS, enable_ipv4=False)
+    yield esp32_target.Extend('m5stack-all-clusters-rpc', board=Esp32Board.M5Stack, app=Esp32App.ALL_CLUSTERS, enable_rpcs=True)
     yield esp32_target.Extend('c3devkit-all-clusters', board=Esp32Board.C3DevKit, app=Esp32App.ALL_CLUSTERS)
 
-    rpc_aware_targets = [
-        esp32_target.Extend('m5stack-all-clusters',
-                            board=Esp32Board.M5Stack, app=Esp32App.ALL_CLUSTERS)
-    ]
-
-    for target in rpc_aware_targets:
-        yield target
-        yield target.Extend('rpc', enable_rpcs=True)
-
     devkitc = esp32_target.Extend('devkitc', board=Esp32Board.DevKitC)
 
     yield devkitc.Extend('all-clusters', app=Esp32App.ALL_CLUSTERS)
+    yield devkitc.Extend('all-clusters-ipv6only', app=Esp32App.ALL_CLUSTERS, enable_ipv4=False)
     yield devkitc.Extend('shell', app=Esp32App.SHELL)
     yield devkitc.Extend('lock', app=Esp32App.LOCK)
     yield devkitc.Extend('bridge', app=Esp32App.BRIDGE)

scripts/build/build_examples.py

@@ -134,9 +134,9 @@ def main(context, log_level, target, target_glob, skip_target_glob, repo, out_pr
 """.strip())
 
     if dry_run:
-        runner = PrintOnlyRunner(dry_run_output)
+        runner = PrintOnlyRunner(dry_run_output, root=repo)
     else:
-        runner = ShellRunner()
+        runner = ShellRunner(root=repo)
 
     if 'all' in target:
         targets = build.ALL_TARGETS

scripts/build/builders/builder.py

@@ -90,8 +90,8 @@ def build(self):
         if self._enable_flashbundle:
             self._generate_flashbundle()
 
-    def _Execute(self, cmdarray, cwd=None, title=None):
-        self._runner.Run(cmdarray, cwd=cwd, title=title)
+    def _Execute(self, cmdarray, title=None):
+        self._runner.Run(cmdarray, title=title)
 
     def CompressArtifacts(self, target_file: str):
         with tarfile.open(target_file, "w:gz") as tar:

scripts/build/builders/esp32.py

@@ -70,8 +70,7 @@ def FlashBundleName(self):
 
 def DefaultsFileName(board: Esp32Board, app: Esp32App, enable_rpcs: bool):
     if app != Esp32App.ALL_CLUSTERS:
-        # only all-clusters has a specific defaults name
-        return None
+        return 'sdkconfig.defaults'
 
     rpc = "_rpc" if enable_rpcs else ""
     if board == Esp32Board.DevKitC:
@@ -91,41 +90,75 @@ def __init__(self,
                  runner,
                  board: Esp32Board = Esp32Board.M5Stack,
                  app: Esp32App = Esp32App.ALL_CLUSTERS,
-                 enable_rpcs: bool = False):
+                 enable_rpcs: bool = False,
+                 enable_ipv4: bool = True
+                 ):
         super(Esp32Builder, self).__init__(root, runner)
         self.board = board
         self.app = app
         self.enable_rpcs = enable_rpcs
+        self.enable_ipv4 = enable_ipv4
 
-    def _IdfEnvExecute(self, cmd, cwd=None, title=None):
+    def _IdfEnvExecute(self, cmd, title=None):
         self._Execute(
             ['bash', '-c', 'source $IDF_PATH/export.sh; %s' % cmd],
-            cwd=cwd,
             title=title)
 
+    @property
+    def ExamplePath(self):
+        return os.path.join('examples', self.app.ExampleName, 'esp32')
+
     def generate(self):
         if os.path.exists(os.path.join(self.output_dir, 'build.ninja')):
             return
 
-        defaults = DefaultsFileName(self.board, self.app, self.enable_rpcs)
+        defaults = os.path.join(self.ExamplePath, DefaultsFileName(
+            self.board, self.app, self.enable_rpcs))
+
+        if not self._runner.dry_run and not os.path.exists(defaults):
+            raise Exception('SDK defaults file missing: %s' % defaults)
 
-        cmd = 'idf.py'
+        defaults_out = os.path.join(self.output_dir, 'sdkconfig.defaults')
+
+        self._Execute(['mkdir', '-p', self.output_dir],
+                      title='Generating ' + self.identifier)
+        self._Execute(['cp', defaults, defaults_out])
+        self._Execute(
+            ['rm', '-f', os.path.join(self.ExamplePath, 'sdkconfig')])
 
-        if defaults:
-            cmd += " -D SDKCONFIG_DEFAULTS='%s'" % defaults
+        if not self.enable_ipv4:
+            self._Execute(
+                ['bash', '-c', 'echo CONFIG_DISABLE_IPV4=y >>%s' % shlex.quote(defaults_out)])
 
-        cmd += ' -C examples/%s/esp32 -B %s reconfigure' % (
-            self.app.ExampleName, shlex.quote(self.output_dir))
+        cmd = "\nexport SDKCONFIG_DEFAULTS={defaults}\nidf.py -C {example_path} -B {out} reconfigure".format(
+            defaults=shlex.quote(defaults_out),
+            example_path=self.ExamplePath,
+            out=shlex.quote(self.output_dir)
+        )
 
         # This will do a 'cmake reconfigure' which will create ninja files without rebuilding
-        self._IdfEnvExecute(
-            cmd, cwd=self.root, title='Generating ' + self.identifier)
+        self._IdfEnvExecute(cmd)
 
     def _build(self):
         logging.info('Compiling Esp32 at %s', self.output_dir)
 
-        self._IdfEnvExecute(
-            "ninja -C '%s'" % self.output_dir, title='Building ' + self.identifier)
+        # Unfortunately sdkconfig is sticky and needs reset on every build
+        self._Execute(
+            ['rm', '-f', os.path.join(self.ExamplePath, 'sdkconfig')])
+
+        defaults_out = os.path.join(self.output_dir, 'sdkconfig.defaults')
+
+        # "ninja -C" is insufficient because sdkconfig changes on every 'config' and results
+        # in a full reconfiguration with default values
+        #
+        # This does a regen + reconfigure.
+        cmd = "\nexport SDKCONFIG_DEFAULTS={defaults}\nidf.py -C {example_path} -B {out} build".format(
+            defaults=shlex.quote(defaults_out),
+            example_path=self.ExamplePath,
+            out=shlex.quote(self.output_dir)
+        )
+
+        self._IdfEnvExecute(cmd, title='Building ' + self.identifier)
 
     def build_outputs(self):
         return {

scripts/build/runner/printonly.py

@@ -16,21 +16,21 @@
 
 
 class PrintOnlyRunner:
-    def __init__(self, output_file):
+    def __init__(self, output_file, root: str):
         self.output_file = output_file
         self.dry_run = True
+        self.root = root
 
-    def Run(self, cmd, cwd=None, title=None):
+    def StartCommandExecution(self):
+        self.output_file.write(
+            "# Commands will be run in CHIP project root.\n")
+        self.output_file.write('cd "%s"\n\n' % self.root)
+
+    def Run(self, cmd, title=None):
         if title:
             self.output_file.write("# " + title + "\n")
 
-        if cwd:
-            self.output_file.write('cd "%s"\n' % cwd)
-
         self.output_file.write(
             " ".join([shlex.quote(part) for part in cmd]) + "\n")
 
-        if cwd:
-            self.output_file.write("cd -\n")
-
         self.output_file.write("\n")

scripts/build/runner/shell.py

@@ -50,17 +50,21 @@ def close(self):
 
 class ShellRunner:
 
-    def __init__(self):
+    def __init__(self, root: str):
         self.dry_run = False
+        self.root_dir = root
 
-    def Run(self, cmd, cwd=None, title=None):
+    def StartCommandExecution(self):
+        pass
+
+    def Run(self, cmd, title=None):
         outpipe = LogPipe(logging.INFO)
         errpipe = LogPipe(logging.WARN)
 
         if title:
             logging.info(title)
 
-        with subprocess.Popen(cmd, cwd=cwd, stdout=outpipe, stderr=errpipe) as s:
+        with subprocess.Popen(cmd, cwd=self.root_dir, stdout=outpipe, stderr=errpipe) as s:
             outpipe.close()
             errpipe.close()
             code = s.wait()

scripts/build/test.py

@@ -82,7 +82,13 @@ def assertCommandOutput(self, expected_file: str, args: List[str]):
         diffs = [line for line in difflib.unified_diff(expected, actual)]
 
         if diffs:
-            msg = "DIFFERENCE between expected and generated output in %s" % expected_file
+            reference = os.path.basename(expected_file) + '.actual'
+            with open(reference, 'wt') as fo:
+                for l in build_actual_output(ROOT, OUT, args):
+                    fo.write(l.replace(ROOT, '{root}').replace(OUT, '{out}'))
+
+            msg = "DIFFERENCE between expected and generated output in %s\n" % expected_file
+            msg += "Expected file can be found in %s" % reference
             for l in diffs:
                 msg += ("\n   " + l.replace(ROOT,
                         '{root}').replace(OUT, '{out}').strip())

scripts/build/testdata/all_targets_except_host.txt

@@ -9,11 +9,13 @@ efr32-brd4161a-lock
 efr32-brd4161a-window-covering
 esp32-c3devkit-all-clusters
 esp32-devkitc-all-clusters
+esp32-devkitc-all-clusters-ipv6only
 esp32-devkitc-bridge
 esp32-devkitc-lock
 esp32-devkitc-shell
 esp32-devkitc-temperature-measurement
 esp32-m5stack-all-clusters
+esp32-m5stack-all-clusters-ipv6only
 esp32-m5stack-all-clusters-rpc
 infineon-p6-lock
 nrf-nrf52840-light