Get configure working again and fix the minimal build (#28547)

* Don't bootstrap in minimal-build workflow

The configure script is a replacement for running PW bootstrap, so running
bootstrap first makes this no longer a valid test of how the configure
script sets up the build environment from scratch.

* configure: bootstrap pip from pypa.io if necessary

This is a workaround for the Debian / Ubuntu missing the ensurepip module
used by venv unless python3-venv is installed, which is "suggested" but
not required by the python3 package, and hence may or may not be installed
in build environments. (https://bugs.launchpad.net/bugs/1290847)

Also avoid := syntax which is unavailable on Python 3.7

* configure: use x64 build of zap even on arm (there is no mac arm build)

* configure: Add matter_ argument prefixes

* Revert pigweed to 0889de6~1

0889de6 introduced https://issues.pigweed.dev/issues/294886611 which breaks the
configure-based build.

.github/workflows/minimal-build.yaml

@@ -36,10 +36,12 @@ jobs:
  steps:
  - name: Checkout
  uses: actions/checkout@v3
- - name: Checkout submodules & Bootstrap
- uses: ./.github/actions/checkout-submodules-and-bootstrap
+
+ - name: Checkout submodules # but don't bootstrap!
+ uses: ./.github/actions/checkout-submodules
  with:
  platform: linux
+
  - name: Configure and build All Clusters App
  run: |
  CC=gcc CXX=g++ scripts/configure --project=examples/all-clusters-app/linux && ./ninja-build

scripts/configure

@@ -226,17 +226,32 @@ function check_build_env() {
 
 function configure_python_env() {
  progress "Setting up Python venv"
- "$PYTHON" -m venv --clear "$BUILD_ENV_PATH"
+ # Debian and Ubuntu ship python3 with a broken venv module unless the
+ # python3-venv package is installed (https://bugs.launchpad.net/bugs/1290847)
+ local withoutpip=() pip="${BUILD_ENV_PATH}/bin/pip"
+ if ! "$PYTHON" -m ensurepip --version >/dev/null 2>&1; then
+ withoutpip=(--without-pip) pip="${pip}.pyz" # bootstrapped below
+ fi
+ "$PYTHON" -m venv --clear "${withoutpip[@]}" "$BUILD_ENV_PATH"
  info "$BUILD_ENV_PATH"
 
+ # Download a standalone pip.pyz from pypa.io if necessary
+ if [[ -n "$withoutpip" ]]; then
+ progress "Bootstrapping pip via pypa.io (venv module is missing ensurepip dependency)"
+ call_impl download https://bootstrap.pypa.io/pip/pip.pyz "$pip"
+ info "ok"
+ fi
+
  # Install our auto-loading venvactivate module so that running scripts via
  # the venv python has the side-effect of fully activating the environment.
  local sitepkgs=("${BUILD_ENV_PATH}/lib/python"*"/site-packages")
  [[ -d "$sitepkgs" ]] || fail "Failed to locate venv site-packages"
  cp "${CHIP_ROOT}/scripts/configure.venv/venvactivate".{pth,py} "${sitepkgs}/"
 
  progress "Installing Python build dependencies"
- "${BUILD_ENV_PATH}/bin/pip" install --require-virtualenv --quiet --upgrade pip wheel
+ # Ensure pip and wheel are up to date first (using pip.pyz if necessary)
+ "${BUILD_ENV_PATH}/bin/python3" "$pip" install --require-virtualenv --quiet --upgrade pip wheel
+
  "${BUILD_ENV_PATH}/bin/pip" install --require-virtualenv --quiet \
  -r "${CHIP_ROOT}/scripts/setup/requirements.build.txt" \
  -c "${CHIP_ROOT}/scripts/setup/constraints.txt"
@@ -254,19 +269,16 @@ function finalize_build_env() {
 }
 
 function download_zap() {
- local version platform arch
+ local version
  read -r version <"${CHIP_ROOT}/scripts/setup/zap.version"
- case "$OSTYPE" in
- linux*) platform=linux ;;
- darwin*) platform=mac ;;
- *) fail "Unable to determine zap platform for OSTYPE='$OSTYPE'" ;;
- esac
- case "$(uname -m)" in
- arm64) arch=arm64 ;;
- x86_64) arch=x64 ;;
- *) fail "Unable to determine zap architecture for 'uname -m'='$(uname -m)'" ;;
+ local platform="$(uname -sm)" flavor
+ case "$platform" in
+ Linux\ x86_64) flavor=linux-x64 ;;
+ Linux\ arm64) flavor=linux-arm64 ;;
+ Darwin\ *) flavor=mac-x64 ;; # there is no mac arm build of zap (can run x64 via Rosetta)
+ *) fail "Unable to determine zap flavor for $platform" ;;
  esac
- local url="https://github.com/project-chip/zap/releases/download/${version}/zap-${platform}-${arch}.zip"
+ local url="https://github.com/project-chip/zap/releases/download/${version}/zap-${flavor}.zip"
 
  progress "Installing zap-cli from $url"
  call_impl download_and_extract_zip "$url" "${BUILD_ENV_PATH}/bin" zap-cli

scripts/configure.impl.py

@@ -24,6 +24,10 @@
 import zipfile
 
 
+def download(url, dest):
+ urllib.request.urlretrieve(url, dest)
+
+
 def download_and_extract_zip(url, dest_dir, *member_names):
  file, *_ = urllib.request.urlretrieve(url)
  with zipfile.ZipFile(file) as zip:
@@ -44,8 +48,10 @@ def process_project_args(gn_args_json_file, *params):
 
 class ProjectArgProcessor:
  # Prefixes to try when mapping parameters to GN arguments
- BOOL_ARG_PREFIXES = ('is_', 'enable_', 'use_', 'chip_', 'chip_enable', 'chip_use_', 'chip_config_')
- GENERIC_ARG_PREFIXES = ('chip_', 'chip_config_')
+ BOOL_ARG_PREFIXES = ('is_', 'enable_', 'use_',
+ 'chip_', 'chip_enable_', 'chip_use_', 'chip_config_',
+ 'matter_', 'matter_enable_', 'matter_use_', 'matter_config_')
+ GENERIC_ARG_PREFIXES = ('chip_', 'chip_config_', 'matter_', 'matter_config_')
 
  gn_args = {} # GN arg -> type ('s'tring, 'b'ool, 'i'integer, '[' list, '{' struct)
  args = collections.OrderedDict() # collected arguments
@@ -84,7 +90,8 @@ def add_env_arg(self, arg, envvar, default=None, list=False):
  value = os.environ.get(envvar, default)
  if not value:
  return
- if not (type := self.gn_args.get(arg, None)):
+ type = self.gn_args.get(arg, None)
+ if not type:
  info("Warning: Not propagating %s, project has no build arg '%s'" % (envvar, arg))
  return
  self.args[arg] = json.dumps(value if type != '[' else value.split() if list else [value])
@@ -94,7 +101,8 @@ def gn_arg(self, name, prefixes=(), type=None):
  arg = name.translate(str.maketrans('-', '_'))
  candidates = [p + arg for p in (('',) + prefixes) if (p + arg) in self.gn_args]
  preferred = [c for c in candidates if self.gn_args[c] == type] if type else []
- if not (match := next(iter(preferred + candidates), None)):
+ match = next(iter(preferred + candidates), None)
+ if not match:
  info("Warning: Project has no build arg '%s'" % arg)
  return match
 
@@ -109,7 +117,8 @@ def process_triplet_parameter(self, name, value):
  self.add_arg(prefix + 'os', triplet[1 if len(triplet) == 2 else 2])
 
  def process_enable_parameter(self, name, value):
- if not (arg := self.gn_arg(name[len('enable-'):], self.BOOL_ARG_PREFIXES, 'b')):
+ arg = self.gn_arg(name[len('enable-'):], self.BOOL_ARG_PREFIXES, 'b')
+ if not arg:
  return
  if self.gn_args[arg] != 'b':
  fail("Project build arg '%s' is not a boolean" % arg)
@@ -118,7 +127,8 @@ def process_enable_parameter(self, name, value):
  self.add_arg(arg, value is None)
 
  def process_generic_parameter(self, name, value):
- if not (arg := self.gn_arg(name, self.GENERIC_ARG_PREFIXES)):
+ arg = self.gn_arg(name, self.GENERIC_ARG_PREFIXES)
+ if not arg:
  return
  if self.gn_args[arg] == 'b':
  fail("Project build arg '%s' is a boolean, use --enable-..." % arg)
@@ -137,9 +147,10 @@ def process_parameter(self, name, value):
  def process_parameters(self, args):
  """Process GNU-style configure command line parameters"""
  for arg in args:
- if not (m := re.fullmatch(r'--([a-z][a-z0-9-]*)(?:=(.*))?', arg)):
+ match = re.fullmatch(r'--([a-z][a-z0-9-]*)(?:=(.*))?', arg)
+ if not match:
  fail("Invalid argument: '%s'" % arg)
- self.process_parameter(m.group(1), m.group(2))
+ self.process_parameter(match.group(1), match.group(2))
 
 
 def info(message):