From 367a0c672c18e0137e0162c15b78adbcb039a33a Mon Sep 17 00:00:00 2001 From: Karsten Sperling <113487422+ksperling-apple@users.noreply.github.com> Date: Tue, 8 Aug 2023 13:20:47 +1200 Subject: [PATCH] Get configure working again and fix the minimal build (#28547) * Don't bootstrap in minimal-build workflow The configure script is a replacement for running PW bootstrap, so running bootstrap first makes this no longer a valid test of how the configure script sets up the build environment from scratch. * configure: bootstrap pip from pypa.io if necessary This is a workaround for the Debian / Ubuntu missing the ensurepip module used by venv unless python3-venv is installed, which is "suggested" but not required by the python3 package, and hence may or may not be installed in build environments. (https://bugs.launchpad.net/bugs/1290847) Also avoid := syntax which is unavailable on Python 3.7 * configure: use x64 build of zap even on arm (there is no mac arm build) * configure: Add matter_ argument prefixes * Revert pigweed to 0889de6~1 0889de6 introduced https://issues.pigweed.dev/issues/294886611 which breaks the configure-based build. --- .github/workflows/minimal-build.yaml | 6 +++-- scripts/configure | 38 ++++++++++++++++++---------- scripts/configure.impl.py | 27 ++++++++++++++------ third_party/pigweed/repo | 2 +- 4 files changed, 49 insertions(+), 24 deletions(-) diff --git a/.github/workflows/minimal-build.yaml b/.github/workflows/minimal-build.yaml index 794d108446fa4d..c980dcd3e0a8e6 100644 --- a/.github/workflows/minimal-build.yaml +++ b/.github/workflows/minimal-build.yaml @@ -36,10 +36,12 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Checkout submodules & Bootstrap - uses: ./.github/actions/checkout-submodules-and-bootstrap + + - name: Checkout submodules # but don't bootstrap! + uses: ./.github/actions/checkout-submodules with: platform: linux + - name: Configure and build All Clusters App run: | CC=gcc CXX=g++ scripts/configure --project=examples/all-clusters-app/linux && ./ninja-build diff --git a/scripts/configure b/scripts/configure index d48971862e2943..a4e7536e627b23 100755 --- a/scripts/configure +++ b/scripts/configure @@ -226,9 +226,22 @@ function check_build_env() { function configure_python_env() { progress "Setting up Python venv" - "$PYTHON" -m venv --clear "$BUILD_ENV_PATH" + # Debian and Ubuntu ship python3 with a broken venv module unless the + # python3-venv package is installed (https://bugs.launchpad.net/bugs/1290847) + local withoutpip=() pip="${BUILD_ENV_PATH}/bin/pip" + if ! "$PYTHON" -m ensurepip --version >/dev/null 2>&1; then + withoutpip=(--without-pip) pip="${pip}.pyz" # bootstrapped below + fi + "$PYTHON" -m venv --clear "${withoutpip[@]}" "$BUILD_ENV_PATH" info "$BUILD_ENV_PATH" + # Download a standalone pip.pyz from pypa.io if necessary + if [[ -n "$withoutpip" ]]; then + progress "Bootstrapping pip via pypa.io (venv module is missing ensurepip dependency)" + call_impl download https://bootstrap.pypa.io/pip/pip.pyz "$pip" + info "ok" + fi + # Install our auto-loading venvactivate module so that running scripts via # the venv python has the side-effect of fully activating the environment. local sitepkgs=("${BUILD_ENV_PATH}/lib/python"*"/site-packages") @@ -236,7 +249,9 @@ function configure_python_env() { cp "${CHIP_ROOT}/scripts/configure.venv/venvactivate".{pth,py} "${sitepkgs}/" progress "Installing Python build dependencies" - "${BUILD_ENV_PATH}/bin/pip" install --require-virtualenv --quiet --upgrade pip wheel + # Ensure pip and wheel are up to date first (using pip.pyz if necessary) + "${BUILD_ENV_PATH}/bin/python3" "$pip" install --require-virtualenv --quiet --upgrade pip wheel + "${BUILD_ENV_PATH}/bin/pip" install --require-virtualenv --quiet \ -r "${CHIP_ROOT}/scripts/setup/requirements.build.txt" \ -c "${CHIP_ROOT}/scripts/setup/constraints.txt" @@ -254,19 +269,16 @@ function finalize_build_env() { } function download_zap() { - local version platform arch + local version read -r version <"${CHIP_ROOT}/scripts/setup/zap.version" - case "$OSTYPE" in - linux*) platform=linux ;; - darwin*) platform=mac ;; - *) fail "Unable to determine zap platform for OSTYPE='$OSTYPE'" ;; - esac - case "$(uname -m)" in - arm64) arch=arm64 ;; - x86_64) arch=x64 ;; - *) fail "Unable to determine zap architecture for 'uname -m'='$(uname -m)'" ;; + local platform="$(uname -sm)" flavor + case "$platform" in + Linux\ x86_64) flavor=linux-x64 ;; + Linux\ arm64) flavor=linux-arm64 ;; + Darwin\ *) flavor=mac-x64 ;; # there is no mac arm build of zap (can run x64 via Rosetta) + *) fail "Unable to determine zap flavor for $platform" ;; esac - local url="https://github.com/project-chip/zap/releases/download/${version}/zap-${platform}-${arch}.zip" + local url="https://github.com/project-chip/zap/releases/download/${version}/zap-${flavor}.zip" progress "Installing zap-cli from $url" call_impl download_and_extract_zip "$url" "${BUILD_ENV_PATH}/bin" zap-cli diff --git a/scripts/configure.impl.py b/scripts/configure.impl.py index b51e40fa00cf55..cedb67a2a5fb05 100644 --- a/scripts/configure.impl.py +++ b/scripts/configure.impl.py @@ -24,6 +24,10 @@ import zipfile +def download(url, dest): + urllib.request.urlretrieve(url, dest) + + def download_and_extract_zip(url, dest_dir, *member_names): file, *_ = urllib.request.urlretrieve(url) with zipfile.ZipFile(file) as zip: @@ -44,8 +48,10 @@ def process_project_args(gn_args_json_file, *params): class ProjectArgProcessor: # Prefixes to try when mapping parameters to GN arguments - BOOL_ARG_PREFIXES = ('is_', 'enable_', 'use_', 'chip_', 'chip_enable', 'chip_use_', 'chip_config_') - GENERIC_ARG_PREFIXES = ('chip_', 'chip_config_') + BOOL_ARG_PREFIXES = ('is_', 'enable_', 'use_', + 'chip_', 'chip_enable_', 'chip_use_', 'chip_config_', + 'matter_', 'matter_enable_', 'matter_use_', 'matter_config_') + GENERIC_ARG_PREFIXES = ('chip_', 'chip_config_', 'matter_', 'matter_config_') gn_args = {} # GN arg -> type ('s'tring, 'b'ool, 'i'integer, '[' list, '{' struct) args = collections.OrderedDict() # collected arguments @@ -84,7 +90,8 @@ def add_env_arg(self, arg, envvar, default=None, list=False): value = os.environ.get(envvar, default) if not value: return - if not (type := self.gn_args.get(arg, None)): + type = self.gn_args.get(arg, None) + if not type: info("Warning: Not propagating %s, project has no build arg '%s'" % (envvar, arg)) return self.args[arg] = json.dumps(value if type != '[' else value.split() if list else [value]) @@ -94,7 +101,8 @@ def gn_arg(self, name, prefixes=(), type=None): arg = name.translate(str.maketrans('-', '_')) candidates = [p + arg for p in (('',) + prefixes) if (p + arg) in self.gn_args] preferred = [c for c in candidates if self.gn_args[c] == type] if type else [] - if not (match := next(iter(preferred + candidates), None)): + match = next(iter(preferred + candidates), None) + if not match: info("Warning: Project has no build arg '%s'" % arg) return match @@ -109,7 +117,8 @@ def process_triplet_parameter(self, name, value): self.add_arg(prefix + 'os', triplet[1 if len(triplet) == 2 else 2]) def process_enable_parameter(self, name, value): - if not (arg := self.gn_arg(name[len('enable-'):], self.BOOL_ARG_PREFIXES, 'b')): + arg = self.gn_arg(name[len('enable-'):], self.BOOL_ARG_PREFIXES, 'b') + if not arg: return if self.gn_args[arg] != 'b': fail("Project build arg '%s' is not a boolean" % arg) @@ -118,7 +127,8 @@ def process_enable_parameter(self, name, value): self.add_arg(arg, value is None) def process_generic_parameter(self, name, value): - if not (arg := self.gn_arg(name, self.GENERIC_ARG_PREFIXES)): + arg = self.gn_arg(name, self.GENERIC_ARG_PREFIXES) + if not arg: return if self.gn_args[arg] == 'b': fail("Project build arg '%s' is a boolean, use --enable-..." % arg) @@ -137,9 +147,10 @@ def process_parameter(self, name, value): def process_parameters(self, args): """Process GNU-style configure command line parameters""" for arg in args: - if not (m := re.fullmatch(r'--([a-z][a-z0-9-]*)(?:=(.*))?', arg)): + match = re.fullmatch(r'--([a-z][a-z0-9-]*)(?:=(.*))?', arg) + if not match: fail("Invalid argument: '%s'" % arg) - self.process_parameter(m.group(1), m.group(2)) + self.process_parameter(match.group(1), match.group(2)) def info(message): diff --git a/third_party/pigweed/repo b/third_party/pigweed/repo index cf4291da443f5b..f04f934b3c6991 160000 --- a/third_party/pigweed/repo +++ b/third_party/pigweed/repo @@ -1 +1 @@ -Subproject commit cf4291da443f5b2db18827747bb66499459de5bd +Subproject commit f04f934b3c69910f866059471625a2bc95e113c4