From c5cbec9fea9dfd46c8d32d650af38d0cc3bd26dc Mon Sep 17 00:00:00 2001 From: kuqin12 <42554914+kuqin12@users.noreply.github.com> Date: Tue, 25 Apr 2023 15:48:19 -0700 Subject: [PATCH] Adding base tool build for Linux ARM (#362) Please ensure you have read the [contribution docs](https://github.com/microsoft/mu/blob/master/CONTRIBUTING.md) prior to submitting the pull request. In particular, [pull request guidelines](https://github.com/microsoft/mu/blob/master/CONTRIBUTING.md#pull-request-best-practices). This change added the build script to cross compile the base tool binaries for Linux ARM system. The needed libuuid system library is pulled from source file and rebuilt to support the corresponding library dependencies. Individual tools' makefiles are also updated to link the cross compiled library as well. This is the first step of https://github.com/microsoft/mu_basecore/issues/369. For each item, place an "x" in between `[` and `]` if true. Example: `[x]`. _(you can also check items in the GitHub UI)_ - [x] Impacts functionality? - **Functionality** - Does the change ultimately impact how firmware functions? - Examples: Add a new library, publish a new PPI, update an algorithm, ... - [ ] Impacts security? - **Security** - Does the change have a direct security impact on an application, flow, or firmware? - Examples: Crypto algorithm change, buffer overflow fix, parameter validation improvement, ... - [ ] Breaking change? - **Breaking change** - Will anyone consuming this change experience a break in build or boot behavior? - Examples: Add a new library class, move a module to a different repo, call a function in a new library class in a pre-existing module, ... - [ ] Includes tests? - **Tests** - Does the change include any explicit test code? - Examples: Unit tests, integration tests, robot tests, ... - [ ] Includes documentation? - **Documentation** - Does the change contain explicit documentation additions outside direct code modifications (and comments)? - Examples: Update readme file, add feature readme file, link to documentation on an a separate Web page, ... This was tested functional on Linux ARM host system building QEMU virtual platforms (Q35 and SBSA). Update the basetool nuget version, once it is published. --- .../BaseTools-Build-For-Publication.yml | 17 ++- BaseTools/Edk2ToolsBuild.py | 100 +++++++++++++++++- BaseTools/Source/C/DevicePath/GNUmakefile | 6 ++ BaseTools/Source/C/GenFv/GNUmakefile | 13 ++- BaseTools/Source/C/GenFv/GenFvInternalLib.c | 12 ++- BaseTools/Source/C/GenFw/GNUmakefile | 6 ++ BaseTools/Source/C/GenSec/GNUmakefile | 6 ++ BaseTools/Source/C/Makefiles/header.makefile | 12 ++- 8 files changed, 156 insertions(+), 16 deletions(-) diff --git a/.azurepipelines/BaseTools-Build-For-Publication.yml b/.azurepipelines/BaseTools-Build-For-Publication.yml index 95a2ad4b83..d62e922cab 100644 --- a/.azurepipelines/BaseTools-Build-For-Publication.yml +++ b/.azurepipelines/BaseTools-Build-For-Publication.yml @@ -30,6 +30,14 @@ variables: jobs: - job: linux_build + # Use matrix to speed up the build process + strategy: + matrix: + TARGET_x64: + Build.Targets: 'X64' + TARGET_AArch64: + Build.Targets: 'AARCH64' + displayName: Linux Build pool: vmImage: ubuntu-20.04 @@ -60,15 +68,15 @@ jobs: displayName: Update Environment inputs: filename: stuart_update - arguments: -c .pytool/CISettings.py + arguments: -c .pytool/CISettings.py TOOL_CHAIN_TAG=GCC5 -a $(Build.Targets) - template: Steps/BuildBaseTools.yml@mu_devops parameters: - extra_parameters: '--skip_path_env' + extra_parameters: '--skip_path_env -a $(Build.Targets)' tool_chain_tag: 'GCC5' - publish: $(linux_build_output_path) - artifact: $(linux_artifact_name) + artifact: $(linux_artifact_name)_$(Build.Targets) displayName: Publish To Pipeline condition: SucceededOrFailed() @@ -176,7 +184,8 @@ jobs: - powershell: mkdir $(Build.StagingDirectory)/$(temp_publication_directory); - mv $(Pipeline.Workspace)/$(linux_artifact_name) $(Build.StagingDirectory)/$(temp_publication_directory)/Linux-x86; + mv $(Pipeline.Workspace)/$(linux_artifact_name)_X64 $(Build.StagingDirectory)/$(temp_publication_directory)/Linux-x86; + mv $(Pipeline.Workspace)/$(linux_artifact_name)_AARCH64 $(Build.StagingDirectory)/$(temp_publication_directory)/Linux-ARM-64; mv $(Pipeline.Workspace)/$(windows_artifact_name)_IA32 $(Build.StagingDirectory)/$(temp_publication_directory)/Windows-x86; mv $(Pipeline.Workspace)/$(windows_artifact_name)_ARM $(Build.StagingDirectory)/$(temp_publication_directory)/Windows-ARM; displayName: Stage Package Files diff --git a/BaseTools/Edk2ToolsBuild.py b/BaseTools/Edk2ToolsBuild.py index c31d32a508..ff70d456d6 100644 --- a/BaseTools/Edk2ToolsBuild.py +++ b/BaseTools/Edk2ToolsBuild.py @@ -12,8 +12,9 @@ import logging import argparse import multiprocessing +import shutil from edk2toolext import edk2_logging -from edk2toolext.environment import self_describing_environment +from edk2toolext.environment import self_describing_environment, shell_environment from edk2toolext.base_abstract_invocable import BaseAbstractInvocable from edk2toollib.utility_functions import RunCmd from edk2toollib.utility_functions import GetHostInfo # MU_CHANGE: Need to check if this is cross compilation @@ -30,8 +31,9 @@ def ParseCommandLineOptions(self): # MU_CHANGE ParserObj.add_argument("-s", "--skip_path_env", dest="skip_env", default=False, action='store_true', help="Skip the creation of the path_env descriptor file") - ParserObj.add_argument("-a", "--target_arch", dest="arch", default='IA32', choices=['IA32', 'ARM'], - help="Specify the architecture of the built base tools") + ParserObj.add_argument("-a", "--target_arch", dest="arch", default=None, choices=[None, 'IA32', 'X64', 'ARM', 'AARCH64'], + help="Specify the architecture of the built base tools. Not specifying this will fall back to the default " + "behavior, for Windows builds, IA32 target will be built, for Linux builds, target arch will be the same as host arch.") args = ParserObj.parse_args() self.tool_chain_tag = args.tct self.target_arch = args.arch @@ -47,8 +49,17 @@ def GetWorkspaceRoot(self): def GetActiveScopes(self): ''' return tuple containing scopes that should be active for this process ''' - # for now don't use scopes - return ('global',) + # MU_CHANGE STARTs: Adding scope for cross compilers when building for ARM/AARCH64 + scopes = ('global',) + if GetHostInfo().os == "Linux" and self.tool_chain_tag.lower().startswith("gcc"): + if self.target_arch is None: + return scopes + if "AARCH64" in self.target_arch: + scopes += ("gcc_aarch64_linux",) + if "ARM" in self.target_arch: + scopes += ("gcc_arm_linux",) + return scopes + # MU_CHANGE ENDs def GetLoggingLevel(self, loggerType): ''' Get the logging level for a given type (return Logging.Level) @@ -120,6 +131,10 @@ def Go(self): if self.tool_chain_tag.lower().startswith("vs"): # MU_CHANGE: Specify target architecture + if self.target_arch is None: + # Put a default as IA32 + self.target_arch = "IA32" + if self.target_arch == "IA32": VcToolChainArch = "x86" TargetInfoArch = "x86" @@ -180,6 +195,81 @@ def Go(self): return ret elif self.tool_chain_tag.lower().startswith("gcc"): + # MU_CHANGE STARTs: Specify target architecture + # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where + # this script is BUILDING the base tools. + prefix = None + TargetInfoArch = None + if self.target_arch is not None: + shell_env.set_shell_var('HOST_ARCH', self.target_arch) + + if "AARCH64" in self.target_arch: + # now check for install dir. If set then set the Prefix + install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_INSTALL") + + # make GCC5_AARCH64_PREFIX to align with tools_def.txt + prefix = os.path.join(install_path, "bin", "aarch64-none-linux-gnu-") + shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix) + TargetInfoArch = "ARM" + + elif "ARM" in self.target_arch: + # now check for install dir. If set then set the Prefix + install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_INSTALL") + + # make GCC5_ARM_PREFIX to align with tools_def.txt + prefix = os.path.join(install_path, "bin", "arm-none-linux-gnueabihf-") + shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix) + TargetInfoArch = "ARM" + + else: + TargetInfoArch = "x86" + + # Otherwise, the built binary arch will be consistent with the host system + + # Added logic to support cross compilation scenarios + HostInfo = GetHostInfo() + if TargetInfoArch != HostInfo.arch: + # this is defaulting to the version that comes with Ubuntu 20.04 + ver = shell_environment.GetBuildVars().GetValue("LIBUUID_VERSION", "2.34") + work_dir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), self.GetLoggingFolderRelativeToRoot()) + pack_name = f"util-linux-{ver}" + unzip_dir = os.path.join(work_dir, pack_name) + + if os.path.isfile(os.path.join(work_dir, f"{pack_name}.tar.gz")): + os.remove(os.path.join(work_dir, f"{pack_name}.tar.gz")) + if os.path.isdir(unzip_dir): + shutil.rmtree(unzip_dir) + + # cross compiling, need to rebuild libuuid for the target + ret = RunCmd("wget", f"https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v{ver}/{pack_name}.tar.gz", workingdir=work_dir) + if ret != 0: + raise Exception(f"Failed to download libuuid version {ver} - {ret}") + + ret = RunCmd("tar", f"xvzf {pack_name}.tar.gz", workingdir=work_dir) + if ret != 0: + raise Exception(f"Failed to untar the downloaded file {ret}") + + # configure the source to use the cross compiler + pack_name = f"util-linux-{ver}" + if "AARCH64" in self.target_arch: + ret = RunCmd("sh", f"./configure --host=aarch64-linux -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir) + elif "ARM" in self.target_arch: + ret = RunCmd("sh", f"./configure --host=arm-linux -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir) + if ret != 0: + raise Exception(f"Failed to configure the util-linux to build with our gcc {ret}") + + ret = RunCmd("make", "", workingdir=unzip_dir) + if ret != 0: + raise Exception(f"Failed to build the libuuid with our gcc {ret}") + + shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID", unzip_dir) + shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID_INC", os.path.join(unzip_dir, "libuuid", "src")) + + ret = RunCmd("make", "clean", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH")) + if ret != 0: + raise Exception("Failed to build.") + # MU_CHANGE ENDs + cpu_count = self.GetCpuThreads() output_stream = edk2_logging.create_output_stream() diff --git a/BaseTools/Source/C/DevicePath/GNUmakefile b/BaseTools/Source/C/DevicePath/GNUmakefile index f61b1b2f17..ee8e3a9058 100644 --- a/BaseTools/Source/C/DevicePath/GNUmakefile +++ b/BaseTools/Source/C/DevicePath/GNUmakefile @@ -30,6 +30,12 @@ ifeq ($(CYGWIN), CYGWIN) endif ifeq ($(LINUX), Linux) +# MU_CHANGE STARTs: Support cross compiling +ifndef CROSS_LIB_UUID LIBS += -luuid +else + LIBS += -L$(CROSS_LIB_UUID) +endif +# MU_CHANGE ENDs endif diff --git a/BaseTools/Source/C/GenFv/GNUmakefile b/BaseTools/Source/C/GenFv/GNUmakefile index 872b981f6a..add64481b9 100644 --- a/BaseTools/Source/C/GenFv/GNUmakefile +++ b/BaseTools/Source/C/GenFv/GNUmakefile @@ -14,6 +14,17 @@ include $(MAKEROOT)/Makefiles/app.makefile LIBS = -lCommon ifeq ($(CYGWIN), CYGWIN) - LIBS += -L/lib/e2fsprogs + LIBS += -L/lib/e2fsprogs -luuid +endif + +ifeq ($(LINUX), Linux) +# MU_CHANGE STARTs: Support cross compiling +ifndef CROSS_LIB_UUID + LIBS += -luuid +else + LIBS += -L$(CROSS_LIB_UUID) + BUILD_CFLAGS += -D__CROSS_LIB_UUID__ -I $(CROSS_LIB_UUID_INC) +endif +# MU_CHANGE ENDs endif diff --git a/BaseTools/Source/C/GenFv/GenFvInternalLib.c b/BaseTools/Source/C/GenFv/GenFvInternalLib.c index 29c3363a50..5d18f145d2 100644 --- a/BaseTools/Source/C/GenFv/GenFvInternalLib.c +++ b/BaseTools/Source/C/GenFv/GenFvInternalLib.c @@ -14,6 +14,17 @@ SPDX-License-Identifier: BSD-2-Clause-Patent // Include files // +#if defined(__FreeBSD__) +#include +#elif defined(__GNUC__) +// MU_CHANGE STARTs: Support cross compiling +#if !defined(__CROSS_LIB_UUID__) +#include +#else +#include +#endif +// MU_CHANGE ENDs +#endif #ifdef __GNUC__ #include #endif @@ -2286,7 +2297,6 @@ Routine Description: bSecCore |= 0x6F; //JAL opcode memcpy(FvImage->FileImage, &bSecCore, sizeof(bSecCore)); - return EFI_SUCCESS; } diff --git a/BaseTools/Source/C/GenFw/GNUmakefile b/BaseTools/Source/C/GenFw/GNUmakefile index 76cda7e7a3..3a38d8872e 100644 --- a/BaseTools/Source/C/GenFw/GNUmakefile +++ b/BaseTools/Source/C/GenFw/GNUmakefile @@ -18,6 +18,12 @@ ifeq ($(CYGWIN), CYGWIN) endif ifeq ($(LINUX), Linux) +# MU_CHANGE STARTs: Support cross compiling +ifndef CROSS_LIB_UUID LIBS += -luuid +else + LIBS += -L$(CROSS_LIB_UUID) +endif +# MU_CHANGE ENDs endif diff --git a/BaseTools/Source/C/GenSec/GNUmakefile b/BaseTools/Source/C/GenSec/GNUmakefile index 9f0844c1b8..4f1949282d 100644 --- a/BaseTools/Source/C/GenSec/GNUmakefile +++ b/BaseTools/Source/C/GenSec/GNUmakefile @@ -18,6 +18,12 @@ ifeq ($(CYGWIN), CYGWIN) endif ifeq ($(LINUX), Linux) +# MU_CHANGE STARTs: Support cross compiling +ifndef CROSS_LIB_UUID LIBS += -luuid +else + LIBS += -L$(CROSS_LIB_UUID) +endif +# MU_CHANGE ENDs endif diff --git a/BaseTools/Source/C/Makefiles/header.makefile b/BaseTools/Source/C/Makefiles/header.makefile index d369908a09..dea5d6090d 100644 --- a/BaseTools/Source/C/Makefiles/header.makefile +++ b/BaseTools/Source/C/Makefiles/header.makefile @@ -54,11 +54,13 @@ AS ?= $(CLANG_BIN)clang AR ?= $(CLANG_BIN)llvm-ar LD ?= $(CLANG_BIN)llvm-ld else ifeq ($(origin CC),default) -CC = gcc -CXX = g++ -AS = gcc -AR = ar -LD = ld +# MU_CHANGE STARTs: Support GCC prefix +CC ?= $(GCC_PREFIX)gcc +CXX ?= $(GCC_PREFIX)g++ +AS ?= $(GCC_PREFIX)gcc +AR ?= $(GCC_PREFIX)ar +LD ?= $(GCC_PREFIX)ld +# MU_CHANGE ENDs endif LINKER ?= $(CC) ifeq ($(HOST_ARCH), IA32)