From 1054a93d154fe78714a14e3af77069c2e9fd2289 Mon Sep 17 00:00:00 2001
From: Mike Lundy <mlundy@splunk.com>
Date: Wed, 15 May 2024 17:01:20 -0700
Subject: [PATCH] WIP: provisional parallelization support

---
 foreign_cc/boost_build.bzl                                | 1 +
 foreign_cc/built_tools/private/built_tools_framework.bzl  | 1 +
 foreign_cc/private/cmake_script.bzl                       | 1 +
 foreign_cc/private/configure_script.bzl                   | 1 +
 foreign_cc/private/framework/toolchains/commands.bzl      | 4 ++++
 .../private/framework/toolchains/freebsd_commands.bzl     | 5 +++++
 .../private/framework/toolchains/linux_commands.bzl       | 8 ++++++++
 .../private/framework/toolchains/macos_commands.bzl       | 8 ++++++++
 .../private/framework/toolchains/windows_commands.bzl     | 5 +++++
 foreign_cc/private/make_script.bzl                        | 1 +
 test/cmake_text_tests.bzl                                 | 8 ++++++++
 11 files changed, 43 insertions(+)

diff --git a/foreign_cc/boost_build.bzl b/foreign_cc/boost_build.bzl
index 246211764..900e08a83 100644
--- a/foreign_cc/boost_build.bzl
+++ b/foreign_cc/boost_build.bzl
@@ -30,6 +30,7 @@ def _create_configure_script(configureParameters):
         "##copy_dir_contents_to_dir## $$EXT_BUILD_ROOT$$/{}/. .".format(root),
         "chmod -R +w .",
         "##enable_tracing##",
+        "##enable_parallel_build##",
         "./bootstrap.sh {}".format(" ".join(ctx.attr.bootstrap_options)),
         "./b2 install {} --prefix=.".format(" ".join(user_options)),
         "##disable_tracing##",
diff --git a/foreign_cc/built_tools/private/built_tools_framework.bzl b/foreign_cc/built_tools/private/built_tools_framework.bzl
index bec5599fb..0a2a9de42 100644
--- a/foreign_cc/built_tools/private/built_tools_framework.bzl
+++ b/foreign_cc/built_tools/private/built_tools_framework.bzl
@@ -78,6 +78,7 @@ def built_tool_rule_impl(ctx, script_lines, out_dir, mnemonic, additional_tools
     ]
 
     script.append("##enable_tracing##")
+    script.append("##enable_parallel_build##")
     script.extend(script_lines)
     script.append("##disable_tracing##")
 
diff --git a/foreign_cc/private/cmake_script.bzl b/foreign_cc/private/cmake_script.bzl
index bfe21bd23..5f568b968 100644
--- a/foreign_cc/private/cmake_script.bzl
+++ b/foreign_cc/private/cmake_script.bzl
@@ -139,6 +139,7 @@ def create_cmake_script(
     directory = "$$EXT_BUILD_ROOT$$/" + root
 
     script.append("##enable_tracing##")
+    script.append("##enable_parallel_build##")
 
     # Configure the CMake generate command
     cmake_prefixes = [cmake_prefix] if cmake_prefix else []
diff --git a/foreign_cc/private/configure_script.bzl b/foreign_cc/private/configure_script.bzl
index 06c516ea5..483a14cfa 100644
--- a/foreign_cc/private/configure_script.bzl
+++ b/foreign_cc/private/configure_script.bzl
@@ -38,6 +38,7 @@ def create_configure_script(
 
     script.append("##export_var## MAKE {}".format(make_path))
     script.append("##enable_tracing##")
+    script.append("##enable_parallel_build##")
 
     if autogen:
         # NOCONFIGURE is pseudo standard and tells the script to not invoke configure.
diff --git a/foreign_cc/private/framework/toolchains/commands.bzl b/foreign_cc/private/framework/toolchains/commands.bzl
index e4f107318..7b589d6ae 100644
--- a/foreign_cc/private/framework/toolchains/commands.bzl
+++ b/foreign_cc/private/framework/toolchains/commands.bzl
@@ -105,6 +105,10 @@ PLATFORM_COMMANDS = {
         arguments = [_argument_info(name = "text", data_type = type(""), doc = "Text to output")],
         doc = "Outputs 'text' to stdout",
     ),
+    "enable_parallel_build": _command_info(
+        arguments = [],
+        doc = "Enable parallization (auto-detect cpu count and set MAKEFLAGS and CMAKE_BUILD_PARALLEL_LEVEL appropriately)",
+    ),
     "enable_tracing": _command_info(
         arguments = [],
         doc = "Enable script tracing. eg: `set -x`",
diff --git a/foreign_cc/private/framework/toolchains/freebsd_commands.bzl b/foreign_cc/private/framework/toolchains/freebsd_commands.bzl
index ccc6e58b1..dd23a006e 100644
--- a/foreign_cc/private/framework/toolchains/freebsd_commands.bzl
+++ b/foreign_cc/private/framework/toolchains/freebsd_commands.bzl
@@ -47,6 +47,10 @@ def enable_tracing():
 def disable_tracing():
     return "set +x"
 
+def enable_parallel_build():
+    # XXX not implemented
+    return ":"
+
 def mkdirs(path):
     return "mkdir -p " + path
 
@@ -282,6 +286,7 @@ commands = struct(
     define_sandbox_paths = define_sandbox_paths,
     disable_tracing = disable_tracing,
     echo = echo,
+    enable_parallel_build = enable_parallel_build,
     enable_tracing = enable_tracing,
     env = env,
     export_var = export_var,
diff --git a/foreign_cc/private/framework/toolchains/linux_commands.bzl b/foreign_cc/private/framework/toolchains/linux_commands.bzl
index ba265ebdf..40c3450f3 100644
--- a/foreign_cc/private/framework/toolchains/linux_commands.bzl
+++ b/foreign_cc/private/framework/toolchains/linux_commands.bzl
@@ -38,6 +38,13 @@ def enable_tracing():
 def disable_tracing():
     return "set +x"
 
+def enable_parallel_build():
+    return """
+    _cpu_count=$(nproc)
+    export MAKEFLAGS="-j$_cpu_count${MAKEFLAGS:+ }${MAKEFLAGS:-}"
+    export CMAKE_BUILD_PARALLEL_LEVEL="$_cpu_count"
+"""
+
 def mkdirs(path):
     return "mkdir -p " + path
 
@@ -264,6 +271,7 @@ commands = struct(
     define_sandbox_paths = define_sandbox_paths,
     disable_tracing = disable_tracing,
     echo = echo,
+    enable_parallel_build = enable_parallel_build,
     enable_tracing = enable_tracing,
     env = env,
     export_var = export_var,
diff --git a/foreign_cc/private/framework/toolchains/macos_commands.bzl b/foreign_cc/private/framework/toolchains/macos_commands.bzl
index c088a9675..f5690be81 100644
--- a/foreign_cc/private/framework/toolchains/macos_commands.bzl
+++ b/foreign_cc/private/framework/toolchains/macos_commands.bzl
@@ -38,6 +38,13 @@ def enable_tracing():
 def disable_tracing():
     return "set +x"
 
+def enable_parallel_build():
+    return """
+    _cpu_count=$(sysctl -n hw.ncpu)
+    export MAKEFLAGS="-j$_cpu_count${MAKEFLAGS:+ }${MAKEFLAGS:-}"
+    export CMAKE_BUILD_PARALLEL_LEVEL="$_cpu_count"
+"""
+
 def mkdirs(path):
     return "mkdir -p " + path
 
@@ -277,6 +284,7 @@ commands = struct(
     define_sandbox_paths = define_sandbox_paths,
     disable_tracing = disable_tracing,
     echo = echo,
+    enable_parallel_build = enable_parallel_build,
     enable_tracing = enable_tracing,
     env = env,
     export_var = export_var,
diff --git a/foreign_cc/private/framework/toolchains/windows_commands.bzl b/foreign_cc/private/framework/toolchains/windows_commands.bzl
index 144f31bfb..b5ac3f561 100644
--- a/foreign_cc/private/framework/toolchains/windows_commands.bzl
+++ b/foreign_cc/private/framework/toolchains/windows_commands.bzl
@@ -38,6 +38,10 @@ def enable_tracing():
 def disable_tracing():
     return "set +x"
 
+def enable_parallel_build():
+    # XXX not implemented
+    return ":"
+
 def mkdirs(path):
     return "mkdir -p " + path
 
@@ -281,6 +285,7 @@ commands = struct(
     define_sandbox_paths = define_sandbox_paths,
     disable_tracing = disable_tracing,
     echo = echo,
+    enable_parallel_build = enable_parallel_build,
     enable_tracing = enable_tracing,
     env = env,
     export_var = export_var,
diff --git a/foreign_cc/private/make_script.bzl b/foreign_cc/private/make_script.bzl
index 5a3754011..30fafadc4 100644
--- a/foreign_cc/private/make_script.bzl
+++ b/foreign_cc/private/make_script.bzl
@@ -19,6 +19,7 @@ def create_make_script(
     script.append("##symlink_contents_to_dir## $$EXT_BUILD_ROOT$$/{} $$BUILD_TMPDIR$$ False".format(root))
 
     script.append("##enable_tracing##")
+    script.append("##enable_parallel_build##")
     configure_vars = get_make_env_vars(workspace_name, tools, flags, env_vars, deps, inputs, make_commands)
     script.extend(["{env_vars} {command}".format(
         env_vars = configure_vars,
diff --git a/test/cmake_text_tests.bzl b/test/cmake_text_tests.bzl
index 105faf5e4..701f8dcfb 100644
--- a/test/cmake_text_tests.bzl
+++ b/test/cmake_text_tests.bzl
@@ -262,6 +262,7 @@ def _merge_flag_values_no_toolchain_file_test(ctx):
 export CXX="/usr/bin/gcc"
 export CXXFLAGS="foo=\\\"bar\\\" -Fbat"
 ##enable_tracing##
+##enable_parallel_build##
 emcmake cmake -DCMAKE_AR="/usr/bin/ar" -DCMAKE_BUILD_TYPE="RelWithDebInfo" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$" -DCMAKE_RANLIB=""  -G 'Unix Makefiles' $$EXT_BUILD_ROOT$$/external/test_rule
 ##disable_tracing##
 """
@@ -315,6 +316,7 @@ export CFLAGS="-U_FORTIFY_SOURCE -fstack-protector -Wall"
 export CXXFLAGS="-U_FORTIFY_SOURCE -fstack-protector -Wall"
 export ASMFLAGS="-U_FORTIFY_SOURCE -fstack-protector -Wall"
 ##enable_tracing##
+##enable_parallel_build##
 cmake -DCMAKE_AR="/usr/bin/ar" -DCMAKE_SHARED_LINKER_FLAGS="-shared -fuse-ld=gold" -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=gold -Wl -no-as-needed" -DNOFORTRAN="on" -DCMAKE_BUILD_TYPE="Debug" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$;/abc/def" -DCMAKE_RANLIB="" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
 ##disable_tracing##
 """
@@ -371,6 +373,7 @@ export CFLAGS="-U_FORTIFY_SOURCE -fstack-protector -Wall"
 export CXXFLAGS="-U_FORTIFY_SOURCE -fstack-protector -Wall"
 export ASMFLAGS="-U_FORTIFY_SOURCE -fstack-protector -Wall"
 ##enable_tracing##
+##enable_parallel_build##
 cmake -DCMAKE_AR="/usr/bin/ar" -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=gold -Wl -no-as-needed" -DCMAKE_BUILD_TYPE="Debug" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$;/abc/def" -DCMAKE_RANLIB="" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
 ##disable_tracing##
 """
@@ -437,6 +440,7 @@ set(CMAKE_SHARED_LINKER_FLAGS_INIT "$$__var_CMAKE_SHARED_LINKER_FLAGS_INIT$$")
 EOF
 
 ##enable_tracing##
+##enable_parallel_build##
 cmake -DNOFORTRAN="on" -DCMAKE_TOOLCHAIN_FILE="$$BUILD_TMPDIR$$/crosstool_bazel.cmake" -DCMAKE_BUILD_TYPE="Debug" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$" -DCMAKE_RANLIB="" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
 ##disable_tracing##
 """
@@ -504,6 +508,7 @@ export CXXFLAGS="--quoted=\\\"abc def\\\" --sysroot=/abc/sysroot --gcc_toolchain
 export ASMFLAGS="assemble assemble-user"
 export CUSTOM_ENV="YES"
 ##enable_tracing##
+##enable_parallel_build##
 cmake -DCMAKE_AR="/cxx_linker_static" -DCMAKE_CXX_LINK_EXECUTABLE="became" -DCMAKE_SHARED_LINKER_FLAGS="shared1 shared2" -DCMAKE_EXE_LINKER_FLAGS="executable" -DCMAKE_BUILD_TYPE="user_type" -DCUSTOM_CACHE="YES" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$" -DCMAKE_RANLIB="" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
 ##disable_tracing##
 """
@@ -571,6 +576,7 @@ export CXXFLAGS="--quoted=\\\"abc def\\\" --sysroot=/abc/sysroot --gcc_toolchain
 export ASMFLAGS="assemble assemble-user"
 export CUSTOM_ENV="YES"
 ##enable_tracing##
+##enable_parallel_build##
 cmake -DCMAKE_AR="/cxx_linker_static" -DCMAKE_CXX_LINK_EXECUTABLE="became" -DCMAKE_SHARED_LINKER_FLAGS="shared1 shared2" -DCMAKE_EXE_LINKER_FLAGS="executable" -DCMAKE_BUILD_TYPE="user_type" -DCUSTOM_CACHE="YES" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$" -DCMAKE_RANLIB="" -DANDROID="YES" -DCMAKE_SYSTEM_NAME="Linux" -DCMAKE_SYSTEM_PROCESSOR="x86_64" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
 ##disable_tracing##
 """
@@ -638,6 +644,7 @@ export CXXFLAGS="--quoted=\\\"abc def\\\" --sysroot=/abc/sysroot --gcc_toolchain
 export ASMFLAGS="assemble assemble-user"
 export CUSTOM_ENV="YES"
 ##enable_tracing##
+##enable_parallel_build##
 cmake -DCMAKE_AR="/cxx_linker_static" -DCMAKE_CXX_LINK_EXECUTABLE="became" -DCMAKE_SHARED_LINKER_FLAGS="shared1 shared2" -DCMAKE_EXE_LINKER_FLAGS="executable" -DCMAKE_BUILD_TYPE="user_type" -DCUSTOM_CACHE="YES" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$" -DCMAKE_RANLIB="" -DCMAKE_SYSTEM_NAME="Linux" -DCMAKE_SYSTEM_PROCESSOR="aarch64" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
 ##disable_tracing##
 """
@@ -726,6 +733,7 @@ EOF
 
 export CUSTOM_ENV="YES"
 ##enable_tracing##
+##enable_parallel_build##
 cmake -DCUSTOM_CACHE="YES" -DCMAKE_TOOLCHAIN_FILE="$$BUILD_TMPDIR$$/crosstool_bazel.cmake" -DCMAKE_BUILD_TYPE="Debug" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$" -DCMAKE_RANLIB="" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
 ##disable_tracing##
 """