From 8ebbd9733c905b04e0263a69a0702f6ead8ba183 Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 4 Aug 2022 09:59:41 -0400 Subject: [PATCH 01/13] Tidy up mysystem etc To avoid running 'mv' if the link command failed --- Signed-off-by: Michael Ferguson --- compiler/dyno/include/chpl/util/subprocess.h | 26 +++---- compiler/dyno/lib/util/subprocess.cpp | 72 +++++++++++++------- compiler/util/mysystem.cpp | 29 ++++---- 3 files changed, 76 insertions(+), 51 deletions(-) diff --git a/compiler/dyno/include/chpl/util/subprocess.h b/compiler/dyno/include/chpl/util/subprocess.h index 40eb416e0016..6ddc9ea5203e 100644 --- a/compiler/dyno/include/chpl/util/subprocess.h +++ b/compiler/dyno/include/chpl/util/subprocess.h @@ -29,21 +29,23 @@ namespace chpl { /** - * Launch a subprocess - * - * commandVec first element is the command to run, the rest are the arguments. - * description a string representation of the command to run - * childPidOut output parameter for the child's pid - * ignoreStatus if true, don't abort if the child exits with a non-zero - * quiet if true, won't print systemCommands - * printSystemCommands if true, print the command to be run - * returns int as program exit status if fork successful, -1 otherwise + Launch a subprocess + + commandVec + first element is the command to run, the rest are the arguments. + + description + a string representation of the command to run + + printSystemCommands if true, print the command to be run + + returns + -1 if there was an error in fork, waitpid, or the child process was killed + 255 if the exec call failed + the exit code from the subprocess (0-255) otherwise */ int executeAndWait(const std::vector commandVec, std::string description, - pid_t& childPidOut, - bool ignoreStatus = false, - bool quiet = false, bool printSystemCommands = false); #endif diff --git a/compiler/dyno/lib/util/subprocess.cpp b/compiler/dyno/lib/util/subprocess.cpp index 07958093f65e..8e4d8b0961db 100644 --- a/compiler/dyno/lib/util/subprocess.cpp +++ b/compiler/dyno/lib/util/subprocess.cpp @@ -28,61 +28,85 @@ #include #include - namespace chpl { // TODO: body of this function should call llvm::sys::ExecuteAndWait instead // see: https://llvm.org/doxygen/namespacellvm_1_1sys.html#a67688ad4697f1d516a7c71e41d78c5ba int executeAndWait(const std::vector commandVec, - std::string description, - pid_t& childPidOut, - bool ignoreStatus, - bool quiet, - bool printSystemCommands) { + std::string description, + bool printSystemCommands) { // if an empty command passed, do nothing - if (commandVec.empty()) { + if (commandVec.empty() || commandVec[0].empty()) { return 0; } - // Join the elements of commandVec into a single space separated string - std::string commandStr = commandVec.front(); - for (unsigned int i = 1; i < commandVec.size(); i++) { - commandStr += " " + commandVec.at(i); + // Treat a '#' at the start of a line as a comment + if (commandVec[0][0] == '#') { + return 0; } int status = -1; std::vector execArgs; - for (unsigned int i = 0; i < commandVec.size(); ++i) { - execArgs.push_back(commandVec[i].c_str()); + for (const auto& elt : commandVec) { + execArgs.push_back(elt.c_str()); } execArgs.push_back(NULL); execArgs.shrink_to_fit(); - if (printSystemCommands && !quiet) { + if (printSystemCommands) { + // Join the elements of commandVec into a single space separated string + std::string commandStr = commandVec.front(); + for (unsigned int i = 1; i < commandVec.size(); i++) { + commandStr += " " + commandVec.at(i); + } + printf("\n# %s\n", description.c_str()); printf("%s\n", commandStr.c_str()); fflush(stdout); fflush(stderr); } - // Treat a '#' at the start of a line as a comment - if (commandStr.c_str()[0] == '#') - return 0; + pid_t childPid = fork(); - childPidOut = fork(); - - if (childPidOut == 0) { + if (childPid == 0) { // in child process execvp(execArgs[0], const_cast (execArgs.data())); - } else if (childPidOut > 0 ) { + // if execvp returned, there was an error + if (printSystemCommands) { + printf("error in exec: %s\n", strerror(errno)); + } + // indicate problem with exec by exiting the child process + exit(-1); + } else if (childPid > 0 ) { // in parent process - waitpid(childPidOut, &status, 0); + int w = waitpid(childPid, &status, 0); + if (w == -1) { + if (printSystemCommands) { + printf("error in waitpid: %s\n", strerror(errno)); + } + // indicate problem with waiting + return -1; + } + if (!WIFEXITED(status)) { + // indicate program did not exit normally + if (printSystemCommands) { + printf("child process did not exit normally\n"); + } + // indicate program did not exit normally + return -1; + } // filter status down to key bits - status = WEXITSTATUS(status); + int code = WEXITSTATUS(status); + if (printSystemCommands && code != 0) { + printf("child process exited with code %i\n", code); + } + return code; } - return status; + + // this can be reached if the 'fork' failed + return -1; } diff --git a/compiler/util/mysystem.cpp b/compiler/util/mysystem.cpp index de0c08f6e6f0..29bf16ad5c24 100644 --- a/compiler/util/mysystem.cpp +++ b/compiler/util/mysystem.cpp @@ -37,9 +37,9 @@ bool printSystemCommands = false; int myshell(const char* command, - const char* description, - bool ignoreStatus, - bool quiet) { + const char* description, + bool ignoreStatus, + bool quiet) { int status = 0; @@ -57,7 +57,7 @@ int myshell(const char* command, status = system(command); if (status == -1) { - USR_FATAL("system() fork failed: %s", strerror(errno)); + USR_FATAL("%s: system() fork failed: %s", description, strerror(errno)); } else if (status != 0 && ignoreStatus == false) { USR_FATAL("%s", description); @@ -81,17 +81,16 @@ int mysystem(const std::vector commandVec, const char* description, bool ignoreStatus, bool quiet) { - pid_t childPid = 0; - int status = chpl::executeAndWait(commandVec, std::string(description), childPid, - ignoreStatus, quiet, printSystemCommands); - if (childPid == 0) { - // if there was an error and we shouldn't ignore them, then exit - if (status != 0 && !ignoreStatus) { - USR_FATAL("%s", description); - } - // handle the case when the child couldn't be forked - } else if (childPid == -1) { - USR_FATAL("fork() failed: %s", strerror(errno)); + int status = chpl::executeAndWait(commandVec, std::string(description), + printSystemCommands && !quiet); + + // if there was an error fork/waiting + if (status == -1) { + USR_FATAL("%s", description); + + } else if (status != 0 && ignoreStatus == false) { + USR_FATAL("%s", description); } + return status; } From 83e693ab8ab8d4769acd577d45fb4bf28def5e49 Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 4 Aug 2022 11:56:31 -0400 Subject: [PATCH 02/13] Add read_bundled_pkg_config_file helper --- Signed-off-by: Michael Ferguson --- util/chplenv/third_party_utils.py | 85 +++++++++++++++++++------------ 1 file changed, 52 insertions(+), 33 deletions(-) diff --git a/util/chplenv/third_party_utils.py b/util/chplenv/third_party_utils.py index 9d9d1ffca03a..d63d26dfd534 100644 --- a/util/chplenv/third_party_utils.py +++ b/util/chplenv/third_party_utils.py @@ -124,31 +124,21 @@ def pkgconfig_get_system_compile_args(pkg): # paths to a previous third-party install directory with the current one. # # pkg is the name of the subdir in third-party -# ucp is the unique config path -# pcfile is the name of the .pc file +# ucp is the unique config path (defaults to default_uniq_cfg_path()) +# pcfile is the name of the .pc file (defaults to .pc) # # This function looks for # third-party//install//lib/pkgconfig/ # # Returns a 2-tuple of lists # (compiler_bundled_args, compiler_system_args) -def pkgconfig_get_bundled_compile_args(pkg, ucp, pcfile): - install_path = get_bundled_install_path(pkg, ucp) +def pkgconfig_get_bundled_compile_args(pkg, ucp='', pcfile=''): + d = read_bundled_pkg_config_file(pkg, ucp, pcfile) - # give up early if the 3rd party package hasn't been built - if not os.path.exists(install_path): + # Return empty tuple if no .pc file was found (e.g. pkg not built yet) + if d == None: return ([ ], [ ]) - pcpath = os.path.join(install_path, 'lib', 'pkgconfig', pcfile) - - # if we get this far, we should have a .pc file. check that it exists. - if not os.access(pcpath, os.R_OK): - error("Could not find '{0}'".format(pcpath), ValueError) - - d = read_pkg_config_file(pcpath, - os.path.join('third-party', pkg, 'install', ucp), - install_path) - if 'Requires' in d and d['Requires']: warning("Simple pkg-config parser does not handle Requires") warning("in {0}".format(pcpath)) @@ -215,6 +205,7 @@ def pkgconfig_get_system_link_args(pkg, static=pkgconfig_default_static()): def pkgconfig_get_bundled_link_args(pkg, ucp='', pcfile='', static=pkgconfig_default_static(), add_rpath=True): + # compute the default ucp if ucp == '': ucp = default_uniq_cfg_path() @@ -223,22 +214,13 @@ def pkgconfig_get_bundled_link_args(pkg, ucp='', pcfile='', pcfile = pkg + '.pc' install_path = get_bundled_install_path(pkg, ucp) - - # give up early if the 3rd party package hasn't been built - if not os.path.exists(install_path): - return ([ ], [ ]) - - lib_dir = os.path.join(install_path, 'lib') - pcpath = os.path.join(lib_dir, 'pkgconfig', pcfile) - # if we get this far, we should have a .pc file. check that it exists. - if not os.access(pcpath, os.R_OK): - error("Could not find '{0}'".format(pcpath), ValueError) + d = read_bundled_pkg_config_file(pkg, ucp, pcfile) - d = read_pkg_config_file(pcpath, - os.path.join('third-party', pkg, 'install', ucp), - install_path) + # Return empty tuple if no .pc file was found (e.g. pkg not built yet) + if d == None: + return ([ ], [ ]) if 'Requires' in d and d['Requires']: warning("Simple pkg-config parser does not handle Requires") @@ -352,7 +334,7 @@ def apply_pkgconfig_subs(s, d): # (i.e. Requires: lines). However this version is intended to # be sufficient for the third-party packages we do have. # -# If find_third_party and replace_third_party are provided, +# If find_third_party and replace_third_party not None, # they should be e.g. # find_third_party=third-party/gasnet/install/some-ucp # replace_third_party=/path/to/chpl-home/third-party/gasnet/install/some-ucp @@ -362,9 +344,7 @@ def apply_pkgconfig_subs(s, d): # $replace_third_party # @memoize -def read_pkg_config_file(pcpath, - find_third_party=None, - replace_third_party=None): +def read_pkg_config_file(pcpath, find_third_party, replace_third_party): ret = { } pattern = None @@ -405,3 +385,42 @@ def read_pkg_config_file(pcpath, ret[key] = val return ret + +# Helper function to call read_pkg_config_file for a bundled +# pkg-config file. +# +# pkg should be the bundled package name (e.g. gasnet) +# ucp is the unique config path (defaults to default_uniq_cfg_path()) +# pcfile is the .pc file name (defaults to .pc) +# +# This function looks for +# third-party//install//lib/pkgconfig/ +# where pcfile defaults to pkg.pc +# +# If the .pc file could not be read, returns None +# Otherwise, returns the dictionary of values read from the .pc file +def read_bundled_pkg_config_file(pkg, ucp='', pcfile=''): + # compute the default ucp + if ucp == '': + ucp = default_uniq_cfg_path() + # compute the default pcfile name + if pcfile == '': + pcfile = pkg + '.pc' + + install_path = get_bundled_install_path(pkg, ucp) + + # give up early if the 3rd party package hasn't been built + if not os.path.exists(install_path): + return None + + pcpath = os.path.join(install_path, 'lib', 'pkgconfig', pcfile) + + # if we get this far, we should have a .pc file. check that it exists. + if not os.access(pcpath, os.R_OK): + error("Could not find '{0}'".format(pcpath), ValueError) + return None + + find_path = os.path.join('third-party', pkg, 'install', ucp) + replace_path = install_path + + return read_pkg_config_file(pcpath, find_path, replace_path) From 8edd2ada89c03d5899602ab09290a686faf1095e Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 4 Aug 2022 11:56:56 -0400 Subject: [PATCH 03/13] Add CHPL_TARGET_LD to printchplenv Only gasnet configurations set it to something other than CHPL_TARGET_CXX. --- Signed-off-by: Michael Ferguson --- util/chplenv/chpl_gasnet.py | 56 ++++++++++++++++++++++++- util/chplenv/compile_link_args_utils.py | 11 +++++ util/chplenv/printchplenv.py | 3 ++ 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/util/chplenv/chpl_gasnet.py b/util/chplenv/chpl_gasnet.py index 60d6a6eb15d5..6c0485b87568 100644 --- a/util/chplenv/chpl_gasnet.py +++ b/util/chplenv/chpl_gasnet.py @@ -1,5 +1,5 @@ import chpl_compiler, chpl_comm_debug, chpl_comm_segment, chpl_comm_substrate -import chpl_home_utils, compiler_utils, third_party_utils +import overrides, chpl_home_utils, compiler_utils, third_party_utils from utils import error, warning, memoize import os @@ -88,3 +88,57 @@ def get_link_args(): tup = third_party_utils.pkgconfig_get_bundled_link_args( 'gasnet', get_uniq_cfg_path(), get_gasnet_pc_file()) return (filter_link_args(tup[0]), filter_link_args(tup[1])) + +# returns the special linker to use, or None if there is none +# (e.g. it might link with mpicc) +@memoize +def get_override_ld(): + d = third_party_utils.read_bundled_pkg_config_file( + 'gasnet', get_uniq_cfg_path(), get_gasnet_pc_file()) + + if d == None: + return None # no ld override + + if 'GASNET_LD' in d: + gasnet_ld = d['GASNET_LD'] + gasnet_cc = d['GASNET_CC'] + gasnet_cxx = d['GASNET_CXX'] + + ld = None # no ld override + + gasnet_ld_requires_mpi = False + ld_name = os.path.basename(gasnet_ld).split()[0] + if 'mpi' in ld_name: + gasnet_ld_requires_mpi = True + + mpi_cxx = overrides.get('MPI_CXX') + if not mpi_cxx: + mpi_cxx = 'mpicxx' + + target_compiler_llvm = False + if chpl_compiler.get('target') == 'llvm': + target_compiler_llvm = True + + # For the linker, we tend to want a C++ linker, in order to support + # linking in components developed in C++ like re2 or user C++ code. + # However, GASNet doesn't provide any guarantees that their linker + # will be a C++ compiler. Based on conversation with the GASNet + # team, we should expect it either to be GASNET_CC, GASNET_CXX, or + # MPI_CC. The following conditional handles these cases, switching + # to the C++ choice in the first and third cases. + if gasnet_ld == gasnet_cxx: + if not target_compiler_llvm: + ld = gasnet_ld # GASNet chose C++ linker so stick with it + elif gasnet_ld == gasnet_cc: + if not target_compiler_llvm: + ld = gasnet_cxx # GASNet chose C, so switch to C++ + elif gasnet_ld_requires_mpi: + ld = mpi_cxx + else: + warning("GASNET_LD unexpectedly set to '{0}'. " + "Please file a Chapel bug against this.".format(gasnet_ld)) + ld = None + + return ld + + return None diff --git a/util/chplenv/compile_link_args_utils.py b/util/chplenv/compile_link_args_utils.py index 73cc6298b5c1..42e4f2840a37 100644 --- a/util/chplenv/compile_link_args_utils.py +++ b/util/chplenv/compile_link_args_utils.py @@ -200,3 +200,14 @@ def compute_internal_compile_link_args(runtime_subdir): 'host_link': host_link, 'target_compile': tgt_compile, 'target_link': tgt_link} + +# return the target linker to use (it can be overriden by gasnet) +# this function returns an array of arguments +# e.g. ['clang++', '--gcc-toolchain=/usr'] +def get_target_link_command(): + if chpl_comm.get() == 'gasnet': + override_ld = chpl_gasnet.get_override_ld() + if override_ld != None: + return override_ld.split() + + return chpl_compiler.get_compiler_command('target', 'cxx') diff --git a/util/chplenv/printchplenv.py b/util/chplenv/printchplenv.py index 861f34809f6c..c75bc245cc31 100755 --- a/util/chplenv/printchplenv.py +++ b/util/chplenv/printchplenv.py @@ -80,6 +80,7 @@ ChapelEnv(' CHPL_TARGET_COMPILER_PRGENV', INTERNAL), ChapelEnv(' CHPL_TARGET_BUNDLED_COMPILE_ARGS', INTERNAL), ChapelEnv(' CHPL_TARGET_SYSTEM_COMPILE_ARGS', INTERNAL), + ChapelEnv(' CHPL_TARGET_LD', RUNTIME | NOPATH), ChapelEnv(' CHPL_TARGET_BUNDLED_LINK_ARGS', INTERNAL), ChapelEnv(' CHPL_TARGET_SYSTEM_LINK_ARGS', INTERNAL), ChapelEnv('CHPL_TARGET_ARCH', RUNTIME | DEFAULT), @@ -166,10 +167,12 @@ def compute_all_values(): target_compiler_c = chpl_compiler.get_compiler_command('target', 'c') target_compiler_cpp = chpl_compiler.get_compiler_command('target', 'c++') target_compiler_prgenv = chpl_compiler.get_prgenv_compiler() + target_linker = compile_link_args_utils.get_target_link_command() ENV_VALS['CHPL_TARGET_COMPILER'] = target_compiler ENV_VALS[' CHPL_TARGET_CC'] = " ".join(target_compiler_c) ENV_VALS[' CHPL_TARGET_CXX'] = " ".join(target_compiler_cpp) ENV_VALS[' CHPL_TARGET_COMPILER_PRGENV'] = target_compiler_prgenv + ENV_VALS[' CHPL_TARGET_LD'] = " ".join(target_linker) ENV_VALS['CHPL_TARGET_ARCH'] = chpl_arch.get('target') ENV_VALS['CHPL_TARGET_CPU'] = chpl_cpu.get('target').cpu From 7e5f3092d78dc3e1e87b690b6c9bb78e58be6f2f Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 4 Aug 2022 13:25:40 -0400 Subject: [PATCH 04/13] Use CHPL_TARGET_LD instead of override-ld in compiler --- Signed-off-by: Michael Ferguson --- compiler/include/driver.h | 1 + compiler/llvm/clangUtil.cpp | 16 ++-------------- compiler/main/driver.cpp | 2 ++ 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/compiler/include/driver.h b/compiler/include/driver.h index 78176788341b..34a49884a45a 100644 --- a/compiler/include/driver.h +++ b/compiler/include/driver.h @@ -140,6 +140,7 @@ extern const char* CHPL_LLVM_CLANG_CXX; extern const char* CHPL_TARGET_BUNDLED_COMPILE_ARGS; extern const char* CHPL_TARGET_SYSTEM_COMPILE_ARGS; +extern const char* CHPL_TARGET_LD; extern const char* CHPL_TARGET_BUNDLED_LINK_ARGS; extern const char* CHPL_TARGET_SYSTEM_LINK_ARGS; diff --git a/compiler/llvm/clangUtil.cpp b/compiler/llvm/clangUtil.cpp index 6865d676571a..300e7da14956 100644 --- a/compiler/llvm/clangUtil.cpp +++ b/compiler/llvm/clangUtil.cpp @@ -4353,29 +4353,17 @@ void makeBinaryLLVM(void) { std::string options = ""; - std::string maino(CHPL_RUNTIME_LIB); maino += "/"; maino += CHPL_RUNTIME_SUBDIR; maino += "/main.o"; - // TODO: move this logic to printchplenv - std::string runtime_ld_override(CHPL_RUNTIME_LIB); - runtime_ld_override += "/"; - runtime_ld_override += CHPL_RUNTIME_SUBDIR; - runtime_ld_override += "/override-ld"; - - std::vector ldOverride; - readArgsFromFile(runtime_ld_override, ldOverride); - // Substitute $CHPL_HOME $CHPL_RUNTIME_LIB etc - expandInstallationPaths(ldOverride); - std::string clangCC = clangInfo->clangCC; std::string clangCXX = clangInfo->clangCXX; std::string useLinkCXX = clangCXX; - if (ldOverride.size() > 0) - useLinkCXX = ldOverride[0]; + if (CHPL_TARGET_LD != nullptr && CHPL_TARGET_LD[0] != '\0') + useLinkCXX = CHPL_TARGET_LD; // start with arguments from CHPL_LLVM_CLANG_C unless // using a non-clang compiler to link diff --git a/compiler/main/driver.cpp b/compiler/main/driver.cpp index ef360c5f314c..0461da5e3674 100644 --- a/compiler/main/driver.cpp +++ b/compiler/main/driver.cpp @@ -109,6 +109,7 @@ const char* CHPL_LLVM_CLANG_CXX = NULL; const char* CHPL_TARGET_BUNDLED_COMPILE_ARGS = NULL; const char* CHPL_TARGET_SYSTEM_COMPILE_ARGS = NULL; +const char* CHPL_TARGET_LD = NULL; const char* CHPL_TARGET_BUNDLED_LINK_ARGS = NULL; const char* CHPL_TARGET_SYSTEM_LINK_ARGS = NULL; @@ -1455,6 +1456,7 @@ static void setChapelEnvs() { CHPL_TARGET_BUNDLED_COMPILE_ARGS = envMap["CHPL_TARGET_BUNDLED_COMPILE_ARGS"]; CHPL_TARGET_SYSTEM_COMPILE_ARGS = envMap["CHPL_TARGET_SYSTEM_COMPILE_ARGS"]; + CHPL_TARGET_LD = envMap["CHPL_TARGET_LD"]; CHPL_TARGET_BUNDLED_LINK_ARGS = envMap["CHPL_TARGET_BUNDLED_LINK_ARGS"]; CHPL_TARGET_SYSTEM_LINK_ARGS = envMap["CHPL_TARGET_SYSTEM_LINK_ARGS"]; From fde45bb173582dd83cea5ee862e4ba47b78d1a2f Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 11 Aug 2022 10:10:47 -0400 Subject: [PATCH 05/13] Update overrides.py to include more printchplenv vars At least include those from printchplenv --all --no-tidy --- Signed-off-by: Michael Ferguson --- util/chplenv/overrides.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/util/chplenv/overrides.py b/util/chplenv/overrides.py index 8d1fe9d2c557..f466986dd394 100755 --- a/util/chplenv/overrides.py +++ b/util/chplenv/overrides.py @@ -9,48 +9,70 @@ from utils import memoize, warning # List of Chapel Environment Variables +# Note: this array lists them in an order that matches +# printchplenv --all --no-tidy +# But some variables are included here so that they can be overridden +# in a chplconfig file even though they are not in that output. +# When such variables are related to others, we include them in the same group. chplvars = [ 'CHPL_HOME', + 'CHPL_HOST_PLATFORM', 'CHPL_HOST_COMPILER', 'CHPL_HOST_CC', 'CHPL_HOST_CXX', 'CHPL_HOST_ARCH', - 'CHPL_HOST_CPU', - 'CHPL_HOST_MEM', + 'CHPL_HOST_CPU', # note: not in printchplenv --all + 'CHPL_TARGET_PLATFORM', 'CHPL_TARGET_COMPILER', 'CHPL_TARGET_CC', 'CHPL_TARGET_CXX', + 'CHPL_TARGET_LD', 'CHPL_TARGET_ARCH', 'CHPL_TARGET_CPU', - 'CHPL_TARGET_MEM', + 'CHPL_LOCALE_MODEL', + 'CHPL_GPU_CODEGEN', + 'CHPL_GPU_RUNTIME', + 'CHPL_CUDA_PATH', + 'CHPL_COMM', 'CHPL_COMM_SUBSTRATE', 'CHPL_GASNET_SEGMENT', 'CHPL_LIBFABRIC', + 'CHPL_TASKS', 'CHPL_LAUNCHER', 'CHPL_TIMERS', 'CHPL_UNWIND', + + 'CHPL_HOST_MEM', + 'CHPL_TARGET_MEM', # note: not in printchplenv --all 'CHPL_MEM', - 'CHPL_MAKE', + 'CHPL_JEMALLOC', # note: these 3 are not in printchplenv --all + 'CHPL_HOST_JEMALLOC', + 'CHPL_TARGET_JEMALLOC', + 'CHPL_ATOMICS', 'CHPL_NETWORK_ATOMICS', + 'CHPL_GMP', 'CHPL_HWLOC', - 'CHPL_JEMALLOC', - 'CHPL_HOST_JEMALLOC', - 'CHPL_TARGET_JEMALLOC', 'CHPL_RE2', + 'CHPL_LLVM', 'CHPL_LLVM_CONFIG', + # CHPL_LLVM_VERSION -- doesn't make sense to override it + 'CHPL_LLVM_GCC_PREFIX', # not in printchplenv --all + 'CHPL_AUX_FILESYS', 'CHPL_LIB_PIC', 'CHPL_SANITIZE', 'CHPL_SANITIZE_EXE', - 'CHPL_LLVM_GCC_PREFIX', + + # These are not in printchplenv --all + 'CHPL_MAKE', 'CHPL_HOST_USE_SYSTEM_LIBCXX', ] From e423cd5a6401cc5e91a8768742e4f50368f7cd9f Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 11 Aug 2022 11:16:21 -0400 Subject: [PATCH 06/13] Fix up chpl-env-gen.h and test of it Use a more comprehensive solution to get to characters that can be in a #define and use the same strategy in the test code as well as the Makefile generating it. --- Signed-off-by: Michael Ferguson --- runtime/Makefile | 10 +++------- test/runtime/sungeun/chpl-env-gen.precomp | 12 +++++++----- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/runtime/Makefile b/runtime/Makefile index b30e6d241085..26eaa3f362ed 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -59,13 +59,9 @@ $(CHPL_ENV_HEADER): $(CHPL_MAKE_HOME)/util/printchplenv $(CHPL_MAKE_HOME)/util/c @echo "#ifndef _CHPL_ENV_GEN_H_" >> $(CHPL_ENV_HEADER) @echo "#define _CHPL_ENV_GEN_H_" >> $(CHPL_ENV_HEADER) @$(CHPL_MAKE_HOME)/util/printchplenv --runtime --no-tidy --anonymize --simple | \ - grep -v CHPL_HOST_CC | \ - grep -v CHPL_HOST_CXX | \ - grep -v CHPL_TARGET_CC | \ - grep -v CHPL_TARGET_CXX | \ - sed s/[-+.\/]/_/g | \ - sed s/\=/' '/ | \ - awk '{ print "#define " $$1 "_" toupper($$2) }' >> $(CHPL_ENV_HEADER) + sed 's/^[ \t]*//;s/[ \t]*$$//' | \ + sed 's/[^0-9A-Za-z]/_/g' | \ + awk '{ print "#define " toupper($$1) }' >> $(CHPL_ENV_HEADER) @echo "#endif /* _CHPL_ENV_GEN_H_ */" >> $(CHPL_ENV_HEADER) THIRD_PARTY_PKGS = $(shell $(CHPL_MAKE_PYTHON) $(CHPL_MAKE_HOME)/util/chplenv/third-party-pkgs) diff --git a/test/runtime/sungeun/chpl-env-gen.precomp b/test/runtime/sungeun/chpl-env-gen.precomp index 42bc95c9dc07..dfcdf0acbf49 100755 --- a/test/runtime/sungeun/chpl-env-gen.precomp +++ b/test/runtime/sungeun/chpl-env-gen.precomp @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import re import shutil import subprocess import sys @@ -19,11 +20,12 @@ testname = sys.argv[1] genfile = testname+'.test.gen.c' with open(genfile, 'w') as f: for key,val in chpl_env.items(): - if (key == 'CHPL_HOST_CC' or key == 'CHPL_HOST_CXX' or - key == 'CHPL_TARGET_CC' or key == 'CHPL_TARGET_CXX'): - continue - - key_val = key+'_'+val.replace('-', '_').replace('/', '_').upper() + key = key.strip() + val = val.strip() + # munge characters similarly to Makefile generating them + key_val = key+'_'+val + key_val = re.sub(r'[^0-9A-Za-z]', '_', key_val) + key_val = key_val.upper() f.write('#ifndef %s\n'%(key_val)) f.write('#error "%s undefined or does not match runtime definition"\n'%(val)) f.write('#endif\n') From 2d3e26c67faab519904e177140b9760320794b71 Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 11 Aug 2022 12:56:32 -0400 Subject: [PATCH 07/13] Tidy up arguments to executeAndWait --- Signed-off-by: Michael Ferguson --- compiler/dyno/include/chpl/util/subprocess.h | 6 +++--- compiler/dyno/lib/util/subprocess.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/dyno/include/chpl/util/subprocess.h b/compiler/dyno/include/chpl/util/subprocess.h index 6ddc9ea5203e..44c6d14e393d 100644 --- a/compiler/dyno/include/chpl/util/subprocess.h +++ b/compiler/dyno/include/chpl/util/subprocess.h @@ -44,9 +44,9 @@ namespace chpl { 255 if the exec call failed the exit code from the subprocess (0-255) otherwise */ -int executeAndWait(const std::vector commandVec, - std::string description, - bool printSystemCommands = false); +int executeAndWait(const std::vector& commandVec, + const std::string& description, + bool printSystemCommands = false); #endif diff --git a/compiler/dyno/lib/util/subprocess.cpp b/compiler/dyno/lib/util/subprocess.cpp index 8e4d8b0961db..e50a4948862d 100644 --- a/compiler/dyno/lib/util/subprocess.cpp +++ b/compiler/dyno/lib/util/subprocess.cpp @@ -33,9 +33,9 @@ namespace chpl { // TODO: body of this function should call llvm::sys::ExecuteAndWait instead // see: https://llvm.org/doxygen/namespacellvm_1_1sys.html#a67688ad4697f1d516a7c71e41d78c5ba -int executeAndWait(const std::vector commandVec, - std::string description, - bool printSystemCommands) { +int executeAndWait(const std::vector& commandVec, + const std::string& description, + bool printSystemCommands) { // if an empty command passed, do nothing if (commandVec.empty() || commandVec[0].empty()) { From c816dc48fd6a84a159c5b5225cfc90b7caf6d0ff Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 11 Aug 2022 12:56:51 -0400 Subject: [PATCH 08/13] Add a test for executeAndWait --- Signed-off-by: Michael Ferguson --- compiler/dyno/test/util/CMakeLists.txt | 5 +- compiler/dyno/test/util/testSubprocess.cpp | 77 ++++++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 compiler/dyno/test/util/testSubprocess.cpp diff --git a/compiler/dyno/test/util/CMakeLists.txt b/compiler/dyno/test/util/CMakeLists.txt index b9cf3e0b57e5..e50a06b2a5ab 100644 --- a/compiler/dyno/test/util/CMakeLists.txt +++ b/compiler/dyno/test/util/CMakeLists.txt @@ -15,7 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -comp_unit_test(testQuoteStrings) -comp_unit_test(testQueryTimingAndTrace) comp_unit_test(testIteratorAdapters) comp_unit_test(testLLVMsupport) +comp_unit_test(testQueryTimingAndTrace) +comp_unit_test(testQuoteStrings) +comp_unit_test(testSubprocess) diff --git a/compiler/dyno/test/util/testSubprocess.cpp b/compiler/dyno/test/util/testSubprocess.cpp new file mode 100644 index 000000000000..c465ee0af049 --- /dev/null +++ b/compiler/dyno/test/util/testSubprocess.cpp @@ -0,0 +1,77 @@ +/* + * Copyright 2021-2022 Hewlett Packard Enterprise Development LP + * Other additional copyright holders may be indicated within. + * + * The entirety of this work is licensed under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "chpl/util/subprocess.h" + +#include + +// always check assertions in this test +#ifdef NDEBUG +#undef NDEBUG +#endif + +using namespace chpl; + +// check that trying to run a nonexistant command returns an error +static void checkNonexistent() { + int rc; + std::vector cmd; + + cmd.clear(); + cmd.push_back("./nonexistent-command"); + rc = executeAndWait(cmd, "nonexistent-command description", true); + assert(rc != 0); + + cmd.clear(); + cmd.push_back("./nonexistent-command"); + cmd.push_back("arg"); + rc = executeAndWait(cmd, "nonexistent-command description", true); + assert(rc != 0); + + cmd.clear(); + cmd.push_back("/this/path/does/not/exist"); + rc = executeAndWait(cmd, "path does not exist description", true); + assert(rc != 0); + + cmd.clear(); + cmd.push_back("/this/path/does/not/exist"); + cmd.push_back("arg"); + rc = executeAndWait(cmd, "path does not exist description", true); + assert(rc != 0); +} + +// check that we can run /usr/bin/env and get 0 return code from it +static void checkEnv() { + int rc; + std::vector cmd; + + cmd.clear(); + cmd.push_back("/usr/bin/env"); + rc = executeAndWait(cmd, "testing that env can be run", true); + assert(rc == 0); + +} + +int main(int argc, char** argv) { + + checkNonexistent(); + checkEnv(); + + return 0; +} From 91b32f431501e6c0b1bcc10b16f4fae36444bf56 Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 11 Aug 2022 13:16:36 -0400 Subject: [PATCH 09/13] Add CHPL_LLVM_SUPPORT to overrides.py to allow it to be set in a chplconfig --- Signed-off-by: Michael Ferguson --- util/chplenv/overrides.py | 1 + 1 file changed, 1 insertion(+) diff --git a/util/chplenv/overrides.py b/util/chplenv/overrides.py index f466986dd394..8d985ccd83f5 100755 --- a/util/chplenv/overrides.py +++ b/util/chplenv/overrides.py @@ -62,6 +62,7 @@ 'CHPL_RE2', 'CHPL_LLVM', + 'CHPL_LLVM_SUPPORT', 'CHPL_LLVM_CONFIG', # CHPL_LLVM_VERSION -- doesn't make sense to override it 'CHPL_LLVM_GCC_PREFIX', # not in printchplenv --all From e4f5b059482ba40b0a2c2246d2777ae104535563 Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 11 Aug 2022 13:19:59 -0400 Subject: [PATCH 10/13] Remove trailing spaces --- Signed-off-by: Michael Ferguson --- compiler/dyno/test/util/testSubprocess.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/dyno/test/util/testSubprocess.cpp b/compiler/dyno/test/util/testSubprocess.cpp index c465ee0af049..52cb96e0bc48 100644 --- a/compiler/dyno/test/util/testSubprocess.cpp +++ b/compiler/dyno/test/util/testSubprocess.cpp @@ -37,13 +37,13 @@ static void checkNonexistent() { cmd.push_back("./nonexistent-command"); rc = executeAndWait(cmd, "nonexistent-command description", true); assert(rc != 0); - + cmd.clear(); cmd.push_back("./nonexistent-command"); cmd.push_back("arg"); rc = executeAndWait(cmd, "nonexistent-command description", true); assert(rc != 0); - + cmd.clear(); cmd.push_back("/this/path/does/not/exist"); rc = executeAndWait(cmd, "path does not exist description", true); From 95b16e192faa261dddb9e75faa24ecbd3665cbf6 Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 11 Aug 2022 14:25:15 -0400 Subject: [PATCH 11/13] Add a comment --- Signed-off-by: Michael Ferguson --- runtime/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runtime/Makefile b/runtime/Makefile index 26eaa3f362ed..6ebbaa0f2d5d 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -58,6 +58,8 @@ $(CHPL_ENV_HEADER): $(CHPL_MAKE_HOME)/util/printchplenv $(CHPL_MAKE_HOME)/util/c @echo "/* THIS FILE IS AUTOMATICALLY GENERATED */" >> $(CHPL_ENV_HEADER) @echo "#ifndef _CHPL_ENV_GEN_H_" >> $(CHPL_ENV_HEADER) @echo "#define _CHPL_ENV_GEN_H_" >> $(CHPL_ENV_HEADER) + # save output of printchplenv into #defines but munge away + # characters that can't be in #defines @$(CHPL_MAKE_HOME)/util/printchplenv --runtime --no-tidy --anonymize --simple | \ sed 's/^[ \t]*//;s/[ \t]*$$//' | \ sed 's/[^0-9A-Za-z]/_/g' | \ From 1ee3d2da80cb02f3c293f2f301a4f43d68dd8d47 Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 11 Aug 2022 14:28:03 -0400 Subject: [PATCH 12/13] Consolidate conditional for comment and empty --- Signed-off-by: Michael Ferguson --- compiler/dyno/lib/util/subprocess.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/compiler/dyno/lib/util/subprocess.cpp b/compiler/dyno/lib/util/subprocess.cpp index e50a4948862d..cc7033914895 100644 --- a/compiler/dyno/lib/util/subprocess.cpp +++ b/compiler/dyno/lib/util/subprocess.cpp @@ -37,13 +37,8 @@ int executeAndWait(const std::vector& commandVec, const std::string& description, bool printSystemCommands) { - // if an empty command passed, do nothing - if (commandVec.empty() || commandVec[0].empty()) { - return 0; - } - - // Treat a '#' at the start of a line as a comment - if (commandVec[0][0] == '#') { + // if an empty command or comment is passed, do nothing + if (commandVec.empty() || commandVec[0].empty() || commandVec[0][0] == '#') { return 0; } From 71ea54fc77af200d43dcefc1a893084aa08caf00 Mon Sep 17 00:00:00 2001 From: Michael Ferguson Date: Thu, 11 Aug 2022 14:28:18 -0400 Subject: [PATCH 13/13] Add test for empty string or comment --- Signed-off-by: Michael Ferguson --- compiler/dyno/test/util/testSubprocess.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/compiler/dyno/test/util/testSubprocess.cpp b/compiler/dyno/test/util/testSubprocess.cpp index 52cb96e0bc48..2bc84e990a09 100644 --- a/compiler/dyno/test/util/testSubprocess.cpp +++ b/compiler/dyno/test/util/testSubprocess.cpp @@ -65,13 +65,34 @@ static void checkEnv() { cmd.push_back("/usr/bin/env"); rc = executeAndWait(cmd, "testing that env can be run", true); assert(rc == 0); +} + +// check that running an empty command or a comment returns 0 +static void checkEmptyOrComment() { + int rc; + std::vector cmd; + + cmd.clear(); + rc = executeAndWait(cmd, "empty command 1", true); + assert(rc == 0); + cmd.clear(); + cmd.push_back(""); + rc = executeAndWait(cmd, "empty command 2", true); + assert(rc == 0); + + cmd.clear(); + cmd.push_back("# this is a comment"); + rc = executeAndWait(cmd, "comment command", true); + assert(rc == 0); } + int main(int argc, char** argv) { checkNonexistent(); checkEnv(); + checkEmptyOrComment(); return 0; }