From 43be7f0468f19b7034969c0c33d6ead912f98aa8 Mon Sep 17 00:00:00 2001 From: Daniel Krupp Date: Wed, 31 Jul 2024 16:03:39 +0200 Subject: [PATCH] Analyzer binary dependent environment If a pre-package analyzer binary (e.g. clang) is found first, the LD_LIBRARY_PATH must be extended. On the other hand, if the analyzer binary is taken from the users machine, the original environment must be used to execute the binary. This patch introduces an analyzer binary dependent environment initialization. --- analyzer/codechecker_analyzer/analyzer.py | 2 +- .../codechecker_analyzer/analyzer_context.py | 37 +++++++--- .../analyzers/analyzer_base.py | 9 ++- .../analyzers/analyzer_types.py | 5 +- .../analyzers/clangsa/analyzer.py | 6 +- .../analyzers/clangsa/version.py | 2 +- .../analyzers/clangtidy/analyzer.py | 9 ++- .../analyzers/cppcheck/analyzer.py | 2 +- .../codechecker_analyzer/cmd/analyzers.py | 2 +- analyzer/codechecker_analyzer/env.py | 19 +++++ analyzer/codechecker_analyzer/host_check.py | 4 +- analyzer/tests/unit/test_env_var.py | 71 ++++++++++++++++++- 12 files changed, 140 insertions(+), 28 deletions(-) diff --git a/analyzer/codechecker_analyzer/analyzer.py b/analyzer/codechecker_analyzer/analyzer.py index 062c770731..9b2d524d48 100644 --- a/analyzer/codechecker_analyzer/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzer.py @@ -250,7 +250,7 @@ def perform_analysis(args, skip_handlers, rs_handler: ReviewStatusHandler, # TODO: cppcheck may require a different environment than clang. version = analyzer_types.supported_analyzers[analyzer] \ - .get_binary_version(context.analyzer_env) + .get_binary_version(context.get_analyzer_env(analyzer)) metadata_info['analyzer_statistics']['version'] = version metadata_tool['analyzers'][analyzer] = metadata_info diff --git a/analyzer/codechecker_analyzer/analyzer_context.py b/analyzer/codechecker_analyzer/analyzer_context.py index 082c508a20..21c7610398 100644 --- a/analyzer/codechecker_analyzer/analyzer_context.py +++ b/analyzer/codechecker_analyzer/analyzer_context.py @@ -65,7 +65,8 @@ def __init__(self): self.__package_build_date = None self.__package_git_hash = None self.__analyzers = {} - self.__analyzer_env = None + self.__analyzer_envs = {} + self.__cc_env = None machine = platform.uname().machine @@ -93,9 +94,9 @@ def __init__(self): def __parse_cc_analyzer_bin(self): env_var_bins = {} - if 'CC_ANALYZER_BIN' in self.analyzer_env: + if 'CC_ANALYZER_BIN' in self.cc_env: had_error = False - for value in self.__analyzer_env['CC_ANALYZER_BIN'].split(';'): + for value in self.__cc_env['CC_ANALYZER_BIN'].split(';'): try: analyzer_name, path = analyzer_binary(value) except ArgumentTypeError as e: @@ -196,17 +197,19 @@ def __set_version(self): def __populate_analyzers(self): """ Set analyzer binaries for each registered analyzers. """ - analyzer_env = None + cc_env = None analyzer_from_path = env.is_analyzer_from_path() if not analyzer_from_path: - analyzer_env = self.analyzer_env + cc_env = self.cc_env env_var_bin = self.__parse_cc_analyzer_bin() compiler_binaries = self.pckg_layout.get('analyzers') for name, value in compiler_binaries.items(): if name in env_var_bin: + # For non-packaged analyzers the original env is set. self.__analyzers[name] = env_var_bin[name] + self.__analyzer_envs[name] = env.get_original_env() continue if analyzer_from_path: @@ -216,8 +219,12 @@ def __populate_analyzers(self): # Check if it is a package relative path. self.__analyzers[name] = os.path.join( self._data_files_dir_path, value) + # For packaged analyzers the ld_library path + # must be extended with the packed libs. + self.__analyzer_envs[name] =\ + env.extend(self.path_env_extra, self.ld_lib_path_extra) else: - env_path = analyzer_env['PATH'] if analyzer_env else None + env_path = cc_env['PATH'] if cc_env else None compiler_binary = which(cmd=value, path=env_path) if not compiler_binary: LOG.debug("'%s' binary can not be found in your PATH!", @@ -226,12 +233,17 @@ def __populate_analyzers(self): continue self.__analyzers[name] = os.path.realpath(compiler_binary) + # For non-packaged analyzers the original env is set. + self.__analyzer_envs[name] = env.get_original_env() # If the compiler binary is a simlink to ccache, use the # original compiler binary. if self.__analyzers[name].endswith("/ccache"): self.__analyzers[name] = compiler_binary + def get_analyzer_env(self, analyzer_name): + return self.__analyzer_envs[analyzer_name] + def __populate_replacer(self): """ Set clang-apply-replacements tool. """ replacer_binary = self.pckg_layout.get('clang-apply-replacements') @@ -240,8 +252,12 @@ def __populate_replacer(self): # Check if it is a package relative path. self.__replacer = os.path.join(self._data_files_dir_path, replacer_binary) + self.__analyzer_envs['clang-apply-replacements'] =\ + env.extend(self.path_env_extra, self.ld_lib_path_extra) else: self.__replacer = which(replacer_binary) + self.__analyzer_envs['clang-apply-replacements'] =\ + env.get_original_env() @property def version(self): @@ -320,11 +336,10 @@ def ld_lib_path_extra(self): return ld_paths @property - def analyzer_env(self): - if not self.__analyzer_env: - self.__analyzer_env = \ - env.extend(self.path_env_extra, self.ld_lib_path_extra) - return self.__analyzer_env + def cc_env(self): + if not self.__cc_env: + self.__cc_env = os.environ.copy() + return self.__cc_env @property def analyzer_binaries(self): diff --git a/analyzer/codechecker_analyzer/analyzers/analyzer_base.py b/analyzer/codechecker_analyzer/analyzers/analyzer_base.py index d6d92b5aa8..f9c02cbe26 100644 --- a/analyzer/codechecker_analyzer/analyzers/analyzer_base.py +++ b/analyzer/codechecker_analyzer/analyzers/analyzer_base.py @@ -111,6 +111,10 @@ def analyze(self, analyzer_cmd, res_handler, proc_callback=None, env=None): LOG.debug_analyzer('\n%s', ' '.join([shlex.quote(x) for x in analyzer_cmd])) + if env is None: + env = analyzer_context.get_context()\ + .get_analyzer_env(self.ANALYZER_NAME) + res_handler.analyzer_cmd = analyzer_cmd try: ret_code, stdout, stderr \ @@ -158,7 +162,10 @@ def signal_handler(signum, _): signal.signal(signal.SIGINT, signal_handler) if env is None: - env = analyzer_context.get_context().analyzer_env + env = analyzer_context.get_context().cc_env + + LOG.debug_analyzer('\nENV:\n') + LOG.debug_analyzer(env) proc = subprocess.Popen( command, diff --git a/analyzer/codechecker_analyzer/analyzers/analyzer_types.py b/analyzer/codechecker_analyzer/analyzers/analyzer_types.py index 6e8a492ecf..fc64304bc4 100644 --- a/analyzer/codechecker_analyzer/analyzers/analyzer_types.py +++ b/analyzer/codechecker_analyzer/analyzers/analyzer_types.py @@ -104,7 +104,8 @@ def is_ignore_conflict_supported(): proc = subprocess.Popen([context.replacer_binary, '--help'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=context.analyzer_env, + env=context + .get_analyzer_env("clang-apply-replacements"), encoding="utf-8", errors="ignore") out, _ = proc.communicate() return '--ignore-insert-conflict' in out @@ -144,7 +145,7 @@ def check_supported_analyzers(analyzers): """ context = analyzer_context.get_context() - check_env = context.analyzer_env + check_env = context.cc_env analyzer_binaries = context.analyzer_binaries diff --git a/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py b/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py index 6bb89697c7..f4e93c2a55 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py @@ -51,7 +51,7 @@ def parse_clang_help_page( help_page = subprocess.check_output( command, stderr=subprocess.STDOUT, - env=analyzer_context.get_context().analyzer_env, + env=analyzer_context.get_context().get_analyzer_env("clangsa"), universal_newlines=True, encoding="utf-8", errors="ignore") @@ -207,7 +207,7 @@ def ctu_capability(cls): if not cls.__ctu_autodetection: cls.__ctu_autodetection = CTUAutodetection( cls.analyzer_binary(), - analyzer_context.get_context().analyzer_env) + analyzer_context.get_context().get_analyzer_env("clangsa")) return cls.__ctu_autodetection @@ -606,7 +606,7 @@ def construct_result_handler(self, buildaction, report_output, def construct_config_handler(cls, args): context = analyzer_context.get_context() - environ = context.analyzer_env + environ = context.get_analyzer_env("clangsa") handler = config_handler.ClangSAConfigHandler(environ) diff --git a/analyzer/codechecker_analyzer/analyzers/clangsa/version.py b/analyzer/codechecker_analyzer/analyzers/clangsa/version.py index 38f7578384..4403f947a0 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangsa/version.py +++ b/analyzer/codechecker_analyzer/analyzers/clangsa/version.py @@ -77,7 +77,7 @@ def get(clang_binary): """ compiler_version = subprocess.check_output( [clang_binary, '--version'], - env=analyzer_context.get_context().analyzer_env, + env=analyzer_context.get_context().get_analyzer_env("clangsa"), encoding="utf-8", errors="ignore") version_parser = ClangVersionInfoParser(clang_binary) diff --git a/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py b/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py index a03d372727..52bd06b976 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py @@ -270,7 +270,8 @@ def get_analyzer_checkers(cls): if cls.__analyzer_checkers: return cls.__analyzer_checkers - environ = analyzer_context.get_context().analyzer_env + environ = analyzer_context\ + .get_context().get_analyzer_env("clang-tidy") result = subprocess.check_output( [cls.analyzer_binary(), "-list-checks", "-checks=*"], env=environ, @@ -297,7 +298,8 @@ def get_checker_config(cls): try: result = subprocess.check_output( [cls.analyzer_binary(), "-dump-config", "-checks=*"], - env=analyzer_context.get_context().analyzer_env, + env=analyzer_context.get_context() + .get_analyzer_env("clang-tidy"), universal_newlines=True, encoding="utf-8", errors="ignore") @@ -313,7 +315,8 @@ def get_analyzer_config(cls): try: result = subprocess.check_output( [cls.analyzer_binary(), "-dump-config", "-checks=*"], - env=analyzer_context.get_context().analyzer_env, + env=analyzer_context.get_context() + .get_analyzer_env("clang-tidy"), universal_newlines=True, encoding="utf-8", errors="ignore") diff --git a/analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py b/analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py index 4977462be3..c74596abdf 100644 --- a/analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py @@ -393,7 +393,7 @@ def construct_config_handler(cls, args): # No cppcheck arguments file was given in the command line. LOG.debug_analyzer(aerr) - check_env = context.analyzer_env + check_env = context.get_analyzer_env("cppcheck") # Overwrite PATH to contain only the parent of the cppcheck binary. if os.path.isabs(Cppcheck.analyzer_binary()): diff --git a/analyzer/codechecker_analyzer/cmd/analyzers.py b/analyzer/codechecker_analyzer/cmd/analyzers.py index aa6e4b2722..69ec73ed50 100644 --- a/analyzer/codechecker_analyzer/cmd/analyzers.py +++ b/analyzer/codechecker_analyzer/cmd/analyzers.py @@ -186,7 +186,7 @@ def uglify(text): rows = [] for analyzer_name, analyzer_class in \ analyzer_types.supported_analyzers.items(): - check_env = context.analyzer_env + check_env = context.get_analyzer_env(analyzer_name) version = analyzer_class.get_binary_version(check_env) if not version: version = 'ERROR' diff --git a/analyzer/codechecker_analyzer/env.py b/analyzer/codechecker_analyzer/env.py index 454a59c54c..a75c384f73 100644 --- a/analyzer/codechecker_analyzer/env.py +++ b/analyzer/codechecker_analyzer/env.py @@ -9,6 +9,7 @@ import os import re +import pickle from codechecker_analyzer import analyzer_context from codechecker_common.logger import get_logger @@ -35,6 +36,24 @@ def get_log_env(logfile, original_env): return new_env +def get_original_env(): + original_env = os.environ + try: + original_env_file = os.environ.get('CODECHECKER_ORIGINAL_BUILD_ENV') + if original_env_file: + LOG.debug_analyzer('Loading original build env from: %s', + original_env_file) + + with open(original_env_file, 'rb') as env_file: + original_env = pickle.load(env_file, encoding='utf-8') + + except Exception as ex: + LOG.warning(str(ex)) + LOG.warning('Failed to get saved original_env ') + original_env = None + return original_env + + def extend(path_env_extra, ld_lib_path_extra): """Extend the checker environment. diff --git a/analyzer/codechecker_analyzer/host_check.py b/analyzer/codechecker_analyzer/host_check.py index 1445db51dc..494818d742 100644 --- a/analyzer/codechecker_analyzer/host_check.py +++ b/analyzer/codechecker_analyzer/host_check.py @@ -59,7 +59,7 @@ def has_analyzer_config_option(clang_bin, config_option_name): cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=analyzer_context.get_context().analyzer_env, + env=analyzer_context.get_context().get_analyzer_env("clangsa"), encoding="utf-8", errors="ignore") out, err = proc.communicate() LOG.debug("stdout:\n%s", out) @@ -92,7 +92,7 @@ def has_analyzer_option(clang_bin, feature): cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=analyzer_context.get_context().analyzer_env, + env=analyzer_context.get_context().get_analyzer_env("clangsa"), encoding="utf-8", errors="ignore") out, err = proc.communicate() LOG.debug("stdout:\n%s", out) diff --git a/analyzer/tests/unit/test_env_var.py b/analyzer/tests/unit/test_env_var.py index 2c1ed54bd9..6531c1a330 100644 --- a/analyzer/tests/unit/test_env_var.py +++ b/analyzer/tests/unit/test_env_var.py @@ -12,6 +12,8 @@ import unittest +import tempfile +import os from codechecker_analyzer import analyzer_context from codechecker_analyzer.analyzers.gcc.analyzer import Gcc @@ -48,7 +50,7 @@ def _get_analyzer_bin_for_cc_analyzer_bin(self, analyzer_bin_conf: str): initialized with the binary that was given by the env var). """ context = analyzer_context.get_context() - context.analyzer_env["CC_ANALYZER_BIN"] = analyzer_bin_conf + context.cc_env["CC_ANALYZER_BIN"] = analyzer_bin_conf context._Context__populate_analyzers() analyzer = create_analyzer_gcc() @@ -80,7 +82,7 @@ def test_cc_analyzer_bin_overrides_cc_analyzers_from_path(self): """ context = analyzer_context.get_context() - context.analyzer_env["CC_ANALYZERS_FROM_PATH"] = '1' + context.cc_env["CC_ANALYZERS_FROM_PATH"] = '1' bin_gcc_var = self._get_analyzer_bin_for_cc_analyzer_bin("gcc:gcc") self.assertTrue(bin_gcc_var.endswith("gcc")) @@ -91,3 +93,68 @@ def test_cc_analyzer_bin_overrides_cc_analyzers_from_path(self): self.assertTrue(not bin_gpp_var.endswith("gcc")) self.assertNotEqual(bin_gcc_var, bin_gpp_var) + + def test_cc_analyzer_internal_env(self): + """ + Check whether the ld_library_path is extended with the internal + lib path if internally packaged analyzer is invoked. + """ + + data_files_dir = tempfile.mkdtemp() + + package_layout_json = """ + { + "ld_lib_path_extra": [], + "path_env_extra": [], + "runtime": { + "analyzers": { + "clang-tidy": "clang-tidy", + "clangsa": "cc-bin/packaged-clang", + "cppcheck": "cppcheck", + "gcc": "g++" + }, + "clang-apply-replacements": "clang-apply-replacements", + "ld_lib_path_extra": [ + "internal_package_lib" + ], + "path_env_extra": [ + ] + } + } + """ + config_dir = os.path.join(data_files_dir, "config") + os.mkdir(config_dir) + layout_cfg_file = os.path.join(config_dir, "package_layout.json") + with open(layout_cfg_file, "w", encoding="utf-8") as text_file: + text_file.write(package_layout_json) + cc_bin_dir = os.path.join(data_files_dir, "cc-bin") + os.mkdir(cc_bin_dir) + packaged_clang_file = os.path.join(cc_bin_dir, "packaged-clang") + with open(packaged_clang_file, "w", encoding="utf-8") as text_file: + text_file.write("") + + context = analyzer_context.get_context() + context._data_files_dir_path = data_files_dir + + lcfg_dict = context._Context__get_package_layout() + context._data_files_dir_path = data_files_dir + context.pckg_layout = lcfg_dict['runtime'] + context._Context__populate_analyzers() + + # clang-19 is part of the codechecker package + # so the internal package lib should be in the ld_library_path + clang_env = context.get_analyzer_env("clangsa") + env_txt = str(clang_env) + self.assertTrue(env_txt.find("internal_package_lib") != -1) + + # clang-tidy is not part of the codechecker package + # so internal package lib should not be in the ld_library_path + clang_env = context.get_analyzer_env("clang-tidy") + env_txt = str(clang_env) + self.assertTrue(env_txt.find("internal_package_lib") == -1) + + os.remove(layout_cfg_file) + os.remove(packaged_clang_file) + os.rmdir(cc_bin_dir) + os.rmdir(config_dir) + os.rmdir(context._data_files_dir_path)