diff --git a/analyzer/codechecker_analyzer/analyzer.py b/analyzer/codechecker_analyzer/analyzer.py index 062c770731..9b2d524d48 100644 --- a/analyzer/codechecker_analyzer/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzer.py @@ -250,7 +250,7 @@ def perform_analysis(args, skip_handlers, rs_handler: ReviewStatusHandler, # TODO: cppcheck may require a different environment than clang. version = analyzer_types.supported_analyzers[analyzer] \ - .get_binary_version(context.analyzer_env) + .get_binary_version(context.get_analyzer_env(analyzer)) metadata_info['analyzer_statistics']['version'] = version metadata_tool['analyzers'][analyzer] = metadata_info diff --git a/analyzer/codechecker_analyzer/analyzer_context.py b/analyzer/codechecker_analyzer/analyzer_context.py index 082c508a20..21c7610398 100644 --- a/analyzer/codechecker_analyzer/analyzer_context.py +++ b/analyzer/codechecker_analyzer/analyzer_context.py @@ -65,7 +65,8 @@ def __init__(self): self.__package_build_date = None self.__package_git_hash = None self.__analyzers = {} - self.__analyzer_env = None + self.__analyzer_envs = {} + self.__cc_env = None machine = platform.uname().machine @@ -93,9 +94,9 @@ def __init__(self): def __parse_cc_analyzer_bin(self): env_var_bins = {} - if 'CC_ANALYZER_BIN' in self.analyzer_env: + if 'CC_ANALYZER_BIN' in self.cc_env: had_error = False - for value in self.__analyzer_env['CC_ANALYZER_BIN'].split(';'): + for value in self.__cc_env['CC_ANALYZER_BIN'].split(';'): try: analyzer_name, path = analyzer_binary(value) except ArgumentTypeError as e: @@ -196,17 +197,19 @@ def __set_version(self): def __populate_analyzers(self): """ Set analyzer binaries for each registered analyzers. """ - analyzer_env = None + cc_env = None analyzer_from_path = env.is_analyzer_from_path() if not analyzer_from_path: - analyzer_env = self.analyzer_env + cc_env = self.cc_env env_var_bin = self.__parse_cc_analyzer_bin() compiler_binaries = self.pckg_layout.get('analyzers') for name, value in compiler_binaries.items(): if name in env_var_bin: + # For non-packaged analyzers the original env is set. self.__analyzers[name] = env_var_bin[name] + self.__analyzer_envs[name] = env.get_original_env() continue if analyzer_from_path: @@ -216,8 +219,12 @@ def __populate_analyzers(self): # Check if it is a package relative path. self.__analyzers[name] = os.path.join( self._data_files_dir_path, value) + # For packaged analyzers the ld_library path + # must be extended with the packed libs. + self.__analyzer_envs[name] =\ + env.extend(self.path_env_extra, self.ld_lib_path_extra) else: - env_path = analyzer_env['PATH'] if analyzer_env else None + env_path = cc_env['PATH'] if cc_env else None compiler_binary = which(cmd=value, path=env_path) if not compiler_binary: LOG.debug("'%s' binary can not be found in your PATH!", @@ -226,12 +233,17 @@ def __populate_analyzers(self): continue self.__analyzers[name] = os.path.realpath(compiler_binary) + # For non-packaged analyzers the original env is set. + self.__analyzer_envs[name] = env.get_original_env() # If the compiler binary is a simlink to ccache, use the # original compiler binary. if self.__analyzers[name].endswith("/ccache"): self.__analyzers[name] = compiler_binary + def get_analyzer_env(self, analyzer_name): + return self.__analyzer_envs[analyzer_name] + def __populate_replacer(self): """ Set clang-apply-replacements tool. """ replacer_binary = self.pckg_layout.get('clang-apply-replacements') @@ -240,8 +252,12 @@ def __populate_replacer(self): # Check if it is a package relative path. self.__replacer = os.path.join(self._data_files_dir_path, replacer_binary) + self.__analyzer_envs['clang-apply-replacements'] =\ + env.extend(self.path_env_extra, self.ld_lib_path_extra) else: self.__replacer = which(replacer_binary) + self.__analyzer_envs['clang-apply-replacements'] =\ + env.get_original_env() @property def version(self): @@ -320,11 +336,10 @@ def ld_lib_path_extra(self): return ld_paths @property - def analyzer_env(self): - if not self.__analyzer_env: - self.__analyzer_env = \ - env.extend(self.path_env_extra, self.ld_lib_path_extra) - return self.__analyzer_env + def cc_env(self): + if not self.__cc_env: + self.__cc_env = os.environ.copy() + return self.__cc_env @property def analyzer_binaries(self): diff --git a/analyzer/codechecker_analyzer/analyzers/analyzer_base.py b/analyzer/codechecker_analyzer/analyzers/analyzer_base.py index d6d92b5aa8..f9c02cbe26 100644 --- a/analyzer/codechecker_analyzer/analyzers/analyzer_base.py +++ b/analyzer/codechecker_analyzer/analyzers/analyzer_base.py @@ -111,6 +111,10 @@ def analyze(self, analyzer_cmd, res_handler, proc_callback=None, env=None): LOG.debug_analyzer('\n%s', ' '.join([shlex.quote(x) for x in analyzer_cmd])) + if env is None: + env = analyzer_context.get_context()\ + .get_analyzer_env(self.ANALYZER_NAME) + res_handler.analyzer_cmd = analyzer_cmd try: ret_code, stdout, stderr \ @@ -158,7 +162,10 @@ def signal_handler(signum, _): signal.signal(signal.SIGINT, signal_handler) if env is None: - env = analyzer_context.get_context().analyzer_env + env = analyzer_context.get_context().cc_env + + LOG.debug_analyzer('\nENV:\n') + LOG.debug_analyzer(env) proc = subprocess.Popen( command, diff --git a/analyzer/codechecker_analyzer/analyzers/analyzer_types.py b/analyzer/codechecker_analyzer/analyzers/analyzer_types.py index 6e8a492ecf..fc64304bc4 100644 --- a/analyzer/codechecker_analyzer/analyzers/analyzer_types.py +++ b/analyzer/codechecker_analyzer/analyzers/analyzer_types.py @@ -104,7 +104,8 @@ def is_ignore_conflict_supported(): proc = subprocess.Popen([context.replacer_binary, '--help'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=context.analyzer_env, + env=context + .get_analyzer_env("clang-apply-replacements"), encoding="utf-8", errors="ignore") out, _ = proc.communicate() return '--ignore-insert-conflict' in out @@ -144,7 +145,7 @@ def check_supported_analyzers(analyzers): """ context = analyzer_context.get_context() - check_env = context.analyzer_env + check_env = context.cc_env analyzer_binaries = context.analyzer_binaries diff --git a/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py b/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py index 6bb89697c7..f4e93c2a55 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py @@ -51,7 +51,7 @@ def parse_clang_help_page( help_page = subprocess.check_output( command, stderr=subprocess.STDOUT, - env=analyzer_context.get_context().analyzer_env, + env=analyzer_context.get_context().get_analyzer_env("clangsa"), universal_newlines=True, encoding="utf-8", errors="ignore") @@ -207,7 +207,7 @@ def ctu_capability(cls): if not cls.__ctu_autodetection: cls.__ctu_autodetection = CTUAutodetection( cls.analyzer_binary(), - analyzer_context.get_context().analyzer_env) + analyzer_context.get_context().get_analyzer_env("clangsa")) return cls.__ctu_autodetection @@ -606,7 +606,7 @@ def construct_result_handler(self, buildaction, report_output, def construct_config_handler(cls, args): context = analyzer_context.get_context() - environ = context.analyzer_env + environ = context.get_analyzer_env("clangsa") handler = config_handler.ClangSAConfigHandler(environ) diff --git a/analyzer/codechecker_analyzer/analyzers/clangsa/version.py b/analyzer/codechecker_analyzer/analyzers/clangsa/version.py index 38f7578384..4403f947a0 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangsa/version.py +++ b/analyzer/codechecker_analyzer/analyzers/clangsa/version.py @@ -77,7 +77,7 @@ def get(clang_binary): """ compiler_version = subprocess.check_output( [clang_binary, '--version'], - env=analyzer_context.get_context().analyzer_env, + env=analyzer_context.get_context().get_analyzer_env("clangsa"), encoding="utf-8", errors="ignore") version_parser = ClangVersionInfoParser(clang_binary) diff --git a/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py b/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py index a03d372727..52bd06b976 100644 --- a/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py @@ -270,7 +270,8 @@ def get_analyzer_checkers(cls): if cls.__analyzer_checkers: return cls.__analyzer_checkers - environ = analyzer_context.get_context().analyzer_env + environ = analyzer_context\ + .get_context().get_analyzer_env("clang-tidy") result = subprocess.check_output( [cls.analyzer_binary(), "-list-checks", "-checks=*"], env=environ, @@ -297,7 +298,8 @@ def get_checker_config(cls): try: result = subprocess.check_output( [cls.analyzer_binary(), "-dump-config", "-checks=*"], - env=analyzer_context.get_context().analyzer_env, + env=analyzer_context.get_context() + .get_analyzer_env("clang-tidy"), universal_newlines=True, encoding="utf-8", errors="ignore") @@ -313,7 +315,8 @@ def get_analyzer_config(cls): try: result = subprocess.check_output( [cls.analyzer_binary(), "-dump-config", "-checks=*"], - env=analyzer_context.get_context().analyzer_env, + env=analyzer_context.get_context() + .get_analyzer_env("clang-tidy"), universal_newlines=True, encoding="utf-8", errors="ignore") diff --git a/analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py b/analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py index 4977462be3..c74596abdf 100644 --- a/analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py +++ b/analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py @@ -393,7 +393,7 @@ def construct_config_handler(cls, args): # No cppcheck arguments file was given in the command line. LOG.debug_analyzer(aerr) - check_env = context.analyzer_env + check_env = context.get_analyzer_env("cppcheck") # Overwrite PATH to contain only the parent of the cppcheck binary. if os.path.isabs(Cppcheck.analyzer_binary()): diff --git a/analyzer/codechecker_analyzer/cmd/analyzers.py b/analyzer/codechecker_analyzer/cmd/analyzers.py index aa6e4b2722..69ec73ed50 100644 --- a/analyzer/codechecker_analyzer/cmd/analyzers.py +++ b/analyzer/codechecker_analyzer/cmd/analyzers.py @@ -186,7 +186,7 @@ def uglify(text): rows = [] for analyzer_name, analyzer_class in \ analyzer_types.supported_analyzers.items(): - check_env = context.analyzer_env + check_env = context.get_analyzer_env(analyzer_name) version = analyzer_class.get_binary_version(check_env) if not version: version = 'ERROR' diff --git a/analyzer/codechecker_analyzer/env.py b/analyzer/codechecker_analyzer/env.py index 454a59c54c..a75c384f73 100644 --- a/analyzer/codechecker_analyzer/env.py +++ b/analyzer/codechecker_analyzer/env.py @@ -9,6 +9,7 @@ import os import re +import pickle from codechecker_analyzer import analyzer_context from codechecker_common.logger import get_logger @@ -35,6 +36,24 @@ def get_log_env(logfile, original_env): return new_env +def get_original_env(): + original_env = os.environ + try: + original_env_file = os.environ.get('CODECHECKER_ORIGINAL_BUILD_ENV') + if original_env_file: + LOG.debug_analyzer('Loading original build env from: %s', + original_env_file) + + with open(original_env_file, 'rb') as env_file: + original_env = pickle.load(env_file, encoding='utf-8') + + except Exception as ex: + LOG.warning(str(ex)) + LOG.warning('Failed to get saved original_env ') + original_env = None + return original_env + + def extend(path_env_extra, ld_lib_path_extra): """Extend the checker environment. diff --git a/analyzer/codechecker_analyzer/host_check.py b/analyzer/codechecker_analyzer/host_check.py index 1445db51dc..494818d742 100644 --- a/analyzer/codechecker_analyzer/host_check.py +++ b/analyzer/codechecker_analyzer/host_check.py @@ -59,7 +59,7 @@ def has_analyzer_config_option(clang_bin, config_option_name): cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=analyzer_context.get_context().analyzer_env, + env=analyzer_context.get_context().get_analyzer_env("clangsa"), encoding="utf-8", errors="ignore") out, err = proc.communicate() LOG.debug("stdout:\n%s", out) @@ -92,7 +92,7 @@ def has_analyzer_option(clang_bin, feature): cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=analyzer_context.get_context().analyzer_env, + env=analyzer_context.get_context().get_analyzer_env("clangsa"), encoding="utf-8", errors="ignore") out, err = proc.communicate() LOG.debug("stdout:\n%s", out) diff --git a/analyzer/tests/unit/test_env_var.py b/analyzer/tests/unit/test_env_var.py index 2c1ed54bd9..6531c1a330 100644 --- a/analyzer/tests/unit/test_env_var.py +++ b/analyzer/tests/unit/test_env_var.py @@ -12,6 +12,8 @@ import unittest +import tempfile +import os from codechecker_analyzer import analyzer_context from codechecker_analyzer.analyzers.gcc.analyzer import Gcc @@ -48,7 +50,7 @@ def _get_analyzer_bin_for_cc_analyzer_bin(self, analyzer_bin_conf: str): initialized with the binary that was given by the env var). """ context = analyzer_context.get_context() - context.analyzer_env["CC_ANALYZER_BIN"] = analyzer_bin_conf + context.cc_env["CC_ANALYZER_BIN"] = analyzer_bin_conf context._Context__populate_analyzers() analyzer = create_analyzer_gcc() @@ -80,7 +82,7 @@ def test_cc_analyzer_bin_overrides_cc_analyzers_from_path(self): """ context = analyzer_context.get_context() - context.analyzer_env["CC_ANALYZERS_FROM_PATH"] = '1' + context.cc_env["CC_ANALYZERS_FROM_PATH"] = '1' bin_gcc_var = self._get_analyzer_bin_for_cc_analyzer_bin("gcc:gcc") self.assertTrue(bin_gcc_var.endswith("gcc")) @@ -91,3 +93,68 @@ def test_cc_analyzer_bin_overrides_cc_analyzers_from_path(self): self.assertTrue(not bin_gpp_var.endswith("gcc")) self.assertNotEqual(bin_gcc_var, bin_gpp_var) + + def test_cc_analyzer_internal_env(self): + """ + Check whether the ld_library_path is extended with the internal + lib path if internally packaged analyzer is invoked. + """ + + data_files_dir = tempfile.mkdtemp() + + package_layout_json = """ + { + "ld_lib_path_extra": [], + "path_env_extra": [], + "runtime": { + "analyzers": { + "clang-tidy": "clang-tidy", + "clangsa": "cc-bin/packaged-clang", + "cppcheck": "cppcheck", + "gcc": "g++" + }, + "clang-apply-replacements": "clang-apply-replacements", + "ld_lib_path_extra": [ + "internal_package_lib" + ], + "path_env_extra": [ + ] + } + } + """ + config_dir = os.path.join(data_files_dir, "config") + os.mkdir(config_dir) + layout_cfg_file = os.path.join(config_dir, "package_layout.json") + with open(layout_cfg_file, "w", encoding="utf-8") as text_file: + text_file.write(package_layout_json) + cc_bin_dir = os.path.join(data_files_dir, "cc-bin") + os.mkdir(cc_bin_dir) + packaged_clang_file = os.path.join(cc_bin_dir, "packaged-clang") + with open(packaged_clang_file, "w", encoding="utf-8") as text_file: + text_file.write("") + + context = analyzer_context.get_context() + context._data_files_dir_path = data_files_dir + + lcfg_dict = context._Context__get_package_layout() + context._data_files_dir_path = data_files_dir + context.pckg_layout = lcfg_dict['runtime'] + context._Context__populate_analyzers() + + # clang-19 is part of the codechecker package + # so the internal package lib should be in the ld_library_path + clang_env = context.get_analyzer_env("clangsa") + env_txt = str(clang_env) + self.assertTrue(env_txt.find("internal_package_lib") != -1) + + # clang-tidy is not part of the codechecker package + # so internal package lib should not be in the ld_library_path + clang_env = context.get_analyzer_env("clang-tidy") + env_txt = str(clang_env) + self.assertTrue(env_txt.find("internal_package_lib") == -1) + + os.remove(layout_cfg_file) + os.remove(packaged_clang_file) + os.rmdir(cc_bin_dir) + os.rmdir(config_dir) + os.rmdir(context._data_files_dir_path)