Analyzer binary dependent environment

If a pre-package analyzer binary (e.g. clang) is found first,
the LD_LIBRARY_PATH must be extended. On the other hand, if the
analyzer binary is taken from the users machine, the original
environment must be used to execute the binary.

This patch introduces an analyzer binary dependent environment
initialization.

analyzer/codechecker_analyzer/analyzer.py

@@ -250,7 +250,7 @@ def perform_analysis(args, skip_handlers, rs_handler: ReviewStatusHandler,
 
         # TODO: cppcheck may require a different environment than clang.
         version = analyzer_types.supported_analyzers[analyzer] \
-            .get_binary_version(context.analyzer_env)
+            .get_binary_version(context.get_analyzer_env(analyzer))
         metadata_info['analyzer_statistics']['version'] = version
 
         metadata_tool['analyzers'][analyzer] = metadata_info

analyzer/codechecker_analyzer/analyzer_context.py

@@ -65,7 +65,8 @@ def __init__(self):
         self.__package_build_date = None
         self.__package_git_hash = None
         self.__analyzers = {}
-        self.__analyzer_env = None
+        self.__analyzer_envs = {}
+        self.__cc_env = None
 
         machine = platform.uname().machine
 
@@ -93,9 +94,9 @@ def __init__(self):
 
     def __parse_cc_analyzer_bin(self):
         env_var_bins = {}
-        if 'CC_ANALYZER_BIN' in self.analyzer_env:
+        if 'CC_ANALYZER_BIN' in self.cc_env:
             had_error = False
-            for value in self.__analyzer_env['CC_ANALYZER_BIN'].split(';'):
+            for value in self.__cc_env['CC_ANALYZER_BIN'].split(';'):
                 try:
                     analyzer_name, path = analyzer_binary(value)
                 except ArgumentTypeError as e:
@@ -196,17 +197,19 @@ def __set_version(self):
 
     def __populate_analyzers(self):
         """ Set analyzer binaries for each registered analyzers. """
-        analyzer_env = None
+        cc_env = None
         analyzer_from_path = env.is_analyzer_from_path()
         if not analyzer_from_path:
-            analyzer_env = self.analyzer_env
+            cc_env = self.cc_env
 
         env_var_bin = self.__parse_cc_analyzer_bin()
 
         compiler_binaries = self.pckg_layout.get('analyzers')
         for name, value in compiler_binaries.items():
             if name in env_var_bin:
+                # For non-packaged analyzers the original env is set.
                 self.__analyzers[name] = env_var_bin[name]
+                self.__analyzer_envs[name] = env.get_original_env()
                 continue
 
             if analyzer_from_path:
@@ -216,8 +219,12 @@ def __populate_analyzers(self):
                 # Check if it is a package relative path.
                 self.__analyzers[name] = os.path.join(
                     self._data_files_dir_path, value)
+                # For packaged analyzers the ld_library path
+                # must be extended with the packed libs.
+                self.__analyzer_envs[name] =\
+                    env.extend(self.path_env_extra, self.ld_lib_path_extra)
             else:
-                env_path = analyzer_env['PATH'] if analyzer_env else None
+                env_path = cc_env['PATH'] if cc_env else None
                 compiler_binary = which(cmd=value, path=env_path)
                 if not compiler_binary:
                     LOG.debug("'%s' binary can not be found in your PATH!",
@@ -226,12 +233,17 @@ def __populate_analyzers(self):
                     continue
 
                 self.__analyzers[name] = os.path.realpath(compiler_binary)
+                # For non-packaged analyzers the original env is set.
+                self.__analyzer_envs[name] = env.get_original_env()
 
                 # If the compiler binary is a simlink to ccache, use the
                 # original compiler binary.
                 if self.__analyzers[name].endswith("/ccache"):
                     self.__analyzers[name] = compiler_binary
 
+    def get_analyzer_env(self, analyzer_name):
+        return self.__analyzer_envs[analyzer_name]
+
     def __populate_replacer(self):
         """ Set clang-apply-replacements tool. """
         replacer_binary = self.pckg_layout.get('clang-apply-replacements')
@@ -240,8 +252,12 @@ def __populate_replacer(self):
             # Check if it is a package relative path.
             self.__replacer = os.path.join(self._data_files_dir_path,
                                            replacer_binary)
+            self.__analyzer_envs['clang-apply-replacements'] =\
+                env.extend(self.path_env_extra, self.ld_lib_path_extra)
         else:
             self.__replacer = which(replacer_binary)
+            self.__analyzer_envs['clang-apply-replacements'] =\
+                env.get_original_env()
 
     @property
     def version(self):
@@ -320,11 +336,10 @@ def ld_lib_path_extra(self):
         return ld_paths
 
     @property
-    def analyzer_env(self):
-        if not self.__analyzer_env:
-            self.__analyzer_env = \
-                env.extend(self.path_env_extra, self.ld_lib_path_extra)
-        return self.__analyzer_env
+    def cc_env(self):
+        if not self.__cc_env:
+            self.__cc_env = os.environ.copy()
+        return self.__cc_env
 
     @property
     def analyzer_binaries(self):

analyzer/codechecker_analyzer/analyzers/analyzer_base.py

@@ -111,6 +111,10 @@ def analyze(self, analyzer_cmd, res_handler, proc_callback=None, env=None):
         LOG.debug_analyzer('\n%s',
                            ' '.join([shlex.quote(x) for x in analyzer_cmd]))
 
+        if env is None:
+            env = analyzer_context.get_context()\
+                .get_analyzer_env(self.ANALYZER_NAME)
+
         res_handler.analyzer_cmd = analyzer_cmd
         try:
             ret_code, stdout, stderr \
@@ -158,7 +162,10 @@ def signal_handler(signum, _):
         signal.signal(signal.SIGINT, signal_handler)
 
         if env is None:
-            env = analyzer_context.get_context().analyzer_env
+            env = analyzer_context.get_context().cc_env
+
+        LOG.debug_analyzer('\nENV:\n')
+        LOG.debug_analyzer(env)
 
         proc = subprocess.Popen(
             command,

analyzer/codechecker_analyzer/analyzers/analyzer_types.py

@@ -104,7 +104,8 @@ def is_ignore_conflict_supported():
     proc = subprocess.Popen([context.replacer_binary, '--help'],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
-                            env=context.analyzer_env,
+                            env=context
+                            .get_analyzer_env("clang-apply-replacements"),
                             encoding="utf-8", errors="ignore")
     out, _ = proc.communicate()
     return '--ignore-insert-conflict' in out
@@ -144,7 +145,7 @@ def check_supported_analyzers(analyzers):
     """
 
     context = analyzer_context.get_context()
-    check_env = context.analyzer_env
+    check_env = context.cc_env
 
     analyzer_binaries = context.analyzer_binaries
 

analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py

@@ -51,7 +51,7 @@ def parse_clang_help_page(
         help_page = subprocess.check_output(
             command,
             stderr=subprocess.STDOUT,
-            env=analyzer_context.get_context().analyzer_env,
+            env=analyzer_context.get_context().get_analyzer_env("clangsa"),
             universal_newlines=True,
             encoding="utf-8",
             errors="ignore")
@@ -207,7 +207,7 @@ def ctu_capability(cls):
         if not cls.__ctu_autodetection:
             cls.__ctu_autodetection = CTUAutodetection(
                 cls.analyzer_binary(),
-                analyzer_context.get_context().analyzer_env)
+                analyzer_context.get_context().get_analyzer_env("clangsa"))
 
         return cls.__ctu_autodetection
 
@@ -606,7 +606,7 @@ def construct_result_handler(self, buildaction, report_output,
     def construct_config_handler(cls, args):
 
         context = analyzer_context.get_context()
-        environ = context.analyzer_env
+        environ = context.get_analyzer_env("clangsa")
 
         handler = config_handler.ClangSAConfigHandler(environ)
 

analyzer/codechecker_analyzer/analyzers/clangsa/version.py

@@ -77,7 +77,7 @@ def get(clang_binary):
     """
     compiler_version = subprocess.check_output(
         [clang_binary, '--version'],
-        env=analyzer_context.get_context().analyzer_env,
+        env=analyzer_context.get_context().get_analyzer_env("clangsa"),
         encoding="utf-8",
         errors="ignore")
     version_parser = ClangVersionInfoParser(clang_binary)

analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py

@@ -270,7 +270,8 @@ def get_analyzer_checkers(cls):
             if cls.__analyzer_checkers:
                 return cls.__analyzer_checkers
 
-            environ = analyzer_context.get_context().analyzer_env
+            environ = analyzer_context\
+                .get_context().get_analyzer_env("clang-tidy")
             result = subprocess.check_output(
                 [cls.analyzer_binary(), "-list-checks", "-checks=*"],
                 env=environ,
@@ -297,7 +298,8 @@ def get_checker_config(cls):
         try:
             result = subprocess.check_output(
                 [cls.analyzer_binary(), "-dump-config", "-checks=*"],
-                env=analyzer_context.get_context().analyzer_env,
+                env=analyzer_context.get_context()
+                .get_analyzer_env("clang-tidy"),
                 universal_newlines=True,
                 encoding="utf-8",
                 errors="ignore")
@@ -313,7 +315,8 @@ def get_analyzer_config(cls):
         try:
             result = subprocess.check_output(
                 [cls.analyzer_binary(), "-dump-config", "-checks=*"],
-                env=analyzer_context.get_context().analyzer_env,
+                env=analyzer_context.get_context()
+                .get_analyzer_env("clang-tidy"),
                 universal_newlines=True,
                 encoding="utf-8",
                 errors="ignore")

analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py

@@ -393,7 +393,7 @@ def construct_config_handler(cls, args):
             # No cppcheck arguments file was given in the command line.
             LOG.debug_analyzer(aerr)
 
-        check_env = context.analyzer_env
+        check_env = context.get_analyzer_env("cppcheck")
 
         # Overwrite PATH to contain only the parent of the cppcheck binary.
         if os.path.isabs(Cppcheck.analyzer_binary()):

analyzer/codechecker_analyzer/cmd/analyzers.py

@@ -186,7 +186,7 @@ def uglify(text):
     rows = []
     for analyzer_name, analyzer_class in \
             analyzer_types.supported_analyzers.items():
-        check_env = context.analyzer_env
+        check_env = context.get_analyzer_env(analyzer_name)
         version = analyzer_class.get_binary_version(check_env)
         if not version:
             version = 'ERROR'

analyzer/codechecker_analyzer/env.py

@@ -9,6 +9,7 @@
 
 import os
 import re
+import pickle
 
 from codechecker_analyzer import analyzer_context
 from codechecker_common.logger import get_logger
@@ -35,6 +36,24 @@ def get_log_env(logfile, original_env):
     return new_env
 
 
+def get_original_env():
+    original_env = os.environ
+    try:
+        original_env_file = os.environ.get('CODECHECKER_ORIGINAL_BUILD_ENV')
+        if original_env_file:
+            LOG.debug_analyzer('Loading original build env from: %s',
+                               original_env_file)
+
+            with open(original_env_file, 'rb') as env_file:
+                original_env = pickle.load(env_file, encoding='utf-8')
+
+    except Exception as ex:
+        LOG.warning(str(ex))
+        LOG.warning('Failed to get saved original_env ')
+        original_env = None
+    return original_env
+
+
 def extend(path_env_extra, ld_lib_path_extra):
     """Extend the checker environment.
 

analyzer/codechecker_analyzer/host_check.py

@@ -59,7 +59,7 @@ def has_analyzer_config_option(clang_bin, config_option_name):
             cmd,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
-            env=analyzer_context.get_context().analyzer_env,
+            env=analyzer_context.get_context().get_analyzer_env("clangsa"),
             encoding="utf-8", errors="ignore")
         out, err = proc.communicate()
         LOG.debug("stdout:\n%s", out)
@@ -92,7 +92,7 @@ def has_analyzer_option(clang_bin, feature):
                 cmd,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
-                env=analyzer_context.get_context().analyzer_env,
+                env=analyzer_context.get_context().get_analyzer_env("clangsa"),
                 encoding="utf-8", errors="ignore")
             out, err = proc.communicate()
             LOG.debug("stdout:\n%s", out)

analyzer/tests/unit/test_env_var.py

@@ -12,6 +12,8 @@
 
 
 import unittest
+import tempfile
+import os
 
 from codechecker_analyzer import analyzer_context
 from codechecker_analyzer.analyzers.gcc.analyzer import Gcc
@@ -48,7 +50,7 @@ def _get_analyzer_bin_for_cc_analyzer_bin(self, analyzer_bin_conf: str):
         initialized with the binary that was given by the env var).
         """
         context = analyzer_context.get_context()
-        context.analyzer_env["CC_ANALYZER_BIN"] = analyzer_bin_conf
+        context.cc_env["CC_ANALYZER_BIN"] = analyzer_bin_conf
         context._Context__populate_analyzers()
 
         analyzer = create_analyzer_gcc()
@@ -80,7 +82,7 @@ def test_cc_analyzer_bin_overrides_cc_analyzers_from_path(self):
         """
 
         context = analyzer_context.get_context()
-        context.analyzer_env["CC_ANALYZERS_FROM_PATH"] = '1'
+        context.cc_env["CC_ANALYZERS_FROM_PATH"] = '1'
 
         bin_gcc_var = self._get_analyzer_bin_for_cc_analyzer_bin("gcc:gcc")
         self.assertTrue(bin_gcc_var.endswith("gcc"))
@@ -91,3 +93,68 @@ def test_cc_analyzer_bin_overrides_cc_analyzers_from_path(self):
         self.assertTrue(not bin_gpp_var.endswith("gcc"))
 
         self.assertNotEqual(bin_gcc_var, bin_gpp_var)
+
+    def test_cc_analyzer_internal_env(self):
+        """
+        Check whether the ld_library_path is extended with the internal
+        lib path if internally packaged analyzer is invoked.
+        """
+
+        data_files_dir = tempfile.mkdtemp()
+
+        package_layout_json = """
+        {
+        "ld_lib_path_extra": [],
+        "path_env_extra": [],
+        "runtime": {
+            "analyzers": {
+            "clang-tidy": "clang-tidy",
+            "clangsa": "cc-bin/packaged-clang",
+            "cppcheck": "cppcheck",
+            "gcc": "g++"
+            },
+            "clang-apply-replacements": "clang-apply-replacements",
+            "ld_lib_path_extra": [
+            "internal_package_lib"
+            ],
+            "path_env_extra": [
+            ]
+        }
+        }
+        """
+        config_dir = os.path.join(data_files_dir, "config")
+        os.mkdir(config_dir)
+        layout_cfg_file = os.path.join(config_dir, "package_layout.json")
+        with open(layout_cfg_file, "w", encoding="utf-8") as text_file:
+            text_file.write(package_layout_json)
+        cc_bin_dir = os.path.join(data_files_dir, "cc-bin")
+        os.mkdir(cc_bin_dir)
+        packaged_clang_file = os.path.join(cc_bin_dir, "packaged-clang")
+        with open(packaged_clang_file, "w", encoding="utf-8") as text_file:
+            text_file.write("")
+
+        context = analyzer_context.get_context()
+        context._data_files_dir_path = data_files_dir
+
+        lcfg_dict = context._Context__get_package_layout()
+        context._data_files_dir_path = data_files_dir
+        context.pckg_layout = lcfg_dict['runtime']
+        context._Context__populate_analyzers()
+
+        # clang-19 is part of the codechecker package
+        # so the internal package lib should be in the ld_library_path
+        clang_env = context.get_analyzer_env("clangsa")
+        env_txt = str(clang_env)
+        self.assertTrue(env_txt.find("internal_package_lib") != -1)
+
+        # clang-tidy is not part of the codechecker package
+        # so internal package lib should not be in the ld_library_path
+        clang_env = context.get_analyzer_env("clang-tidy")
+        env_txt = str(clang_env)
+        self.assertTrue(env_txt.find("internal_package_lib") == -1)
+
+        os.remove(layout_cfg_file)
+        os.remove(packaged_clang_file)
+        os.rmdir(cc_bin_dir)
+        os.rmdir(config_dir)
+        os.rmdir(context._data_files_dir_path)