Merge pull request #4305 from dkrupp/env_fix

Analyzer binary dependent environment

analyzer/codechecker_analyzer/analyzer.py

@@ -248,9 +248,8 @@ def perform_analysis(args, skip_handlers, rs_handler: ReviewStatusHandler,
             if state == CheckerState.ENABLED:
                 enabled_checkers[analyzer].append(check)
 
-        # TODO: cppcheck may require a different environment than clang.
         version = analyzer_types.supported_analyzers[analyzer] \
-            .get_binary_version(context.analyzer_env)
+            .get_binary_version(context.get_analyzer_env(analyzer))
         metadata_info['analyzer_statistics']['version'] = version
 
         metadata_tool['analyzers'][analyzer] = metadata_info

analyzer/codechecker_analyzer/analyzer_context.py

@@ -65,7 +65,8 @@ def __init__(self):
         self.__package_build_date = None
         self.__package_git_hash = None
         self.__analyzers = {}
-        self.__analyzer_env = None
+        self.__analyzer_envs = {}
+        self.__cc_env = None
 
         machine = platform.uname().machine
 
@@ -93,9 +94,9 @@ def __init__(self):
 
     def __parse_cc_analyzer_bin(self):
         env_var_bins = {}
-        if 'CC_ANALYZER_BIN' in self.analyzer_env:
+        if 'CC_ANALYZER_BIN' in self.cc_env:
             had_error = False
-            for value in self.__analyzer_env['CC_ANALYZER_BIN'].split(';'):
+            for value in self.__cc_env['CC_ANALYZER_BIN'].split(';'):
                 try:
                     analyzer_name, path = analyzer_binary(value)
                 except ArgumentTypeError as e:
@@ -196,17 +197,19 @@ def __set_version(self):
 
     def __populate_analyzers(self):
         """ Set analyzer binaries for each registered analyzers. """
-        analyzer_env = None
+        cc_env = None
         analyzer_from_path = env.is_analyzer_from_path()
         if not analyzer_from_path:
-            analyzer_env = self.analyzer_env
+            cc_env = self.cc_env
 
         env_var_bin = self.__parse_cc_analyzer_bin()
 
         compiler_binaries = self.pckg_layout.get('analyzers')
         for name, value in compiler_binaries.items():
             if name in env_var_bin:
+                # For non-packaged analyzers the original env is set.
                 self.__analyzers[name] = env_var_bin[name]
+                self.__analyzer_envs[name] = env.get_original_env()
                 continue
 
             if analyzer_from_path:
@@ -216,8 +219,12 @@ def __populate_analyzers(self):
                 # Check if it is a package relative path.
                 self.__analyzers[name] = os.path.join(
                     self._data_files_dir_path, value)
+                # For packaged analyzers the ld_library path
+                # must be extended with the packed libs.
+                self.__analyzer_envs[name] =\
+                    env.extend(self.path_env_extra, self.ld_lib_path_extra)
             else:
-                env_path = analyzer_env['PATH'] if analyzer_env else None
+                env_path = cc_env['PATH'] if cc_env else None
                 compiler_binary = which(cmd=value, path=env_path)
                 if not compiler_binary:
                     LOG.debug("'%s' binary can not be found in your PATH!",
@@ -226,12 +233,17 @@ def __populate_analyzers(self):
                     continue
 
                 self.__analyzers[name] = os.path.realpath(compiler_binary)
+                # For non-packaged analyzers the original env is set.
+                self.__analyzer_envs[name] = env.get_original_env()
 
                 # If the compiler binary is a simlink to ccache, use the
                 # original compiler binary.
                 if self.__analyzers[name].endswith("/ccache"):
                     self.__analyzers[name] = compiler_binary
 
+    def get_analyzer_env(self, analyzer_name):
+        return self.__analyzer_envs[analyzer_name]
+
     def __populate_replacer(self):
         """ Set clang-apply-replacements tool. """
         replacer_binary = self.pckg_layout.get('clang-apply-replacements')
@@ -240,8 +252,12 @@ def __populate_replacer(self):
             # Check if it is a package relative path.
             self.__replacer = os.path.join(self._data_files_dir_path,
                                            replacer_binary)
+            self.__analyzer_envs['clang-apply-replacements'] =\
+                env.extend(self.path_env_extra, self.ld_lib_path_extra)
         else:
             self.__replacer = which(replacer_binary)
+            self.__analyzer_envs['clang-apply-replacements'] =\
+                env.get_original_env()
 
     @property
     def version(self):
@@ -320,11 +336,10 @@ def ld_lib_path_extra(self):
         return ld_paths
 
     @property
-    def analyzer_env(self):
-        if not self.__analyzer_env:
-            self.__analyzer_env = \
-                env.extend(self.path_env_extra, self.ld_lib_path_extra)
-        return self.__analyzer_env
+    def cc_env(self):
+        if not self.__cc_env:
+            self.__cc_env = os.environ.copy()
+        return self.__cc_env
 
     @property
     def analyzer_binaries(self):

analyzer/codechecker_analyzer/analyzers/analyzer_base.py

@@ -111,6 +111,10 @@ def analyze(self, analyzer_cmd, res_handler, proc_callback=None, env=None):
         LOG.debug_analyzer('\n%s',
                            ' '.join([shlex.quote(x) for x in analyzer_cmd]))
 
+        if env is None:
+            env = analyzer_context.get_context()\
+                .get_analyzer_env(self.ANALYZER_NAME)
+
         res_handler.analyzer_cmd = analyzer_cmd
         try:
             ret_code, stdout, stderr \
@@ -158,7 +162,10 @@ def signal_handler(signum, _):
         signal.signal(signal.SIGINT, signal_handler)
 
         if env is None:
-            env = analyzer_context.get_context().analyzer_env
+            env = analyzer_context.get_context().cc_env
+
+        LOG.debug_analyzer('\nENV:\n')
+        LOG.debug_analyzer(env)
 
         proc = subprocess.Popen(
             command,

analyzer/codechecker_analyzer/analyzers/analyzer_types.py

@@ -104,7 +104,9 @@ def is_ignore_conflict_supported():
     proc = subprocess.Popen([context.replacer_binary, '--help'],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
-                            env=context.analyzer_env,
+                            env=context
+                            .get_analyzer_env(
+                                os.path.basename(context.replacer_binary)),
                             encoding="utf-8", errors="ignore")
     out, _ = proc.communicate()
     return '--ignore-insert-conflict' in out
@@ -144,7 +146,7 @@ def check_supported_analyzers(analyzers):
     """
 
     context = analyzer_context.get_context()
-    check_env = context.analyzer_env
+    check_env = context.cc_env
 
     analyzer_binaries = context.analyzer_binaries
 

analyzer/codechecker_analyzer/analyzers/clangsa/analyzer.py

@@ -51,7 +51,8 @@ def parse_clang_help_page(
         help_page = subprocess.check_output(
             command,
             stderr=subprocess.STDOUT,
-            env=analyzer_context.get_context().analyzer_env,
+            env=analyzer_context.get_context()
+            .get_analyzer_env(ClangSA.ANALYZER_NAME),
             universal_newlines=True,
             encoding="utf-8",
             errors="ignore")
@@ -207,7 +208,8 @@ def ctu_capability(cls):
         if not cls.__ctu_autodetection:
             cls.__ctu_autodetection = CTUAutodetection(
                 cls.analyzer_binary(),
-                analyzer_context.get_context().analyzer_env)
+                analyzer_context.get_context()
+                .get_analyzer_env(ClangSA.ANALYZER_NAME))
 
         return cls.__ctu_autodetection
 
@@ -606,7 +608,7 @@ def construct_result_handler(self, buildaction, report_output,
     def construct_config_handler(cls, args):
 
         context = analyzer_context.get_context()
-        environ = context.analyzer_env
+        environ = context.get_analyzer_env(ClangSA.ANALYZER_NAME)
 
         handler = config_handler.ClangSAConfigHandler(environ)
 

analyzer/codechecker_analyzer/analyzers/clangsa/version.py

@@ -77,7 +77,7 @@ def get(clang_binary):
     """
     compiler_version = subprocess.check_output(
         [clang_binary, '--version'],
-        env=analyzer_context.get_context().analyzer_env,
+        env=analyzer_context.get_context().get_analyzer_env("clangsa"),
         encoding="utf-8",
         errors="ignore")
     version_parser = ClangVersionInfoParser(clang_binary)

analyzer/codechecker_analyzer/analyzers/clangtidy/analyzer.py

@@ -270,7 +270,8 @@ def get_analyzer_checkers(cls):
             if cls.__analyzer_checkers:
                 return cls.__analyzer_checkers
 
-            environ = analyzer_context.get_context().analyzer_env
+            environ = analyzer_context\
+                .get_context().get_analyzer_env(cls.ANALYZER_NAME)
             result = subprocess.check_output(
                 [cls.analyzer_binary(), "-list-checks", "-checks=*"],
                 env=environ,
@@ -297,7 +298,8 @@ def get_checker_config(cls):
         try:
             result = subprocess.check_output(
                 [cls.analyzer_binary(), "-dump-config", "-checks=*"],
-                env=analyzer_context.get_context().analyzer_env,
+                env=analyzer_context.get_context()
+                .get_analyzer_env(cls.ANALYZER_NAME),
                 universal_newlines=True,
                 encoding="utf-8",
                 errors="ignore")
@@ -313,7 +315,8 @@ def get_analyzer_config(cls):
         try:
             result = subprocess.check_output(
                 [cls.analyzer_binary(), "-dump-config", "-checks=*"],
-                env=analyzer_context.get_context().analyzer_env,
+                env=analyzer_context.get_context()
+                .get_analyzer_env(cls.ANALYZER_NAME),
                 universal_newlines=True,
                 encoding="utf-8",
                 errors="ignore")

analyzer/codechecker_analyzer/analyzers/cppcheck/analyzer.py

@@ -13,7 +13,6 @@
 from packaging.version import Version
 from pathlib import Path
 import os
-import pickle
 import re
 import shlex
 import shutil
@@ -272,18 +271,6 @@ def get_checker_config(cls):
         """
         return []
 
-    def analyze(self, analyzer_cmd, res_handler, proc_callback=None, _=None):
-        environment = None
-
-        original_env_file = os.environ.get(
-            'CODECHECKER_ORIGINAL_BUILD_ENV')
-        if original_env_file:
-            with open(original_env_file, 'rb') as env_file:
-                environment = pickle.load(env_file, encoding='utf-8')
-
-        return super().analyze(
-            analyzer_cmd, res_handler, proc_callback, environment)
-
     def post_analyze(self, result_handler):
         """
         Post process the reuslts after the analysis.
@@ -393,7 +380,7 @@ def construct_config_handler(cls, args):
             # No cppcheck arguments file was given in the command line.
             LOG.debug_analyzer(aerr)
 
-        check_env = context.analyzer_env
+        check_env = context.get_analyzer_env(cls.ANALYZER_NAME)
 
         # Overwrite PATH to contain only the parent of the cppcheck binary.
         if os.path.isabs(Cppcheck.analyzer_binary()):

analyzer/codechecker_analyzer/analyzers/gcc/analyzer.py

@@ -7,8 +7,6 @@
 # -------------------------------------------------------------------------
 from collections import defaultdict
 from packaging.version import Version
-import os
-import pickle
 import shlex
 import subprocess
 
@@ -132,17 +130,6 @@ def get_checker_config(cls):
         # TODO
         return []
 
-    def analyze(self, analyzer_cmd, res_handler, proc_callback=None, _=None):
-        env = None
-
-        original_env_file = os.environ.get(
-            'CODECHECKER_ORIGINAL_BUILD_ENV')
-        if original_env_file:
-            with open(original_env_file, 'rb') as env_file:
-                env = pickle.load(env_file, encoding='utf-8')
-
-        return super().analyze(analyzer_cmd, res_handler, proc_callback, env)
-
     def post_analyze(self, result_handler: GccResultHandler):
         """
         Post process the reuslts after the analysis.

analyzer/codechecker_analyzer/cmd/analyzers.py

@@ -186,7 +186,7 @@ def uglify(text):
     rows = []
     for analyzer_name, analyzer_class in \
             analyzer_types.supported_analyzers.items():
-        check_env = context.analyzer_env
+        check_env = context.get_analyzer_env(analyzer_name)
         version = analyzer_class.get_binary_version(check_env)
         if not version:
             version = 'ERROR'

analyzer/codechecker_analyzer/env.py

@@ -9,6 +9,7 @@
 
 import os
 import re
+import pickle
 
 from codechecker_analyzer import analyzer_context
 from codechecker_common.logger import get_logger
@@ -35,6 +36,24 @@ def get_log_env(logfile, original_env):
     return new_env
 
 
+def get_original_env():
+    original_env = os.environ
+    try:
+        original_env_file = os.environ.get('CODECHECKER_ORIGINAL_BUILD_ENV')
+        if original_env_file:
+            LOG.debug_analyzer('Loading original build env from: %s',
+                               original_env_file)
+
+            with open(original_env_file, 'rb') as env_file:
+                original_env = pickle.load(env_file, encoding='utf-8')
+
+    except Exception as ex:
+        LOG.warning(str(ex))
+        LOG.warning('Failed to get saved original_env ')
+        original_env = None
+    return original_env
+
+
 def extend(path_env_extra, ld_lib_path_extra):
     """Extend the checker environment.
 

analyzer/codechecker_analyzer/host_check.py

@@ -59,7 +59,7 @@ def has_analyzer_config_option(clang_bin, config_option_name):
             cmd,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
-            env=analyzer_context.get_context().analyzer_env,
+            env=analyzer_context.get_context().get_analyzer_env("clangsa"),
             encoding="utf-8", errors="ignore")
         out, err = proc.communicate()
         LOG.debug("stdout:\n%s", out)
@@ -92,7 +92,7 @@ def has_analyzer_option(clang_bin, feature):
                 cmd,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
-                env=analyzer_context.get_context().analyzer_env,
+                env=analyzer_context.get_context().get_analyzer_env("clangsa"),
                 encoding="utf-8", errors="ignore")
             out, err = proc.communicate()
             LOG.debug("stdout:\n%s", out)