[analyzer] Modernize analyzer's Python scripts

Summary:
Fix read/write in binary format, which crashes Python 3.
Additionally, clean up redundant (as for Python 3) code and
fix a handful of flake8 warnings.

Differential Revision: https://reviews.llvm.org/D79932

clang/utils/analyzer/CmpRuns.py

@@ -39,26 +39,30 @@
 
 STATS_REGEXP = re.compile(r"Statistics: (\{.+\})", re.MULTILINE | re.DOTALL)
 
-class Colors(object):
+
+class Colors:
     """
     Color for terminal highlight.
     """
     RED = '\x1b[2;30;41m'
     GREEN = '\x1b[6;30;42m'
     CLEAR = '\x1b[0m'
 
-# Information about analysis run:
-# path - the analysis output directory
-# root - the name of the root directory, which will be disregarded when
-# determining the source file name
-class SingleRunInfo(object):
+
+class SingleRunInfo:
+    """
+    Information about analysis run:
+    path - the analysis output directory
+    root - the name of the root directory, which will be disregarded when
+    determining the source file name
+    """
     def __init__(self, path, root="", verboseLog=None):
         self.path = path
         self.root = root.rstrip("/\\")
         self.verboseLog = verboseLog
 
 
-class AnalysisDiagnostic(object):
+class AnalysisDiagnostic:
     def __init__(self, data, report, htmlReport):
         self._data = data
         self._loc = self._data['location']
@@ -80,7 +84,7 @@ def getRootFileName(self):
         p = path[0]
         if 'location' in p:
             fIdx = p['location']['file']
-        else: # control edge
+        else:  # control edge
             fIdx = path[0]['edges'][0]['start'][0]['file']
         out = self._report.files[fIdx]
         root = self._report.run.root
@@ -139,14 +143,14 @@ def getRawData(self):
         return self._data
 
 
-class AnalysisReport(object):
+class AnalysisReport:
     def __init__(self, run, files):
         self.run = run
         self.files = files
         self.diagnostics = []
 
 
-class AnalysisRun(object):
+class AnalysisRun:
     def __init__(self, info):
         self.path = info.path
         self.root = info.root
@@ -303,12 +307,14 @@ def compareResults(A, B, opts):
 
     return res
 
+
 def computePercentile(l, percentile):
     """
     Return computed percentile.
     """
     return sorted(l)[int(round(percentile * len(l) + 0.5)) - 1]
 
+
 def deriveStats(results):
     # Assume all keys are the same in each statistics bucket.
     combined_data = defaultdict(list)
@@ -355,6 +361,7 @@ def compareStats(resultsA, resultsB):
                         report = Colors.RED + report + Colors.CLEAR
             print("\t %s %s" % (kkey, report))
 
+
 def dumpScanBuildResultsDiff(dirA, dirB, opts, deleteEmpty=True,
                              Stdout=sys.stdout):
     # Load the run results.
@@ -367,7 +374,7 @@ def dumpScanBuildResultsDiff(dirA, dirB, opts, deleteEmpty=True,
 
     # Open the verbose log, if given.
     if opts.verboseLog:
-        auxLog = open(opts.verboseLog, "wb")
+        auxLog = open(opts.verboseLog, "w")
     else:
         auxLog = None
 
@@ -405,6 +412,7 @@ def dumpScanBuildResultsDiff(dirA, dirB, opts, deleteEmpty=True,
 
     return foundDiffs, len(resultsA.diagnostics), len(resultsB.diagnostics)
 
+
 def generate_option_parser():
     parser = OptionParser("usage: %prog [options] [dir A] [dir B]")
     parser.add_option("", "--rootA", dest="rootA",

clang/utils/analyzer/SATestAdd.py

@@ -52,8 +52,8 @@
 
 def isExistingProject(PMapFile, projectID):
     PMapReader = csv.reader(PMapFile)
-    for I in PMapReader:
-        if projectID == I[0]:
+    for ProjectInfo in PMapReader:
+        if projectID == ProjectInfo[0]:
             return True
     return False
 
@@ -71,21 +71,24 @@ def addNewProject(ID, BuildMode):
         sys.exit(-1)
 
     # Build the project.
+    # TODO: Repair this call.  We give it a wrong amount wrong arguments and it
+    #       is not trivial to construct argparse arguments in here.
+    #       Requires refactoring of the 'testProject' function.
     SATestBuild.testProject(ID, BuildMode, IsReferenceBuild=True)
 
     # Add the project ID to the project map.
     ProjectMapPath = os.path.join(CurDir, SATestBuild.ProjectMapFile)
 
     if os.path.exists(ProjectMapPath):
-        FileMode = "r+b"
+        FileMode = "r+"
     else:
         print("Warning: Creating the Project Map file!!")
-        FileMode = "w+b"
+        FileMode = "w+"
 
     with open(ProjectMapPath, FileMode) as PMapFile:
         if (isExistingProject(PMapFile, ID)):
-            print('Warning: Project with ID \'', ID, \
-                                 '\' already exists.', file=sys.stdout)
+            print('Warning: Project with ID \'', ID,
+                  '\' already exists.', file=sys.stdout)
             print("Reference output has been regenerated.", file=sys.stdout)
         else:
             PMapWriter = csv.writer(PMapFile)
@@ -97,12 +100,12 @@ def addNewProject(ID, BuildMode):
 # TODO: Set the path to the Repository directory.
 if __name__ == '__main__':
     if len(sys.argv) < 2 or sys.argv[1] in ('-h', '--help'):
-        print('Add a new project for testing to the analyzer'\
-                             '\nUsage: ', sys.argv[0],\
-                             'project_ID <mode>\n' \
-                             'mode: 0 for single file project, ' \
-                             '1 for scan_build, ' \
-                             '2 for single file c++11 project', file=sys.stderr)
+        print('Add a new project for testing to the analyzer'
+              '\nUsage: ', sys.argv[0],
+              'project_ID <mode>\n'
+              'mode: 0 for single file project, '
+              '1 for scan_build, '
+              '2 for single file c++11 project', file=sys.stderr)
         sys.exit(-1)
 
     BuildMode = 1

clang/utils/analyzer/SATestBuild.py

@@ -75,7 +75,8 @@
     level=logging.DEBUG,
     format='%(asctime)s:%(levelname)s:%(name)s: %(message)s')
 
-class StreamToLogger(object):
+
+class StreamToLogger:
     def __init__(self, logger, log_level=logging.INFO):
         self.logger = logger
         self.log_level = log_level
@@ -377,7 +378,7 @@ def runAnalyzePreprocessed(Args, Dir, SBOutputDir, Mode):
         # Build and call the analyzer command.
         OutputOption = "-o '%s.plist' " % os.path.join(PlistPath, FileName)
         Command = CmdPrefix + OutputOption + ("'%s'" % FileName)
-        LogFile = open(os.path.join(FailPath, FileName + ".stderr.txt"), "w+b")
+        LogFile = open(os.path.join(FailPath, FileName + ".stderr.txt"), "w+")
         try:
             if Verbose == 1:
                 Local.stdout.write("  Executing: %s\n" % (Command,))
@@ -432,7 +433,7 @@ def buildProject(Args, Dir, SBOutputDir, ProjectBuildMode, IsReferenceBuild):
     os.makedirs(os.path.join(SBOutputDir, LogFolderName))
 
     # Build and analyze the project.
-    with open(BuildLogPath, "wb+") as PBuildLogFile:
+    with open(BuildLogPath, "w+") as PBuildLogFile:
         if (ProjectBuildMode == 1):
             downloadAndPatch(Dir, PBuildLogFile)
             runCleanupScript(Dir, PBuildLogFile)
@@ -646,7 +647,7 @@ def __init__(self, Args, TasksQueue, ResultsDiffer, FailureFlag):
         self.TasksQueue = TasksQueue
         self.ResultsDiffer = ResultsDiffer
         self.FailureFlag = FailureFlag
-        super(TestProjectThread, self).__init__()
+        super().__init__()
 
         # Needed to gracefully handle interrupts with Ctrl-C
         self.daemon = True
@@ -700,7 +701,7 @@ def testProject(Args, ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0
 
 
 def projectFileHandler():
-    return open(getProjectMapPath(), "rb")
+    return open(getProjectMapPath(), "r")
 
 
 def iterateOverProjects(PMapFile):
@@ -709,25 +710,26 @@ def iterateOverProjects(PMapFile):
     from the start.
     """
     PMapFile.seek(0)
-    for I in csv.reader(PMapFile):
-        if (SATestUtils.isCommentCSVLine(I)):
+    for ProjectInfo in csv.reader(PMapFile):
+        if (SATestUtils.isCommentCSVLine(ProjectInfo)):
             continue
-        yield I
+        yield ProjectInfo
 
 
 def validateProjectFile(PMapFile):
     """
     Validate project file.
     """
-    for I in iterateOverProjects(PMapFile):
-        if len(I) != 2:
+    for ProjectInfo in iterateOverProjects(PMapFile):
+        if len(ProjectInfo) != 2:
             print("Error: Rows in the ProjectMapFile should have 2 entries.")
             raise Exception()
-        if I[1] not in ('0', '1', '2'):
-            print("Error: Second entry in the ProjectMapFile should be 0" \
+        if ProjectInfo[1] not in ('0', '1', '2'):
+            print("Error: Second entry in the ProjectMapFile should be 0"
                   " (single file), 1 (project), or 2(single file c++11).")
             raise Exception()
 
+
 def singleThreadedTestAll(Args, ProjectsToTest):
     """
     Run all projects.
@@ -738,6 +740,7 @@ def singleThreadedTestAll(Args, ProjectsToTest):
         Success &= testProject(Args, *ProjArgs)
     return Success
 
+
 def multiThreadedTestAll(Args, ProjectsToTest, Jobs):
     """
     Run each project in a separate thread.

clang/utils/analyzer/SATestUpdateDiffs.py

@@ -31,14 +31,14 @@ def updateReferenceResults(ProjName, ProjBuildMode):
         SATestBuild.getSBOutputDirName(IsReferenceBuild=False))
 
     if not os.path.exists(CreatedResultsPath):
-        print("New results not found, was SATestBuild.py "\
-                             "previously run?", file=sys.stderr)
+        print("New results not found, was SATestBuild.py "
+              "previously run?", file=sys.stderr)
         sys.exit(1)
 
     BuildLogPath = SATestBuild.getBuildLogPath(RefResultsPath)
     Dirname = os.path.dirname(os.path.abspath(BuildLogPath))
     runCmd("mkdir -p '%s'" % Dirname)
-    with open(BuildLogPath, "wb+") as PBuildLogFile:
+    with open(BuildLogPath, "w+") as PBuildLogFile:
         # Remove reference results: in git, and then again for a good measure
         # with rm, as git might not remove things fully if there are empty
         # directories involved.
@@ -63,9 +63,10 @@ def updateReferenceResults(ProjName, ProjBuildMode):
 
 def main(argv):
     if len(argv) == 2 and argv[1] in ('-h', '--help'):
-        print("Update static analyzer reference results based "\
-                             "\non the previous run of SATestBuild.py.\n"\
-                             "\nN.B.: Assumes that SATestBuild.py was just run", file=sys.stderr)
+        print("Update static analyzer reference results based "
+              "\non the previous run of SATestBuild.py.\n"
+              "\nN.B.: Assumes that SATestBuild.py was just run",
+              file=sys.stderr)
         sys.exit(1)
 
     with SATestBuild.projectFileHandler() as f:

clang/utils/analyzer/SATestUtils.py

@@ -5,6 +5,7 @@
 
 Verbose = 1
 
+
 def which(command, paths=None):
     """which(command, [paths]) - Look up the given command in the paths string
     (or the PATH environment variable, if unspecified)."""

clang/utils/analyzer/SumTimerInfo.py

@@ -12,8 +12,8 @@
 
 if __name__ == '__main__':
     if len(sys.argv) < 2:
-        print('Usage: ', sys.argv[0],\
-                             'scan_build_output_file', file=sys.stderr)
+        print('Usage: ', sys.argv[0],
+              'scan_build_output_file', file=sys.stderr)
         sys.exit(-1)
 
     f = open(sys.argv[1], 'r')