Merge pull request #48 from IBM/incremental-analysis

Implementation of incremental analysis at analysis level 1

Author: rangeetpan

src/main/java/com/ibm/northstar/CodeAnalyzer.java

@@ -13,8 +13,8 @@
 
 package com.ibm.northstar;
 
-
 import com.github.javaparser.Problem;
+import com.google.common.reflect.TypeToken;
 import com.google.gson.*;
 import com.ibm.northstar.entities.JavaCompilationUnit;
 import com.ibm.northstar.utils.BuildProject;
@@ -27,8 +27,10 @@
 import picocli.CommandLine.Option;
 
 import java.io.File;
+import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.lang.reflect.Type;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -44,6 +46,9 @@ public class CodeAnalyzer implements Runnable {
     @Option(names = {"-i", "--input"}, description = "Path to the project root directory.")
     private static String input;
 
+    @Option(names = {"-t", "--target-files"}, description = "Paths to files to be analyzed from the input application.")
+    private static List<String> targetFiles;
+
     @Option(names = {"-s", "--source-analysis"}, description = "Analyze a single string of java source code instead the project.")
     private static String sourceAnalysis;
 
@@ -62,6 +67,8 @@ public class CodeAnalyzer implements Runnable {
     @Option(names = {"-v", "--verbose"}, description = "Print logs to console.")
     private static boolean verbose = false;
 
+    private static final String outputFileName = "analysis.json";
+
 
     public static Gson gson = new GsonBuilder()
             .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
@@ -109,11 +116,46 @@ private static void analyze() throws IOException, ClassHierarchyException, CallG
             } else {
                 Log.warn("Failed to download library dependencies of project");
             }
-            // construct symbol table for project, write parse problems to file in output directory if specified
-            Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>> symbolTableExtractionResult =
-                SymbolTable.extractAll(Paths.get(input));
+            boolean analysisFileExists = output != null && Files.exists(Paths.get(output + File.separator + outputFileName));
+
+            // if target files are specified, compute symbol table information for the given files
+            if (targetFiles != null) {
+                Log.info(targetFiles.size() + "target files specified for analysis: " + targetFiles);
+
+                // if target files specified for analysis level 2, downgrade to analysis level 1
+                if (analysisLevel > 1) {
+                    Log.warn("Incremental analysis is supported at analysis level 1 only; " +
+                            "performing analysis level 1 for target files");
+                    analysisLevel = 1;
+                }
+
+                // extract symbol table for the specified files
+                symbolTable = SymbolTable.extract(Paths.get(input), targetFiles.stream().map(Paths::get).toList()).getLeft();
+
+                // if analysis file exists, update it with new symbol table information for the specified fiels
+                if (analysisFileExists) {
+                    // read symbol table information from existing analysis file
+                    Map<String, JavaCompilationUnit> existingSymbolTable = readSymbolTableFromFile(
+                            new File(output, outputFileName));
+                    if (existingSymbolTable != null) {
+                        // for each file, tag its symbol table information as "updated" and update existing symbol table
+                        for (String targetFile : targetFiles) {
+                            String targetPathAbs = Paths.get(targetFile).toAbsolutePath().toString();
+                            JavaCompilationUnit javaCompilationUnit = symbolTable.get(targetPathAbs);
+                            javaCompilationUnit.setModified(true);
+                            existingSymbolTable.put(targetPathAbs, javaCompilationUnit);
+                        }
+                    }
+                    symbolTable = existingSymbolTable;
+                }
+            }
+            else {
+                // construct symbol table for project, write parse problems to file in output directory if specified
+                Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>> symbolTableExtractionResult =
+                        SymbolTable.extractAll(Paths.get(input));
 
-            symbolTable = symbolTableExtractionResult.getLeft();
+                symbolTable = symbolTableExtractionResult.getLeft();
+            }
 
             if (analysisLevel > 1) {
                 // Save SDG, and Call graph as JSON
@@ -166,4 +208,15 @@ private static void emit(String consolidatedJSONString) throws IOException {
             }
         }
     }
+
+    private static Map<String, JavaCompilationUnit> readSymbolTableFromFile(File analysisJsonFile) {
+        Type symbolTableType = new TypeToken<Map<String, JavaCompilationUnit>>() {}.getType();
+        try (FileReader reader = new FileReader(analysisJsonFile)) {
+            JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject();
+            return gson.fromJson(jsonObject.get("symbol_table"), symbolTableType);
+        } catch (IOException e) {
+            Log.error("Error reading analysis file: " + e.getMessage());
+        }
+        return null;
+    }
 }

src/main/java/com/ibm/northstar/SymbolTable.java

@@ -17,6 +17,7 @@
 import com.github.javaparser.resolution.types.ResolvedType;
 import com.github.javaparser.symbolsolver.JavaSymbolSolver;
 import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver;
+import com.github.javaparser.symbolsolver.resolution.typesolvers.JarTypeSolver;
 import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver;
 import com.github.javaparser.symbolsolver.utils.SymbolSolverCollectionStrategy;
 import com.github.javaparser.utils.ProjectRoot;
@@ -640,8 +641,8 @@ public static Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>>
         SymbolSolverCollectionStrategy symbolSolverCollectionStrategy = new SymbolSolverCollectionStrategy();
         ProjectRoot projectRoot = symbolSolverCollectionStrategy.collect(projectRootPath);
         javaSymbolSolver = (JavaSymbolSolver)symbolSolverCollectionStrategy.getParserConfiguration().getSymbolResolver().get();
-        Map symbolTable = new LinkedHashMap<String, JavaCompilationUnit>();
-        Map parseProblems = new HashMap<String, List<Problem>>();
+        Map<String, JavaCompilationUnit> symbolTable = new LinkedHashMap<>();
+        Map<String, List<Problem>> parseProblems = new HashMap<>();
         for (SourceRoot sourceRoot : projectRoot.getSourceRoots()) {
             for (ParseResult<CompilationUnit> parseResult : sourceRoot.tryToParse()) {
                 if (parseResult.isSuccessful()) {
@@ -681,6 +682,46 @@ public static Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>>
         return Pair.of(symbolTable, parseProblems);
     }
 
+    /**
+     * Parses the given set of Java source files from the given project and constructs the symbol table.
+     * @param projectRootPath
+     * @param javaFilePaths
+     * @return
+     * @throws IOException
+     */
+    public static Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>> extract(
+            Path projectRootPath,
+            List<Path> javaFilePaths
+    ) throws IOException {
+
+        // create symbol solver and parser configuration
+        SymbolSolverCollectionStrategy symbolSolverCollectionStrategy = new SymbolSolverCollectionStrategy();
+        ProjectRoot projectRoot = symbolSolverCollectionStrategy.collect(projectRootPath);
+        javaSymbolSolver = (JavaSymbolSolver)symbolSolverCollectionStrategy.getParserConfiguration().getSymbolResolver().get();
+        ParserConfiguration parserConfiguration = new ParserConfiguration();
+        parserConfiguration.setSymbolResolver(javaSymbolSolver);
+
+        // create java parser with the configuration
+        JavaParser javaParser = new JavaParser(parserConfiguration);
+
+        Map symbolTable = new LinkedHashMap<String, JavaCompilationUnit>();
+        Map parseProblems = new HashMap<String, List<Problem>>();
+
+        // parse all given files and return pair of symbol table and parse problems
+        for (Path javaFilePath : javaFilePaths) {
+            ParseResult<CompilationUnit> parseResult = javaParser.parse(javaFilePath);
+            if (parseResult.isSuccessful()) {
+                CompilationUnit compilationUnit = parseResult.getResult().get();
+                symbolTable.put(compilationUnit.getStorage().get().getPath().toString(),
+                        processCompilationUnit(compilationUnit));
+            } else {
+                Log.error(parseResult.getProblems().toString());
+                parseProblems.put(javaFilePath.toString(), parseResult.getProblems());
+            }
+        }
+        return Pair.of(symbolTable, parseProblems);
+    }
+
     public static void main(String[] args) throws IOException {
         extractAll(Paths.get(args[0]));
     }

src/main/java/com/ibm/northstar/entities/JavaCompilationUnit.java

@@ -10,4 +10,5 @@ public class JavaCompilationUnit {
     private String comment;
     private List<String> imports;
     private Map<String, Type> typeDeclarations;
+    private boolean isModified;
 }