Skip to content

Enable analysis over a single file provided as a string to codeanalyzer. #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 16, 2024
Merged

Enable analysis over a single file provided as a string to codeanalyzer. #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c289cc6
Enable analysis over a single file provided as a string to codeanalyzer.
rahlk May 14, 2024
f548a23
Merge branch 'main' into analysis-over-a-single-file
rahlk May 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 54 additions & 41 deletions src/main/java/com/ibm/northstar/CodeAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,27 +43,27 @@
@Command(name = "codeanalyzer", mixinStandardHelpOptions = true, sortOptions = false, version = "codeanalyzer v1.1", description = "Convert java binary (*.jar, *.ear, *.war) into a comprehensive system dependency graph.")
public class CodeAnalyzer implements Runnable {

@Option(names = {"-i", "--input"}, required = true, description = "Path to the project root directory.")
@Option(names = {"-i", "--input"}, description = "Path to the project root directory.")
private static String input;

@Option(names = {"-s", "--source-analysis"}, description = "Analyze a single string of java source code instead the project.")
private static String sourceAnalysis;

@Option(names = {"-o", "--output"}, description = "Destination directory to save the output graphs. By default, the SDG formatted as a JSON will be printed to the console.")
private static String output;

@Option(names = {"-b", "--build-cmd"}, description = "Custom build command. Defaults to auto build.")
private static String build;

@Option(names = {"--no-build"}, description = "Do not build your application. Use this option if you have already built your application.")
private static boolean noBuild = false;

@Option(names = {"-a", "--analysis-level"}, description = "[Optional] Level of analysis to perform. Options: 1 (for just symbol table) or 2 (for full analysis including the system depenedency graph). Default: 1")
@Option(names = {"-a", "--analysis-level"}, description = "Level of analysis to perform. Options: 1 (for just symbol table) or 2 (for full analysis including the system depenedency graph). Default: 1")
private static int analysisLevel = 1;

@Option(names = {"-o", "--output"}, description = "[Optional] Destination directory to save the output graphs. By default, the SDG formatted as a JSON will be printed to the console.")
private static String output;

@Option(names = {"-d", "--dependencies"}, description = "[Optional] Path to the application 3rd party dependencies that may be helpful in analyzing the application.")
@Option(names = {"-d", "--dependencies"}, description = "Path to the application 3rd party dependencies that may be helpful in analyzing the application.")
private static String dependencies;

@Option(names = {"-s", "--source-analysis"}, description = "[Experimental] Analyze the source code instead directly of the binary. Warning: This option is experimental and may not work as expected.")
private static boolean analyzeSource = false;

@Option(names = {"-v", "--verbose"}, description = "Print logs to console.")
private static boolean verbose = false;

Expand Down Expand Up @@ -96,42 +96,55 @@ public void run() {

private static void analyze() throws IOException, ClassHierarchyException, CallGraphBuilderCancelException {

// download library dependencies of project for type resolution
if (!BuildProject.downloadLibraryDependencies(input)) {
Log.warn("Failed to download library dependencies of project");
JsonObject combinedJsonObject = new JsonObject();
Map<String, JavaCompilationUnit> symbolTable;
// First of all if, sourceAnalysis is provided, we will analyze the source code instead of the project.
if (sourceAnalysis != null) {
// Construct symbol table for source code
Log.debug("Single file analysis.");
Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>> symbolTableExtractionResult = SymbolTable.extractSingle(sourceAnalysis);
symbolTable = symbolTableExtractionResult.getLeft();
}
// construct symbol table for project, write parse problems to file in output directory if specified
Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>> symbolTableExtractionResult =
SymbolTable.extractAll(Paths.get(input));
Map<String, JavaCompilationUnit> symbolTable = symbolTableExtractionResult.getLeft();
if (output != null) {
Path outputPath = Paths.get(output);
if (!Files.exists(outputPath)) {
Files.createDirectories(outputPath);

else {

// download library dependencies of project for type resolution
if (!BuildProject.downloadLibraryDependencies(input)) {
Log.warn("Failed to download library dependencies of project");
}
// construct symbol table for project, write parse problems to file in output directory if specified
Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>> symbolTableExtractionResult =
SymbolTable.extractAll(Paths.get(input));

symbolTable = symbolTableExtractionResult.getLeft();
if (output != null) {
Path outputPath = Paths.get(output);
if (!Files.exists(outputPath)) {
Files.createDirectories(outputPath);
}
gson.toJson(symbolTableExtractionResult.getRight(), new FileWriter(new File(outputPath.toString(), "parse_errors.json")));
}
gson.toJson(symbolTableExtractionResult.getRight(), new FileWriter(new File(outputPath.toString(), "parse_errors.json")));
}

JsonObject combinedJsonObject = new JsonObject();
if (analysisLevel > 1) {
// Save SDG, IPCFG, and Call graph as JSON
// If noBuild is not true, and build is also not provided, we will use "auto" as the build command
build = build == null ? "auto" : build;
// Is noBuild is true, we will not build the project
build = noBuild ? null : build;
String sdgAsJSONString = SystemDependencyGraph.construct(input, dependencies, build);
JsonElement sdgAsJSONElement = gson.fromJson(sdgAsJSONString, JsonElement.class);
JsonObject sdgAsJSONObject = sdgAsJSONElement.getAsJsonObject();

// We don't really need these fields, so we'll remove it.
sdgAsJSONObject.remove("nodes");
sdgAsJSONObject.remove("creator");
sdgAsJSONObject.remove("version");

// Remove the 'edges' element and move the list of edges up one level
JsonElement edges = sdgAsJSONObject.get("edges");
combinedJsonObject.add("system_dependency_graph", edges);
if (analysisLevel > 1) {
// Save SDG, and Call graph as JSON
// If noBuild is not true, and build is also not provided, we will use "auto" as the build command
build = build == null ? "auto" : build;
// Is noBuild is true, we will not build the project
build = noBuild ? null : build;
String sdgAsJSONString = SystemDependencyGraph.construct(input, dependencies, build);
JsonElement sdgAsJSONElement = gson.fromJson(sdgAsJSONString, JsonElement.class);
JsonObject sdgAsJSONObject = sdgAsJSONElement.getAsJsonObject();

// We don't really need these fields, so we'll remove it.
sdgAsJSONObject.remove("nodes");
sdgAsJSONObject.remove("creator");
sdgAsJSONObject.remove("version");

// Remove the 'edges' element and move the list of edges up one level
JsonElement edges = sdgAsJSONObject.get("edges");
combinedJsonObject.add("system_dependency_graph", edges);

}
}

// Convert the JavaCompilationUnit to JSON and add to consolidated json object
Expand Down
29 changes: 28 additions & 1 deletion src/main/java/com/ibm/northstar/SymbolTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package com.ibm.northstar;

import com.github.javaparser.JavaParser;
import com.github.javaparser.ParseResult;
import com.github.javaparser.ParserConfiguration;
import com.github.javaparser.Problem;
import com.github.javaparser.ast.CompilationUnit;
import com.github.javaparser.ast.NodeList;
Expand All @@ -16,6 +18,8 @@
import com.github.javaparser.resolution.UnsolvedSymbolException;
import com.github.javaparser.resolution.types.ResolvedType;
import com.github.javaparser.symbolsolver.JavaSymbolSolver;
import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver;
import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver;
import com.github.javaparser.symbolsolver.utils.SymbolSolverCollectionStrategy;
import com.github.javaparser.utils.ProjectRoot;
import com.github.javaparser.utils.SourceRoot;
Expand Down Expand Up @@ -90,7 +94,7 @@ private boolean isMethodSignatureMatch(String fullSignature, String searchSignat
private static JavaCompilationUnit processCompilationUnit(CompilationUnit parseResult) {
JavaCompilationUnit cUnit = new JavaCompilationUnit();

cUnit.setFilePath(parseResult.getStorage().get().getFileName());
cUnit.setFilePath(parseResult.getStorage().map(s -> s.getPath().toString()).orElse("<in-memory>"));

// Add the comment field to the compilation unit
cUnit.setComment(parseResult.getComment().isPresent() ? parseResult.getComment().get().asString() : "");
Expand Down Expand Up @@ -573,6 +577,29 @@ public static Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>>
return Pair.of(symbolTable, parseProblems);
}

public static Pair<Map<String, JavaCompilationUnit>, Map<String, List<Problem>>> extractSingle(String code) throws IOException {
Map symbolTable = new LinkedHashMap<String, JavaCompilationUnit>();
Map parseProblems = new HashMap<String, List<Problem>>();
// Setting up symbol solvers
CombinedTypeSolver combinedTypeSolver = new CombinedTypeSolver();
combinedTypeSolver.add(new ReflectionTypeSolver());

ParserConfiguration parserConfiguration = new ParserConfiguration();
parserConfiguration.setSymbolResolver(new JavaSymbolSolver(combinedTypeSolver));

JavaParser javaParser = new JavaParser(parserConfiguration);
ParseResult<CompilationUnit> parseResult = javaParser.parse(code);
if (parseResult.isSuccessful()) {
CompilationUnit compilationUnit = parseResult.getResult().get();
Log.debug("Successfully parsed code. Now processing compilation unit");
symbolTable.put("<pseudo-path>", processCompilationUnit(compilationUnit));
} else {
Log.error(parseResult.getProblems().toString());
parseProblems.put("code", parseResult.getProblems());
}
return Pair.of(symbolTable, parseProblems);
}

public static void main(String[] args) throws IOException {
extractAll(Paths.get(args[0]));
}
Expand Down
99 changes: 49 additions & 50 deletions src/main/java/com/ibm/northstar/SystemDependencyGraph.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,58 +104,57 @@ private static org.jgrapht.Graph<Pair<String, Callable>, AbstractGraphEdge> buil
// We'll use forward and backward search on the DFS to identify which CFG nodes
// are dominant
// This is a forward DFS search (or exit time first search)
// int dfsNumber = 0;
// Map<Statement, Integer> dfsFinish = HashMapFactory.make();
// Iterator<Statement> search = DFS.iterateFinishTime(sdg, entryPoints.get());
//
// while (search.hasNext()) {
// dfsFinish.put(search.next(), dfsNumber++);
// }
//
// // This is a reverse DFS search (or entry time first search)
// int reverseDfsNumber = 0;
// Map<Statement, Integer> dfsStart = HashMapFactory.make();
// Iterator<Statement> reverseSearch = DFS.iterateDiscoverTime(sdg, entryPoints.get());
//
// while (reverseSearch.hasNext()) {
// dfsStart.put(reverseSearch.next(), reverseDfsNumber++);
// }
int dfsNumber = 0;
Map<Statement, Integer> dfsFinish = HashMapFactory.make();
Iterator<Statement> search = DFS.iterateFinishTime(sdg, entryPoints.get());

while (search.hasNext()) {
dfsFinish.put(search.next(), dfsNumber++);
}

// This is a reverse DFS search (or entry time first search)
int reverseDfsNumber = 0;
Map<Statement, Integer> dfsStart = HashMapFactory.make();
Iterator<Statement> reverseSearch = DFS.iterateDiscoverTime(sdg, entryPoints.get());

while (reverseSearch.hasNext()) {
dfsStart.put(reverseSearch.next(), reverseDfsNumber++);
}

// Populate graph
// sdg.stream()
// .filter(dfsFinish::containsKey)
// .sorted(Comparator.comparingInt(dfsFinish::get))
// .forEach(p -> sdg.getSuccNodes(p).forEachRemaining(s -> {
// if (dfsFinish.containsKey(s)
// && dfsStart.get(p) != null && dfsStart.get(s) != null
// && !((dfsStart.get(p) >= dfsStart.get(s))
// && (dfsFinish.get(p) <= dfsFinish.get(s)))
// && !p.getNode().getMethod().equals(s.getNode().getMethod())) {
//
// // Add the source nodes to the graph as vertices
// Pair<String, Callable> source = Optional.ofNullable(getCallableFromSymbolTable(p.getNode().getMethod())).orElseGet(() -> createAndPutNewCallableInSymbolTable(p.getNode().getMethod()));
// graph.addVertex(source);
//
// // Add the target nodes to the graph as vertices
// Pair<String, Callable> target = Optional.ofNullable(getCallableFromSymbolTable(s.getNode().getMethod())).orElseGet(() -> createAndPutNewCallableInSymbolTable(s.getNode().getMethod()));
// graph.addVertex(target);
//
//
// String edgeType = edgeLabels.apply(p, s);
// SystemDepEdge graphEdge = new SystemDepEdge(p, s, edgeType);
// SystemDepEdge cgEdge = (SystemDepEdge) graph.getEdge(source, target);
// if (source.getRight() != null && target.getRight() != null) {
// if (cgEdge == null || !cgEdge.equals(graphEdge)) {
// graph.addEdge(
// source,
// target,
// graphEdge);
// } else {
// graphEdge.incrementWeight();
// }
// }
// }
// }));
sdg.stream()
.filter(dfsFinish::containsKey)
.sorted(Comparator.comparingInt(dfsFinish::get))
.forEach(p -> sdg.getSuccNodes(p).forEachRemaining(s -> {
if (dfsFinish.containsKey(s)
&& dfsStart.get(p) != null && dfsStart.get(s) != null
&& !((dfsStart.get(p) >= dfsStart.get(s))
&& (dfsFinish.get(p) <= dfsFinish.get(s)))
&& !p.getNode().getMethod().equals(s.getNode().getMethod())) {

// Add the source nodes to the graph as vertices
Pair<String, Callable> source = Optional.ofNullable(getCallableFromSymbolTable(p.getNode().getMethod())).orElseGet(() -> createAndPutNewCallableInSymbolTable(p.getNode().getMethod()));
graph.addVertex(source);

// Add the target nodes to the graph as vertices
Pair<String, Callable> target = Optional.ofNullable(getCallableFromSymbolTable(s.getNode().getMethod())).orElseGet(() -> createAndPutNewCallableInSymbolTable(s.getNode().getMethod()));
graph.addVertex(target);

String edgeType = edgeLabels.apply(p, s);
SystemDepEdge graphEdge = new SystemDepEdge(p, s, edgeType);
SystemDepEdge cgEdge = (SystemDepEdge) graph.getEdge(source, target);
if (source.getRight() != null && target.getRight() != null) {
if (cgEdge == null || !cgEdge.equals(graphEdge)) {
graph.addEdge(
source,
target,
graphEdge);
} else {
graphEdge.incrementWeight();
}
}
}
}));

callGraph.getEntrypointNodes()
.forEach(p -> {
Expand Down
15 changes: 15 additions & 0 deletions src/main/java/com/ibm/northstar/entities/SystemDepEdge.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ public class SystemDepEdge extends AbstractGraphEdge {
* The Type.
*/
public final String type;
public final String sourceKind;
public final String destinationKind;

/**
* Instantiates a new System dep edge.
Expand All @@ -51,6 +53,8 @@ public class SystemDepEdge extends AbstractGraphEdge {
*/
public SystemDepEdge(Statement sourceStatement, Statement destinationStatement, String type) {
super();
this.sourceKind = sourceStatement.getKind().toString();
this.destinationKind = destinationStatement.getKind().toString();
this.type = type;
this.sourcePos = getStatementPosition(sourceStatement);
this.destinationPos = getStatementPosition(destinationStatement);
Expand All @@ -69,6 +73,15 @@ public boolean equals(Object o) {
&& this.type.equals(((SystemDepEdge) o).getType());
}


public String getSourceKind() {
return sourceKind;
}

public String getDestinationKind() {
return destinationKind;
}

/**
* Gets type.
*
Expand Down Expand Up @@ -98,7 +111,9 @@ public Integer getDestinationPos() {

public Map<String, Attribute> getAttributes() {
Map<String, Attribute> map = new LinkedHashMap<>();
map.put("source_kind", DefaultAttribute.createAttribute(getSourceKind()));
map.put("type", DefaultAttribute.createAttribute(getType()));
map.put("destination_kind", DefaultAttribute.createAttribute(getDestinationKind()));
map.put("weight", DefaultAttribute.createAttribute(String.valueOf(getWeight())));
return map;
}
Expand Down