Skip to content

Commit

Permalink
Merge pull request #333 from knrc/performance
Browse files Browse the repository at this point in the history 
Fix performance issue for aggregates, fixes #324
  • Loading branch information
@hboutemy
hboutemy authored Apr 13, 2023
2 parents 81c9668 + e839807 commit bbcf2ff
Show file tree
Hide file tree
Showing 6 changed files with 133 additions and 174 deletions.
91 changes: 79 additions & 12 deletions src/main/java/org/cyclonedx/maven/BaseCycloneDxMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.project.MavenProject;
import org.apache.maven.project.MavenProjectHelper;
import org.apache.maven.shared.dependency.analyzer.ProjectDependencyAnalysis;
import org.cyclonedx.BomGeneratorFactory;
import org.cyclonedx.CycloneDxSchema;
import org.cyclonedx.exception.GeneratorException;
Expand All @@ -45,8 +46,10 @@
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;

Expand Down Expand Up @@ -230,12 +233,13 @@ protected Component convert(Artifact artifact) {
/**
* Analyze the current Maven project to extract the BOM components list and their dependencies.
*
* @param components the components set to fill
* @param dependencies the dependencies set to fill
* @param topLevelComponents the PURLs for all top level components
* @param components the components map to fill
* @param dependencies the dependencies map to fill
* @return the name of the analysis done to store as a BOM, or {@code null} to not save result.
* @throws MojoExecutionException something weird happened...
*/
protected abstract String extractComponentsAndDependencies(Set<Component> components, Set<Dependency> dependencies) throws MojoExecutionException;
protected abstract String extractComponentsAndDependencies(Set<String> topLevelComponents, Map<String, Component> components, Map<String, Dependency> dependencies) throws MojoExecutionException;

public void execute() throws MojoExecutionException {
final boolean shouldSkip = Boolean.parseBoolean(System.getProperty("cyclonedx.skip", Boolean.toString(skip)));
Expand All @@ -245,10 +249,12 @@ public void execute() throws MojoExecutionException {
}
logParameters();

final Set<Component> components = new LinkedHashSet<>();
final Set<Dependency> dependencies = new LinkedHashSet<>();
// top level components do not currently set their scope, we track these to prevent merging of scopes
final Set<String> topLevelComponents = new LinkedHashSet<>();
final Map<String, Component> componentMap = new LinkedHashMap<>();
final Map<String, Dependency> dependencyMap = new LinkedHashMap<>();

String analysis = extractComponentsAndDependencies(components, dependencies);
String analysis = extractComponentsAndDependencies(topLevelComponents, componentMap, dependencyMap);
if (analysis != null) {
List<String> scopes = new ArrayList<>();
if (includeCompileScope) scopes.add("compile");
Expand All @@ -258,25 +264,29 @@ public void execute() throws MojoExecutionException {
if (includeTestScope) scopes.add("test");

final Metadata metadata = modelConverter.convert(project, analysis + " " + String.join("+", scopes), projectType, schemaVersion(), includeLicenseText);
projectDependenciesConverter.cleanupBomDependencies(metadata, components, dependencies);

generateBom(analysis, metadata, components, dependencies);
final Component rootComponent = metadata.getComponent();
componentMap.remove(rootComponent.getPurl());

projectDependenciesConverter.cleanupBomDependencies(metadata, componentMap, dependencyMap);

generateBom(analysis, metadata, new ArrayList<>(componentMap.values()), new ArrayList<>(dependencyMap.values()));
}
}

private void generateBom(String analysis, Metadata metadata, Set<Component> components, Set<Dependency> dependencies) throws MojoExecutionException {
private void generateBom(String analysis, Metadata metadata, List<Component> components, List<Dependency> dependencies) throws MojoExecutionException {
try {
getLog().info(String.format(MESSAGE_CREATING_BOM, schemaVersion, components.size()));
final Bom bom = new Bom();
bom.setComponents(new ArrayList<>(components));
bom.setComponents(components);

if (schemaVersion().getVersion() >= 1.1 && includeBomSerialNumber) {
bom.setSerialNumber("urn:uuid:" + UUID.randomUUID());
}

if (schemaVersion().getVersion() >= 1.2) {
bom.setMetadata(metadata);
bom.setDependencies(new ArrayList<>(dependencies));
bom.setDependencies(dependencies);
}

/*if (schemaVersion().getVersion() >= 1.3) {
Expand Down Expand Up @@ -333,7 +343,7 @@ private void saveBomToFile(String bomString, String extension, Parser bomParser)
}
}

protected Set<Dependency> extractBOMDependencies(MavenProject mavenProject) throws MojoExecutionException {
protected Map<String, Dependency> extractBOMDependencies(MavenProject mavenProject) throws MojoExecutionException {
ProjectDependenciesConverter.MavenDependencyScopes include = new ProjectDependenciesConverter.MavenDependencyScopes(includeCompileScope, includeProvidedScope, includeRuntimeScope, includeTestScope, includeSystemScope);
return projectDependenciesConverter.extractBOMDependencies(mavenProject, include, excludeTypes);
}
Expand Down Expand Up @@ -378,4 +388,61 @@ protected void logParameters() {
getLog().info("------------------------------------------------------------------------");
}
}

protected void populateComponents(final Set<String> topLevelComponents, final Map<String, Component> components, final Set<Artifact> artifacts, final ProjectDependencyAnalysis dependencyAnalysis) {
for (Artifact artifact: artifacts) {
final String purl = generatePackageUrl(artifact);
final Component.Scope artifactScope = (dependencyAnalysis != null ? inferComponentScope(artifact, dependencyAnalysis) : null);
final Component component = components.get(purl);
if (component == null) {
final Component newComponent = convert(artifact);
newComponent.setScope(artifactScope);
components.put(purl, newComponent);
} else if (!topLevelComponents.contains(purl)) {
component.setScope(mergeScopes(component.getScope(), artifactScope));
}
}
}

/**
* Infer BOM component scope (required/optional/excluded) based on Maven project dependency analysis.
*
* @param artifact Artifact from maven project
* @param projectDependencyAnalysis Maven Project Dependency Analysis data
*
* @return Component.Scope - REQUIRED: If the component is used (as detected by project dependency analysis). OPTIONAL: If it is unused
*/
protected Component.Scope inferComponentScope(Artifact artifact, ProjectDependencyAnalysis projectDependencyAnalysis) {
if (projectDependencyAnalysis == null) {
return null;
}

Set<Artifact> usedDeclaredArtifacts = projectDependencyAnalysis.getUsedDeclaredArtifacts();
Set<Artifact> usedUndeclaredArtifacts = projectDependencyAnalysis.getUsedUndeclaredArtifacts();
Set<Artifact> unusedDeclaredArtifacts = projectDependencyAnalysis.getUnusedDeclaredArtifacts();
Set<Artifact> testArtifactsWithNonTestScope = projectDependencyAnalysis.getTestArtifactsWithNonTestScope();

// Is the artifact used?
if (usedDeclaredArtifacts.contains(artifact) || usedUndeclaredArtifacts.contains(artifact)) {
return Component.Scope.REQUIRED;
}

// Is the artifact unused or test?
if (unusedDeclaredArtifacts.contains(artifact) || testArtifactsWithNonTestScope.contains(artifact)) {
return Component.Scope.OPTIONAL;
}

return null;
}

// Merging of scopes follows the method previously implemented in the aggregate code. This needs to be fixed in a future PR.
private Component.Scope mergeScopes(final Component.Scope existing, final Component.Scope project) {
if ((Component.Scope.REQUIRED == existing) || (Component.Scope.REQUIRED == project)) {
return Component.Scope.REQUIRED;
}
if (null == existing) {
return project;
}
return existing;
}
}
88 changes: 13 additions & 75 deletions src/main/java/org/cyclonedx/maven/CycloneDxAggregateMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,16 @@
*/
package org.cyclonedx.maven;

import org.apache.maven.artifact.Artifact;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
import org.apache.maven.plugins.annotations.Parameter;
import org.apache.maven.plugins.annotations.ResolutionScope;
import org.apache.maven.project.MavenProject;
import org.apache.maven.shared.dependency.analyzer.ProjectDependencyAnalysis;
import org.cyclonedx.model.Component;
import org.cyclonedx.model.Dependency;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -106,57 +101,35 @@ protected void logAdditionalParameters() {
getLog().info("outputReactorProjects : " + outputReactorProjects);
}

protected String extractComponentsAndDependencies(final Set<Component> components, final Set<Dependency> dependencies) throws MojoExecutionException {
@Override
protected String extractComponentsAndDependencies(final Set<String> topLevelComponents, final Map<String, Component> components, final Map<String, Dependency> dependencies) throws MojoExecutionException {
if (! getProject().isExecutionRoot()) {
// non-root project: let parent class create a module-only BOM?
if (outputReactorProjects) {
return super.extractComponentsAndDependencies(components, dependencies);
return super.extractComponentsAndDependencies(topLevelComponents, components, dependencies);
}
getLog().info("Skipping CycloneDX on non-execution root");
return null;
}

// root project: analyze and aggregate all the modules
getLog().info((reactorProjects.size() <= 1) ? MESSAGE_RESOLVING_DEPS : MESSAGE_RESOLVING_AGGREGATED_DEPS);
final Set<String> componentRefs = new LinkedHashSet<>();

// Perform used/unused dependencies analysis for all projects upfront
final List<ProjectDependencyAnalysis> projectsDependencyAnalysis = prepareMavenDependencyAnalysis();

// Add reference to BOM metadata component.
// Without this, direct dependencies of the Maven project cannot be determined.
final Component bomComponent = convert(getProject().getArtifact());
componentRefs.add(bomComponent.getBomRef());

for (final MavenProject mavenProject : reactorProjects) {
if (shouldExclude(mavenProject)) {
getLog().info("Excluding " + mavenProject.getArtifactId());
continue;
}

// Add reference to BOM metadata component.
// Without this, direct dependencies of the Maven project cannot be determined.
final Map<String, Dependency> projectDependencies = extractBOMDependencies(mavenProject);

final Component projectBomComponent = convert(mavenProject.getArtifact());
if (! mavenProject.isExecutionRoot()) {
// DO NOT include root project as it's already been included as a bom metadata component
// Also, ensure that only one project component with the same bom-ref exists in the BOM
if (!componentRefs.contains(projectBomComponent.getBomRef())) {
components.add(projectBomComponent);
}
}
componentRefs.add(projectBomComponent.getBomRef());
components.put(projectBomComponent.getPurl(), projectBomComponent);
topLevelComponents.add(projectBomComponent.getPurl());

for (final Artifact artifact : mavenProject.getArtifacts()) {
final Component component = convert(artifact);
populateComponents(topLevelComponents, components, mavenProject.getArtifacts(), doProjectDependencyAnalysis(mavenProject));

// ensure that only one component with the same bom-ref exists in the BOM
if (componentRefs.add(component.getBomRef())) {
component.setScope(inferComponentScope(artifact, projectsDependencyAnalysis));
components.add(component);
}
}

dependencies.addAll(extractBOMDependencies(mavenProject));
projectDependencies.forEach(dependencies::putIfAbsent);
}

addMavenProjectsAsParentDependencies(reactorProjects, dependencies);
Expand All @@ -172,51 +145,16 @@ protected String extractComponentsAndDependencies(final Set<Component> component
* @param reactorProjects the Maven projects from the reactor
* @param dependencies all BOM dependencies found in reactor
*/
private void addMavenProjectsAsParentDependencies(List<MavenProject> reactorProjects, Set<Dependency> dependencies) {
Map<String, Dependency> dependenciesByRef = new HashMap<>();
dependencies.forEach(d -> dependenciesByRef.put(d.getRef(), d));

private void addMavenProjectsAsParentDependencies(List<MavenProject> reactorProjects, Map<String, Dependency> dependencies) {
for (final MavenProject project: reactorProjects) {
if (project.hasParent()) {
final String parentRef = generatePackageUrl(project.getParentArtifact());
Dependency parentDependency = dependenciesByRef.get(parentRef);
Dependency parentDependency = dependencies.get(parentRef);
if (parentDependency != null) {
final Dependency child = new Dependency(generatePackageUrl(project.getArtifact()));
parentDependency.addDependency(child);
final String projectRef = generatePackageUrl(project.getArtifact());
parentDependency.addDependency(new Dependency(projectRef));
}
}
}
}

private List<ProjectDependencyAnalysis> prepareMavenDependencyAnalysis() throws MojoExecutionException {
final List<ProjectDependencyAnalysis> dependencyAnalysisMap = new ArrayList<>();
for (final MavenProject mavenProject : reactorProjects) {
if (shouldExclude(mavenProject)) {
continue;
}
ProjectDependencyAnalysis dependencyAnalysis = doProjectDependencyAnalysis(mavenProject);
if (dependencyAnalysis != null) {
dependencyAnalysisMap.add(dependencyAnalysis);
}
}
return dependencyAnalysisMap;
}

private Component.Scope inferComponentScope(Artifact artifact, List<ProjectDependencyAnalysis> projectsDependencyAnalysis) {
Component.Scope componentScope = null;
for (ProjectDependencyAnalysis dependencyAnalysis : projectsDependencyAnalysis) {
Component.Scope currentProjectScope = inferComponentScope(artifact, dependencyAnalysis);

// Set scope to required if the component is used in any project
if (Component.Scope.REQUIRED.equals(currentProjectScope)) {
return Component.Scope.REQUIRED;
}

if (componentScope == null && currentProjectScope != null) {
// Set optional or excluded scope
componentScope = currentProjectScope;
}
}
return componentScope;
}
}
62 changes: 10 additions & 52 deletions src/main/java/org/cyclonedx/maven/CycloneDxMojo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
*/
package org.cyclonedx.maven;

import org.apache.maven.artifact.Artifact;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.maven.plugins.annotations.LifecyclePhase;
import org.apache.maven.plugins.annotations.Mojo;
Expand All @@ -32,7 +31,8 @@
import org.codehaus.plexus.component.repository.exception.ComponentLookupException;
import org.cyclonedx.model.Component;
import org.cyclonedx.model.Dependency;
import java.util.LinkedHashSet;

import java.util.Map;
import java.util.Set;

/**
Expand Down Expand Up @@ -90,61 +90,19 @@ protected ProjectDependencyAnalysis doProjectDependencyAnalysis(MavenProject mav
return null;
}

protected String extractComponentsAndDependencies(final Set<Component> components, final Set<Dependency> dependencies) throws MojoExecutionException {
final Set<String> componentRefs = new LinkedHashSet<>();

protected String extractComponentsAndDependencies(final Set<String> topLevelComponents, final Map<String, Component> components, final Map<String, Dependency> dependencies) throws MojoExecutionException {
getLog().info(MESSAGE_RESOLVING_DEPS);
if (getProject() != null && getProject().getArtifacts() != null) {
ProjectDependencyAnalysis dependencyAnalysis = doProjectDependencyAnalysis(getProject());

// Add reference to BOM metadata component.
// Without this, direct dependencies of the Maven project cannot be determined.
final Component bomComponent = convert(getProject().getArtifact());
componentRefs.add(bomComponent.getBomRef());
final Map<String, Dependency> projectDependencies = extractBOMDependencies(getProject());

for (final Artifact artifact : getProject().getArtifacts()) {
final Component component = convert(artifact);
// ensure that only one component with the same bom-ref exists in the BOM
if (componentRefs.add(component.getBomRef())) {
component.setScope(inferComponentScope(artifact, dependencyAnalysis));
components.add(component);
}
}
}
final Component projectBomComponent = convert(getProject().getArtifact());
components.put(projectBomComponent.getPurl(), projectBomComponent);
topLevelComponents.add(projectBomComponent.getPurl());

dependencies.addAll(extractBOMDependencies(getProject()));
populateComponents(topLevelComponents, components, getProject().getArtifacts(), doProjectDependencyAnalysis(getProject()));

return "makeBom";
}
projectDependencies.forEach(dependencies::putIfAbsent);

/**
* Infer BOM component scope based on Maven project dependency analysis.
*
* @param artifact Artifact from maven project
* @param projectDependencyAnalysis Maven Project Dependency Analysis data
*
* @return Component.Scope - Required: If the component is used (as detected by project dependency analysis). Optional: If it is unused
*/
protected Component.Scope inferComponentScope(Artifact artifact, ProjectDependencyAnalysis projectDependencyAnalysis) {
if (projectDependencyAnalysis == null) {
return null;
}

Set<Artifact> usedDeclaredArtifacts = projectDependencyAnalysis.getUsedDeclaredArtifacts();
Set<Artifact> usedUndeclaredArtifacts = projectDependencyAnalysis.getUsedUndeclaredArtifacts();
Set<Artifact> unusedDeclaredArtifacts = projectDependencyAnalysis.getUnusedDeclaredArtifacts();
Set<Artifact> testArtifactsWithNonTestScope = projectDependencyAnalysis.getTestArtifactsWithNonTestScope();

// Is the artifact used?
if (usedDeclaredArtifacts.contains(artifact) || usedUndeclaredArtifacts.contains(artifact)) {
return Component.Scope.REQUIRED;
}

// Is the artifact unused or test?
if (unusedDeclaredArtifacts.contains(artifact) || testArtifactsWithNonTestScope.contains(artifact)) {
return Component.Scope.OPTIONAL;
}

return null;
return "makeBom";
}
}
Loading

0 comments on commit bbcf2ff

Please sign in to comment.