Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce database round-trips during BOM processing #4486

Merged
merged 1 commit into from
Dec 23, 2024
Merged

Reduce database round-trips during BOM processing #4486

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/main/java/org/dependencytrack/model/Component.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,9 @@
@Persistent(name = "properties"),
@Persistent(name = "vulnerabilities"),
}),
@FetchGroup(name = "BOM_UPLOAD_PROCESSING", members = {
@Persistent(name = "properties")
}),
@FetchGroup(name = "INTERNAL_IDENTIFICATION", members = {
@Persistent(name = "id"),
@Persistent(name = "group"),
Expand Down Expand Up @@ -107,6 +110,7 @@ public class Component implements Serializable {
*/
public enum FetchGroup {
ALL,
BOM_UPLOAD_PROCESSING,
INTERNAL_IDENTIFICATION,
METRICS_UPDATE,
REPO_META_ANALYSIS
Expand Down
14 changes: 14 additions & 0 deletions src/main/java/org/dependencytrack/model/ComponentIdentity.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,13 @@ public ComponentIdentity(final Component component) {
this.objectType = ObjectType.COMPONENT;
}

public ComponentIdentity(final Component component, final boolean excludeUuid) {
this(component);
if (excludeUuid) {
this.uuid = null;
}
}

public ComponentIdentity(final org.cyclonedx.model.Component component) {
try {
this.purl = new PackageURL(component.getPurl());
Expand All @@ -95,6 +102,13 @@ public ComponentIdentity(final ServiceComponent service) {
this.objectType = ObjectType.SERVICE;
}

public ComponentIdentity(final ServiceComponent service, final boolean excludeUuid) {
this(service);
if (excludeUuid) {
this.uuid = null;
}
}

public ComponentIdentity(final org.cyclonedx.model.Service service) {
this.group = service.getGroup();
this.name = service.getName();
Expand Down
132 changes: 1 addition & 131 deletions src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,55 +460,6 @@ public void recursivelyDelete(Component component, boolean commitIndex) {

}

/**
* Returns a component by matching its identity information.
* <p>
* Note that this method employs a stricter matching logic than {@link #matchIdentity(ComponentIdentity)}.
* For example, if {@code purl} of the given {@link ComponentIdentity} is {@code null},
* this method will use a query that explicitly checks for the {@code purl} column to be {@code null}.
* Whereas other methods will simply not include {@code purl} in the query in such cases.
*
* @param project the Project the component is a dependency of
* @param cid the identity values of the component
* @return a Component object, or null if not found
* @since 4.11.0
*/
public Component matchSingleIdentityExact(final Project project, final ComponentIdentity cid) {
final Pair<String, Map<String, Object>> queryFilterParamsPair = buildExactComponentIdentityQuery(project, cid);
final Query<Component> query = pm.newQuery(Component.class, queryFilterParamsPair.getKey());
query.setNamedParameters(queryFilterParamsPair.getRight());
try {
return query.executeUnique();
} finally {
query.closeAll();
}
}

/**
* Returns the first component matching a given {@link ComponentIdentity} in a {@link Project}.
*
* @param project the Project the component is a dependency of
* @param cid the identity values of the component
* @return a Component object, or null if not found
* @since 4.11.0
*/
public Component matchFirstIdentityExact(final Project project, final ComponentIdentity cid) {
final Pair<String, Map<String, Object>> queryFilterParamsPair = buildExactComponentIdentityQuery(project, cid);
final Query<Component> query = pm.newQuery(Component.class, queryFilterParamsPair.getKey());
query.setNamedParameters(queryFilterParamsPair.getRight());
query.setRange(0, 1);
try {
final List<Component> result = query.executeList();
if (result.isEmpty()) {
return null;
}

return result.get(0);
} finally {
query.closeAll();
}
}

/**
* Returns a list of components by matching its identity information.
* @param project the Project the component is a dependency of
Expand Down Expand Up @@ -597,87 +548,6 @@ private static Pair<String, Map<String, Object>> buildComponentIdentityQuery(fin
return Pair.of(filter, params);
}

private static Pair<String, Map<String, Object>> buildExactComponentIdentityQuery(final Project project, final ComponentIdentity cid) {
var filterParts = new ArrayList<String>();
final var params = new HashMap<String, Object>();

if (cid.getPurl() != null) {
filterParts.add("(purl != null && purl == :purl)");
params.put("purl", cid.getPurl().canonicalize());
} else {
filterParts.add("purl == null");
}

if (cid.getCpe() != null) {
filterParts.add("(cpe != null && cpe == :cpe)");
params.put("cpe", cid.getCpe());
} else {
filterParts.add("cpe == null");
}

if (cid.getSwidTagId() != null) {
filterParts.add("(swidTagId != null && swidTagId == :swidTagId)");
params.put("swidTagId", cid.getSwidTagId());
} else {
filterParts.add("swidTagId == null");
}

var coordinatesFilter = "(";
if (cid.getGroup() != null) {
coordinatesFilter += "group == :group";
params.put("group", cid.getGroup());
} else {
coordinatesFilter += "group == null";
}
coordinatesFilter += " && name == :name";
params.put("name", cid.getName());
if (cid.getVersion() != null) {
coordinatesFilter += " && version == :version";
params.put("version", cid.getVersion());
} else {
coordinatesFilter += " && version == null";
}
coordinatesFilter += ")";
filterParts.add(coordinatesFilter);

final var filter = "project == :project && (" + String.join(" && ", filterParts) + ")";
params.put("project", project);

return Pair.of(filter, params);
}

/**
* Intelligently adds dependencies for components that are not already a dependency
* of the specified project and removes the dependency relationship for components
* that are not in the list of specified components.
* @param project the project to bind components to
* @param existingProjectComponents the complete list of existing dependent components
* @param components the complete list of components that should be dependencies of the project
*/
public void reconcileComponents(Project project, List<Component> existingProjectComponents, List<Component> components) {
// Removes components as dependencies to the project for all
// components not included in the list provided
List<Component> markedForDeletion = new ArrayList<>();
for (final Component existingComponent: existingProjectComponents) {
boolean keep = false;
for (final Component component: components) {
if (component.getId() == existingComponent.getId()) {
keep = true;
break;
}
}
if (!keep) {
markedForDeletion.add(existingComponent);
}
}
if (!markedForDeletion.isEmpty()) {
for (Component c: markedForDeletion) {
this.recursivelyDelete(c, false);
}
//this.delete(markedForDeletion);
}
}

public Map<String, Component> getDependencyGraphForComponents(Project project, List<Component> components) {
Map<String, Component> dependencyGraph = new HashMap<>();
if (project.getDirectDependencies() == null || project.getDirectDependencies().isBlank()) {
Expand Down Expand Up @@ -857,7 +727,7 @@ public void synchronizeComponentProperties(final Component component, final List
// counter-intuitive to some users, who might expect their manual changes to persist.
// If we want to support that, we need a way to track which properties were added and / or
// modified manually.
if (component.getProperties() != null) {
if (component.getProperties() != null && !component.getProperties().isEmpty()) {
pm.deletePersistentAll(component.getProperties());
}

Expand Down
16 changes: 0 additions & 16 deletions src/main/java/org/dependencytrack/persistence/QueryManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -959,14 +959,6 @@ public List<VulnerableSoftware> getAllVulnerableSoftware(final String cpePart, f
return getVulnerableSoftwareQueryManager().getAllVulnerableSoftware(cpePart, cpeVendor, cpeProduct, purl);
}

public Component matchSingleIdentityExact(final Project project, final ComponentIdentity cid) {
return getComponentQueryManager().matchSingleIdentityExact(project, cid);
}

public Component matchFirstIdentityExact(final Project project, final ComponentIdentity cid) {
return getComponentQueryManager().matchFirstIdentityExact(project, cid);
}

public List<Component> matchIdentity(final Project project, final ComponentIdentity cid) {
return getComponentQueryManager().matchIdentity(project, cid);
}
Expand All @@ -975,10 +967,6 @@ public List<Component> matchIdentity(final ComponentIdentity cid) {
return getComponentQueryManager().matchIdentity(cid);
}

public void reconcileComponents(Project project, List<Component> existingProjectComponents, List<Component> components) {
getComponentQueryManager().reconcileComponents(project, existingProjectComponents, components);
}

public List<Component> getAllComponents(Project project) {
return getComponentQueryManager().getAllComponents(project);
}
Expand All @@ -999,10 +987,6 @@ public ServiceComponent matchServiceIdentity(final Project project, final Compon
return getServiceComponentQueryManager().matchServiceIdentity(project, cid);
}

public void reconcileServiceComponents(Project project, List<ServiceComponent> existingProjectServices, List<ServiceComponent> services) {
getServiceComponentQueryManager().reconcileServiceComponents(project, existingProjectServices, services);
}

public ServiceComponent createServiceComponent(ServiceComponent service, boolean commitIndex) {
return getServiceComponentQueryManager().createServiceComponent(service, commitIndex);
}
Expand Down
33 changes: 0 additions & 33 deletions src/main/java/org/dependencytrack/persistence/ServiceComponentQueryManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import javax.jdo.FetchPlan;
import javax.jdo.PersistenceManager;
import javax.jdo.Query;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

Expand Down Expand Up @@ -65,38 +64,6 @@ public ServiceComponent matchServiceIdentity(final Project project, final Compon
return singleResult(query.executeWithArray(project, cid.getGroup(), cid.getName(), cid.getVersion()));
}

/**
* Intelligently adds service components that are not already a dependency
* of the specified project and removes the dependency relationship for service components
* that are not in the list of specified components.
* @param project the project to bind components to
* @param existingProjectServices the complete list of existing dependent service components
* @param services the complete list of service components that should be dependencies of the project
*/
public void reconcileServiceComponents(Project project, List<ServiceComponent> existingProjectServices, List<ServiceComponent> services) {
// Removes components as dependencies to the project for all
// components not included in the list provided
List<ServiceComponent> markedForDeletion = new ArrayList<>();
for (final ServiceComponent existingService: existingProjectServices) {
boolean keep = false;
for (final ServiceComponent service: services) {
if (service.getId() == existingService.getId()) {
keep = true;
break;
}
}
if (!keep) {
markedForDeletion.add(existingService);
}
}
if (!markedForDeletion.isEmpty()) {
for (ServiceComponent sc: markedForDeletion) {
this.recursivelyDelete(sc, false);
}
//this.delete(markedForDeletion);
}
}

/**
* Creates a new ServiceComponent.
* @param service the ServiceComponent to persist
Expand Down
Loading
Loading