diff --git a/src/main/java/org/dependencytrack/model/Component.java b/src/main/java/org/dependencytrack/model/Component.java index c7ad08262..2a988cbf1 100644 --- a/src/main/java/org/dependencytrack/model/Component.java +++ b/src/main/java/org/dependencytrack/model/Component.java @@ -79,6 +79,9 @@ @Persistent(name = "properties"), @Persistent(name = "vulnerabilities"), }), + @FetchGroup(name = "BOM_UPLOAD_PROCESSING", members = { + @Persistent(name = "properties") + }), @FetchGroup(name = "INTERNAL_IDENTIFICATION", members = { @Persistent(name = "id"), @Persistent(name = "group"), @@ -107,6 +110,7 @@ public class Component implements Serializable { */ public enum FetchGroup { ALL, + BOM_UPLOAD_PROCESSING, INTERNAL_IDENTIFICATION, METRICS_UPDATE, REPO_META_ANALYSIS diff --git a/src/main/java/org/dependencytrack/model/ComponentIdentity.java b/src/main/java/org/dependencytrack/model/ComponentIdentity.java index 242b1ae72..8015c7a83 100644 --- a/src/main/java/org/dependencytrack/model/ComponentIdentity.java +++ b/src/main/java/org/dependencytrack/model/ComponentIdentity.java @@ -72,6 +72,13 @@ public ComponentIdentity(final Component component) { this.objectType = ObjectType.COMPONENT; } + public ComponentIdentity(final Component component, final boolean excludeUuid) { + this(component); + if (excludeUuid) { + this.uuid = null; + } + } + public ComponentIdentity(final org.cyclonedx.model.Component component) { try { this.purl = new PackageURL(component.getPurl()); @@ -95,6 +102,13 @@ public ComponentIdentity(final ServiceComponent service) { this.objectType = ObjectType.SERVICE; } + public ComponentIdentity(final ServiceComponent service, final boolean excludeUuid) { + this(service); + if (excludeUuid) { + this.uuid = null; + } + } + public ComponentIdentity(final org.cyclonedx.model.Service service) { this.group = service.getGroup(); this.name = service.getName(); diff --git a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java index 353516b2c..fa700e321 100644 --- a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java @@ -460,55 +460,6 @@ public void recursivelyDelete(Component component, boolean commitIndex) { } - /** - * Returns a component by matching its identity information. - *

- * Note that this method employs a stricter matching logic than {@link #matchIdentity(ComponentIdentity)}. - * For example, if {@code purl} of the given {@link ComponentIdentity} is {@code null}, - * this method will use a query that explicitly checks for the {@code purl} column to be {@code null}. - * Whereas other methods will simply not include {@code purl} in the query in such cases. - * - * @param project the Project the component is a dependency of - * @param cid the identity values of the component - * @return a Component object, or null if not found - * @since 4.11.0 - */ - public Component matchSingleIdentityExact(final Project project, final ComponentIdentity cid) { - final Pair> queryFilterParamsPair = buildExactComponentIdentityQuery(project, cid); - final Query query = pm.newQuery(Component.class, queryFilterParamsPair.getKey()); - query.setNamedParameters(queryFilterParamsPair.getRight()); - try { - return query.executeUnique(); - } finally { - query.closeAll(); - } - } - - /** - * Returns the first component matching a given {@link ComponentIdentity} in a {@link Project}. - * - * @param project the Project the component is a dependency of - * @param cid the identity values of the component - * @return a Component object, or null if not found - * @since 4.11.0 - */ - public Component matchFirstIdentityExact(final Project project, final ComponentIdentity cid) { - final Pair> queryFilterParamsPair = buildExactComponentIdentityQuery(project, cid); - final Query query = pm.newQuery(Component.class, queryFilterParamsPair.getKey()); - query.setNamedParameters(queryFilterParamsPair.getRight()); - query.setRange(0, 1); - try { - final List result = query.executeList(); - if (result.isEmpty()) { - return null; - } - - return result.get(0); - } finally { - query.closeAll(); - } - } - /** * Returns a list of components by matching its identity information. * @param project the Project the component is a dependency of @@ -597,87 +548,6 @@ private static Pair> buildComponentIdentityQuery(fin return Pair.of(filter, params); } - private static Pair> buildExactComponentIdentityQuery(final Project project, final ComponentIdentity cid) { - var filterParts = new ArrayList(); - final var params = new HashMap(); - - if (cid.getPurl() != null) { - filterParts.add("(purl != null && purl == :purl)"); - params.put("purl", cid.getPurl().canonicalize()); - } else { - filterParts.add("purl == null"); - } - - if (cid.getCpe() != null) { - filterParts.add("(cpe != null && cpe == :cpe)"); - params.put("cpe", cid.getCpe()); - } else { - filterParts.add("cpe == null"); - } - - if (cid.getSwidTagId() != null) { - filterParts.add("(swidTagId != null && swidTagId == :swidTagId)"); - params.put("swidTagId", cid.getSwidTagId()); - } else { - filterParts.add("swidTagId == null"); - } - - var coordinatesFilter = "("; - if (cid.getGroup() != null) { - coordinatesFilter += "group == :group"; - params.put("group", cid.getGroup()); - } else { - coordinatesFilter += "group == null"; - } - coordinatesFilter += " && name == :name"; - params.put("name", cid.getName()); - if (cid.getVersion() != null) { - coordinatesFilter += " && version == :version"; - params.put("version", cid.getVersion()); - } else { - coordinatesFilter += " && version == null"; - } - coordinatesFilter += ")"; - filterParts.add(coordinatesFilter); - - final var filter = "project == :project && (" + String.join(" && ", filterParts) + ")"; - params.put("project", project); - - return Pair.of(filter, params); - } - - /** - * Intelligently adds dependencies for components that are not already a dependency - * of the specified project and removes the dependency relationship for components - * that are not in the list of specified components. - * @param project the project to bind components to - * @param existingProjectComponents the complete list of existing dependent components - * @param components the complete list of components that should be dependencies of the project - */ - public void reconcileComponents(Project project, List existingProjectComponents, List components) { - // Removes components as dependencies to the project for all - // components not included in the list provided - List markedForDeletion = new ArrayList<>(); - for (final Component existingComponent: existingProjectComponents) { - boolean keep = false; - for (final Component component: components) { - if (component.getId() == existingComponent.getId()) { - keep = true; - break; - } - } - if (!keep) { - markedForDeletion.add(existingComponent); - } - } - if (!markedForDeletion.isEmpty()) { - for (Component c: markedForDeletion) { - this.recursivelyDelete(c, false); - } - //this.delete(markedForDeletion); - } - } - public Map getDependencyGraphForComponents(Project project, List components) { Map dependencyGraph = new HashMap<>(); if (project.getDirectDependencies() == null || project.getDirectDependencies().isBlank()) { @@ -857,7 +727,7 @@ public void synchronizeComponentProperties(final Component component, final List // counter-intuitive to some users, who might expect their manual changes to persist. // If we want to support that, we need a way to track which properties were added and / or // modified manually. - if (component.getProperties() != null) { + if (component.getProperties() != null && !component.getProperties().isEmpty()) { pm.deletePersistentAll(component.getProperties()); } diff --git a/src/main/java/org/dependencytrack/persistence/QueryManager.java b/src/main/java/org/dependencytrack/persistence/QueryManager.java index 56b3cc312..fb78e9cb0 100644 --- a/src/main/java/org/dependencytrack/persistence/QueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/QueryManager.java @@ -959,14 +959,6 @@ public List getAllVulnerableSoftware(final String cpePart, f return getVulnerableSoftwareQueryManager().getAllVulnerableSoftware(cpePart, cpeVendor, cpeProduct, purl); } - public Component matchSingleIdentityExact(final Project project, final ComponentIdentity cid) { - return getComponentQueryManager().matchSingleIdentityExact(project, cid); - } - - public Component matchFirstIdentityExact(final Project project, final ComponentIdentity cid) { - return getComponentQueryManager().matchFirstIdentityExact(project, cid); - } - public List matchIdentity(final Project project, final ComponentIdentity cid) { return getComponentQueryManager().matchIdentity(project, cid); } @@ -975,10 +967,6 @@ public List matchIdentity(final ComponentIdentity cid) { return getComponentQueryManager().matchIdentity(cid); } - public void reconcileComponents(Project project, List existingProjectComponents, List components) { - getComponentQueryManager().reconcileComponents(project, existingProjectComponents, components); - } - public List getAllComponents(Project project) { return getComponentQueryManager().getAllComponents(project); } @@ -999,10 +987,6 @@ public ServiceComponent matchServiceIdentity(final Project project, final Compon return getServiceComponentQueryManager().matchServiceIdentity(project, cid); } - public void reconcileServiceComponents(Project project, List existingProjectServices, List services) { - getServiceComponentQueryManager().reconcileServiceComponents(project, existingProjectServices, services); - } - public ServiceComponent createServiceComponent(ServiceComponent service, boolean commitIndex) { return getServiceComponentQueryManager().createServiceComponent(service, commitIndex); } diff --git a/src/main/java/org/dependencytrack/persistence/ServiceComponentQueryManager.java b/src/main/java/org/dependencytrack/persistence/ServiceComponentQueryManager.java index da78ff80d..68a3464ca 100644 --- a/src/main/java/org/dependencytrack/persistence/ServiceComponentQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/ServiceComponentQueryManager.java @@ -30,7 +30,6 @@ import javax.jdo.FetchPlan; import javax.jdo.PersistenceManager; import javax.jdo.Query; -import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -65,38 +64,6 @@ public ServiceComponent matchServiceIdentity(final Project project, final Compon return singleResult(query.executeWithArray(project, cid.getGroup(), cid.getName(), cid.getVersion())); } - /** - * Intelligently adds service components that are not already a dependency - * of the specified project and removes the dependency relationship for service components - * that are not in the list of specified components. - * @param project the project to bind components to - * @param existingProjectServices the complete list of existing dependent service components - * @param services the complete list of service components that should be dependencies of the project - */ - public void reconcileServiceComponents(Project project, List existingProjectServices, List services) { - // Removes components as dependencies to the project for all - // components not included in the list provided - List markedForDeletion = new ArrayList<>(); - for (final ServiceComponent existingService: existingProjectServices) { - boolean keep = false; - for (final ServiceComponent service: services) { - if (service.getId() == existingService.getId()) { - keep = true; - break; - } - } - if (!keep) { - markedForDeletion.add(existingService); - } - } - if (!markedForDeletion.isEmpty()) { - for (ServiceComponent sc: markedForDeletion) { - this.recursivelyDelete(sc, false); - } - //this.delete(markedForDeletion); - } - } - /** * Creates a new ServiceComponent. * @param service the ServiceComponent to persist diff --git a/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java b/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java index ec1d3dc41..82e399324 100644 --- a/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java +++ b/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java @@ -25,12 +25,10 @@ import alpine.notification.NotificationLevel; import org.apache.commons.collections4.MultiValuedMap; import org.apache.commons.collections4.multimap.HashSetValuedHashMap; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.cyclonedx.exception.ParseException; import org.cyclonedx.parsers.BomParserFactory; import org.cyclonedx.parsers.Parser; import org.datanucleus.flush.FlushMode; -import org.datanucleus.store.query.QueryNotUniqueException; import org.dependencytrack.event.BomUploadEvent; import org.dependencytrack.event.NewVulnerableDependencyAnalysisEvent; import org.dependencytrack.event.PolicyEvaluationEvent; @@ -61,7 +59,6 @@ import org.json.JSONArray; import org.slf4j.MDC; -import javax.jdo.JDOUserException; import javax.jdo.PersistenceManager; import javax.jdo.Query; import java.util.ArrayList; @@ -77,8 +74,11 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Collectors; +import static javax.jdo.FetchPlan.FETCH_SIZE_GREEDY; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.trim; import static org.apache.commons.lang3.StringUtils.trimToNull; @@ -368,10 +368,6 @@ private Map processComponents( ) { assertPersistent(project, "Project mut be persistent"); - // Fetch IDs of all components that exist in the project already. - // We'll need them later to determine which components to delete. - final Set idsOfComponentsToDelete = getAllComponentIds(qm, project, Component.class); - // Avoid redundant queries by caching resolved licenses. // It is likely that if license IDs were present in a BOM, // they appear multiple times for different components. @@ -382,26 +378,34 @@ private Map processComponents( final var customLicenseCache = new HashMap(); final var internalComponentIdentifier = new InternalComponentIdentifier(); - final var persistentComponents = new HashMap(); + + final List persistentComponents = getAllComponents(qm, project); + + // Group existing components by their identity for easier lookup. + // Note that we exclude the UUID from the identity here, + // since incoming non-persistent components won't have one yet. + final Map persistentComponentByIdentity = persistentComponents.stream() + .collect(Collectors.toMap( + component -> new ComponentIdentity(component, /* excludeUuid */ true), + Function.identity(), + (previous, duplicate) -> { + LOGGER.warn(""" + More than one existing component matches the identity %s; \ + Proceeding with first match, others will be deleted\ + """.formatted(new ComponentIdentity(previous, /* excludeUuid */ true))); + return previous; + })); + + final Set idsOfComponentsToDelete = persistentComponents.stream() + .map(Component::getId) + .collect(Collectors.toSet()); + for (final Component component : components) { component.setInternal(internalComponentIdentifier.isInternal(component)); resolveAndApplyLicense(qm, component, licenseCache, customLicenseCache); final var componentIdentity = new ComponentIdentity(component); - Component persistentComponent; - try { - persistentComponent = qm.matchSingleIdentityExact(project, componentIdentity); - } catch (JDOUserException e) { - if (!(ExceptionUtils.getRootCause(e) instanceof QueryNotUniqueException)) { - throw e; - } - - LOGGER.warn(""" - More than one existing component match the identity %s; \ - Proceeding with first match, others will be deleted\ - """.formatted(componentIdentity.toJSON())); - persistentComponent = qm.matchFirstIdentityExact(project, componentIdentity); - } + Component persistentComponent = persistentComponentByIdentity.get(componentIdentity); if (persistentComponent == null) { component.setProject(project); persistentComponent = qm.persist(component); @@ -450,9 +454,21 @@ private Map processComponents( identitiesByBomRef.put(bomRef, newIdentity); } - persistentComponents.put(newIdentity, persistentComponent); + persistentComponentByIdentity.put(newIdentity, persistentComponent); } + persistentComponentByIdentity.entrySet().removeIf(entry -> { + // Remove entries for identities without UUID, those were only needed for matching. + final ComponentIdentity identity = entry.getKey(); + if (identity.getUuid() == null) { + return true; + } + + // Remove entries for components marked for deletion. + final Component component = entry.getValue(); + return idsOfComponentsToDelete.contains(component.getId()); + }); + qm.getPersistenceManager().flush(); final long componentsDeleted = deleteComponentsById(qm, idsOfComponentsToDelete); @@ -460,7 +476,7 @@ private Map processComponents( qm.getPersistenceManager().flush(); } - return persistentComponents; + return persistentComponentByIdentity; } private Map processServices( @@ -472,17 +488,30 @@ private Map processServices( ) { assertPersistent(project, "Project must be persistent"); - final PersistenceManager pm = qm.getPersistenceManager(); - - // Fetch IDs of all services that exist in the project already. - // We'll need them later to determine which services to delete. - final Set idsOfServicesToDelete = getAllComponentIds(qm, project, ServiceComponent.class); - - final var persistentServices = new HashMap(); + final List persistentServices = getAllServices(qm, project); + + // Group existing services by their identity for easier lookup. + // Note that we exclude the UUID from the identity here, + // since incoming non-persistent services won't have one yet. + final Map persistentServiceByIdentity = persistentServices.stream() + .collect(Collectors.toMap( + service -> new ComponentIdentity(service, /* excludeUuid */ true), + Function.identity(), + (previous, duplicate) -> { + LOGGER.warn(""" + More than one existing service matches the identity %s; \ + Proceeding with first match, others will be deleted\ + """.formatted(new ComponentIdentity(previous, /* excludeUuid */ true))); + return previous; + })); + + final Set idsOfServicesToDelete = persistentServices.stream() + .map(ServiceComponent::getId) + .collect(Collectors.toSet()); for (final ServiceComponent service : services) { final var componentIdentity = new ComponentIdentity(service); - ServiceComponent persistentService = qm.matchServiceIdentity(project, componentIdentity); + ServiceComponent persistentService = persistentServiceByIdentity.get(componentIdentity); if (persistentService == null) { service.setProject(project); persistentService = qm.persist(service); @@ -509,9 +538,21 @@ private Map processServices( identitiesByBomRef.put(bomRef, newIdentity); } - persistentServices.put(newIdentity, persistentService); + persistentServiceByIdentity.put(newIdentity, persistentService); } + persistentServiceByIdentity.entrySet().removeIf(entry -> { + // Remove entries for identities without UUID, those were only needed for matching. + final ComponentIdentity identity = entry.getKey(); + if (identity.getUuid() == null) { + return true; + } + + // Remove entries for services marked for deletion. + final ServiceComponent service = entry.getValue(); + return idsOfServicesToDelete.contains(service.getId()); + }); + qm.getPersistenceManager().flush(); final long servicesDeleted = deleteServicesById(qm, idsOfServicesToDelete); @@ -520,7 +561,7 @@ private Map processServices( } - return persistentServices; + return persistentServiceByIdentity; } private void recordBomImport(final Context ctx, final QueryManager qm, final Project project) { @@ -728,14 +769,28 @@ private static void resolveAndApplyLicense( } } - private static Set getAllComponentIds(final QueryManager qm, final Project project, final Class clazz) { - final Query query = qm.getPersistenceManager().newQuery(clazz); - query.setFilter("project == :project"); - query.setParameters(project); - query.setResult("id"); + private static List getAllComponents(final QueryManager qm, final Project project) { + final Query query = qm.getPersistenceManager().newQuery(Component.class); + query.getFetchPlan().addGroup(Component.FetchGroup.BOM_UPLOAD_PROCESSING.name()); + query.getFetchPlan().setFetchSize(FETCH_SIZE_GREEDY); + query.setFilter("project.id == :projectId"); + query.setParameters(project.getId()); + + try { + return List.copyOf(query.executeList()); + } finally { + query.closeAll(); + } + } + + private static List getAllServices(final QueryManager qm, final Project project) { + final Query query = qm.getPersistenceManager().newQuery(ServiceComponent.class); + query.getFetchPlan().setFetchSize(FETCH_SIZE_GREEDY); + query.setFilter("project.id == :projectId"); + query.setParameters(project.getId()); try { - return new HashSet<>(query.executeResultList(Long.class)); + return List.copyOf(query.executeList()); } finally { query.closeAll(); }