diff --git a/docs/_posts/2024-xx-xx-v4.11.0.md b/docs/_posts/2024-xx-xx-v4.11.0.md index 47b81f2877..47ac06cfe1 100644 --- a/docs/_posts/2024-xx-xx-v4.11.0.md +++ b/docs/_posts/2024-xx-xx-v4.11.0.md @@ -10,14 +10,12 @@ overhauled to be more reliable and efficient. Further, BOM processing is now an occurring midway do not cause a partial state to be left behind. De-duplication of components and services is more predictable, and log messages emitted during processing contain additional context, making them easier to correlate. Because the new implementation can have a big impact on how Dependency-Track behaves regarding BOM uploads, -it is disabled by default for this release. It may be enabled by setting the environment variable `BOM_PROCESSING_TASK_V2_ENABLED` -to `true`. Users are highly encouraged to do so. +it is disabled by default for this release. It may be enabled in the experimental panel in administration. * **BOM Validation**. Historically, Dependency-Track did not validate uploaded BOMs and VEXs against the CycloneDX schema. While this allowed BOMs to be processed that did not strictly adhere to the schema, it could lead to confusion when uploaded files were accepted, but then failed to be ingested during asynchronous processing. Starting with this release, uploaded files will be rejected if they fail schema validation. Note that this may reveal issues in BOM -generators that currently produce invalid CycloneDX documents. Validation may be turned off by setting the -environment variable `BOM_VALIDATION_ENABLED` to `false`. +generators that currently produce invalid CycloneDX documents. Validation may be turned off by disabling it in the experimental panel in administration. * *This feature was demoed in our April community meeting! Watch it [here](https://www.youtube.com/watch?v=3iIeajRJK8o&t=450s)* * **Global Vulnerability Audit View**. This new interface allows users to discover and filter vulnerabilities that affect their portfolio, across all projects. When portfolio access control is enabled, this view is limited to projects a user diff --git a/src/main/java/org/dependencytrack/common/ConfigKey.java b/src/main/java/org/dependencytrack/common/ConfigKey.java index 6e543bffed..a4e8b4ed08 100644 --- a/src/main/java/org/dependencytrack/common/ConfigKey.java +++ b/src/main/java/org/dependencytrack/common/ConfigKey.java @@ -39,9 +39,7 @@ public enum ConfigKey implements Config.Key { REPO_META_ANALYZER_CACHE_STAMPEDE_BLOCKER_ENABLED("repo.meta.analyzer.cacheStampedeBlocker.enabled", true), REPO_META_ANALYZER_CACHE_STAMPEDE_BLOCKER_LOCK_BUCKETS("repo.meta.analyzer.cacheStampedeBlocker.lock.buckets", 1000), REPO_META_ANALYZER_CACHE_STAMPEDE_BLOCKER_MAX_ATTEMPTS("repo.meta.analyzer.cacheStampedeBlocker.max.attempts", 10), - SYSTEM_REQUIREMENT_CHECK_ENABLED("system.requirement.check.enabled", true), - BOM_PROCESSING_TASK_V2_ENABLED("bom.processing.task.v2.enabled", false), - BOM_VALIDATION_ENABLED("bom.validation.enabled", true); + SYSTEM_REQUIREMENT_CHECK_ENABLED("system.requirement.check.enabled", true); private final String propertyName; private final Object defaultValue; diff --git a/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java b/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java index f53471ef58..d0f7ccc91b 100644 --- a/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java +++ b/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java @@ -18,15 +18,12 @@ */ package org.dependencytrack.event; -import alpine.Config; import alpine.common.logging.Logger; import alpine.event.LdapSyncEvent; import alpine.event.framework.EventService; import alpine.event.framework.SingleThreadedEventService; import alpine.server.tasks.LdapSyncTask; import org.dependencytrack.RequirementsVerifier; -import org.dependencytrack.common.ConfigKey; -import org.dependencytrack.tasks.BomUploadProcessingTask; import org.dependencytrack.tasks.BomUploadProcessingTaskV2; import org.dependencytrack.tasks.CallbackTask; import org.dependencytrack.tasks.ClearComponentAnalysisCacheTask; @@ -87,11 +84,8 @@ public void contextInitialized(final ServletContextEvent event) { if (RequirementsVerifier.failedValidation()) { return; } - if (Config.getInstance().getPropertyAsBoolean(ConfigKey.BOM_PROCESSING_TASK_V2_ENABLED)) { - EVENT_SERVICE.subscribe(BomUploadEvent.class, BomUploadProcessingTaskV2.class); - } else { - EVENT_SERVICE.subscribe(BomUploadEvent.class, BomUploadProcessingTask.class); - } + + EVENT_SERVICE.subscribe(BomUploadEvent.class, BomUploadProcessingTaskV2.class); EVENT_SERVICE.subscribe(VexUploadEvent.class, VexUploadProcessingTask.class); EVENT_SERVICE.subscribe(LdapSyncEvent.class, LdapSyncTask.class); EVENT_SERVICE.subscribe(InternalAnalysisEvent.class, InternalAnalysisTask.class); @@ -135,11 +129,8 @@ public void contextDestroyed(final ServletContextEvent event) { LOGGER.info("Shutting down asynchronous event subsystem"); TaskScheduler.getInstance().shutdown(); - if (Config.getInstance().getPropertyAsBoolean(ConfigKey.BOM_PROCESSING_TASK_V2_ENABLED)) { - EVENT_SERVICE.unsubscribe(BomUploadProcessingTaskV2.class); - } else { - EVENT_SERVICE.unsubscribe(BomUploadProcessingTask.class); - } + + EVENT_SERVICE.unsubscribe(BomUploadProcessingTaskV2.class); EVENT_SERVICE.unsubscribe(VexUploadProcessingTask.class); EVENT_SERVICE.unsubscribe(LdapSyncTask.class); EVENT_SERVICE.unsubscribe(InternalAnalysisTask.class); diff --git a/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java b/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java index 5d6cc55848..e1527d1cac 100644 --- a/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java +++ b/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java @@ -108,7 +108,9 @@ public enum ConfigPropertyConstants { TASK_SCHEDULER_COMPONENT_ANALYSIS_CACHE_CLEAR_CADENCE("task-scheduler", "component.analysis.cache.clear.cadence", "24", PropertyType.INTEGER, "Cleanup cadence (in hours) for component analysis cache"), SEARCH_INDEXES_CONSISTENCY_CHECK_ENABLED("search-indexes", "consistency.check.enabled", "true", PropertyType.BOOLEAN, "Flag to enable lucene indexes periodic consistency check"), SEARCH_INDEXES_CONSISTENCY_CHECK_CADENCE("search-indexes", "consistency.check.cadence", "4320", PropertyType.INTEGER, "Lucene indexes consistency check cadence (in minutes)"), - SEARCH_INDEXES_CONSISTENCY_CHECK_DELTA_THRESHOLD("search-indexes", "consistency.check.delta.threshold", "20", PropertyType.INTEGER, "Threshold used to trigger an index rebuild when comparing database table and corresponding lucene index (in percentage). It must be an integer between 1 and 100"); + SEARCH_INDEXES_CONSISTENCY_CHECK_DELTA_THRESHOLD("search-indexes", "consistency.check.delta.threshold", "20", PropertyType.INTEGER, "Threshold used to trigger an index rebuild when comparing database table and corresponding lucene index (in percentage). It must be an integer between 1 and 100"), + BOM_PROCESSING_TASK_V2_ENABLED("experimental", "bom.processing.task.v2.enabled", "false", PropertyType.BOOLEAN, "Flag to enable BOM UPLOAD V2"), + BOM_VALIDATION_ENABLED("artifact", "bom.validation.enabled", "true", PropertyType.BOOLEAN, "Flag to control bom validation"); private String groupName; private String propertyName; diff --git a/src/main/java/org/dependencytrack/resources/v1/BomResource.java b/src/main/java/org/dependencytrack/resources/v1/BomResource.java index 982a7fed05..7cf1febd4d 100644 --- a/src/main/java/org/dependencytrack/resources/v1/BomResource.java +++ b/src/main/java/org/dependencytrack/resources/v1/BomResource.java @@ -18,7 +18,6 @@ */ package org.dependencytrack.resources.v1; -import alpine.Config; import alpine.common.logging.Logger; import alpine.event.framework.Event; import alpine.server.auth.PermissionRequired; @@ -35,9 +34,9 @@ import org.cyclonedx.CycloneDxMediaType; import org.cyclonedx.exception.GeneratorException; import org.dependencytrack.auth.Permissions; -import org.dependencytrack.common.ConfigKey; import org.dependencytrack.event.BomUploadEvent; import org.dependencytrack.model.Component; +import org.dependencytrack.model.ConfigPropertyConstants; import org.dependencytrack.model.Project; import org.dependencytrack.model.validation.ValidUuid; import org.dependencytrack.parser.cyclonedx.CycloneDXExporter; @@ -468,8 +467,10 @@ private Response process(QueryManager qm, Project project, List data() { return Arrays.asList(new Object[][]{ - {BomUploadProcessingTask.class.getSimpleName(), (Supplier) BomUploadProcessingTask::new}, + {BomUploadProcessingTask.class.getSimpleName(), (Supplier) BomUploadProcessingTaskV2::new}, {BomUploadProcessingTaskV2.class.getSimpleName(), (Supplier) BomUploadProcessingTaskV2::new} }); } + private final String ignoredBomUploadProcessingTaskName; private final Supplier bomUploadProcessingTaskSupplier; public BomUploadProcessingTaskTest(final String ignoredBomUploadProcessingTaskName, final Supplier bomUploadProcessingTaskSupplier) { + + this.ignoredBomUploadProcessingTaskName = ignoredBomUploadProcessingTaskName; this.bomUploadProcessingTaskSupplier = bomUploadProcessingTaskSupplier; } @@ -119,6 +124,14 @@ public void setUp() { EventService.getInstance().subscribe(VulnerabilityAnalysisEvent.class, EventSubscriber.class); NotificationService.getInstance().subscribe(new Subscription(NotificationSubscriber.class)); + qm.createConfigProperty( + BOM_PROCESSING_TASK_V2_ENABLED.getGroupName(), + BOM_PROCESSING_TASK_V2_ENABLED.getPropertyName(), + (this.ignoredBomUploadProcessingTaskName == BomUploadProcessingTaskV2.class.getSimpleName()) ? "true" : "false", + BOM_PROCESSING_TASK_V2_ENABLED.getPropertyType(), + null + ); + // Enable processing of CycloneDX BOMs qm.createConfigProperty(ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getGroupName(), ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getPropertyName(), "true", @@ -287,7 +300,7 @@ public void informTest() throws Exception { @Test public void informWithEmptyBomTest() throws Exception { // Known to now work with old task implementation. - if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) { + if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) { return; } @@ -308,6 +321,20 @@ public void informWithEmptyBomTest() throws Exception { @Test public void informWithInvalidCycloneDxBomTest() throws Exception { + + // Known to now work with old task implementation. + if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) { + return; + } + + qm.createConfigProperty( + BOM_VALIDATION_ENABLED.getGroupName(), + BOM_VALIDATION_ENABLED.getPropertyName(), + "true", + BOM_VALIDATION_ENABLED.getPropertyType(), + null + ); + final Project project = qm.createProject("Acme Example", null, "1.0", null, null, null, true, false); final byte[] bomBytes = """ @@ -344,7 +371,7 @@ public void informWithInvalidCycloneDxBomTest() throws Exception { @Test public void informWithNonExistentProjectTest() throws Exception { // Known to now work with old task implementation. - if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) { + if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) { return; } @@ -368,7 +395,7 @@ public void informWithNonExistentProjectTest() throws Exception { @Test public void informWithComponentsUnderMetadataBomTest() throws Exception { // Known to now work with old task implementation. - if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) { + if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) { return; } @@ -404,7 +431,7 @@ public void informWithComponentsUnderMetadataBomTest() throws Exception { @Test public void informWithExistingDuplicateComponentsTest() { // Known to now work with old task implementation. - if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) { + if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) { return; } @@ -490,7 +517,7 @@ public void informWithExistingDuplicateComponentsTest() { @Test public void informWithBloatedBomTest() throws Exception { // Known to now work with old task implementation. - if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) { + if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) { return; } @@ -546,7 +573,7 @@ public void informWithBloatedBomTest() throws Exception { @Test public void informWithCustomLicenseResolutionTest() throws Exception { // Known to now work with old task implementation. - if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) { + if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) { return; } @@ -1021,7 +1048,7 @@ public void informWithExistingComponentPropertiesAndBomWithComponentProperties() @Test // https://github.com/DependencyTrack/dependency-track/issues/1905 public void informIssue1905Test() throws Exception { // Known to now work with old task implementation. - if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) { + if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) { return; } @@ -1060,7 +1087,7 @@ public void informIssue1905Test() throws Exception { @Test // https://github.com/DependencyTrack/dependency-track/issues/2519 public void informIssue2519Test() throws Exception { // Known to now work with old task implementation. - if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) { + if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) { return; } @@ -1138,7 +1165,7 @@ public void informIssue3309Test() { @Test // https://github.com/DependencyTrack/dependency-track/issues/3371 public void informIssue3371Test() throws Exception { // Known to now work with old task implementation. - if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) { + if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) { return; }