diff --git a/fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java b/fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java index f28cddfad7..7738c6efd9 100644 --- a/fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java +++ b/fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java @@ -25,7 +25,7 @@ import com.fortify.cli.common.output.writer.record.RecordWriterConfig; import com.fortify.cli.common.output.writer.record.RecordWriterConfig.RecordWriterConfigBuilder; import com.fortify.cli.common.rest.paging.INextPageUrlProducer; -import com.fortify.cli.common.rest.paging.PagingHelper; +import com.fortify.cli.common.rest.paging.LinkHeaderPagingHelper; import com.fortify.cli.common.rest.unirest.IfFailureHandler; import com.fortify.cli.common.util.PicocliSpecHelper; import com.fortify.cli.common.variable.DefaultVariablePropertyName; @@ -123,7 +123,7 @@ private final void writeRecords(IRecordWriter recordWriter, HttpRequest httpR * @param nextPageUrlProducer */ private final void writeRecords(IRecordWriter recordWriter, HttpRequest httpRequest, INextPageUrlProducer nextPageUrlProducer) { - PagingHelper.pagedRequest(httpRequest, nextPageUrlProducer) + LinkHeaderPagingHelper.pagedRequest(httpRequest, nextPageUrlProducer) .ifSuccess(r->writeRecords(recordWriter, r)) .ifFailure(IfFailureHandler::handle); // Just in case no error interceptor was registered for this request } diff --git a/fcli-common/src/main/java/com/fortify/cli/common/report/collector/IReportResultsCollector.java b/fcli-common/src/main/java/com/fortify/cli/common/report/collector/IReportResultsCollector.java index 3364799053..ac16ad5d7c 100644 --- a/fcli-common/src/main/java/com/fortify/cli/common/report/collector/IReportResultsCollector.java +++ b/fcli-common/src/main/java/com/fortify/cli/common/report/collector/IReportResultsCollector.java @@ -1,9 +1,9 @@ package com.fortify.cli.common.report.collector; -import com.fortify.cli.common.report.writer.entry.IReportErrorEntryWriter; +import com.fortify.cli.common.report.logger.IReportLogger; public interface IReportResultsCollector extends AutoCloseable { - IReportErrorEntryWriter errorWriter(); + IReportLogger logger(); /** * Override default close method to not throw any exception. */ diff --git a/fcli-common/src/main/java/com/fortify/cli/common/report/generator/AbstractReportUnirestResultsGenerator.java b/fcli-common/src/main/java/com/fortify/cli/common/report/generator/AbstractReportUnirestResultsGenerator.java index 050e36a9bd..e99faba0fe 100644 --- a/fcli-common/src/main/java/com/fortify/cli/common/report/generator/AbstractReportUnirestResultsGenerator.java +++ b/fcli-common/src/main/java/com/fortify/cli/common/report/generator/AbstractReportUnirestResultsGenerator.java @@ -73,7 +73,7 @@ private final UnirestInstance createUnirestInstance() { * report output. */ private final void handleSourceError(Exception e) { - resultsCollector().errorWriter().addReportError(String.format("Error processing %s source: %s", getType(), sourceConfig().getUrl()), e); + resultsCollector().logger().error(String.format("Error processing %s source: %s", getType(), sourceConfig().getUrl()), e); } /** diff --git a/fcli-common/src/main/java/com/fortify/cli/common/report/logger/IReportLogger.java b/fcli-common/src/main/java/com/fortify/cli/common/report/logger/IReportLogger.java new file mode 100644 index 0000000000..dd3dcf5c76 --- /dev/null +++ b/fcli-common/src/main/java/com/fortify/cli/common/report/logger/IReportLogger.java @@ -0,0 +1,11 @@ +package com.fortify.cli.common.report.logger; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +public interface IReportLogger { + void warn(String msg, Object... args); + void warn(String msg, Exception e, Object... args); + void error(String msg, Object... args); + void error(String msg, Exception e, Object... args); + void updateSummary(ObjectNode summary); +} \ No newline at end of file diff --git a/fcli-common/src/main/java/com/fortify/cli/common/report/logger/ReportLogger.java b/fcli-common/src/main/java/com/fortify/cli/common/report/logger/ReportLogger.java new file mode 100644 index 0000000000..5ceaeee0da --- /dev/null +++ b/fcli-common/src/main/java/com/fortify/cli/common/report/logger/ReportLogger.java @@ -0,0 +1,93 @@ +package com.fortify.cli.common.report.logger; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.time.LocalDateTime; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.common.json.JsonHelper; +import com.fortify.cli.common.progress.helper.IProgressHelper; +import com.fortify.cli.common.report.writer.IReportWriter; +import com.fortify.cli.common.util.Counter; + +import lombok.SneakyThrows; + +public final class ReportLogger implements IReportLogger { + private final IProgressHelper progressHelper; + private final BufferedWriter logWriter; + private Counter errorCounter = new Counter(); + private Counter warnCounter = new Counter(); + + public ReportLogger(IReportWriter reportWriter, IProgressHelper progressHelper) { + this.logWriter = reportWriter.bufferedWriter("report.log"); + this.progressHelper = progressHelper; + } + + @Override + public void updateSummary(ObjectNode summary) { + summary.set("logCounts", + JsonHelper.getObjectMapper().createObjectNode() + .put("error", errorCounter.getCount()) + .put("warn", warnCounter.getCount()) + ); + } + + @Override + public final void warn(String msg, Object... msgArgs) { + write("WARN", warnCounter, msg, null, msgArgs); + } + + @Override + public final void warn(String msg, Exception e, Object... msgArgs) { + write("WARN", warnCounter, msg, e, msgArgs); + } + + @Override + public final void error(String msg, Object... msgArgs) { + write("ERROR", errorCounter, msg, null, msgArgs); + } + + @Override + public final void error(String msg, Exception e, Object... msgArgs) { + // We want to fail completely in case of IOExceptions as this likely + // means we cannot write the report, so we rethrow as ReportWriterIOException + if ( e instanceof IOException ) { throw new ReportWriterIOException((IOException)e); } + // Rethrow previously thrown ReportWriterIOException + if ( e instanceof ReportWriterIOException ) { throw (ReportWriterIOException)e; } + write("ERROR", errorCounter, msg, e, msgArgs); + } + + @SneakyThrows + private void write(String level, Counter counter, String msg, Exception e, Object[] msgArgs) { + counter.increase(); + var fullMsg = msgArgs==null ? msg : String.format(msg, msgArgs); + if ( e!=null ) { + fullMsg = String.format("%s: %s: %s", fullMsg, e.getClass().getSimpleName(), e.getMessage()); + } + progressHelper.writeWarning(String.format("%s: %s", level, fullMsg)); + logWriter.append(String.format("[%s] %s %s\n", LocalDateTime.now(), level, fullMsg)); + if ( e!=null ) { + logWriter.append(String.format("%s\n", toString(e))); + } + } + + private String toString(Exception e) { + try ( var sw = new StringWriter(); var pw = new PrintWriter(sw) ) { + e.printStackTrace(pw); + return sw.toString(); + } catch ( IOException ioe ) { + return e.getClass().getName()+": "+e.getMessage(); + } + } + + private static final class ReportWriterIOException extends IllegalStateException { + private static final long serialVersionUID = 1L; + + public ReportWriterIOException(IOException cause) { + super("Error writing report; report contents may be incomplete", cause); + } + } + +} diff --git a/fcli-common/src/main/java/com/fortify/cli/common/report/writer/entry/IReportErrorEntryWriter.java b/fcli-common/src/main/java/com/fortify/cli/common/report/writer/entry/IReportErrorEntryWriter.java deleted file mode 100644 index b393799133..0000000000 --- a/fcli-common/src/main/java/com/fortify/cli/common/report/writer/entry/IReportErrorEntryWriter.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.fortify.cli.common.report.writer.entry; - -public interface IReportErrorEntryWriter { - void addReportError(String operation, String message); - void addReportError(String operation, Exception e); - int getErrorCount(); -} \ No newline at end of file diff --git a/fcli-common/src/main/java/com/fortify/cli/common/report/writer/entry/ReportErrorEntryWriter.java b/fcli-common/src/main/java/com/fortify/cli/common/report/writer/entry/ReportErrorEntryWriter.java deleted file mode 100644 index b7f2eb90dc..0000000000 --- a/fcli-common/src/main/java/com/fortify/cli/common/report/writer/entry/ReportErrorEntryWriter.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.fortify.cli.common.report.writer.entry; - -import java.io.IOException; - -import com.fortify.cli.common.json.JsonHelper; -import com.fortify.cli.common.output.OutputFormat; -import com.fortify.cli.common.output.writer.record.IRecordWriter; -import com.fortify.cli.common.progress.helper.IProgressHelper; -import com.fortify.cli.common.report.writer.IReportWriter; - -import lombok.Getter; - -public final class ReportErrorEntryWriter implements IReportErrorEntryWriter { - private IProgressHelper progressHelper; - private IRecordWriter recordWriter; - @Getter private int errorCount = 0; - - public ReportErrorEntryWriter(IReportWriter reportWriter, IProgressHelper progressHelper) { - this.recordWriter = reportWriter.recordWriter(OutputFormat.csv, "report-errors.csv", false, null); - this.progressHelper = progressHelper; - } - - @Override - public final void addReportError(String operation, String message) { - errorCount++; - progressHelper.writeWarning("WARN: %s: %s", operation, message); - recordWriter.writeRecord(JsonHelper.getObjectMapper().createObjectNode() - .put("operation", operation) - .put("message", message)); - } - - @Override - public final void addReportError(String operation, Exception e) { - if ( e instanceof IOException ) { throw new ReportWriterIOException((IOException)e); } - if ( e instanceof ReportWriterIOException ) { throw (ReportWriterIOException)e; } - addReportError(operation, e.getClass().getName()+": "+e.getMessage()); - } - - public static final class ReportWriterIOException extends IllegalStateException { - private static final long serialVersionUID = 1L; - - public ReportWriterIOException(IOException cause) { - super("Error writing report; report contents may be incomplete", cause); - } - } - -} diff --git a/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java b/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/LinkHeaderPagingHelper.java similarity index 90% rename from fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java rename to fcli-common/src/main/java/com/fortify/cli/common/rest/paging/LinkHeaderPagingHelper.java index c264591f36..9bdc1998dd 100644 --- a/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java +++ b/fcli-common/src/main/java/com/fortify/cli/common/rest/paging/LinkHeaderPagingHelper.java @@ -8,14 +8,14 @@ import kong.unirest.HttpRequest; import kong.unirest.PagedList; -@SuppressWarnings("unchecked") // TODO Can we get rid of these warnings in a better way? -public class PagingHelper { +public class LinkHeaderPagingHelper { private static final Pattern linkHeaderPattern = Pattern.compile("<([^>]*)>; *rel=\"([^\"]*)\""); public static final PagedList pagedRequest(HttpRequest request, INextPageUrlProducer nextPageUrlProducer) { return pagedRequest(request, nextPageUrlProducer, JsonNode.class); } + @SuppressWarnings("unchecked") // TODO Can we get rid of these warnings in a better way? public static final PagedList pagedRequest(HttpRequest request, INextPageUrlProducer nextPageUrlProducer, Class returnType) { return request.asPaged(r->r.asObject(returnType), nextPageUrlProducer::getNextPageUrl); } diff --git a/fcli-common/src/main/java/com/fortify/cli/common/util/Counter.java b/fcli-common/src/main/java/com/fortify/cli/common/util/Counter.java new file mode 100644 index 0000000000..f1f1fb531d --- /dev/null +++ b/fcli-common/src/main/java/com/fortify/cli/common/util/Counter.java @@ -0,0 +1,24 @@ +package com.fortify.cli.common.util; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +@ToString @EqualsAndHashCode +public class Counter { + @Getter private long count = 0; + + public Counter increase() { + count++; + return this; + } + + public Counter increase(long value) { + count+=value; + return this; + } + + public Counter increase(Counter counter) { + return increase(counter.getCount()); + } +} diff --git a/fcli-common/src/main/java/com/fortify/cli/common/util/StringUtils.java b/fcli-common/src/main/java/com/fortify/cli/common/util/StringUtils.java index 897fb0f2ae..e7411f7451 100644 --- a/fcli-common/src/main/java/com/fortify/cli/common/util/StringUtils.java +++ b/fcli-common/src/main/java/com/fortify/cli/common/util/StringUtils.java @@ -29,4 +29,10 @@ public static final String substringAfterLast(String str, String separator) { final int pos = str.lastIndexOf(separator); return pos==-1 ? "" : str.substring(pos + separator.length()); } + + public static final String capitalize(String str) { + return str==null + ? null + : str.substring(0,1).toUpperCase() + str.substring(1).toLowerCase(); + } } diff --git a/fcli-fod/src/main/java/com/fortify/cli/fod/rest/helper/FoDPagingHelper.java b/fcli-fod/src/main/java/com/fortify/cli/fod/rest/helper/FoDPagingHelper.java index a9bba0a3d4..401e3bcbf6 100644 --- a/fcli-fod/src/main/java/com/fortify/cli/fod/rest/helper/FoDPagingHelper.java +++ b/fcli-fod/src/main/java/com/fortify/cli/fod/rest/helper/FoDPagingHelper.java @@ -5,14 +5,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fortify.cli.common.rest.paging.INextPageUrlProducer; -import com.fortify.cli.common.rest.paging.PagingHelper; +import com.fortify.cli.common.rest.paging.LinkHeaderPagingHelper; import kong.unirest.HttpRequest; import kong.unirest.PagedList; public class FoDPagingHelper { public static final PagedList pagedRequest(HttpRequest request) { - return PagingHelper.pagedRequest(request, nextPageUrlProducer(request)); + return LinkHeaderPagingHelper.pagedRequest(request, nextPageUrlProducer(request)); } public static final INextPageUrlProducer nextPageUrlProducer(HttpRequest originalRequest) { return nextPageUrlProducer(originalRequest.getUrl()); diff --git a/fcli-sc-dast/src/main/java/com/fortify/cli/sc_dast/rest/helper/SCDastPagingHelper.java b/fcli-sc-dast/src/main/java/com/fortify/cli/sc_dast/rest/helper/SCDastPagingHelper.java index 82664660be..dfccf3813b 100644 --- a/fcli-sc-dast/src/main/java/com/fortify/cli/sc_dast/rest/helper/SCDastPagingHelper.java +++ b/fcli-sc-dast/src/main/java/com/fortify/cli/sc_dast/rest/helper/SCDastPagingHelper.java @@ -4,7 +4,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fortify.cli.common.rest.paging.INextPageUrlProducer; -import com.fortify.cli.common.rest.paging.PagingHelper; +import com.fortify.cli.common.rest.paging.LinkHeaderPagingHelper; import io.micronaut.http.uri.UriBuilder; import kong.unirest.HttpRequest; @@ -12,7 +12,7 @@ public class SCDastPagingHelper { public static final PagedList pagedRequest(HttpRequest request) { - return PagingHelper.pagedRequest(request, nextPageUrlProducer(request)); + return LinkHeaderPagingHelper.pagedRequest(request, nextPageUrlProducer(request)); } public static final INextPageUrlProducer nextPageUrlProducer(HttpRequest originalRequest) { return nextPageUrlProducer(originalRequest.getUrl()); diff --git a/fcli-ssc/src/main/java/com/fortify/cli/ssc/rest/bulk/SSCBulkEmbedder.java b/fcli-ssc/src/main/java/com/fortify/cli/ssc/rest/bulk/SSCBulkEmbedder.java index edb8dff796..5c5c3ea8cc 100644 --- a/fcli-ssc/src/main/java/com/fortify/cli/ssc/rest/bulk/SSCBulkEmbedder.java +++ b/fcli-ssc/src/main/java/com/fortify/cli/ssc/rest/bulk/SSCBulkEmbedder.java @@ -5,7 +5,7 @@ import java.util.stream.Stream; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fortify.cli.common.json.JsonHelper; import com.fortify.cli.ssc.rest.helper.SSCInputTransformer; @@ -30,11 +30,11 @@ public SSCBulkEmbedder(ISSCEntityEmbedderSupplier... suppliers) { .collect(Collectors.toList()); } - public JsonNode transformInput(UnirestInstance unirest, JsonNode input) { - var records = SSCInputTransformer.getDataOrSelf(input); - if ( records instanceof ObjectNode ) { - records = JsonHelper.toArrayNode(records); - } + public ArrayNode transformInput(UnirestInstance unirest, JsonNode input) { + var data = SSCInputTransformer.getDataOrSelf(input); + var records = data instanceof ArrayNode + ? (ArrayNode) data + : JsonHelper.toArrayNode(data); if ( embedders!=null ) { SSCBulkRequestBuilder builder = new SSCBulkRequestBuilder(); for ( var record : records ) { diff --git a/fcli-ssc/src/main/java/com/fortify/cli/ssc/rest/helper/SSCPagingHelper.java b/fcli-ssc/src/main/java/com/fortify/cli/ssc/rest/helper/SSCPagingHelper.java index e357d6458e..86cc9e30ba 100644 --- a/fcli-ssc/src/main/java/com/fortify/cli/ssc/rest/helper/SSCPagingHelper.java +++ b/fcli-ssc/src/main/java/com/fortify/cli/ssc/rest/helper/SSCPagingHelper.java @@ -5,20 +5,22 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fortify.cli.common.json.JsonHelper; import com.fortify.cli.common.rest.paging.INextPageUrlProducer; -import com.fortify.cli.common.rest.paging.PagingHelper; +import com.fortify.cli.common.rest.paging.LinkHeaderPagingHelper; import kong.unirest.HttpRequest; import kong.unirest.PagedList; +import lombok.Setter; public class SSCPagingHelper { + private static final SSCContinueNextPageSupplier continueNextPageSupplier = new SSCContinueNextPageSupplier(); public static final PagedList pagedRequest(HttpRequest request) { - return pagedRequest(request, SSCPagingHelper::getTrue); + return pagedRequest(request, continueNextPageSupplier); } public static final PagedList pagedRequest(HttpRequest request, Supplier continueSupplier) { - return PagingHelper.pagedRequest(request, nextPageUrlProducer(continueSupplier)); + return LinkHeaderPagingHelper.pagedRequest(request, nextPageUrlProducer(continueSupplier)); } public static final INextPageUrlProducer nextPageUrlProducer() { - return nextPageUrlProducer(SSCPagingHelper::getTrue); + return nextPageUrlProducer(continueNextPageSupplier); } public static final INextPageUrlProducer nextPageUrlProducer(Supplier continueSupplier) { return r -> { @@ -31,5 +33,9 @@ public static final INextPageUrlProducer nextPageUrlProducer(Supplier c return null; }; } - private static final boolean getTrue() { return true; } + public static final class SSCContinueNextPageSupplier implements Supplier { + @Setter private boolean loadNextPage = true; + @Override + public Boolean get() { return loadNextPage; } + } } diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportAppArtifactCollector.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportAppArtifactCollector.java new file mode 100644 index 0000000000..cc513e4f1f --- /dev/null +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportAppArtifactCollector.java @@ -0,0 +1,231 @@ +package com.fortify.cli.util.msp_report.collector; + +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.fortify.cli.common.rest.unirest.config.IUrlConfig; +import com.fortify.cli.common.util.Counter; +import com.fortify.cli.util.msp_report.config.MspReportConfig; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportLicenseType; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppDescriptor; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppSummaryDescriptor; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCArtifactDescriptor; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCArtifactDescriptor.MspReportSSCArtifactScanType; +import com.fortify.cli.util.msp_report.writer.MspReportResultsWriters; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; + +/** + *

This class is responsible for collecting and outputting + * {@link MspReportSSCArtifactDescriptor} instances.

+ * + * @author rsenden + * + */ +@RequiredArgsConstructor +public final class MspReportAppArtifactCollector implements AutoCloseable { + private final MspReportConfig reportConfig; + private final MspReportResultsWriters writers; + private final IUrlConfig urlConfig; + private final MspReportSSCAppDescriptor appDescriptor; + private final MspReportSSCAppSummaryDescriptor summaryDescriptor = new MspReportSSCAppSummaryDescriptor(); + private final List processedArtifacts = new ArrayList<>(); + private final Map> applicationLicenseArtifacts = new LinkedHashMap<>(); + + public MspReportSSCAppSummaryDescriptor summary() { + return summaryDescriptor; + } + + /** + * Report an SSC application version artifact. + * @return {@link MspReportArtifactCollectorState#DONE} is we're done processing + * this application version, {@link MspReportArtifactCollectorState#DONE} + * otherwise. + */ + @SneakyThrows + public MspReportArtifactCollectorState report(MspReportSSCArtifactDescriptor artifactDescriptor) { + summaryDescriptor.getArtifactsProcessedCounter().increase(); + var licenseType = appDescriptor.getMspLicenseType(); + + if ( isIgnoredAndContinue(artifactDescriptor) ) { + return MspReportArtifactCollectorState.CONTINUE; + } + if ( isIgnoredAndDone(licenseType, artifactDescriptor) ) { + return MspReportArtifactCollectorState.DONE; + } + if ( !isBeforeReportingStartDate(artifactDescriptor.getLastScanDate()) ) { + processWithinReportingPeriod(licenseType, artifactDescriptor); + } else { + return processOutsideReportingPeriod(licenseType, artifactDescriptor); + } + return MspReportArtifactCollectorState.CONTINUE; + } + + /** + * This method returns true if we want to ignore the current artifact, + * and don't need to process any further artifacts for this application + * version. We're done processing artifacts if license type is Scan + * or Demo, and upload date is before reporting start date. Note that + * we need to check upload date instead of scan date as artifacts are + * ordered by descending upload date, and upload date should always be + * more recent than scan date. In the following example, we still want + * to process artifact #2 even if scan date for #1 is before reporting + * period start date: + *
    + *
  1. uploadDate within reporting period, lastScanDate older than + * reporting period start date
    2. + *
  2. Both uploadDate and lastScanDate within reporting period + *
+ */ + private boolean isIgnoredAndDone(MspReportLicenseType licenseType, MspReportSSCArtifactDescriptor artifactDescriptor) { + var uploadDateTime = artifactDescriptor.getUploadDate(); + return licenseType!=MspReportLicenseType.Application + && uploadDateTime!=null + && isBeforeReportingStartDate(uploadDateTime) + ? true : false; + } + + /** + * This method returns true if we want to ignore the current artifact, + * but want to continue processing further artifacts. We want to ignore + * all artifacts that either: + *
    + *
  • Have no scan date (usually because there was an error processing + * the artifact, or the artifact needs to be approved)
    • + *
  • Are more recent than reporting period end date
    • + *
+ * Note that we're not ignoring artifacts for which the scan or upload + * date is before the reporting period start date, as these may still + * need to be processed for Application license reporting. + * @return true if we want to ignore the current artifact, + * false if we want to process the current artifact + */ + private boolean isIgnoredAndContinue(MspReportSSCArtifactDescriptor artifactDescriptor) { + var lastScanDateTime = artifactDescriptor.getLastScanDate(); + return lastScanDateTime==null || isAfterReportingEndDate(lastScanDateTime); + } + + private void processWithinReportingPeriod(MspReportLicenseType licenseType, MspReportSSCArtifactDescriptor artifactDescriptor) { + summaryDescriptor.getArtifactsInReportingPeriodCounter().increase(); + switch ( licenseType ) { + case Application: + storeForApplicationLicense(artifactDescriptor); + break; + case Demo: + storeForDemoOrScanLicense(artifactDescriptor, false, null); + break; + case Scan: + storeForDemoOrScanLicense(artifactDescriptor, true, summaryDescriptor.getConsumedScanEntitlementsCounter()); + break; + } + } + + private void storeForApplicationLicense(MspReportSSCArtifactDescriptor artifactDescriptor) { + artifactDescriptor.getScanTypes().forEach( + type->applicationLicenseArtifacts + .computeIfAbsent(type, x->new ArrayList<>()) + .add(artifactDescriptor)); + } + + private void storeForDemoOrScanLicense(MspReportSSCArtifactDescriptor artifactDescriptor, boolean consumeEntitlementForFortifyScans, Counter counter) { + artifactDescriptor.getScanTypes() + .forEach(type->store( + artifactDescriptor, type, + consumeEntitlementForFortifyScans, + artifactDescriptor.getLastScanDate(), + counter)); + } + + private void store(MspReportSSCArtifactDescriptor artifactDescriptor, MspReportSSCArtifactScanType entitlementScanType, boolean consumeEntitlementForFortifyScans, LocalDateTime entitlementConsumptionDate, Counter counter) { + var entitlementConsumed = entitlementScanType.isFortifyScan() && consumeEntitlementForFortifyScans; + if ( !entitlementConsumed ) { + entitlementConsumptionDate = null; + } else { + counter.increase(); + } + processedArtifacts.add(new MspReportProcessedArtifactDescriptor( + artifactDescriptor, + entitlementScanType, + entitlementConsumed, + entitlementConsumptionDate)); + } + + private MspReportArtifactCollectorState processOutsideReportingPeriod(MspReportLicenseType licenseType, MspReportSSCArtifactDescriptor artifactDescriptor) { + if ( licenseType!=MspReportLicenseType.Application ) { + // We're only interested in the artifact if license type is Application, + // however this method may be invoked if uploadDate is within reporting + // period but lastScanDate is not. + return MspReportArtifactCollectorState.CONTINUE; + } + var matchingScanTypes = artifactDescriptor.getMatchingFortifyScanTypes(applicationLicenseArtifacts.keySet()); + for ( var matchingScanType : matchingScanTypes ) { + applicationLicenseArtifacts.get(matchingScanType) + .forEach(matchingArtifact->store( + matchingArtifact, + matchingScanType, + false, + artifactDescriptor.getLastScanDate(), + summaryDescriptor.getConsumedApplicationEntitlementsCounter())); + } + matchingScanTypes.forEach(applicationLicenseArtifacts::remove); + return matchingScanTypes.isEmpty() && !applicationLicenseArtifacts.isEmpty() + ? MspReportArtifactCollectorState.CONTINUE + : MspReportArtifactCollectorState.DONE; + } + + private boolean isAfterReportingEndDate(LocalDateTime dateTime) { + var reportingEndDateTime = reportConfig.getReportingEndDate().atTime(LocalTime.MAX); + return dateTime.isAfter(reportingEndDateTime); + } + + private boolean isBeforeReportingStartDate(LocalDateTime dateTime) { + var reportingStartDateTime = reportConfig.getReportingStartDate().atStartOfDay(); + return dateTime.isBefore(reportingStartDateTime); + } + + public void close() { + storeRemainingArtifacts(); + processedArtifacts.forEach(this::writeResult); + } + + private void storeRemainingArtifacts() { + applicationLicenseArtifacts.entrySet().forEach(this::storeRemainingArtifacts); + } + + private void storeRemainingArtifacts(Map.Entry> entry) { + var entitlementScanType = entry.getKey(); + var descriptors = entry.getValue(); + var size = descriptors.size(); + var entitlementConsumedDate = descriptors.get(size-1).getLastScanDate(); + for ( int i = 0 ; i < size ; i++ ) { + var artifactDescriptor = descriptors.get(i); + store(artifactDescriptor, + entitlementScanType, + i==size-1, + entitlementConsumedDate, + summaryDescriptor.getConsumedApplicationEntitlementsCounter()); + } + } + + private void writeResult(MspReportProcessedArtifactDescriptor descriptor) { + writers.artifactsWriter().write(urlConfig, appDescriptor, descriptor.artifactDescriptor, descriptor.entitlementScanType, descriptor.entitlementConsumed, descriptor.entitlementConsumptionDate); + } + + public static enum MspReportArtifactCollectorState { + CONTINUE, DONE + } + + @Data + public static final class MspReportProcessedArtifactDescriptor { + private final MspReportSSCArtifactDescriptor artifactDescriptor; + private final MspReportSSCArtifactScanType entitlementScanType; + private final boolean entitlementConsumed; + private final LocalDateTime entitlementConsumptionDate; + } +} diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportAppCollector.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportAppCollector.java new file mode 100644 index 0000000000..1bade736e7 --- /dev/null +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportAppCollector.java @@ -0,0 +1,109 @@ +package com.fortify.cli.util.msp_report.collector; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.common.json.JsonHelper; +import com.fortify.cli.common.rest.unirest.config.IUrlConfig; +import com.fortify.cli.common.util.Counter; +import com.fortify.cli.common.util.StringUtils; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportLicenseType; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportProcessingStatus; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCProcessedAppDescriptor; +import com.fortify.cli.util.msp_report.writer.MspReportResultsWriters; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; + +/** + *

This class is responsible for collecting and outputting + * {@link MspReportSSCProcessedAppDescriptor} instances.

+ * + * @author rsenden + * + */ +@RequiredArgsConstructor +public final class MspReportAppCollector { + private final MspReportResultsWriters writers; + private final ObjectNode summary; + + private Counter totalAppCounter = new Counter(); + private Counter applicationEntitlementsConsumedCounter = new Counter(); + private Counter scanEntitlementsConsumedCounter = new Counter(); + private Map countsByProcessingStatus = new HashMap<>(); + private Map countsByAttrStatus = new HashMap<>(); + + + @SneakyThrows + public void report(IUrlConfig urlConfig, MspReportSSCProcessedAppDescriptor descriptor) { + totalAppCounter.increase(); + var summaryDescriptor = descriptor.getAppSummaryDescriptor(); + applicationEntitlementsConsumedCounter.increase(summaryDescriptor.getConsumedApplicationEntitlementsCounter()); + scanEntitlementsConsumedCounter.increase(summaryDescriptor.getConsumedScanEntitlementsCounter()); + increaseCountByProcessingStatus(descriptor.getStatus()); + increaseMspLicenseTypeAttrStatusCounts(descriptor.getAppDescriptor().getMspLicenseType()); + writers.appsWriter().write(urlConfig, descriptor); + } + + void writeResults() { + writeEntitlementsConsumedCounts(); + writeAppCounts(); + } + + private void writeAppCounts() { + ObjectNode appCounts = JsonHelper.getObjectMapper().createObjectNode(); + appCounts.put("total", totalAppCounter.getCount()); + Stream.of(MspReportProcessingStatus.values()) + .forEach(status->appCounts.put("appsWith"+StringUtils.capitalize(status.name()), getCounterByProcessingStatus(status).getCount())); + Stream.of(MspReportSSCAppAttrStatus.values()) + .forEach(status->appCounts.put(status.name(), getCounterByAttrStatus(status).getCount())); + summary.set("applicationCounts", appCounts); + } + + private void writeEntitlementsConsumedCounts() { + ObjectNode entitlementsConsumedCounts = JsonHelper.getObjectMapper().createObjectNode(); + entitlementsConsumedCounts.put("application", applicationEntitlementsConsumedCounter.getCount()); + entitlementsConsumedCounts.put("scan", scanEntitlementsConsumedCounter.getCount()); + summary.set("entitlementsConsumed", entitlementsConsumedCounts); + } + + private void increaseMspLicenseTypeAttrStatusCounts(MspReportLicenseType mspLicenseType) { + if ( mspLicenseType==null ) { + throw new IllegalStateException("MSP license type not defined"); + } else { + switch (mspLicenseType) { + case Application: + increaseCountByAttrStatus(MspReportSSCAppAttrStatus.applicationLicenseType); + break; + case Demo: + increaseCountByAttrStatus(MspReportSSCAppAttrStatus.demoLicenseType); + break; + case Scan: + increaseCountByAttrStatus(MspReportSSCAppAttrStatus.scanLicenseType); + break; + } + } + } + + private void increaseCountByAttrStatus(MspReportSSCAppAttrStatus status) { + getCounterByAttrStatus(status).increase(); + } + + private Counter getCounterByAttrStatus(MspReportSSCAppAttrStatus status) { + return countsByAttrStatus.computeIfAbsent(status, x->new Counter()); + } + + private void increaseCountByProcessingStatus(MspReportProcessingStatus status) { + getCounterByProcessingStatus(status).increase(); + } + + private Counter getCounterByProcessingStatus(MspReportProcessingStatus status) { + return countsByProcessingStatus.computeIfAbsent(status, x->new Counter()); + } + + private static enum MspReportSSCAppAttrStatus { + demoLicenseType, applicationLicenseType, scanLicenseType + } +} diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportAppVersionCollector.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportAppVersionCollector.java index 857e837936..b6c4e02107 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportAppVersionCollector.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportAppVersionCollector.java @@ -7,13 +7,11 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fortify.cli.common.json.JsonHelper; import com.fortify.cli.common.rest.unirest.config.IUrlConfig; -import com.fortify.cli.util.msp_report.generator.ssc.MspReportLicenseType; -import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppVersionDescriptor; +import com.fortify.cli.common.util.Counter; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportProcessingStatus; import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCProcessedAppVersionDescriptor; -import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCProcessedAppVersionDescriptor.MspReportSSCAppVersionProcessingStatus; import com.fortify.cli.util.msp_report.writer.MspReportResultsWriters; -import io.micrometer.common.util.StringUtils; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -29,99 +27,33 @@ public final class MspReportAppVersionCollector { private final MspReportResultsWriters writers; private final ObjectNode summary; - private int totalAppVersionCount = 0; - private int applicationEntitlementsConsumed = 0; - private int scanEntitlementsConsumed = 0; - private Map countsByProcessingStatus = new HashMap<>(); - private Map countsByAttrStatus = new HashMap<>(); - + private Counter totalAppVersionCounter = new Counter(); + private Map countsByProcessingStatus = new HashMap<>(); @SneakyThrows public void report(IUrlConfig urlConfig, MspReportSSCProcessedAppVersionDescriptor descriptor) { - totalAppVersionCount++; - applicationEntitlementsConsumed += descriptor.getAppVersionEntitlementSummaryDescriptor().getApplicationEntitlementsConsumed(); - scanEntitlementsConsumed += descriptor.getAppVersionEntitlementSummaryDescriptor().getScanEntitlementsConsumed(); + totalAppVersionCounter.increase(); increaseCountByProcessingStatus(descriptor.getStatus()); - increaseAttrStatusCounts(descriptor.getAppVersionDescriptor()); - writers.appVersionWriter().write(urlConfig, descriptor); + writers.appVersionsWriter().write(urlConfig, descriptor); } void writeResults() { writeAppVersionCounts(); - writeEntitlementsConsumedCounts(); } private void writeAppVersionCounts() { ObjectNode appVersionCounts = JsonHelper.getObjectMapper().createObjectNode(); - appVersionCounts.put("total", totalAppVersionCount); - Stream.of(MspReportSSCAppVersionProcessingStatus.values()) - .forEach(status->appVersionCounts.put(status.name(), getCountByProcessingStatus(status))); - Stream.of(MspReportSSCAppVersionAttrStatus.values()) - .forEach(status->appVersionCounts.put(status.name(), getCountByAttrStatus(status))); + appVersionCounts.put("total", totalAppVersionCounter.getCount()); + Stream.of(MspReportProcessingStatus.values()) + .forEach(status->appVersionCounts.put(status.name(), getCounterByProcessingStatus(status).getCount())); summary.set("applicationVersionCounts", appVersionCounts); } - private void writeEntitlementsConsumedCounts() { - ObjectNode entitlementsConsumedCounts = JsonHelper.getObjectMapper().createObjectNode(); - entitlementsConsumedCounts.put("application", applicationEntitlementsConsumed); - entitlementsConsumedCounts.put("scan", scanEntitlementsConsumed); - summary.set("entitlementsConsumed", entitlementsConsumedCounts); - } - - private void increaseCountByProcessingStatus(MspReportSSCAppVersionProcessingStatus status) { - countsByProcessingStatus.put(status, getCountByProcessingStatus(status)+1); - } - - private Integer getCountByProcessingStatus(MspReportSSCAppVersionProcessingStatus status) { - return countsByProcessingStatus.getOrDefault(status, 0); - } - - private void increaseAttrStatusCounts(MspReportSSCAppVersionDescriptor appVersionDescriptor) { - increaseMspLicenseTypeAttrStatusCounts(appVersionDescriptor.getMspLicenseType()); - increaseEndCustomerNameAttrStatusCounts(appVersionDescriptor.getMspEndCustomerName()); - increaseEndCustomerLocationAttrStatusCounts(appVersionDescriptor.getMspEndCustomerLocation()); - } - - private void increaseMspLicenseTypeAttrStatusCounts(MspReportLicenseType mspLicenseType) { - if ( mspLicenseType==null ) { - increaseCountByAttrStatus(MspReportSSCAppVersionAttrStatus.missingLicenseType); - } else { - switch (mspLicenseType) { - case Application: - increaseCountByAttrStatus(MspReportSSCAppVersionAttrStatus.applicationLicenseType); - break; - case Demo: - increaseCountByAttrStatus(MspReportSSCAppVersionAttrStatus.demoLicenseType); - break; - case Scan: - increaseCountByAttrStatus(MspReportSSCAppVersionAttrStatus.scanLicenseType); - break; - } - } + private void increaseCountByProcessingStatus(MspReportProcessingStatus status) { + getCounterByProcessingStatus(status).increase(); } - private void increaseEndCustomerNameAttrStatusCounts(String mspEndCustomerName) { - if ( StringUtils.isBlank(mspEndCustomerName) ) { - increaseCountByAttrStatus(MspReportSSCAppVersionAttrStatus.missingEndCustomerName); - } - } - - private void increaseEndCustomerLocationAttrStatusCounts(String mspEndCustomerLocation) { - if ( StringUtils.isBlank(mspEndCustomerLocation) ) { - increaseCountByAttrStatus(MspReportSSCAppVersionAttrStatus.missingEndCustomerLocation); - } - } - - private void increaseCountByAttrStatus(MspReportSSCAppVersionAttrStatus status) { - countsByAttrStatus.put(status, getCountByAttrStatus(status)+1); - } - - - private Integer getCountByAttrStatus(MspReportSSCAppVersionAttrStatus status) { - return countsByAttrStatus.getOrDefault(status, 0); - } - - private static enum MspReportSSCAppVersionAttrStatus { - demoLicenseType, applicationLicenseType, scanLicenseType, missingLicenseType, missingEndCustomerName, missingEndCustomerLocation + private Counter getCounterByProcessingStatus(MspReportProcessingStatus status) { + return countsByProcessingStatus.computeIfAbsent(status, x->new Counter()); } } diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportResultsCollector.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportResultsCollector.java index fe386dd937..35e1990c42 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportResultsCollector.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/collector/MspReportResultsCollector.java @@ -4,11 +4,13 @@ import com.fortify.cli.common.progress.helper.IProgressHelperI18n; import com.fortify.cli.common.report.collector.IReportResultsCollector; +import com.fortify.cli.common.report.logger.IReportLogger; +import com.fortify.cli.common.report.logger.ReportLogger; import com.fortify.cli.common.report.writer.IReportWriter; -import com.fortify.cli.common.report.writer.entry.IReportErrorEntryWriter; -import com.fortify.cli.common.report.writer.entry.ReportErrorEntryWriter; +import com.fortify.cli.common.rest.unirest.config.IUrlConfig; import com.fortify.cli.util.msp_report.cli.cmd.MspReportGenerateCommand; import com.fortify.cli.util.msp_report.config.MspReportConfig; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppDescriptor; import com.fortify.cli.util.msp_report.writer.MspReportResultsWriters; import lombok.Getter; @@ -19,7 +21,7 @@ * This class is the primary entry point for collecting and outputting report data. * An instance of this class is created by the {@link MspReportGenerateCommand} * and passed to the source-specific generators. Source-specific generators can use - * this class to access the {@link IReportErrorEntryWriter} and various result collectors. + * this class to access the {@link IReportLogger} and various result collectors. * * @author rsenden * @@ -30,6 +32,7 @@ public final class MspReportResultsCollector implements IReportResultsCollector @Getter private final IProgressHelperI18n progressHelper; private final IReportWriter reportWriter; private final MspReportResultsWriters writers; + @Getter private final MspReportAppCollector appCollector; @Getter private final MspReportAppVersionCollector appVersionCollector; public MspReportResultsCollector(MspReportConfig reportConfig, IReportWriter reportWriter, IProgressHelperI18n progressHelper) { @@ -37,16 +40,21 @@ public MspReportResultsCollector(MspReportConfig reportConfig, IReportWriter rep this.progressHelper = progressHelper; this.reportWriter = reportWriter; this.writers = new MspReportResultsWriters(reportWriter, progressHelper); + this.appCollector = new MspReportAppCollector(this.writers, reportWriter.summary()); this.appVersionCollector = new MspReportAppVersionCollector(this.writers, reportWriter.summary()); } + public MspReportAppArtifactCollector artifactCollector(IUrlConfig urlConfig, MspReportSSCAppDescriptor appDescriptor) { + return new MspReportAppArtifactCollector(reportConfig, writers, urlConfig, appDescriptor); + } + /** - * We provide public access to {@link ReportErrorEntryWriter}, all + * We provide public access to {@link ReportLogger}, all * other writers are for internal use by this class only. * @return */ - public final IReportErrorEntryWriter errorWriter() { - return writers.errorWriter(); + public final IReportLogger logger() { + return writers.logger(); } @Override @SneakyThrows @@ -55,7 +63,8 @@ public void close() { reportWriter.summary().put("contractStartDate", reportConfig.getContractStartDate().format(DateTimeFormatter.ISO_LOCAL_DATE)); reportWriter.summary().put("reportingStartDate", reportConfig.getReportingStartDate().format(DateTimeFormatter.ISO_LOCAL_DATE)); reportWriter.summary().put("reportingEndDate", reportConfig.getReportingEndDate().format(DateTimeFormatter.ISO_LOCAL_DATE)); + appCollector.writeResults(); appVersionCollector.writeResults(); - reportWriter.summary().put("errorCount", errorWriter().getErrorCount()); + logger().updateSummary(reportWriter.summary()); } } diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportLicenseType.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportLicenseType.java index fded2e8c76..e6decc606c 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportLicenseType.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportLicenseType.java @@ -1,5 +1,12 @@ package com.fortify.cli.util.msp_report.generator.ssc; +import java.util.Arrays; +import java.util.List; + public enum MspReportLicenseType { Application, Scan, Demo; + + public static final List allOrderedByPriority() { + return Arrays.asList(Application, Scan, Demo); + } } diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportProcessingStatus.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportProcessingStatus.java new file mode 100644 index 0000000000..c2285800a3 --- /dev/null +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportProcessingStatus.java @@ -0,0 +1,5 @@ +package com.fortify.cli.util.msp_report.generator.ssc; + +public enum MspReportProcessingStatus { + error, warn, success +} \ No newline at end of file diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppDescriptor.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppDescriptor.java new file mode 100644 index 0000000000..99dbc3495c --- /dev/null +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppDescriptor.java @@ -0,0 +1,98 @@ +package com.fortify.cli.util.msp_report.generator.ssc; + +import static com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppVersionAttribute.MSP_End_Customer_Location; +import static com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppVersionAttribute.MSP_End_Customer_Name; +import static com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppVersionAttribute.MSP_License_Type; + +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.BiFunction; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.common.json.JsonNodeHolder; +import com.fortify.cli.common.report.logger.IReportLogger; +import com.fortify.cli.common.util.Counter; + +import io.micrometer.common.util.StringUtils; +import io.micronaut.core.annotation.ReflectiveAccess; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@ReflectiveAccess @Data @EqualsAndHashCode(callSuper = false) +public class MspReportSSCAppDescriptor extends JsonNodeHolder { + private String id; + private String name; + private ZonedDateTime creationDate; + private MspReportLicenseType mspLicenseType; + private String mspEndCustomerName; + private String mspEndCustomerLocation; + private final List versionDescriptors = new ArrayList<>(); + private final Counter warnCounter = new Counter(); + + public ObjectNode updateReportRecord(ObjectNode objectNode) { + return objectNode + .put("applicationId", id) + .put("applicationName", name) + .put("applicationCreationDate", creationDate.toString()) + .put("mspLicenseType", mspLicenseType.name()) + .put("mspEndCustomerName", mspEndCustomerName) + .put("mspEndCustomerLocation", mspEndCustomerLocation); + } + + public void check(IReportLogger logger) { + if ( mspLicenseType==null ) { + throw new IllegalStateException("Missing license type"); + } + + if ( mspLicenseType!=MspReportLicenseType.Demo ) { + if ( StringUtils.isBlank(mspEndCustomerName) ) { + warn(logger, "Missing MSP end customer name"); + } + if ( StringUtils.isBlank(mspEndCustomerLocation) ) { + warn(logger, "Missing MSP end customer location"); + } + } + } + + public void addVersionDescriptor(IReportLogger logger, MspReportSSCAppVersionDescriptor appVersionDescriptor) { + appVersionDescriptor.setAppDescriptor(this); + versionDescriptors.add(appVersionDescriptor); + mspLicenseType = get(logger, MSP_License_Type.name(), mspLicenseType, appVersionDescriptor.getMspLicenseType(), this::onMismatch); + mspEndCustomerName = get(logger, MSP_End_Customer_Name.name(), mspEndCustomerName, appVersionDescriptor.getMspEndCustomerName(), null); + mspEndCustomerLocation = get(logger, MSP_End_Customer_Location.name(), mspEndCustomerLocation, appVersionDescriptor.getMspEndCustomerLocation(), null); + } + + private MspReportLicenseType onMismatch(MspReportLicenseType appType, MspReportLicenseType versionType) { + var allByPriority = new ArrayList<>(MspReportLicenseType.allOrderedByPriority()); + var currentTypes = Arrays.asList(appType, versionType); + allByPriority.retainAll(currentTypes); + return allByPriority.get(0); + } + + private T get(IReportLogger logger, String attrName, T appValue, T versionValue, BiFunction onMismatch) { + if ( appValue==null || (appValue instanceof String && ((String) appValue).isBlank())) { + return versionValue; + } + if ( versionValue==null || (versionValue instanceof String && ((String) versionValue).isBlank())) { + return appValue; + } + if ( appValue.equals(versionValue) ) { return appValue; } + var msg = String.format("%s mismatch (app: %s, version: %s)", attrName, appValue, versionValue); + if ( onMismatch!=null ) { + var value = onMismatch.apply(appValue, versionValue); + warn(logger, "%s, using %s", msg, value); + return value; + } else { + throw new IllegalStateException(msg); + } + } + + private void warn(IReportLogger logger, String msg, Object... msgArgs) { + var fullMsg = String.format("Application %s (%s): %s", name, mspLicenseType, msg); + logger.warn(fullMsg, msgArgs); + warnCounter.increase(); + } + +} diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppSummaryDescriptor.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppSummaryDescriptor.java new file mode 100644 index 0000000000..dceef7a951 --- /dev/null +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppSummaryDescriptor.java @@ -0,0 +1,22 @@ +package com.fortify.cli.util.msp_report.generator.ssc; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.common.util.Counter; + +import lombok.Data; + +@Data +public class MspReportSSCAppSummaryDescriptor { + private final Counter artifactsProcessedCounter = new Counter(); + private final Counter artifactsInReportingPeriodCounter = new Counter(); + private final Counter consumedApplicationEntitlementsCounter = new Counter(); + private final Counter consumedScanEntitlementsCounter = new Counter(); + + public ObjectNode updateReportRecord(ObjectNode objectNode) { + return objectNode + .put("artifactsProcessed", artifactsProcessedCounter.getCount()) + .put("artifactsInReportingPeriod", artifactsInReportingPeriodCounter.getCount()) + .put("consumedApplicationEntitlements", consumedApplicationEntitlementsCounter.getCount()) + .put("consumedScanEntitlements", consumedScanEntitlementsCounter.getCount()); + } +} diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppVersionDescriptor.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppVersionDescriptor.java index d2c8f5e1f4..53a56c3708 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppVersionDescriptor.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppVersionDescriptor.java @@ -4,6 +4,8 @@ import static com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppVersionAttribute.MSP_End_Customer_Name; import static com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppVersionAttribute.MSP_License_Type; +import java.time.ZonedDateTime; + import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -11,22 +13,26 @@ import com.fortify.cli.common.json.JsonNodeHolder; import io.micronaut.core.annotation.ReflectiveAccess; +import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; @ReflectiveAccess @Data @EqualsAndHashCode(callSuper = false) public class MspReportSSCAppVersionDescriptor extends JsonNodeHolder { + // We need to exclude this field from toString, equals and hashCode to + // avoid endless recursion + @ToString.Exclude @EqualsAndHashCode.Exclude private MspReportSSCAppDescriptor appDescriptor; @JsonProperty("id") private String versionId; - private String applicationName; @JsonProperty("name") private String versionName; + @JsonProperty("creationDate") private ZonedDateTime versionCreationDate; private boolean active; - private MspReportLicenseType mspLicenseType; - private String mspEndCustomerName; - private String mspEndCustomerLocation; - - public void setProject(ObjectNode project) { - applicationName = project.get("name").asText(); - } + // MSP attribute values should only be accessed by MspReportSSCAppDescriptor, + // hence access level PACKAGE + @Getter(AccessLevel.PACKAGE) private MspReportLicenseType mspLicenseType; + @Getter(AccessLevel.PACKAGE) private String mspEndCustomerName; + @Getter(AccessLevel.PACKAGE) private String mspEndCustomerLocation; public void setAttrValuesByName(ObjectNode attrValuesByName) { mspLicenseType = JsonHelper.evaluateSpelExpression(attrValuesByName, MSP_License_Type.asExpression(), MspReportLicenseType.class); @@ -34,29 +40,20 @@ public void setAttrValuesByName(ObjectNode attrValuesByName) { mspEndCustomerLocation = JsonHelper.evaluateSpelExpression(attrValuesByName, MSP_End_Customer_Location.asExpression(), String.class); } - public MspReportSSCAppVersionDescriptor check() { - // Even though we have already validated the attribute definitions, - // attribute values may still be blank if an application version - // was created before the attributes were created/set to required. - checkNotBlankAttr("mspLicenseType", mspLicenseType); - return this; + public void setAppDescriptor(MspReportSSCAppDescriptor appDescriptor) { + this.appDescriptor = appDescriptor; } @JsonIgnore public String getAppAndVersionName() { - return applicationName+":"+versionName; - } - - private void checkNotBlankAttr(String name, Object value) { - if ( value==null || (value instanceof String && ((String) value).isBlank()) ) { - throw new IllegalStateException(String.format("Blank value not allowed for attribute %s (%s)", name, getAppAndVersionName())); - } + return appDescriptor.getName()+":"+versionName; } public ObjectNode updateReportRecord(ObjectNode objectNode) { - return objectNode.put("applicationName", applicationName) - .put("versionName", versionName) + return appDescriptor.updateReportRecord(objectNode) .put("versionId", versionId) + .put("versionName", versionName) + .put("versionCreationDate", versionCreationDate.toString()) .put("active", active) .put("mspLicenseType", mspLicenseType.name()) .put("mspEndCustomerName", mspEndCustomerName) diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppVersionEntitlementSummaryDescriptor.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppVersionEntitlementSummaryDescriptor.java deleted file mode 100644 index 352c3429f8..0000000000 --- a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCAppVersionEntitlementSummaryDescriptor.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.fortify.cli.util.msp_report.generator.ssc; - -import com.fasterxml.jackson.databind.node.ObjectNode; - -import lombok.Data; -import lombok.RequiredArgsConstructor; - -@Data @RequiredArgsConstructor -public class MspReportSSCAppVersionEntitlementSummaryDescriptor { - private final MspReportLicenseType mspLicenseType; - private int numberOfScansInReportingPeriod; - - public void increaseNumberOfScansInReportingPeriod() { - numberOfScansInReportingPeriod++; - } - - public int getApplicationEntitlementsConsumed() { - return mspLicenseType==null || mspLicenseType==MspReportLicenseType.Application - ? 1 - : 0; - } - - public int getScanEntitlementsConsumed() { - return mspLicenseType==MspReportLicenseType.Scan - ? numberOfScansInReportingPeriod - : 0; - } - - public ObjectNode updateReportRecord(ObjectNode objectNode) { - return objectNode - .put("numberOfScansInReportingPeriod", numberOfScansInReportingPeriod) - .put("applicationEntitlementsConsumed", getApplicationEntitlementsConsumed()) - .put("scanEntitlementsConsumed", getScanEntitlementsConsumed()); - } -} diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCArtifactDescriptor.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCArtifactDescriptor.java new file mode 100644 index 0000000000..c551fe18ed --- /dev/null +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCArtifactDescriptor.java @@ -0,0 +1,111 @@ +package com.fortify.cli.util.msp_report.generator.ssc; + +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.common.json.JsonNodeHolder; + +import io.micronaut.core.annotation.ReflectiveAccess; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@ReflectiveAccess @Data @EqualsAndHashCode(callSuper = false) +public final class MspReportSSCArtifactDescriptor extends JsonNodeHolder { + private String id; + private LocalDateTime lastScanDate; + private LocalDateTime uploadDate; + private boolean purged; + private MspReportSSCArtifactScanStatus scaStatus; + private MspReportSSCArtifactScanStatus webInspectStatus; + private MspReportSSCArtifactScanStatus runtimeStatus; + private MspReportSSCArtifactScanStatus otherStatus; + @JsonIgnore private Set scanTypes; + @JsonIgnore private Set fortifyScanTypes; + + public void setLastScanDate(ZonedDateTime zonedDateTime) { + this.lastScanDate = zonedDateTime==null ? null : zonedDateTime.toLocalDateTime(); + } + + public void setUploadDate(ZonedDateTime zonedDateTime) { + this.uploadDate = zonedDateTime==null ? null : zonedDateTime.toLocalDateTime(); + } + + public boolean isFortifySastScan() { + return hasScan(scaStatus); + } + + public boolean isFortifyDastScan() { + return hasScan(webInspectStatus); + } + + public boolean isFortifyRuntimeScan() { + return hasScan(runtimeStatus); + } + + public boolean isOtherScan() { + return hasScan(otherStatus); + } + + public Set getScanTypes() { + if ( scanTypes==null ) { + scanTypes = new HashSet<>(4); + if ( isFortifySastScan() ) { scanTypes.add(MspReportSSCArtifactScanType.SAST); } + if ( isFortifyDastScan() ) { scanTypes.add(MspReportSSCArtifactScanType.DAST); } + if ( isFortifyRuntimeScan() ) { scanTypes.add(MspReportSSCArtifactScanType.RUNTIME); } + if ( isOtherScan() ) { scanTypes.add(MspReportSSCArtifactScanType.OTHER); } + } + return scanTypes; + } + + public Set getMatchingFortifyScanTypes(Collection scanTypes) { + var result = new HashSet<>(getFortifyScanTypes()); + result.retainAll(scanTypes); + return result; + } + + public String getScanTypesString() { + var scanTypes = getScanTypes(); + return scanTypes.isEmpty() + ? "N/A" + : scanTypes.stream() + .map(Enum::name) + .collect(Collectors.joining("+")); + } + + public ObjectNode updateReportRecord(ObjectNode objectNode) { + return objectNode + .put("artifactId", id) + .put("scanType", getScanTypesString()) + .put("uploadDate", toString(uploadDate)) + .put("lastScanDate", toString(lastScanDate)) + .put("purged", purged); + } + + private String toString(LocalDateTime zonedDateTime) { + return zonedDateTime==null ? "N/A" : zonedDateTime.toString(); + } + + public static final boolean hasScan(MspReportSSCArtifactScanStatus status) { + return status==MspReportSSCArtifactScanStatus.IGNORED + || status==MspReportSSCArtifactScanStatus.PROCESSED; + } + + public static enum MspReportSSCArtifactScanStatus { + NONE, NOT_EXIST, IGNORED, PROCESSED; + } + + @RequiredArgsConstructor + public static enum MspReportSSCArtifactScanType { + SAST(true), DAST(true), RUNTIME(true), OTHER(false); + + @Getter private final boolean fortifyScan; + } +} diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCProcessedAppDescriptor.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCProcessedAppDescriptor.java new file mode 100644 index 0000000000..640290ac80 --- /dev/null +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCProcessedAppDescriptor.java @@ -0,0 +1,22 @@ +package com.fortify.cli.util.msp_report.generator.ssc; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +import lombok.Data; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor @Data +public class MspReportSSCProcessedAppDescriptor { + private final MspReportSSCAppDescriptor appDescriptor; + private final MspReportProcessingStatus status; + private final String reason; + private final MspReportSSCAppSummaryDescriptor appSummaryDescriptor; + + public ObjectNode updateReportRecord(ObjectNode objectNode) { + return appSummaryDescriptor.updateReportRecord( + appDescriptor.updateReportRecord(objectNode) + .put("status", status.name()) + .put("reson", reason)) + ; + } +} diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCProcessedAppVersionDescriptor.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCProcessedAppVersionDescriptor.java index 8a28c4715d..c63cae0b83 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCProcessedAppVersionDescriptor.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCProcessedAppVersionDescriptor.java @@ -8,19 +8,13 @@ @RequiredArgsConstructor @Data public class MspReportSSCProcessedAppVersionDescriptor { private final MspReportSSCAppVersionDescriptor appVersionDescriptor; - private final MspReportSSCAppVersionProcessingStatus status; + private final MspReportProcessingStatus status; private final String reason; - private final MspReportSSCAppVersionEntitlementSummaryDescriptor appVersionEntitlementSummaryDescriptor; - + public ObjectNode updateReportRecord(ObjectNode objectNode) { - return appVersionEntitlementSummaryDescriptor.updateReportRecord( - appVersionDescriptor.updateReportRecord(objectNode) + return appVersionDescriptor.updateReportRecord(objectNode) .put("status", status.name()) - .put("reson", reason)) + .put("reson", reason) ; } - - public static enum MspReportSSCAppVersionProcessingStatus { - success, error - } } diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCResultsGenerator.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCResultsGenerator.java index 38654c17c3..4d11bcc318 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCResultsGenerator.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/generator/ssc/MspReportSSCResultsGenerator.java @@ -4,23 +4,25 @@ import static com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppVersionAttribute.MSP_End_Customer_Name; import static com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppVersionAttribute.MSP_License_Type; -import java.util.Random; - import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fortify.cli.common.json.JsonHelper; import com.fortify.cli.ssc.entity.appversion.helper.SSCAppVersionEmbedderSupplier; import com.fortify.cli.ssc.entity.attribute_definition.domain.SSCAttributeDefinitionType; import com.fortify.cli.ssc.entity.attribute_definition.helper.SSCAttributeDefinitionHelper; import com.fortify.cli.ssc.entity.token.helper.SSCTokenConverter; import com.fortify.cli.ssc.rest.bulk.SSCBulkEmbedder; +import com.fortify.cli.ssc.rest.helper.SSCInputTransformer; import com.fortify.cli.ssc.rest.helper.SSCPagingHelper; -import com.fortify.cli.util.msp_report.collector.MspReportAppVersionCollector; +import com.fortify.cli.ssc.rest.helper.SSCPagingHelper.SSCContinueNextPageSupplier; +import com.fortify.cli.util.msp_report.collector.MspReportAppArtifactCollector; +import com.fortify.cli.util.msp_report.collector.MspReportAppArtifactCollector.MspReportArtifactCollectorState; import com.fortify.cli.util.msp_report.collector.MspReportResultsCollector; import com.fortify.cli.util.msp_report.config.MspReportSSCSourceConfig; import com.fortify.cli.util.msp_report.generator.AbstractMspReportUnirestResultsGenerator; -import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCProcessedAppVersionDescriptor.MspReportSSCAppVersionProcessingStatus; import io.micrometer.common.util.StringUtils; +import kong.unirest.HttpRequest; import kong.unirest.HttpResponse; import kong.unirest.UnirestInstance; @@ -51,8 +53,8 @@ public MspReportSSCResultsGenerator(MspReportSSCSourceConfig sourceConfig, MspRe @Override protected void generateResults() { validateSSCAttributes(); - SSCPagingHelper.pagedRequest(unirest().get("/api/v1/projectVersions?limit=100")) - .forEach(this::processAppVersionPage); + SSCPagingHelper.pagedRequest(unirest().get("/api/v1/projects?limit=100")) + .forEach(this::processAppPage); } /** @@ -68,52 +70,84 @@ private void validateSSCAttributes() { .check(false, SSCAttributeDefinitionType.TEXT); } - /** - * This method processes a single page of application versions, - * embedding additional data using {@link #appVersionBulkEmbedder} - * and then invoking {@link #processAppVersion(JsonNode)} for each - * application version. - */ - private void processAppVersionPage(HttpResponse response) { - appVersionBulkEmbedder.transformInput(unirest(), response.getBody()) - .forEach(this::processAppVersion); + private void processAppPage(HttpResponse response) { + var apps = ((ArrayNode)SSCInputTransformer.getDataOrSelf(response.getBody())); + JsonHelper.stream(apps).forEach(this::processApp); } - /** - * This method handles a single application version represented by - * the given {@link JsonNode}. The node will be converted to an - * {@link MspReportSSCAppVersionDescriptor} instance, which is passed - * to the {@link #processAppVersion(MspReportSSCAppVersionDescriptor)} - * method for further processing. Based on processing state, a new - * {@link MspReportSSCProcessedAppVersionDescriptor} instance will - * then be created and passed to the {@link MspReportAppVersionCollector} - * provided by the configured {@link MspReportResultsCollector}. - */ - private void processAppVersion(JsonNode appVersionNode) { - var descriptor = JsonHelper.treeToValue(appVersionNode, MspReportSSCAppVersionDescriptor.class); + private void processApp(JsonNode appNode) { + var descriptor = JsonHelper.treeToValue(appNode, MspReportSSCAppDescriptor.class); + resultsCollector().progressHelper().writeI18nProgress("processing.app", descriptor.getName()); try { - var summary = processAppVersion(descriptor.check()); - resultsCollector().appVersionCollector().report(sourceConfig(), new MspReportSSCProcessedAppVersionDescriptor(descriptor, MspReportSSCAppVersionProcessingStatus.success, "Successfully processed", summary)); + loadVersionsForApp(descriptor); + descriptor.check(resultsCollector().logger()); + var summary = processApp(descriptor); + var status = descriptor.getWarnCounter().getCount()>0 + ? MspReportProcessingStatus.warn + : MspReportProcessingStatus.success; + var reason = status==MspReportProcessingStatus.warn + ? "Processed with warnings" + : "Successfully processed"; + resultsCollector().appCollector() + .report(sourceConfig(), new MspReportSSCProcessedAppDescriptor(descriptor, status, reason, summary)); } catch ( Exception e ) { - resultsCollector().errorWriter().addReportError("Error loading data for application version "+descriptor.getAppAndVersionName(), e); - resultsCollector().appVersionCollector().report(sourceConfig(), new MspReportSSCProcessedAppVersionDescriptor(descriptor, MspReportSSCAppVersionProcessingStatus.error, e.getMessage(), createSummaryDescriptor(descriptor))); + resultsCollector().logger().error("Error Processing application %s", e, descriptor.getName()); + resultsCollector().appCollector() + .report(sourceConfig(), new MspReportSSCProcessedAppDescriptor(descriptor, MspReportProcessingStatus.error, e.getMessage(), new MspReportSSCAppSummaryDescriptor())); } } + + private void loadVersionsForApp(MspReportSSCAppDescriptor descriptor) { + SSCPagingHelper.pagedRequest( + unirest().get("/api/v1/projects/{id}/versions?limit=100") + .routeParam("id", descriptor.getId())) + .forEach(r->loadAppVersionPage(descriptor, r.getBody())); + } + + private void loadAppVersionPage(MspReportSSCAppDescriptor appDescriptor, JsonNode body) { + var appVersions = appVersionBulkEmbedder.transformInput(unirest(), body); + JsonHelper.stream(appVersions) + .map(node->JsonHelper.treeToValue(node, MspReportSSCAppVersionDescriptor.class)) + .forEach(versionDescriptor -> appDescriptor.addVersionDescriptor(resultsCollector().logger(), versionDescriptor)); + } - private MspReportSSCAppVersionEntitlementSummaryDescriptor processAppVersion(MspReportSSCAppVersionDescriptor descriptor) { - var result = createSummaryDescriptor(descriptor); - // TODO - Load artifacts within reporting period - // - Increase count for every artifact found - // - Write artifact details to CSV file - int nrOfScans = new Random().nextInt(500); - for ( int i=0 ; iprocessAppVersion(versionDescriptor, artifactCollector)); + return artifactCollector.summary(); + } + } + + private void processAppVersion(MspReportSSCAppVersionDescriptor descriptor, MspReportAppArtifactCollector artifactCollector) { + try { + var continueNextPageSupplier = new SSCContinueNextPageSupplier(); + HttpRequest req = unirest().get("/api/v1/projectVersions/{pvId}/artifacts?limit=100") + .routeParam("pvId", descriptor.getVersionId()); + SSCPagingHelper.pagedRequest(req, continueNextPageSupplier) + .forEach(r->processArtifactPage(r.getBody(), artifactCollector, continueNextPageSupplier)); + resultsCollector().appVersionCollector() + .report(sourceConfig(), new MspReportSSCProcessedAppVersionDescriptor(descriptor, MspReportProcessingStatus.success, "Successfully processed")); + } catch ( Exception e ) { + resultsCollector().logger().error("Error loading artifacts for application version %s", e, descriptor.getAppAndVersionName()); + resultsCollector().appVersionCollector() + .report(sourceConfig(), new MspReportSSCProcessedAppVersionDescriptor(descriptor, MspReportProcessingStatus.error, e.getMessage())); + throw e; } - return result; } - private MspReportSSCAppVersionEntitlementSummaryDescriptor createSummaryDescriptor(MspReportSSCAppVersionDescriptor descriptor) { - return new MspReportSSCAppVersionEntitlementSummaryDescriptor(descriptor.getMspLicenseType()); + private void processArtifactPage(JsonNode body, MspReportAppArtifactCollector artifactCollector, SSCContinueNextPageSupplier continueNextPageSupplier) { + var done = JsonHelper.stream((ArrayNode)SSCInputTransformer.getDataOrSelf(body)) + .map(this::createArtifactDescriptor) + .map(artifactCollector::report) + .filter(MspReportArtifactCollectorState.DONE::equals) + .findFirst() + .isPresent(); + continueNextPageSupplier.setLoadNextPage(!done); + } + + private MspReportSSCArtifactDescriptor createArtifactDescriptor(JsonNode artifactNode) { + return JsonHelper.treeToValue(artifactNode, MspReportSSCArtifactDescriptor.class); } /** diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/IMspReportAppArtifactsWriter.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/IMspReportAppArtifactsWriter.java new file mode 100644 index 0000000000..b85071cf79 --- /dev/null +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/IMspReportAppArtifactsWriter.java @@ -0,0 +1,17 @@ +package com.fortify.cli.util.msp_report.writer; + +import java.time.LocalDateTime; + +import com.fortify.cli.common.rest.unirest.config.IUrlConfig; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppDescriptor; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCArtifactDescriptor; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCArtifactDescriptor.MspReportSSCArtifactScanType; + +public interface IMspReportAppArtifactsWriter { + void write(IUrlConfig urlConfig, + MspReportSSCAppDescriptor appDescriptor, + MspReportSSCArtifactDescriptor artifactDescriptor, + MspReportSSCArtifactScanType entitlementScanType, + boolean entitlementConsumed, + LocalDateTime entitlementConsumptionDate); +} \ No newline at end of file diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/IMspReportAppsWriter.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/IMspReportAppsWriter.java new file mode 100644 index 0000000000..ab058fb110 --- /dev/null +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/IMspReportAppsWriter.java @@ -0,0 +1,8 @@ +package com.fortify.cli.util.msp_report.writer; + +import com.fortify.cli.common.rest.unirest.config.IUrlConfig; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCProcessedAppDescriptor; + +public interface IMspReportAppsWriter { + void write(IUrlConfig urlConfig, MspReportSSCProcessedAppDescriptor descriptor); +} \ No newline at end of file diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/MspReportAppArtifactsWriter.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/MspReportAppArtifactsWriter.java new file mode 100644 index 0000000000..9f817c074c --- /dev/null +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/MspReportAppArtifactsWriter.java @@ -0,0 +1,37 @@ +package com.fortify.cli.util.msp_report.writer; + +import java.time.LocalDateTime; + +import com.fortify.cli.common.json.JsonHelper; +import com.fortify.cli.common.output.OutputFormat; +import com.fortify.cli.common.output.writer.record.IRecordWriter; +import com.fortify.cli.common.report.writer.IReportWriter; +import com.fortify.cli.common.rest.unirest.config.IUrlConfig; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCAppDescriptor; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCArtifactDescriptor; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCArtifactDescriptor.MspReportSSCArtifactScanType; + +public final class MspReportAppArtifactsWriter implements IMspReportAppArtifactsWriter { + private final IRecordWriter recordWriter; + + public MspReportAppArtifactsWriter(IReportWriter reportWriter) { + this.recordWriter = reportWriter.recordWriter(OutputFormat.csv, "details/artifacts.csv", false, null); + } + + @Override + public void write(IUrlConfig urlConfig, + MspReportSSCAppDescriptor appDescriptor, + MspReportSSCArtifactDescriptor artifactDescriptor, + MspReportSSCArtifactScanType entitlementScanType, + boolean entitlementConsumed, + LocalDateTime entitlementConsumptionDate) { + recordWriter.writeRecord( + artifactDescriptor.updateReportRecord( + appDescriptor.updateReportRecord( + JsonHelper.getObjectMapper().createObjectNode() + .put("url", urlConfig.getUrl()))) + .put("entitlementScanType", entitlementScanType.name()) + .put("entitlementConsumed", entitlementConsumed) + .put("entitlementConsumptionDate", entitlementConsumptionDate==null?"N/A":entitlementConsumptionDate.toString())); + } +} diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/MspReportAppsWriter.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/MspReportAppsWriter.java new file mode 100644 index 0000000000..a67fecd3e7 --- /dev/null +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/MspReportAppsWriter.java @@ -0,0 +1,25 @@ +package com.fortify.cli.util.msp_report.writer; + +import com.fortify.cli.common.json.JsonHelper; +import com.fortify.cli.common.output.OutputFormat; +import com.fortify.cli.common.output.writer.record.IRecordWriter; +import com.fortify.cli.common.report.writer.IReportWriter; +import com.fortify.cli.common.rest.unirest.config.IUrlConfig; +import com.fortify.cli.util.msp_report.generator.ssc.MspReportSSCProcessedAppDescriptor; + +public final class MspReportAppsWriter implements IMspReportAppsWriter { + private final IRecordWriter recordWriter; + + public MspReportAppsWriter(IReportWriter reportWriter) { + this.recordWriter = reportWriter.recordWriter(OutputFormat.csv, "details/applications.csv", false, null); + } + + @Override + public void write(IUrlConfig urlConfig, MspReportSSCProcessedAppDescriptor descriptor) { + recordWriter.writeRecord( + descriptor.updateReportRecord( + JsonHelper.getObjectMapper().createObjectNode() + .put("url", urlConfig.getUrl()))); + + } +} diff --git a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/MspReportResultsWriters.java b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/MspReportResultsWriters.java index b3a264cc10..8b3383a97c 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/MspReportResultsWriters.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/msp_report/writer/MspReportResultsWriters.java @@ -1,9 +1,9 @@ package com.fortify.cli.util.msp_report.writer; import com.fortify.cli.common.progress.helper.IProgressHelperI18n; +import com.fortify.cli.common.report.logger.IReportLogger; +import com.fortify.cli.common.report.logger.ReportLogger; import com.fortify.cli.common.report.writer.IReportWriter; -import com.fortify.cli.common.report.writer.entry.IReportErrorEntryWriter; -import com.fortify.cli.common.report.writer.entry.ReportErrorEntryWriter; import lombok.Getter; import lombok.experimental.Accessors; @@ -17,12 +17,16 @@ @Getter @Accessors(fluent=true) public class MspReportResultsWriters { private final IProgressHelperI18n progressHelper; - private final IReportErrorEntryWriter errorWriter; - private final IMspReportAppVersionsWriter appVersionWriter; + private final IReportLogger logger; + private final IMspReportAppsWriter appsWriter; + private final IMspReportAppVersionsWriter appVersionsWriter; + private final IMspReportAppArtifactsWriter artifactsWriter; public MspReportResultsWriters(IReportWriter reportWriter, IProgressHelperI18n progressHelper) { this.progressHelper = progressHelper; - this.errorWriter = new ReportErrorEntryWriter(reportWriter, progressHelper); - this.appVersionWriter = new MspReportAppVersionsWriter(reportWriter); + this.logger = new ReportLogger(reportWriter, progressHelper); + this.appsWriter = new MspReportAppsWriter(reportWriter); + this.appVersionsWriter = new MspReportAppVersionsWriter(reportWriter); + this.artifactsWriter = new MspReportAppArtifactsWriter(reportWriter); } } diff --git a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/collector/NcdReportRepositoryCollector.java b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/collector/NcdReportRepositoryCollector.java index 02ee4a85b3..b0d8990855 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/collector/NcdReportRepositoryCollector.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/collector/NcdReportRepositoryCollector.java @@ -40,7 +40,7 @@ void reportRepository(INcdReportRepositoryDescriptor descriptor, NcdReportReposi void reportRepositoryError(INcdReportRepositoryDescriptor descriptor, Exception e) { // TODO Log error - writers.errorWriter().addReportError("Error loading repository: "+descriptor.getUrl(), e); + writers.logger().error("Error loading repository: "+descriptor.getUrl(), e); reportRepository(descriptor, NcdReportRepositoryReportingStatus.error, e.getMessage()); } diff --git a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/collector/NcdReportResultsCollector.java b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/collector/NcdReportResultsCollector.java index 815b54fb1f..2348dc68db 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/collector/NcdReportResultsCollector.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/collector/NcdReportResultsCollector.java @@ -2,9 +2,9 @@ import com.fortify.cli.common.progress.helper.IProgressHelperI18n; import com.fortify.cli.common.report.collector.IReportResultsCollector; +import com.fortify.cli.common.report.logger.IReportLogger; +import com.fortify.cli.common.report.logger.ReportLogger; import com.fortify.cli.common.report.writer.IReportWriter; -import com.fortify.cli.common.report.writer.entry.IReportErrorEntryWriter; -import com.fortify.cli.common.report.writer.entry.ReportErrorEntryWriter; import com.fortify.cli.util.ncd_report.cli.cmd.NcdReportGenerateCommand; import com.fortify.cli.util.ncd_report.config.NcdReportConfig; import com.fortify.cli.util.ncd_report.writer.NcdReportResultsWriters; @@ -17,7 +17,7 @@ * This class is the primary entry point for collecting and outputting report data. * An instance of this class is created by the {@link NcdReportGenerateCommand} * and passed to the source-specific generators. Source-specific generators can use - * this class to access the {@link IReportErrorEntryWriter} and + * this class to access the {@link IReportLogger} and * {@link NcdReportRepositoryProcessor} instances. * * @author rsenden @@ -40,12 +40,12 @@ public NcdReportResultsCollector(NcdReportConfig reportConfig, IReportWriter rep } /** - * We provide public access to {@link ReportErrorEntryWriter}, all + * We provide public access to {@link ReportLogger}, all * other writers are for internal use by this class only. * @return */ - public final IReportErrorEntryWriter errorWriter() { - return writers.errorWriter(); + public final IReportLogger logger() { + return writers.logger(); } public INcdReportRepositoryProcessor repositoryProcessor() { @@ -54,7 +54,7 @@ public INcdReportRepositoryProcessor repositoryProcessor() { @Override @SneakyThrows public void close() { - reportWriter.summary().put("errorCount", errorWriter().getErrorCount()); repositoryProcessor.writeResults(); + logger().updateSummary(reportWriter.summary()); } } diff --git a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/github/GitHubPagingHelper.java b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/github/GitHubPagingHelper.java index 4b77f67dd8..823d717839 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/github/GitHubPagingHelper.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/github/GitHubPagingHelper.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fortify.cli.common.rest.paging.INextPageUrlProducer; -import com.fortify.cli.common.rest.paging.PagingHelper; +import com.fortify.cli.common.rest.paging.LinkHeaderPagingHelper; import kong.unirest.HttpRequest; import kong.unirest.PagedList; @@ -16,10 +16,10 @@ public class GitHubPagingHelper { private GitHubPagingHelper() {} public static final PagedList pagedRequest(HttpRequest request, Class returnType) { - return PagingHelper.pagedRequest(request, nextPageUrlProducer(), returnType); + return LinkHeaderPagingHelper.pagedRequest(request, nextPageUrlProducer(), returnType); } public static final INextPageUrlProducer nextPageUrlProducer() { - return PagingHelper.linkHeaderNextPageUrlProducer("Link", "next"); + return LinkHeaderPagingHelper.linkHeaderNextPageUrlProducer("Link", "next"); } } diff --git a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/github/NcdReportGitHubResultsGenerator.java b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/github/NcdReportGitHubResultsGenerator.java index c93e835a58..1deba299c6 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/github/NcdReportGitHubResultsGenerator.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/github/NcdReportGitHubResultsGenerator.java @@ -68,7 +68,7 @@ private void generateResults(NcdReportGitHubOrganizationConfig orgConfig) { .ifSuccess(r->r.getBody().forEach(repo-> resultsCollector().repositoryProcessor().processRepository(new NcdReportCombinedRepoSelectorConfig(sourceConfig(), orgConfig), getRepoDescriptor(repo), this::generateCommitData))); } catch ( Exception e ) { - resultsCollector().errorWriter().addReportError(String.format("Error processing organization: %s (%s)", orgName, sourceConfig().getApiUrl()), e); + resultsCollector().logger().error(String.format("Error processing organization: %s (%s)", orgName, sourceConfig().getApiUrl()), e); } } diff --git a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/gitlab/GitLabPagingHelper.java b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/gitlab/GitLabPagingHelper.java index 9f62d3e34c..808e4a2f81 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/gitlab/GitLabPagingHelper.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/gitlab/GitLabPagingHelper.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fortify.cli.common.rest.paging.INextPageUrlProducer; -import com.fortify.cli.common.rest.paging.PagingHelper; +import com.fortify.cli.common.rest.paging.LinkHeaderPagingHelper; import kong.unirest.HttpRequest; import kong.unirest.PagedList; @@ -16,10 +16,10 @@ public class GitLabPagingHelper { private GitLabPagingHelper() {} public static final PagedList pagedRequest(HttpRequest request, Class returnType) { - return PagingHelper.pagedRequest(request, nextPageUrlProducer(), returnType); + return LinkHeaderPagingHelper.pagedRequest(request, nextPageUrlProducer(), returnType); } public static final INextPageUrlProducer nextPageUrlProducer() { - return PagingHelper.linkHeaderNextPageUrlProducer("link", "next"); + return LinkHeaderPagingHelper.linkHeaderNextPageUrlProducer("link", "next"); } } diff --git a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/gitlab/NcdReportGitLabResultsGenerator.java b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/gitlab/NcdReportGitLabResultsGenerator.java index 53d526b8c3..a63e796b95 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/gitlab/NcdReportGitLabResultsGenerator.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/generator/gitlab/NcdReportGitLabResultsGenerator.java @@ -74,7 +74,7 @@ private void generateResults(NcdReportGitLabGroupConfig groupConfig) { .ifSuccess(r->r.getBody().forEach(project-> resultsCollector().repositoryProcessor().processRepository(new NcdReportCombinedRepoSelectorConfig(sourceConfig(), groupConfig), getRepoDescriptor(project), this::generateCommitData))); } catch ( Exception e ) { - resultsCollector().errorWriter().addReportError(String.format("Error processing group: %s (%s)", groupId, sourceConfig().getBaseUrl()), e); + resultsCollector().logger().error(String.format("Error processing group: %s (%s)", groupId, sourceConfig().getBaseUrl()), e); } } diff --git a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/writer/NcdReportResultsWriters.java b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/writer/NcdReportResultsWriters.java index 1473a9ac30..557ab231de 100644 --- a/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/writer/NcdReportResultsWriters.java +++ b/fcli-util/src/main/java/com/fortify/cli/util/ncd_report/writer/NcdReportResultsWriters.java @@ -1,9 +1,9 @@ package com.fortify.cli.util.ncd_report.writer; import com.fortify.cli.common.progress.helper.IProgressHelperI18n; +import com.fortify.cli.common.report.logger.IReportLogger; +import com.fortify.cli.common.report.logger.ReportLogger; import com.fortify.cli.common.report.writer.IReportWriter; -import com.fortify.cli.common.report.writer.entry.IReportErrorEntryWriter; -import com.fortify.cli.common.report.writer.entry.ReportErrorEntryWriter; import lombok.Getter; import lombok.experimental.Accessors; @@ -17,7 +17,7 @@ @Getter @Accessors(fluent=true) public class NcdReportResultsWriters { private final IProgressHelperI18n progressHelper; - private final IReportErrorEntryWriter errorWriter; + private final IReportLogger logger; private final INcdReportRepositoriesWriter repositoryWriter; private final INcdReportCommitsByBranchWriter commitsByBranchWriter; private final INcdReportCommitsByRepositoryWriter commitsByRepositoryWriter; @@ -26,7 +26,7 @@ public class NcdReportResultsWriters { public NcdReportResultsWriters(IReportWriter reportWriter, IProgressHelperI18n progressHelper) { this.progressHelper = progressHelper; - this.errorWriter = new ReportErrorEntryWriter(reportWriter, progressHelper); + this.logger = new ReportLogger(reportWriter, progressHelper); this.repositoryWriter = new NcdReportRepositoriesWriter(reportWriter); this.commitsByBranchWriter = new NcdReportCommitsByBranchWriter(reportWriter); this.commitsByRepositoryWriter = new NcdReportCommitsByRepositoryWriter(reportWriter); diff --git a/fcli-util/src/main/resources/com/fortify/cli/util/i18n/UtilMessages.properties b/fcli-util/src/main/resources/com/fortify/cli/util/i18n/UtilMessages.properties index a26f725a60..39a7d92777 100644 --- a/fcli-util/src/main/resources/com/fortify/cli/util/i18n/UtilMessages.properties +++ b/fcli-util/src/main/resources/com/fortify/cli/util/i18n/UtilMessages.properties @@ -32,6 +32,9 @@ fcli.util.msp-report.generate.start-date = Reporting period start date. Format: fcli.util.msp-report.generate.end-date = Reporting period start date. Format: yyyy-MM-dd, for example 2023-03-31 fcli.util.msp-report.generate.confirm = Confirm delete of existing report output location fcli.util.msp-report.generate.confirmPrompt = Confirm delete of existing output location %s? +fcli.util.msp-report.generate.processing.app = Processing application %s +fcli.util.msp-report.generate.processing.appversion = Processing application version %s +fcli.util.msp-report.generate.warn.usingUploadDate = WARN: No scan date found for artifact id %s (%s), using upload date instead fcli.util.msp-report.generate-config.usage.header = Generate a sample configuration file for use by the 'generate' command fcli.util.msp-report.generate-config.config = Name of the sample configuration file to be generated