chore: Refactoring, complete MSP report implementation

feat: Add `fcli util msp-report` commands

fcli-common/src/main/java/com/fortify/cli/common/output/writer/output/standard/StandardOutputWriter.java

@@ -25,7 +25,7 @@
 import com.fortify.cli.common.output.writer.record.RecordWriterConfig;
 import com.fortify.cli.common.output.writer.record.RecordWriterConfig.RecordWriterConfigBuilder;
 import com.fortify.cli.common.rest.paging.INextPageUrlProducer;
-import com.fortify.cli.common.rest.paging.PagingHelper;
+import com.fortify.cli.common.rest.paging.LinkHeaderPagingHelper;
 import com.fortify.cli.common.rest.unirest.IfFailureHandler;
 import com.fortify.cli.common.util.PicocliSpecHelper;
 import com.fortify.cli.common.variable.DefaultVariablePropertyName;
@@ -123,7 +123,7 @@ private final void writeRecords(IRecordWriter recordWriter, HttpRequest<?> httpR
      * @param nextPageUrlProducer
      */
     private final void writeRecords(IRecordWriter recordWriter, HttpRequest<?> httpRequest, INextPageUrlProducer nextPageUrlProducer) {
-        PagingHelper.pagedRequest(httpRequest, nextPageUrlProducer)
+        LinkHeaderPagingHelper.pagedRequest(httpRequest, nextPageUrlProducer)
             .ifSuccess(r->writeRecords(recordWriter, r))
             .ifFailure(IfFailureHandler::handle); // Just in case no error interceptor was registered for this request
     }

fcli-common/src/main/java/com/fortify/cli/common/report/collector/IReportResultsCollector.java

@@ -1,9 +1,9 @@
 package com.fortify.cli.common.report.collector;
 
-import com.fortify.cli.common.report.writer.entry.IReportErrorEntryWriter;
+import com.fortify.cli.common.report.logger.IReportLogger;
 
 public interface IReportResultsCollector extends AutoCloseable {
-    IReportErrorEntryWriter errorWriter();
+    IReportLogger logger();
     /**
      * Override default close method to not throw any exception.
      */

fcli-common/src/main/java/com/fortify/cli/common/report/generator/AbstractReportUnirestResultsGenerator.java

@@ -73,7 +73,7 @@ private final UnirestInstance createUnirestInstance() {
      * report output. 
      */
     private final void handleSourceError(Exception e) {
-        resultsCollector().errorWriter().addReportError(String.format("Error processing %s source: %s", getType(), sourceConfig().getUrl()), e);
+        resultsCollector().logger().error(String.format("Error processing %s source: %s", getType(), sourceConfig().getUrl()), e);
     }
 
     /**

fcli-common/src/main/java/com/fortify/cli/common/report/logger/IReportLogger.java

@@ -0,0 +1,11 @@
+package com.fortify.cli.common.report.logger;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public interface IReportLogger {
+    void warn(String msg, Object... args);
+    void warn(String msg, Exception e, Object... args);
+    void error(String msg, Object... args);
+    void error(String msg, Exception e, Object... args);
+    void updateSummary(ObjectNode summary);
+}

fcli-common/src/main/java/com/fortify/cli/common/report/logger/ReportLogger.java

@@ -0,0 +1,93 @@
+package com.fortify.cli.common.report.logger;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.time.LocalDateTime;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fortify.cli.common.json.JsonHelper;
+import com.fortify.cli.common.progress.helper.IProgressHelper;
+import com.fortify.cli.common.report.writer.IReportWriter;
+import com.fortify.cli.common.util.Counter;
+
+import lombok.SneakyThrows;
+
+public final class ReportLogger implements IReportLogger {
+    private final IProgressHelper progressHelper;
+    private final BufferedWriter logWriter;
+    private Counter errorCounter = new Counter();
+    private Counter warnCounter = new Counter();
+
+    public ReportLogger(IReportWriter reportWriter, IProgressHelper progressHelper) {
+        this.logWriter = reportWriter.bufferedWriter("report.log");
+        this.progressHelper = progressHelper;
+    }
+
+    @Override
+    public void updateSummary(ObjectNode summary) {
+        summary.set("logCounts", 
+                JsonHelper.getObjectMapper().createObjectNode()
+                    .put("error", errorCounter.getCount())
+                    .put("warn", warnCounter.getCount())
+                );
+    }
+
+    @Override
+    public final void warn(String msg, Object... msgArgs) {
+        write("WARN", warnCounter, msg, null, msgArgs);
+    }
+
+    @Override
+    public final void warn(String msg, Exception e, Object... msgArgs) {
+        write("WARN", warnCounter, msg, e, msgArgs);
+    }
+
+    @Override
+    public final void error(String msg, Object... msgArgs) {
+        write("ERROR", errorCounter, msg, null, msgArgs);
+    }
+
+    @Override
+    public final void error(String msg, Exception e, Object... msgArgs) {
+        // We want to fail completely in case of IOExceptions as this likely
+        // means we cannot write the report, so we rethrow as ReportWriterIOException
+        if ( e instanceof IOException ) { throw new ReportWriterIOException((IOException)e); }
+        // Rethrow previously thrown ReportWriterIOException
+        if ( e instanceof ReportWriterIOException ) { throw (ReportWriterIOException)e; }
+        write("ERROR", errorCounter, msg, e, msgArgs);
+    }
+
+    @SneakyThrows
+    private void write(String level, Counter counter, String msg, Exception e, Object[] msgArgs) {
+        counter.increase();
+        var fullMsg = msgArgs==null ? msg : String.format(msg, msgArgs);
+        if ( e!=null ) {
+            fullMsg = String.format("%s: %s: %s", fullMsg, e.getClass().getSimpleName(), e.getMessage());
+        }
+        progressHelper.writeWarning(String.format("%s: %s", level, fullMsg));
+        logWriter.append(String.format("[%s] %s %s\n", LocalDateTime.now(), level, fullMsg));
+        if ( e!=null ) {
+            logWriter.append(String.format("%s\n", toString(e)));
+        }
+    }
+
+    private String toString(Exception e) {
+        try ( var sw = new StringWriter(); var pw = new PrintWriter(sw) ) {
+            e.printStackTrace(pw);
+            return sw.toString();
+        } catch ( IOException ioe ) {
+            return e.getClass().getName()+": "+e.getMessage();
+        }
+    }
+
+    private static final class ReportWriterIOException extends IllegalStateException {
+        private static final long serialVersionUID = 1L;
+
+        public ReportWriterIOException(IOException cause) {
+            super("Error writing report; report contents may be incomplete", cause);
+        }
+    }
+
+}

fcli-common/src/main/java/com/fortify/cli/common/rest/paging/PagingHelper.java → fcli-common/src/main/java/com/fortify/cli/common/rest/paging/LinkHeaderPagingHelper.java

@@ -8,14 +8,14 @@
 import kong.unirest.HttpRequest;
 import kong.unirest.PagedList;
 
-@SuppressWarnings("unchecked") // TODO Can we get rid of these warnings in a better way?
-public class PagingHelper {
+public class LinkHeaderPagingHelper {
     private static final Pattern linkHeaderPattern = Pattern.compile("<([^>]*)>; *rel=\"([^\"]*)\"");
 
     public static final PagedList<JsonNode> pagedRequest(HttpRequest<?> request, INextPageUrlProducer nextPageUrlProducer) {
         return pagedRequest(request, nextPageUrlProducer, JsonNode.class);
     }
 
+    @SuppressWarnings("unchecked") // TODO Can we get rid of these warnings in a better way?
     public static final <R extends JsonNode> PagedList<R> pagedRequest(HttpRequest<?> request, INextPageUrlProducer nextPageUrlProducer, Class<R> returnType) {
         return request.asPaged(r->r.asObject(returnType), nextPageUrlProducer::getNextPageUrl);
     }

fcli-common/src/main/java/com/fortify/cli/common/util/Counter.java

@@ -0,0 +1,24 @@
+package com.fortify.cli.common.util;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.ToString;
+
+@ToString @EqualsAndHashCode
+public class Counter {
+    @Getter private long count = 0;
+
+    public Counter increase() {
+        count++;
+        return this;
+    }
+
+    public Counter increase(long value) {
+        count+=value;
+        return this;
+    }
+
+    public Counter increase(Counter counter) {
+        return increase(counter.getCount());
+    }
+}

fcli-common/src/main/java/com/fortify/cli/common/util/StringUtils.java

@@ -29,4 +29,10 @@ public static final String substringAfterLast(String str, String separator) {
         final int pos = str.lastIndexOf(separator);
         return pos==-1 ? "" : str.substring(pos + separator.length());
     }
+
+    public static final String capitalize(String str) {
+        return str==null 
+                ? null 
+                : str.substring(0,1).toUpperCase() + str.substring(1).toLowerCase(); 
+    }
 }

fcli-fod/src/main/java/com/fortify/cli/fod/rest/helper/FoDPagingHelper.java

@@ -5,14 +5,14 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fortify.cli.common.rest.paging.INextPageUrlProducer;
-import com.fortify.cli.common.rest.paging.PagingHelper;
+import com.fortify.cli.common.rest.paging.LinkHeaderPagingHelper;
 
 import kong.unirest.HttpRequest;
 import kong.unirest.PagedList;
 
 public class FoDPagingHelper {
     public static final PagedList<JsonNode> pagedRequest(HttpRequest<?> request) {
-        return PagingHelper.pagedRequest(request, nextPageUrlProducer(request));
+        return LinkHeaderPagingHelper.pagedRequest(request, nextPageUrlProducer(request));
     }
     public static final INextPageUrlProducer nextPageUrlProducer(HttpRequest<?> originalRequest) {
         return nextPageUrlProducer(originalRequest.getUrl());

fcli-sc-dast/src/main/java/com/fortify/cli/sc_dast/rest/helper/SCDastPagingHelper.java

@@ -4,15 +4,15 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fortify.cli.common.rest.paging.INextPageUrlProducer;
-import com.fortify.cli.common.rest.paging.PagingHelper;
+import com.fortify.cli.common.rest.paging.LinkHeaderPagingHelper;
 
 import io.micronaut.http.uri.UriBuilder;
 import kong.unirest.HttpRequest;
 import kong.unirest.PagedList;
 
 public class SCDastPagingHelper {
     public static final PagedList<JsonNode> pagedRequest(HttpRequest<?> request) {
-        return PagingHelper.pagedRequest(request, nextPageUrlProducer(request));
+        return LinkHeaderPagingHelper.pagedRequest(request, nextPageUrlProducer(request));
     }
     public static final INextPageUrlProducer nextPageUrlProducer(HttpRequest<?> originalRequest) {
         return nextPageUrlProducer(originalRequest.getUrl());

fcli-ssc/src/main/java/com/fortify/cli/ssc/rest/bulk/SSCBulkEmbedder.java

@@ -5,7 +5,7 @@
 import java.util.stream.Stream;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fortify.cli.common.json.JsonHelper;
 import com.fortify.cli.ssc.rest.helper.SSCInputTransformer;
 
@@ -30,11 +30,11 @@ public SSCBulkEmbedder(ISSCEntityEmbedderSupplier... suppliers) {
                 .collect(Collectors.toList());
     }
 
-    public JsonNode transformInput(UnirestInstance unirest, JsonNode input) {
-        var records = SSCInputTransformer.getDataOrSelf(input);
-        if ( records instanceof ObjectNode ) {
-            records = JsonHelper.toArrayNode(records);
-        }
+    public ArrayNode transformInput(UnirestInstance unirest, JsonNode input) {
+        var data = SSCInputTransformer.getDataOrSelf(input);
+        var records = data instanceof ArrayNode 
+                ? (ArrayNode) data
+                : JsonHelper.toArrayNode(data);
         if ( embedders!=null ) {
             SSCBulkRequestBuilder builder = new SSCBulkRequestBuilder();
             for ( var record : records ) {

fcli-ssc/src/main/java/com/fortify/cli/ssc/rest/helper/SSCPagingHelper.java

@@ -5,20 +5,22 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fortify.cli.common.json.JsonHelper;
 import com.fortify.cli.common.rest.paging.INextPageUrlProducer;
-import com.fortify.cli.common.rest.paging.PagingHelper;
+import com.fortify.cli.common.rest.paging.LinkHeaderPagingHelper;
 
 import kong.unirest.HttpRequest;
 import kong.unirest.PagedList;
+import lombok.Setter;
 
 public class SSCPagingHelper {
+    private static final SSCContinueNextPageSupplier continueNextPageSupplier = new SSCContinueNextPageSupplier();
     public static final PagedList<JsonNode> pagedRequest(HttpRequest<?> request) {
-        return pagedRequest(request, SSCPagingHelper::getTrue);
+        return pagedRequest(request, continueNextPageSupplier);
     }
     public static final PagedList<JsonNode> pagedRequest(HttpRequest<?> request, Supplier<Boolean> continueSupplier) {
-        return PagingHelper.pagedRequest(request, nextPageUrlProducer(continueSupplier));
+        return LinkHeaderPagingHelper.pagedRequest(request, nextPageUrlProducer(continueSupplier));
     }
     public static final INextPageUrlProducer nextPageUrlProducer() {
-        return nextPageUrlProducer(SSCPagingHelper::getTrue);
+        return nextPageUrlProducer(continueNextPageSupplier);
     }
     public static final INextPageUrlProducer nextPageUrlProducer(Supplier<Boolean> continueSupplier) {
         return r -> {
@@ -31,5 +33,9 @@ public static final INextPageUrlProducer nextPageUrlProducer(Supplier<Boolean> c
             return null;
         };
     }
-    private static final boolean getTrue() { return true; }
+    public static final class SSCContinueNextPageSupplier implements Supplier<Boolean> {
+        @Setter private boolean loadNextPage = true;
+        @Override
+        public Boolean get() { return loadNextPage; }
+    }
 }