Maven-like rolling output when the build happens to be linear #269

common/src/main/java/org/mvndaemon/mvnd/common/logging/TerminalOutput.java

@@ -173,6 +173,11 @@ private boolean doAccept(Message entry) {
             this.maxThreads = bs.getMaxThreads();
             final int maxThreadsDigits = (int) (Math.log10(maxThreads) + 1);
             this.threadsFormat = "%" + (maxThreadsDigits * 3 + 2) + "s";
+            if (maxThreads == 1) {
+                this.noBuffering = true;
+                display.update(Collections.emptyList(), 0);
+                applyNoBuffering();
+            }
             break;
         }
         case Message.CANCEL_BUILD: {
@@ -322,8 +327,7 @@ private boolean doAccept(Message entry) {
             case CTRL_B:
                 noBuffering = !noBuffering;
                 if (noBuffering) {
-                    projects.values().stream().flatMap(p -> p.log.stream()).forEach(log);
-                    projects.clear();
+                    applyNoBuffering();
                 } else {
                     clearDisplay();
                 }
@@ -345,6 +349,11 @@ private boolean doAccept(Message entry) {
         return true;
     }
 
+    private void applyNoBuffering() {
+        projects.values().stream().flatMap(p -> p.log.stream()).forEach(log);
+        projects.clear();
+    }
+
     @Override
     public void describeTerminal() {
         StringBuilder sb = new StringBuilder();

daemon/src/main/java/org/mvndaemon/mvnd/builder/DependencyGraph.java

@@ -15,28 +15,139 @@
  */
 package org.mvndaemon.mvnd.builder;
 
+import groovy.lang.Binding;
+import groovy.lang.GroovyShell;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import org.apache.maven.execution.MavenSession;
 import org.apache.maven.execution.ProjectDependencyGraph;
 import org.apache.maven.project.MavenProject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * File origin:
  * https://github.com/takari/takari-smart-builder/blob/takari-smart-builder-0.6.1/src/main/java/io/takari/maven/builder/smart/DependencyGraph.java
  */
-interface DependencyGraph<K> {
+public class DependencyGraph implements ProjectDependencyGraph {
 
-    static DependencyGraph<MavenProject> fromMaven(ProjectDependencyGraph graph, String rules) {
-        List<MavenProject> projects = graph.getSortedProjects();
+    private static final Logger logger = LoggerFactory.getLogger(DependencyGraph.class);
+    static final Pattern mvndRuleSanitizerPattern = Pattern.compile("[,\\s]+");
+
+    private final List<MavenProject> sortedProjects;
+    private final Map<MavenProject, List<MavenProject>> upstreams;
+    private final Map<MavenProject, List<MavenProject>> downstreams;
+    private final ProjectDependencyGraph delegate;
+    private final boolean serial;
+
+    public static DependencyGraph fromMaven(MavenSession session) {
+
+        List<String> list = new ArrayList<>();
+
+        String providerScript = null;
+        String providerUrl = session.getTopLevelProject().getProperties()
+                .getProperty(SmartBuilder.MVND_BUILDER_RULES_PROVIDER_URL);
+        if (providerUrl != null) {
+            logger.warn(SmartBuilder.MVND_BUILDER_RULES_PROVIDER_URL
+                    + " property is deprecated and the support for it will be removed in mvnd 0.3. See https://github.com/mvndaemon/mvnd/issues/264");
+
+            URL url;
+            try {
+                url = new URL(providerUrl);
+            } catch (MalformedURLException e) {
+                try {
+                    url = new File(providerUrl).toURI().toURL();
+                } catch (MalformedURLException ex) {
+                    url = null;
+                }
+            }
+            if (url == null) {
+                throw new RuntimeException("Bad syntax for " + SmartBuilder.MVND_BUILDER_RULES_PROVIDER_URL, null);
+            }
+            try (BufferedReader r = new BufferedReader(new InputStreamReader(url.openStream()))) {
+                StringBuilder sb = new StringBuilder();
+                char[] buf = new char[8192];
+                int l;
+                while ((l = r.read(buf)) >= 0) {
+                    sb.append(buf, 0, l);
+                }
+                providerScript = sb.toString();
+            } catch (IOException e) {
+                throw new RuntimeException("Unable to read provider url " + SmartBuilder.MVND_BUILDER_RULES_PROVIDER_URL,
+                        e);
+            }
+        }
+        if (providerScript == null) {
+            providerScript = session.getTopLevelProject().getProperties()
+                    .getProperty(SmartBuilder.MVND_BUILDER_RULES_PROVIDER_SCRIPT);
+        }
+        if (providerScript != null) {
+            logger.warn(SmartBuilder.MVND_BUILDER_RULES_PROVIDER_SCRIPT
+                    + " property is deprecated and the support for it will be removed in mvnd 0.3. See https://github.com/mvndaemon/mvnd/issues/264");
+
+            Binding binding = new Binding();
+            GroovyShell shell = new GroovyShell(binding);
+            binding.setProperty("session", session);
+            Object result = shell.evaluate(providerScript);
+            if (result instanceof Iterable) {
+                for (Object r : (Iterable) result) {
+                    list.add(r.toString());
+                }
+            } else if (result != null) {
+                list.add(result.toString());
+            } else {
+                throw new RuntimeException("The provider script did not return a valid string or string collection", null);
+            }
+            list.add(result.toString());
+        }
+
+        String topRule = session.getTopLevelProject().getProperties().getProperty(SmartBuilder.MVND_BUILDER_RULES);
+        if (topRule != null) {
+            logger.warn(SmartBuilder.MVND_BUILDER_RULES
+                    + " property is deprecated and the support for it will be removed in mvnd 0.3. See https://github.com/mvndaemon/mvnd/issues/264");
+            list.add(topRule);
+        }
+
+        session.getAllProjects().forEach(p -> {
+            String rule = p.getProperties().getProperty(SmartBuilder.MVND_BUILDER_RULE);
+            if (rule != null) {
+                logger.warn(SmartBuilder.MVND_BUILDER_RULE
+                        + " property is deprecated and the support for it will be removed in mvnd 0.3. See https://github.com/mvndaemon/mvnd/issues/264");
+                rule = rule.trim();
+                if (!rule.isEmpty()) {
+                    rule = mvndRuleSanitizerPattern.matcher(rule).replaceAll(",");
+                    list.add(rule + " before " + p.getGroupId() + ":" + p.getArtifactId());
+                }
+            }
+        });
+        String rules = null;
+        if (!list.isEmpty()) {
+            rules = String.join("\n", list);
+        }
+
+        return fromMaven(session.getProjectDependencyGraph(), rules);
+    }
+
+    static DependencyGraph fromMaven(ProjectDependencyGraph graph, String rules) {
+        final List<MavenProject> projects = graph.getSortedProjects();
         Map<MavenProject, List<MavenProject>> upstreams = projects.stream()
                 .collect(Collectors.toMap(p -> p, p -> graph.getUpstreamProjects(p, false)));
         Map<MavenProject, List<MavenProject>> downstreams = projects.stream()
-                .collect(Collectors.toMap(p -> p, p -> graph.getDownstreamProjects(p, false)));
+                .collect(
+                        Collectors.toMap(p -> p, p -> graph.getDownstreamProjects(p, false)));
 
         if (rules != null) {
             for (String rule : rules.split("\\s*;\\s*|\n")) {
@@ -80,35 +191,56 @@ static DependencyGraph<MavenProject> fromMaven(ProjectDependencyGraph graph, Str
                 deps.get(0).forEach(p -> downstreams.get(p).addAll(deps.get(1)));
             }
         }
-        return new DependencyGraph<MavenProject>() {
-            @Override
-            public Stream<MavenProject> getDownstreamProjects(MavenProject project) {
-                return downstreams.get(project).stream();
-            }
+        final boolean serial = !projects.stream().anyMatch(p -> downstreams.get(p).size() > 1);
+        return new DependencyGraph(Collections.unmodifiableList(projects), upstreams, downstreams, serial, graph);
+    }
 
-            @Override
-            public Stream<MavenProject> getProjects() {
-                return projects.stream();
-            }
+    public DependencyGraph(List<MavenProject> sortedProjects, Map<MavenProject, List<MavenProject>> upstreams,
+            Map<MavenProject, List<MavenProject>> downstreams, boolean serial, ProjectDependencyGraph delegate) {
+        this.sortedProjects = sortedProjects;
+        this.upstreams = upstreams;
+        this.downstreams = downstreams;
+        this.serial = serial;
+        this.delegate = delegate;
+    }
 
-            @Override
-            public Stream<MavenProject> getUpstreamProjects(MavenProject project) {
-                return upstreams.get(project).stream();
-            }
+    public Stream<MavenProject> getDownstreamProjects(MavenProject project) {
+        return downstreams.get(project).stream();
+    }
 
-            @Override
-            public boolean isRoot(MavenProject project) {
-                return upstreams.get(project).isEmpty();
-            }
-        };
+    public Stream<MavenProject> getUpstreamProjects(MavenProject project) {
+        return upstreams.get(project).stream();
+    }
+
+    public boolean isRoot(MavenProject project) {
+        return upstreams.get(project).isEmpty();
+    }
+
+    @Override
+    public List<MavenProject> getAllProjects() {
+        return sortedProjects;
     }
 
-    Stream<K> getProjects();
+    @Override
+    public List<MavenProject> getSortedProjects() {
+        return sortedProjects;
+    }
 
-    boolean isRoot(K project);
+    @Override
+    public List<MavenProject> getDownstreamProjects(MavenProject project, boolean transitive) {
+        return delegate.getDownstreamProjects(project, transitive);
+    }
 
-    Stream<K> getDownstreamProjects(K project);
+    @Override
+    public List<MavenProject> getUpstreamProjects(MavenProject project, boolean transitive) {
+        return delegate.getUpstreamProjects(project, transitive);
+    }
 
-    Stream<K> getUpstreamProjects(K project);
+    /**
+     * @return {@code true} if all projects in this graph have at most one downstream projects; {@code false} otherwise
+     */
+    public boolean isSerial() {
+        return serial;
+    }
 
 }

daemon/src/main/java/org/mvndaemon/mvnd/builder/ProjectComparator.java

@@ -51,29 +51,29 @@
  */
 class ProjectComparator {
 
-    public static Comparator<MavenProject> create(DependencyGraph<MavenProject> graph) {
+    public static Comparator<MavenProject> create(DependencyGraph graph) {
         return create0(graph, Collections.emptyMap(), ProjectComparator::id);
     }
 
-    static <K> Comparator<K> create0(DependencyGraph<K> dependencyGraph,
+    static Comparator<MavenProject> create0(DependencyGraph dependencyGraph,
             Map<String, AtomicLong> historicalServiceTimes,
-            Function<K, String> toKey) {
+            Function<MavenProject, String> toKey) {
         final long defaultServiceTime = average(historicalServiceTimes.values());
 
-        final Map<K, Long> serviceTimes = new HashMap<>();
+        final Map<MavenProject, Long> serviceTimes = new HashMap<>();
 
-        final Set<K> rootProjects = new HashSet<>();
-        dependencyGraph.getProjects().forEach(project -> {
+        final Set<MavenProject> rootProjects = new HashSet<>();
+        dependencyGraph.getAllProjects().stream().forEach(project -> {
             long serviceTime = getServiceTime(historicalServiceTimes, project, defaultServiceTime, toKey);
             serviceTimes.put(project, serviceTime);
             if (dependencyGraph.isRoot(project)) {
                 rootProjects.add(project);
             }
         });
 
-        final Map<K, Long> projectWeights = calculateWeights(dependencyGraph, serviceTimes, rootProjects);
+        final Map<MavenProject, Long> projectWeights = calculateWeights(dependencyGraph, serviceTimes, rootProjects);
 
-        return Comparator.comparingLong((ToLongFunction<K>) projectWeights::get)
+        return Comparator.comparingLong((ToLongFunction<MavenProject>) projectWeights::get)
                 .thenComparing(toKey, String::compareTo)
                 .reversed();
     }
@@ -83,16 +83,16 @@ private static long average(Collection<AtomicLong> values) {
                 .average().orElse(1.0d));
     }
 
-    private static <K> long getServiceTime(Map<String, AtomicLong> serviceTimes, K project,
-            long defaultServiceTime, Function<K, String> toKey) {
+    private static long getServiceTime(Map<String, AtomicLong> serviceTimes, MavenProject project,
+            long defaultServiceTime, Function<MavenProject, String> toKey) {
         AtomicLong serviceTime = serviceTimes.get(toKey.apply(project));
         return serviceTime != null ? serviceTime.longValue() : defaultServiceTime;
     }
 
-    private static <K> Map<K, Long> calculateWeights(DependencyGraph<K> dependencyGraph,
-            Map<K, Long> serviceTimes, Collection<K> rootProjects) {
-        Map<K, Long> weights = new HashMap<>();
-        for (K rootProject : rootProjects) {
+    private static Map<MavenProject, Long> calculateWeights(DependencyGraph dependencyGraph,
+            Map<MavenProject, Long> serviceTimes, Collection<MavenProject> rootProjects) {
+        Map<MavenProject, Long> weights = new HashMap<>();
+        for (MavenProject rootProject : rootProjects) {
             calculateWeights(dependencyGraph, serviceTimes, rootProject, weights);
         }
         return weights;
@@ -102,8 +102,8 @@ private static <K> Map<K, Long> calculateWeights(DependencyGraph<K> dependencyGr
      * Returns the maximum sum of build time along a path from the project to an exit project. An
      * "exit project" is a project without downstream dependencies.
      */
-    private static <K> long calculateWeights(DependencyGraph<K> dependencyGraph,
-            Map<K, Long> serviceTimes, K project, Map<K, Long> weights) {
+    private static long calculateWeights(DependencyGraph dependencyGraph,
+            Map<MavenProject, Long> serviceTimes, MavenProject project, Map<MavenProject, Long> weights) {
         long weight = serviceTimes.get(project)
                 + dependencyGraph.getDownstreamProjects(project)
                         .mapToLong(successor -> {

daemon/src/main/java/org/mvndaemon/mvnd/builder/ReactorBuildQueue.java

@@ -30,7 +30,7 @@
  */
 class ReactorBuildQueue {
 
-    private final DependencyGraph<MavenProject> graph;
+    private final DependencyGraph graph;
 
     private final Set<MavenProject> rootProjects;
 
@@ -44,13 +44,13 @@ class ReactorBuildQueue {
     private final Set<MavenProject> finishedProjects;
 
     public ReactorBuildQueue(Collection<MavenProject> projects,
-            DependencyGraph<MavenProject> graph) {
+            DependencyGraph graph) {
         this.graph = graph;
         this.projects = new HashSet<>();
         this.rootProjects = new HashSet<>();
         this.blockedProjects = new HashSet<>();
         this.finishedProjects = new HashSet<>();
-        this.graph.getProjects().forEach(project -> {
+        this.graph.getAllProjects().stream().forEach(project -> {
             this.projects.add(project);
             if (this.graph.isRoot(project)) {
                 this.rootProjects.add(project);

daemon/src/main/java/org/mvndaemon/mvnd/builder/ReactorBuildStats.java

@@ -107,18 +107,18 @@ public long walltimeTime(TimeUnit unit) {
         return unit.convert(stopTime - startTime, TimeUnit.NANOSECONDS);
     }
 
-    public String renderCriticalPath(DependencyGraph<MavenProject> graph) {
+    public String renderCriticalPath(DependencyGraph graph) {
         return renderCriticalPath(graph, ReactorBuildStats::projectGA);
     }
 
-    public <K> String renderCriticalPath(DependencyGraph<K> graph, Function<K, String> toKey) {
+    public String renderCriticalPath(DependencyGraph graph, Function<MavenProject, String> toKey) {
         StringBuilder result = new StringBuilder();
 
         // render critical path
 
         long criticalPathServiceTime = 0;
         result.append("Build critical path service times (and bottleneck** times):");
-        for (K project : calculateCriticalPath(graph, toKey)) {
+        for (MavenProject project : calculateCriticalPath(graph, toKey)) {
             result.append('\n');
             String key = toKey.apply(project);
             criticalPathServiceTime += serviceTimes.get(key).get();
@@ -178,11 +178,11 @@ private String formatDuration(long nanos) {
         return String.format("%5d s", secs);
     }
 
-    private <K> List<K> calculateCriticalPath(DependencyGraph<K> graph, Function<K, String> toKey) {
-        Comparator<K> comparator = ProjectComparator.create0(graph, serviceTimes, toKey);
-        Stream<K> rootProjects = graph.getProjects().filter(graph::isRoot);
-        List<K> criticalPath = new ArrayList<>();
-        K project = getCriticalProject(rootProjects, comparator);
+    private List<MavenProject> calculateCriticalPath(DependencyGraph graph, Function<MavenProject, String> toKey) {
+        Comparator<MavenProject> comparator = ProjectComparator.create0(graph, serviceTimes, toKey);
+        Stream<MavenProject> rootProjects = graph.getAllProjects().stream().filter(graph::isRoot);
+        List<MavenProject> criticalPath = new ArrayList<>();
+        MavenProject project = getCriticalProject(rootProjects, comparator);
         do {
             criticalPath.add(project);
         } while ((project = getCriticalProject(graph.getDownstreamProjects(project), comparator)) != null);