From 5f996d565dcda100b861828b6731e3d732e5af49 Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Wed, 13 Apr 2016 22:50:10 -0400 Subject: [PATCH 01/16] Initial implementation of predicate-based flow scanner and an incremental analyzer for flow graphs --- .../plugins/workflow/graph/FlowScanner.java | 317 ++++++++++++++++++ .../graph/IncrementalFlowAnalysis.java | 84 +++++ 2 files changed, 401 insertions(+) create mode 100644 src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java create mode 100644 src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java new file mode 100644 index 00000000..fc294c1c --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java @@ -0,0 +1,317 @@ +package org.jenkinsci.plugins.workflow.graph; + +/* + * The MIT License + * + * Copyright (c) 2013-2014, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import hudson.model.Action; +import org.jenkinsci.plugins.workflow.actions.ErrorAction; +import org.jenkinsci.plugins.workflow.actions.LabelAction; +import org.jenkinsci.plugins.workflow.actions.LogAction; +import org.jenkinsci.plugins.workflow.actions.StageAction; +import org.jenkinsci.plugins.workflow.actions.WorkspaceAction; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * Generified algorithms for scanning flows for information + * Supports a variety of algorithms for searching, and pluggable conditions + * Worth noting: predicates may be stateful here + * + * ANALYSIS method will + * @author Sam Van Oort + */ +public class FlowScanner { + /** Different ways of scannning the flow graph starting from one or more head nodes + * DEPTH_FIRST_ALL_PARENTS is the same as FlowWalker + * - scan through the first parents (depth first search), then come back to visit parallel branches + * BLOCK_SCOPES just skims through the blocks from the inside out, in reverse order + * SINGLE_PARENT only walks through the hierarchy of the first parent in the head (or heads) + */ + public enum ScanType { + DEPTH_FIRST_ALL_PARENTS, + BLOCK_SCOPES, + SINGLE_PARENT + } + + /** + * Create a predicate that will match on all FlowNodes having a specific action present + * @param actionClass Action class to look for + * @param Action type + * @return Predicate that will match when FlowNode has the action given + */ + public static Predicate createPredicateWhereActionExists(@Nonnull final Class actionClass) { + return new Predicate() { + @Override + public boolean apply(FlowNode input) { + return (input != null && input.getAction(actionClass) != null); + } + }; + } + + // Default predicates + static final Predicate MATCH_HAS_LABEL = createPredicateWhereActionExists(LabelAction.class); + static final Predicate MATCH_IS_STAGE = createPredicateWhereActionExists(StageAction.class); + static final Predicate MATCH_HAS_WORKSPACE = createPredicateWhereActionExists(WorkspaceAction.class); + static final Predicate MATCH_HAS_ERROR = createPredicateWhereActionExists(ErrorAction.class); + static final Predicate MATCH_HAS_LOG = createPredicateWhereActionExists(LogAction.class); + + /** One of many ways to scan the flowgraph */ + public interface ScanAlgorithm { + + /** + * Search for first node (walking from the heads through parents) that matches the condition + * @param heads Nodes to start searching from + * @param stopNodes Search doesn't go beyond any of these nodes, null or empty will run to end of flow + * @param matchPredicate Matching condition for search + * @return First node matching condition, or null if none found + */ + @CheckForNull + public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate); + + /** + * Search for first node (walking from the heads through parents) that matches the condition + * @param heads Nodes to start searching from + * @param stopNodes Search doesn't go beyond any of these nodes, null or empty will run to end of flow + * @param matchPredicate Matching condition for search + * @return All nodes matching condition + */ + @Nonnull + public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate); + } + + /** Does a simple and efficient depth-first search */ + public static class DepthFirstScanner implements ScanAlgorithm { + + @Override + public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { + if (heads == null || heads.size() == 0) { + return null; + } + + HashSet visited = new HashSet(); + ArrayDeque queue = new ArrayDeque(heads); // Only used for parallel branches + + // Do what we need to for fast tests + Collection fastStopNodes = (stopNodes == null || stopNodes.size() == 0) ? Collections.EMPTY_SET : stopNodes; + if (fastStopNodes.size() > 10 && !(fastStopNodes instanceof Set)) { + fastStopNodes = new HashSet(fastStopNodes); + } + + // TODO this will probably be more efficient if we work with the first node + // or use a recursive solution for parallel forks + while (!queue.isEmpty()) { + FlowNode f = queue.pop(); + if (matchPredicate.apply(f)) { + return f; + } + visited.add(f); + List parents = f.getParents(); // Parents never null + for (FlowNode p : parents) { + if (!visited.contains(p) && !fastStopNodes.contains(p)) { + queue.push(p); + } + } + } + return null; + } + + @Override + public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { + if (heads == null || heads.size() == 0) { + return null; + } + + HashSet visited = new HashSet(); + ArrayDeque queue = new ArrayDeque(heads); // Only used for parallel branches + ArrayList matches = new ArrayList(); + + // Do what we need to for fast tests + Collection fastStopNodes = (stopNodes == null || stopNodes.size() == 0) ? Collections.EMPTY_SET : stopNodes; + if (fastStopNodes.size() > 10 && !(fastStopNodes instanceof Set)) { + fastStopNodes = new HashSet(fastStopNodes); + } + + // TODO this will probably be more efficient if use a variable for non-parallel flows and don't constantly push/pop array + while (!queue.isEmpty()) { + FlowNode f = queue.pop(); + if (matchPredicate.apply(f)) { + matches.add(f); + } + visited.add(f); + List parents = f.getParents(); // Parents never null + for (FlowNode p : parents) { + if (!visited.contains(p) && !fastStopNodes.contains(p)) { + queue.push(p); + } + } + } + return matches; + } + } + + /** + * Scans through a single ancestry, does not cover parallel branches + */ + public static class LinearScanner implements ScanAlgorithm { + + @Override + public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { + if (heads == null || heads.size() == 0) { + return null; + } + + // Do what we need to for fast tests + Collection fastStopNodes = (stopNodes == null || stopNodes.size() == 0) ? Collections.EMPTY_SET : stopNodes; + if (fastStopNodes.size() > 10 && !(fastStopNodes instanceof Set)) { + fastStopNodes = new HashSet(fastStopNodes); + } + + FlowNode current = heads.iterator().next(); + while (current != null) { + if (matchPredicate.apply(current)) { + return current; + } + List parents = current.getParents(); // Parents never null + current = null; + for (FlowNode p : parents) { + if (!fastStopNodes.contains(p)) { + current = p; + break; + } + } + } + return current; + } + + @Override + public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { + if (heads == null || heads.size() == 0) { + return null; + } + + // Do what we need to for fast tests + Collection fastStopNodes = (stopNodes == null || stopNodes.size() == 0) ? Collections.EMPTY_SET : stopNodes; + if (fastStopNodes.size() > 10 && !(fastStopNodes instanceof Set)) { + fastStopNodes = new HashSet(fastStopNodes); + } + ArrayList matches = new ArrayList(); + + FlowNode current = heads.iterator().next(); + while (current != null) { + if (matchPredicate.apply(current)) { + matches.add(current); + } + List parents = current.getParents(); // Parents never null + current = null; + for (FlowNode p : parents) { + if (!fastStopNodes.contains(p)) { + current = p; + break; + } + } + } + return matches; + } + } + + /** + * Scanner that jumps over nested blocks + */ + public static class BlockHoppingScanner implements ScanAlgorithm { + + @Override + public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { + if (heads == null || heads.size() == 0) { + return null; + } + + // Do what we need to for fast tests + Collection fastStopNodes = (stopNodes == null || stopNodes.size() == 0) ? Collections.EMPTY_SET : stopNodes; + if (fastStopNodes.size() > 10 && !(fastStopNodes instanceof Set)) { + fastStopNodes = new HashSet(fastStopNodes); + } + + FlowNode current = heads.iterator().next(); + while (current != null) { + if (!(current instanceof BlockEndNode) && matchPredicate.apply(current)) { + return current; + } else { // Hop the block + current = ((BlockEndNode) current).getStartNode(); + } + List parents = current.getParents(); // Parents never null + current = null; + for (FlowNode p : parents) { + if (!fastStopNodes.contains(p)) { + current = p; + break; + } + } + } + return current; + } + + @Override + public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { + if (heads == null || heads.size() == 0) { + return null; + } + + // Do what we need to for fast tests + Collection fastStopNodes = (stopNodes == null || stopNodes.size() == 0) ? Collections.EMPTY_SET : stopNodes; + if (fastStopNodes.size() > 10 && !(fastStopNodes instanceof Set)) { + fastStopNodes = new HashSet(fastStopNodes); + } + ArrayList matches = new ArrayList(); + + FlowNode current = heads.iterator().next(); + while (current != null) { + if (!(current instanceof BlockEndNode) && matchPredicate.apply(current)) { + matches.add(current); + } else { // Hop the block + current = ((BlockEndNode) current).getStartNode(); + } + List parents = current.getParents(); // Parents never null + current = null; + for (FlowNode p : parents) { + if (!fastStopNodes.contains(p)) { + current = p; + break; + } + } + } + return matches; + } + } +} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java new file mode 100644 index 00000000..56eaff55 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java @@ -0,0 +1,84 @@ +package org.jenkinsci.plugins.workflow.graph; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.jenkinsci.plugins.workflow.flow.FlowExecution; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Provides incremental analysis of flow graphs, where updates are on the head + * @author Sam Van Oort + */ +public class IncrementalFlowAnalysis { + + protected static class IncrementalAnalysis { + protected List lastHeadIds = new ArrayList(); + protected T lastValue; + + /** Gets value from a flownode */ + protected Function valueExtractor; + + protected Predicate nodeMatchCondition; + + public IncrementalAnalysis(@Nonnull Predicate nodeMatchCondition, @Nonnull Function valueExtractFunction){ + this.nodeMatchCondition = nodeMatchCondition; + this.valueExtractor = valueExtractFunction; + } + + /** + * Look up a value scanned from the flow + * If the heads haven't changed in the flow, return the current heads + * If they have, only hunt from the current value until the last one + * @param exec + * @return + */ + @CheckForNull + public T getUpdatedValue(@CheckForNull FlowExecution exec) { + if (exec == null) { + return null; + } + List heads = exec.getCurrentHeads(); + if (heads != null && heads.size() == lastHeadIds.size()) { + boolean useCache = false; + for (FlowNode f : heads) { + if (lastHeadIds.contains(f.getId())) { + useCache = true; + break; + } + } + if (!useCache) { + update(exec); + } + return lastValue; + } + return null; + } + + protected void update(@Nonnull FlowExecution exec) { + ArrayList nodes = new ArrayList(); + for (String nodeId : this.lastHeadIds) { + try { + nodes.add(exec.getNode(nodeId)); + } catch (IOException ioe) { + throw new IllegalStateException(ioe); + } + } + FlowNode matchNode = new FlowScanner.BlockHoppingScanner().findFirstMatch(exec.getCurrentHeads(), nodes, this.nodeMatchCondition); + this.lastValue = this.valueExtractor.apply(matchNode); + + this.lastHeadIds.clear(); + for (FlowNode f : exec.getCurrentHeads()) { + lastHeadIds.add(f.getId()); + } + } + } + + static Cache analysisCache = CacheBuilder.newBuilder().initialCapacity(100).build(); +} From ddb608dc2bea2c36a9aa470b0b38aad39f479937 Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Thu, 14 Apr 2016 00:34:05 -0400 Subject: [PATCH 02/16] Pacify FindBugs --- .../org/jenkinsci/plugins/workflow/graph/FlowScanner.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java index fc294c1c..36418551 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java @@ -150,7 +150,7 @@ public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckF @Override public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { if (heads == null || heads.size() == 0) { - return null; + return Collections.EMPTY_LIST; } HashSet visited = new HashSet(); @@ -218,7 +218,7 @@ public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckF @Override public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { if (heads == null || heads.size() == 0) { - return null; + return Collections.EMPTY_LIST; } // Do what we need to for fast tests @@ -285,7 +285,7 @@ public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckF @Override public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { if (heads == null || heads.size() == 0) { - return null; + return Collections.EMPTY_LIST; } // Do what we need to for fast tests From 2b51497f8e5d5db546647267139824e7f651a942 Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Mon, 18 Apr 2016 01:49:23 -0400 Subject: [PATCH 03/16] Tests for the unrefactored FlowScanner --- pom.xml | 20 +- .../plugins/workflow/graph/FlowScanner.java | 19 +- .../workflow/graph/TestFlowScanner.java | 184 ++++++++++++++++++ 3 files changed, 219 insertions(+), 4 deletions(-) create mode 100644 src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java diff --git a/pom.xml b/pom.xml index 204a85f0..5fa4333b 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,7 @@ scm:git:git@github.com:jenkinsci/${project.artifactId}-plugin.git https://github.com/jenkinsci/${project.artifactId}-plugin HEAD - + repo.jenkins-ci.org @@ -70,5 +70,23 @@ workflow-step-api 1.15 + + ${project.groupId} + workflow-job + 1.15 + test + + + + ${project.groupId} + workflow-cps + 2.2-SNAPSHOT + + + ${project.groupId} + workflow-basic-steps + 1.15 + test + diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java index 36418551..1f993a97 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java @@ -3,7 +3,7 @@ /* * The MIT License * - * Copyright (c) 2013-2014, CloudBees, Inc. + * Copyright (c) 2016, CloudBees, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,7 +24,6 @@ * THE SOFTWARE. */ -import com.google.common.base.Function; import com.google.common.base.Predicate; import hudson.model.Action; import org.jenkinsci.plugins.workflow.actions.ErrorAction; @@ -32,6 +31,7 @@ import org.jenkinsci.plugins.workflow.actions.LogAction; import org.jenkinsci.plugins.workflow.actions.StageAction; import org.jenkinsci.plugins.workflow.actions.WorkspaceAction; +import org.jenkinsci.plugins.workflow.cps.nodes.StepAtomNode; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; @@ -40,7 +40,6 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; @@ -87,6 +86,20 @@ public boolean apply(FlowNode input) { static final Predicate MATCH_HAS_ERROR = createPredicateWhereActionExists(ErrorAction.class); static final Predicate MATCH_HAS_LOG = createPredicateWhereActionExists(LogAction.class); + public static Predicate createPredicateForStepNodeWithDescriptor(final String descriptorId) { + Predicate outputPredicate = new Predicate() { + @Override + public boolean apply(FlowNode input) { + if (input instanceof StepAtomNode) { + StepAtomNode san = (StepAtomNode)input; + return descriptorId.equals(san.getDescriptor().getId()); + } + return false; + } + }; + return outputPredicate; + } + /** One of many ways to scan the flowgraph */ public interface ScanAlgorithm { diff --git a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java new file mode 100644 index 00000000..9738372e --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java @@ -0,0 +1,184 @@ +/* + * The MIT License + * + * Copyright (c) 2016, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.graph; + +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; +import org.jenkinsci.plugins.workflow.flow.FlowExecution; +import org.jenkinsci.plugins.workflow.job.WorkflowJob; +import org.jenkinsci.plugins.workflow.job.WorkflowRun; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.jvnet.hudson.test.BuildWatcher; +import org.jvnet.hudson.test.JenkinsRule; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +public class TestFlowScanner { + + @ClassRule + public static BuildWatcher buildWatcher = new BuildWatcher(); + + @Rule public JenkinsRule r = new JenkinsRule(); + + + /** Tests the basic scan algorithm, predicate use, start/stop nodes */ + @Test + public void testSimpleScan() throws Exception { + WorkflowJob job = r.jenkins.createProject(WorkflowJob.class, "Convoluted"); + job.setDefinition(new CpsFlowDefinition( + "sleep 2 \n" + + "echo 'donothing'\n" + + "echo 'doitagain'" + )); + WorkflowRun b = r.assertBuildStatusSuccess(job.scheduleBuild2(0)); + FlowExecution exec = b.getExecution(); + FlowScanner.ScanAlgorithm[] scans = {new FlowScanner.LinearScanner(), + new FlowScanner.DepthFirstScanner(), + new FlowScanner.BlockHoppingScanner()}; + + Predicate echoPredicate = FlowScanner.createPredicateForStepNodeWithDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); + List heads = b.getExecution().getCurrentHeads(); + + // Test expected scans with no stop nodes given (different ways of specifying none) + for (FlowScanner.ScanAlgorithm sa : scans) { + FlowNode node = sa.findFirstMatch(heads, null, echoPredicate); + Assert.assertEquals(exec.getNode("5"), node); + node = sa.findFirstMatch(heads, Collections.EMPTY_LIST, echoPredicate); + Assert.assertEquals(exec.getNode("5"), node); + node = sa.findFirstMatch(heads, Collections.EMPTY_SET, echoPredicate); + Assert.assertEquals(exec.getNode("5"), node); + + Collection nodeList = sa.findAllMatches(heads, null, echoPredicate); + FlowNode[] expected = new FlowNode[]{exec.getNode("5"), exec.getNode("4")}; + Assert.assertArrayEquals(expected, nodeList.toArray()); + nodeList = sa.findAllMatches(heads, Collections.EMPTY_LIST, echoPredicate); + Assert.assertArrayEquals(expected, nodeList.toArray()); + nodeList = sa.findAllMatches(heads, Collections.EMPTY_SET, echoPredicate); + Assert.assertArrayEquals(expected, nodeList.toArray()); + } + + // Test with no matches + for (FlowScanner.ScanAlgorithm sa : scans) { + FlowNode node = sa.findFirstMatch(heads, null, (Predicate)Predicates.alwaysFalse()); + Assert.assertNull(node); + + Collection nodeList = sa.findAllMatches(heads, null, (Predicate)Predicates.alwaysFalse()); + Assert.assertNotNull(nodeList); + Assert.assertEquals(0, nodeList.size()); + } + + // Test with a stop node given, sometimes no matches + Collection noMatchEndNode = Collections.singleton(exec.getNode("5")); + Collection singleMatchEndNode = Collections.singleton(exec.getNode("4")); + for (FlowScanner.ScanAlgorithm sa : scans) { + FlowNode node = sa.findFirstMatch(heads, noMatchEndNode, echoPredicate); + Assert.assertNull(node); + + Collection nodeList = sa.findAllMatches(heads, noMatchEndNode, echoPredicate); + Assert.assertNotNull(nodeList); + Assert.assertEquals(0, nodeList.size()); + + // Now we try with a stop list the reduces node set for multiple matches + node = sa.findFirstMatch(heads, singleMatchEndNode, echoPredicate); + Assert.assertEquals(exec.getNode("5"), node); + nodeList = sa.findAllMatches(heads, singleMatchEndNode, echoPredicate); + Assert.assertNotNull(nodeList); + Assert.assertEquals(1, nodeList.size()); + Assert.assertEquals(exec.getNode("5"), nodeList.iterator().next()); + } + } + + /** Tests the basic scan algorithm where blocks are involved */ + @Test + public void blockScan() throws Exception { + WorkflowJob job = r.jenkins.createProject(WorkflowJob.class, "Convoluted"); + job.setDefinition(new CpsFlowDefinition( + "echo 'first'\n" + + "timeout(time: 10, unit: 'SECONDS') {\n" + + " echo 'second'\n" + + " echo 'third'\n" + + "}\n" + + "sleep 1" + )); + WorkflowRun b = r.assertBuildStatusSuccess(job.scheduleBuild2(0)); + FlowScanner.ScanAlgorithm[] scans = {new FlowScanner.LinearScanner(), + new FlowScanner.DepthFirstScanner(), + new FlowScanner.BlockHoppingScanner()}; + for (FlowScanner.ScanAlgorithm sa : scans) { + + } + FlowGraphWalker walk = new FlowGraphWalker(b.getExecution()); + ArrayList flows = new ArrayList(); + for (FlowNode f : walk) { + flows.add(f); + } + + } + + + @Test + public void testme() throws Exception { + WorkflowJob job = r.jenkins.createProject(WorkflowJob.class, "Convoluted"); + job.setDefinition(new CpsFlowDefinition( + "echo 'pre-stage command'\n" + + "sleep 1\n" + + "stage 'first'\n" + + "echo 'I ran'\n" + + "stage 'second'\n" + + "node {\n" + + " def steps = [:]\n" + + " steps['2a-dir'] = {\n" + + " echo 'do 2a stuff'\n" + + " echo 'do more 2a stuff'\n" + + " timeout(time: 10, unit: 'SECONDS') {\n" + + " stage 'invalid'\n" + + " echo 'time seconds'\n" + + " }\n" + + " sleep 15\n" + + " }\n" + + " steps['2b'] = {\n" + + " echo 'do 2b stuff'\n" + + " sleep 10\n" + + " echo 'echo_label_me'\n" + + " }\n" + + " parallel steps\n" + + "}\n" + + "\n" + + "stage 'final'\n" + + "echo 'ran final 1'\n" + + "echo 'ran final 2'" + )); + WorkflowRun b = r.assertBuildStatusSuccess(job.scheduleBuild2(0)); + FlowGraphWalker walker = new FlowGraphWalker(); + walker.addHeads(b.getExecution().getCurrentHeads()); + } +} \ No newline at end of file From bb0864e9cef64fd5dd92c34abc14af5de70d2f3d Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Mon, 18 Apr 2016 19:15:20 -0400 Subject: [PATCH 04/16] WIP on pulling out generalized flowscanning algorithms --- .../plugins/workflow/graph/FlowScanner.java | 189 +++++++++++++++--- .../workflow/graph/TestFlowScanner.java | 2 +- 2 files changed, 157 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java index 1f993a97..77637454 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java @@ -40,6 +40,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; @@ -100,7 +101,8 @@ public boolean apply(FlowNode input) { return outputPredicate; } - /** One of many ways to scan the flowgraph */ + /** Interface to be used for scanning/analyzing FlowGraphs with support for different visit orders + */ public interface ScanAlgorithm { /** @@ -124,23 +126,142 @@ public interface ScanAlgorithm { public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate); } + /** + * Base class for flow scanners, which offers basic methods and stubs for algorithms + * Scanners store state internally, and are not thread-safe but are reusable + * Scans/analysis of graphs is implemented via internal iteration to allow reusing algorithm bodies + * However internal iteration has access to additional information + */ + public static abstract class AbstractFlowScanner implements ScanAlgorithm { + + // State variables, not all need be used + protected ArrayDeque _queue; + protected FlowNode _current; + + // Public APIs need to invoke this before searches + protected abstract void initialize(); + + /** + * Actual meat of the iteration, get the next node to visit, using & updating state as needed + * @param f Node to look for parents of (usually _current) + * @param blackList Nodes that are not eligible for visiting + * @return Next node to visit, or null if we've exhausted the node list + */ + @CheckForNull + protected abstract FlowNode next(@CheckForNull FlowNode f, @Nonnull Collection blackList); + + + /** Fast internal scan from start through single-parent (unbranched) nodes until we hit a node with one of the following: + * - Multiple parents + * - No parents + * - Satisfies the endCondition predicate + * + * @param endCondition Predicate that ends search + * @return Node satisfying condition + */ + @CheckForNull + protected static FlowNode linearScanUntil(@Nonnull FlowNode start, @Nonnull Predicate endCondition) { + while(true) { + if (endCondition.apply(start)){ + break; + } + List parents = start.getParents(); + if (parents == null || parents.size() == 0 || parents.size() > 1) { + break; + } + start = parents.get(0); + } + return start; + } + + /** Convert stop nodes to a collection that can efficiently be checked for membership, handling nulls if needed */ + @Nonnull + protected Collection convertToFastCheckable(@CheckForNull Collection nodeCollection) { + if (nodeCollection == null || nodeCollection.size()==0) { + return Collections.EMPTY_SET; + } else if (nodeCollection instanceof Set) { + return nodeCollection; + } + return nodeCollection.size() > 5 ? new HashSet(nodeCollection) : nodeCollection; + } + + public FlowNode findFirstMatch(@CheckForNull Collection heads, @Nonnull Predicate matchPredicate) { + return this.findFirstMatch(heads, null, matchPredicate); + } + + public Collection findAllMatches(@CheckForNull Collection heads, @Nonnull Predicate matchPredicate) { + return this.findAllMatches(heads, null, matchPredicate); + } + + // Basic algo impl + protected FlowNode findFirstMatchBasic(@CheckForNull Collection heads, + @CheckForNull Collection endNodes, + Predicate matchCondition) { + if (heads == null || heads.size() == 0) { + return null; + } + initialize(); + Collection fastEndNodes = convertToFastCheckable(endNodes); + + while((this._current = this.next(_current, fastEndNodes)) != null) { + if (matchCondition.apply(this._current)) { + return this._current; + } + } + return null; + } + + // Basic algo impl + protected List findAllMatchesBasic(@CheckForNull Collection heads, + @CheckForNull Collection endNodes, + Predicate matchCondition) { + if (heads == null || heads.size() == 0) { + return null; + } + initialize(); + Collection fastEndNodes = convertToFastCheckable(endNodes); + ArrayList nodes = new ArrayList(); + + while((this._current = this.next(_current, fastEndNodes)) != null) { + if (matchCondition.apply(this._current)) { + nodes.add(this._current); + } + } + return nodes; + } + } + /** Does a simple and efficient depth-first search */ - public static class DepthFirstScanner implements ScanAlgorithm { + public static class DepthFirstScanner extends AbstractFlowScanner { + + protected HashSet _visited = new HashSet(); + + protected void initialize() { + if (this._queue == null) { + this._queue = new ArrayDeque(); + } else { + this._queue.clear(); + } + this._visited.clear(); + this._current = null; + } + + @Override + protected FlowNode next(@CheckForNull FlowNode f, @Nonnull Collection blackList) { + // Check for visited and stuff? + return null; + } @Override public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { if (heads == null || heads.size() == 0) { return null; } + initialize(); HashSet visited = new HashSet(); ArrayDeque queue = new ArrayDeque(heads); // Only used for parallel branches - - // Do what we need to for fast tests - Collection fastStopNodes = (stopNodes == null || stopNodes.size() == 0) ? Collections.EMPTY_SET : stopNodes; - if (fastStopNodes.size() > 10 && !(fastStopNodes instanceof Set)) { - fastStopNodes = new HashSet(fastStopNodes); - } + Collection fastStopNodes = convertToFastCheckable(stopNodes); // TODO this will probably be more efficient if we work with the first node // or use a recursive solution for parallel forks @@ -169,12 +290,7 @@ public Collection findAllMatches(@CheckForNull Collection he HashSet visited = new HashSet(); ArrayDeque queue = new ArrayDeque(heads); // Only used for parallel branches ArrayList matches = new ArrayList(); - - // Do what we need to for fast tests - Collection fastStopNodes = (stopNodes == null || stopNodes.size() == 0) ? Collections.EMPTY_SET : stopNodes; - if (fastStopNodes.size() > 10 && !(fastStopNodes instanceof Set)) { - fastStopNodes = new HashSet(fastStopNodes); - } + Collection fastStopNodes = convertToFastCheckable(stopNodes); // TODO this will probably be more efficient if use a variable for non-parallel flows and don't constantly push/pop array while (!queue.isEmpty()) { @@ -197,7 +313,7 @@ public Collection findAllMatches(@CheckForNull Collection he /** * Scans through a single ancestry, does not cover parallel branches */ - public static class LinearScanner implements ScanAlgorithm { + public static class LinearScanner extends AbstractFlowScanner { @Override public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { @@ -205,11 +321,7 @@ public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckF return null; } - // Do what we need to for fast tests - Collection fastStopNodes = (stopNodes == null || stopNodes.size() == 0) ? Collections.EMPTY_SET : stopNodes; - if (fastStopNodes.size() > 10 && !(fastStopNodes instanceof Set)) { - fastStopNodes = new HashSet(fastStopNodes); - } + Collection fastStopNodes = convertToFastCheckable(stopNodes); FlowNode current = heads.iterator().next(); while (current != null) { @@ -235,10 +347,7 @@ public Collection findAllMatches(@CheckForNull Collection he } // Do what we need to for fast tests - Collection fastStopNodes = (stopNodes == null || stopNodes.size() == 0) ? Collections.EMPTY_SET : stopNodes; - if (fastStopNodes.size() > 10 && !(fastStopNodes instanceof Set)) { - fastStopNodes = new HashSet(fastStopNodes); - } + Collection fastStopNodes = convertToFastCheckable(stopNodes); ArrayList matches = new ArrayList(); FlowNode current = heads.iterator().next(); @@ -257,12 +366,22 @@ public Collection findAllMatches(@CheckForNull Collection he } return matches; } + + @Override + protected void initialize() { + // no-op for us + } + + @Override + protected FlowNode next(@CheckForNull FlowNode f, @Nonnull Collection blackList) { + return null; + } } /** * Scanner that jumps over nested blocks */ - public static class BlockHoppingScanner implements ScanAlgorithm { + public static class BlockHoppingScanner extends AbstractFlowScanner { @Override public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { @@ -271,10 +390,7 @@ public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckF } // Do what we need to for fast tests - Collection fastStopNodes = (stopNodes == null || stopNodes.size() == 0) ? Collections.EMPTY_SET : stopNodes; - if (fastStopNodes.size() > 10 && !(fastStopNodes instanceof Set)) { - fastStopNodes = new HashSet(fastStopNodes); - } + Collection fastStopNodes = convertToFastCheckable(stopNodes); FlowNode current = heads.iterator().next(); while (current != null) { @@ -302,10 +418,7 @@ public Collection findAllMatches(@CheckForNull Collection he } // Do what we need to for fast tests - Collection fastStopNodes = (stopNodes == null || stopNodes.size() == 0) ? Collections.EMPTY_SET : stopNodes; - if (fastStopNodes.size() > 10 && !(fastStopNodes instanceof Set)) { - fastStopNodes = new HashSet(fastStopNodes); - } + Collection fastStopNodes = convertToFastCheckable(stopNodes); ArrayList matches = new ArrayList(); FlowNode current = heads.iterator().next(); @@ -326,5 +439,15 @@ public Collection findAllMatches(@CheckForNull Collection he } return matches; } + + @Override + protected void initialize() { + + } + + @Override + protected FlowNode next(@CheckForNull FlowNode f, @Nonnull Collection blackList) { + return null; + } } } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java index 9738372e..9028396b 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java @@ -66,7 +66,7 @@ public void testSimpleScan() throws Exception { new FlowScanner.BlockHoppingScanner()}; Predicate echoPredicate = FlowScanner.createPredicateForStepNodeWithDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); - List heads = b.getExecution().getCurrentHeads(); + List heads = exec.getCurrentHeads(); // Test expected scans with no stop nodes given (different ways of specifying none) for (FlowScanner.ScanAlgorithm sa : scans) { From f4de17485cd42042caafaa3449645c3e7f3653ea Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Tue, 19 Apr 2016 02:08:31 -0400 Subject: [PATCH 05/16] Flesh out/fix most of the flow scanner implementations --- .../plugins/workflow/graph/FlowScanner.java | 223 ++++++------------ .../workflow/graph/TestFlowScanner.java | 25 +- 2 files changed, 83 insertions(+), 165 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java index 77637454..b77ae46d 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java @@ -53,17 +53,6 @@ * @author Sam Van Oort */ public class FlowScanner { - /** Different ways of scannning the flow graph starting from one or more head nodes - * DEPTH_FIRST_ALL_PARENTS is the same as FlowWalker - * - scan through the first parents (depth first search), then come back to visit parallel branches - * BLOCK_SCOPES just skims through the blocks from the inside out, in reverse order - * SINGLE_PARENT only walks through the hierarchy of the first parent in the head (or heads) - */ - public enum ScanType { - DEPTH_FIRST_ALL_PARENTS, - BLOCK_SCOPES, - SINGLE_PARENT - } /** * Create a predicate that will match on all FlowNodes having a specific action present @@ -71,6 +60,7 @@ public enum ScanType { * @param Action type * @return Predicate that will match when FlowNode has the action given */ + @Nonnull public static Predicate createPredicateWhereActionExists(@Nonnull final Class actionClass) { return new Predicate() { @Override @@ -87,7 +77,7 @@ public boolean apply(FlowNode input) { static final Predicate MATCH_HAS_ERROR = createPredicateWhereActionExists(ErrorAction.class); static final Predicate MATCH_HAS_LOG = createPredicateWhereActionExists(LogAction.class); - public static Predicate createPredicateForStepNodeWithDescriptor(final String descriptorId) { + public static Predicate predicateMatchStepDescriptor(final String descriptorId) { Predicate outputPredicate = new Predicate() { @Override public boolean apply(FlowNode input) { @@ -141,14 +131,15 @@ public static abstract class AbstractFlowScanner implements ScanAlgorithm { // Public APIs need to invoke this before searches protected abstract void initialize(); + protected abstract void setHeads(@Nonnull Collection heads); + /** * Actual meat of the iteration, get the next node to visit, using & updating state as needed - * @param f Node to look for parents of (usually _current) * @param blackList Nodes that are not eligible for visiting * @return Next node to visit, or null if we've exhausted the node list */ @CheckForNull - protected abstract FlowNode next(@CheckForNull FlowNode f, @Nonnull Collection blackList); + protected abstract FlowNode next(@Nonnull Collection blackList); /** Fast internal scan from start through single-parent (unbranched) nodes until we hit a node with one of the following: @@ -185,46 +176,50 @@ protected Collection convertToFastCheckable(@CheckForNull Collection 5 ? new HashSet(nodeCollection) : nodeCollection; } + @CheckForNull public FlowNode findFirstMatch(@CheckForNull Collection heads, @Nonnull Predicate matchPredicate) { return this.findFirstMatch(heads, null, matchPredicate); } + @Nonnull public Collection findAllMatches(@CheckForNull Collection heads, @Nonnull Predicate matchPredicate) { return this.findAllMatches(heads, null, matchPredicate); } // Basic algo impl - protected FlowNode findFirstMatchBasic(@CheckForNull Collection heads, + public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckForNull Collection endNodes, Predicate matchCondition) { if (heads == null || heads.size() == 0) { return null; } initialize(); + this.setHeads(heads); Collection fastEndNodes = convertToFastCheckable(endNodes); - while((this._current = this.next(_current, fastEndNodes)) != null) { - if (matchCondition.apply(this._current)) { - return this._current; + while ((_current = next(fastEndNodes)) != null) { + if (matchCondition.apply(_current)) { + return _current; } } return null; } // Basic algo impl - protected List findAllMatchesBasic(@CheckForNull Collection heads, + public List findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection endNodes, Predicate matchCondition) { if (heads == null || heads.size() == 0) { return null; } initialize(); + this.setHeads(heads); Collection fastEndNodes = convertToFastCheckable(endNodes); ArrayList nodes = new ArrayList(); - while((this._current = this.next(_current, fastEndNodes)) != null) { - if (matchCondition.apply(this._current)) { - nodes.add(this._current); + while ((_current = next(fastEndNodes)) != null) { + if (matchCondition.apply(_current)) { + nodes.add(_current); } } return nodes; @@ -247,66 +242,34 @@ protected void initialize() { } @Override - protected FlowNode next(@CheckForNull FlowNode f, @Nonnull Collection blackList) { - // Check for visited and stuff? - return null; + protected void setHeads(@Nonnull Collection heads) { + // Needs to handle blacklist + _queue.addAll(heads); } @Override - public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { - if (heads == null || heads.size() == 0) { - return null; - } - initialize(); - - HashSet visited = new HashSet(); - ArrayDeque queue = new ArrayDeque(heads); // Only used for parallel branches - Collection fastStopNodes = convertToFastCheckable(stopNodes); - - // TODO this will probably be more efficient if we work with the first node - // or use a recursive solution for parallel forks - while (!queue.isEmpty()) { - FlowNode f = queue.pop(); - if (matchPredicate.apply(f)) { - return f; - } - visited.add(f); - List parents = f.getParents(); // Parents never null - for (FlowNode p : parents) { - if (!visited.contains(p) && !fastStopNodes.contains(p)) { - queue.push(p); + protected FlowNode next(@Nonnull Collection blackList) { + FlowNode output = null; + if (_current != null) { + List parents = _current.getParents(); + if (parents != null) { + for (FlowNode f : parents) { + if (!blackList.contains(f) && !_visited.contains(f)) { + if (output != null ) { + output = f; + } else { + _queue.push(f); + } + } } } } - return null; - } - @Override - public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { - if (heads == null || heads.size() == 0) { - return Collections.EMPTY_LIST; - } - - HashSet visited = new HashSet(); - ArrayDeque queue = new ArrayDeque(heads); // Only used for parallel branches - ArrayList matches = new ArrayList(); - Collection fastStopNodes = convertToFastCheckable(stopNodes); - - // TODO this will probably be more efficient if use a variable for non-parallel flows and don't constantly push/pop array - while (!queue.isEmpty()) { - FlowNode f = queue.pop(); - if (matchPredicate.apply(f)) { - matches.add(f); - } - visited.add(f); - List parents = f.getParents(); // Parents never null - for (FlowNode p : parents) { - if (!visited.contains(p) && !fastStopNodes.contains(p)) { - queue.push(p); - } - } + if (output == null && _queue.size() > 0) { + output = _queue.pop(); } - return matches; + _visited.add(output); + return output; } } @@ -316,64 +279,30 @@ public Collection findAllMatches(@CheckForNull Collection he public static class LinearScanner extends AbstractFlowScanner { @Override - public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { - if (heads == null || heads.size() == 0) { - return null; - } - - Collection fastStopNodes = convertToFastCheckable(stopNodes); + protected void initialize() { + // no-op for us + } - FlowNode current = heads.iterator().next(); - while (current != null) { - if (matchPredicate.apply(current)) { - return current; - } - List parents = current.getParents(); // Parents never null - current = null; - for (FlowNode p : parents) { - if (!fastStopNodes.contains(p)) { - current = p; - break; - } - } + @Override + protected void setHeads(@Nonnull Collection heads) { + if (heads.size() > 0) { + this._current = heads.iterator().next(); } - return current; } @Override - public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { - if (heads == null || heads.size() == 0) { - return Collections.EMPTY_LIST; + protected FlowNode next(@Nonnull Collection blackList) { + if (_current == null) { + return null; } - - // Do what we need to for fast tests - Collection fastStopNodes = convertToFastCheckable(stopNodes); - ArrayList matches = new ArrayList(); - - FlowNode current = heads.iterator().next(); - while (current != null) { - if (matchPredicate.apply(current)) { - matches.add(current); - } - List parents = current.getParents(); // Parents never null - current = null; - for (FlowNode p : parents) { - if (!fastStopNodes.contains(p)) { - current = p; - break; + List parents = _current.getParents(); + if (parents != null || parents.size() > 0) { + for (FlowNode f : parents) { + if (!blackList.contains(f)) { + return f; } } } - return matches; - } - - @Override - protected void initialize() { - // no-op for us - } - - @Override - protected FlowNode next(@CheckForNull FlowNode f, @Nonnull Collection blackList) { return null; } } @@ -411,33 +340,9 @@ public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckF return current; } - @Override - public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { - if (heads == null || heads.size() == 0) { - return Collections.EMPTY_LIST; - } - - // Do what we need to for fast tests - Collection fastStopNodes = convertToFastCheckable(stopNodes); - ArrayList matches = new ArrayList(); - - FlowNode current = heads.iterator().next(); - while (current != null) { - if (!(current instanceof BlockEndNode) && matchPredicate.apply(current)) { - matches.add(current); - } else { // Hop the block - current = ((BlockEndNode) current).getStartNode(); - } - List parents = current.getParents(); // Parents never null - current = null; - for (FlowNode p : parents) { - if (!fastStopNodes.contains(p)) { - current = p; - break; - } - } - } - return matches; + protected FlowNode jumpBlock(FlowNode current) { + return (current instanceof BlockEndNode) ? + ((BlockEndNode)current).getStartNode() : current; } @Override @@ -446,7 +351,23 @@ protected void initialize() { } @Override - protected FlowNode next(@CheckForNull FlowNode f, @Nonnull Collection blackList) { + protected void setHeads(@Nonnull Collection heads) { + _queue.addAll(heads); + } + + @Override + protected FlowNode next(@Nonnull Collection blackList) { + if (_current == null) { + return null; + } + List parents = _current.getParents(); + if (parents != null || parents.size() > 0) { + for (FlowNode f : parents) { + if (!blackList.contains(f)) { + return f; + } + } + } return null; } } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java index 9028396b..d502ae70 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java @@ -49,7 +49,6 @@ public class TestFlowScanner { @Rule public JenkinsRule r = new JenkinsRule(); - /** Tests the basic scan algorithm, predicate use, start/stop nodes */ @Test public void testSimpleScan() throws Exception { @@ -63,13 +62,15 @@ public void testSimpleScan() throws Exception { FlowExecution exec = b.getExecution(); FlowScanner.ScanAlgorithm[] scans = {new FlowScanner.LinearScanner(), new FlowScanner.DepthFirstScanner(), - new FlowScanner.BlockHoppingScanner()}; + new FlowScanner.BlockHoppingScanner() + }; - Predicate echoPredicate = FlowScanner.createPredicateForStepNodeWithDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); + Predicate echoPredicate = FlowScanner.predicateMatchStepDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); List heads = exec.getCurrentHeads(); // Test expected scans with no stop nodes given (different ways of specifying none) for (FlowScanner.ScanAlgorithm sa : scans) { + System.out.println("Testing class: "+sa.getClass()); FlowNode node = sa.findFirstMatch(heads, null, echoPredicate); Assert.assertEquals(exec.getNode("5"), node); node = sa.findFirstMatch(heads, Collections.EMPTY_LIST, echoPredicate); @@ -119,7 +120,7 @@ public void testSimpleScan() throws Exception { /** Tests the basic scan algorithm where blocks are involved */ @Test - public void blockScan() throws Exception { + public void testBlockScan() throws Exception { WorkflowJob job = r.jenkins.createProject(WorkflowJob.class, "Convoluted"); job.setDefinition(new CpsFlowDefinition( "echo 'first'\n" + @@ -130,18 +131,14 @@ public void blockScan() throws Exception { "sleep 1" )); WorkflowRun b = r.assertBuildStatusSuccess(job.scheduleBuild2(0)); - FlowScanner.ScanAlgorithm[] scans = {new FlowScanner.LinearScanner(), - new FlowScanner.DepthFirstScanner(), - new FlowScanner.BlockHoppingScanner()}; - for (FlowScanner.ScanAlgorithm sa : scans) { - } - FlowGraphWalker walk = new FlowGraphWalker(b.getExecution()); - ArrayList flows = new ArrayList(); - for (FlowNode f : walk) { - flows.add(f); - } + // Test blockhopping + FlowScanner.BlockHoppingScanner blockHoppingScanner = new FlowScanner.BlockHoppingScanner(); + Collection matches = blockHoppingScanner.findAllMatches(b.getExecution().getCurrentHeads(), null, + FlowScanner.predicateMatchStepDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep")); + // This means we jumped the blocks + Assert.assertEquals(1, matches.size()); } From d41472305c2db9144592c8c01dc9abdf73a2dcf8 Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Tue, 19 Apr 2016 05:13:30 -0400 Subject: [PATCH 06/16] Finish tests for FlowScanner algorithms and fix remaining edge cases --- .../plugins/workflow/graph/FlowScanner.java | 68 ++++++---------- .../workflow/graph/TestFlowScanner.java | 77 +++++++++++-------- 2 files changed, 69 insertions(+), 76 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java index b77ae46d..86f4468c 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java @@ -193,9 +193,12 @@ public FlowNode findFirstMatch(@CheckForNull Collection heads, if (heads == null || heads.size() == 0) { return null; } + initialize(); - this.setHeads(heads); Collection fastEndNodes = convertToFastCheckable(endNodes); + Collection filteredHeads = new HashSet(heads); + filteredHeads.removeAll(fastEndNodes); + this.setHeads(filteredHeads); while ((_current = next(fastEndNodes)) != null) { if (matchCondition.apply(_current)) { @@ -213,8 +216,10 @@ public List findAllMatches(@CheckForNull Collection heads, return null; } initialize(); - this.setHeads(heads); Collection fastEndNodes = convertToFastCheckable(endNodes); + Collection filteredHeads = new HashSet(heads); + filteredHeads.removeAll(fastEndNodes); + this.setHeads(filteredHeads); ArrayList nodes = new ArrayList(); while ((_current = next(fastEndNodes)) != null) { @@ -277,10 +282,11 @@ protected FlowNode next(@Nonnull Collection blackList) { * Scans through a single ancestry, does not cover parallel branches */ public static class LinearScanner extends AbstractFlowScanner { + protected boolean isFirst = true; @Override protected void initialize() { - // no-op for us + isFirst = true; } @Override @@ -295,6 +301,10 @@ protected FlowNode next(@Nonnull Collection blackList) { if (_current == null) { return null; } + if (isFirst) { // Kind of cheating, but works + isFirst = false; + return _current; + } List parents = _current.getParents(); if (parents != null || parents.size() > 0) { for (FlowNode f : parents) { @@ -310,61 +320,33 @@ protected FlowNode next(@Nonnull Collection blackList) { /** * Scanner that jumps over nested blocks */ - public static class BlockHoppingScanner extends AbstractFlowScanner { - - @Override - public FlowNode findFirstMatch(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate) { - if (heads == null || heads.size() == 0) { - return null; - } - - // Do what we need to for fast tests - Collection fastStopNodes = convertToFastCheckable(stopNodes); - - FlowNode current = heads.iterator().next(); - while (current != null) { - if (!(current instanceof BlockEndNode) && matchPredicate.apply(current)) { - return current; - } else { // Hop the block - current = ((BlockEndNode) current).getStartNode(); - } - List parents = current.getParents(); // Parents never null - current = null; - for (FlowNode p : parents) { - if (!fastStopNodes.contains(p)) { - current = p; - break; - } - } - } - return current; - } + public static class BlockHoppingScanner extends LinearScanner { protected FlowNode jumpBlock(FlowNode current) { return (current instanceof BlockEndNode) ? ((BlockEndNode)current).getStartNode() : current; } - @Override - protected void initialize() { - - } - - @Override - protected void setHeads(@Nonnull Collection heads) { - _queue.addAll(heads); - } - @Override protected FlowNode next(@Nonnull Collection blackList) { if (_current == null) { return null; } + if (isFirst) { // Hax, but solves the problem + isFirst = false; + return _current; + } List parents = _current.getParents(); if (parents != null || parents.size() > 0) { for (FlowNode f : parents) { if (!blackList.contains(f)) { - return f; + FlowNode jumped = jumpBlock(f); + if (jumped != f) { + _current = jumped; + return next(blackList); + } else { + return f; + } } } } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java index d502ae70..094781ec 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java @@ -89,6 +89,7 @@ public void testSimpleScan() throws Exception { // Test with no matches for (FlowScanner.ScanAlgorithm sa : scans) { + System.out.println("Testing class: "+sa.getClass()); FlowNode node = sa.findFirstMatch(heads, null, (Predicate)Predicates.alwaysFalse()); Assert.assertNull(node); @@ -97,6 +98,13 @@ public void testSimpleScan() throws Exception { Assert.assertEquals(0, nodeList.size()); } + // Verify we touch head and foot nodes too + for (FlowScanner.ScanAlgorithm sa : scans) { + System.out.println("Testing class: "+sa.getClass()); + Collection nodeList = sa.findAllMatches(heads, null, (Predicate)Predicates.alwaysTrue()); + Assert.assertEquals(5, nodeList.size()); + } + // Test with a stop node given, sometimes no matches Collection noMatchEndNode = Collections.singleton(exec.getNode("5")); Collection singleMatchEndNode = Collections.singleton(exec.getNode("4")); @@ -131,51 +139,54 @@ public void testBlockScan() throws Exception { "sleep 1" )); WorkflowRun b = r.assertBuildStatusSuccess(job.scheduleBuild2(0)); + Predicate matchEchoStep = FlowScanner.predicateMatchStepDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); // Test blockhopping FlowScanner.BlockHoppingScanner blockHoppingScanner = new FlowScanner.BlockHoppingScanner(); - Collection matches = blockHoppingScanner.findAllMatches(b.getExecution().getCurrentHeads(), null, - FlowScanner.predicateMatchStepDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep")); + Collection matches = blockHoppingScanner.findAllMatches(b.getExecution().getCurrentHeads(), null, matchEchoStep); // This means we jumped the blocks Assert.assertEquals(1, matches.size()); - } + FlowScanner.DepthFirstScanner depthFirstScanner = new FlowScanner.DepthFirstScanner(); + matches = depthFirstScanner.findAllMatches(b.getExecution().getCurrentHeads(), null, matchEchoStep); + + // Nodes all covered + Assert.assertEquals(3, matches.size()); + } + /** And the parallel case */ @Test - public void testme() throws Exception { + public void testParallelScan() throws Exception { WorkflowJob job = r.jenkins.createProject(WorkflowJob.class, "Convoluted"); job.setDefinition(new CpsFlowDefinition( - "echo 'pre-stage command'\n" + - "sleep 1\n" + - "stage 'first'\n" + - "echo 'I ran'\n" + - "stage 'second'\n" + - "node {\n" + - " def steps = [:]\n" + - " steps['2a-dir'] = {\n" + - " echo 'do 2a stuff'\n" + - " echo 'do more 2a stuff'\n" + - " timeout(time: 10, unit: 'SECONDS') {\n" + - " stage 'invalid'\n" + - " echo 'time seconds'\n" + - " }\n" + - " sleep 15\n" + - " }\n" + - " steps['2b'] = {\n" + - " echo 'do 2b stuff'\n" + - " sleep 10\n" + - " echo 'echo_label_me'\n" + - " }\n" + - " parallel steps\n" + - "}\n" + - "\n" + - "stage 'final'\n" + - "echo 'ran final 1'\n" + - "echo 'ran final 2'" + "echo 'first'\n" + + "def steps = [:]\n" + + "steps['1'] = {\n" + + " echo 'do 1 stuff'\n" + + "}\n" + + "steps['2'] = {\n" + + " echo '2a'\n" + + " echo '2b'\n" + + "}\n" + + "parallel steps\n" + + "echo 'final'" )); WorkflowRun b = r.assertBuildStatusSuccess(job.scheduleBuild2(0)); - FlowGraphWalker walker = new FlowGraphWalker(); - walker.addHeads(b.getExecution().getCurrentHeads()); + Collection heads = b.getExecution().getCurrentHeads(); + Predicate matchEchoStep = FlowScanner.predicateMatchStepDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); + + FlowScanner.ScanAlgorithm scanner = new FlowScanner.LinearScanner(); + Collection matches = scanner.findAllMatches(heads, null, matchEchoStep); + Assert.assertTrue(matches.size() >= 3 && matches.size() <= 4); + + scanner = new FlowScanner.DepthFirstScanner(); + matches = scanner.findAllMatches(heads, null, matchEchoStep); + Assert.assertTrue(matches.size() == 5); + + scanner = new FlowScanner.BlockHoppingScanner(); + matches = scanner.findAllMatches(heads, null, matchEchoStep); + Assert.assertTrue(matches.size() == 2); } + } \ No newline at end of file From 4c8faa47f10fd99f2a3ea875e496e80f3a3de1ea Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Tue, 19 Apr 2016 05:36:50 -0400 Subject: [PATCH 07/16] Add visitor method to FlowScanner for collecting stats while walking the graph --- .../plugins/workflow/graph/FlowScanner.java | 28 +++++++++++++++++++ .../workflow/graph/TestFlowScanner.java | 26 ++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java index 86f4468c..1f366aec 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java @@ -25,6 +25,7 @@ */ import com.google.common.base.Predicate; +import com.sun.tools.javac.comp.Flow; import hudson.model.Action; import org.jenkinsci.plugins.workflow.actions.ErrorAction; import org.jenkinsci.plugins.workflow.actions.LabelAction; @@ -91,6 +92,15 @@ public boolean apply(FlowNode input) { return outputPredicate; } + public interface FlowNodeVisitor { + /** + * Visit the flow node, and indicate if we should continue analysis + * @param f Node to visit + * @return False if node is done + */ + public boolean visit(@Nonnull FlowNode f); + } + /** Interface to be used for scanning/analyzing FlowGraphs with support for different visit orders */ public interface ScanAlgorithm { @@ -114,6 +124,9 @@ public interface ScanAlgorithm { */ @Nonnull public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate); + + /** Used for extracting metrics from the flow graph */ + public void visitAll(@CheckForNull Collection heads, FlowNodeVisitor visitor); } /** @@ -229,6 +242,21 @@ public List findAllMatches(@CheckForNull Collection heads, } return nodes; } + + /** Used for extracting metrics from the flow graph */ + public void visitAll(@CheckForNull Collection heads, FlowNodeVisitor visitor) { + if (heads == null || heads.size() == 0) { + return; + } + initialize(); + this.setHeads(heads); + Collection endNodes = Collections.EMPTY_SET; + + boolean continueAnalysis = true; + while (continueAnalysis && (_current = next(endNodes)) != null) { + continueAnalysis = visitor.visit(_current); + } + } } /** Does a simple and efficient depth-first search */ diff --git a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java index 094781ec..b2dce9d4 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java @@ -37,6 +37,7 @@ import org.jvnet.hudson.test.BuildWatcher; import org.jvnet.hudson.test.JenkinsRule; +import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -49,6 +50,24 @@ public class TestFlowScanner { @Rule public JenkinsRule r = new JenkinsRule(); + static final class CollectingVisitor implements FlowScanner.FlowNodeVisitor { + ArrayList visited = new ArrayList(); + + @Override + public boolean visit(@Nonnull FlowNode f) { + visited.add(f); + return true; + } + + public void reset() { + this.visited.clear(); + } + + public ArrayList getVisited() { + return visited; + } + }; + /** Tests the basic scan algorithm, predicate use, start/stop nodes */ @Test public void testSimpleScan() throws Exception { @@ -98,11 +117,16 @@ public void testSimpleScan() throws Exception { Assert.assertEquals(0, nodeList.size()); } + + CollectingVisitor vis = new CollectingVisitor(); // Verify we touch head and foot nodes too for (FlowScanner.ScanAlgorithm sa : scans) { - System.out.println("Testing class: "+sa.getClass()); + System.out.println("Testing class: " + sa.getClass()); Collection nodeList = sa.findAllMatches(heads, null, (Predicate)Predicates.alwaysTrue()); + vis.reset(); + sa.visitAll(heads, vis); Assert.assertEquals(5, nodeList.size()); + Assert.assertEquals(5, vis.getVisited().size()); } // Test with a stop node given, sometimes no matches From eaf47694aa3d488efe4c6ec23e27b03817b0a7bd Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Tue, 19 Apr 2016 05:58:27 -0400 Subject: [PATCH 08/16] Fix FindBugs complaints --- .../plugins/workflow/graph/FlowScanner.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java index 1f366aec..b6eba1a3 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java @@ -25,7 +25,6 @@ */ import com.google.common.base.Predicate; -import com.sun.tools.javac.comp.Flow; import hudson.model.Action; import org.jenkinsci.plugins.workflow.actions.ErrorAction; import org.jenkinsci.plugins.workflow.actions.LabelAction; @@ -33,6 +32,7 @@ import org.jenkinsci.plugins.workflow.actions.StageAction; import org.jenkinsci.plugins.workflow.actions.WorkspaceAction; import org.jenkinsci.plugins.workflow.cps.nodes.StepAtomNode; +import org.jenkinsci.plugins.workflow.steps.StepDescriptor; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; @@ -41,7 +41,6 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; @@ -78,13 +77,14 @@ public boolean apply(FlowNode input) { static final Predicate MATCH_HAS_ERROR = createPredicateWhereActionExists(ErrorAction.class); static final Predicate MATCH_HAS_LOG = createPredicateWhereActionExists(LogAction.class); - public static Predicate predicateMatchStepDescriptor(final String descriptorId) { + public static Predicate predicateMatchStepDescriptor(@Nonnull final String descriptorId) { Predicate outputPredicate = new Predicate() { @Override public boolean apply(FlowNode input) { if (input instanceof StepAtomNode) { StepAtomNode san = (StepAtomNode)input; - return descriptorId.equals(san.getDescriptor().getId()); + StepDescriptor sd = san.getDescriptor(); + return sd != null && descriptorId.equals(sd.getId()); } return false; } @@ -226,7 +226,7 @@ public List findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection endNodes, Predicate matchCondition) { if (heads == null || heads.size() == 0) { - return null; + return Collections.EMPTY_LIST; } initialize(); Collection fastEndNodes = convertToFastCheckable(endNodes); @@ -288,7 +288,7 @@ protected FlowNode next(@Nonnull Collection blackList) { if (parents != null) { for (FlowNode f : parents) { if (!blackList.contains(f) && !_visited.contains(f)) { - if (output != null ) { + if (output == null ) { output = f; } else { _queue.push(f); @@ -334,7 +334,7 @@ protected FlowNode next(@Nonnull Collection blackList) { return _current; } List parents = _current.getParents(); - if (parents != null || parents.size() > 0) { + if (parents != null && parents.size() > 0) { for (FlowNode f : parents) { if (!blackList.contains(f)) { return f; @@ -365,7 +365,7 @@ protected FlowNode next(@Nonnull Collection blackList) { return _current; } List parents = _current.getParents(); - if (parents != null || parents.size() > 0) { + if (parents != null && parents.size() > 0) { for (FlowNode f : parents) { if (!blackList.contains(f)) { FlowNode jumped = jumpBlock(f); From 0ed42f803786569e57c19e6f3215ffcd0e913c41 Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Tue, 19 Apr 2016 05:58:46 -0400 Subject: [PATCH 09/16] Flesh out incremental analysis cache skeleton --- .../graph/IncrementalFlowAnalysis.java | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java index 56eaff55..7c20820e 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java @@ -4,6 +4,7 @@ import com.google.common.base.Predicate; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.jenkinsci.plugins.workflow.flow.FlowExecution; import javax.annotation.CheckForNull; @@ -80,5 +81,36 @@ protected void update(@Nonnull FlowExecution exec) { } } - static Cache analysisCache = CacheBuilder.newBuilder().initialCapacity(100).build(); + public static class IncrementalAnalysisCache { + Function analysisFunction; + Predicate matchCondition; + Cache> analysisCache = CacheBuilder.newBuilder().initialCapacity(100).build(); + + public T getAnalysisValue(@CheckForNull FlowExecution f) { + if (f != null) { + String url; + try { + url = f.getUrl(); + } catch (IOException ioe) { + throw new IllegalStateException(ioe); + } + IncrementalAnalysis analysis = analysisCache.getIfPresent(url); + if (analysis != null) { + return analysis.getUpdatedValue(f); + } else { + IncrementalAnalysis newAnalysis = new IncrementalAnalysis(matchCondition, analysisFunction); + T value = newAnalysis.getUpdatedValue(f); + analysisCache.put(url, newAnalysis); + return value; + } + } + + return null; + } + + public IncrementalAnalysisCache(Predicate matchCondition, Function analysisFunction) { + this.matchCondition = matchCondition; + this.analysisFunction = analysisFunction; + } + } } From 7f4d4882ff14bcca93e4f17093f148f1a469eeca Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Tue, 19 Apr 2016 06:02:07 -0400 Subject: [PATCH 10/16] Move FlowScanner method to search by StepDescriptor into test only to avoid a cyclic plugin dependency --- pom.xml | 2 +- .../plugins/workflow/graph/FlowScanner.java | 17 -------------- .../graph/IncrementalFlowAnalysis.java | 1 - .../workflow/graph/TestFlowScanner.java | 23 ++++++++++++++++--- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/pom.xml b/pom.xml index 5fa4333b..41db0596 100644 --- a/pom.xml +++ b/pom.xml @@ -77,10 +77,10 @@ test - ${project.groupId} workflow-cps 2.2-SNAPSHOT + test ${project.groupId} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java index b6eba1a3..1d9c0018 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java @@ -31,8 +31,6 @@ import org.jenkinsci.plugins.workflow.actions.LogAction; import org.jenkinsci.plugins.workflow.actions.StageAction; import org.jenkinsci.plugins.workflow.actions.WorkspaceAction; -import org.jenkinsci.plugins.workflow.cps.nodes.StepAtomNode; -import org.jenkinsci.plugins.workflow.steps.StepDescriptor; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; @@ -77,21 +75,6 @@ public boolean apply(FlowNode input) { static final Predicate MATCH_HAS_ERROR = createPredicateWhereActionExists(ErrorAction.class); static final Predicate MATCH_HAS_LOG = createPredicateWhereActionExists(LogAction.class); - public static Predicate predicateMatchStepDescriptor(@Nonnull final String descriptorId) { - Predicate outputPredicate = new Predicate() { - @Override - public boolean apply(FlowNode input) { - if (input instanceof StepAtomNode) { - StepAtomNode san = (StepAtomNode)input; - StepDescriptor sd = san.getDescriptor(); - return sd != null && descriptorId.equals(sd.getId()); - } - return false; - } - }; - return outputPredicate; - } - public interface FlowNodeVisitor { /** * Visit the flow node, and indicate if we should continue analysis diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java index 7c20820e..5a0b4190 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java @@ -4,7 +4,6 @@ import com.google.common.base.Predicate; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.jenkinsci.plugins.workflow.flow.FlowExecution; import javax.annotation.CheckForNull; diff --git a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java index b2dce9d4..79898d3f 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java @@ -27,9 +27,11 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; +import org.jenkinsci.plugins.workflow.cps.nodes.StepAtomNode; import org.jenkinsci.plugins.workflow.flow.FlowExecution; import org.jenkinsci.plugins.workflow.job.WorkflowJob; import org.jenkinsci.plugins.workflow.job.WorkflowRun; +import org.jenkinsci.plugins.workflow.steps.StepDescriptor; import org.junit.Assert; import org.junit.ClassRule; import org.junit.Rule; @@ -50,6 +52,21 @@ public class TestFlowScanner { @Rule public JenkinsRule r = new JenkinsRule(); + public static Predicate predicateMatchStepDescriptor(@Nonnull final String descriptorId) { + Predicate outputPredicate = new Predicate() { + @Override + public boolean apply(FlowNode input) { + if (input instanceof StepAtomNode) { + StepAtomNode san = (StepAtomNode)input; + StepDescriptor sd = san.getDescriptor(); + return sd != null && descriptorId.equals(sd.getId()); + } + return false; + } + }; + return outputPredicate; + } + static final class CollectingVisitor implements FlowScanner.FlowNodeVisitor { ArrayList visited = new ArrayList(); @@ -84,7 +101,7 @@ public void testSimpleScan() throws Exception { new FlowScanner.BlockHoppingScanner() }; - Predicate echoPredicate = FlowScanner.predicateMatchStepDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); + Predicate echoPredicate = predicateMatchStepDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); List heads = exec.getCurrentHeads(); // Test expected scans with no stop nodes given (different ways of specifying none) @@ -163,7 +180,7 @@ public void testBlockScan() throws Exception { "sleep 1" )); WorkflowRun b = r.assertBuildStatusSuccess(job.scheduleBuild2(0)); - Predicate matchEchoStep = FlowScanner.predicateMatchStepDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); + Predicate matchEchoStep = predicateMatchStepDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); // Test blockhopping FlowScanner.BlockHoppingScanner blockHoppingScanner = new FlowScanner.BlockHoppingScanner(); @@ -198,7 +215,7 @@ public void testParallelScan() throws Exception { )); WorkflowRun b = r.assertBuildStatusSuccess(job.scheduleBuild2(0)); Collection heads = b.getExecution().getCurrentHeads(); - Predicate matchEchoStep = FlowScanner.predicateMatchStepDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); + Predicate matchEchoStep = predicateMatchStepDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); FlowScanner.ScanAlgorithm scanner = new FlowScanner.LinearScanner(); Collection matches = scanner.findAllMatches(heads, null, matchEchoStep); From deb585b03a23b444da1f6484195c98f589b004f0 Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Mon, 25 Apr 2016 17:43:54 -0400 Subject: [PATCH 11/16] Rename findAllMatches to filter --- .../plugins/workflow/graph/FlowScanner.java | 10 ++++---- .../workflow/graph/TestFlowScanner.java | 24 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java index 1d9c0018..a7676dcf 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/FlowScanner.java @@ -106,7 +106,7 @@ public interface ScanAlgorithm { * @return All nodes matching condition */ @Nonnull - public Collection findAllMatches(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate); + public Collection filter(@CheckForNull Collection heads, @CheckForNull Collection stopNodes, @Nonnull Predicate matchPredicate); /** Used for extracting metrics from the flow graph */ public void visitAll(@CheckForNull Collection heads, FlowNodeVisitor visitor); @@ -179,7 +179,7 @@ public FlowNode findFirstMatch(@CheckForNull Collection heads, @Nonnul @Nonnull public Collection findAllMatches(@CheckForNull Collection heads, @Nonnull Predicate matchPredicate) { - return this.findAllMatches(heads, null, matchPredicate); + return this.filter(heads, null, matchPredicate); } // Basic algo impl @@ -205,9 +205,9 @@ public FlowNode findFirstMatch(@CheckForNull Collection heads, } // Basic algo impl - public List findAllMatches(@CheckForNull Collection heads, - @CheckForNull Collection endNodes, - Predicate matchCondition) { + public List filter(@CheckForNull Collection heads, + @CheckForNull Collection endNodes, + Predicate matchCondition) { if (heads == null || heads.size() == 0) { return Collections.EMPTY_LIST; } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java index 79898d3f..3983664c 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestFlowScanner.java @@ -114,12 +114,12 @@ public void testSimpleScan() throws Exception { node = sa.findFirstMatch(heads, Collections.EMPTY_SET, echoPredicate); Assert.assertEquals(exec.getNode("5"), node); - Collection nodeList = sa.findAllMatches(heads, null, echoPredicate); + Collection nodeList = sa.filter(heads, null, echoPredicate); FlowNode[] expected = new FlowNode[]{exec.getNode("5"), exec.getNode("4")}; Assert.assertArrayEquals(expected, nodeList.toArray()); - nodeList = sa.findAllMatches(heads, Collections.EMPTY_LIST, echoPredicate); + nodeList = sa.filter(heads, Collections.EMPTY_LIST, echoPredicate); Assert.assertArrayEquals(expected, nodeList.toArray()); - nodeList = sa.findAllMatches(heads, Collections.EMPTY_SET, echoPredicate); + nodeList = sa.filter(heads, Collections.EMPTY_SET, echoPredicate); Assert.assertArrayEquals(expected, nodeList.toArray()); } @@ -129,7 +129,7 @@ public void testSimpleScan() throws Exception { FlowNode node = sa.findFirstMatch(heads, null, (Predicate)Predicates.alwaysFalse()); Assert.assertNull(node); - Collection nodeList = sa.findAllMatches(heads, null, (Predicate)Predicates.alwaysFalse()); + Collection nodeList = sa.filter(heads, null, (Predicate) Predicates.alwaysFalse()); Assert.assertNotNull(nodeList); Assert.assertEquals(0, nodeList.size()); } @@ -139,7 +139,7 @@ public void testSimpleScan() throws Exception { // Verify we touch head and foot nodes too for (FlowScanner.ScanAlgorithm sa : scans) { System.out.println("Testing class: " + sa.getClass()); - Collection nodeList = sa.findAllMatches(heads, null, (Predicate)Predicates.alwaysTrue()); + Collection nodeList = sa.filter(heads, null, (Predicate) Predicates.alwaysTrue()); vis.reset(); sa.visitAll(heads, vis); Assert.assertEquals(5, nodeList.size()); @@ -153,14 +153,14 @@ public void testSimpleScan() throws Exception { FlowNode node = sa.findFirstMatch(heads, noMatchEndNode, echoPredicate); Assert.assertNull(node); - Collection nodeList = sa.findAllMatches(heads, noMatchEndNode, echoPredicate); + Collection nodeList = sa.filter(heads, noMatchEndNode, echoPredicate); Assert.assertNotNull(nodeList); Assert.assertEquals(0, nodeList.size()); // Now we try with a stop list the reduces node set for multiple matches node = sa.findFirstMatch(heads, singleMatchEndNode, echoPredicate); Assert.assertEquals(exec.getNode("5"), node); - nodeList = sa.findAllMatches(heads, singleMatchEndNode, echoPredicate); + nodeList = sa.filter(heads, singleMatchEndNode, echoPredicate); Assert.assertNotNull(nodeList); Assert.assertEquals(1, nodeList.size()); Assert.assertEquals(exec.getNode("5"), nodeList.iterator().next()); @@ -184,13 +184,13 @@ public void testBlockScan() throws Exception { // Test blockhopping FlowScanner.BlockHoppingScanner blockHoppingScanner = new FlowScanner.BlockHoppingScanner(); - Collection matches = blockHoppingScanner.findAllMatches(b.getExecution().getCurrentHeads(), null, matchEchoStep); + Collection matches = blockHoppingScanner.filter(b.getExecution().getCurrentHeads(), null, matchEchoStep); // This means we jumped the blocks Assert.assertEquals(1, matches.size()); FlowScanner.DepthFirstScanner depthFirstScanner = new FlowScanner.DepthFirstScanner(); - matches = depthFirstScanner.findAllMatches(b.getExecution().getCurrentHeads(), null, matchEchoStep); + matches = depthFirstScanner.filter(b.getExecution().getCurrentHeads(), null, matchEchoStep); // Nodes all covered Assert.assertEquals(3, matches.size()); @@ -218,15 +218,15 @@ public void testParallelScan() throws Exception { Predicate matchEchoStep = predicateMatchStepDescriptor("org.jenkinsci.plugins.workflow.steps.EchoStep"); FlowScanner.ScanAlgorithm scanner = new FlowScanner.LinearScanner(); - Collection matches = scanner.findAllMatches(heads, null, matchEchoStep); + Collection matches = scanner.filter(heads, null, matchEchoStep); Assert.assertTrue(matches.size() >= 3 && matches.size() <= 4); scanner = new FlowScanner.DepthFirstScanner(); - matches = scanner.findAllMatches(heads, null, matchEchoStep); + matches = scanner.filter(heads, null, matchEchoStep); Assert.assertTrue(matches.size() == 5); scanner = new FlowScanner.BlockHoppingScanner(); - matches = scanner.findAllMatches(heads, null, matchEchoStep); + matches = scanner.filter(heads, null, matchEchoStep); Assert.assertTrue(matches.size() == 2); } From 72e49df68ac14865c79056cdd4ccbd5cd0fa6b7e Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Mon, 25 Apr 2016 18:15:03 -0400 Subject: [PATCH 12/16] Refactor flow graph incremental analysis, add stub of a test until I can figure out how to get it to use something like semaphore step --- ...java => IncrementalFlowAnalysisCache.java} | 86 ++++++++++++------- .../graph/TestIncrementalFlowAnalysis.java | 85 ++++++++++++++++++ 2 files changed, 142 insertions(+), 29 deletions(-) rename src/main/java/org/jenkinsci/plugins/workflow/graph/{IncrementalFlowAnalysis.java => IncrementalFlowAnalysisCache.java} (53%) create mode 100644 src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java similarity index 53% rename from src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java rename to src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java index 5a0b4190..f0f1c6e5 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysis.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java @@ -1,3 +1,27 @@ +/* + * The MIT License + * + * Copyright (c) 2016, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + package org.jenkinsci.plugins.workflow.graph; import com.google.common.base.Function; @@ -16,7 +40,11 @@ * Provides incremental analysis of flow graphs, where updates are on the head * @author Sam Van Oort */ -public class IncrementalFlowAnalysis { +public class IncrementalFlowAnalysisCache { + + Function analysisFunction; + Predicate matchCondition; + Cache> analysisCache = CacheBuilder.newBuilder().initialCapacity(100).build(); protected static class IncrementalAnalysis { protected List lastHeadIds = new ArrayList(); @@ -80,36 +108,36 @@ protected void update(@Nonnull FlowExecution exec) { } } - public static class IncrementalAnalysisCache { - Function analysisFunction; - Predicate matchCondition; - Cache> analysisCache = CacheBuilder.newBuilder().initialCapacity(100).build(); - - public T getAnalysisValue(@CheckForNull FlowExecution f) { - if (f != null) { - String url; - try { - url = f.getUrl(); - } catch (IOException ioe) { - throw new IllegalStateException(ioe); - } - IncrementalAnalysis analysis = analysisCache.getIfPresent(url); - if (analysis != null) { - return analysis.getUpdatedValue(f); - } else { - IncrementalAnalysis newAnalysis = new IncrementalAnalysis(matchCondition, analysisFunction); - T value = newAnalysis.getUpdatedValue(f); - analysisCache.put(url, newAnalysis); - return value; - } + public T getAnalysisValue(@CheckForNull FlowExecution f) { + if (f != null) { + String url; + try { + url = f.getUrl(); + } catch (IOException ioe) { + throw new IllegalStateException(ioe); + } + IncrementalAnalysis analysis = analysisCache.getIfPresent(url); + if (analysis != null) { + return analysis.getUpdatedValue(f); + } else { + IncrementalAnalysis newAnalysis = new IncrementalAnalysis(matchCondition, analysisFunction); + T value = newAnalysis.getUpdatedValue(f); + analysisCache.put(url, newAnalysis); + return value; } - - return null; } - public IncrementalAnalysisCache(Predicate matchCondition, Function analysisFunction) { - this.matchCondition = matchCondition; - this.analysisFunction = analysisFunction; - } + return null; + } + + public IncrementalFlowAnalysisCache(Predicate matchCondition, Function analysisFunction) { + this.matchCondition = matchCondition; + this.analysisFunction = analysisFunction; + } + + public IncrementalFlowAnalysisCache(Predicate matchCondition, Function analysisFunction, Cache myCache) { + this.matchCondition = matchCondition; + this.analysisFunction = analysisFunction; + this.analysisCache = myCache; } } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java new file mode 100644 index 00000000..aef6864a --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java @@ -0,0 +1,85 @@ +/* + * The MIT License + * + * Copyright (c) 2016, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.graph; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import org.jenkinsci.plugins.workflow.actions.LabelAction; +import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition +import org.jenkinsci.plugins.workflow.job.WorkflowJob; +import org.jenkinsci.plugins.workflow.job.WorkflowRun; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.jvnet.hudson.test.BuildWatcher; +import org.jvnet.hudson.test.JenkinsRule; +import org.jenkinsci.plugins.workflow.test.steps.SemaphoreStep; + +import java.util.Collection; + +/** + * @author svanoort + */ +public class TestIncrementalFlowAnalysis { + @ClassRule + public static BuildWatcher buildWatcher = new BuildWatcher(); + + @Rule + public JenkinsRule r = new JenkinsRule(); + + /** Tests the basic incremental analysis */ + @Test + public void testIncrementalAnalysis() throws Exception { + WorkflowJob job = r.jenkins.createProject(WorkflowJob.class, "Convoluted"); + job.setDefinition(new CpsFlowDefinition( + "for (int i=0; i<4; i++) {\n" + + " stage \"stage-$i\"\n" + + " echo \"Doing $i\"\n" + + " semaphore 'wait'\n" + + "}" + )); + + // Search conditions + Predicate labelledNode = FlowScanner.createPredicateWhereActionExists(LabelAction.class); + Function getLabelFunction = new Function() { + @Override + public String apply(FlowNode input) { + LabelAction labelled = input.getAction(LabelAction.class); + return (labelled != null) ? labelled.getDisplayName() : null; + } + }; + + IncrementalFlowAnalysisCache incrementalAnalysis = new IncrementalFlowAnalysisCache(labelledNode, getLabelFunction); + + // TODO how the devil do I test this, when SemaphoreStep is part of another repo's test classes? + } + + /** Tests analysis where there are multiple heads (parallel excecution blocks) */ + @Test + public void testIncrementalAnalysisParallel() throws Exception { + // TODO figure out a case where this is actually a thing? + } +} From eb485520d01816854c7051d0866a833a179e6e9b Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Mon, 25 Apr 2016 18:16:12 -0400 Subject: [PATCH 13/16] Fix imports in incremental flow analysis --- .../plugins/workflow/graph/TestIncrementalFlowAnalysis.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java index aef6864a..716d444a 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java @@ -27,18 +27,14 @@ import com.google.common.base.Function; import com.google.common.base.Predicate; import org.jenkinsci.plugins.workflow.actions.LabelAction; -import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition +import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; import org.jenkinsci.plugins.workflow.job.WorkflowJob; -import org.jenkinsci.plugins.workflow.job.WorkflowRun; -import org.junit.Assert; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import org.jvnet.hudson.test.BuildWatcher; import org.jvnet.hudson.test.JenkinsRule; -import org.jenkinsci.plugins.workflow.test.steps.SemaphoreStep; -import java.util.Collection; /** * @author svanoort From e640fd33a333e12357b52d7e4e31aa0197a54103 Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Tue, 26 Apr 2016 15:05:39 -0400 Subject: [PATCH 14/16] Refactor the incremental flow analysis to allow testing by running incrementally --- .../graph/IncrementalFlowAnalysisCache.java | 81 ++++++++++++++----- .../graph/TestIncrementalFlowAnalysis.java | 14 ++-- 2 files changed, 67 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java index f0f1c6e5..6742dc76 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java @@ -37,7 +37,11 @@ import java.util.List; /** - * Provides incremental analysis of flow graphs, where updates are on the head + * Provides an efficient way to find the most recent (closest to head) node matching a condition, and get info about it + * + * This is useful in cases where we are watching an in-progress pipeline execution. + * It uses caching and only looks at new nodes (the delta since last execution). + * @TODO Thread safety? * @author Sam Van Oort */ public class IncrementalFlowAnalysisCache { @@ -47,7 +51,7 @@ public class IncrementalFlowAnalysisCache { Cache> analysisCache = CacheBuilder.newBuilder().initialCapacity(100).build(); protected static class IncrementalAnalysis { - protected List lastHeadIds = new ArrayList(); + protected List lastHeadIds = new ArrayList(); // We don't want to hold refs to the actual nodes protected T lastValue; /** Gets value from a flownode */ @@ -73,32 +77,55 @@ public T getUpdatedValue(@CheckForNull FlowExecution exec) { return null; } List heads = exec.getCurrentHeads(); - if (heads != null && heads.size() == lastHeadIds.size()) { - boolean useCache = false; + if (heads == null || heads.size() == 0) { + return null; + } + return getUpdatedValueInternal(exec, heads); + } + + @CheckForNull + public T getUpdatedValue(@CheckForNull FlowExecution exec, @Nonnull List heads) { + if (exec == null || heads.size() == 0) { + return null; + } + return getUpdatedValueInternal(exec, heads); + } + + /** + * Internal implementation + * @param exec Execution, used in obtaining node instances + * @param heads Heads to scan from, cannot be empty + * @return Updated value or null if not present + */ + @CheckForNull + protected T getUpdatedValueInternal(@Nonnull FlowExecution exec, @Nonnull List heads) { + boolean hasChanged = heads.size() == lastHeadIds.size(); + if (hasChanged) { for (FlowNode f : heads) { - if (lastHeadIds.contains(f.getId())) { - useCache = true; + if (!lastHeadIds.contains(f.getId())) { + hasChanged = false; break; } } - if (!useCache) { - update(exec); - } - return lastValue; } - return null; + if (!hasChanged) { + updateInternal(exec, heads); + } + return lastValue; } - protected void update(@Nonnull FlowExecution exec) { - ArrayList nodes = new ArrayList(); + // FlowExecution is used for look + protected void updateInternal(@Nonnull FlowExecution exec, @Nonnull List heads) { + ArrayList stopNodes = new ArrayList(); + // Fetch the actual flow nodes to use as halt conditions for (String nodeId : this.lastHeadIds) { try { - nodes.add(exec.getNode(nodeId)); + stopNodes.add(exec.getNode(nodeId)); } catch (IOException ioe) { throw new IllegalStateException(ioe); } } - FlowNode matchNode = new FlowScanner.BlockHoppingScanner().findFirstMatch(exec.getCurrentHeads(), nodes, this.nodeMatchCondition); + FlowNode matchNode = new FlowScanner.BlockHoppingScanner().findFirstMatch(heads, stopNodes, this.nodeMatchCondition); this.lastValue = this.valueExtractor.apply(matchNode); this.lastHeadIds.clear(); @@ -108,25 +135,39 @@ protected void update(@Nonnull FlowExecution exec) { } } + /** + * Get the latest value, using the heads of a FlowExecutions + * @param f Flow executions + * @return Analysis value, or null no nodes match condition/flow has not begun + */ + @CheckForNull public T getAnalysisValue(@CheckForNull FlowExecution f) { - if (f != null) { + if (f == null) { + return null; + } else { + return getAnalysisValue(f, f.getCurrentHeads()); + } + } + + @CheckForNull + public T getAnalysisValue(@CheckForNull FlowExecution exec, @CheckForNull List heads) { + if (exec != null && heads == null && heads.size() != 0) { String url; try { - url = f.getUrl(); + url = exec.getUrl(); } catch (IOException ioe) { throw new IllegalStateException(ioe); } IncrementalAnalysis analysis = analysisCache.getIfPresent(url); if (analysis != null) { - return analysis.getUpdatedValue(f); + return analysis.getUpdatedValue(exec, heads); } else { IncrementalAnalysis newAnalysis = new IncrementalAnalysis(matchCondition, analysisFunction); - T value = newAnalysis.getUpdatedValue(f); + T value = newAnalysis.getUpdatedValue(exec, heads); analysisCache.put(url, newAnalysis); return value; } } - return null; } diff --git a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java index 716d444a..5e1c87ee 100644 --- a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java +++ b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java @@ -28,7 +28,9 @@ import com.google.common.base.Predicate; import org.jenkinsci.plugins.workflow.actions.LabelAction; import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; +import org.jenkinsci.plugins.workflow.flow.FlowExecution; import org.jenkinsci.plugins.workflow.job.WorkflowJob; +import org.jenkinsci.plugins.workflow.job.WorkflowRun; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -54,7 +56,6 @@ public void testIncrementalAnalysis() throws Exception { "for (int i=0; i<4; i++) {\n" + " stage \"stage-$i\"\n" + " echo \"Doing $i\"\n" + - " semaphore 'wait'\n" + "}" )); @@ -69,13 +70,10 @@ public String apply(FlowNode input) { }; IncrementalFlowAnalysisCache incrementalAnalysis = new IncrementalFlowAnalysisCache(labelledNode, getLabelFunction); + WorkflowRun b = r.assertBuildStatusSuccess(job.scheduleBuild2(0)); + FlowExecution exec = b.getExecution(); + FlowNode test = exec.getNode("4"); - // TODO how the devil do I test this, when SemaphoreStep is part of another repo's test classes? - } - - /** Tests analysis where there are multiple heads (parallel excecution blocks) */ - @Test - public void testIncrementalAnalysisParallel() throws Exception { - // TODO figure out a case where this is actually a thing? + // TODO add tests based on calling incremental analysis from points further along flow, possible in some paralle cases } } From 7c1e6df4fae63b0c7c2de4f8e7bfb754dfa6e7b4 Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Tue, 26 Apr 2016 15:06:46 -0400 Subject: [PATCH 15/16] Remove the incremental flow analysis so we can push it into a separate PR --- .../graph/IncrementalFlowAnalysisCache.java | 184 ------------------ .../graph/TestIncrementalFlowAnalysis.java | 79 -------- 2 files changed, 263 deletions(-) delete mode 100644 src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java delete mode 100644 src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java deleted file mode 100644 index 6742dc76..00000000 --- a/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * The MIT License - * - * Copyright (c) 2016, CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -package org.jenkinsci.plugins.workflow.graph; - -import com.google.common.base.Function; -import com.google.common.base.Predicate; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import org.jenkinsci.plugins.workflow.flow.FlowExecution; - -import javax.annotation.CheckForNull; -import javax.annotation.Nonnull; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Provides an efficient way to find the most recent (closest to head) node matching a condition, and get info about it - * - * This is useful in cases where we are watching an in-progress pipeline execution. - * It uses caching and only looks at new nodes (the delta since last execution). - * @TODO Thread safety? - * @author Sam Van Oort - */ -public class IncrementalFlowAnalysisCache { - - Function analysisFunction; - Predicate matchCondition; - Cache> analysisCache = CacheBuilder.newBuilder().initialCapacity(100).build(); - - protected static class IncrementalAnalysis { - protected List lastHeadIds = new ArrayList(); // We don't want to hold refs to the actual nodes - protected T lastValue; - - /** Gets value from a flownode */ - protected Function valueExtractor; - - protected Predicate nodeMatchCondition; - - public IncrementalAnalysis(@Nonnull Predicate nodeMatchCondition, @Nonnull Function valueExtractFunction){ - this.nodeMatchCondition = nodeMatchCondition; - this.valueExtractor = valueExtractFunction; - } - - /** - * Look up a value scanned from the flow - * If the heads haven't changed in the flow, return the current heads - * If they have, only hunt from the current value until the last one - * @param exec - * @return - */ - @CheckForNull - public T getUpdatedValue(@CheckForNull FlowExecution exec) { - if (exec == null) { - return null; - } - List heads = exec.getCurrentHeads(); - if (heads == null || heads.size() == 0) { - return null; - } - return getUpdatedValueInternal(exec, heads); - } - - @CheckForNull - public T getUpdatedValue(@CheckForNull FlowExecution exec, @Nonnull List heads) { - if (exec == null || heads.size() == 0) { - return null; - } - return getUpdatedValueInternal(exec, heads); - } - - /** - * Internal implementation - * @param exec Execution, used in obtaining node instances - * @param heads Heads to scan from, cannot be empty - * @return Updated value or null if not present - */ - @CheckForNull - protected T getUpdatedValueInternal(@Nonnull FlowExecution exec, @Nonnull List heads) { - boolean hasChanged = heads.size() == lastHeadIds.size(); - if (hasChanged) { - for (FlowNode f : heads) { - if (!lastHeadIds.contains(f.getId())) { - hasChanged = false; - break; - } - } - } - if (!hasChanged) { - updateInternal(exec, heads); - } - return lastValue; - } - - // FlowExecution is used for look - protected void updateInternal(@Nonnull FlowExecution exec, @Nonnull List heads) { - ArrayList stopNodes = new ArrayList(); - // Fetch the actual flow nodes to use as halt conditions - for (String nodeId : this.lastHeadIds) { - try { - stopNodes.add(exec.getNode(nodeId)); - } catch (IOException ioe) { - throw new IllegalStateException(ioe); - } - } - FlowNode matchNode = new FlowScanner.BlockHoppingScanner().findFirstMatch(heads, stopNodes, this.nodeMatchCondition); - this.lastValue = this.valueExtractor.apply(matchNode); - - this.lastHeadIds.clear(); - for (FlowNode f : exec.getCurrentHeads()) { - lastHeadIds.add(f.getId()); - } - } - } - - /** - * Get the latest value, using the heads of a FlowExecutions - * @param f Flow executions - * @return Analysis value, or null no nodes match condition/flow has not begun - */ - @CheckForNull - public T getAnalysisValue(@CheckForNull FlowExecution f) { - if (f == null) { - return null; - } else { - return getAnalysisValue(f, f.getCurrentHeads()); - } - } - - @CheckForNull - public T getAnalysisValue(@CheckForNull FlowExecution exec, @CheckForNull List heads) { - if (exec != null && heads == null && heads.size() != 0) { - String url; - try { - url = exec.getUrl(); - } catch (IOException ioe) { - throw new IllegalStateException(ioe); - } - IncrementalAnalysis analysis = analysisCache.getIfPresent(url); - if (analysis != null) { - return analysis.getUpdatedValue(exec, heads); - } else { - IncrementalAnalysis newAnalysis = new IncrementalAnalysis(matchCondition, analysisFunction); - T value = newAnalysis.getUpdatedValue(exec, heads); - analysisCache.put(url, newAnalysis); - return value; - } - } - return null; - } - - public IncrementalFlowAnalysisCache(Predicate matchCondition, Function analysisFunction) { - this.matchCondition = matchCondition; - this.analysisFunction = analysisFunction; - } - - public IncrementalFlowAnalysisCache(Predicate matchCondition, Function analysisFunction, Cache myCache) { - this.matchCondition = matchCondition; - this.analysisFunction = analysisFunction; - this.analysisCache = myCache; - } -} diff --git a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java deleted file mode 100644 index 5e1c87ee..00000000 --- a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * The MIT License - * - * Copyright (c) 2016, CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -package org.jenkinsci.plugins.workflow.graph; - -import com.google.common.base.Function; -import com.google.common.base.Predicate; -import org.jenkinsci.plugins.workflow.actions.LabelAction; -import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; -import org.jenkinsci.plugins.workflow.flow.FlowExecution; -import org.jenkinsci.plugins.workflow.job.WorkflowJob; -import org.jenkinsci.plugins.workflow.job.WorkflowRun; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.jvnet.hudson.test.BuildWatcher; -import org.jvnet.hudson.test.JenkinsRule; - - -/** - * @author svanoort - */ -public class TestIncrementalFlowAnalysis { - @ClassRule - public static BuildWatcher buildWatcher = new BuildWatcher(); - - @Rule - public JenkinsRule r = new JenkinsRule(); - - /** Tests the basic incremental analysis */ - @Test - public void testIncrementalAnalysis() throws Exception { - WorkflowJob job = r.jenkins.createProject(WorkflowJob.class, "Convoluted"); - job.setDefinition(new CpsFlowDefinition( - "for (int i=0; i<4; i++) {\n" + - " stage \"stage-$i\"\n" + - " echo \"Doing $i\"\n" + - "}" - )); - - // Search conditions - Predicate labelledNode = FlowScanner.createPredicateWhereActionExists(LabelAction.class); - Function getLabelFunction = new Function() { - @Override - public String apply(FlowNode input) { - LabelAction labelled = input.getAction(LabelAction.class); - return (labelled != null) ? labelled.getDisplayName() : null; - } - }; - - IncrementalFlowAnalysisCache incrementalAnalysis = new IncrementalFlowAnalysisCache(labelledNode, getLabelFunction); - WorkflowRun b = r.assertBuildStatusSuccess(job.scheduleBuild2(0)); - FlowExecution exec = b.getExecution(); - FlowNode test = exec.getNode("4"); - - // TODO add tests based on calling incremental analysis from points further along flow, possible in some paralle cases - } -} From 66682d17049fc0fb7559182650d39bc34304084f Mon Sep 17 00:00:00 2001 From: Sam Van Oort Date: Tue, 26 Apr 2016 15:07:10 -0400 Subject: [PATCH 16/16] Revert "Remove the incremental flow analysis so we can push it into a separate PR" This reverts commit 7c1e6df4fae63b0c7c2de4f8e7bfb754dfa6e7b4. --- .../graph/IncrementalFlowAnalysisCache.java | 184 ++++++++++++++++++ .../graph/TestIncrementalFlowAnalysis.java | 79 ++++++++ 2 files changed, 263 insertions(+) create mode 100644 src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java create mode 100644 src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java diff --git a/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java new file mode 100644 index 00000000..6742dc76 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/graph/IncrementalFlowAnalysisCache.java @@ -0,0 +1,184 @@ +/* + * The MIT License + * + * Copyright (c) 2016, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.graph; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.jenkinsci.plugins.workflow.flow.FlowExecution; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Provides an efficient way to find the most recent (closest to head) node matching a condition, and get info about it + * + * This is useful in cases where we are watching an in-progress pipeline execution. + * It uses caching and only looks at new nodes (the delta since last execution). + * @TODO Thread safety? + * @author Sam Van Oort + */ +public class IncrementalFlowAnalysisCache { + + Function analysisFunction; + Predicate matchCondition; + Cache> analysisCache = CacheBuilder.newBuilder().initialCapacity(100).build(); + + protected static class IncrementalAnalysis { + protected List lastHeadIds = new ArrayList(); // We don't want to hold refs to the actual nodes + protected T lastValue; + + /** Gets value from a flownode */ + protected Function valueExtractor; + + protected Predicate nodeMatchCondition; + + public IncrementalAnalysis(@Nonnull Predicate nodeMatchCondition, @Nonnull Function valueExtractFunction){ + this.nodeMatchCondition = nodeMatchCondition; + this.valueExtractor = valueExtractFunction; + } + + /** + * Look up a value scanned from the flow + * If the heads haven't changed in the flow, return the current heads + * If they have, only hunt from the current value until the last one + * @param exec + * @return + */ + @CheckForNull + public T getUpdatedValue(@CheckForNull FlowExecution exec) { + if (exec == null) { + return null; + } + List heads = exec.getCurrentHeads(); + if (heads == null || heads.size() == 0) { + return null; + } + return getUpdatedValueInternal(exec, heads); + } + + @CheckForNull + public T getUpdatedValue(@CheckForNull FlowExecution exec, @Nonnull List heads) { + if (exec == null || heads.size() == 0) { + return null; + } + return getUpdatedValueInternal(exec, heads); + } + + /** + * Internal implementation + * @param exec Execution, used in obtaining node instances + * @param heads Heads to scan from, cannot be empty + * @return Updated value or null if not present + */ + @CheckForNull + protected T getUpdatedValueInternal(@Nonnull FlowExecution exec, @Nonnull List heads) { + boolean hasChanged = heads.size() == lastHeadIds.size(); + if (hasChanged) { + for (FlowNode f : heads) { + if (!lastHeadIds.contains(f.getId())) { + hasChanged = false; + break; + } + } + } + if (!hasChanged) { + updateInternal(exec, heads); + } + return lastValue; + } + + // FlowExecution is used for look + protected void updateInternal(@Nonnull FlowExecution exec, @Nonnull List heads) { + ArrayList stopNodes = new ArrayList(); + // Fetch the actual flow nodes to use as halt conditions + for (String nodeId : this.lastHeadIds) { + try { + stopNodes.add(exec.getNode(nodeId)); + } catch (IOException ioe) { + throw new IllegalStateException(ioe); + } + } + FlowNode matchNode = new FlowScanner.BlockHoppingScanner().findFirstMatch(heads, stopNodes, this.nodeMatchCondition); + this.lastValue = this.valueExtractor.apply(matchNode); + + this.lastHeadIds.clear(); + for (FlowNode f : exec.getCurrentHeads()) { + lastHeadIds.add(f.getId()); + } + } + } + + /** + * Get the latest value, using the heads of a FlowExecutions + * @param f Flow executions + * @return Analysis value, or null no nodes match condition/flow has not begun + */ + @CheckForNull + public T getAnalysisValue(@CheckForNull FlowExecution f) { + if (f == null) { + return null; + } else { + return getAnalysisValue(f, f.getCurrentHeads()); + } + } + + @CheckForNull + public T getAnalysisValue(@CheckForNull FlowExecution exec, @CheckForNull List heads) { + if (exec != null && heads == null && heads.size() != 0) { + String url; + try { + url = exec.getUrl(); + } catch (IOException ioe) { + throw new IllegalStateException(ioe); + } + IncrementalAnalysis analysis = analysisCache.getIfPresent(url); + if (analysis != null) { + return analysis.getUpdatedValue(exec, heads); + } else { + IncrementalAnalysis newAnalysis = new IncrementalAnalysis(matchCondition, analysisFunction); + T value = newAnalysis.getUpdatedValue(exec, heads); + analysisCache.put(url, newAnalysis); + return value; + } + } + return null; + } + + public IncrementalFlowAnalysisCache(Predicate matchCondition, Function analysisFunction) { + this.matchCondition = matchCondition; + this.analysisFunction = analysisFunction; + } + + public IncrementalFlowAnalysisCache(Predicate matchCondition, Function analysisFunction, Cache myCache) { + this.matchCondition = matchCondition; + this.analysisFunction = analysisFunction; + this.analysisCache = myCache; + } +} diff --git a/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java new file mode 100644 index 00000000..5e1c87ee --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/workflow/graph/TestIncrementalFlowAnalysis.java @@ -0,0 +1,79 @@ +/* + * The MIT License + * + * Copyright (c) 2016, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.graph; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import org.jenkinsci.plugins.workflow.actions.LabelAction; +import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; +import org.jenkinsci.plugins.workflow.flow.FlowExecution; +import org.jenkinsci.plugins.workflow.job.WorkflowJob; +import org.jenkinsci.plugins.workflow.job.WorkflowRun; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.jvnet.hudson.test.BuildWatcher; +import org.jvnet.hudson.test.JenkinsRule; + + +/** + * @author svanoort + */ +public class TestIncrementalFlowAnalysis { + @ClassRule + public static BuildWatcher buildWatcher = new BuildWatcher(); + + @Rule + public JenkinsRule r = new JenkinsRule(); + + /** Tests the basic incremental analysis */ + @Test + public void testIncrementalAnalysis() throws Exception { + WorkflowJob job = r.jenkins.createProject(WorkflowJob.class, "Convoluted"); + job.setDefinition(new CpsFlowDefinition( + "for (int i=0; i<4; i++) {\n" + + " stage \"stage-$i\"\n" + + " echo \"Doing $i\"\n" + + "}" + )); + + // Search conditions + Predicate labelledNode = FlowScanner.createPredicateWhereActionExists(LabelAction.class); + Function getLabelFunction = new Function() { + @Override + public String apply(FlowNode input) { + LabelAction labelled = input.getAction(LabelAction.class); + return (labelled != null) ? labelled.getDisplayName() : null; + } + }; + + IncrementalFlowAnalysisCache incrementalAnalysis = new IncrementalFlowAnalysisCache(labelledNode, getLabelFunction); + WorkflowRun b = r.assertBuildStatusSuccess(job.scheduleBuild2(0)); + FlowExecution exec = b.getExecution(); + FlowNode test = exec.getNode("4"); + + // TODO add tests based on calling incremental analysis from points further along flow, possible in some paralle cases + } +}