diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index 510f24fb..a2d496cc 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -2,6 +2,6 @@ io.jenkins.tools.incrementals git-changelist-maven-extension - 1.0-beta-3 + 1.0-beta-4 diff --git a/pom.xml b/pom.xml index 602454e9..c0e7f32f 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.jenkins-ci.plugins plugin - 3.14 + 3.21 org.jenkins-ci.plugins.workflow diff --git a/src/main/java/org/jenkinsci/plugins/workflow/flow/FlowExecutionOwner.java b/src/main/java/org/jenkinsci/plugins/workflow/flow/FlowExecutionOwner.java index b0d23e2d..2a09113e 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/flow/FlowExecutionOwner.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/flow/FlowExecutionOwner.java @@ -35,6 +35,7 @@ import javax.annotation.CheckForNull; import javax.annotation.Nonnull; import jenkins.model.TransientActionFactory; +import org.jenkinsci.plugins.workflow.log.LogStorage; import org.jenkinsci.plugins.workflow.steps.StepContext; /** @@ -121,9 +122,15 @@ public String getUrlOfExecution() throws IOException { /** * Gets a listener to which we may print general messages. * Normally {@link StepContext#get} should be used, but in some cases there is no associated step. + *

The listener should be remotable: if sent to an agent, messages printed to it should still appear in the log. + * The same will then apply to calls to {@link StepContext#get} on {@link TaskListener}. */ public @Nonnull TaskListener getListener() throws IOException { - return TaskListener.NULL; + try { + return LogStorage.of(this).overallListener(); + } catch (InterruptedException x) { + throw new IOException(x); + } } /** @@ -167,6 +174,9 @@ private static class DummyOwner extends FlowExecutionOwner { @Override public int hashCode() { return 0; } + @Override public TaskListener getListener() throws IOException { + return TaskListener.NULL; + } } } diff --git a/src/main/java/org/jenkinsci/plugins/workflow/log/BrokenLogStorage.java b/src/main/java/org/jenkinsci/plugins/workflow/log/BrokenLogStorage.java new file mode 100644 index 00000000..3b600903 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/log/BrokenLogStorage.java @@ -0,0 +1,86 @@ +/* + * The MIT License + * + * Copyright 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.log; + +import hudson.Functions; +import hudson.console.AnnotatedLargeText; +import hudson.model.BuildListener; +import hudson.model.TaskListener; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; +import org.jenkinsci.plugins.workflow.graph.FlowNode; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.Beta; +import org.kohsuke.stapler.framework.io.ByteBuffer; + +/** + * Placeholder for storage broken by some kind of access error. + */ +@Restricted(Beta.class) +public final class BrokenLogStorage implements LogStorage { + + private final Throwable x; + + public BrokenLogStorage(Throwable x) { + this.x = x; + } + + @Override public BuildListener overallListener() throws IOException, InterruptedException { + throw new IOException(x); + } + + @Override public TaskListener nodeListener(FlowNode node) throws IOException, InterruptedException { + throw new IOException(x); + } + + @Override public AnnotatedLargeText overallLog(FlowExecutionOwner.Executable build, boolean complete) { + return new BrokenAnnotatedLargeText<>(); + } + + @Override public AnnotatedLargeText stepLog(FlowNode node, boolean complete) { + return new BrokenAnnotatedLargeText<>(); + } + + private class BrokenAnnotatedLargeText extends AnnotatedLargeText { + + BrokenAnnotatedLargeText() { + super(makeByteBuffer(), StandardCharsets.UTF_8, true, null); + } + + } + + private ByteBuffer makeByteBuffer() { + ByteBuffer buf = new ByteBuffer(); + byte[] stack = Functions.printThrowable(x).getBytes(StandardCharsets.UTF_8); + try { + buf.write(stack, 0, stack.length); + } catch (IOException x2) { + assert false : x2; + } + return buf; + } + +} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/log/ConsoleAnnotators.java b/src/main/java/org/jenkinsci/plugins/workflow/log/ConsoleAnnotators.java new file mode 100644 index 00000000..aec53cde --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/log/ConsoleAnnotators.java @@ -0,0 +1,107 @@ +/* + * The MIT License + * + * Copyright 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.log; + +import com.jcraft.jzlib.GZIPInputStream; +import com.jcraft.jzlib.GZIPOutputStream; +import com.trilead.ssh2.crypto.Base64; +import hudson.console.AnnotatedLargeText; +import hudson.console.ConsoleAnnotationOutputStream; +import hudson.console.ConsoleAnnotator; +import hudson.remoting.ClassFilter; +import hudson.remoting.ObjectInputStreamEx; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import static java.lang.Math.abs; +import java.util.concurrent.TimeUnit; +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.CipherOutputStream; +import jenkins.model.Jenkins; +import jenkins.security.CryptoConfidentialKey; +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.Beta; +import org.kohsuke.stapler.Stapler; +import org.kohsuke.stapler.StaplerRequest; +import org.kohsuke.stapler.StaplerResponse; + +/** + * Some utility code extracted from {@link AnnotatedLargeText} which probably belongs in {@link ConsoleAnnotator} or {@link ConsoleAnnotationOutputStream}. + */ +@Restricted(Beta.class) +public class ConsoleAnnotators { + + private static final CryptoConfidentialKey PASSING_ANNOTATOR = new CryptoConfidentialKey(ConsoleAnnotators.class, "consoleAnnotator"); + + /** + * What to pass to {@link ConsoleAnnotationOutputStream#ConsoleAnnotationOutputStream} when overriding {@link AnnotatedLargeText#writeHtmlTo}. + */ + public static ConsoleAnnotator createAnnotator(T context) throws IOException { + StaplerRequest req = Stapler.getCurrentRequest(); + try { + String base64 = req != null ? req.getHeader("X-ConsoleAnnotator") : null; + if (base64 != null) { + @SuppressWarnings("deprecation") // TODO still used in the AnnotatedLargeText version + Cipher sym = PASSING_ANNOTATOR.decrypt(); + try (ObjectInputStream ois = new ObjectInputStreamEx(new GZIPInputStream( + new CipherInputStream(new ByteArrayInputStream(Base64.decode(base64.toCharArray())), sym)), + Jenkins.get().pluginManager.uberClassLoader, + ClassFilter.DEFAULT)) { + long timestamp = ois.readLong(); + if (TimeUnit.HOURS.toMillis(1) > abs(System.currentTimeMillis() - timestamp)) { + @SuppressWarnings("unchecked") ConsoleAnnotator annotator = (ConsoleAnnotator) ois.readObject(); + return annotator; + } + } + } + } catch (ClassNotFoundException e) { + throw new IOException(e); + } + return ConsoleAnnotator.initial(context); + } + + /** + * What to call at the end of an override of {@link AnnotatedLargeText#writeHtmlTo}. + */ + public static void setAnnotator(ConsoleAnnotator annotator) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + @SuppressWarnings("deprecation") // TODO still used in the AnnotatedLargeText version + Cipher sym = PASSING_ANNOTATOR.encrypt(); + try (ObjectOutputStream oos = new ObjectOutputStream(new GZIPOutputStream(new CipherOutputStream(baos, sym)))) { + oos.writeLong(System.currentTimeMillis()); + oos.writeObject(annotator); + } + StaplerResponse rsp = Stapler.getCurrentResponse(); + if (rsp != null) { + rsp.setHeader("X-ConsoleAnnotator", new String(Base64.encode(baos.toByteArray()))); + } + } + + private ConsoleAnnotators() {} + +} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/log/FileLogStorage.java b/src/main/java/org/jenkinsci/plugins/workflow/log/FileLogStorage.java new file mode 100644 index 00000000..ba6854fc --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/log/FileLogStorage.java @@ -0,0 +1,310 @@ +/* + * The MIT License + * + * Copyright 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.log; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import hudson.console.AnnotatedLargeText; +import hudson.console.ConsoleAnnotationOutputStream; +import hudson.model.BuildListener; +import hudson.model.StreamBuildListener; +import hudson.model.TaskListener; +import hudson.util.StreamTaskListener; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.RandomAccessFile; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.logging.Logger; +import org.apache.commons.io.input.NullReader; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; +import org.jenkinsci.plugins.workflow.graph.FlowNode; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.Beta; +import org.kohsuke.stapler.framework.io.ByteBuffer; + +/** + * Simple implementation of log storage in a single file that maintains a side file with an index indicating where node transitions occur. + * Each line in the index file is a byte offset, optionally followed by a space and then a node ID. + */ +@Restricted(Beta.class) +public final class FileLogStorage implements LogStorage { + + private static final Logger LOGGER = Logger.getLogger(FileLogStorage.class.getName()); + + private static final Map openStorages = Collections.synchronizedMap(new HashMap<>()); + + public static synchronized LogStorage forFile(File log) { + return openStorages.computeIfAbsent(log, FileLogStorage::new); + } + + private final File log; + private final File index; + @SuppressFBWarnings(value = "IS2_INCONSISTENT_SYNC", justification = "FB apparently gets confused by what the lock is, and anyway we only care about synchronizing writes") + private FileOutputStream os; + private Writer indexOs; + private String lastId; + + private FileLogStorage(File log) { + this.log = log; + this.index = new File(log + "-index"); + } + + private synchronized void open() throws IOException { + if (os == null) { + os = new FileOutputStream(log, true); + if (index.isFile()) { + try (BufferedReader r = Files.newBufferedReader(index.toPath(), StandardCharsets.UTF_8)) { + // TODO would be faster to scan the file backwards for the penultimate \n, then convert the byte sequence from there to EOF to UTF-8 and set lastId accordingly + String lastLine = null; + while (true) { + // Note that BufferedReader tolerates final lines without a line separator, so if for some reason the last write has been truncated this result could be incorrect. + // In practice this seems unlikely since we explicitly flush after the newline, so we should be sending a single small block to the filesystem to persist. + // Anyway at worst the result would be a (perhaps temporarily) incorrect line → step mapping, which is tolerable for one step of one build, and barely affects the overall build log. + String line = r.readLine(); + if (line == null) { + break; + } else { + lastLine = line; + } + } + if (lastLine != null) { + int space = lastLine.indexOf(' '); + lastId = space == -1 ? null : lastLine.substring(space + 1); + } + } + } + indexOs = new OutputStreamWriter(new FileOutputStream(index, true), StandardCharsets.UTF_8); + } + } + + @Override public BuildListener overallListener() throws IOException, InterruptedException { + return new StreamBuildListener(new IndexOutputStream(null), StandardCharsets.UTF_8); + } + + @Override public TaskListener nodeListener(FlowNode node) throws IOException, InterruptedException { + return new StreamTaskListener(new IndexOutputStream(node.getId()), StandardCharsets.UTF_8); + } + + private void checkId(String id) throws IOException { + assert Thread.holdsLock(this); + if (!Objects.equals(id, lastId)) { + long pos = os.getChannel().position(); + if (id == null) { + indexOs.write(pos + "\n"); + } else { + indexOs.write(pos + " " + id + "\n"); + } + // Could call FileChannel.force(true) like hudson.util.FileChannelWriter does for AtomicFileWriter, + // though making index-log writes slower is likely a poor tradeoff for slightly more reliable log display, + // since logs are often never read and this is transient data rather than configuration or valuable state. + indexOs.flush(); + lastId = id; + } + } + + private final class IndexOutputStream extends OutputStream { + + private final String id; + + IndexOutputStream(String id) throws IOException { + this.id = id; + open(); + } + + @Override public void write(int b) throws IOException { + synchronized (FileLogStorage.this) { + checkId(id); + os.write(b); + } + } + + @Override public void write(byte[] b) throws IOException { + synchronized (FileLogStorage.this) { + checkId(id); + os.write(b); + } + } + + @Override public void write(byte[] b, int off, int len) throws IOException { + synchronized (FileLogStorage.this) { + checkId(id); + os.write(b, off, len); + } + } + + @Override public void flush() throws IOException { + os.flush(); + } + + @Override public void close() throws IOException { + if (id == null) { + openStorages.remove(log); + try { + os.close(); + } finally { + indexOs.close(); + } + } + } + + } + + @Override public AnnotatedLargeText overallLog(FlowExecutionOwner.Executable build, boolean complete) { + return new AnnotatedLargeText(log, StandardCharsets.UTF_8, complete, build) { + @Override public long writeHtmlTo(long start, Writer w) throws IOException { + try (BufferedReader indexBR = index.isFile() ? Files.newBufferedReader(index.toPath(), StandardCharsets.UTF_8) : new BufferedReader(new NullReader(0))) { + ConsoleAnnotationOutputStream caos = new ConsoleAnnotationOutputStream<>(w, ConsoleAnnotators.createAnnotator(build), build, StandardCharsets.UTF_8); + long r = this.writeRawLogTo(start, new FilterOutputStream(caos) { + // To insert startStep/endStep annotations into the overall log, we need to simultaneously read index-log. + // We use the standard LargeText.FileSession to get the raw log text (we need not think about ConsoleNote here), having seeked to the start position. + // Then we read index-log in order, looking for transitions from one step to the next (or to or from non-step overall output). + // Whenever we are about to write a byte which is at a boundary, or if there is a boundary at EOF, the HTML annotations are injected into the output; + // the read of index-log is advanced lazily (it is not necessary to have the whole mapping in memory). + long lastTransition = -1; + boolean eof; // NullReader is strict and throws IOException (not EOFException) if you read() again after having already gotten -1 + String lastId; + long pos = start; + boolean hadLastId; + @Override public void write(int b) throws IOException { + while (lastTransition < pos && !eof) { + String line = indexBR.readLine(); + if (line == null) { + eof = true; + break; + } + int space = line.indexOf(' '); + try { + lastTransition = Long.parseLong(space == -1 ? line : line.substring(0, space)); + } catch (NumberFormatException x) { + LOGGER.warning("Ignoring corrupt index file " + index); + } + lastId = space == -1 ? null : line.substring(space + 1); + } + if (pos == lastTransition) { + if (hadLastId) { + w.write(LogStorage.endStep()); + } + hadLastId = lastId != null; + if (lastId != null) { + w.write(LogStorage.startStep(lastId)); + } + } + super.write(b); + pos++; + } + @Override public void flush() throws IOException { + if (lastId != null) { + w.write(LogStorage.endStep()); + } + super.flush(); + } + }); + ConsoleAnnotators.setAnnotator(caos.getConsoleAnnotator()); + return r; + } + } + }; + } + + @Override public AnnotatedLargeText stepLog(FlowNode node, boolean complete) { + String id = node.getId(); + try (ByteBuffer buf = new ByteBuffer(); + RandomAccessFile raf = new RandomAccessFile(log, "r"); + BufferedReader indexBR = index.isFile() ? Files.newBufferedReader(index.toPath(), StandardCharsets.UTF_8) : new BufferedReader(new NullReader(0))) { + // Check this _before_ reading index-log to reduce the chance of a race condition resulting in recent content being associated with the wrong step: + long end = raf.length(); + // To produce just the output for a single step (again we do not need to pay attention to ConsoleNote here since AnnotatedLargeText handles it), + // index-log is read looking for transitions that pertain to this step: beginning or ending its content, including at EOF if applicable. + // (Other transitions, such as to or from unrelated steps, are irrelevant). + // Once a start and end position have been identified, that block is copied to a memory buffer. + String line; + long pos = -1; // -1 if not currently in this node, start position if we are + while ((line = indexBR.readLine()) != null) { + int space = line.indexOf(' '); + long lastTransition = -1; + try { + lastTransition = Long.parseLong(space == -1 ? line : line.substring(0, space)); + } catch (NumberFormatException x) { + LOGGER.warning("Ignoring corrupt index file " + index); + // If index-log is corrupt for whatever reason, we given up on this step in this build; + // there is no way we would be able to produce accurate output anyway. + // Note that NumberFormatException is nonfatal in the case of the overall build log: + // the whole-build HTML output always includes exactly what is in the main log file, + // at worst with some missing or inaccurate startStep/endStep annotations. + continue; + } + if (pos == -1) { + if (space != -1 && line.substring(space + 1).equals(id)) { + pos = lastTransition; + } + } else if (lastTransition > pos) { + raf.seek(pos); + if (lastTransition > pos + Integer.MAX_VALUE) { + throw new IOException("Cannot read more than 2Gib at a time"); // ByteBuffer does not support it anyway + } + // Could perhaps be done a bit more efficiently with FileChannel methods, + // at least if org.kohsuke.stapler.framework.io.ByteBuffer were replaced by java.nio.[Heap]ByteBuffer. + // The overall bottleneck here is however the need to use a memory buffer to begin with: + // LargeText.Source/Session are not public so, pending improvements to Stapler, + // we cannot lazily stream per-step content the way we do for the overall log. + // (Except perhaps by extending ByteBuffer and then overriding every public method!) + // LargeText also needs to be improved to support opaque (non-long) cursors + // (and callers such as progressiveText.jelly and Blue Ocean updated accordingly), + // which is a hard requirement for efficient rendering of cloud-backed logs, + // though for this implementation we do not need it since we can work with byte offsets. + byte[] data = new byte[(int) (lastTransition - pos)]; + raf.readFully(data); + buf.write(data); + pos = -1; + } // else some sort of mismatch + } + if (pos != -1 && /* otherwise race condition? */ end > pos) { + // In case the build is ongoing and we are still actively writing content for this step, + // we will hit EOF before any other transition. Otherwise identical to normal case above. + raf.seek(pos); + if (end > pos + Integer.MAX_VALUE) { + throw new IOException("Cannot read more than 2Gib at a time"); + } + byte[] data = new byte[(int) (end - pos)]; + raf.readFully(data); + buf.write(data); + } + return new AnnotatedLargeText<>(buf, StandardCharsets.UTF_8, complete, node); + } catch (IOException x) { + return new BrokenLogStorage(x).stepLog(node, complete); + } + } + +} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/log/LogStorage.java b/src/main/java/org/jenkinsci/plugins/workflow/log/LogStorage.java new file mode 100644 index 00000000..dc57362f --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/log/LogStorage.java @@ -0,0 +1,131 @@ +/* + * The MIT License + * + * Copyright 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.log; + +import hudson.ExtensionList; +import hudson.console.AnnotatedLargeText; +import hudson.console.ConsoleAnnotationOutputStream; +import hudson.model.BuildListener; +import hudson.model.TaskListener; +import java.io.File; +import java.io.IOException; +import javax.annotation.Nonnull; +import org.jenkinsci.plugins.workflow.actions.LogAction; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; +import org.jenkinsci.plugins.workflow.graph.FlowNode; +import org.jenkinsci.plugins.workflow.steps.StepContext; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.Beta; + +/** + * Means of replacing how logs are stored for a Pipeline build as a whole or for one step. + * UTF-8 encoding is assumed throughout. + * @see JEP-210: Pluggable log storage + */ +@Restricted(Beta.class) +public interface LogStorage { + + + /** + * Provides an alternate way of emitting output from a build. + *

May implement {@link AutoCloseable} to clean up at the end of a build; + * it may or may not be closed during Jenkins shutdown while a build is running. + * @return a (remotable) build listener; do not bother overriding anything except {@link TaskListener#getLogger} + * @see FlowExecutionOwner#getListener + */ + @Nonnull BuildListener overallListener() throws IOException, InterruptedException; + + /** + * Provides an alternate way of emitting output from a node (such as a step). + *

May implement {@link AutoCloseable} to clean up at the end of a node ({@link FlowNode#isActive}); + * it may or may not be closed during Jenkins shutdown while a build is running. + * @param node a running node + * @return a (remotable) task listener; do not bother overriding anything except {@link TaskListener#getLogger} + * @see StepContext#get + */ + @Nonnull TaskListener nodeListener(@Nonnull FlowNode node) throws IOException, InterruptedException; + + /** + * Provides an alternate way of retrieving output from a build. + *

In an {@link AnnotatedLargeText#writeHtmlTo} override, {@link ConsoleAnnotationOutputStream#eol} + * should apply {@link #startStep} and {@link #endStep} to delineate blocks contributed by steps. + * (Also see {@link ConsoleAnnotators}.) + * @param complete if true, we claim to be serving the complete log for a build, + * so implementations should be sure to retrieve final log lines + * @return a log + */ + @Nonnull AnnotatedLargeText overallLog(@Nonnull FlowExecutionOwner.Executable build, boolean complete); + + /** + * Introduces an HTML block with a {@code pipeline-node-} CSS class based on {@link FlowNode#getId}. + * @see #endStep + * @see #overallLog + */ + static @Nonnull String startStep(@Nonnull String id) { + return ""; + } + + /** + * Closes an HTML step block. + * @see #startStep + * @see #overallLog + */ + static @Nonnull String endStep() { + return ""; + } + + /** + * Provides an alternate way of retrieving output from a build. + * @param node a running node + * @param complete if true, we claim to be serving the complete log for a node, + * so implementations should be sure to retrieve final log lines + * @return a log for this just this node + * @see LogAction + */ + @Nonnull AnnotatedLargeText stepLog(@Nonnull FlowNode node, boolean complete); + + /** + * Gets the available log storage method for a given build. + * @param b a build about to start + * @return the mechanism for handling this build, including any necessary fallback + * @see LogStorageFactory + */ + static @Nonnull LogStorage of(@Nonnull FlowExecutionOwner b) { + try { + for (LogStorageFactory factory : ExtensionList.lookup(LogStorageFactory.class)) { + LogStorage storage = factory.forBuild(b); + if (storage != null) { + // Pending integration with JEP-207 / JEP-212, this choice is not persisted. + return storage; + } + } + // Similar to Run.getLogFile, but not supporting gzip: + return FileLogStorage.forFile(new File(b.getRootDir(), "log")); + } catch (Exception x) { + return new BrokenLogStorage(x); + } + } + +} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/log/LogStorageFactory.java b/src/main/java/org/jenkinsci/plugins/workflow/log/LogStorageFactory.java new file mode 100644 index 00000000..f5382836 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/log/LogStorageFactory.java @@ -0,0 +1,47 @@ +/* + * The MIT License + * + * Copyright 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.log; + +import hudson.ExtensionPoint; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.Beta; + +/** + * Factory interface for {@link LogStorage}. + */ +@Restricted(Beta.class) +public interface LogStorageFactory extends ExtensionPoint { + + /** + * Checks whether we should handle a given build. + * @param b a build about to start + * @return a mechanism for handling this build, or null to fall back to the next implementation or the default + */ + @CheckForNull LogStorage forBuild(@Nonnull FlowExecutionOwner b); + +} diff --git a/src/main/java/org/jenkinsci/plugins/workflow/log/package-info.java b/src/main/java/org/jenkinsci/plugins/workflow/log/package-info.java new file mode 100644 index 00000000..0b0dabd0 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/workflow/log/package-info.java @@ -0,0 +1,39 @@ +/* + * The MIT License + * + * Copyright 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/** + * APIs supporting the production and retrieval of log messages associated with Pipeline builds ({@link FlowExecutionOwner}) and individual steps ({@link FlowNode}). + *

Note that the term “step” is used loosely in documentation here to refer to a {@link FlowNode}, + * which is only precise in the case of {@link AtomNode}s. + * Block-scoped {@link Step}s which use {@link BodyInvoker} can be producing output interleaved with their children, + * something the {@link FlowNode#getId} should track. + * @see JEP-210 + */ +package org.jenkinsci.plugins.workflow.log; + +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; +import org.jenkinsci.plugins.workflow.graph.AtomNode; +import org.jenkinsci.plugins.workflow.graph.FlowNode; +import org.jenkinsci.plugins.workflow.steps.BodyInvoker; +import org.jenkinsci.plugins.workflow.steps.Step; diff --git a/src/test/java/org/jenkinsci/plugins/workflow/log/FileLogStorageTest.java b/src/test/java/org/jenkinsci/plugins/workflow/log/FileLogStorageTest.java new file mode 100644 index 00000000..ae92a4fe --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/workflow/log/FileLogStorageTest.java @@ -0,0 +1,57 @@ +/* + * The MIT License + * + * Copyright 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.log; + +import hudson.model.TaskListener; +import java.io.File; +import static org.junit.Assert.*; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class FileLogStorageTest extends LogStorageTestBase { + + @Rule public TemporaryFolder tmp = new TemporaryFolder(); + private File log; + + @Before public void log() throws Exception { + log = tmp.newFile(); + } + + @Override protected LogStorage createStorage() { + return FileLogStorage.forFile(log); + } + + @Test public void oldFormat() throws Exception { + LogStorage ls = createStorage(); + TaskListener overall = ls.overallListener(); + overall.getLogger().println("stuff"); + close(overall); + assertTrue(new File(log + "-index").delete()); + assertOverallLog(0, "stuff\n", true); + } + +} diff --git a/src/test/java/org/jenkinsci/plugins/workflow/log/LogStorageTestBase.java b/src/test/java/org/jenkinsci/plugins/workflow/log/LogStorageTestBase.java new file mode 100644 index 00000000..d11b2a69 --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/workflow/log/LogStorageTestBase.java @@ -0,0 +1,286 @@ +/* + * The MIT License + * + * Copyright 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.log; + +import hudson.console.AnnotatedLargeText; +import hudson.console.HyperlinkNote; +import hudson.model.TaskListener; +import hudson.remoting.VirtualChannel; +import java.io.EOFException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.Callable; +import java.util.function.BiFunction; +import java.util.logging.Logger; +import jenkins.security.ConfidentialStore; +import jenkins.security.MasterToSlaveCallable; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.NullOutputStream; +import org.apache.commons.io.output.NullWriter; +import org.apache.commons.io.output.WriterOutputStream; +import org.jenkinsci.plugins.workflow.graph.FlowNode; +import static org.junit.Assert.*; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.jvnet.hudson.test.JenkinsRule; + +/** + * Foundation for compliance tests of {@link LogStorage} implementations. + */ +public abstract class LogStorageTestBase { + + static { + System.setProperty("line.separator", "\n"); + } + + /** Needed since {@link ConsoleAnnotators} will not work without encryption, and currently {@link ConfidentialStore#get} has no fallback mode for unit tests accessible except via package-local. */ + @ClassRule public static JenkinsRule r = new JenkinsRule(); + + /** Create a new storage implementation, but potentially reusing any data initialized in the last {@link Before} setup. */ + protected abstract LogStorage createStorage() throws Exception; + + @Test public void smokes() throws Exception { + LogStorage ls = createStorage(); + TaskListener overall = ls.overallListener(); + overall.getLogger().println("starting"); + TaskListener step1 = ls.nodeListener(new MockNode("1")); + step1.getLogger().println("one #1"); + TaskListener step2 = ls.nodeListener(new MockNode("2")); + step2.getLogger().println("two #1"); + long betweenStep2Lines = text().writeHtmlTo(0, new NullWriter()); + step2.getLogger().println("two #2"); + overall.getLogger().println("interrupting"); + /* We do not really care much whether nodes are annotated when we start display in the middle; the UI will not do anything with it anyway: + assertOverallLog(betweenStep2Lines, "two #2\ninterrupting\n", true); + */ + long overallHtmlPos = assertOverallLog(0, "starting\none #1\ntwo #1\ntwo #2\ninterrupting\n", true); + assertEquals(overallHtmlPos, assertOverallLog(overallHtmlPos, "", true)); + assertLength(overallHtmlPos); + try { // either tolerate OOB, or not + assertOverallLog(999, "", true); + assertOverallLog(999, "", false); + } catch (EOFException x) {} + long step1Pos = assertStepLog("1", 0, "one #1\n", true); + long step2Pos = assertStepLog("2", 0, "two #1\ntwo #2\n", true); + step1.getLogger().println("one #2"); + step1.getLogger().println("one #3"); + overall.getLogger().println("pausing"); + overallHtmlPos = assertOverallLog(overallHtmlPos, "one #2\none #3\npausing\n", true); + step1Pos = assertStepLog("1", step1Pos, "one #2\none #3\n", true); + assertLength("1", step1Pos); + try { // as above + assertStepLog("1", 999, "", true); + assertStepLog("1", 999, "", false); + } catch (EOFException x) {} + step2Pos = assertStepLog("2", step2Pos, "", true); + close(overall); + ls = createStorage(); + overall = ls.overallListener(); + overall.getLogger().println("resuming"); + step1 = ls.nodeListener(new MockNode("1")); + step1.getLogger().println("one #4"); + close(step1); + TaskListener step3 = ls.nodeListener(new MockNode("3")); + step3.getLogger().println("three #1"); + close(step3); + overall.getLogger().println("ending"); + close(overall); + overallHtmlPos = assertOverallLog(overallHtmlPos, "resuming\none #4\nthree #1\nending\n", true); + assertEquals(overallHtmlPos, assertOverallLog(overallHtmlPos, "", true)); + assertLength(overallHtmlPos); + step1Pos = assertStepLog("1", step1Pos, "one #4\n", true); + assertLength("1", step1Pos); + assertStepLog("1", 0, "one #1\none #2\none #3\none #4\n", false); + step2Pos = assertStepLog("2", step2Pos, "", true); + assertStepLog("3", 0, "three #1\n", true); + ls = createStorage(); + TaskListener step4 = ls.nodeListener(new MockNode("4")); + step4.getLogger().println(HyperlinkNote.encodeTo("http://nowhere.net/", "nikde")); + close(overall); + long step4Pos = assertStepLog("4", 0, "nikde\n", true); + assertLength("4", step4Pos); + overall = ls.overallListener(); + overall.getLogger().println("really ending"); + close(overall); + overallHtmlPos = assertOverallLog(overallHtmlPos, "nikde\nreally ending\n", true); + assertEquals(overallHtmlPos, assertOverallLog(overallHtmlPos, "", true)); + assertLength(overallHtmlPos); + } + + protected static void close(TaskListener listener) throws Exception { + if (listener instanceof AutoCloseable) { + ((AutoCloseable) listener).close(); + } + } + + @Test public void remoting() throws Exception { + LogStorage ls = createStorage(); + TaskListener overall = ls.overallListener(); + overall.getLogger().println("overall from master"); + TaskListener step = ls.nodeListener(new MockNode("1")); + step.getLogger().println("step from master"); + long overallPos = assertOverallLog(0, "overall from master\nstep from master\n", true); + long stepPos = assertStepLog("1", 0, "step from master\n", true); + VirtualChannel channel = r.createOnlineSlave().getChannel(); + channel.call(new RemotePrint("overall from agent", overall)); + channel.call(new RemotePrint("step from agent", step)); + while (!IOUtils.toString(text().readAll()).contains("overall from agent") || !IOUtils.toString(text().readAll()).contains("step from agent")) { + // TODO current cloud implementations may be unable to honor the completed flag on remotely printed messages, pending some way to have all affected loggers confirm they have flushed + Logger.getLogger(LogStorageTestBase.class.getName()).info("waiting for remote content to appear"); + Thread.sleep(1000); + } + overallPos = assertOverallLog(overallPos, "overall from agent\nstep from agent\n", true); + stepPos = assertStepLog("1", stepPos, "step from agent\n", true); + assertEquals(overallPos, assertOverallLog(overallPos, "", true)); + assertEquals(stepPos, assertStepLog("1", stepPos, "", true)); + } + private static final class RemotePrint extends MasterToSlaveCallable { + static { + System.setProperty("line.separator", "\n"); + } + private final String message; + private final TaskListener listener; + RemotePrint(String message, TaskListener listener) { + this.message = message; + this.listener = listener; + } + @Override public Void call() throws Exception { + listener.getLogger().println(message); + return null; + } + } + + /** + * Checks what happens when code using {@link TaskListener#getLogger} prints a line with inadequate synchronization. + * Normally you use something like {@link PrintWriter#println(String)} which synchronizes and so delivers a complete line. + * Failures to do this can cause output from different steps (or general build output) to be interleaved at a sub-line level. + * This might not render well (depending on the implementation), but we need to ensure that the entire build log is not broken as a result. + */ + @Test public void mangledLines() throws Exception { + Random r = new Random(); + BiFunction thread = (c, l) -> new Thread(() -> { + for (int i = 0; i < 1000; i++) { + l.getLogger().print(c); + if (r.nextDouble() < 0.1) { + l.getLogger().println(); + } + if (r.nextDouble() < 0.1) { + try { + Thread.sleep(r.nextInt(10)); + } catch (InterruptedException x) { + x.printStackTrace(); + } + } + } + }); + List threads = new ArrayList<>(); + LogStorage ls = createStorage(); + threads.add(thread.apply('.', ls.overallListener())); + threads.add(thread.apply('1', ls.nodeListener(new MockNode("1")))); + threads.add(thread.apply('2', ls.nodeListener(new MockNode("2")))); + threads.forEach(Thread::start); + threads.forEach(t -> { + try { + t.join(); + } catch (InterruptedException x) { + x.printStackTrace(); + } + }); + long pos = text().writeHtmlTo(0, new NullWriter()); + // TODO detailed assertions would need to take into account completion flag: + // assertLength(pos); + // assertOverallLog(pos, "", true); + text().writeRawLogTo(0, new NullOutputStream()); + pos = text("1").writeHtmlTo(0, new NullWriter()); + // assertLength("1", pos); + // assertStepLog("1", pos, "", true); + text("1").writeRawLogTo(0, new NullOutputStream()); + pos = text("2").writeHtmlTo(0, new NullWriter()); + // assertLength("2", pos); + // assertStepLog("2", pos, "", true); + text("2").writeRawLogTo(0, new NullOutputStream()); + } + + // TODO test missing final newline + + protected final long assertOverallLog(long start, String expected, boolean html) throws Exception { + return assertLog(() -> text(), start, expected, html, html); + } + + protected final long assertStepLog(String id, long start, String expected, boolean html) throws Exception { + return assertLog(() -> text(id), start, expected, html, false); + } + + private long assertLog(Callable> text, long start, String expected, boolean html, boolean coalesceSpans) throws Exception { + long pos = start; + StringWriter sw = new StringWriter(); + AnnotatedLargeText oneText; + do { + oneText = text.call(); + if (html) { + pos = oneText.writeHtmlTo(pos, sw); + } else { + pos = oneText.writeRawLogTo(pos, new WriterOutputStream(sw, StandardCharsets.UTF_8)); + } + } while (!oneText.isComplete()); + String result = sw.toString(); + if (coalesceSpans) { + result = SpanCoalescerTest.coalesceSpans(result); + } + assertEquals(expected, result); + return pos; + } + + protected final void assertLength(long length) throws Exception { + assertLength(text(), length); + } + + protected final void assertLength(String id, long length) throws Exception { + assertLength(text(id), length); + } + + private void assertLength(AnnotatedLargeText text, long length) throws Exception { + assertEquals(length, text.length()); + } + + private AnnotatedLargeText text() throws Exception { + return createStorage().overallLog(null, true); + } + + private AnnotatedLargeText text(String id) throws Exception { + return createStorage().stepLog(new MockNode(id), true); + } + + private static class MockNode extends FlowNode { + MockNode(String id) {super(null, id);} + @Override protected String getTypeDisplayName() {return null;} + } + +} diff --git a/src/test/java/org/jenkinsci/plugins/workflow/log/SpanCoalescerTest.java b/src/test/java/org/jenkinsci/plugins/workflow/log/SpanCoalescerTest.java new file mode 100644 index 00000000..d84d8987 --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/workflow/log/SpanCoalescerTest.java @@ -0,0 +1,74 @@ +/* + * The MIT License + * + * Copyright 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow.log; + +import hudson.console.AnnotatedLargeText; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import static org.junit.Assert.*; +import org.junit.Test; + +public class SpanCoalescerTest { + + @Test public void works() throws Exception { + assertUncoalesced("plain\n"); + assertUncoalesced("one\n"); + assertUncoalesced("plain\n1a\n1b\n2a\n2b\nmore plain\n"); + assertUncoalesced("1a\nplain\n2a\n"); + assertCoalesced("plain\n1a\n1b\n1c\n1d\nmore plain\n", + "plain\n1a\n1b\n1c\n1d\nmore plain\n"); + assertCoalesced("1a\n1b\n2a\n3a\n3b\n", + "1a\n1b\n2a\n3a\n3b\n"); + } + + private static void assertUncoalesced(String text) { + assertEquals(text, coalesceSpans(text)); + } + + private static void assertCoalesced(String text, String collapsed) { + assertEquals(collapsed, coalesceSpans(text)); + } + + private static final Pattern COALESCIBLE = Pattern.compile("[^\"]+)\">(?.*?)\">", Pattern.DOTALL); + + /** + * Coalesces sequences of {@link LogStorage#startStep} and {@link LogStorage#endStep} annotations referring to the same ID. + * This is necessary as we may be doing progressive logging (!{@link AnnotatedLargeText#isComplete}), + * in which case a block of output from a single step might be broken across two requests, + * each of which would emit its own HTML {@code span}. + */ + static String coalesceSpans(String text) { + while (true) { + Matcher m = COALESCIBLE.matcher(text); + if (m.find()) { + text = m.replaceFirst("${first}"); + } else { + break; + } + } + return text; + } + +}