Skip to content

Commit

Permalink
Variable recursion and directory delete helper.
Browse files Browse the repository at this point in the history
  • Loading branch information
@ericsciple
ericsciple committed Mar 22, 2016
1 parent a702a3b commit a16213a
Show file tree
Hide file tree
Showing 21 changed files with 985 additions and 309 deletions.
19 changes: 1 addition & 18 deletions src/Agent.Worker/Build/BuildDirectoryManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,25 +195,8 @@ private void DeleteDirectory(IExecutionContext executionContext, string descript
Trace.Info($"Checking if {description} exists: '{path}'");
if (Directory.Exists(path))
{
// Delete the files.
executionContext.Debug($"Deleting {description}: '{path}'");
Trace.Info("Deleting files.");
foreach (string file in Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories))
{
executionContext.CancellationToken.ThrowIfCancellationRequested();
// TODO: Test for readonly files.
File.Delete(file);
}

// Delete the directories.
Trace.Info("Deleting directories.");
foreach (string directory in Directory.GetDirectories(path, "*", SearchOption.AllDirectories).OrderByDescending(x => x.Length))
{
executionContext.CancellationToken.ThrowIfCancellationRequested();
Directory.Delete(directory);
}

Directory.Delete(path);
IOUtil.DeleteDirectory(path, executionContext.CancellationToken);
}
}
}
Expand Down
14 changes: 11 additions & 3 deletions src/Agent.Worker/ExecutionContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using Microsoft.VisualStudio.Services.Agent.Util;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;

namespace Microsoft.VisualStudio.Services.Agent.Worker
Expand Down Expand Up @@ -244,12 +245,19 @@ public void InitializeJob(JobRequestMessage message)

// Initialize the environment.
Endpoints = message.Environment.Endpoints;
Variables = new Variables(HostContext, message.Environment.Variables);
List<string> warnings;
Variables = new Variables(HostContext, message.Environment.Variables, out warnings);

// Initialize the job timeline record.
// the job timeline record is at order 1.
InitializeTimelineRecord(message.Timeline.Id, message.JobId, null, ExecutionContextType.Job, message.JobName, 1);
}

// Log any warnings.
foreach (string warning in (warnings ?? new List<string>()))
{
this.Warning(warning);
}
}

// Do not add a format string overload. In general, execution context messages are user facing and
// therefore should be localized. Use the Loc methods from the StringUtil class. The exception to
Expand All @@ -260,7 +268,7 @@ public void Write(string tag, string message)
lock (_loggerLock)
{
_logger.Write(msg);
}
}

_jobServerQueue.QueueWebConsoleLine(msg);
}
Expand Down
8 changes: 6 additions & 2 deletions src/Agent.Worker/JobRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ public async Task<TaskResult> RunAsync(JobRequestMessage message)
Trace.Info("Starting the job execution context.");
jobContext.Start();

// Expand the endpoint data values.
foreach (ServiceEndpoint endpoint in jobContext.Endpoints)
{
jobContext.Variables.ExpandValues(target: endpoint.Data);
}

// Get the job extensions.
Trace.Info("Getting job extensions.");
string hostType = jobContext.Variables.System_HostType;
Expand Down Expand Up @@ -108,8 +114,6 @@ public async Task<TaskResult> RunAsync(JobRequestMessage message)
return jobContext.Result.Value;
}

// TODO: Recursive expand variables before running the steps. Detect cycles and warn if a cyclical reference is encountered. Depth limit of 50. Use a stack, not recursive function.

// Run the steps.
var stepsRunner = HostContext.GetService<IStepsRunner>();
try
Expand Down
7 changes: 4 additions & 3 deletions src/Agent.Worker/TaskManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Threading.Tasks;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

namespace Microsoft.VisualStudio.Services.Agent.Worker
{
Expand Down Expand Up @@ -69,11 +70,11 @@ private async Task DownloadAsync(IExecutionContext executionContext, TaskReferen
{
try
{
//sometimes the temp folder is not deleted -> wipe it
//if the temp folder wasn't moved -> wipe it
if (Directory.Exists(tempDirectory))
{
Trace.Verbose("Deleting task temp folder: {0}", tempDirectory);
Directory.Delete(tempDirectory, true);
IOUtil.DeleteDirectory(tempDirectory, CancellationToken.None); // Don't cancel this cleanup and should be pretty fast.
}
}
catch (Exception ex)
Expand Down
39 changes: 1 addition & 38 deletions src/Agent.Worker/TaskRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,44 +77,7 @@ public async Task RunAsync()

// Expand the inputs.
Trace.Verbose("Expanding inputs.");
foreach (string inputKey in inputs.Keys.ToArray())
{
// Bump the start index with each replacement to prevent recursive replacement.
Trace.Verbose($"Expanding input '{inputKey}'.");
int startIndex = 0;
int prefixIndex;
int suffixIndex;
string inputValue = inputs[inputKey] ?? string.Empty;
while (startIndex < inputValue.Length &&
(prefixIndex = inputValue.IndexOf(Constants.Variables.MacroPrefix, startIndex, StringComparison.Ordinal)) >= 0 &&
(suffixIndex = inputValue.IndexOf(Constants.Variables.MacroSuffix, prefixIndex + Constants.Variables.MacroPrefix.Length, StringComparison.Ordinal)) >= 0)
{
// A variable macro candidate was found.
string variableKey = inputValue.Substring(
startIndex: prefixIndex + Constants.Variables.MacroPrefix.Length,
length: suffixIndex - prefixIndex - Constants.Variables.MacroPrefix.Length);
Trace.Verbose($"Variable macro candidate '{variableKey}'.");
string variableValue;
if (!string.IsNullOrEmpty(variableKey) &&
ExecutionContext.Variables.TryGetValue(variableKey, out variableValue))
{
// Update the input value.
Trace.Verbose("Candidate found.");
inputValue = string.Concat(
inputValue.Substring(0, prefixIndex),
variableValue ?? string.Empty,
inputValue.Substring(suffixIndex + Constants.Variables.MacroSuffix.Length));
startIndex = prefixIndex + (variableValue ?? string.Empty).Length;
}
else
{
Trace.Verbose("Candidate not found.");
startIndex += Constants.Variables.MacroPrefix.Length;
}
}

inputs[inputKey] = inputValue;
}
ExecutionContext.Variables.ExpandValues(target: inputs);

// TODO: Delegate to the source provider to fixup the file path inputs.

Expand Down
190 changes: 189 additions & 1 deletion src/Agent.Worker/Variables.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;

namespace Microsoft.VisualStudio.Services.Agent.Worker
{
Expand All @@ -15,8 +16,9 @@ public sealed class Variables
private readonly IHostContext _hostContext;
private readonly Tracing _trace;

public Variables(IHostContext hostContext, IDictionary<string, string> copy)
public Variables(IHostContext hostContext, IDictionary<string, string> copy, out List<string> warnings)
{
// Store/Validate args.
_hostContext = hostContext;
_trace = _hostContext.GetTrace(nameof(Variables));
ArgUtil.NotNull(hostContext, nameof(hostContext));
Expand All @@ -25,6 +27,9 @@ public Variables(IHostContext hostContext, IDictionary<string, string> copy)
{
_store[key] = copy[key];
}

// Recursively expand the variables.
RecursivelyExpand(out warnings);
}

public BuildCleanOption? Build_Clean { get { return GetEnum<BuildCleanOption>(Constants.Variables.Build.Clean); } }
Expand All @@ -36,6 +41,59 @@ public Variables(IHostContext hostContext, IDictionary<string, string> copy)
public string System_TFCollectionUrl { get { return Get(WellKnownDistributedTaskVariables.TFCollectionUrl); } }
public bool? System_EnableAccessToken { get { return GetBoolean(Constants.Variables.System.EnableAccessToken); } }

public void ExpandValues(IDictionary<string, string> target)
{
_trace.Entering();
target = target ?? new Dictionary<string, string>();

// This algorithm does not perform recursive replacement.

// Process each key in the target dictionary.
foreach (string targetKey in target.Keys.ToArray())
{
_trace.Verbose($"Expanding key: '{targetKey}'");
int startIndex = 0;
int prefixIndex;
int suffixIndex;
string targetValue = target[targetKey] ?? string.Empty;

// Find the next macro within the target value.
while (startIndex < targetValue.Length &&
(prefixIndex = targetValue.IndexOf(Constants.Variables.MacroPrefix, startIndex, StringComparison.Ordinal)) >= 0 &&
(suffixIndex = targetValue.IndexOf(Constants.Variables.MacroSuffix, prefixIndex + Constants.Variables.MacroPrefix.Length, StringComparison.Ordinal)) >= 0)
{
// A candidate was found.
string variableKey = targetValue.Substring(
startIndex: prefixIndex + Constants.Variables.MacroPrefix.Length,
length: suffixIndex - prefixIndex - Constants.Variables.MacroPrefix.Length);
_trace.Verbose($"Found macro candidate: '{variableKey}'");
string variableValue;
if (!string.IsNullOrEmpty(variableKey) &&
TryGetValue(variableKey, out variableValue))
{
// A matching variable was found.
// Update the target value.
_trace.Verbose("Macro found.");
targetValue = string.Concat(
targetValue.Substring(0, prefixIndex),
variableValue ?? string.Empty,
targetValue.Substring(suffixIndex + Constants.Variables.MacroSuffix.Length));

// Bump the start index to prevent recursive replacement.
startIndex = prefixIndex + (variableValue ?? string.Empty).Length;
}
else
{
// A matching variable was not found.
_trace.Verbose("Macro not found.");
startIndex = prefixIndex + 1;
}
}

target[targetKey] = targetValue;
}
}

public string Get(string name)
{
string val;
Expand Down Expand Up @@ -75,6 +133,7 @@ public void Set(string name, string val)
{
// TODO: Determine whether this line should be uncommented again: ArgUtil.NotNull(val, nameof(val));
// Can variables not be cleared? Can a null variable come across the wire? What if the user does ##setvariable from a script and we interpret as null instead of empty string. This feels brittle.

//TODO: Determine if variable should be added to SecretMasker

_trace.Verbose($"Set '{name}' = '{val}'");
Expand All @@ -92,6 +151,135 @@ public bool TryGetValue(string name, out string val)
_trace.Verbose($"Get '{name}' (not found)");
return false;
}

private void RecursivelyExpand(out List<string> warnings)
{
// TODO: Should tracing be omitted when expanding secret vales?
const int MaxDepth = 50;
// TODO: Max size?
_trace.Entering();
warnings = new List<string>();

// Make a copy of the original dictionary so the expansion results are predictable. Otherwise,
// depending on the order of expansion, an actual max depth restriction may not be encountered.
var original = new Dictionary<string, string>(_store, StringComparer.OrdinalIgnoreCase);

// Process each variable in the dictionary.
foreach (string key in original.Keys)
{
_trace.Verbose($"Expanding variable: '{key}'");

// This algorithm handles recursive replacement using a stack.
// 1) Max depth is enforced by leveraging the stack count.
// 2) Cyclical references are detected by walking the stack.
// 3) Additional call frames are avoided.
bool exceedsMaxDepth = false;
bool hasCycle = false;
var stack = new Stack<RecursionState>();
RecursionState state = new RecursionState(key: key, value: original[key]);

// The outer while loop is used to manage popping items from the stack (of state objects).
while (true)
{
// The inner while loop is used to manage replacement within the current state object.

// Find the next macro within the current value.
while (state.StartIndex < state.Value.Length &&
(state.PrefixIndex = state.Value.IndexOf(Constants.Variables.MacroPrefix, state.StartIndex, StringComparison.Ordinal)) >= 0 &&
(state.SuffixIndex = state.Value.IndexOf(Constants.Variables.MacroSuffix, state.PrefixIndex + Constants.Variables.MacroPrefix.Length, StringComparison.Ordinal)) >= 0)
{
// A candidate was found.
string nestedKey = state.Value.Substring(
startIndex: state.PrefixIndex + Constants.Variables.MacroPrefix.Length,
length: state.SuffixIndex - state.PrefixIndex - Constants.Variables.MacroPrefix.Length);
_trace.Verbose($"Found macro candidate: '{nestedKey}'");
string nestedValue;
if (!string.IsNullOrEmpty(nestedKey) &&
original.TryGetValue(nestedKey, out nestedValue))
{
// A matching variable was found.
// Push the current state onto the stack.
_trace.Verbose("Macro found.");

// Check for max depth.
int currentDepth = stack.Count + 1; // Add 1 since the current state isn't on the stack.
if (currentDepth == MaxDepth)
{
// Warn and break out of the while loops.
_trace.Warning("Exceeds max depth.");
exceedsMaxDepth = true;
warnings.Add(StringUtil.Loc("Variable0ExceedsMaxDepth1", key, MaxDepth));
break;
}
// Check for a cyclical reference.
else if (string.Equals(state.Key, nestedKey, StringComparison.OrdinalIgnoreCase) ||
stack.Any(x => string.Equals(x.Key, nestedKey, StringComparison.OrdinalIgnoreCase)))
{
// Warn and break out of the while loops.
_trace.Warning("Cyclical reference detected.");
hasCycle = true;
warnings.Add(StringUtil.Loc("Variable0ContainsCyclicalReference", key));
break;
}
else
{
// Push the current state and start a new state. There is no need to break out
// of the inner while loop. It will continue processing the new current state.
_trace.Verbose($"Expanding nested variable: '{nestedKey}'");
stack.Push(state);
state = new RecursionState(key: nestedKey, value: nestedValue);
}
}
else
{
// A matching variable was not found.
_trace.Verbose("Macro not found.");
state.StartIndex = state.PrefixIndex + 1;
}
} // End of inner while loop for processing the variable.

// No replacement is performed if something went wrong.
if (exceedsMaxDepth || hasCycle)
{
break;
}

// Check if finished processing the stack.
if (stack.Count == 0)
{
// Store the final value and break out of the outer while loop.
Set(state.Key, state.Value ?? string.Empty);
break;
}

// Adjust and pop the parent state.
_trace.Verbose("Popping recursion state.");
RecursionState parent = stack.Pop();
parent.Value = string.Concat(
parent.Value.Substring(0, parent.PrefixIndex),
state.Value ?? string.Empty,
parent.Value.Substring(parent.SuffixIndex + Constants.Variables.MacroSuffix.Length));
parent.StartIndex = parent.PrefixIndex + (state.Value ?? string.Empty).Length;
state = parent;
_trace.Verbose($"Intermediate state '{state.Key}': '{state.Value}'");
} // End of outer while loop for recursively processing the variable.
} // End of foreach loop over each key in the dictionary.
}

private sealed class RecursionState
{
public RecursionState(string key, string value)
{
Key = key;
Value = value;
}

public string Key { get; private set; }
public string Value { get; set; }
public int StartIndex { get; set; }
public int PrefixIndex { get; set; }
public int SuffixIndex { get; set; }
}
}

public enum BuildCleanOption
Expand Down
Loading

0 comments on commit a16213a

Please sign in to comment.