Skip to content

Commit

Permalink
Composite Actions Support for Multiple Run Steps (actions#549)
Browse files Browse the repository at this point in the history
* Composite Action Run Steps

* Clean up trace messages + add Trace debug in ActionManager

* Change String to string

* Add comma to Composite

* Change JobSteps to a List, Change Register Step function name

* Add TODO, remove unn. content

* Remove unnecessary code

* Fix unit tests

* Add verbose trace logs which are only viewable by devs

* Sort usings in Composite Action Handler

* Change 0 to location

* Update context variables in composite action yaml

* Add helpful error message for null steps
  • Loading branch information
ethanchewy authored Jun 23, 2020
1 parent ee5b7ce commit 5e0e4ff
Show file tree
Hide file tree
Showing 13 changed files with 624 additions and 36 deletions.
20 changes: 20 additions & 0 deletions src/Runner.Worker/ActionManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,12 @@ public Definition LoadAction(IExecutionContext executionContext, Pipelines.Actio
Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}.");
}
}
else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Composite && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
var compositeAction = definition.Data.Execution as CompositeActionExecutionData;
Trace.Info($"Load {compositeAction.Steps.Count} action steps.");
Trace.Verbose($"Details: {StringUtil.ConvertToJson(compositeAction.Steps)}");
}
else
{
throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString());
Expand Down Expand Up @@ -1038,6 +1044,11 @@ private ActionContainer PrepareRepositoryActionAsync(IExecutionContext execution
Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation.");
return null;
}
else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Composite && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
Trace.Info($"Action composite: {(actionDefinitionData.Execution as CompositeActionExecutionData).Steps}, no more preparation.");
return null;
}
else
{
throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString());
Expand Down Expand Up @@ -1148,6 +1159,7 @@ public enum ActionExecutionType
NodeJS,
Plugin,
Script,
Composite,
}

public sealed class ContainerActionExecutionData : ActionExecutionData
Expand Down Expand Up @@ -1204,6 +1216,14 @@ public sealed class ScriptActionExecutionData : ActionExecutionData
public override bool HasPost => false;
}

public sealed class CompositeActionExecutionData : ActionExecutionData
{
public override ActionExecutionType ExecutionType => ActionExecutionType.Composite;
public override bool HasPre => false;
public override bool HasPost => false;
public List<Pipelines.ActionStep> Steps { get; set; }
}

public abstract class ActionExecutionData
{
private string _initCondition = $"{Constants.Expressions.Always}()";
Expand Down
32 changes: 30 additions & 2 deletions src/Runner.Worker/ActionManifestManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
using YamlDotNet.Core.Events;
using System.Globalization;
using System.Linq;
using Pipelines = GitHub.DistributedTask.Pipelines;

namespace GitHub.Runner.Worker
{
Expand Down Expand Up @@ -92,7 +93,7 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani
break;

case "runs":
actionDefinition.Execution = ConvertRuns(context, actionPair.Value);
actionDefinition.Execution = ConvertRuns(executionContext, context, actionPair.Value);
break;
default:
Trace.Info($"Ignore action property {propertyName}.");
Expand Down Expand Up @@ -284,7 +285,7 @@ private TemplateContext CreateContext(
// Add the file table
if (_fileTable?.Count > 0)
{
for (var i = 0 ; i < _fileTable.Count ; i++)
for (var i = 0; i < _fileTable.Count; i++)
{
result.GetFileId(_fileTable[i]);
}
Expand All @@ -294,6 +295,7 @@ private TemplateContext CreateContext(
}

private ActionExecutionData ConvertRuns(
IExecutionContext executionContext,
TemplateContext context,
TemplateToken inputsToken)
{
Expand All @@ -311,6 +313,8 @@ private ActionExecutionData ConvertRuns(
var postToken = default(StringToken);
var postEntrypointToken = default(StringToken);
var postIfToken = default(StringToken);
var stepsLoaded = default(List<Pipelines.ActionStep>);

foreach (var run in runsMapping)
{
var runsKey = run.Key.AssertString("runs key").Value;
Expand Down Expand Up @@ -355,6 +359,15 @@ private ActionExecutionData ConvertRuns(
case "pre-if":
preIfToken = run.Value.AssertString("pre-if");
break;
case "steps":
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
var steps = run.Value.AssertSequence("steps");
var evaluator = executionContext.ToPipelineTemplateEvaluator();
stepsLoaded = evaluator.LoadCompositeSteps(steps);
break;
}
throw new Exception("You aren't supposed to be using Composite Actions yet!");
default:
Trace.Info($"Ignore run property {runsKey}.");
break;
Expand Down Expand Up @@ -402,6 +415,21 @@ private ActionExecutionData ConvertRuns(
};
}
}
else if (string.Equals(usingToken.Value, "composite", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
if (stepsLoaded == null)
{
// TODO: Add a more helpful error message + including file name, etc. to show user that it's because of their yaml file
throw new ArgumentNullException($"No steps provided.");
}
else
{
return new CompositeActionExecutionData()
{
Steps = stepsLoaded,
};
}
}
else
{
throw new ArgumentOutOfRangeException($"'using: {usingToken.Value}' is not supported, use 'docker' or 'node12' instead.");
Expand Down
22 changes: 18 additions & 4 deletions src/Runner.Worker/ExecutionContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public interface IExecutionContext : IRunnerService
JobContext JobContext { get; }

// Only job level ExecutionContext has JobSteps
Queue<IStep> JobSteps { get; }
List<IStep> JobSteps { get; }

// Only job level ExecutionContext has PostJobSteps
Stack<IStep> PostJobSteps { get; }
Expand Down Expand Up @@ -105,6 +105,7 @@ public interface IExecutionContext : IRunnerService
// others
void ForceTaskComplete();
void RegisterPostJobStep(IStep step);
void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location);
}

public sealed class ExecutionContext : RunnerService, IExecutionContext
Expand Down Expand Up @@ -159,7 +160,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext
public List<ContainerInfo> ServiceContainers { get; private set; }

// Only job level ExecutionContext has JobSteps
public Queue<IStep> JobSteps { get; private set; }
public List<IStep> JobSteps { get; private set; }

// Only job level ExecutionContext has PostJobSteps
public Stack<IStep> PostJobSteps { get; private set; }
Expand All @@ -169,7 +170,6 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext

public bool EchoOnActionCommand { get; set; }


public TaskResult? Result
{
get
Expand Down Expand Up @@ -266,6 +266,20 @@ public void RegisterPostJobStep(IStep step)
Root.PostJobSteps.Push(step);
}

/// <summary>
/// Helper function used in CompositeActionHandler::RunAsync to
/// add a child node, aka a step, to the current job to the Root.JobSteps based on the location.
/// </summary>
public void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location)
{
// TODO: For UI purposes, look at figuring out how to condense steps in one node => maybe use the same previous GUID
var newGuid = Guid.NewGuid();
step.ExecutionContext = Root.CreateChild(newGuid, step.DisplayName, newGuid.ToString("N"), null, null);
step.ExecutionContext.ExpressionValues["inputs"] = inputsData;
// TODO: confirm whether not copying message contexts is safe
Root.JobSteps.Insert(location, step);
}

public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary<string, string> intraActionState = null, int? recordOrder = null)
{
Trace.Entering();
Expand Down Expand Up @@ -660,7 +674,7 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation
PrependPath = new List<string>();

// JobSteps for job ExecutionContext
JobSteps = new Queue<IStep>();
JobSteps = new List<IStep>();

// PostJobSteps for job ExecutionContext
PostJobSteps = new Stack<IStep>();
Expand Down
98 changes: 98 additions & 0 deletions src/Runner.Worker/Handlers/CompositeActionHandler.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
using Pipelines = GitHub.DistributedTask.Pipelines;


namespace GitHub.Runner.Worker.Handlers
{
[ServiceLocator(Default = typeof(CompositeActionHandler))]
public interface ICompositeActionHandler : IHandler
{
CompositeActionExecutionData Data { get; set; }
}
public sealed class CompositeActionHandler : Handler, ICompositeActionHandler
{
public CompositeActionExecutionData Data { get; set; }

public Task RunAsync(ActionRunStage stage)
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));

var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext;
ArgUtil.NotNull(githubContext, nameof(githubContext));

var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp);

// Resolve action steps
var actionSteps = Data.Steps;

// Create Context Data to reuse for each composite action step
var inputsData = new DictionaryContextData();
foreach (var i in Inputs)
{
inputsData[i.Key] = new StringContextData(i.Value);
}

// Add each composite action step to the front of the queue
int location = 0;
foreach (Pipelines.ActionStep aStep in actionSteps)
{
// Ex:
// runs:
// using: "composite"
// steps:
// - uses: example/test-composite@v2 (a)
// - run echo hello world (b)
// - run echo hello world 2 (c)
//
// ethanchewy/test-composite/action.yaml
// runs:
// using: "composite"
// steps:
// - run echo hello world 3 (d)
// - run echo hello world 4 (e)
//
// Steps processed as follow:
// | a |
// | a | => | d |
// (Run step d)
// | a |
// | a | => | e |
// (Run step e)
// | a |
// (Run step a)
// | b |
// (Run step b)
// | c |
// (Run step c)
// Done.

var actionRunner = HostContext.CreateService<IActionRunner>();
actionRunner.Action = aStep;
actionRunner.Stage = stage;
actionRunner.Condition = aStep.Condition;
actionRunner.DisplayName = aStep.DisplayName;
// TODO: Do we need to add any context data from the job message?
// (See JobExtension.cs ~line 236)

ExecutionContext.RegisterNestedStep(actionRunner, inputsData, location);
location++;
}

return Task.CompletedTask;
}

}
}
5 changes: 5 additions & 0 deletions src/Runner.Worker/Handlers/HandlerFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,11 @@ public IHandler Create(
handler = HostContext.CreateService<IRunnerPluginHandler>();
(handler as IRunnerPluginHandler).Data = data as PluginActionExecutionData;
}
else if (data.ExecutionType == ActionExecutionType.Composite)
{
handler = HostContext.CreateService<ICompositeActionHandler>();
(handler as ICompositeActionHandler).Data = data as CompositeActionExecutionData;
}
else
{
// This should never happen.
Expand Down
2 changes: 1 addition & 1 deletion src/Runner.Worker/JobRunner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ public async Task<TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message,
{
foreach (var step in jobSteps)
{
jobContext.JobSteps.Enqueue(step);
jobContext.JobSteps.Add(step);
}

await stepsRunner.RunAsync(jobContext);
Expand Down
19 changes: 14 additions & 5 deletions src/Runner.Worker/StepsRunner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,15 @@ public async Task RunAsync(IExecutionContext jobContext)
checkPostJobActions = true;
while (jobContext.PostJobSteps.TryPop(out var postStep))
{
jobContext.JobSteps.Enqueue(postStep);
jobContext.JobSteps.Add(postStep);
}

continue;
}

var step = jobContext.JobSteps.Dequeue();
var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps.Peek() : null;
var step = jobContext.JobSteps[0];
jobContext.JobSteps.RemoveAt(0);
var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps[0] : null;

Trace.Info($"Processing step: DisplayName='{step.DisplayName}'");
ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext));
Expand Down Expand Up @@ -409,7 +410,11 @@ private bool InitializeScope(IStep step, Dictionary<string, PipelineContextData>
scope = scopesToInitialize.Pop();
executionContext.Debug($"Initializing scope '{scope.Name}'");
executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.ParentName);
executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null;
// TODO: Fix this temporary workaround for Composite Actions
if (!executionContext.ExpressionValues.ContainsKey("inputs") && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null;
}
var templateEvaluator = executionContext.ToPipelineTemplateEvaluator();
var inputs = default(DictionaryContextData);
try
Expand All @@ -432,7 +437,11 @@ private bool InitializeScope(IStep step, Dictionary<string, PipelineContextData>
// Setup expression values
var scopeName = executionContext.ScopeName;
executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scopeName);
executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName];
// TODO: Fix this temporary workaround for Composite Actions
if (!executionContext.ExpressionValues.ContainsKey("inputs") && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA")))
{
executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName];
}

return true;
}
Expand Down
Loading

0 comments on commit 5e0e4ff

Please sign in to comment.