From 5e0e4ff6275be12ad1874595fecbe9e278323576 Mon Sep 17 00:00:00 2001 From: Ethan Chiu <17chiue@gmail.com> Date: Tue, 23 Jun 2020 15:35:32 -0400 Subject: [PATCH] Composite Actions Support for Multiple Run Steps (#549) * Composite Action Run Steps * Clean up trace messages + add Trace debug in ActionManager * Change String to string * Add comma to Composite * Change JobSteps to a List, Change Register Step function name * Add TODO, remove unn. content * Remove unnecessary code * Fix unit tests * Add verbose trace logs which are only viewable by devs * Sort usings in Composite Action Handler * Change 0 to location * Update context variables in composite action yaml * Add helpful error message for null steps --- src/Runner.Worker/ActionManager.cs | 20 + src/Runner.Worker/ActionManifestManager.cs | 32 +- src/Runner.Worker/ExecutionContext.cs | 22 +- .../Handlers/CompositeActionHandler.cs | 98 +++++ src/Runner.Worker/Handlers/HandlerFactory.cs | 5 + src/Runner.Worker/JobRunner.cs | 2 +- src/Runner.Worker/StepsRunner.cs | 19 +- src/Runner.Worker/action_yaml.json | 33 +- .../PipelineTemplateConstants.cs | 1 + .../PipelineTemplateConverter.cs | 347 +++++++++++++++++- .../PipelineTemplateEvaluator.cs | 48 ++- .../Pipelines/PipelineConstants.cs | 7 + src/Test/L0/Worker/StepsRunnerL0.cs | 26 +- 13 files changed, 624 insertions(+), 36 deletions(-) create mode 100644 src/Runner.Worker/Handlers/CompositeActionHandler.cs diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index b9720610d92..242ab79e052 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -395,6 +395,12 @@ public Definition LoadAction(IExecutionContext executionContext, Pipelines.Actio Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}."); } } + else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Composite && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + var compositeAction = definition.Data.Execution as CompositeActionExecutionData; + Trace.Info($"Load {compositeAction.Steps.Count} action steps."); + Trace.Verbose($"Details: {StringUtil.ConvertToJson(compositeAction.Steps)}"); + } else { throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString()); @@ -1038,6 +1044,11 @@ private ActionContainer PrepareRepositoryActionAsync(IExecutionContext execution Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation."); return null; } + else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Composite && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + Trace.Info($"Action composite: {(actionDefinitionData.Execution as CompositeActionExecutionData).Steps}, no more preparation."); + return null; + } else { throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString()); @@ -1148,6 +1159,7 @@ public enum ActionExecutionType NodeJS, Plugin, Script, + Composite, } public sealed class ContainerActionExecutionData : ActionExecutionData @@ -1204,6 +1216,14 @@ public sealed class ScriptActionExecutionData : ActionExecutionData public override bool HasPost => false; } + public sealed class CompositeActionExecutionData : ActionExecutionData + { + public override ActionExecutionType ExecutionType => ActionExecutionType.Composite; + public override bool HasPre => false; + public override bool HasPost => false; + public List Steps { get; set; } + } + public abstract class ActionExecutionData { private string _initCondition = $"{Constants.Expressions.Always}()"; diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 4e9149d26b6..9095f498ddd 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -14,6 +14,7 @@ using YamlDotNet.Core.Events; using System.Globalization; using System.Linq; +using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker { @@ -92,7 +93,7 @@ public ActionDefinitionData Load(IExecutionContext executionContext, string mani break; case "runs": - actionDefinition.Execution = ConvertRuns(context, actionPair.Value); + actionDefinition.Execution = ConvertRuns(executionContext, context, actionPair.Value); break; default: Trace.Info($"Ignore action property {propertyName}."); @@ -284,7 +285,7 @@ private TemplateContext CreateContext( // Add the file table if (_fileTable?.Count > 0) { - for (var i = 0 ; i < _fileTable.Count ; i++) + for (var i = 0; i < _fileTable.Count; i++) { result.GetFileId(_fileTable[i]); } @@ -294,6 +295,7 @@ private TemplateContext CreateContext( } private ActionExecutionData ConvertRuns( + IExecutionContext executionContext, TemplateContext context, TemplateToken inputsToken) { @@ -311,6 +313,8 @@ private ActionExecutionData ConvertRuns( var postToken = default(StringToken); var postEntrypointToken = default(StringToken); var postIfToken = default(StringToken); + var stepsLoaded = default(List); + foreach (var run in runsMapping) { var runsKey = run.Key.AssertString("runs key").Value; @@ -355,6 +359,15 @@ private ActionExecutionData ConvertRuns( case "pre-if": preIfToken = run.Value.AssertString("pre-if"); break; + case "steps": + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + var steps = run.Value.AssertSequence("steps"); + var evaluator = executionContext.ToPipelineTemplateEvaluator(); + stepsLoaded = evaluator.LoadCompositeSteps(steps); + break; + } + throw new Exception("You aren't supposed to be using Composite Actions yet!"); default: Trace.Info($"Ignore run property {runsKey}."); break; @@ -402,6 +415,21 @@ private ActionExecutionData ConvertRuns( }; } } + else if (string.Equals(usingToken.Value, "composite", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + if (stepsLoaded == null) + { + // TODO: Add a more helpful error message + including file name, etc. to show user that it's because of their yaml file + throw new ArgumentNullException($"No steps provided."); + } + else + { + return new CompositeActionExecutionData() + { + Steps = stepsLoaded, + }; + } + } else { throw new ArgumentOutOfRangeException($"'using: {usingToken.Value}' is not supported, use 'docker' or 'node12' instead."); diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 0318974b01d..cea1e2fd4d7 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -63,7 +63,7 @@ public interface IExecutionContext : IRunnerService JobContext JobContext { get; } // Only job level ExecutionContext has JobSteps - Queue JobSteps { get; } + List JobSteps { get; } // Only job level ExecutionContext has PostJobSteps Stack PostJobSteps { get; } @@ -105,6 +105,7 @@ public interface IExecutionContext : IRunnerService // others void ForceTaskComplete(); void RegisterPostJobStep(IStep step); + void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location); } public sealed class ExecutionContext : RunnerService, IExecutionContext @@ -159,7 +160,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext public List ServiceContainers { get; private set; } // Only job level ExecutionContext has JobSteps - public Queue JobSteps { get; private set; } + public List JobSteps { get; private set; } // Only job level ExecutionContext has PostJobSteps public Stack PostJobSteps { get; private set; } @@ -169,7 +170,6 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext public bool EchoOnActionCommand { get; set; } - public TaskResult? Result { get @@ -266,6 +266,20 @@ public void RegisterPostJobStep(IStep step) Root.PostJobSteps.Push(step); } + /// + /// Helper function used in CompositeActionHandler::RunAsync to + /// add a child node, aka a step, to the current job to the Root.JobSteps based on the location. + /// + public void RegisterNestedStep(IStep step, DictionaryContextData inputsData, int location) + { + // TODO: For UI purposes, look at figuring out how to condense steps in one node => maybe use the same previous GUID + var newGuid = Guid.NewGuid(); + step.ExecutionContext = Root.CreateChild(newGuid, step.DisplayName, newGuid.ToString("N"), null, null); + step.ExecutionContext.ExpressionValues["inputs"] = inputsData; + // TODO: confirm whether not copying message contexts is safe + Root.JobSteps.Insert(location, step); + } + public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary intraActionState = null, int? recordOrder = null) { Trace.Entering(); @@ -660,7 +674,7 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation PrependPath = new List(); // JobSteps for job ExecutionContext - JobSteps = new Queue(); + JobSteps = new List(); // PostJobSteps for job ExecutionContext PostJobSteps = new Stack(); diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs new file mode 100644 index 00000000000..7c5b25ed5fb --- /dev/null +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines.ContextData; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Sdk; +using Pipelines = GitHub.DistributedTask.Pipelines; + + +namespace GitHub.Runner.Worker.Handlers +{ + [ServiceLocator(Default = typeof(CompositeActionHandler))] + public interface ICompositeActionHandler : IHandler + { + CompositeActionExecutionData Data { get; set; } + } + public sealed class CompositeActionHandler : Handler, ICompositeActionHandler + { + public CompositeActionExecutionData Data { get; set; } + + public Task RunAsync(ActionRunStage stage) + { + // Validate args. + Trace.Entering(); + ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); + ArgUtil.NotNull(Inputs, nameof(Inputs)); + + var githubContext = ExecutionContext.ExpressionValues["github"] as GitHubContext; + ArgUtil.NotNull(githubContext, nameof(githubContext)); + + var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp); + + // Resolve action steps + var actionSteps = Data.Steps; + + // Create Context Data to reuse for each composite action step + var inputsData = new DictionaryContextData(); + foreach (var i in Inputs) + { + inputsData[i.Key] = new StringContextData(i.Value); + } + + // Add each composite action step to the front of the queue + int location = 0; + foreach (Pipelines.ActionStep aStep in actionSteps) + { + // Ex: + // runs: + // using: "composite" + // steps: + // - uses: example/test-composite@v2 (a) + // - run echo hello world (b) + // - run echo hello world 2 (c) + // + // ethanchewy/test-composite/action.yaml + // runs: + // using: "composite" + // steps: + // - run echo hello world 3 (d) + // - run echo hello world 4 (e) + // + // Steps processed as follow: + // | a | + // | a | => | d | + // (Run step d) + // | a | + // | a | => | e | + // (Run step e) + // | a | + // (Run step a) + // | b | + // (Run step b) + // | c | + // (Run step c) + // Done. + + var actionRunner = HostContext.CreateService(); + actionRunner.Action = aStep; + actionRunner.Stage = stage; + actionRunner.Condition = aStep.Condition; + actionRunner.DisplayName = aStep.DisplayName; + // TODO: Do we need to add any context data from the job message? + // (See JobExtension.cs ~line 236) + + ExecutionContext.RegisterNestedStep(actionRunner, inputsData, location); + location++; + } + + return Task.CompletedTask; + } + + } +} diff --git a/src/Runner.Worker/Handlers/HandlerFactory.cs b/src/Runner.Worker/Handlers/HandlerFactory.cs index 0f2413ef5b7..db4d6559c88 100644 --- a/src/Runner.Worker/Handlers/HandlerFactory.cs +++ b/src/Runner.Worker/Handlers/HandlerFactory.cs @@ -66,6 +66,11 @@ public IHandler Create( handler = HostContext.CreateService(); (handler as IRunnerPluginHandler).Data = data as PluginActionExecutionData; } + else if (data.ExecutionType == ActionExecutionType.Composite) + { + handler = HostContext.CreateService(); + (handler as ICompositeActionHandler).Data = data as CompositeActionExecutionData; + } else { // This should never happen. diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 31dfb17145b..33b291adb6d 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -152,7 +152,7 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, { foreach (var step in jobSteps) { - jobContext.JobSteps.Enqueue(step); + jobContext.JobSteps.Add(step); } await stepsRunner.RunAsync(jobContext); diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index 485a4cdf980..e75d2e106f8 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -59,14 +59,15 @@ public async Task RunAsync(IExecutionContext jobContext) checkPostJobActions = true; while (jobContext.PostJobSteps.TryPop(out var postStep)) { - jobContext.JobSteps.Enqueue(postStep); + jobContext.JobSteps.Add(postStep); } continue; } - var step = jobContext.JobSteps.Dequeue(); - var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps.Peek() : null; + var step = jobContext.JobSteps[0]; + jobContext.JobSteps.RemoveAt(0); + var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps[0] : null; Trace.Info($"Processing step: DisplayName='{step.DisplayName}'"); ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext)); @@ -409,7 +410,11 @@ private bool InitializeScope(IStep step, Dictionary scope = scopesToInitialize.Pop(); executionContext.Debug($"Initializing scope '{scope.Name}'"); executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.ParentName); - executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null; + // TODO: Fix this temporary workaround for Composite Actions + if (!executionContext.ExpressionValues.ContainsKey("inputs") && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null; + } var templateEvaluator = executionContext.ToPipelineTemplateEvaluator(); var inputs = default(DictionaryContextData); try @@ -432,7 +437,11 @@ private bool InitializeScope(IStep step, Dictionary // Setup expression values var scopeName = executionContext.ScopeName; executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scopeName); - executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName]; + // TODO: Fix this temporary workaround for Composite Actions + if (!executionContext.ExpressionValues.ContainsKey("inputs") && !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TESTING_COMPOSITE_ACTIONS_ALPHA"))) + { + executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName]; + } return true; } diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index 7a8b847d31f..cb1d90b2e0b 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -32,7 +32,8 @@ "one-of": [ "container-runs", "node12-runs", - "plugin-runs" + "plugin-runs", + "composite-runs" ] }, "container-runs": { @@ -83,6 +84,36 @@ } } }, + "composite-runs": { + "mapping": { + "properties": { + "using": "non-empty-string", + "steps": "composite-steps" + } + } + }, + "composite-steps": { + "context": [ + "github", + "needs", + "strategy", + "matrix", + "secrets", + "steps", + "inputs", + "job", + "runner", + "env", + "always(0,0)", + "failure(0,0)", + "cancelled(0,0)", + "success(0,0)", + "hashFiles(1,255)" + ], + "sequence": { + "item-type": "any" + } + }, "container-runs-context": { "context": [ "inputs" diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs index d1c886dd891..f2609462b23 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs @@ -65,6 +65,7 @@ public sealed class PipelineTemplateConstants public const String StepEnv = "step-env"; public const String StepIfResult = "step-if-result"; public const String Steps = "steps"; + public const String StepsInTemplate = "steps-in-template"; public const String StepsScopeInputs = "steps-scope-inputs"; public const String StepsScopeOutputs = "steps-scope-outputs"; public const String StepsTemplateRoot = "steps-template-root"; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index 43be43d3375..a952f58fbb9 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -29,7 +29,6 @@ internal static Boolean ConvertToIfResult( var evaluationResult = EvaluationResult.CreateIntermediateResult(null, ifResult); return evaluationResult.IsTruthy; } - internal static Boolean? ConvertToStepContinueOnError( TemplateContext context, TemplateToken token, @@ -264,5 +263,351 @@ internal static List> ConvertToJobServiceCont return result; } + + //Note: originally was List but we need to change to List to use the "Inputs" attribute + internal static List ConvertToSteps( + TemplateContext context, + TemplateToken steps) + { + var stepsSequence = steps.AssertSequence($"job {PipelineTemplateConstants.Steps}"); + + var result = new List(); + foreach (var stepsItem in stepsSequence) + { + var step = ConvertToStep(context, stepsItem); + if (step != null) // step = null means we are hitting error during step conversion, there should be an error in context.errors + { + if (step.Enabled) + { + result.Add(step); + } + } + } + + return result; + } + + private static ActionStep ConvertToStep( + TemplateContext context, + TemplateToken stepsItem) + { + var step = stepsItem.AssertMapping($"{PipelineTemplateConstants.Steps} item"); + var continueOnError = default(ScalarToken); + var env = default(TemplateToken); + var id = default(StringToken); + var ifCondition = default(String); + var ifToken = default(ScalarToken); + var name = default(ScalarToken); + var run = default(ScalarToken); + var scope = default(StringToken); + var timeoutMinutes = default(ScalarToken); + var uses = default(StringToken); + var with = default(TemplateToken); + var workingDir = default(ScalarToken); + var path = default(ScalarToken); + var clean = default(ScalarToken); + var fetchDepth = default(ScalarToken); + var lfs = default(ScalarToken); + var submodules = default(ScalarToken); + var shell = default(ScalarToken); + + foreach (var stepProperty in step) + { + var propertyName = stepProperty.Key.AssertString($"{PipelineTemplateConstants.Steps} item key"); + + switch (propertyName.Value) + { + case PipelineTemplateConstants.Clean: + clean = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Clean}"); + break; + + case PipelineTemplateConstants.ContinueOnError: + ConvertToStepContinueOnError(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + continueOnError = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} {PipelineTemplateConstants.ContinueOnError}"); + break; + + case PipelineTemplateConstants.Env: + ConvertToStepEnvironment(context, stepProperty.Value, StringComparer.Ordinal, allowExpressions: true); // Validate early if possible + env = stepProperty.Value; + break; + + case PipelineTemplateConstants.FetchDepth: + fetchDepth = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.FetchDepth}"); + break; + + case PipelineTemplateConstants.Id: + id = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Id}"); + if (!NameValidation.IsValid(id.Value, true)) + { + context.Error(id, $"Step id {id.Value} is invalid. Ids must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'"); + } + break; + + case PipelineTemplateConstants.If: + ifToken = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.If}"); + break; + + case PipelineTemplateConstants.Lfs: + lfs = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Lfs}"); + break; + + case PipelineTemplateConstants.Name: + name = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Name}"); + break; + + case PipelineTemplateConstants.Path: + path = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Path}"); + break; + + case PipelineTemplateConstants.Run: + run = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Run}"); + break; + + case PipelineTemplateConstants.Shell: + shell = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Shell}"); + break; + + case PipelineTemplateConstants.Scope: + scope = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Scope}"); + break; + + case PipelineTemplateConstants.Submodules: + submodules = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Submodules}"); + break; + + case PipelineTemplateConstants.TimeoutMinutes: + ConvertToStepTimeout(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + timeoutMinutes = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.TimeoutMinutes}"); + break; + + case PipelineTemplateConstants.Uses: + uses = stepProperty.Value.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Uses}"); + break; + + case PipelineTemplateConstants.With: + ConvertToStepInputs(context, stepProperty.Value, allowExpressions: true); // Validate early if possible + with = stepProperty.Value; + break; + + case PipelineTemplateConstants.WorkingDirectory: + workingDir = stepProperty.Value.AssertScalar($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.WorkingDirectory}"); + break; + + default: + propertyName.AssertUnexpectedValue($"{PipelineTemplateConstants.Steps} item key"); // throws + break; + } + } + + // Fixup the if-condition + var isDefaultScope = String.IsNullOrEmpty(scope?.Value); + ifCondition = ConvertToIfCondition(context, ifToken, false, isDefaultScope); + + if (run != null) + { + var result = new ActionStep + { + ScopeName = scope?.Value, + ContextName = id?.Value, + ContinueOnError = continueOnError, + DisplayNameToken = name, + Condition = ifCondition, + TimeoutInMinutes = timeoutMinutes, + Environment = env, + Reference = new ScriptReference(), + }; + + var inputs = new MappingToken(null, null, null); + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.Script), run); + + if (workingDir != null) + { + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.WorkingDirectory), workingDir); + } + + if (shell != null) + { + inputs.Add(new StringToken(null, null, null, PipelineConstants.ScriptStepInputs.Shell), shell); + } + + result.Inputs = inputs; + + return result; + } + else + { + uses.AssertString($"{PipelineTemplateConstants.Steps} item {PipelineTemplateConstants.Uses}"); + var result = new ActionStep + { + ScopeName = scope?.Value, + ContextName = id?.Value, + ContinueOnError = continueOnError, + DisplayNameToken = name, + Condition = ifCondition, + TimeoutInMinutes = timeoutMinutes, + Inputs = with, + Environment = env, + }; + + if (uses.Value.StartsWith("docker://", StringComparison.Ordinal)) + { + var image = uses.Value.Substring("docker://".Length); + result.Reference = new ContainerRegistryReference { Image = image }; + } + else if (uses.Value.StartsWith("./") || uses.Value.StartsWith(".\\")) + { + result.Reference = new RepositoryPathReference + { + RepositoryType = PipelineConstants.SelfAlias, + Path = uses.Value + }; + } + else + { + var usesSegments = uses.Value.Split('@'); + var pathSegments = usesSegments[0].Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries); + var gitRef = usesSegments.Length == 2 ? usesSegments[1] : String.Empty; + + if (usesSegments.Length != 2 || + pathSegments.Length < 2 || + String.IsNullOrEmpty(pathSegments[0]) || + String.IsNullOrEmpty(pathSegments[1]) || + String.IsNullOrEmpty(gitRef)) + { + // todo: loc + context.Error(uses, $"Expected format {{org}}/{{repo}}[/path]@ref. Actual '{uses.Value}'"); + } + else + { + var repositoryName = $"{pathSegments[0]}/{pathSegments[1]}"; + var directoryPath = pathSegments.Length > 2 ? String.Join("/", pathSegments.Skip(2)) : String.Empty; + + result.Reference = new RepositoryPathReference + { + RepositoryType = RepositoryTypes.GitHub, + Name = repositoryName, + Ref = gitRef, + Path = directoryPath, + }; + } + } + + return result; + } + } + + /// + /// When empty, default to "success()". + /// When a status function is not referenced, format as "success() && <CONDITION>". + /// + private static String ConvertToIfCondition( + TemplateContext context, + TemplateToken token, + Boolean isJob, + Boolean isDefaultScope) + { + String condition; + if (token is null) + { + condition = null; + } + else if (token is BasicExpressionToken expressionToken) + { + condition = expressionToken.Expression; + } + else + { + var stringToken = token.AssertString($"{(isJob ? "job" : "step")} {PipelineTemplateConstants.If}"); + condition = stringToken.Value; + } + + if (String.IsNullOrWhiteSpace(condition)) + { + return $"{PipelineTemplateConstants.Success}()"; + } + + var expressionParser = new ExpressionParser(); + var functions = default(IFunctionInfo[]); + var namedValues = default(INamedValueInfo[]); + if (isJob) + { + namedValues = s_jobIfNamedValues; + // TODO: refactor into seperate functions + // functions = PhaseCondition.FunctionInfo; + } + else + { + namedValues = isDefaultScope ? s_stepNamedValues : s_stepInTemplateNamedValues; + functions = s_stepConditionFunctions; + } + + var node = default(ExpressionNode); + try + { + node = expressionParser.CreateTree(condition, null, namedValues, functions) as ExpressionNode; + } + catch (Exception ex) + { + context.Error(token, ex); + return null; + } + + if (node == null) + { + return $"{PipelineTemplateConstants.Success}()"; + } + + var hasStatusFunction = node.Traverse().Any(x => + { + if (x is Function function) + { + return String.Equals(function.Name, PipelineTemplateConstants.Always, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Cancelled, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Failure, StringComparison.OrdinalIgnoreCase) || + String.Equals(function.Name, PipelineTemplateConstants.Success, StringComparison.OrdinalIgnoreCase); + } + + return false; + }); + + return hasStatusFunction ? condition : $"{PipelineTemplateConstants.Success}() && ({condition})"; + } + + private static readonly INamedValueInfo[] s_jobIfNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly INamedValueInfo[] s_stepNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.Strategy), + new NamedValueInfo(PipelineTemplateConstants.Matrix), + new NamedValueInfo(PipelineTemplateConstants.Steps), + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Job), + new NamedValueInfo(PipelineTemplateConstants.Runner), + new NamedValueInfo(PipelineTemplateConstants.Env), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly INamedValueInfo[] s_stepInTemplateNamedValues = new INamedValueInfo[] + { + new NamedValueInfo(PipelineTemplateConstants.Strategy), + new NamedValueInfo(PipelineTemplateConstants.Matrix), + new NamedValueInfo(PipelineTemplateConstants.Steps), + new NamedValueInfo(PipelineTemplateConstants.Inputs), + new NamedValueInfo(PipelineTemplateConstants.GitHub), + new NamedValueInfo(PipelineTemplateConstants.Job), + new NamedValueInfo(PipelineTemplateConstants.Runner), + new NamedValueInfo(PipelineTemplateConstants.Env), + new NamedValueInfo(PipelineTemplateConstants.Needs), + }; + private static readonly IFunctionInfo[] s_stepConditionFunctions = new IFunctionInfo[] + { + new FunctionInfo(PipelineTemplateConstants.Always, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Cancelled, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Failure, 0, 0), + new FunctionInfo(PipelineTemplateConstants.Success, 0, 0), + new FunctionInfo(PipelineTemplateConstants.HashFiles, 1, Byte.MaxValue), + }; } } diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs index a36f5b7e3aa..55076e670e5 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs @@ -159,6 +159,32 @@ public String EvaluateStepDisplayName( return result; } + public List LoadCompositeSteps( + TemplateToken token + ) + { + var result = default(List); + if (token != null && token.Type != TokenType.Null) + { + var context = CreateContext(null, null, setMissingContext: false); + // TODO: we might want to to have a bool to prevent it from filling in with missing context w/ dummy variables + try + { + token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepsInTemplate, token, 0, null, omitHeader: true); + context.Errors.Check(); + result = PipelineTemplateConverter.ConvertToSteps(context, token); + } + catch (Exception ex) when (!(ex is TemplateValidationException)) + { + context.Errors.Add(ex); + } + + context.Errors.Check(); + } + return result; + } + + public Dictionary EvaluateStepEnvironment( TemplateToken token, DictionaryContextData contextData, @@ -400,7 +426,8 @@ public IList> EvaluateJobServiceContainers( private TemplateContext CreateContext( DictionaryContextData contextData, IList expressionFunctions, - IEnumerable> expressionState = null) + IEnumerable> expressionState = null, + bool setMissingContext = true) { var result = new TemplateContext { @@ -449,18 +476,21 @@ private TemplateContext CreateContext( // - Evaluating early when all referenced contexts are available, even though all allowed // contexts may not yet be available. For example, evaluating step display name can often // be performed early. - foreach (var name in s_expressionValueNames) + if (setMissingContext) { - if (!result.ExpressionValues.ContainsKey(name)) + foreach (var name in s_expressionValueNames) { - result.ExpressionValues[name] = null; + if (!result.ExpressionValues.ContainsKey(name)) + { + result.ExpressionValues[name] = null; + } } - } - foreach (var name in s_expressionFunctionNames) - { - if (!functionNames.Contains(name)) + foreach (var name in s_expressionFunctionNames) { - result.ExpressionFunctions.Add(new FunctionInfo(name, 0, Int32.MaxValue)); + if (!functionNames.Contains(name)) + { + result.ExpressionFunctions.Add(new FunctionInfo(name, 0, Int32.MaxValue)); + } } } diff --git a/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs b/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs index 2d599dd9c55..2e03671fbb2 100644 --- a/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/PipelineConstants.cs @@ -94,5 +94,12 @@ public static class WorkspaceCleanOptions public static readonly String Resources = "resources"; public static readonly String All = "all"; } + + public static class ScriptStepInputs + { + public static readonly String Script = "script"; + public static readonly String WorkingDirectory = "workingDirectory"; + public static readonly String Shell = "shell"; + } } } diff --git a/src/Test/L0/Worker/StepsRunnerL0.cs b/src/Test/L0/Worker/StepsRunnerL0.cs index 2d7cb9fb0c4..1dfee2252ad 100644 --- a/src/Test/L0/Worker/StepsRunnerL0.cs +++ b/src/Test/L0/Worker/StepsRunnerL0.cs @@ -80,7 +80,7 @@ public async Task RunNormalStepsAllStepPass() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -115,7 +115,7 @@ public async Task RunNormalStepsContinueOnError() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -154,7 +154,7 @@ public async Task RunsAfterFailureBasedOnCondition() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Steps.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Steps.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -208,7 +208,7 @@ public async Task RunsAlwaysSteps() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Steps.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Steps.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -287,7 +287,7 @@ public async Task SetsJobResultCorrectly() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Steps.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Steps.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -330,7 +330,7 @@ public async Task SkipsAfterFailureOnlyBaseOnCondition() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Step.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Step.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -361,7 +361,7 @@ public async Task AlwaysMeansAlways() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -391,7 +391,7 @@ public async Task TreatsConditionErrorAsFailure() { _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(variableSet.Select(x => x.Object).ToList())); + _ec.Setup(x => x.JobSteps).Returns(new List(variableSet.Select(x => x.Object).ToList())); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -417,7 +417,7 @@ public async Task StepEnvOverrideJobEnvContext() _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -455,7 +455,7 @@ public async Task PopulateEnvContextForEachStep() _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object, step2.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -493,7 +493,7 @@ public async Task PopulateEnvContextAfterSetupStepsContext() _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object, step2.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -524,7 +524,7 @@ public async Task StepContextOutcome() _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object, step3.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object, step2.Object, step3.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); @@ -560,7 +560,7 @@ public async Task StepContextConclusion() _ec.Object.Result = null; - _ec.Setup(x => x.JobSteps).Returns(new Queue(new[] { step1.Object, step2.Object, step3.Object })); + _ec.Setup(x => x.JobSteps).Returns(new List(new[] { step1.Object, step2.Object, step3.Object })); // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object);