Merge branch 'grant-archibald-ms/variables-collections-377' into gran…

…t-archibald-md/integration-merge

samples/coe-kit-setup-wizard/setFirstStep.te.yaml

@@ -0,0 +1,30 @@
+testSuite:
+ testSuiteName: CoE Starter Kit Setup Wizard
+ testSuiteDescription: Verify custom page of CoE Starter Kit Setup Wizard and step through install
+ persona: User1
+ appLogicalName: NotNeeded
+
+ testCases:
+ - testCaseName: Verify
+ testCaseDescription: Verify setup and upgrade Wizard of the CoE Starter Kit
+ testSteps: |
+ =
+ Set(configStep, 1); 
+ Assert(configStep=1);
+ TestEngine.Pause()
+
+testSettings:
+ headless: false
+ locale: "en-US"
+ recordVideo: true
+ extensionModules:
+ enable: true
+ browserConfigurations:
+ - browser: Chromium
+ timeout: 480000
+
+environmentVariables:
+ users:
+ - personaName: User1
+ emailKey: NotNeeded
+ passwordKey: NotNeeded

src/Microsoft.PowerApps.TestEngine.Tests/PowerFx/PowerFxEngineTests.cs

@@ -151,6 +151,9 @@ public void ExecuteOneFunctionTest()
  {
  MockTestState.Setup(x => x.GetTestSettings()).Returns(new TestSettings());
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
 
  var powerFxEngine = new PowerFxEngine(MockTestInfraFunctions.Object, MockTestWebProvider.Object, MockSingleTestInstanceState.Object, MockTestState.Object, MockFileSystem.Object);
  MockTestState.Setup(x => x.GetTestSettings()).Returns<TestSettings>(null);
@@ -169,6 +172,12 @@ public void ExecuteMultipleFunctionsTest()
  {
  MockTestState.Setup(x => x.GetTestSettings()).Returns(new TestSettings());
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
 
  var powerFxExpression = "1+1; //some comment \n 2+2;\n Concatenate(\"hello\", \"world\");";
  var powerFxEngine = new PowerFxEngine(MockTestInfraFunctions.Object, MockTestWebProvider.Object, MockSingleTestInstanceState.Object, MockTestState.Object, MockFileSystem.Object);
@@ -188,6 +197,9 @@ public void ExecuteMultipleFunctionsWithDifferentLocaleTest()
  {
  MockTestState.Setup(x => x.GetTestSettings()).Returns(new TestSettings());
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
 
  // en-US locale
  var culture = new CultureInfo("en-US");
@@ -249,6 +261,9 @@ public async Task ExecuteWithVariablesTest()
  var testSettings = new TestSettings() { Timeout = 3000 };
  MockTestState.Setup(x => x.GetTestSettings()).Returns(testSettings);
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
 
  var powerFxEngine = new PowerFxEngine(MockTestInfraFunctions.Object, MockTestWebProvider.Object, MockSingleTestInstanceState.Object, MockTestState.Object, MockFileSystem.Object);
  powerFxEngine.Setup();
@@ -274,6 +289,9 @@ public async Task ExecuteFailsWhenPowerFXThrowsTest()
  {
  MockTestState.Setup(x => x.GetTestSettings()).Returns(new TestSettings());
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
 
  var powerFxExpression = "someNonExistentPowerFxFunction(1, 2, 3)";
  MockTestWebProvider.Setup(x => x.LoadObjectModelAsync()).Returns(Task.FromResult(new Dictionary<string, ControlRecordValue>()));
@@ -287,6 +305,9 @@ public async Task ExecuteFailsWhenUsingNonExistentVariableTest()
  {
  MockTestState.Setup(x => x.GetTestSettings()).Returns(new TestSettings());
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
 
  var powerFxExpression = "Concatenate(Label1.Text, Label2.Text)";
  var powerFxEngine = new PowerFxEngine(MockTestInfraFunctions.Object, MockTestWebProvider.Object, MockSingleTestInstanceState.Object, MockTestState.Object, MockFileSystem.Object);
@@ -299,6 +320,9 @@ public void ExecuteAssertFunctionTest()
  {
  MockTestState.Setup(x => x.GetTestSettings()).Returns(new TestSettings());
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
 
  var powerFxExpression = "Assert(1+1=2, \"Adding 1 + 1\")";
  var powerFxEngine = new PowerFxEngine(MockTestInfraFunctions.Object, MockTestWebProvider.Object, MockSingleTestInstanceState.Object, MockTestState.Object, MockFileSystem.Object);
@@ -315,6 +339,9 @@ public async Task ExecuteScreenshotFunctionTest()
  {
  MockTestState.Setup(x => x.GetTestSettings()).Returns(new TestSettings());
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
 
  MockSingleTestInstanceState.Setup(x => x.GetTestResultsDirectory()).Returns("C:\\testResults");
  MockFileSystem.Setup(x => x.IsValidFilePath(It.IsAny<string>())).Returns(true);
@@ -342,6 +369,9 @@ public async Task ExecuteSelectFunctionTest()
  var testSettings = new TestSettings() { Timeout = 3000 };
  MockTestState.Setup(x => x.GetTestSettings()).Returns(testSettings);
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
 
  var powerFxExpression = "Select(Button1)";
  var powerFxEngine = new PowerFxEngine(MockTestInfraFunctions.Object, MockTestWebProvider.Object, MockSingleTestInstanceState.Object, MockTestState.Object, MockFileSystem.Object);
@@ -412,6 +442,9 @@ public async Task ExecuteSetPropertyFunctionTest()
  var testSettings = new TestSettings() { Timeout = 3000 };
  MockTestState.Setup(x => x.GetTestSettings()).Returns(testSettings);
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
 
  var powerFxExpression = "SetProperty(Button1.Text, \"10\")";
  var powerFxEngine = new PowerFxEngine(MockTestInfraFunctions.Object, MockTestWebProvider.Object, MockSingleTestInstanceState.Object, MockTestState.Object, MockFileSystem.Object);
@@ -472,6 +505,9 @@ public async Task ExecuteWaitFunctionTest()
  var testSettings = new TestSettings() { Timeout = 3000 };
  MockTestState.Setup(x => x.GetTestSettings()).Returns(testSettings);
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
 
  var powerFxExpression = "Wait(Label1, \"Text\", \"1\")";
  var powerFxEngine = new PowerFxEngine(MockTestInfraFunctions.Object, MockTestWebProvider.Object, MockSingleTestInstanceState.Object, MockTestState.Object, MockFileSystem.Object);
@@ -576,6 +612,9 @@ private PowerFxEngine GetPowerFxEngine()
  public async Task ExecuteFooFromModuleFunction()
  {
  var testSettings = new TestSettings() { ExtensionModules = new TestSettingExtensions { Enable = true } };
+ MockTestState.SetupGet(x => x.ExecuteStepByStep).Returns(false);
+ MockTestState.Setup(x => x.OnBeforeTestStepExecuted(It.IsAny<TestStepEventArgs>()));
+ MockTestState.Setup(x => x.OnAfterTestStepExecuted(It.IsAny<TestStepEventArgs>()));
 
  var mockModule = new Mock<ITestEngineModule>();
  var modules = new List<ITestEngineModule>() { mockModule.Object };

src/Microsoft.PowerApps.TestEngine/Config/ITestState.cs

@@ -146,5 +146,34 @@ public interface ITestState
  public List<ITestWebProvider> GetTestEngineWebProviders();
 
  public List<IUserCertificateProvider> GetTestEngineAuthProviders();
+
+ /// <summary>
+ /// Determine if the steps of the test steps should be executed step by step or as one action
+ /// </summary>
+ public bool ExecuteStepByStep { get; set; }
+
+ /// <summary>
+ /// Event triggered before a test step is executed
+ /// </summary>
+ event EventHandler<TestStepEventArgs> BeforeTestStepExecuted;
+
+ /// <summary>
+ /// Event triggered after a test step is executed
+ /// </summary>
+ event EventHandler<TestStepEventArgs> AfterTestStepExecuted;
+
+ /// <summary>
+ /// This method is called before a test step is executed.
+ /// It allows for any necessary setup or logging before the test step runs.
+ /// </summary>
+ /// <param name="e">The event arguments containing details about the test step.</param>
+ public void OnBeforeTestStepExecuted(TestStepEventArgs e);
+
+ /// <summary>
+ /// This method is called after a test step is executed.
+ /// It allows for any necessary cleanup or logging after the test step runs.
+ /// </summary>
+ /// <param name="e">The event arguments containing details about the test step.</param>
+ public void OnAfterTestStepExecuted(TestStepEventArgs e);
  }
 }

src/Microsoft.PowerApps.TestEngine/Config/TestState.cs

@@ -19,6 +19,9 @@ public class TestState : ITestState
  {
  private readonly ITestConfigParser _testConfigParser;
 
+ public event EventHandler<TestStepEventArgs> BeforeTestStepExecuted;
+ public event EventHandler<TestStepEventArgs> AfterTestStepExecuted;
+
  private TestPlanDefinition TestPlanDefinition { get; set; }
  private List<TestCase> TestCases { get; set; } = new List<TestCase>();
  private string EnvironmentId { get; set; }
@@ -42,6 +45,9 @@ public class TestState : ITestState
 
  private bool IsValid { get; set; } = false;
 
+ // Determine if Power FX expressions delimited by ; should be executed step by step
+ public bool ExecuteStepByStep { get; set; } = false;
+
  public TestState(ITestConfigParser testConfigParser)
  {
  _testConfigParser = testConfigParser;
@@ -367,5 +373,15 @@ public List<IUserCertificateProvider> GetTestEngineAuthProviders()
  {
  return CertificateProviders;
  }
+
+ public void OnBeforeTestStepExecuted(TestStepEventArgs e)
+ {
+ BeforeTestStepExecuted?.Invoke(this, e);
+ }
+
+ public void OnAfterTestStepExecuted(TestStepEventArgs e)
+ {
+ AfterTestStepExecuted?.Invoke(this, e);
+ }
  }
 }

src/Microsoft.PowerApps.TestEngine/Config/TestStepEventArgs.cs

@@ -0,0 +1,31 @@
+using Microsoft.PowerFx;
+using Microsoft.PowerFx.Types;
+
+namespace Microsoft.PowerApps.TestEngine.Config
+{
+ /// <summary>
+ /// Represents the event arguments for a test step event.
+ /// </summary>
+ public class TestStepEventArgs : EventArgs
+ {
+ /// <summary>
+ /// Gets or sets the name of the test step.
+ /// </summary>
+ public string TestStep { get; set; }
+
+ /// <summary>
+ /// Gets or sets the result of the test step.
+ /// </summary>
+ public FormulaValue Result { get; set; }
+
+ /// <summary>
+ /// Gets or sets the step number of the test step.
+ /// </summary>
+ public int? StepNumber { get; set; }
+
+ /// <summary>
+ /// Gets or sets the recalculation engine used for the test step.
+ /// </summary>
+ public RecalcEngine Engine { get; set; }
+ }
+}

src/Microsoft.PowerApps.TestEngine/PowerFx/PowerFxEngine.cs

@@ -51,9 +51,8 @@ public void Setup()
  symbols.EnableMutationFunctions();
  powerFxConfig.SymbolTable = symbols;
 
- //TODO: Remove
+ // Enabled to allow ability to set variable and collection state that can be used with providers and as test variables
  powerFxConfig.EnableSetFunction();
- //TODO: End
 
  powerFxConfig.AddFunction(new SelectOneParamFunction(_testWebProvider, async () => await UpdatePowerFxModelAsync(), Logger));
  powerFxConfig.AddFunction(new SelectTwoParamsFunction(_testWebProvider, async () => await UpdatePowerFxModelAsync(), Logger));
@@ -131,7 +130,7 @@ public FormulaValue Execute(string testSteps, CultureInfo culture)
  testSteps = testSteps.Remove(0, 1);
  }
 
- var goStepByStep = false;
+ var goStepByStep = TestState.ExecuteStepByStep;
 
  // Check if the syntax is correct
  var checkResult = Engine.Check(testSteps, null, GetPowerFxParserOptions(culture));
@@ -147,18 +146,28 @@ public FormulaValue Execute(string testSteps, CultureInfo culture)
  var splitSteps = PowerFxHelper.ExtractFormulasSeparatedByChainingOperator(Engine, checkResult, culture);
  FormulaValue result = FormulaValue.NewBlank();
 
+ int stepNumber = 0;
+
  foreach (var step in splitSteps)
  {
+ TestState.OnBeforeTestStepExecuted(new TestStepEventArgs { TestStep = step, StepNumber = stepNumber, Engine = Engine });
+
  Logger.LogTrace($"Attempting:{step.Replace("\n", "").Replace("\r", "")}");
  result = Engine.Eval(step, null, new ParserOptions() { AllowsSideEffects = true, Culture = culture, NumberIsFloat = true });
+
+ TestState.OnAfterTestStepExecuted(new TestStepEventArgs { TestStep = step, Result = result, StepNumber = stepNumber, Engine = Engine });
+ stepNumber++;
  }
  return result;
  }
  else
  {
  var values = new SymbolValues();
  Logger.LogTrace($"Attempting:\n\n{{\n{testSteps}}}");
- return Engine.Eval(testSteps, null, new ParserOptions() { AllowsSideEffects = true, Culture = culture, NumberIsFloat = true });
+ TestState.OnBeforeTestStepExecuted(new TestStepEventArgs { TestStep = testSteps, StepNumber = null, Engine = Engine });
+ var result = Engine.Eval(testSteps, null, new ParserOptions() { AllowsSideEffects = true, Culture = culture, NumberIsFloat = true });
+ TestState.OnAfterTestStepExecuted(new TestStepEventArgs { TestStep = testSteps, Result = result, StepNumber = 1 });
+ return result;
  }
  }