Merge branch 'integration' into grant-archibald-ms/docs

samples/buttonclicker/README.md

@@ -0,0 +1,20 @@
+# Overview
+
+This Power Apps Test Engine sample demonstrates how to clicking button of a canvas application
+
+## Usage
+
+1. Build the Test Engine solution
+
+2. Get the Environment Id and Tenant of the environment that the samplication solution has been imported into
+
+3. Execute the test for custom page changing the example below to the url of your organization using browser persistant cookies
+
+```pwsh
+cd bin\Debug\PowerAppsEngine
+dotnet PowerAppsTestEngine.dll -i ..\..\..\samples\buttonclicker\testPlan.fx.yaml -e 00000000-0000-0000-0000-11112223333 -t 11112222-3333-4444-5555-666677778888 -u browser -p canvas
+```
+
+NOTES:
+- If the BrowserCache folder does not exist with valid Persistent Session cookies an interactive login will be required
+- After an interactive login has been completed a headless test session can be run

samples/mda/testPlan.fx.yaml

@@ -18,6 +18,8 @@ testSettings:
  enable: true
  browserConfigurations:
  - browser: Chromium
+ channel: msedge
+ timeout: 600000
 
 environmentVariables:
  users:

samples/testSettings.yaml

@@ -1,6 +1,8 @@
 locale: "en-US"
+headless: false
 recordVideo: true
 extensionModules:
  enable: true
 browserConfigurations:
  - browser: Chromium
+ channel: msedge

src/Microsoft.PowerApps.TestEngine.Tests/Config/SingleTestInstanceStateTests.cs

@@ -44,7 +44,7 @@ public void SingleTestInstanceStateSuccessTest()
  [Theory]
  [InlineData("")]
  [InlineData(null)]
- public void SetTestRunIdThrowsOnInvalidInputTest(string invalidInput)
+ public void SetTestRunIdThrowsOnInvalidInputTest(string? invalidInput)
  {
  var state = new SingleTestInstanceState();
  Assert.Throws<ArgumentNullException>(() => state.SetTestRunId(invalidInput));
@@ -53,7 +53,7 @@ public void SetTestRunIdThrowsOnInvalidInputTest(string invalidInput)
  [Theory]
  [InlineData("")]
  [InlineData(null)]
- public void SetTestIdThrowsOnInvalidInputTest(string invalidInput)
+ public void SetTestIdThrowsOnInvalidInputTest(string? invalidInput)
  {
  var state = new SingleTestInstanceState();
  Assert.Throws<ArgumentNullException>(() => state.SetTestId(invalidInput));
@@ -62,7 +62,7 @@ public void SetTestIdThrowsOnInvalidInputTest(string invalidInput)
  [Theory]
  [InlineData("")]
  [InlineData(null)]
- public void SetTestResultsDirectoryThrowsOnInvalidInputTest(string invalidInput)
+ public void SetTestResultsDirectoryThrowsOnInvalidInputTest(string? invalidInput)
  {
  var state = new SingleTestInstanceState();
  Assert.Throws<ArgumentNullException>(() => state.SetTestResultsDirectory(invalidInput));

src/Microsoft.PowerApps.TestEngine.Tests/Config/TestStateTests.cs

@@ -189,7 +189,7 @@ public void TestStateSuccessOnFilePathTest()
  [Theory]
  [InlineData(null)]
  [InlineData("")]
- public void ParseAndSetTestStateThrowsOnInvalidTestConfigFile(string testConfigPath)
+ public void ParseAndSetTestStateThrowsOnInvalidTestConfigFile(string? testConfigPath)
  {
  var state = new TestState(MockTestConfigParser.Object);
  MockLoggerFactory.Setup(x => x.CreateLogger(It.IsAny<string>())).Returns(MockLogger.Object);
@@ -218,7 +218,7 @@ public void ParseAndSetTestStateThrowsOnNoTestSuiteDefinition()
  [Theory]
  [InlineData(null)]
  [InlineData("")]
- public void ParseAndSetTestStateThrowsOnNoNameInTestSuiteDefinition(string testName)
+ public void ParseAndSetTestStateThrowsOnNoNameInTestSuiteDefinition(string? testName)
  {
  var state = new TestState(MockTestConfigParser.Object);
  var testConfigFile = "testPlan.fx.yaml";
@@ -237,7 +237,7 @@ public void ParseAndSetTestStateThrowsOnNoNameInTestSuiteDefinition(string testN
  [Theory]
  [InlineData(null)]
  [InlineData("")]
- public void ParseAndSetTestStateThrowsOnNoPersonaInTestSuiteDefinition(string persona)
+ public void ParseAndSetTestStateThrowsOnNoPersonaInTestSuiteDefinition(string? persona)
  {
  var state = new TestState(MockTestConfigParser.Object);
  var testConfigFile = "testPlan.fx.yaml";
@@ -258,7 +258,7 @@ public void ParseAndSetTestStateThrowsOnNoPersonaInTestSuiteDefinition(string pe
  [InlineData("", "")]
  [InlineData(null, "")]
  [InlineData("", null)]
- public void ParseAndSetTestStateThrowsOnNoAppLogicalNameOrAppIdInTestSuiteDefinition(string appLogicalName, string appId)
+ public void ParseAndSetTestStateThrowsOnNoAppLogicalNameOrAppIdInTestSuiteDefinition(string? appLogicalName, string? appId)
  {
  var state = new TestState(MockTestConfigParser.Object);
  var testConfigFile = "testPlan.fx.yaml";
@@ -278,7 +278,7 @@ public void ParseAndSetTestStateThrowsOnNoAppLogicalNameOrAppIdInTestSuiteDefini
  [Theory]
  [InlineData("appLogicalName", null)]
  [InlineData(null, "appId")]
- public void ParseAndSetTestStateDoesNotThrowWhenEitherOfAppLogicalNameOrAppIdInTestSuiteDefinition(string appLogicalName, string appId)
+ public void ParseAndSetTestStateDoesNotThrowWhenEitherOfAppLogicalNameOrAppIdInTestSuiteDefinition(string? appLogicalName, string? appId)
  {
  var state = new TestState(MockTestConfigParser.Object);
  var testConfigFile = "testPlan.fx.yaml";
@@ -314,7 +314,7 @@ public void ParseAndSetTestStateThrowsOnNoTestCaseInTestSuiteDefinition()
  [Theory]
  [InlineData(null)]
  [InlineData("")]
- public void ParseAndSetTestStateThrowsOnNoTestCaseNameInTestCase(string testCaseName)
+ public void ParseAndSetTestStateThrowsOnNoTestCaseNameInTestCase(string? testCaseName)
  {
  var state = new TestState(MockTestConfigParser.Object);
  var testConfigFile = "testPlan.fx.yaml";
@@ -333,7 +333,7 @@ public void ParseAndSetTestStateThrowsOnNoTestCaseNameInTestCase(string testCase
  [Theory]
  [InlineData(null)]
  [InlineData("")]
- public void ParseAndSetTestStateThrowsOnNoTestStepsInTestCase(string testSteps)
+ public void ParseAndSetTestStateThrowsOnNoTestStepsInTestCase(string? testSteps)
  {
  var state = new TestState(MockTestConfigParser.Object);
  var testConfigFile = "testPlan.fx.yaml";
@@ -403,7 +403,7 @@ public void ParseAndSetTestStateThrowsOnNoBrowserConfigurationInTestSettings()
  [Theory]
  [InlineData(null)]
  [InlineData("")]
- public void ParseAndSetTestStateThrowsOnNoBrowserInBrowserConfigurationInTestSettings(string browser)
+ public void ParseAndSetTestStateThrowsOnNoBrowserInBrowserConfigurationInTestSettings(string? browser)
  {
  var state = new TestState(MockTestConfigParser.Object);
  var testConfigFile = "testPlan.fx.yaml";
@@ -497,7 +497,7 @@ public void ParseAndSetTestStateThrowsOnNoUserConfigurationInEnvironmentVariable
  [Theory]
  [InlineData("")]
  [InlineData(null)]
- public void ParseAndSetTestStateThrowsOnNoPersonaNameInUserConfigurationInEnvironmentVariables(string personaName)
+ public void ParseAndSetTestStateThrowsOnNoPersonaNameInUserConfigurationInEnvironmentVariables(string? personaName)
  {
  var state = new TestState(MockTestConfigParser.Object);
  var testConfigFile = "testPlan.fx.yaml";
@@ -516,7 +516,7 @@ public void ParseAndSetTestStateThrowsOnNoPersonaNameInUserConfigurationInEnviro
  [Theory]
  [InlineData("")]
  [InlineData(null)]
- public void ParseAndSetTestStateThrowsOnNoEmailKeyInUserConfigurationInEnvironmentVariables(string emailKey)
+ public void ParseAndSetTestStateThrowsOnNoEmailKeyInUserConfigurationInEnvironmentVariables(string? emailKey)
  {
  var state = new TestState(MockTestConfigParser.Object);
  var testConfigFile = "testPlan.fx.yaml";
@@ -574,7 +574,7 @@ public void ParseAndSetTestStateThrowsOnMulitpleMissingValues()
  [Theory]
  [InlineData("")]
  [InlineData(null)]
- public void SetEnvironmentThrowsOnNullInput(string environment)
+ public void SetEnvironmentThrowsOnNullInput(string? environment)
  {
  var state = new TestState(MockTestConfigParser.Object);
  Assert.Throws<ArgumentNullException>(() => state.SetEnvironment(environment));
@@ -583,7 +583,7 @@ public void SetEnvironmentThrowsOnNullInput(string environment)
  [Theory]
  [InlineData("")]
  [InlineData(null)]
- public void SetDomainThrowsOnNullInput(string domain)
+ public void SetDomainThrowsOnNullInput(string? domain)
  {
  var state = new TestState(MockTestConfigParser.Object);
  Assert.Throws<ArgumentNullException>(() => state.SetDomain(domain));
@@ -592,7 +592,7 @@ public void SetDomainThrowsOnNullInput(string domain)
  [Theory]
  [InlineData("")]
  [InlineData(null)]
- public void SetTenantThrowsOnNullInput(string tenant)
+ public void SetTenantThrowsOnNullInput(string? tenant)
  {
  var state = new TestState(MockTestConfigParser.Object);
  Assert.Throws<ArgumentNullException>(() => state.SetTenant(tenant));
@@ -601,7 +601,7 @@ public void SetTenantThrowsOnNullInput(string tenant)
  [Theory]
  [InlineData("")]
  [InlineData(null)]
- public void SetOutputDirectoryThrowsOnNullInput(string outputDirectory)
+ public void SetOutputDirectoryThrowsOnNullInput(string? outputDirectory)
  {
  var state = new TestState(MockTestConfigParser.Object);
  Assert.Throws<ArgumentNullException>(() => state.SetOutputDirectory(outputDirectory));

src/Microsoft.PowerApps.TestEngine.Tests/Config/YamlTestConfigParserTests.cs

@@ -360,7 +360,7 @@ public void YamlTestConfigParserParseTestSettingsWithLocaleTest()
  [Theory]
  [InlineData("")]
  [InlineData(null)]
- public void YamlTestConfigParserThrowsOnNullArguments(string filePath)
+ public void YamlTestConfigParserThrowsOnNullArguments(string? filePath)
  {
  var mockFileSystem = new Mock<IFileSystem>(MockBehavior.Strict);
  var parser = new YamlTestConfigParser(mockFileSystem.Object);

src/Microsoft.PowerApps.TestEngine.Tests/Helpers/PollingHelpersTest.cs

@@ -58,10 +58,10 @@ public async Task PollingAsyncSucceedsTest()
  }
 
  [Fact]
- public void PollingAsyncTimeoutTest()
+ public async Task PollingAsyncTimeoutTest()
  {
  LoggingTestHelper.SetupMock(MockLogger);
- Assert.ThrowsAsync<TimeoutException>(() => PollingHelper.PollAsync(false, conditionToCheck, () => functionToCallAsync(), _notEnoughRuntime, MockLogger.Object));
+ await Assert.ThrowsAsync<TimeoutException>(() => PollingHelper.PollAsync(false, conditionToCheck, () => functionToCallAsync(), _notEnoughRuntime, MockLogger.Object));
  }
 
  [Fact]
@@ -72,10 +72,10 @@ public void PollingThrowsOnInvalidArgumentsTest()
  }
 
  [Fact]
- public void PollingAsyncThrowsOnInvalidArgumentsTest()
+ public async Task PollingAsyncThrowsOnInvalidArgumentsTest()
  {
  LoggingTestHelper.SetupMock(MockLogger);
- Assert.ThrowsAsync<ArgumentOutOfRangeException>(() => PollingHelper.PollAsync(false, conditionToCheck, () => functionToCallAsync(), _invalidRuntime, MockLogger.Object));
+ await Assert.ThrowsAsync<ArgumentOutOfRangeException>(() => PollingHelper.PollAsync(false, conditionToCheck, () => functionToCallAsync(), _invalidRuntime, MockLogger.Object));
  }
 
  }

src/Microsoft.PowerApps.TestEngine.Tests/Microsoft.PowerApps.TestEngine.Tests.csproj

@@ -11,17 +11,20 @@
  </PropertyGroup>
 
  <ItemGroup>
- <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="4.5.0" />
- <PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.1" />
- <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
- <PackageReference Include="Moq" Version="4.17.2" />
+ <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
+ <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Scripting" Version="4.11.0" />
+ <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
+ <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+ <PackageReference Include="Moq" Version="4.20.72" />
+ <PackageReference Include="System.Formats.Asn1" Version="8.0.1" />
  <PackageReference Include="System.Security.Cryptography.Pkcs" Version="8.0.0" />
- <PackageReference Include="xunit" Version="2.4.1" />
- <PackageReference Include="xunit.runner.visualstudio" Version="2.4.2">
+ <PackageReference Include="System.Text.Json" Version="8.0.4" />
+ <PackageReference Include="xunit" Version="2.9.2" />
+ <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
  <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
  <PrivateAssets>all</PrivateAssets>
  </PackageReference>
- <PackageReference Include="coverlet.collector" Version="3.1.0">
+ <PackageReference Include="coverlet.collector" Version="6.0.2">
  <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
  <PrivateAssets>all</PrivateAssets>
  </PackageReference>

src/Microsoft.PowerApps.TestEngine.Tests/Modules/TestEngineModuleMEFLoaderTests.cs

@@ -28,7 +28,7 @@ public void DisabledEmptyCatalog()
  var catalog = loader.LoadModules(setting);
 
  Assert.NotNull(catalog);
- Assert.Equal(0, catalog.Catalogs.Count);
+ Assert.Empty(catalog.Catalogs);
  }
 
  [Theory]

src/Microsoft.PowerApps.TestEngine.Tests/PowerFx/PowerFxEngineTests.cs

@@ -66,15 +66,15 @@ public void ExecuteThrowsOnNoSetupTest()
  }
 
  [Fact]
- public void UpdatePowerFxModelAsyncThrowsOnNoSetupTest()
+ public async Task UpdatePowerFxModelAsyncThrowsOnNoSetupTest()
  {
  var powerFxEngine = new PowerFxEngine(MockTestInfraFunctions.Object, MockTestWebProvider.Object, MockSingleTestInstanceState.Object, MockTestState.Object, MockFileSystem.Object);
- Assert.ThrowsAsync<InvalidOperationException>(() => powerFxEngine.UpdatePowerFxModelAsync());
+ await Assert.ThrowsAsync<InvalidOperationException>(() => powerFxEngine.UpdatePowerFxModelAsync());
  LoggingTestHelper.VerifyLogging(MockLogger, "Engine is null, make sure to call Setup first", LogLevel.Error, Times.Once());
  }
 
  [Fact]
- public async void UpdatePowerFxModelAsyncThrowsOnCantGetAppStatusTest()
+ public async Task UpdatePowerFxModelAsyncThrowsOnCantGetAppStatusTest()
  {
  var recordType = RecordType.Empty().Add("Text", FormulaType.String);
  var button1 = new ControlRecordValue(recordType, MockTestWebProvider.Object, "Button1");
@@ -93,7 +93,7 @@ public async void UpdatePowerFxModelAsyncThrowsOnCantGetAppStatusTest()
  }
 
  [Fact]
- public async void RunRequirementsCheckAsyncTest()
+ public async Task RunRequirementsCheckAsyncTest()
  {
  MockTestState.Setup(x => x.GetTestSettings()).Returns(new TestSettings());
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
@@ -111,7 +111,7 @@ public async void RunRequirementsCheckAsyncTest()
  }
 
  [Fact]
- public async void RunRequirementsCheckAsyncThrowsOnCheckAndHandleIfLegacyPlayerTest()
+ public async Task RunRequirementsCheckAsyncThrowsOnCheckAndHandleIfLegacyPlayerTest()
  {
  MockTestState.Setup(x => x.GetTestSettings()).Returns(new TestSettings());
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
@@ -129,7 +129,7 @@ public async void RunRequirementsCheckAsyncThrowsOnCheckAndHandleIfLegacyPlayerT
  }
 
  [Fact]
- public async void RunRequirementsCheckAsyncThrowsOnTestEngineReadyTest()
+ public async Task RunRequirementsCheckAsyncThrowsOnTestEngineReadyTest()
  {
  MockTestState.Setup(x => x.GetTestSettings()).Returns(new TestSettings());
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
@@ -270,7 +270,7 @@ public async Task ExecuteWithVariablesTest()
  }
 
  [Fact]
- public void ExecuteFailsWhenPowerFXThrowsTest()
+ public async Task ExecuteFailsWhenPowerFXThrowsTest()
  {
  MockTestState.Setup(x => x.GetTestSettings()).Returns(new TestSettings());
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
@@ -279,19 +279,19 @@ public void ExecuteFailsWhenPowerFXThrowsTest()
  MockTestWebProvider.Setup(x => x.LoadObjectModelAsync()).Returns(Task.FromResult(new Dictionary<string, ControlRecordValue>()));
  var powerFxEngine = new PowerFxEngine(MockTestInfraFunctions.Object, MockTestWebProvider.Object, MockSingleTestInstanceState.Object, MockTestState.Object, MockFileSystem.Object);
  powerFxEngine.Setup();
- Assert.ThrowsAsync<Exception>(async () => await powerFxEngine.ExecuteWithRetryAsync(powerFxExpression, It.IsAny<CultureInfo>()));
+ await Assert.ThrowsAsync<AggregateException>(async () => await powerFxEngine.ExecuteWithRetryAsync(powerFxExpression, It.IsAny<CultureInfo>()));
  }
 
  [Fact]
- public void ExecuteFailsWhenUsingNonExistentVariableTest()
+ public async Task ExecuteFailsWhenUsingNonExistentVariableTest()
  {
  MockTestState.Setup(x => x.GetTestSettings()).Returns(new TestSettings());
  MockTestState.Setup(x => x.GetTestEngineModules()).Returns(new List<ITestEngineModule>());
 
  var powerFxExpression = "Concatenate(Label1.Text, Label2.Text)";
  var powerFxEngine = new PowerFxEngine(MockTestInfraFunctions.Object, MockTestWebProvider.Object, MockSingleTestInstanceState.Object, MockTestState.Object, MockFileSystem.Object);
  powerFxEngine.Setup();
- Assert.ThrowsAsync<Exception>(async () => await powerFxEngine.ExecuteWithRetryAsync(powerFxExpression, It.IsAny<CultureInfo>()));
+ await Assert.ThrowsAsync<AggregateException>(async () => await powerFxEngine.ExecuteWithRetryAsync(powerFxExpression, It.IsAny<CultureInfo>()));
  }
 
  [Fact]
@@ -505,7 +505,7 @@ public async Task ExecuteWaitFunctionThrowsOnDifferentRecordTypeTest()
  MockTestWebProvider.Verify(x => x.LoadObjectModelAsync(), Times.Once());
  }
 
- public async Task TestStepByStep()
+ private async Task TestStepByStep()
  {
  // Arrange
  var powerFxEngine = GetPowerFxEngine();

src/Microsoft.PowerApps.TestEngine.Tests/Reporting/TestReporterTests.cs

@@ -31,7 +31,7 @@ public TestReporterTests()
  [InlineData(null)]
  [InlineData("")]
  [InlineData("nonexistentid")]
- public void ThrowsOnInvalidTestRunIdTest(string testRunId)
+ public void ThrowsOnInvalidTestRunIdTest(string? testRunId)
  {
  var testReporter = new TestReporter(MockFileSystem.Object);
  Assert.Throws<ArgumentException>(() => testReporter.GetTestRun(testRunId));
@@ -290,7 +290,7 @@ public void StartTestTest()
  [InlineData(true, "some logs", new string[] { "file1.txt", "file2.txt", "file3.txt" }, null)]
  [InlineData(false, "some logs", new string[] { }, null)]
  [InlineData(true, "some logs", new string[] { }, "error message")]
- public void EndTestTest(bool success, string stdout, string[] additionalFiles, string errorMessage)
+ public void EndTestTest(bool success, string stdout, string[] additionalFiles, string? errorMessage)
  {
  var testRunName = "testRunName";
  var testUser = "testUser";

src/Microsoft.PowerApps.TestEngine.Tests/SingleTestRunnerTests.cs

@@ -318,7 +318,7 @@ public async Task SingleTestRunnerPowerFxTestFail()
  VerifyFinallyExecution(testData.testResultDirectory, 1, 0, 1);
  }
 
- public async Task SingleTestRunnerHandlesExceptionsThrownCorrectlyHelper(Action<Exception> additionalMockSetup)
+ private async Task SingleTestRunnerHandlesExceptionsThrownCorrectlyHelper(Action<Exception> additionalMockSetup)
  {
  var singleTestRunner = new SingleTestRunner(MockTestReporter.Object,
  MockPowerFxEngine.Object,