test: fix NUnit Console.Error and friends to appear in test output (#…

…3027)

.github/workflows/tests.yml

@@ -19,7 +19,7 @@ jobs:
  fail-fast: false
  matrix:
  browser: [chromium, firefox, webkit]
- os: [windows-latest, ubuntu-latest, macos-13]
+ os: [windows-latest, ubuntu-latest, macos-latest]
  steps:
  - uses: actions/checkout@v3
  - name: Setup .NET Core

src/Playwright.Tests.TestServer/Playwright.Tests.TestServer.csproj

@@ -16,6 +16,9 @@
  <ItemGroup>
  <Folder Include="assets\" />
  </ItemGroup>
+ <ItemGroup>
+ <PackageReference Include="NUnit" Version="4.2.2" />
+ </ItemGroup>
  <Target Name="CheckAssetsFolderExists" BeforeTargets="Build">
  <Error Text="assets folder prerequisites are missing. Ensure you've ran `.\build.sh --init` from the root of the solution." Condition="!Exists('$(MSBuildProjectDirectory)\assets\empty.html')" />
  </Target>

src/Playwright.Tests.TestServer/SimpleServer.cs

@@ -30,6 +30,7 @@
 using System.IO.Compression;
 using System.Net;
 using System.Net.WebSockets;
+using System.Reflection;
 using System.Security.Cryptography.X509Certificates;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -41,6 +42,8 @@
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.StaticFiles;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using NUnit.Framework.Internal;
 
 namespace Microsoft.Playwright.Tests.TestServer;
 
@@ -80,6 +83,7 @@ public SimpleServer(int port, string contentRoot, bool isHttps)
 
  EmptyPage = $"{Prefix}/empty.html";
 
+ var currentExecutionContext = TestExecutionContext.CurrentContext;
  _requestWaits = new ConcurrentDictionary<string, Action<HttpContext>>();
  _waitForWebSocketConnectionRequestsWaits = [];
  _routes = new ConcurrentDictionary<string, Func<HttpContext, Task>>();
@@ -89,11 +93,21 @@ public SimpleServer(int port, string contentRoot, bool isHttps)
  _contentRoot = contentRoot;
 
  _webHost = new WebHostBuilder()
+ .ConfigureLogging(logging =>
+ {
+ // Allow seeing exceptions in the console output.
+ logging.AddConsole();
+ logging.SetMinimumLevel(LogLevel.Error);
+ })
  .Configure((app) => app
  .UseWebSockets()
- .UseDeveloperExceptionPage()
  .Use(middleware: async (HttpContext context, Func<Task> next) =>
  {
+ {
+ // This hack allows us to have Console.WriteLine etc. appear in the test output.
+ var currentContext = typeof(TestExecutionContext).GetField("AsyncLocalCurrentContext", BindingFlags.NonPublic | BindingFlags.Static).GetValue(null) as AsyncLocal<TestExecutionContext>;
+ currentContext.Value = currentExecutionContext;
+ }
  if (context.Request.Path == "/ws")
  {
  if (context.WebSockets.IsWebSocketRequest)
@@ -236,7 +250,7 @@ private string GetContentType(string fileName)
 
  public void SetCSP(string path, string csp) => _csp.Add(path, csp);
 
- public Task StartAsync() => _webHost.StartAsync();
+ public Task StartAsync(CancellationToken cancellationToken) => _webHost.StartAsync(cancellationToken);
 
  public Task StopAsync()
  {

src/Playwright.Tests/BaseTests/HttpService.cs

@@ -42,7 +42,7 @@ public static Task<HttpService> Register(WorkerAwareTest test)
  Server = SimpleServer.Create(8907 + workerIndex * 2, assetDir),
  HttpsServer = SimpleServer.CreateHttps(8907 + workerIndex * 2 + 1, assetDir)
  };
- await Task.WhenAll(http.Server.StartAsync(), http.HttpsServer.StartAsync());
+ await Task.WhenAll(http.Server.StartAsync(TestContext.CurrentContext.CancellationToken), http.HttpsServer.StartAsync(TestContext.CurrentContext.CancellationToken));
  return http;
  });
  }

src/Playwright.Tests/BaseTests/PlaywrightAssert.cs

@@ -23,7 +23,6 @@
  */
 
 using System.Text.Json;
-using NUnit.Framework.Constraints;
 using SixLabors.ImageSharp;
 using SixLabors.ImageSharp.PixelFormats;
 

src/Playwright.Tests/BrowserContextViewportMobileTests.cs

@@ -70,8 +70,7 @@ public async Task ShouldBeDetectableByModernizr()
  {
  await using var context = await Browser.NewContextAsync(Playwright.Devices["iPhone 6"]);
  var page = await context.NewPageAsync();
- await page.GotoAsync(Server.Prefix + "/detect-touch.html");
- Assert.AreEqual("YES", await page.EvaluateAsync<string>("document.body.textContent.trim()"));
+ Assert.AreEqual(true, await page.EvaluateAsync<bool>("'ontouchstart' in window || !!window.TouchEvent"));
  }
 
  [PlaywrightTest("browsercontext-viewport-mobile.spec.ts", "should detect touch when applying viewport with touches")]

src/Playwright.Tests/BrowserContextViewportTests.cs

@@ -86,8 +86,7 @@ public async Task ShouldNotHaveTouchByDefault()
  {
  await Page.GotoAsync(Server.Prefix + "/mobile.html");
  Assert.False(await Page.EvaluateAsync<bool>("'ontouchstart' in window"));
- await Page.GotoAsync(Server.Prefix + "/detect-touch.html");
- Assert.AreEqual("NO", await Page.EvaluateAsync<string>("document.body.textContent.trim()"));
+ Assert.AreEqual(false, await Page.EvaluateAsync<bool>("'ontouchstart' in window"));
  }
 
  [PlaywrightTest("browsercontext-viewport.spec.ts", "should support touch with null viewport")]

src/Playwright.Tests/BrowserTypeConnectTests.cs

@@ -453,7 +453,7 @@ public async Task ShouldRecordContextTraceWithNoDirectoryName()
  }
 
  [PlaywrightTest("browsertype-connect.spec.ts", "should upload large file")]
- [Timeout(TestConstants.SlowTestTimeout)]
+ [CancelAfter(TestConstants.SlowTestTimeout)]
  public async Task ShouldUploadLargeFile()
  {
  var browser = await BrowserType.ConnectAsync(_remoteServer.WSEndpoint);

src/Playwright.Tests/GlobalUsings.cs

@@ -0,0 +1,29 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and / or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+// We should eventually migrate to the constrained assertions:
+// https://docs.nunit.org/articles/nunit/release-notes/Nunit4.0-MigrationGuide.html#use-global-using-aliases
+global using Assert = NUnit.Framework.Legacy.ClassicAssert;
+global using CollectionAssert = NUnit.Framework.Legacy.CollectionAssert;
+global using StringAssert = NUnit.Framework.Legacy.StringAssert;

src/Playwright.Tests/PageGotoTests.cs

@@ -279,7 +279,7 @@ public async Task ShouldFailWhenMainResourcesFailedToLoad()
  }
  else if (TestConstants.IsWebKit && TestConstants.IsWindows)
  {
- StringAssert.Contains("Couldn't connect to server", exception.Message);
+ StringAssert.Contains("Could not connect to server", exception.Message);
  }
  else if (TestConstants.IsWebKit)
  {

src/Playwright.Tests/PageSetInputFilesTests.cs

@@ -353,7 +353,7 @@ public async Task ShouldWorkForWebkitdirectory()
  }
 
  [PlaywrightTest("page-set-input-files.spec.ts", "should upload large file")]
- [Timeout(TestConstants.SlowTestTimeout)]
+ [CancelAfter(TestConstants.SlowTestTimeout)]
  public async Task ShouldUploadLargeFile()
  {
  await Page.GotoAsync(Server.Prefix + "/input/fileupload.html");

src/Playwright.Tests/PageWaitForNavigationTests.cs

@@ -307,15 +307,15 @@ public async Task ShouldWorkOnFrame()
  }
 
  [PlaywrightTest]
- [Timeout(45_000)]
+ [CancelAfter(45_000)]
  public async Task ShouldHaveADefaultTimeout()
  {
  await Page.GotoAsync(Server.Prefix + "/frames/one-frame.html");
  await PlaywrightAssert.ThrowsAsync<TimeoutException>(async () => await Page.RunAndWaitForNavigationAsync(() => Task.CompletedTask));
  }
 
  [PlaywrightTest]
- [Timeout(5_000)]
+ [CancelAfter(5_000)]
  public async Task ShouldTakeTimeoutIntoAccount()
  {
  await Page.GotoAsync(Server.Prefix + "/frames/one-frame.html");

src/Playwright.Tests/Playwright.Tests.csproj

@@ -21,9 +21,9 @@
  <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
  <PrivateAssets>all</PrivateAssets>
  </PackageReference>
- <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
- <PackageReference Include="NUnit" Version="3.14.0" />
- <PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
+ <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+ <PackageReference Include="NUnit" Version="4.2.2" />
+ <PackageReference Include="NUnit3TestAdapter" Version="4.6.0" />
  <PackageReference Include="SixLabors.ImageSharp" Version="2.1.9" />
  </ItemGroup>
  <ItemGroup>

src/Playwright.Tests/ScreencastTests.cs

@@ -191,7 +191,7 @@ public async Task ShouldCaptureStaticPageInPersistentContext()
 
 
  [PlaywrightTest("screencast.spec.ts", "video.path()/saveAs() does not hang immediately after launchPersistentContext and context.close()")]
- [Timeout(30_000)]
+ [CancelAfter(30_000)]
  public async Task VideoPathSaveAsDoesNotHangImmediatelyAfterLaunchPersistentContextAndContextClose()
  {
  using var userDirectory = new TempDirectory();