From ed19db1a235fa7ca60c3d111b3544dcbe5e00bb0 Mon Sep 17 00:00:00 2001 From: Piotr Karczmarz Date: Fri, 1 Nov 2024 08:15:57 +0100 Subject: [PATCH] Support for automatically taking screenshots at the end of each UI test (passed or failed). (#130) Implemented ability to automatically take screenshots for UI tests, locally and when using GitHub Actions. A screenshot is taken at the end of every UI test (all tests inherited from `PlaywrightTestsBase` class). Currently `VsTestSettings(TakeScreenshotOnFailure = true)` attribute doesn't work for async tests (`Task async` or `void async`), which is way the custom solution was implemented. - Updated Cake Build and Nightly Build to include support for taking screenshots via GitHub Actions. - Additional logging for Solution_Name_Is_Added_To_Chat_Input(), which very rarely fails. - Additional logging for PlaywrightTestsBase.GetChatContextTags() - Fixed logging for TestBase.OpenSolution() --- .github/workflows/cake-build.yml | 12 +++++ .github/workflows/nightly.yml | 12 +++++ .../CustomConfigurationTests.cs | 2 +- .../ChatLoggedBasicTests.cs | 10 ++-- .../ChatNotLoggedStateTests.cs | 8 ++- .../Cody.VisualStudio.Tests.csproj | 2 + .../PlaywrightTestsBase.cs | 14 ++++- .../Properties/AssemblyInfo.cs | 1 + src/Cody.VisualStudio.Tests/ScreenshotUtil.cs | 52 +++++++++++++++++++ src/Cody.VisualStudio.Tests/TestsBase.cs | 36 ++++++++++++- src/Cody.sln | 1 + 11 files changed, 141 insertions(+), 9 deletions(-) create mode 100644 src/Cody.VisualStudio.Tests/ScreenshotUtil.cs diff --git a/.github/workflows/cake-build.yml b/.github/workflows/cake-build.yml index 2a96c529..66a73b38 100644 --- a/.github/workflows/cake-build.yml +++ b/.github/workflows/cake-build.yml @@ -28,6 +28,10 @@ jobs: - name: Add msbuild to PATH uses: microsoft/setup-msbuild@v1.3 + - name: Change Screen Resolution + shell: pwsh + run: Set-DisplayResolution -Width 1920 -Height 1080 -Force + - name: ⚙️ Prepare Visual Studio run: '&"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\IDE\devenv.exe" /RootSuffix Exp /ResetSettings General.vssettings' @@ -69,6 +73,14 @@ jobs: cd src dotnet test .\Cody.Core.Tests\bin\Debug\Cody.Core.Tests.dll .\Cody.VisualStudio.Tests\bin\Debug\Cody.VisualStudio.Tests.dll -v detailed -l:trx + - name: Upload screenshots for UI tests + uses: actions/upload-artifact@v4 + if: always() + with: + name: UI Tests Screenshots + path: src/Cody.VisualStudio.Tests/bin/Debug/Screenshots + retention-days: 20 + - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action/windows@v2 if: always() diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ac8a05a5..c20c856b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -20,6 +20,10 @@ jobs: - name: Add msbuild to PATH uses: microsoft/setup-msbuild@v1.3 + - name: Change Screen Resolution + shell: pwsh + run: Set-DisplayResolution -Width 1920 -Height 1080 -Force + - name: ⚙️ Prepare Visual Studio run: '&"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\IDE\devenv.exe" /RootSuffix Exp /ResetSettings General.vssettings' @@ -53,6 +57,14 @@ jobs: cd src dotnet test .\Cody.VisualStudio.Tests\bin\Debug\Cody.VisualStudio.Tests.dll -v detailed -l:trx + - name: Upload screenshots for UI tests + uses: actions/upload-artifact@v4 + if: always() + with: + name: UI Tests Screenshots + path: src/Cody.VisualStudio.Tests/bin/Debug/Screenshots + retention-days: 5 + - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action/windows@v2 if: always() diff --git a/src/Cody.Core.Tests/CustomConfigurationTests.cs b/src/Cody.Core.Tests/CustomConfigurationTests.cs index 99f354a0..a98bcecd 100644 --- a/src/Cody.Core.Tests/CustomConfigurationTests.cs +++ b/src/Cody.Core.Tests/CustomConfigurationTests.cs @@ -64,7 +64,7 @@ public void Complex_JSON_Data_Should_Be_Serializable() // given var key1 = "cody.autocomplete.enabled"; var key2 = "cody.customHeaders"; - var key3 = "cody.customHeaders"; + var key3 = "cody.excludeFiles"; var configurationJson = $@"{{ ""{key1}"": true, ""{key2}"": {{ diff --git a/src/Cody.VisualStudio.Tests/ChatLoggedBasicTests.cs b/src/Cody.VisualStudio.Tests/ChatLoggedBasicTests.cs index 3b4207a5..5a9dc1fd 100644 --- a/src/Cody.VisualStudio.Tests/ChatLoggedBasicTests.cs +++ b/src/Cody.VisualStudio.Tests/ChatLoggedBasicTests.cs @@ -1,14 +1,12 @@ using System; -using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using EnvDTE; using Xunit; using Xunit.Abstractions; namespace Cody.VisualStudio.Tests { - public class ChatLoggedBasicTests : PlaywrightTestsBase + public class ChatLoggedBasicTests : PlaywrightTestsBase, IDisposable { public ChatLoggedBasicTests(ITestOutputHelper output) : base(output) { @@ -80,5 +78,11 @@ public async Task Entered_Prompt_Show_Up_In_Today_History() Assert.Contains(chatHistoryEntries, x => x.Contains(prompt)); } + + public void Dispose() + { + var testName = GetTestName(); + TakeScreenshot(testName); + } } } diff --git a/src/Cody.VisualStudio.Tests/ChatNotLoggedStateTests.cs b/src/Cody.VisualStudio.Tests/ChatNotLoggedStateTests.cs index 688ac464..35780455 100644 --- a/src/Cody.VisualStudio.Tests/ChatNotLoggedStateTests.cs +++ b/src/Cody.VisualStudio.Tests/ChatNotLoggedStateTests.cs @@ -8,7 +8,7 @@ namespace Cody.VisualStudio.Tests { - public class ChatNotLoggedStateTests : PlaywrightTestsBase + public class ChatNotLoggedStateTests : PlaywrightTestsBase, IDisposable { public ChatNotLoggedStateTests(ITestOutputHelper output) : base(output) { @@ -43,5 +43,11 @@ public async Task Loads_Properly_InNotLoggedState() // then Assert.Equal(text, textContents.First()); } + + public void Dispose() + { + var testName = GetTestName(); + TakeScreenshot(testName); + } } } diff --git a/src/Cody.VisualStudio.Tests/Cody.VisualStudio.Tests.csproj b/src/Cody.VisualStudio.Tests/Cody.VisualStudio.Tests.csproj index b3a053a9..d0976726 100644 --- a/src/Cody.VisualStudio.Tests/Cody.VisualStudio.Tests.csproj +++ b/src/Cody.VisualStudio.Tests/Cody.VisualStudio.Tests.csproj @@ -37,6 +37,7 @@ + @@ -52,6 +53,7 @@ + diff --git a/src/Cody.VisualStudio.Tests/PlaywrightTestsBase.cs b/src/Cody.VisualStudio.Tests/PlaywrightTestsBase.cs index f5f05813..8eb03074 100644 --- a/src/Cody.VisualStudio.Tests/PlaywrightTestsBase.cs +++ b/src/Cody.VisualStudio.Tests/PlaywrightTestsBase.cs @@ -127,7 +127,11 @@ protected async Task SetAccessToken(string accessToken) protected async Task ShowChatTab() => await Page.GetByTestId("tab-chat").ClickAsync(); - protected async Task ShowHistoryTab() => await Page.GetByTestId("tab-history").ClickAsync(); + protected async Task ShowHistoryTab() + { + await Page.GetByTestId("tab-history").ClickAsync(); + await Task.Delay(500); + } protected async Task ShowPromptsTab() => await Page.GetByTestId("tab-prompts").ClickAsync(); @@ -162,8 +166,14 @@ protected async Task> GetChatContextTags() { var tagsList = new List(); + WriteLog("Searching for Chat ..."); var chatBox = await Page.QuerySelectorAsync("[aria-label='Chat message']"); - if (chatBox == null) throw new Exception("ChatBox is null. Probably not authenticated."); + if (chatBox == null) + { + WriteLog("Chat NOT found."); + throw new Exception("ChatBox is null. Probably not authenticated."); + } + WriteLog("Chat found."); var list = await chatBox.QuerySelectorAllAsync("span[data-lexical-decorator='true']"); foreach (var item in list) diff --git a/src/Cody.VisualStudio.Tests/Properties/AssemblyInfo.cs b/src/Cody.VisualStudio.Tests/Properties/AssemblyInfo.cs index fa85bb76..684744d2 100644 --- a/src/Cody.VisualStudio.Tests/Properties/AssemblyInfo.cs +++ b/src/Cody.VisualStudio.Tests/Properties/AssemblyInfo.cs @@ -33,3 +33,4 @@ // [assembly: AssemblyVersion("1.0.*")] [assembly: TestFramework("Xunit.VsTestFramework", "VsixTesting.Xunit")] [assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly, DisableTestParallelization = true, MaxParallelThreads = 1)] +[assembly: VsTestSettings(TakeScreenshotOnFailure = true)] diff --git a/src/Cody.VisualStudio.Tests/ScreenshotUtil.cs b/src/Cody.VisualStudio.Tests/ScreenshotUtil.cs new file mode 100644 index 00000000..6e6a22bd --- /dev/null +++ b/src/Cody.VisualStudio.Tests/ScreenshotUtil.cs @@ -0,0 +1,52 @@ +using System; +using System.Runtime.InteropServices; +using System.Drawing; +using System.Drawing.Imaging; + +namespace Cody.VisualStudio.Tests +{ + public class ScreenshotUtil + { + public static void CaptureWindow(IntPtr hwnd, string path) + { + var rect = default(RECT); + GetWindowRect(hwnd, ref rect); + CaptureScreenArea( + path, + left: rect.Left, + top: rect.Top, + width: rect.Right - rect.Left, + height: rect.Bottom - rect.Top); + } + + public static void CaptureScreenArea(string path, int left, int top, int width, int height) + { + using (var bitmap = new Bitmap(width, height)) + using (var image = Graphics.FromImage(bitmap)) + { + image.CopyFromScreen( + sourceX: left, + sourceY: top, + blockRegionSize: new Size(width, height), + copyPixelOperation: CopyPixelOperation.SourceCopy, + destinationX: 0, + destinationY: 0); + + bitmap.Save(path, ImageFormat.Png); + } + } + + [DllImport("user32.dll")] + private static extern bool GetWindowRect(IntPtr hWnd, ref RECT rect); + + [StructLayout(LayoutKind.Sequential)] + private struct RECT + { + public int Left; + public int Top; + public int Right; + public int Bottom; + } + } + +} diff --git a/src/Cody.VisualStudio.Tests/TestsBase.cs b/src/Cody.VisualStudio.Tests/TestsBase.cs index 01753f4e..539ffec1 100644 --- a/src/Cody.VisualStudio.Tests/TestsBase.cs +++ b/src/Cody.VisualStudio.Tests/TestsBase.cs @@ -1,14 +1,17 @@ using System; +using System.Drawing.Imaging; +using System.IO; +using System.Reflection; using System.Runtime.CompilerServices; using System.Threading.Tasks; using System.Windows; using Cody.Core.Logging; -using EnvDTE; using EnvDTE80; using Microsoft.VisualStudio.Shell; using Microsoft.VisualStudio.Shell.Interop; using Microsoft.VisualStudio.TextManager.Interop; using Xunit.Abstractions; +using System.Diagnostics; using Thread = System.Threading.Thread; namespace Cody.VisualStudio.Tests @@ -38,17 +41,46 @@ public void WriteLog(string message, string type = "", [CallerMemberName] string } } + protected string GetTestName() + { + var test = (ITest)_logger.GetType() + .GetField("test", BindingFlags.Instance | BindingFlags.NonPublic) + ?.GetValue(_logger); + + var testName = test?.DisplayName; + return testName; + } + + protected void TakeScreenshot(string name) + { + var safeName = string.Join("_", name.Split(Path.GetInvalidFileNameChars())); + var date = DateTime.Now.ToString("yyyy-MM-dd hh.mm.ss"); + + var directory = "Screenshots"; + var directoryPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), directory)); + var path = Path.Combine(directoryPath, $"{date} {safeName}.png"); + + WriteLog($"Saving screenshots to:{directoryPath}"); + Directory.CreateDirectory(directoryPath); + + ScreenshotUtil.CaptureWindow(Process.GetCurrentProcess().MainWindowHandle, path); + WriteLog($"Screenshot saved to:{path}"); + } + private IVsUIShell _uiShell; protected IVsUIShell UIShell => _uiShell ?? (_uiShell = (IVsUIShell)Package.GetGlobalService(typeof(SVsUIShell))); private DTE2 _dte; - protected DTE2 Dte => _dte ?? (_dte = (DTE2)Package.GetGlobalService(typeof(DTE))); + protected DTE2 Dte => _dte ?? (_dte = (DTE2)Package.GetGlobalService(typeof(EnvDTE.DTE))); protected async Task OpenSolution(string path) { + + WriteLog($"Opening solution '{path}' ..."); Dte.Solution.Open(path); await Task.Delay(TimeSpan.FromSeconds(5)); + WriteLog("Delay after solution open stopped."); } protected void CloseSolution() => Dte.Solution.Close(); diff --git a/src/Cody.sln b/src/Cody.sln index 0009fd6c..7f70471f 100644 --- a/src/Cody.sln +++ b/src/Cody.sln @@ -24,6 +24,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution build.cake = build.cake ..\.github\workflows\cake-build.yml = ..\.github\workflows\cake-build.yml ..\.github\workflows\code-style.yml = ..\.github\workflows\code-style.yml + ..\.github\workflows\nightly.yml = ..\.github\workflows\nightly.yml ..\.github\workflows\publish.yml = ..\.github\workflows\publish.yml ..\.github\workflows\release-preview.yml = ..\.github\workflows\release-preview.yml EndProjectSection