Support for automatically taking screenshots at the end of each UI te…

…st (passed or failed). (#130)

Implemented ability to automatically take screenshots for UI tests,
locally and when using GitHub Actions. A screenshot is taken at the end
of every UI test (all tests inherited from `PlaywrightTestsBase` class).

Currently `VsTestSettings(TakeScreenshotOnFailure = true)` attribute
doesn't work for async tests (`Task async` or `void async`), which is
way the custom solution was implemented.

- Updated Cake Build and Nightly Build to include support for taking
screenshots via GitHub Actions.
- Additional logging for Solution_Name_Is_Added_To_Chat_Input(), which
very rarely fails.
- Additional logging for PlaywrightTestsBase.GetChatContextTags()
- Fixed logging for TestBase.OpenSolution()

.github/workflows/cake-build.yml

@@ -28,6 +28,10 @@ jobs:
       - name: Add msbuild to PATH
         uses: microsoft/setup-msbuild@v1.3
 
+      - name: Change Screen Resolution
+        shell: pwsh
+        run: Set-DisplayResolution -Width 1920 -Height 1080 -Force
+
       - name: ⚙️ Prepare Visual Studio
         run: '&"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\IDE\devenv.exe" /RootSuffix Exp /ResetSettings General.vssettings'
 
@@ -69,6 +73,14 @@ jobs:
           cd src
           dotnet test .\Cody.Core.Tests\bin\Debug\Cody.Core.Tests.dll .\Cody.VisualStudio.Tests\bin\Debug\Cody.VisualStudio.Tests.dll -v detailed -l:trx
 
+      - name: Upload screenshots for UI tests
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: UI Tests Screenshots
+          path: src/Cody.VisualStudio.Tests/bin/Debug/Screenshots
+          retention-days: 20
+
       - name: Publish Test Results
         uses: EnricoMi/publish-unit-test-result-action/windows@v2
         if: always()

.github/workflows/nightly.yml

@@ -20,6 +20,10 @@ jobs:
       - name: Add msbuild to PATH
         uses: microsoft/setup-msbuild@v1.3
 
+      - name: Change Screen Resolution
+        shell: pwsh
+        run: Set-DisplayResolution -Width 1920 -Height 1080 -Force
+
       - name: ⚙️ Prepare Visual Studio
         run: '&"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\IDE\devenv.exe" /RootSuffix Exp /ResetSettings General.vssettings'
 
@@ -53,6 +57,14 @@ jobs:
           cd src
           dotnet test .\Cody.VisualStudio.Tests\bin\Debug\Cody.VisualStudio.Tests.dll -v detailed -l:trx
 
+      - name: Upload screenshots for UI tests
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: UI Tests Screenshots
+          path: src/Cody.VisualStudio.Tests/bin/Debug/Screenshots
+          retention-days: 5
+
       - name: Publish Test Results
         uses: EnricoMi/publish-unit-test-result-action/windows@v2
         if: always()

src/Cody.Core.Tests/CustomConfigurationTests.cs

@@ -64,7 +64,7 @@ public void Complex_JSON_Data_Should_Be_Serializable()
             // given
             var key1 = "cody.autocomplete.enabled";
             var key2 = "cody.customHeaders";
-            var key3 = "cody.customHeaders";
+            var key3 = "cody.excludeFiles";
             var configurationJson = $@"{{
                         	""{key1}"": true,
 	                        ""{key2}"": {{

src/Cody.VisualStudio.Tests/ChatLoggedBasicTests.cs

@@ -1,14 +1,12 @@
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
-using EnvDTE;
 using Xunit;
 using Xunit.Abstractions;
 
 namespace Cody.VisualStudio.Tests
 {
-    public class ChatLoggedBasicTests : PlaywrightTestsBase
+    public class ChatLoggedBasicTests : PlaywrightTestsBase, IDisposable
     {
         public ChatLoggedBasicTests(ITestOutputHelper output) : base(output)
         {
@@ -80,5 +78,11 @@ public async Task Entered_Prompt_Show_Up_In_Today_History()
             Assert.Contains(chatHistoryEntries, x => x.Contains(prompt));
 
         }
+
+        public void Dispose()
+        {
+            var testName = GetTestName();
+            TakeScreenshot(testName);
+        }
     }
 }

src/Cody.VisualStudio.Tests/ChatNotLoggedStateTests.cs

@@ -8,7 +8,7 @@
 
 namespace Cody.VisualStudio.Tests
 {
-    public class ChatNotLoggedStateTests : PlaywrightTestsBase
+    public class ChatNotLoggedStateTests : PlaywrightTestsBase, IDisposable
     {
         public ChatNotLoggedStateTests(ITestOutputHelper output) : base(output)
         {
@@ -43,5 +43,11 @@ public async Task Loads_Properly_InNotLoggedState()
             // then
             Assert.Equal(text, textContents.First());
         }
+
+        public void Dispose()
+        {
+            var testName = GetTestName();
+            TakeScreenshot(testName);
+        }
     }
 }

src/Cody.VisualStudio.Tests/Cody.VisualStudio.Tests.csproj

@@ -37,6 +37,7 @@
     <Reference Include="System" />
     <Reference Include="System.Configuration" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Drawing" />
     <Reference Include="System.Runtime" />
     <Reference Include="System.Runtime.Serialization" />
     <Reference Include="System.Xml.Linq" />
@@ -52,6 +53,7 @@
     </Compile>
     <Compile Include="ChatNotLoggedStateTests.cs" />
     <Compile Include="ChatLoggedBasicTests.cs" />
+    <Compile Include="ScreenshotUtil.cs" />
     <Compile Include="PlaywrightTestsBase.cs" />
     <Compile Include="PlaywrightInitializationTests.cs" />
     <Compile Include="CodyPackageTests.cs" />

src/Cody.VisualStudio.Tests/PlaywrightTestsBase.cs

@@ -127,7 +127,11 @@ protected async Task SetAccessToken(string accessToken)
 
         protected async Task ShowChatTab() => await Page.GetByTestId("tab-chat").ClickAsync();
 
-        protected async Task ShowHistoryTab() => await Page.GetByTestId("tab-history").ClickAsync();
+        protected async Task ShowHistoryTab()
+        {
+            await Page.GetByTestId("tab-history").ClickAsync();
+            await Task.Delay(500);
+        }
 
         protected async Task ShowPromptsTab() => await Page.GetByTestId("tab-prompts").ClickAsync();
 
@@ -162,8 +166,14 @@ protected async Task<IReadOnlyCollection<ContextTag>> GetChatContextTags()
         {
             var tagsList = new List<ContextTag>();
 
+            WriteLog("Searching for Chat ...");
             var chatBox = await Page.QuerySelectorAsync("[aria-label='Chat message']");
-            if (chatBox == null) throw new Exception("ChatBox is null. Probably not authenticated.");
+            if (chatBox == null)
+            {
+                WriteLog("Chat NOT found.");
+                throw new Exception("ChatBox is null. Probably not authenticated.");
+            }
+            WriteLog("Chat found.");
 
             var list = await chatBox.QuerySelectorAllAsync("span[data-lexical-decorator='true']");
             foreach (var item in list)

src/Cody.VisualStudio.Tests/Properties/AssemblyInfo.cs

@@ -33,3 +33,4 @@
 // [assembly: AssemblyVersion("1.0.*")]
 [assembly: TestFramework("Xunit.VsTestFramework", "VsixTesting.Xunit")]
 [assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly, DisableTestParallelization = true, MaxParallelThreads = 1)]
+[assembly: VsTestSettings(TakeScreenshotOnFailure = true)]

src/Cody.VisualStudio.Tests/ScreenshotUtil.cs

@@ -0,0 +1,52 @@
+using System;
+using System.Runtime.InteropServices;
+using System.Drawing;
+using System.Drawing.Imaging;
+
+namespace Cody.VisualStudio.Tests
+{
+    public class ScreenshotUtil
+    {
+        public static void CaptureWindow(IntPtr hwnd, string path)
+        {
+            var rect = default(RECT);
+            GetWindowRect(hwnd, ref rect);
+            CaptureScreenArea(
+                path,
+                left: rect.Left,
+                top: rect.Top,
+                width: rect.Right - rect.Left,
+                height: rect.Bottom - rect.Top);
+        }
+
+        public static void CaptureScreenArea(string path, int left, int top, int width, int height)
+        {
+            using (var bitmap = new Bitmap(width, height))
+            using (var image = Graphics.FromImage(bitmap))
+            {
+                image.CopyFromScreen(
+                    sourceX: left,
+                    sourceY: top,
+                    blockRegionSize: new Size(width, height),
+                    copyPixelOperation: CopyPixelOperation.SourceCopy,
+                    destinationX: 0,
+                    destinationY: 0);
+
+                bitmap.Save(path, ImageFormat.Png);
+            }
+        }
+
+        [DllImport("user32.dll")]
+        private static extern bool GetWindowRect(IntPtr hWnd, ref RECT rect);
+
+        [StructLayout(LayoutKind.Sequential)]
+        private struct RECT
+        {
+            public int Left;
+            public int Top;
+            public int Right;
+            public int Bottom;
+        }
+    }
+
+}

src/Cody.VisualStudio.Tests/TestsBase.cs

@@ -1,14 +1,17 @@
 using System;
+using System.Drawing.Imaging;
+using System.IO;
+using System.Reflection;
 using System.Runtime.CompilerServices;
 using System.Threading.Tasks;
 using System.Windows;
 using Cody.Core.Logging;
-using EnvDTE;
 using EnvDTE80;
 using Microsoft.VisualStudio.Shell;
 using Microsoft.VisualStudio.Shell.Interop;
 using Microsoft.VisualStudio.TextManager.Interop;
 using Xunit.Abstractions;
+using System.Diagnostics;
 using Thread = System.Threading.Thread;
 
 namespace Cody.VisualStudio.Tests
@@ -38,17 +41,46 @@ public void WriteLog(string message, string type = "", [CallerMemberName] string
             }
         }
 
+        protected string GetTestName()
+        {
+            var test = (ITest)_logger.GetType()
+                .GetField("test", BindingFlags.Instance | BindingFlags.NonPublic)
+                ?.GetValue(_logger);
+
+            var testName = test?.DisplayName;
+            return testName;
+        }
+
+        protected void TakeScreenshot(string name)
+        {
+            var safeName = string.Join("_", name.Split(Path.GetInvalidFileNameChars()));
+            var date = DateTime.Now.ToString("yyyy-MM-dd hh.mm.ss");
+
+            var directory = "Screenshots";
+            var directoryPath = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), directory));
+            var path = Path.Combine(directoryPath, $"{date} {safeName}.png");
+
+            WriteLog($"Saving screenshots to:{directoryPath}");
+            Directory.CreateDirectory(directoryPath);
+
+            ScreenshotUtil.CaptureWindow(Process.GetCurrentProcess().MainWindowHandle, path);
+            WriteLog($"Screenshot saved to:{path}");
+        }
+
         private IVsUIShell _uiShell;
         protected IVsUIShell UIShell => _uiShell ?? (_uiShell = (IVsUIShell)Package.GetGlobalService(typeof(SVsUIShell)));
 
         private DTE2 _dte;
-        protected DTE2 Dte => _dte ?? (_dte = (DTE2)Package.GetGlobalService(typeof(DTE)));
+        protected DTE2 Dte => _dte ?? (_dte = (DTE2)Package.GetGlobalService(typeof(EnvDTE.DTE)));
 
         protected async Task OpenSolution(string path)
         {
+
+            WriteLog($"Opening solution '{path}' ...");
             Dte.Solution.Open(path);
 
             await Task.Delay(TimeSpan.FromSeconds(5));
+            WriteLog("Delay after solution open stopped.");
         }
 
         protected void CloseSolution() => Dte.Solution.Close();

src/Cody.sln

@@ -24,6 +24,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		build.cake = build.cake
 		..\.github\workflows\cake-build.yml = ..\.github\workflows\cake-build.yml
 		..\.github\workflows\code-style.yml = ..\.github\workflows\code-style.yml
+		..\.github\workflows\nightly.yml = ..\.github\workflows\nightly.yml
 		..\.github\workflows\publish.yml = ..\.github\workflows\publish.yml
 		..\.github\workflows\release-preview.yml = ..\.github\workflows\release-preview.yml
 	EndProjectSection