feat: Add ability to always save test file (#948)

#### Details

#935 requests that we provide a way for CLI users to create an
`.a11ytest` file even when no errors are reported. This adds a
`--alwayssavetestfile` option to the CLI, then plumbs it through to take
effect. People using the automation interface can access this directly
via the `Config.Builder.WithAlwaysSaveTestFile` method.

##### Motivation

Address #935 

##### Context

<!-- Are there any parts that you've intentionally left out-of-scope for
a later PR to handle? -->

<!-- Were there any alternative approaches you considered? What
tradeoffs did you consider? -->

#### Pull request checklist
<!-- If a checklist item is not applicable to this change, write "n/a"
in the checkbox -->
- [x] Addresses an existing issue: #935

---------

Co-authored-by: Dave Tryon <45672944@DaveTryon@users.noreply.github.com>

docs/AutomationReference.md

@@ -92,6 +92,14 @@ path | `string` | The path to the configuration file.
 
 The `WithCustomUIAConfig` method returns the  `Config.Builder` configured with the specified custom UIA configuration file.
 
+##### `WithAlwaysSaveTestFile`
+
+Cause each scan to save a test file, even if no errors are reported. The default is to save a test file only is errors are found.
+
+###### Return object
+
+The `WithAlwaysSaveTestFile` method returns the  `Config.Builder` configured to always save a test file.
+
 ##### Build
 
 Build an instance of `Config`.

src/Automation/Data/Config.cs

@@ -39,6 +39,9 @@ public class Config
         ///  <summary>The path to a file containing configuration instructing Axe Windows how to interpret custom UI Automation data.</summary>
         public string CustomUIAConfigPath { get; private set; }
 
+        /// <summary>Override the default behavior of only saving a11ytest files if errors are found.</summary>
+        public bool AlwaysSaveTestFile { get; private set; }
+
         /// <summary>
         /// Custom handling of DPI awareness. The default handling is to set the entire process as DPI-aware
         /// before running the scan, and to leave it in that state after the scan completes. If your process
@@ -117,6 +120,16 @@ public Builder WithCustomUIAConfig(string path)
                 return this;
             }
 
+            /// <summary>
+            /// Configure Axe.Windows to always save test files in the specified format, even if no errors are found.
+            /// By default, test files are saved only if errors are found or if multiple top-level windows exist.
+            /// </summary>
+            public Builder WithAlwaysSaveTestFile()
+            {
+                _config.AlwaysSaveTestFile = true;
+                return this;
+            }
+
             /// <summary>
             /// Build an instance of <see cref="Config"/>
             /// </summary>
@@ -130,6 +143,7 @@ public Config Build()
                     OutputDirectory = _config.OutputDirectory,
                     CustomUIAConfigPath = _config.CustomUIAConfigPath,
                     DPIAwareness = _config.DPIAwareness,
+                    AlwaysSaveTestFile = _config.AlwaysSaveTestFile,
                 };
             }
         } // Builder

src/Automation/SnapshotCommand.cs

@@ -126,7 +126,7 @@ private static WindowScanOutput ProcessResults(A11yElement element, Guid element
                 }
                 else
                 {
-                    if (results.ErrorCount > 0)
+                    if (results.ErrorCount > 0 || config.AlwaysSaveTestFile)
                     {
                         results.OutputFile = WriteOutputFiles(config.OutputFileFormat, scanTools, element, elementId, null, actionContext);
                     }

src/CLI/IOptions.cs

@@ -12,5 +12,6 @@ internal interface IOptions
         VerbosityLevel VerbosityLevel { get; }
         int DelayInSeconds { get; }
         string CustomUia { get; }
+        bool AlwaysSaveTestFile { get; }
     }
 }

src/CLI/Options.cs

@@ -32,6 +32,9 @@ public class Options : IOptions
         [Option(Required = false, HelpText = "CustomUia", ResourceType = typeof(Resources.OptionsHelpText))]
         public string CustomUia { get; set; }
 
+        [Option(Required = false, HelpText = "AlwaysSaveTestFile", ResourceType = typeof(Resources.OptionsHelpText))]
+        public bool AlwaysSaveTestFile { get; set; }
+
         // CommandLineParser will never set this value!
         public VerbosityLevel VerbosityLevel { get; set; } = VerbosityLevel.Default;
     }

src/CLI/OptionsEvaluator.cs

@@ -64,6 +64,7 @@ public static IOptions ProcessInputs(Options rawInputs, IProcessHelper processHe
                 VerbosityLevel = verbosityLevel,
                 DelayInSeconds = delayInSeconds,
                 CustomUia = rawInputs.CustomUia,
+                AlwaysSaveTestFile = rawInputs.AlwaysSaveTestFile,
             };
         }
 

src/CLI/README.MD

@@ -16,7 +16,7 @@ You invoke the scanner by running `AxeWindowsCLI.exe`. By default, this tool get
 If you run the tool without parameters, you'll be presented with the help screen. An example follows:
 
 ```
-AxeWindowsCLI 2.1.1
+AxeWindowsCLI 2.1.4
 Copyright c 2020
 
   --processid                Process Id
@@ -39,6 +39,10 @@ Copyright c 2020
 
   --customuia                The path to a configuration file specifying custom
                              UI Automation attributes
+
+  --alwayssavetestfile       If specified, always save the test file. By
+                             default, the test file is saved only if errors are
+                             found.
 ```
 
 To scan an application, you need to specify the application's process via either the `--processId` or `--processName` parameters
@@ -55,9 +59,10 @@ verbosity|Identifies the level of detail you want in the output. Valid values ar
 showThirdPartyNotices|If specified, displays the third party notices for components used by AxeWindowsCLI. This information is also available in the `thirdpartynotices.html` file that is installed with AxeWindowsCLI.
 delayInSeconds|Optionally inserts a delay before triggering the scan. This allows transient controls (menus, drop-down-lists, etc.) to be scanned.
 customUia|Optionally provides a path to a [custom UIA configuration file](../../docs/CustomUIA.md). By default, only system-defined UIA properties will be included in the scan.
+alwayssavetestfile|Optionally causes the test file to always be saved. By default, the test file is saved only if errors are found.
 
 ### Scan results
-A summary of scan results will be displayed after the scan is run. In addition, an `.a11ytest` file will be generated if 1 or more errors were detected. The location of this file will be reported in the tool output (see the documentation of the `--outputDirectory` and `--scanId` parameters for ways to alter the name or location of the output file). This file can then be opened with **Accessibility Insights for Windows**, which is freely available at http://accessibilityinsights.io. If you scan an application with multiple top-level windows, an `.a11ytest` file will be generated for _each_ top-level window, even if no errors are detected.
+A summary of scan results will be displayed after the scan is run. In addition, an `.a11ytest` file will be generated if 1 or more errors were detected or if the `alwayssavetestfile` option was specified. The location of this file will be reported in the tool output (see the documentation of the `--outputDirectory` and `--scanId` parameters for ways to alter the name or location of the output file). This file can then be opened with **Accessibility Insights for Windows**, which is freely available at http://accessibilityinsights.io. If you scan an application with multiple top-level windows, an `.a11ytest` file will be generated for _each_ top-level window, even if no errors are detected.
 
 A detailed description of scan results will also be displayed if you specify verbose output (see the `verbosity` parameter).
 

src/CLI/Resources/OptionsHelpText.resx

@@ -117,6 +117,9 @@
   <resheader name="writer">
     <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
   </resheader>
+  <data name="AlwaysSaveTestFile" xml:space="preserve">
+    <value>If specified, always save the test file. By default, the test file is saved only if errors are found.</value>
+  </data>
   <data name="CustomUia" xml:space="preserve">
     <value>The path to a configuration file specifying custom UI Automation attributes</value>
   </data>

src/CLI/ScanRunner.cs

@@ -25,6 +25,9 @@ private static IScanner BuildScanner(IOptions options)
             if (!string.IsNullOrEmpty(options.OutputDirectory))
                 builder = builder.WithOutputDirectory(options.OutputDirectory);
 
+            if (options.AlwaysSaveTestFile)
+                builder = builder.WithAlwaysSaveTestFile();
+
             return ScannerFactory.CreateScanner(builder.Build());
         }
     }

src/CLITests/OptionsEvaluatorTests.cs

@@ -31,14 +31,16 @@ private void VerifyAllMocks()
 
         private static void ValidateOptions(IOptions options, string processName = TestProcessName,
             int processId = TestProcessId, string outputDirectory = null, string scanId = null,
-            VerbosityLevel verbosityLevel = VerbosityLevel.Default, int delayInSeconds = 0)
+            VerbosityLevel verbosityLevel = VerbosityLevel.Default, int delayInSeconds = 0,
+            bool alwaysWriteTestFile = false)
         {
             Assert.AreEqual(processName, options.ProcessName);
             Assert.AreEqual(processId, options.ProcessId);
             Assert.AreEqual(scanId, options.ScanId);
             Assert.AreEqual(outputDirectory, options.OutputDirectory);
             Assert.AreEqual(verbosityLevel, options.VerbosityLevel);
             Assert.AreEqual(delayInSeconds, options.DelayInSeconds);
+            Assert.AreEqual(alwaysWriteTestFile, options.AlwaysSaveTestFile);
         }
 
         [TestMethod]
@@ -287,5 +289,20 @@ public void ProcessInputs_SpecifiesScanOptions_SetsMultiscan()
                 processId: TestProcessId);
             VerifyAllMocks();
         }
+
+        [TestMethod]
+        [Timeout(1000)]
+        public void ProcessInputs_SpecifiesAlwaysSaveTestFile_SetsSaveTestFile()
+        {
+            _processHelperMock.Setup(x => x.ProcessIdFromName(TestProcessName)).Returns(TestProcessId);
+            Options input = new Options
+            {
+                ProcessName = TestProcessName,
+                AlwaysSaveTestFile = true,
+            };
+            ValidateOptions(OptionsEvaluator.ProcessInputs(input, _processHelperMock.Object),
+                processId: TestProcessId, alwaysWriteTestFile: true);
+            VerifyAllMocks();
+        }
     }
 }

src/CLITests/OptionsTests.cs

@@ -22,6 +22,7 @@ public class OptionsTests
         const string ShowThirdPartyNoticesKey = "--showthirdpartynotices";
         const string DelayInSecondsKey = "--delayinseconds";
         const string CustomUiaKey = "--customuia";
+        const string AlwaysSaveTestFileKey = "--alwayssavetestfile";
 
         const string TestProcessName = "MyProcess";
         const int TestProcessId = 42;
@@ -35,7 +36,7 @@ private int FailIfCalled(IEnumerable<Error> errors)
         private static int ValidateOptions(Options options, string processName = null,
             int processId = 0, string outputDirectory = null, string scanId = null,
             string verbosity = null, bool showThirdPartyNotices = false,
-            int delayInSeconds = 0, string customUia = null)
+            int delayInSeconds = 0, string customUia = null, bool alwaysSaveTestFile = false)
         {
             Assert.AreEqual(processName, options.ProcessName);
             Assert.AreEqual(processId, options.ProcessId);
@@ -46,6 +47,8 @@ private static int ValidateOptions(Options options, string processName = null,
             Assert.AreEqual(showThirdPartyNotices, options.ShowThirdPartyNotices);
             Assert.AreEqual(delayInSeconds, options.DelayInSeconds);
             Assert.AreEqual(customUia, options.CustomUia);
+            Assert.AreEqual(alwaysSaveTestFile, options.AlwaysSaveTestFile);
+
             return ExpectedParseSuccess;
         }
 
@@ -155,5 +158,21 @@ public void ParseArguments_CustomUiaIsSet_SucceedsWithCustomUia()
 
             Assert.AreEqual(ExpectedParseSuccess, parseResult);
         }
+
+        [TestMethod]
+        [Timeout(1000)]
+        public void ParseArguments_AlwaysSaveTestFileIsSpecified_SucceedsWithAlwaysSaveTestFile()
+        {
+            bool expectedAlwaysSaveTestFile = true;
+
+            string[] args = { AlwaysSaveTestFileKey };
+
+            int parseResult = Parser.Default.ParseArguments<Options>(args)
+                .MapResult(
+                    (o) => ValidateOptions(o, alwaysSaveTestFile: expectedAlwaysSaveTestFile),
+                    FailIfCalled);
+
+            Assert.AreEqual(ExpectedParseSuccess, parseResult);
+        }
     }
 }