Support Bulk Sanitizer Add (#6926)

* add code and tests for bulk sanitizers
* add docs reflecting new bulk sanitizer add

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs

@@ -3,6 +3,7 @@
 using Azure.Sdk.Tools.TestProxy.Matchers;
 using Azure.Sdk.Tools.TestProxy.Sanitizers;
 using Azure.Sdk.Tools.TestProxy.Transforms;
+using Microsoft.AspNetCore.DataProtection.KeyManagement;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging.Abstractions;
@@ -11,6 +12,7 @@
 using System.Linq;
 using System.Net;
 using System.Reflection;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Xunit;
 
@@ -29,6 +31,123 @@ public class AdminTests
     {
         private NullLoggerFactory _nullLogger = new NullLoggerFactory();
 
+        [Fact]
+        public async void TestAddSanitizersThrowsOnEmptyArray()
+        {
+            RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory());
+            var httpContext = new DefaultHttpContext();
+
+            string requestBody = @"[]";
+
+            httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody(requestBody);
+            httpContext.Request.ContentLength = httpContext.Request.Body.Length;
+            testRecordingHandler.Sanitizers.Clear();
+
+            var controller = new Admin(testRecordingHandler, _nullLogger)
+            {
+                ControllerContext = new ControllerContext()
+                {
+                    HttpContext = httpContext
+                }
+            };
+            var assertion = await Assert.ThrowsAsync<HttpException>(
+                async () => await controller.AddSanitizers()
+            );
+
+            assertion.StatusCode.Equals(HttpStatusCode.BadRequest);
+        }
+
+        [Fact]
+
+        public async void TestAddSanitizersHandlesPopulatedArray()
+        {
+            RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory());
+            var httpContext = new DefaultHttpContext();
+
+            string requestBody = @"[
+    {
+        ""Name"": ""GeneralRegexSanitizer"",
+        ""Body"": { 
+            ""regex"": ""[a-zA-Z]?"",
+            ""value"": ""hello_there"",
+            ""condition"": {
+                ""UriRegex"": "".+/Tables""
+            }
+        }
+    },
+    {
+        ""Name"": ""HeaderRegexSanitizer"",
+        ""Body"": { 
+            ""key"": ""Location"",
+            ""value"": ""https://fakeazsdktestaccount.table.core.windows.net/Tables""
+        }
+    }
+]";
+
+            httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody(requestBody);
+            httpContext.Request.ContentLength = httpContext.Request.Body.Length;
+            testRecordingHandler.Sanitizers.Clear();
+
+            var controller = new Admin(testRecordingHandler, _nullLogger)
+            {
+                ControllerContext = new ControllerContext()
+                {
+                    HttpContext = httpContext
+                }
+            };
+            await controller.AddSanitizers();
+
+
+            Assert.Equal(2, testRecordingHandler.Sanitizers.Count);
+
+            Assert.True(testRecordingHandler.Sanitizers[0] is GeneralRegexSanitizer);
+            Assert.True(testRecordingHandler.Sanitizers[1] is HeaderRegexSanitizer);
+        }
+
+        [Fact]
+        public async void TestAddSanitizersThrowsOnSingleBadInput()
+        {
+            RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory());
+            var httpContext = new DefaultHttpContext();
+
+            string requestBody = @"[
+    {
+        ""Name"": ""GeneralRegexSanitizer"",
+        ""Body"": { 
+            ""regex"": ""[a-zA-Z]?"",
+            ""value"": ""hello_there"",
+            ""condition"": {
+                ""UriRegex"": "".+/Tables""
+            }
+        }
+    },
+    {
+        ""Name"": ""BadRegexIdentifier"",
+        ""Body"": { 
+            ""key"": ""Location"",
+            ""value"": ""https://fakeazsdktestaccount.table.core.windows.net/Tables""
+        }
+    }
+]";
+
+            httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody(requestBody);
+            httpContext.Request.ContentLength = httpContext.Request.Body.Length;
+            testRecordingHandler.Sanitizers.Clear();
+
+            var controller = new Admin(testRecordingHandler, _nullLogger)
+            {
+                ControllerContext = new ControllerContext()
+                {
+                    HttpContext = httpContext
+                }
+            };
+            var assertion = await Assert.ThrowsAsync<HttpException>(
+                async () => await controller.AddSanitizers()
+            );
+
+            assertion.StatusCode.Equals(HttpStatusCode.BadRequest);
+        }
+
         [Fact]
         public async void TestAddSanitizerThrowsOnInvalidAbstractionId()
         {

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
 using Azure.Sdk.Tools.TestProxy.Common;
@@ -8,6 +8,7 @@
 using Microsoft.Extensions.Logging;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Net;
 using System.Text.Json;
 using System.Threading.Tasks;
@@ -81,6 +82,36 @@ public async Task AddSanitizer()
             }
         }
 
+        [HttpPost]
+        public async Task AddSanitizers()
+        {
+            DebugLogger.LogAdminRequestDetails(_logger, Request);
+            var recordingId = RecordingHandler.GetHeader(Request, "x-recording-id", allowNulls: true);
+
+            // parse all of them first, any exceptions should pop here
+            var workload = (await HttpRequestInteractions.GetBody<List<SanitizerBody>>(Request)).Select(s => (RecordedTestSanitizer)GetSanitizer(s.Name, s.Body)).ToList();
+
+            if (workload.Count == 0)
+            {
+                throw new HttpException(HttpStatusCode.BadRequest, "When bulk adding sanitizers, ensure there is at least one sanitizer added in each batch. Received 0 work items.");
+            }
+
+            // register them all
+            foreach(var sanitizer in workload)
+            {
+                if (recordingId != null)
+                {
+                    _recordingHandler.AddSanitizerToRecording(recordingId, sanitizer);
+                }
+                else
+                {
+                    _recordingHandler.Sanitizers.Add(sanitizer);
+                }
+            }
+
+        }
+
+
         [HttpPost]
         public async Task SetMatcher()
         {

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/HttpRequestInteractions.cs

@@ -4,6 +4,7 @@
 using Azure.Core;
 using Azure.Sdk.Tools.TestProxy.Common.Exceptions;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using System;
 using System.IO;
@@ -61,6 +62,32 @@ public static string GetBodyKey(JsonDocument document, string key, bool allowNul
             return value;
         }
 
+        public async static Task<T> GetBody<T>(HttpRequest req)
+        {
+            if (req.ContentLength > 0)
+            {
+                try
+                {
+                    using (var jsonDocument = await JsonDocument.ParseAsync(req.Body, options: new JsonDocumentOptions() { AllowTrailingCommas = true }))
+                    {
+                        return JsonSerializer.Deserialize<T>(jsonDocument.RootElement.GetRawText(), new JsonSerializerOptions() { });
+                    }
+
+                }
+                catch (Exception e)
+                {
+                    req.Body.Position = 0;
+                    using (StreamReader readstream = new StreamReader(req.Body, Encoding.UTF8))
+                    {
+                        string bodyContent = readstream.ReadToEnd();
+                        throw new HttpException(HttpStatusCode.BadRequest, $"The body of this request is invalid JSON. Content: {bodyContent}. Exception detail: {e.Message}");
+                    }
+                }
+            }
+
+            return default(T);
+        }
+
         public async static Task<JsonDocument> GetBody(HttpRequest req)
         {
             if (req.ContentLength > 0)

tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/SanitizerList.cs

@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+using System.Text.Json;
+
+namespace Azure.Sdk.Tools.TestProxy.Common
+{
+    public class SanitizerBody {
+        public string Name { get; set; }
+        public JsonDocument Body { get; set; }
+    }
+}

tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md

@@ -524,7 +524,7 @@ Add a more expansive Header sanitizer that uses a target group instead of filter
 
 ```jsonc
 // POST to URI <proxyURL>/Admin/AddSanitizer
-// dictionary dictionary
+// headers
 {
     "x-abstraction-identifier": "HeaderRegexSanitizer"
 }
@@ -549,6 +549,36 @@ Each sanitizer is optionally prefaced with the **specific part** of the request/
 
 A sanitizer that does _not_ include this prefix is something different, and probably applies at the session level instead on an individual request/response pair.
 
+#### Passing sanitizers in bulk
+
+In some cases, users need to register a lot (10+) of sanitizers. In this case, going back and forth with the proxy server individually is not very efficient. To ameliorate this, the proxy honors multiple sanitizers in the same request if the user utilizes `/Admin/AddSanitizers`.
+
+```jsonc
+// POST to URI <proxyURL>/Admin/AddSanitizers
+// note the request body is simply an array of objects
+[
+    {
+        "Name": "GeneralRegexSanitizer",
+        "Body": { 
+            "regex": "[a-zA-Z]?",
+            "value": "hello_there",
+            "condition": {
+                "UriRegex": ".+/Tables"
+            }
+        }
+    },
+    {
+        "Name": "HeaderRegexSanitizer",
+        "Body": { // <-- the contents of this property mirror what would be passed in the request body for individual AddSanitizer()
+            "key": "Location",
+            "value": "fakeaccount",
+            "regex": "https\\:\\/\\/(?<account>[a-z]+)\\.(?:table|blob|queue)\\.core\\.windows\\.net",
+            "groupForReplace": "account"
+        }
+    }
+]
+```
+
 ### For Sanitizers, Matchers, or Transforms in general
 
 When invoked as basic requests to the `Admin` controller, these settings will be applied to **all** further requests and responses. Both `Playback` and `Recording`. Where applicable.