diff --git a/benchmarks/TurnerSoftware.RobotsExclusionTools.Benchmarks/TurnerSoftware.RobotsExclusionTools.Benchmarks.csproj b/benchmarks/TurnerSoftware.RobotsExclusionTools.Benchmarks/TurnerSoftware.RobotsExclusionTools.Benchmarks.csproj index 9af491e..27ac77b 100644 --- a/benchmarks/TurnerSoftware.RobotsExclusionTools.Benchmarks/TurnerSoftware.RobotsExclusionTools.Benchmarks.csproj +++ b/benchmarks/TurnerSoftware.RobotsExclusionTools.Benchmarks/TurnerSoftware.RobotsExclusionTools.Benchmarks.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net9.0 false @@ -17,7 +17,7 @@ - + diff --git a/src/TurnerSoftware.RobotsExclusionTools/TurnerSoftware.RobotsExclusionTools.csproj b/src/TurnerSoftware.RobotsExclusionTools/TurnerSoftware.RobotsExclusionTools.csproj index 583d96b..54167cb 100644 --- a/src/TurnerSoftware.RobotsExclusionTools/TurnerSoftware.RobotsExclusionTools.csproj +++ b/src/TurnerSoftware.RobotsExclusionTools/TurnerSoftware.RobotsExclusionTools.csproj @@ -1,7 +1,7 @@  - netstandard2.0;netstandard2.1;net6.0 + netstandard2.0;netstandard2.1;net8.0;net9.0 TurnerSoftware.RobotsExclusionTools Robots.txt parsing and querying utility $(PackageBaseTags) @@ -10,9 +10,14 @@ - - - + + + + + + + + diff --git a/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/Controllers/RobotsController.cs b/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/Controllers/RobotsController.cs deleted file mode 100644 index 8658a7c..0000000 --- a/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/Controllers/RobotsController.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Net; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Mvc; - -namespace TurnerSoftware.RobotsExclusionTools.Tests.TestSite.Controllers -{ - [Route("/")] - public class RobotsController : ControllerBase - { - private SiteContext Context { get; } - - public RobotsController(SiteContext context) - { - Context = context; - } - - [Route("robots.txt")] - public IActionResult RobotsWithError() - { - return StatusCode(Context.StatusCode); - } - } -} diff --git a/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/Program.cs b/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/Program.cs new file mode 100644 index 0000000..e7a0b44 --- /dev/null +++ b/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/Program.cs @@ -0,0 +1,35 @@ +using System.IO; + +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; + +using TurnerSoftware.RobotsExclusionTools.Tests.TestSite; + + +var builder = WebApplication.CreateBuilder(args); + +var app = builder.Build(); + +app.UseRouting(); + + +app.MapGet("robots.txt", async (ctx) => +{ + var context = ctx.RequestServices.GetRequiredService(); + if (context.ExpectedStatusCode == 200) + { + ctx.Response.Headers.ContentType = new("text/plain"); + ctx.Response.StatusCode = 200; + await ctx.Response.WriteAsync("# This file is used by the Test Server\n# This allows testing loading by URI\nUser-agent: MyCustom-UserAgent\nDisallow: "); + } + else + { + ctx.Response.StatusCode = context.ExpectedStatusCode; + } + await ctx.Response.CompleteAsync(); + +}); + +app.Run(); +public partial class Program { } \ No newline at end of file diff --git a/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/SiteContext.cs b/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/SiteContext.cs index 573ec7a..868c051 100644 --- a/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/SiteContext.cs +++ b/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/SiteContext.cs @@ -1,11 +1,7 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace TurnerSoftware.RobotsExclusionTools.Tests.TestSite +namespace TurnerSoftware.RobotsExclusionTools.Tests.TestSite { - public class SiteContext + public class SiteContext(int expectedStatusCode) { - public int StatusCode { get; set; } + public int ExpectedStatusCode { get; set; } = expectedStatusCode; } } diff --git a/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/Startup.cs b/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/Startup.cs deleted file mode 100644 index c723b6d..0000000 --- a/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/Startup.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.IO; -using Microsoft.AspNetCore.Builder; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.FileProviders; - -namespace TurnerSoftware.RobotsExclusionTools.Tests.TestSite -{ - public class Startup - { - private SiteContext Context { get; } - - public Startup(SiteContext context) - { - Context = context; - } - - public void ConfigureServices(IServiceCollection services) - { - services.AddMvcCore(); - } - - public void Configure(IApplicationBuilder app) - { - if (Context.StatusCode == 200) - { - app.UseStaticFiles(new StaticFileOptions - { - FileProvider = new PhysicalFileProvider( - Path.Combine(Directory.GetCurrentDirectory(), "Resources")) - }); - } - else - { - app.UseMvc(); - } - } - } -} diff --git a/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/TestSiteManager.cs b/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/TestSiteManager.cs deleted file mode 100644 index 5029758..0000000 --- a/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/TestSiteManager.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Text; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.TestHost; -using Microsoft.Extensions.DependencyInjection; - -namespace TurnerSoftware.RobotsExclusionTools.Tests.TestSite -{ - public sealed class TestSiteManager : IDisposable - { - private TestServer Server { get; set; } - private HttpClient Client { get; set; } - - public TestSiteManager(SiteContext context) - { - var builder = new WebHostBuilder() - .ConfigureServices(s => - { - s.AddSingleton(context); - }) - .UseStartup(); - - Server = new TestServer(builder); - Client = Server.CreateClient(); - } - - public HttpClient GetHttpClient() - { - return Client; - } - - public void Dispose() - { - if (Server != null) - { - Server.Dispose(); - Server = null; - - Client.Dispose(); - Client = null; - } - } - } -} diff --git a/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/TurnerSoftware.RobotsExclusionTools.Tests.TestSite.csproj b/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/TurnerSoftware.RobotsExclusionTools.Tests.TestSite.csproj index cb407ec..a3d4725 100644 --- a/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/TurnerSoftware.RobotsExclusionTools.Tests.TestSite.csproj +++ b/tests/TurnerSoftware.RobotsExclusionTools.Tests.TestSite/TurnerSoftware.RobotsExclusionTools.Tests.TestSite.csproj @@ -1,25 +1,19 @@ - - + - netstandard2.0 - false - - - - - - - - - PreserveNewest - - - - - - - - - - - + net9.0 + false + + + + + + + PreserveNewest + + + + + + + + \ No newline at end of file diff --git a/tests/TurnerSoftware.RobotsExclusionTools.Tests/Helpers/RobotsExclusionProtocolHelperTests.cs b/tests/TurnerSoftware.RobotsExclusionTools.Tests/Helpers/RobotsExclusionProtocolHelperTests.cs index f4b15a9..f92c94b 100644 --- a/tests/TurnerSoftware.RobotsExclusionTools.Tests/Helpers/RobotsExclusionProtocolHelperTests.cs +++ b/tests/TurnerSoftware.RobotsExclusionTools.Tests/Helpers/RobotsExclusionProtocolHelperTests.cs @@ -1,5 +1,5 @@ -using System; -using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestTools.UnitTesting; + using TurnerSoftware.RobotsExclusionTools.Helpers; namespace TurnerSoftware.RobotsExclusionTools.Tests.Helpers; @@ -19,9 +19,9 @@ public class RobotsExclusionProtocolHelperTests [DataRow(9, new byte[] { 0xF2, 0xBF, 0xBF, 0x80 }, 4)] [DataRow(10, new byte[] { 0xF4, 0x8F, 0x80, 0x80 }, 4)] [DataTestMethod] - public void TryReadUtf8ByteSequenceTest(int _, object[] rawInput, int? expectedNumberOfBytes) + public void TryReadUtf8ByteSequenceTest(int _, byte[] rawInput, int? expectedNumberOfBytes) { - var bytes = Array.ConvertAll(rawInput, (input) => (byte)Convert.ToInt32(input)); + var bytes = rawInput; //Array.ConvertAll(rawInput, (input) => (byte)Convert.ToInt32(input)); var result = RobotsExclusionProtocolHelper.TryReadUtf8ByteSequence(bytes, out var actual); if (expectedNumberOfBytes is null) { diff --git a/tests/TurnerSoftware.RobotsExclusionTools.Tests/RobotsFile/RobotsFileParserTests.cs b/tests/TurnerSoftware.RobotsExclusionTools.Tests/RobotsFile/RobotsFileParserTests.cs index b7493ff..1281f3b 100644 --- a/tests/TurnerSoftware.RobotsExclusionTools.Tests/RobotsFile/RobotsFileParserTests.cs +++ b/tests/TurnerSoftware.RobotsExclusionTools.Tests/RobotsFile/RobotsFileParserTests.cs @@ -1,28 +1,31 @@ using System; using System.Linq; using System.Threading.Tasks; + +using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.VisualStudio.TestTools.UnitTesting; -using TurnerSoftware.RobotsExclusionTools.Tests.TestSite; namespace TurnerSoftware.RobotsExclusionTools.Tests.RobotsFile { [TestClass] public class RobotsFileParserTests : TestBase { - private static TestSiteManager GetRobotsSiteManager(int statusCode) + private static WebApplicationFactory GetTestSiteFactory(int statusCode) { - return new TestSiteManager(new SiteContext { StatusCode = statusCode }); + var factory = new TestWebApplicationFactory(statusCode); + + return factory; } [TestMethod] public async Task FromUriLoading_200_OK_PerFileRules() { - using (var siteManager = GetRobotsSiteManager(200)) + using (var factory = GetTestSiteFactory(200)) { - var client = siteManager.GetHttpClient(); + var client = factory.CreateClient(); var robotsFile = await new RobotsFileParser(client).FromUriAsync(new Uri("http://localhost/robots.txt")); Assert.IsTrue(robotsFile.SiteAccessEntries.Any(s => - s.UserAgents.Contains("MyCustom-UserAgent") && + s.UserAgents.Contains("MyCustom-UserAgent") && s.PathRules.Any(p => string.IsNullOrEmpty(p.Path) && p.RuleType == PathRuleType.Disallow) )); } @@ -31,9 +34,9 @@ public async Task FromUriLoading_200_OK_PerFileRules() [TestMethod] public async Task FromUriLoading_AccessRules_404_NotFound_AllowAll() { - using (var siteManager = GetRobotsSiteManager(404)) + using (var webFactory = GetTestSiteFactory(404)) { - var client = siteManager.GetHttpClient(); + var client = webFactory.CreateClient(); var robotsFile = await new RobotsFileParser(client).FromUriAsync(new Uri("http://localhost/robots.txt"), new RobotsFileAccessRules { AllowAllWhen404NotFound = true @@ -44,9 +47,9 @@ public async Task FromUriLoading_AccessRules_404_NotFound_AllowAll() [TestMethod] public async Task FromUriLoading_AccessRules_404_NotFound_DenyAll() { - using (var siteManager = GetRobotsSiteManager(404)) + using (var webFactory = GetTestSiteFactory(404)) { - var client = siteManager.GetHttpClient(); + var client = webFactory.CreateClient(); var robotsFile = await new RobotsFileParser(client).FromUriAsync(new Uri("http://localhost/robots.txt"), new RobotsFileAccessRules { AllowAllWhen404NotFound = false @@ -60,9 +63,9 @@ public async Task FromUriLoading_AccessRules_404_NotFound_DenyAll() [TestMethod] public async Task FromUriLoading_AccessRules_401_Unauthorized_AllowAll() { - using (var siteManager = GetRobotsSiteManager(401)) + using (var webFactory = GetTestSiteFactory(401)) { - var client = siteManager.GetHttpClient(); + var client = webFactory.CreateClient(); var robotsFile = await new RobotsFileParser(client).FromUriAsync(new Uri("http://localhost/robots.txt"), new RobotsFileAccessRules { AllowAllWhen401Unauthorized = true @@ -73,9 +76,9 @@ public async Task FromUriLoading_AccessRules_401_Unauthorized_AllowAll() [TestMethod] public async Task FromUriLoading_AccessRules_401_Unauthorized_DenyAll() { - using (var siteManager = GetRobotsSiteManager(401)) + using (var webFactory = GetTestSiteFactory(401)) { - var client = siteManager.GetHttpClient(); + var client = webFactory.CreateClient(); var robotsFile = await new RobotsFileParser(client).FromUriAsync(new Uri("http://localhost/robots.txt"), new RobotsFileAccessRules { AllowAllWhen401Unauthorized = false @@ -89,9 +92,9 @@ public async Task FromUriLoading_AccessRules_401_Unauthorized_DenyAll() [TestMethod] public async Task FromUriLoading_AccessRules_403_Forbidden_AllowAll() { - using (var siteManager = GetRobotsSiteManager(403)) + using (var webFactory = GetTestSiteFactory(403)) { - var client = siteManager.GetHttpClient(); + var client = webFactory.CreateClient(); var robotsFile = await new RobotsFileParser(client).FromUriAsync(new Uri("http://localhost/robots.txt"), new RobotsFileAccessRules { AllowAllWhen403Forbidden = true @@ -102,9 +105,9 @@ public async Task FromUriLoading_AccessRules_403_Forbidden_AllowAll() [TestMethod] public async Task FromUriLoading_AccessRules_403_Forbidden_DenyAll() { - using (var siteManager = GetRobotsSiteManager(403)) + using (var webFactory = GetTestSiteFactory(403)) { - var client = siteManager.GetHttpClient(); + var client = webFactory.CreateClient(); var robotsFile = await new RobotsFileParser(client).FromUriAsync(new Uri("http://localhost/robots.txt"), new RobotsFileAccessRules { AllowAllWhen403Forbidden = false @@ -118,9 +121,9 @@ public async Task FromUriLoading_AccessRules_403_Forbidden_DenyAll() [TestMethod] public async Task FromUriLoading_OtherHttpStatus_AllowAll() { - using (var siteManager = GetRobotsSiteManager(418)) + using (var webFactory = GetTestSiteFactory(418)) { - var client = siteManager.GetHttpClient(); + var client = webFactory.CreateClient(); var robotsFile = await new RobotsFileParser(client).FromUriAsync(new Uri("http://localhost/robots.txt")); Assert.IsFalse(robotsFile.SiteAccessEntries.Any()); } @@ -129,9 +132,9 @@ public async Task FromUriLoading_OtherHttpStatus_AllowAll() [TestMethod] public async Task FromUriLoading_DefaultNoRobotsRFCRules() { - using (var siteManager = GetRobotsSiteManager(401)) + using (var webFactory = GetTestSiteFactory(401)) { - var client = siteManager.GetHttpClient(); + var client = webFactory.CreateClient(); var robotsFile = await new RobotsFileParser(client).FromUriAsync(new Uri("http://localhost/robots.txt")); Assert.IsTrue(robotsFile.SiteAccessEntries.Any(s => s.UserAgents.Contains("*") && s.PathRules.Any(p => p.Path == "/" && p.RuleType == PathRuleType.Disallow) diff --git a/tests/TurnerSoftware.RobotsExclusionTools.Tests/TestWebApplicationFactory.cs b/tests/TurnerSoftware.RobotsExclusionTools.Tests/TestWebApplicationFactory.cs new file mode 100644 index 0000000..c2139a9 --- /dev/null +++ b/tests/TurnerSoftware.RobotsExclusionTools.Tests/TestWebApplicationFactory.cs @@ -0,0 +1,21 @@ +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; + +using TurnerSoftware.RobotsExclusionTools.Tests.TestSite; + +namespace TurnerSoftware.RobotsExclusionTools.Tests +{ + internal class TestWebApplicationFactory(int expectedStatusCode) : WebApplicationFactory + where TProgram : class + { + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.ConfigureServices(services => + { + services.Add( + new Microsoft.Extensions.DependencyInjection.ServiceDescriptor(typeof(SiteContext), (sp) => new SiteContext(expectedStatusCode), Microsoft.Extensions.DependencyInjection.ServiceLifetime.Singleton) + ); + }); + } + } +} diff --git a/tests/TurnerSoftware.RobotsExclusionTools.Tests/TurnerSoftware.RobotsExclusionTools.Tests.csproj b/tests/TurnerSoftware.RobotsExclusionTools.Tests/TurnerSoftware.RobotsExclusionTools.Tests.csproj index 2185a0f..402bc62 100644 --- a/tests/TurnerSoftware.RobotsExclusionTools.Tests/TurnerSoftware.RobotsExclusionTools.Tests.csproj +++ b/tests/TurnerSoftware.RobotsExclusionTools.Tests/TurnerSoftware.RobotsExclusionTools.Tests.csproj @@ -1,7 +1,7 @@  - net461;netcoreapp3.1;net6.0 + net9.0 Latest false @@ -21,16 +21,15 @@ - - - - - - + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - +