feat: Add masstransit outbox system (#1277)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Added a new constructor for initializing validation options by
scanning assemblies.
- Introduced interfaces and a class for configuring publish-subscribe
capabilities.
- Enhanced infrastructure settings with new MassTransit configuration
options.
- Implemented notification processing context management for better
resource handling.
- Added idempotency management for domain events to prevent duplicate
processing.
  - Introduced new configuration options for local development settings.

- **Bug Fixes**
- Improved reliability of event consumption assertions in integration
tests.

- **Tests**
- Enhanced integration tests for cloud events related to dialog
operations.

- **Documentation**
- Updated local development setup instructions and configuration
settings in the README.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Ole Jørgen Skogstad <skogstad@softis.net>

Author: MagnusSandgren

.azure/applications/service/main.bicep

@@ -82,6 +82,10 @@ var containerAppEnvVars = [
     name: 'AZURE_CLIENT_ID'
     value: managedIdentity.properties.clientId
   }
+  {
+    name: 'Infrastructure__MassTransit__Host'
+    value: 'sb://${serviceBusNamespaceName}.servicebus.windows.net/'
+  }
 ]
 
 resource environmentKeyVaultResource 'Microsoft.KeyVault/vaults@2023-07-01' existing = {
@@ -152,9 +156,7 @@ module containerApp '../../modules/containerApp/main.bicep' = {
   name: containerAppName
   params: {
     name: containerAppName
-    // todo: make this dynamic based on service name. Using webapi for now.
-    // image: '${baseImageUrl}${serviceName}:${imageTag}'
-    image: '${baseImageUrl}webapi:${imageTag}'
+    image: '${baseImageUrl}${serviceName}:${imageTag}'
     location: location
     envVariables: containerAppEnvVars
     containerAppEnvId: containerAppEnvironment.id

.azure/applications/web-api-so/main.bicep

@@ -75,10 +75,6 @@ var containerAppEnvVars = [
     name: 'ASPNETCORE_URLS'
     value: 'http://+:8080'
   }
-  {
-    name: 'RUN_OUTBOX_SCHEDULER'
-    value: 'true'
-  }
 ]
 
 resource environmentKeyVaultResource 'Microsoft.KeyVault/vaults@2023-07-01' existing = {

README.md

@@ -126,16 +126,16 @@ To generate test tokens, see https://github.com/Altinn/AltinnTestTools. There is
 We are able to toggle some external resources in local development. This is done through the `appsettings.Development.json` file. The following settings are available:
 ```json
 "LocalDevelopment": {
-  "UseLocalDevelopmentUser": true,
-  "UseLocalDevelopmentResourceRegister": true,
-  "UseLocalDevelopmentOrganizationRegister": true,
-  "UseLocalDevelopmentNameRegister": true,
-  "UseLocalDevelopmentAltinnAuthorization": true,
-  "UseLocalDevelopmentCloudEventBus": true,
-  "UseLocalDevelopmentCompactJwsGenerator": true,
-  "DisableShortCircuitOutboxDispatcher": true,
-  "DisableCache": false,
-  "DisableAuth": true
+    "UseLocalDevelopmentUser": true,
+    "UseLocalDevelopmentResourceRegister": true,
+    "UseLocalDevelopmentOrganizationRegister": true,
+    "UseLocalDevelopmentNameRegister": true,
+    "UseLocalDevelopmentAltinnAuthorization": true,
+    "UseLocalDevelopmentCloudEventBus": true,
+    "UseLocalDevelopmentCompactJwsGenerator": true,
+    "DisableCache": true,
+    "DisableAuth": true,
+    "UseInMemoryServiceBusTransport": true
 }
 ```
 Toggling these flags will enable/disable the external resources. The `DisableAuth` flag, for example, will disable authentication in the WebAPI project. This is useful when debugging the WebAPI project in an IDE. These settings will only be respected in the `Development` environment.

src/Digdir.Domain.Dialogporten.Application/ApplicationAssemblyMarker.cs

@@ -2,7 +2,7 @@
 
 namespace Digdir.Domain.Dialogporten.Application;
 
-public static class ApplicationAssemblyMarker
+public sealed class ApplicationAssemblyMarker
 {
     public static readonly Assembly Assembly = typeof(ApplicationAssemblyMarker).Assembly;
 }

src/Digdir.Domain.Dialogporten.Application/ApplicationExtensions.cs

@@ -9,7 +9,7 @@
 using Microsoft.Extensions.Hosting;
 using System.Reflection;
 using Digdir.Domain.Dialogporten.Application.Common.Authorization;
-using Digdir.Domain.Dialogporten.Application.Features.V1.Common.Content;
+using MediatR.NotificationPublishers;
 
 namespace Digdir.Domain.Dialogporten.Application;
 
@@ -34,7 +34,12 @@ public static IServiceCollection AddApplication(this IServiceCollection services
         services
             // Framework
             .AddAutoMapper(thisAssembly)
-            .AddMediatR(x => x.RegisterServicesFromAssembly(thisAssembly))
+            .AddMediatR(x =>
+            {
+                x.RegisterServicesFromAssembly(thisAssembly);
+                x.TypeEvaluator = type => !type.IsAssignableTo(typeof(IIgnoreOnAssemblyScan));
+                x.NotificationPublisherType = typeof(TaskWhenAllPublisher);
+            })
             .AddValidatorsFromAssembly(thisAssembly, ServiceLifetime.Transient, includeInternalTypes: true,
                 filter: type => !type.ValidatorType.IsAssignableTo(typeof(IIgnoreOnAssemblyScan)))
 

src/Digdir.Domain.Dialogporten.Application/Common/Extensions/OptionExtensions/FluentValidationOptions.cs

@@ -1,11 +1,14 @@
-using FluentValidation;
+using System.Diagnostics;
+using System.Reflection;
+using FluentValidation;
 using Microsoft.Extensions.Options;
 
 namespace Digdir.Domain.Dialogporten.Application.Common.Extensions.OptionExtensions;
 
 public sealed class FluentValidationOptions<TOptions> : IValidateOptions<TOptions>
     where TOptions : class
 {
+    private static readonly Type OptionType = typeof(TOptions);
     private readonly IEnumerable<IValidator<TOptions>> _validators;
     public string? Name { get; }
 
@@ -15,6 +18,26 @@ public FluentValidationOptions(string? name, IEnumerable<IValidator<TOptions>> v
         _validators = validators;
     }
 
+    /// <summary>
+    /// Initializes a new instance of the <see cref="FluentValidationOptions{TOptions}"/> class
+    /// by scanning the specified assemblies for validators.
+    /// </summary>
+    /// <param name="assemblies">The assemblies to scan for validators.</param>
+    /// <remarks>
+    /// This constructor scans the provided assemblies for types that implement the 
+    /// <see cref="IValidator{T}"/> interface for the specified <typeparamref name="TOptions"/> type.
+    /// It includes both public and internal validators in the search. <b>Use this constructor sparingly
+    /// as it uses reflection to find validators.</b>
+    /// </remarks>
+    public FluentValidationOptions(params Assembly[] assemblies)
+    {
+        _validators = AssemblyScanner
+            .FindValidatorsInAssemblies(assemblies, includeInternalTypes: true)
+            .Where(x => x.InterfaceType.GenericTypeArguments.First() == OptionType)
+            .Select(x => (IValidator<TOptions>)Activator.CreateInstance(x.ValidatorType, nonPublic: true)! ?? throw new UnreachableException())
+            .ToList();
+    }
+
     public ValidateOptionsResult Validate(string? name, TOptions options)
     {
         // Null name is used to configure all named options.

src/Digdir.Domain.Dialogporten.Application/Common/IIgnoreOnAssemblyScan.cs

@@ -0,0 +1,6 @@
+namespace Digdir.Domain.Dialogporten.Application.Common;
+
+/// <summary>
+/// Marker interface to indicate that a class should be ignored during assembly scanning.
+/// </summary>
+public interface IIgnoreOnAssemblyScan;

src/Digdir.Domain.Dialogporten.Application/Externals/IDialogDbContext.cs

@@ -1,7 +1,6 @@
 using Digdir.Domain.Dialogporten.Domain.Dialogs.Entities;
 using Digdir.Domain.Dialogporten.Domain.Dialogs.Entities.Actions;
 using Digdir.Domain.Dialogporten.Domain.Dialogs.Entities.Activities;
-using Digdir.Domain.Dialogporten.Domain.Outboxes;
 using Digdir.Library.Entity.Abstractions.Features.Identifiable;
 using Microsoft.EntityFrameworkCore;
 using System.Linq.Expressions;
@@ -40,8 +39,6 @@ public interface IDialogDbContext
     DbSet<DialogContent> DialogContents { get; }
     DbSet<DialogContentType> DialogContentTypes { get; }
 
-    DbSet<OutboxMessage> OutboxMessages { get; }
-    DbSet<OutboxMessageConsumer> OutboxMessageConsumers { get; }
     DbSet<SubjectResource> SubjectResources { get; }
     DbSet<DialogEndUserContext> DialogEndUserContexts { get; }
     DbSet<LabelAssignmentLog> LabelAssignmentLogs { get; }

src/Digdir.Domain.Dialogporten.Application/Features/V1/Common/Content/ContentValueDtoValidator.cs

@@ -1,5 +1,6 @@
-using System.Diagnostics.CodeAnalysis;
 using Digdir.Domain.Dialogporten.Application.Common.Authorization;
+using System.Diagnostics.CodeAnalysis;
+using Digdir.Domain.Dialogporten.Application.Common;
 using Digdir.Domain.Dialogporten.Application.Common.Extensions;
 using Digdir.Domain.Dialogporten.Application.Common.Extensions.FluentValidation;
 using Digdir.Domain.Dialogporten.Application.Externals.Presentation;
@@ -12,13 +13,10 @@
 namespace Digdir.Domain.Dialogporten.Application.Features.V1.Common.Content;
 
 // DialogContentValueDtoValidator has constructor parameter input, and can't be registered in DI assembly scan
-// This interface is used to ignore the class when scanning for validators
+// IIgnoreOnAssemblyScan is used to ignore the class when scanning for validators
 // The validator is manually created in the Create and Update validators
-internal interface IIgnoreOnAssemblyScan;
-
 internal sealed class ContentValueDtoValidator : AbstractValidator<ContentValueDto>, IIgnoreOnAssemblyScan
 {
-
     public ContentValueDtoValidator(DialogTransmissionContentType contentType)
     {
         RuleFor(x => x.MediaType)

src/Digdir.Domain.Dialogporten.Application/Features/V1/Outboxes/Commands/Delete/DeleteOutboxMessagesCommand.cs

@@ -9,12 +9,12 @@ public sealed class LocalDevelopmentSettings
     public bool UseLocalDevelopmentResourceRegister { get; init; } = true;
     public bool UseLocalDevelopmentAltinnAuthorization { get; init; } = true;
     public bool UseLocalDevelopmentCloudEventBus { get; init; } = true;
-    public bool DisableShortCircuitOutboxDispatcher { get; init; } = true;
     public bool DisableCache { get; init; } = true;
     public bool DisableAuth { get; init; } = true;
     public bool UseLocalDevelopmentNameRegister { get; set; } = true;
     public bool UseLocalDevelopmentOrganizationRegister { get; set; } = true;
     public bool UseLocalDevelopmentCompactJwsGenerator { get; set; } = true;
+    public bool UseInMemoryServiceBusTransport { get; set; } = true;
 }
 
 public static class LocalDevelopmentSettingsExtensions

src/Digdir.Domain.Dialogporten.Application/LocalDevelopmentSettings.cs

@@ -2,7 +2,7 @@
 
 namespace Digdir.Domain.Dialogporten.ChangeDataCapture;
 
-public static class ChangeDataCaptureAssemblyMarker
+public sealed class ChangeDataCaptureAssemblyMarker
 {
     public static readonly Assembly Assembly = typeof(ChangeDataCaptureAssemblyMarker).Assembly;
 }

src/Digdir.Domain.Dialogporten.ChangeDataCapture/ChangeDataCaptureAssemblyMarker.cs

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
     <ItemGroup>
-        <PackageReference Include="Azure.Identity" Version="1.12.1"/>
+        <PackageReference Include="Azure.Identity" Version="1.13.0" />
         <PackageReference Include="MassTransit" Version="8.2.5"/>
         <PackageReference Include="Npgsql" Version="8.0.5" />
         <PackageReference Include="Microsoft.Azure.AppConfiguration.AspNetCore" Version="8.0.0" />

src/Digdir.Domain.Dialogporten.ChangeDataCapture/Digdir.Domain.Dialogporten.ChangeDataCapture.csproj

@@ -1,5 +1,5 @@
 using System.Text.Json.Serialization;
-using Digdir.Library.Entity.Abstractions.Features.EventPublisher;
+using Digdir.Domain.Dialogporten.Domain.Common.EventPublisher;
 
 namespace Digdir.Domain.Dialogporten.Domain.Common;
 

src/Digdir.Domain.Dialogporten.Domain/Common/DomainEvent.cs

@@ -1,6 +1,6 @@
 using MediatR;
 
-namespace Digdir.Library.Entity.Abstractions.Features.EventPublisher;
+namespace Digdir.Domain.Dialogporten.Domain.Common.EventPublisher;
 
 /// <summary>
 /// Represents a domain event.

src/Digdir.Library.Entity.Abstractions/Features/EventPublisher/IDomainEvent.cs src/Digdir.Domain.Dialogporten.Domain/Common/EventPublisher/IDomainEvent.cs

@@ -1,4 +1,4 @@
-namespace Digdir.Library.Entity.Abstractions.Features.EventPublisher;
+namespace Digdir.Domain.Dialogporten.Domain.Common.EventPublisher;
 
 /// <summary>
 /// Abstraction representing functionality to publish domain events.

src/Digdir.Library.Entity.Abstractions/Features/EventPublisher/IEventPublisher.cs src/Digdir.Domain.Dialogporten.Domain/Common/EventPublisher/IEventPublisher.cs

@@ -1,9 +1,9 @@
 using Digdir.Domain.Dialogporten.Domain.Actors;
+using Digdir.Domain.Dialogporten.Domain.Common.EventPublisher;
 using Digdir.Domain.Dialogporten.Domain.Dialogs.Entities.Transmissions;
 using Digdir.Domain.Dialogporten.Domain.Dialogs.Events.Activities;
 using Digdir.Domain.Dialogporten.Domain.Localizations;
 using Digdir.Library.Entity.Abstractions.Features.Aggregate;
-using Digdir.Library.Entity.Abstractions.Features.EventPublisher;
 using Digdir.Library.Entity.Abstractions.Features.Immutable;
 
 namespace Digdir.Domain.Dialogporten.Domain.Dialogs.Entities.Activities;

src/Digdir.Domain.Dialogporten.Domain/Dialogs/Entities/Activities/DialogActivity.cs

@@ -1,5 +1,6 @@
 using Digdir.Domain.Dialogporten.Domain.Actors;
 using Digdir.Domain.Dialogporten.Domain.Attachments;
+using Digdir.Domain.Dialogporten.Domain.Common.EventPublisher;
 using Digdir.Domain.Dialogporten.Domain.DialogEndUserContexts.Entities;
 using Digdir.Domain.Dialogporten.Domain.Dialogs.Entities.Actions;
 using Digdir.Domain.Dialogporten.Domain.Dialogs.Entities.Activities;
@@ -8,7 +9,6 @@
 using Digdir.Domain.Dialogporten.Domain.Dialogs.Events;
 using Digdir.Library.Entity.Abstractions;
 using Digdir.Library.Entity.Abstractions.Features.Aggregate;
-using Digdir.Library.Entity.Abstractions.Features.EventPublisher;
 using Digdir.Library.Entity.Abstractions.Features.SoftDeletable;
 using Digdir.Library.Entity.Abstractions.Features.Versionable;
 

src/Digdir.Domain.Dialogporten.Domain/Dialogs/Entities/DialogEntity.cs

@@ -2,7 +2,7 @@
 
 namespace Digdir.Domain.Dialogporten.Domain;
 
-public static class DomainAssemblyMarker
+public sealed class DomainAssemblyMarker
 {
     public static readonly Assembly Assembly = typeof(DomainAssemblyMarker).Assembly;
 }

src/Digdir.Domain.Dialogporten.Domain/DomainAssemblyMarker.cs

@@ -0,0 +1,12 @@
+using Digdir.Domain.Dialogporten.Domain.Common.EventPublisher;
+
+namespace Digdir.Domain.Dialogporten.Domain;
+
+public static class DomainExtensions
+{
+    public static IEnumerable<Type> GetDomainEventTypes()
+        => DomainAssemblyMarker.Assembly
+            .GetTypes()
+            .Where(x => !x.IsAbstract && !x.IsInterface && !x.IsGenericType)
+            .Where(x => x.IsAssignableTo(typeof(IDomainEvent)));
+}

src/Digdir.Domain.Dialogporten.Domain/DomainExtensions.cs

@@ -1,5 +1,5 @@
 using System.Text.Json;
-using Digdir.Library.Entity.Abstractions.Features.EventPublisher;
+using Digdir.Domain.Dialogporten.Domain.Common.EventPublisher;
 
 namespace Digdir.Domain.Dialogporten.Domain.Outboxes;