diff --git a/.gitignore b/.gitignore index a5eddaf2..da9560b0 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ bld/ coverage.json coverage.info .vs +[Tt]est[Rr]esults/ # Docs _site diff --git a/docs/docs/operator/advanced-configuration.mdx b/docs/docs/operator/advanced-configuration.mdx new file mode 100644 index 00000000..59f2c37f --- /dev/null +++ b/docs/docs/operator/advanced-configuration.mdx @@ -0,0 +1,591 @@ +--- +title: Advanced Configuration +description: Advanced Operator Configuration Options +sidebar_position: 8 +--- + +# Advanced Configuration + +This guide covers advanced configuration options for KubeOps operators, including finalizer management, custom leader election, and durable requeue mechanisms. + +## Finalizer Management + +KubeOps provides automatic finalizer attachment and detachment to ensure proper resource cleanup. These features can be configured through `OperatorSettings`. + +### Auto-Attach Finalizers + +By default, KubeOps automatically attaches finalizers to entities during reconciliation. This ensures that cleanup operations are performed before resources are deleted. + +```csharp +builder.Services + .AddKubernetesOperator(settings => + { + // Enable automatic finalizer attachment (default: true) + settings.AutoAttachFinalizers = true; + }); +``` + +When `AutoAttachFinalizers` is enabled: +- Finalizers are automatically added to entities during reconciliation +- You don't need to manually call the `EntityFinalizerAttacher` delegate +- All registered finalizers for an entity type are automatically attached + +When disabled: +```csharp +settings.AutoAttachFinalizers = false; +``` + +You must manually attach finalizers in your controller: + +```csharp +public class V1DemoEntityController( + ILogger logger, + EntityFinalizerAttacher finalizerAttacher) + : IEntityController +{ + public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) + { + // Manually attach finalizer + entity = await finalizerAttacher(entity, cancellationToken); + + // Continue with reconciliation logic + logger.LogInformation("Reconciling entity {Entity}", entity); + return ReconciliationResult.Success(entity); + } + + public Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) + { + return Task.FromResult(ReconciliationResult.Success(entity)); + } +} +``` + +### Auto-Detach Finalizers + +KubeOps automatically removes finalizers after successful finalization. This can also be configured: + +```csharp +builder.Services + .AddKubernetesOperator(settings => + { + // Enable automatic finalizer removal (default: true) + settings.AutoDetachFinalizers = true; + }); +``` + +When `AutoDetachFinalizers` is enabled: +- Finalizers are automatically removed when `FinalizeAsync` returns success + +When disabled: +```csharp +settings.AutoDetachFinalizers = false; +``` + +You must manually manage finalizer removal, which is typically not recommended unless you have specific requirements. + +### Use Cases + +**Keep defaults enabled** when: +- You want standard finalizer behavior +- Your finalizers follow the typical pattern +- You don't need fine-grained control + +**Disable auto-attach** when: +- You need conditional finalizer attachment +- Different instances should have different finalizers +- You want to attach finalizers based on specific conditions + +**Disable auto-detach** when: +- You need custom finalizer removal logic +- You want to coordinate multiple finalizers manually +- You have external systems that need to confirm cleanup + +## Custom Leader Election + +KubeOps supports different leader election mechanisms through the `LeaderElectionType` setting. This allows you to control how multiple operator instances coordinate in a cluster. + +### Leader Election Types + +```csharp +public enum LeaderElectionType +{ + None = 0, // No leader election - all instances process events + Single = 1, // Single leader election - only one instance processes events + Custom = 2 // Custom implementation - user-defined coordination +} +``` + +### Configuration + +```csharp +builder.Services + .AddKubernetesOperator(settings => + { + settings.LeaderElectionType = LeaderElectionType.Single; + settings.LeaderElectionLeaseDuration = TimeSpan.FromSeconds(15); + settings.LeaderElectionRenewDeadline = TimeSpan.FromSeconds(10); + settings.LeaderElectionRetryPeriod = TimeSpan.FromSeconds(2); + }); +``` + +### Custom Leader Election + +The `Custom` leader election type allows you to implement your own coordination logic, such as namespace-based leader election. + +#### Example: Namespace-Based Leader Election + +In some scenarios, you may want different operator instances to handle different namespaces. This enables horizontal scaling while maintaining isolation. + +**Step 1: Implement a custom ResourceWatcher** + +```csharp +public sealed class NamespacedLeaderElectionResourceWatcher( + ActivitySource activitySource, + ILogger> logger, + IReconciler reconciler, + OperatorSettings settings, + IEntityLabelSelector labelSelector, + IKubernetesClient client, + INamespaceLeadershipManager namespaceLeadershipManager) + : ResourceWatcher( + activitySource, + logger, + reconciler, + settings, + labelSelector, + client) + where TEntity : IKubernetesObject +{ + protected override async Task> OnEventAsync( + WatchEventType eventType, + TEntity entity, + CancellationToken cancellationToken) + { + // Check if this instance is responsible for the entity's namespace + if (!await namespaceLeadershipManager.IsResponsibleForNamespace( + entity.Namespace(), + cancellationToken)) + { + // Skip processing - another instance handles this namespace + return ReconciliationResult.Success(entity); + } + + // Process the event + return await base.OnEventAsync(eventType, entity, cancellationToken); + } +} +``` + +**Step 2: Implement the leadership manager** + +```csharp +public interface INamespaceLeadershipManager +{ + Task IsResponsibleForNamespace(string @namespace, CancellationToken cancellationToken); +} + +public class NamespaceLeadershipManager : INamespaceLeadershipManager +{ + private readonly ILeaderElector _leaderElector; + private readonly ConcurrentDictionary _namespaceResponsibility = new(); + + public async Task IsResponsibleForNamespace( + string @namespace, + CancellationToken cancellationToken) + { + // Implement your logic here: + // - Consistent hashing of namespace names + // - Lease-based namespace assignment + // - External coordination service (e.g., etcd, Consul) + + return _namespaceResponsibility.GetOrAdd( + @namespace, + ns => CalculateResponsibility(ns)); + } + + private bool CalculateResponsibility(string @namespace) + { + // Example: Simple hash-based distribution + var instanceId = Environment.GetEnvironmentVariable("POD_NAME") ?? "instance-0"; + var instanceCount = int.Parse( + Environment.GetEnvironmentVariable("REPLICA_COUNT") ?? "1"); + + var namespaceHash = @namespace.GetHashCode(); + var assignedInstance = Math.Abs(namespaceHash % instanceCount); + var currentInstance = int.Parse(instanceId.Split('-').Last()); + + return assignedInstance == currentInstance; + } +} +``` + +**Step 3: Register the custom watcher** + +```csharp +builder.Services + .AddKubernetesOperator(settings => + { + settings.LeaderElectionType = LeaderElectionType.Custom; + }) + .AddSingleton() + .AddHostedService>(); +``` + +### Benefits of Custom Leader Election + +- **Horizontal Scaling**: Multiple instances can process different subsets of resources +- **Namespace Isolation**: Different teams or environments can have dedicated operator instances +- **Geographic Distribution**: Route requests to instances in specific regions +- **Load Balancing**: Distribute work across multiple instances + +## Custom Requeue Mechanism + +By default, KubeOps uses an in-memory queue for requeuing entities. This queue is volatile and does not survive operator restarts. For production scenarios, you may want to implement a durable queue. + +### Default Behavior + +The default `ITimedEntityQueue` implementation: +- Stores requeue entries in memory +- Processes them after the specified delay +- Loses pending requeues on operator restart + +### Implementing a Durable Queue + +You can implement `ITimedEntityQueue` to use external queue systems like Azure Service Bus, RabbitMQ, or AWS SQS. + +#### Example: Azure Service Bus Integration + +**Step 1: Implement ITimedEntityQueue** + +```csharp +public sealed class DurableTimedEntityQueue( + ServiceBusClient serviceBusClient, + IEntityRequeueQueueNameProvider queueNameProvider, + TimeProvider timeProvider) + : ITimedEntityQueue + where TEntity : IKubernetesObject +{ + private readonly ServiceBusSender _sender = serviceBusClient.CreateSender( + queueNameProvider.GetRequeueQueueName()); + + public async Task Enqueue( + TEntity entity, + RequeueType type, + TimeSpan requeueIn, + CancellationToken cancellationToken) + { + var entry = new RequeueEntry() { Entity = entity, RequeueType = type }; + var message = new ServiceBusMessage(BinaryData.FromObjectAsJson(entry)); + + // Schedule the message for future delivery + await _sender.ScheduleMessageAsync( + message, + timeProvider.GetUtcNow().Add(requeueIn), + cancellationToken); + } + + public Task Remove(TEntity entity, CancellationToken cancellationToken) + { + // will be automatically removed when the message is processed + return Task.CompletedTask; + } + + public async IAsyncEnumerator> GetAsyncEnumerator( + CancellationToken cancellationToken = default) + { + // Not used with external queue - processing happens via the processor + await Task.CompletedTask; + yield break; + } + + public void Dispose() + { + _sender.DisposeAsync().AsTask().GetAwaiter().GetResult(); + } +} +``` + +**Step 2: Implement a background service to process messages** + +```csharp +public sealed class DurableEntityRequeueBackgroundService( + ServiceBusClient serviceBusClient, + IKubernetesClient kubernetesClient, + IReconciler reconciler, + IEntityRequeueQueueNameProvider queueNameProvider, + ILogger> logger) + : BackgroundService + where TEntity : IKubernetesObject +{ + private readonly ServiceBusProcessor _processor = serviceBusClient.CreateProcessor( + queueNameProvider.GetRequeueQueueName(), + new ServiceBusProcessorOptions + { + MaxConcurrentCalls = 1, + AutoCompleteMessages = false + }); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _processor.ProcessMessageAsync += ProcessMessageAsync; + _processor.ProcessErrorAsync += ProcessErrorAsync; + + await _processor.StartProcessingAsync(stoppingToken); + + try + { + await Task.Delay(Timeout.Infinite, stoppingToken); + } + catch (OperationCanceledException) + { + // Expected during shutdown + } + } + + private async Task ProcessMessageAsync(ProcessMessageEventArgs args) + { + var entry = args.Message.Body.ToObjectFromJson>(); + + // Verify entity still exists + var entity = await kubernetesClient.GetAsync( + entry.Entity.Name(), + entry.Entity.Namespace(), + args.CancellationToken); + + if (entity == null) + { + logger.LogInformation( + "Skipping reconciliation for deleted entity {Name}", + entry.Entity.Name()); + await args.CompleteMessageAsync(args.Message, args.CancellationToken); + return; + } + + // Complete message before reconciliation to avoid reprocessing + await args.CompleteMessageAsync(args.Message, args.CancellationToken); + + // Trigger reconciliation + await reconciler.Reconcile( + ReconciliationContext.CreateFromOperatorEvent( + entity, + entry.RequeueType.ToWatchEventType()), + args.CancellationToken); + } + + private Task ProcessErrorAsync(ProcessErrorEventArgs args) + { + logger.LogError(args.Exception, "Error processing requeue message"); + return Task.CompletedTask; + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + await _processor.StopProcessingAsync(cancellationToken); + _processor.ProcessMessageAsync -= ProcessMessageAsync; + _processor.ProcessErrorAsync -= ProcessErrorAsync; + await _processor.DisposeAsync(); + await base.StopAsync(cancellationToken); + } +} +``` + +**Step 3: Register the durable queue** + +```csharp +builder.Services + .AddSingleton(sp => + new ServiceBusClient(configuration["ServiceBus:ConnectionString"])) + .AddSingleton() + .AddKubernetesOperator() + .RegisterComponents(); + +// Replace the default queue with the durable implementation +builder.Services.Replace(ServiceDescriptor.Singleton( + typeof(ITimedEntityQueue<>), + typeof(DurableTimedEntityQueue<>))); + +// Add the background service to process messages +builder.Services.AddHostedService>(); +``` + +**Step 4: Create the queue name provider** + +```csharp +public interface IEntityRequeueQueueNameProvider +{ + string GetRequeueQueueName() where TEntity : IKubernetesObject; +} + +public class EntityRequeueQueueNameProvider : IEntityRequeueQueueNameProvider +{ + public string GetRequeueQueueName() + where TEntity : IKubernetesObject + { + return $"operator-requeue-{typeof(TEntity).Name.ToLowerInvariant()}"; + } +} +``` + +### Benefits of Durable Requeues + +- **Persistence**: Requeue requests survive operator restarts +- **Reliability**: Messages are not lost during failures +- **Scalability**: External queue systems can handle high volumes +- **Observability**: Queue metrics provide insights into requeue patterns +- **Coordination**: Multiple operator instances can share the same queue + +### Combining Custom Leader Election with Durable Queues + +For advanced scenarios, you can combine namespace-based leader election with durable queues: + +```csharp +public sealed class NamespacedLeaderElectionEntityRequeueBackgroundService( + ServiceBusClient serviceBusClient, + IKubernetesClient kubernetesClient, + IReconciler reconciler, + IEntityRequeueQueueNameProvider queueNameProvider, + INamespaceLeadershipManager namespaceLeadershipManager, + ILogger> logger) + : BackgroundService + where TEntity : IKubernetesObject +{ + private readonly ServiceBusProcessor _processor = serviceBusClient.CreateProcessor( + queueNameProvider.GetRequeueQueueName(), + new ServiceBusProcessorOptions + { + MaxConcurrentCalls = 1, + AutoCompleteMessages = false + }); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _processor.ProcessMessageAsync += ProcessMessageAsync; + _processor.ProcessErrorAsync += ProcessErrorAsync; + await _processor.StartProcessingAsync(stoppingToken); + + try + { + await Task.Delay(Timeout.Infinite, stoppingToken); + } + catch (OperationCanceledException) + { + // Expected during shutdown + } + } + + private async Task ProcessMessageAsync(ProcessMessageEventArgs args) + { + var entry = args.Message.Body.ToObjectFromJson>(); + + // Verify entity still exists + var entity = await kubernetesClient.GetAsync( + entry.Entity.Name(), + entry.Entity.Namespace(), + args.CancellationToken); + + if (entity == null) + { + logger.LogInformation("Entity no longer exists, completing message"); + await args.CompleteMessageAsync(args.Message, args.CancellationToken); + return; + } + + // Check if this instance is responsible for the namespace + if (!await namespaceLeadershipManager.IsResponsibleForNamespace( + entity.Namespace(), + args.CancellationToken)) + { + logger.LogInformation( + "Not responsible for namespace {Namespace}, abandoning message", + entity.Namespace()); + + // Abandon the message so another instance can process it + await args.AbandonMessageAsync(args.Message, cancellationToken: args.CancellationToken); + return; + } + + // Complete message and trigger reconciliation + await args.CompleteMessageAsync(args.Message, args.CancellationToken); + + await reconciler.Reconcile( + ReconciliationContext.CreateFromOperatorEvent( + entity, + entry.RequeueType.ToWatchEventType()), + args.CancellationToken); + } + + private Task ProcessErrorAsync(ProcessErrorEventArgs args) + { + logger.LogError(args.Exception, "Error processing requeue message"); + return Task.CompletedTask; + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + await _processor.StopProcessingAsync(cancellationToken); + _processor.ProcessMessageAsync -= ProcessMessageAsync; + _processor.ProcessErrorAsync -= ProcessErrorAsync; + await _processor.DisposeAsync(); + await base.StopAsync(cancellationToken); + } +} +``` + +This combination provides: +- **Namespace-based work distribution** across operator instances +- **Durable requeue persistence** for reliability +- **Message abandonment** when an instance is not responsible, allowing proper routing + +## Best Practices + +### Finalizer Management + +1. **Keep defaults enabled** for most use cases +2. **Monitor finalizer attachment** in your logs +3. **Test finalizer behavior** in development before production +4. **Handle finalizer failures** gracefully with proper error messages + +### Leader Election + +1. **Start with `Single`** for simple deployments +2. **Use `Custom`** only when you need advanced coordination +3. **Test failover scenarios** to ensure seamless transitions +4. **Monitor leader election** status in your logs +5. **Set appropriate lease durations** based on your workload + +### Requeue Mechanisms + +1. **Use in-memory queue** for development and simple operators +2. **Implement durable queues** for production workloads +3. **Monitor queue depth** to detect processing issues +4. **Set appropriate requeue delays** to avoid overwhelming your system +5. **Handle message deduplication** to prevent duplicate processing +6. **Test restart scenarios** to ensure requeues survive failures + +## Troubleshooting + +### Finalizers Not Attaching + +- Check `settings.AutoAttachFinalizers` is `true` +- Verify finalizer is registered with `AddFinalizer()` +- Check logs for finalizer attachment errors + +### Leader Election Issues + +- Verify RBAC permissions for lease resources +- Check network connectivity between instances +- Review lease duration settings +- Monitor logs for leader election events +- Ensure cluster time and local time are synchronized (time drift can cause lease issues) + +### Requeue Problems + +- Verify queue connection (for durable queues) +- Check queue permissions and quotas +- Monitor message processing errors +- Review requeue delay settings +- Ensure entities still exist before reprocessing \ No newline at end of file diff --git a/docs/docs/operator/building-blocks/controllers.mdx b/docs/docs/operator/building-blocks/controllers.mdx index 1aba0b3b..79493a33 100644 --- a/docs/docs/operator/building-blocks/controllers.mdx +++ b/docs/docs/operator/building-blocks/controllers.mdx @@ -17,16 +17,22 @@ public class V1DemoEntityController( ILogger logger, IKubernetesClient client) : IEntityController { - public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) + public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); // Implement your reconciliation logic here + return ReconciliationResult.Success(entity); } - public async Task DeletedAsync(V1DemoEntity entity, CancellationToken token) + public async Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { logger.LogInformation("Deleting entity {Entity}.", entity); // Implement your cleanup logic here + return ReconciliationResult.Success(entity); } } ``` @@ -52,41 +58,48 @@ This method is called when: - The operator starts up and discovers existing resources ```csharp -public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Check if required resources exist var deployment = await client.GetAsync( entity.Spec.DeploymentName, - entity.Namespace()); + entity.Namespace(), + cancellationToken); if (deployment == null) { // Create the deployment if it doesn't exist - await client.CreateAsync(new V1Deployment - { - Metadata = new V1ObjectMeta + await client.CreateAsync( + new V1Deployment { - Name = entity.Spec.DeploymentName, - NamespaceProperty = entity.Namespace() + Metadata = new V1ObjectMeta + { + Name = entity.Spec.DeploymentName, + NamespaceProperty = entity.Namespace() + }, + Spec = new V1DeploymentSpec + { + Replicas = entity.Spec.Replicas, + // ... other deployment configuration + } }, - Spec = new V1DeploymentSpec - { - Replicas = entity.Spec.Replicas, - // ... other deployment configuration - } - }); + cancellationToken); } // Update status to reflect current state entity.Status.LastReconciled = DateTime.UtcNow; - await client.UpdateStatusAsync(entity); + await client.UpdateStatusAsync(entity, cancellationToken); + + return ReconciliationResult.Success(entity); } ``` ### DeletedAsync :::warning Important -The `DeletedAsync` method is purely informative and "fire and forget". It is called when a resource is deleted, but it cannot guarantee proper cleanup of resources. For reliable resource cleanup, you must use [Finalizers](./finalizer). +The `DeletedAsync` method is informational only and executes asynchronously without guarantees. While it is called when a resource is deleted, it cannot ensure proper cleanup. For reliable resource cleanup, use [finalizers](./finalizer). ::: This method is called when a resource is deleted, but should only be used for: @@ -96,13 +109,130 @@ This method is called when a resource is deleted, but should only be used for: - Updating external systems about the deletion ```csharp -public async Task DeletedAsync(V1DemoEntity entity, CancellationToken token) +public async Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Log the deletion event logger.LogInformation("Entity {Entity} was deleted.", entity); // Update external systems if needed - await NotifyExternalSystem(entity); + await NotifyExternalSystem(entity, cancellationToken); + + return ReconciliationResult.Success(entity); +} +``` + +## Reconciliation Results + +All reconciliation methods must return a `ReconciliationResult`. This provides a standardized way to communicate the outcome of reconciliation operations. + +### Success Results + +Return a success result when reconciliation completes without errors: + +```csharp +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) +{ + // Perform reconciliation + await ApplyDesiredState(entity, cancellationToken); + + // Return success + return ReconciliationResult.Success(entity); +} +``` + +### Failure Results + +Return a failure result when reconciliation encounters an error: + +```csharp +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) +{ + try + { + await ApplyDesiredState(entity, cancellationToken); + return ReconciliationResult.Success(entity); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to reconcile entity {Name}", entity.Name()); + return ReconciliationResult.Failure( + entity, + "Failed to apply desired state", + ex); + } +} +``` + +### Requeuing Entities + +You can request automatic requeuing by specifying a `requeueAfter` parameter: + +```csharp +// Requeue after 5 minutes +return ReconciliationResult.Success(entity, TimeSpan.FromMinutes(5)); + +// Or set it after creation +var result = ReconciliationResult.Success(entity); +result.RequeueAfter = TimeSpan.FromSeconds(30); +return result; +``` + +This is useful for: +- Polling external resources +- Implementing retry logic with backoff +- Periodic status checks +- Waiting for external dependencies + +:::info Durable Requeue Mechanisms +By default, requeue requests are stored in memory and will be lost on operator restart. For production scenarios requiring persistence, see [Advanced Configuration - Custom Requeue Mechanism](../advanced-configuration#custom-requeue-mechanism) to learn how to implement durable queues using Azure Service Bus, RabbitMQ, or other messaging systems. +::: + +### Error Handling with Results + +The `ReconciliationResult` provides structured error handling: + +```csharp +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) +{ + if (!await ValidateConfiguration(entity)) + { + return ReconciliationResult.Failure( + entity, + "Configuration validation failed: Required field 'DeploymentName' is empty", + requeueAfter: TimeSpan.FromMinutes(1)); + } + + try + { + await ReconcileInternal(entity, cancellationToken); + return ReconciliationResult.Success(entity); + } + catch (KubernetesException ex) when (ex.Status.Code == 409) + { + // Conflict - retry after short delay + return ReconciliationResult.Failure( + entity, + "Resource conflict detected", + ex, + requeueAfter: TimeSpan.FromSeconds(5)); + } + catch (Exception ex) + { + logger.LogError(ex, "Unexpected error during reconciliation"); + return ReconciliationResult.Failure( + entity, + $"Reconciliation failed: {ex.Message}", + ex, + requeueAfter: TimeSpan.FromMinutes(5)); + } } ``` @@ -145,38 +275,47 @@ For more details about RBAC configuration, see the [RBAC documentation](../rbac) - Always check the current state before making changes ```csharp -public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Check if required resources exist - if (await IsDesiredState(entity)) + if (await IsDesiredState(entity, cancellationToken)) { - return; + return ReconciliationResult.Success(entity); } // Only make changes if needed - await ApplyDesiredState(entity); + await ApplyDesiredState(entity, cancellationToken); + return ReconciliationResult.Success(entity); } ``` ### Error Handling -- Handle errors gracefully -- Log errors with appropriate context -- Consider implementing retry logic for transient failures +- Use `ReconciliationResult.Failure()` for errors +- Include meaningful error messages +- Use the `requeueAfter` parameter for retry logic +- Preserve exception information ```csharp -public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { try { - await ReconcileInternal(entity, token); + await ReconcileInternal(entity, cancellationToken); + return ReconciliationResult.Success(entity); } catch (Exception ex) { - logger.LogError(ex, "Error reconciling entity {Entity}", entity); - // Update status to reflect the error - entity.Status.Error = ex.Message; - await client.UpdateStatusAsync(entity); + logger.LogError(ex, "Error reconciling entity {Name}", entity.Name()); + return ReconciliationResult.Failure( + entity, + $"Reconciliation failed: {ex.Message}", + ex, + requeueAfter: TimeSpan.FromMinutes(1)); } } ``` @@ -198,5 +337,5 @@ public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) 1. **Infinite Loops**: Avoid creating reconciliation loops that trigger themselves 2. **Missing Error Handling**: Always handle potential errors 3. **Resource Leaks**: Ensure proper cleanup of resources -4. **Missing RBAC**: Configure appropriate permissions +4. **Missing RBAC Configuration**: Configure appropriate permissions 5. **Status Updates**: Remember that status updates don't trigger reconciliation diff --git a/docs/docs/operator/building-blocks/finalizer.mdx b/docs/docs/operator/building-blocks/finalizer.mdx index fe349c86..d59052b9 100644 --- a/docs/docs/operator/building-blocks/finalizer.mdx +++ b/docs/docs/operator/building-blocks/finalizer.mdx @@ -25,10 +25,10 @@ When a resource is marked for deletion: 3. Each finalizer must explicitly remove itself after completing its cleanup 4. Only when all finalizers are removed is the resource actually deleted -This mechanism ensures that cleanup operations are: +This mechanism guarantees that cleanup operations are: - Guaranteed to run -- Run in a controlled manner +- Executed in a controlled sequence - Completed before resource deletion ## Implementing Finalizers @@ -40,19 +40,26 @@ public class DemoEntityFinalizer( ILogger logger, IKubernetesClient client) : IEntityFinalizer { - public async Task FinalizeAsync(V1DemoEntity entity, CancellationToken token) + public async Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { logger.LogInformation("Finalizing entity {Entity}", entity); try { // Clean up resources - await CleanupResources(entity); + await CleanupResources(entity, cancellationToken); + return ReconciliationResult.Success(entity); } catch (Exception ex) { logger.LogError(ex, "Error finalizing entity {Entity}", entity); - throw; // Re-throw to prevent finalizer removal + // Return failure to prevent finalizer removal + return ReconciliationResult.Failure( + entity, + $"Finalization failed: {ex.Message}", + ex); } } } @@ -60,7 +67,9 @@ public class DemoEntityFinalizer( ## Using Finalizers -Finalizers are automatically attached to entities using the `EntityFinalizerAttacher` delegate. This delegate is injected into your controller and handles the finalizer attachment: +By default, KubeOps automatically attaches finalizers to entities during reconciliation. This behavior can be configured through `OperatorSettings`. See [Advanced Configuration](../advanced-configuration#finalizer-management) for details on controlling automatic finalizer attachment. + +If you need manual control, you can use the `EntityFinalizerAttacher` delegate, which is injected into your controller: ```csharp [EntityRbac(typeof(V1DemoEntity), Verbs = RbacVerb.All)] @@ -69,18 +78,25 @@ public class V1DemoEntityController( EntityFinalizerAttacher finalizer) : IEntityController { - public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) + public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Attach the finalizer to the entity - entity = await finalizer(entity, token); + entity = await finalizer(entity, cancellationToken); // Continue with reconciliation logic logger.LogInformation("Reconciling entity {Entity}", entity); + + return ReconciliationResult.Success(entity); } - public async Task DeletedAsync(V1DemoEntity entity, CancellationToken token) + public async Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { logger.LogInformation("Entity {Entity} was deleted", entity); + return ReconciliationResult.Success(entity); } } ``` @@ -94,39 +110,50 @@ public class V1DemoEntityController( - Check resource existence before attempting cleanup ```csharp -public async Task FinalizeAsync(V1DemoEntity entity, CancellationToken token) +public async Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Check if resources still exist before cleanup - var resources = await GetResources(entity); + var resources = await GetResources(entity, cancellationToken); if (!resources.Any()) { // Resources already cleaned up - return; + return ReconciliationResult.Success(entity); } // Perform cleanup - await CleanupResources(resources); + await CleanupResources(resources, cancellationToken); + return ReconciliationResult.Success(entity); } ``` ### 2. Error Handling -- Handle errors gracefully +- Use `ReconciliationResult.Failure()` for errors +- Return failure results to prevent finalizer removal - Log errors with appropriate context -- Consider implementing retry logic for transient failures +- Include retry delays when appropriate ```csharp -public async Task FinalizeAsync(V1DemoEntity entity, CancellationToken token) +public async Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { try { - await FinalizeInternal(entity, token); + await FinalizeInternal(entity, cancellationToken); + return ReconciliationResult.Success(entity); } catch (Exception ex) { logger.LogError(ex, "Error finalizing entity {Entity}", entity); - // Re-throw to prevent finalizer removal - throw; + // Return failure to prevent finalizer removal + return ReconciliationResult.Failure( + entity, + $"Finalization failed: {ex.Message}", + ex, + requeueAfter: TimeSpan.FromMinutes(1)); } } ``` @@ -137,6 +164,50 @@ public async Task FinalizeAsync(V1DemoEntity entity, CancellationToken token) - Handle dependencies between resources - Consider cleanup order (e.g., delete pods before services) +### 4. Finalizer Results + +The finalizer must return a `ReconciliationResult`: + +- **Success**: The finalizer is removed, and the entity is deleted +- **Failure**: The finalizer remains, and the entity is requeued for retry + +```csharp +public async Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) +{ + // Check if external resources exist + var externalResource = await GetExternalResource(entity.Spec.ResourceId); + + if (externalResource == null) + { + // Already cleaned up + return ReconciliationResult.Success(entity); + } + + try + { + await DeleteExternalResource(externalResource, cancellationToken); + return ReconciliationResult.Success(entity); + } + catch (Exception ex) when (IsRetryable(ex)) + { + // Transient error - retry after delay + return ReconciliationResult.Failure( + entity, + "Failed to delete external resource", + ex, + requeueAfter: TimeSpan.FromSeconds(30)); + } + catch (Exception ex) + { + // Permanent error - log and succeed to prevent stuck resource + logger.LogError(ex, "Permanent error during finalization, allowing deletion"); + return ReconciliationResult.Success(entity); + } +} +``` + ## Common Pitfalls ### 1. Stuck Resources @@ -145,13 +216,15 @@ If a finalizer fails to complete: - The resource will remain in the cluster - It will be marked for deletion but never actually deleted -- Manual intervention may be required +- The finalizer will be retried based on the `requeueAfter` value +- Manual intervention may be required for permanent failures To fix stuck resources: 1. Identify the failing finalizer 2. Fix the underlying issue -3. Manually remove the finalizer only if necessary: +3. Check if the finalizer is being retried +4. Manually remove the finalizer only if necessary: ```bash kubectl patch -p '{"metadata":{"finalizers":[]}}' --type=merge ``` diff --git a/docs/docs/operator/deployment.mdx b/docs/docs/operator/deployment.mdx index 8dfd83cb..04df353c 100644 --- a/docs/docs/operator/deployment.mdx +++ b/docs/docs/operator/deployment.mdx @@ -1,7 +1,7 @@ --- title: Deployment description: Deploying your KubeOps Operator -sidebar_position: 8 +sidebar_position: 9 --- # Deployment @@ -177,7 +177,14 @@ kubectl apply -f https://github.com/your-org/your-operator/releases/download/v1. - Configure resource limits - Implement logging and metrics -4. **Updates**: +4. **High Availability**: + + - Run multiple replicas for redundancy + - Configure leader election for coordinated operation (see [Advanced Configuration - Leader Election](./advanced-configuration#custom-leader-election)) + - Consider namespace-based distribution for horizontal scaling + - Implement durable requeue mechanisms for reliability (see [Advanced Configuration - Custom Requeue Mechanism](./advanced-configuration#custom-requeue-mechanism)) + +5. **Updates**: - Document upgrade procedures - Test upgrades in staging - Provide rollback instructions diff --git a/docs/docs/operator/events.mdx b/docs/docs/operator/events.mdx index 1e3cd645..252e0496 100644 --- a/docs/docs/operator/events.mdx +++ b/docs/docs/operator/events.mdx @@ -64,7 +64,9 @@ KubeOps provides an `EventPublisher` to create and update events for your custom ```csharp public class DemoController(EventPublisher eventPublisher) : IEntityController { - public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) + public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { try { @@ -74,7 +76,9 @@ public class DemoController(EventPublisher eventPublisher) : IEntityController.Success(entity); } catch (Exception ex) { @@ -83,9 +87,21 @@ public class DemoController(EventPublisher eventPublisher) : IEntityController.Failure( + entity, + "Reconciliation failed", + ex); } } + + public Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) + { + return Task.FromResult(ReconciliationResult.Success(entity)); + } } ``` @@ -100,8 +116,8 @@ KubeOps provides two event types: 1. **Event Naming**: - - Use consistent, machine-readable reasons - - Make messages human-readable and descriptive + - Use consistent, machine-readable reason codes + - Write human-readable, descriptive messages - Include relevant details in the message 2. **Event Frequency**: diff --git a/docs/docs/operator/getting-started.mdx b/docs/docs/operator/getting-started.mdx index 1221c77e..35f86599 100644 --- a/docs/docs/operator/getting-started.mdx +++ b/docs/docs/operator/getting-started.mdx @@ -16,7 +16,7 @@ Before you begin, ensure you have the following installed: - A local Kubernetes cluster (like [kind](https://kind.sigs.k8s.io/) or [minikube](https://minikube.sigs.k8s.io/)) :::warning Development Environment -For local development, we recommend using `kind` or Docker Desktop as it provides a lightweight Kubernetes cluster that's perfect for operator development. Make sure your cluster is running before proceeding with the installation steps. +For local development, we recommend using `kind` or Docker Desktop, which provide lightweight Kubernetes clusters ideal for operator development. Ensure your cluster is running before proceeding with the installation steps. ::: ## Installing KubeOps Templates @@ -150,10 +150,20 @@ Controllers implement the reconciliation logic for your custom resources: [EntityRbac(typeof(V1DemoEntity), Verbs = RbacVerb.All)] public class DemoController : IEntityController { - public Task ReconcileAsync(V1DemoEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Implement your reconciliation logic here - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); + } + + public Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) + { + // Handle deletion event + return Task.FromResult(ReconciliationResult.Success(entity)); } } ``` @@ -165,10 +175,12 @@ Finalizers handle cleanup when resources are deleted: ```csharp public class DemoFinalizer : IEntityFinalizer { - public Task FinalizeAsync(V1DemoEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Implement your cleanup logic here - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } ``` @@ -181,6 +193,7 @@ The following sections will dive deeper into: - [Controllers](./building-blocks/controllers) - Implementing reconciliation logic - [Finalizers](./building-blocks/finalizer) - Handling resource cleanup - [Webhooks](./building-blocks/webhooks) - Implementing validation and mutation webhooks +- [Advanced Configuration](./advanced-configuration) - Leader election, durable queues, and finalizer management Make sure to read these sections before deploying your operator to production. ::: diff --git a/docs/docs/operator/testing/_category_.json b/docs/docs/operator/testing/_category_.json index 3ea28e1c..ccb9a78a 100644 --- a/docs/docs/operator/testing/_category_.json +++ b/docs/docs/operator/testing/_category_.json @@ -1,5 +1,5 @@ { - "position": 9, + "position": 10, "label": "Testing", "collapsible": true, "collapsed": true diff --git a/docs/docs/operator/testing/integration-tests.mdx b/docs/docs/operator/testing/integration-tests.mdx index 2724f57b..a0f4c5d8 100644 --- a/docs/docs/operator/testing/integration-tests.mdx +++ b/docs/docs/operator/testing/integration-tests.mdx @@ -238,10 +238,19 @@ public class EntityControllerIntegrationTest : IntegrationTestBase _svc = svc; } - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync( + V1OperatorIntegrationTestEntity entity, + CancellationToken cancellationToken) { _svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); + } + + public Task> DeletedAsync( + V1OperatorIntegrationTestEntity entity, + CancellationToken cancellationToken) + { + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/docs/docs/operator/utilities.mdx b/docs/docs/operator/utilities.mdx index 3cdfa14e..3e05b3d0 100644 --- a/docs/docs/operator/utilities.mdx +++ b/docs/docs/operator/utilities.mdx @@ -1,7 +1,7 @@ --- title: Utilities description: Utilities for your Operator and Development -sidebar_position: 10 +sidebar_position: 11 --- # Development and Operator Utilities diff --git a/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs b/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs index 9916626c..deca3529 100644 --- a/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs +++ b/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs @@ -4,23 +4,26 @@ using ConversionWebhookOperator.Entities; -using KubeOps.Abstractions.Controller; +using k8s.Models; + using KubeOps.Abstractions.Rbac; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; namespace ConversionWebhookOperator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -public class V1TestEntityController(ILogger logger) : IEntityController +public sealed class V1TestEntityController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.CompletedTask; + logger.LogInformation("Reconciling entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Deleted entity {Entity}.", entity); - return Task.CompletedTask; + logger.LogInformation("Deleted entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/examples/ConversionWebhookOperator/Webhooks/TestConversionWebhook.cs b/examples/ConversionWebhookOperator/Webhooks/TestConversionWebhook.cs index 00f0d5be..070dd4b6 100644 --- a/examples/ConversionWebhookOperator/Webhooks/TestConversionWebhook.cs +++ b/examples/ConversionWebhookOperator/Webhooks/TestConversionWebhook.cs @@ -9,7 +9,7 @@ namespace ConversionWebhookOperator.Webhooks; [ConversionWebhook(typeof(V3TestEntity))] -public class TestConversionWebhook : ConversionWebhook +public sealed class TestConversionWebhook : ConversionWebhook { protected override IEnumerable> Converters => new IEntityConverter[] { diff --git a/examples/Operator/Controller/V1TestEntityController.cs b/examples/Operator/Controller/V1TestEntityController.cs index 7527ad6f..a593e872 100644 --- a/examples/Operator/Controller/V1TestEntityController.cs +++ b/examples/Operator/Controller/V1TestEntityController.cs @@ -2,10 +2,11 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Events; -using KubeOps.Abstractions.Queue; +using k8s.Models; + using KubeOps.Abstractions.Rbac; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using Microsoft.Extensions.Logging; @@ -14,18 +15,20 @@ namespace Operator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -public class V1TestEntityController(ILogger logger) +public sealed class V1TestEntityController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync( + V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.CompletedTask; + logger.LogInformation("Reconciling entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync( + V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Deleting entity {Entity}.", entity); - return Task.CompletedTask; + logger.LogInformation("Deleted entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/examples/Operator/Finalizer/FinalizerOne.cs b/examples/Operator/Finalizer/FinalizerOne.cs index c7179900..f80b8537 100644 --- a/examples/Operator/Finalizer/FinalizerOne.cs +++ b/examples/Operator/Finalizer/FinalizerOne.cs @@ -2,16 +2,15 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Finalizer; using Operator.Entities; namespace Operator.Finalizer; -public class FinalizerOne : IEntityFinalizer +public sealed class FinalizerOne : IEntityFinalizer { - public Task FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) - { - return Task.CompletedTask; - } + public Task> FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } diff --git a/examples/WebhookOperator/Controller/V1TestEntityController.cs b/examples/WebhookOperator/Controller/V1TestEntityController.cs index ac96eba4..72bae63d 100644 --- a/examples/WebhookOperator/Controller/V1TestEntityController.cs +++ b/examples/WebhookOperator/Controller/V1TestEntityController.cs @@ -2,25 +2,28 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Controller; +using k8s.Models; + using KubeOps.Abstractions.Rbac; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using WebhookOperator.Entities; namespace WebhookOperator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -public class V1TestEntityController(ILogger logger) : IEntityController +public sealed class V1TestEntityController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.CompletedTask; + logger.LogInformation("Reconciling entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Deleted entity {Entity}.", entity); - return Task.CompletedTask; + logger.LogInformation("Deleted entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/examples/WebhookOperator/Webhooks/TestMutationWebhook.cs b/examples/WebhookOperator/Webhooks/TestMutationWebhook.cs index 3124c5a4..8f3b7a01 100644 --- a/examples/WebhookOperator/Webhooks/TestMutationWebhook.cs +++ b/examples/WebhookOperator/Webhooks/TestMutationWebhook.cs @@ -9,7 +9,7 @@ namespace WebhookOperator.Webhooks; [MutationWebhook(typeof(V1TestEntity))] -public class TestMutationWebhook : MutationWebhook +public sealed class TestMutationWebhook : MutationWebhook { public override MutationResult Create(V1TestEntity entity, bool dryRun) { diff --git a/examples/WebhookOperator/Webhooks/TestValidationWebhook.cs b/examples/WebhookOperator/Webhooks/TestValidationWebhook.cs index 5c539a44..1b3428fe 100644 --- a/examples/WebhookOperator/Webhooks/TestValidationWebhook.cs +++ b/examples/WebhookOperator/Webhooks/TestValidationWebhook.cs @@ -9,7 +9,7 @@ namespace WebhookOperator.Webhooks; [ValidationWebhook(typeof(V1TestEntity))] -public class TestValidationWebhook : ValidationWebhook +public sealed class TestValidationWebhook : ValidationWebhook { public override ValidationResult Create(V1TestEntity entity, bool dryRun) { diff --git a/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs b/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs index fbc6ad57..2cc630f2 100644 --- a/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs +++ b/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs @@ -5,10 +5,10 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Crds; using KubeOps.Abstractions.Entities; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; using Microsoft.Extensions.DependencyInjection; @@ -24,6 +24,11 @@ public interface IOperatorBuilder /// IServiceCollection Services { get; } + /// + /// Configuration settings for the operator. + /// + OperatorSettings Settings { get; } + /// /// Add a controller implementation for a specific entity to the operator. /// The metadata for the entity must be added as well. diff --git a/src/KubeOps.Abstractions/Builder/LeaderElectionType.cs b/src/KubeOps.Abstractions/Builder/LeaderElectionType.cs new file mode 100644 index 00000000..7e68a7aa --- /dev/null +++ b/src/KubeOps.Abstractions/Builder/LeaderElectionType.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Abstractions.Builder; + +/// +/// Specifies the types of leader election mechanisms to be used in distributed systems or workloads. +/// +public enum LeaderElectionType +{ + /// + /// Represents the absence of a leader election mechanism. + /// This option is used when no leader election is required, and all instances + /// are expected to operate without coordination or exclusivity. + /// + None = 0, + + /// + /// Represents the leader election mechanism where only a single instance of the application + /// assumes the leader role at any given time. This is used to coordinate operations + /// that require exclusivity or to manage shared resources in distributed systems. + /// + Single = 1, + + /// + /// Represents a custom leader election mechanism determined by the user. + /// This option allows the integration of user-defined logic for handling + /// leader election, enabling tailored coordination strategies beyond the + /// provided defaults. + /// + Custom = 2, +} diff --git a/src/KubeOps.Abstractions/Builder/OperatorSettings.cs b/src/KubeOps.Abstractions/Builder/OperatorSettings.cs index d02b642e..7101d124 100644 --- a/src/KubeOps.Abstractions/Builder/OperatorSettings.cs +++ b/src/KubeOps.Abstractions/Builder/OperatorSettings.cs @@ -36,19 +36,18 @@ public sealed partial class OperatorSettings public string? Namespace { get; set; } /// - /// - /// Whether the leader elector should run. You should enable - /// this if you plan to run the operator redundantly. - /// - /// - /// If this is disabled and an operator runs in multiple instances - /// (in the same namespace), it can lead to a "split brain" problem. - /// - /// - /// Defaults to `false`. - /// + /// Defines the type of leader election mechanism to be used by the operator. + /// Determines how resources and controllers are coordinated in a distributed environment. + /// Defaults to indicating no leader election is configured. /// - public bool EnableLeaderElection { get; set; } = false; + public LeaderElectionType LeaderElectionType { get; set; } = LeaderElectionType.None; + + /// + /// Defines the strategy for requeuing reconciliation events within the operator. + /// Determines how reconciliation events are managed and requeued during operator execution. + /// Defaults to when not explicitly configured. + /// + public RequeueStrategy RequeueStrategy { get; set; } = RequeueStrategy.InMemory; /// /// Defines how long one lease is valid for any leader. @@ -73,6 +72,19 @@ public sealed partial class OperatorSettings /// public Action? ConfigureResourceWatcherEntityCache { get; set; } + /// + /// Indicates whether finalizers should be automatically attached to Kubernetes entities during reconciliation. + /// When enabled, the operator will ensure that all defined finalizers for the entity are added if they are not already present. + /// Defaults to true. + /// + public bool AutoAttachFinalizers { get; set; } = true; + + /// + /// Indicates whether finalizers should be automatically removed from Kubernetes resources + /// upon successful completion of their finalization process. Defaults to true. + /// + public bool AutoDetachFinalizers { get; set; } = true; + [GeneratedRegex(@"(\W|_)", RegexOptions.CultureInvariant)] private static partial Regex OperatorNameRegex(); } diff --git a/src/KubeOps.Abstractions/Builder/RequeueStrategy.cs b/src/KubeOps.Abstractions/Builder/RequeueStrategy.cs new file mode 100644 index 00000000..b2d21cf3 --- /dev/null +++ b/src/KubeOps.Abstractions/Builder/RequeueStrategy.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Abstractions.Builder; + +/// +/// Defines the strategy for requeuing reconciliation events within the operator. +/// +public enum RequeueStrategy +{ + /// + /// Represents an in-memory requeue strategy where reconciliation events + /// are managed and requeued without external persistence or reliance on third-party systems. + /// Suitable for scenarios requiring lightweight or transient processing. + /// + InMemory, + + /// + /// Represents a custom requeue strategy where the logic for managing and + /// handling reconciliation events is fully defined and implemented by the user. + /// This provides maximum flexibility for scenarios with specific or complex requirements. + /// + Custom, +} diff --git a/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs b/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs index 408db1a5..2f26f16b 100644 --- a/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs +++ b/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. +using System.Reflection; + using k8s; using k8s.Models; @@ -91,6 +93,14 @@ public static V1OwnerReference MakeOwnerReference(this IKubernetesObject + /// Retrieves the applied to the specified Kubernetes object type, if it exists. + /// + /// The Kubernetes object to inspect for the attribute. + /// The if found; otherwise, null. + public static KubernetesEntityAttribute? GetKubernetesEntityAttribute(this IKubernetesObject entity) + => entity.GetType().GetCustomAttribute(true); + private static IList EnsureOwnerReferences(this V1ObjectMeta meta) => meta.OwnerReferences ??= new List(); } diff --git a/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj b/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj index 14c815cf..5e9c4edc 100644 --- a/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj +++ b/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj @@ -20,4 +20,4 @@ - + \ No newline at end of file diff --git a/src/KubeOps.Abstractions/Queue/EntityRequeue.cs b/src/KubeOps.Abstractions/Queue/EntityRequeue.cs deleted file mode 100644 index b3518138..00000000 --- a/src/KubeOps.Abstractions/Queue/EntityRequeue.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information. - -using k8s; -using k8s.Models; - -namespace KubeOps.Abstractions.Queue; - -/// -/// Injectable delegate for requeueing entities. -/// -/// Use this delegate when you need to pro-actively reconcile an entity after a -/// certain amount of time. This is useful, if you want to check your entities -/// periodically. -/// -/// -/// After the timeout is reached, the entity is fetched -/// from the API and passed to the controller for reconciliation. -/// If the entity was deleted in the meantime, the controller will not be called. -/// -/// -/// If the entity gets modified while the timeout is running, the timer -/// is canceled and restarted, if another requeue is requested. -/// -/// -/// The type of the entity. -/// The instance of the entity that should be requeued. -/// The time to wait before another reconcile loop is fired. -/// -/// Use the requeue delegate to repeatedly reconcile an entity after 5 seconds. -/// -/// [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -/// public class V1TestEntityController : IEntityController<V1TestEntity> -/// { -/// private readonly EntityRequeue<V1TestEntity> _requeue; -/// -/// public V1TestEntityController(EntityRequeue<V1TestEntity> requeue) -/// { -/// _requeue = requeue; -/// } -/// -/// public async Task ReconcileAsync(V1TestEntity entity, CancellationToken token) -/// { -/// _requeue(entity, TimeSpan.FromSeconds(5)); -/// } -/// } -/// -/// -public delegate void EntityRequeue(TEntity entity, TimeSpan requeueIn) - where TEntity : IKubernetesObject; diff --git a/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/Controller/IEntityController{TEntity}.cs similarity index 63% rename from src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs rename to src/KubeOps.Abstractions/Reconciliation/Controller/IEntityController{TEntity}.cs index fc6e23ab..e650b76a 100644 --- a/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Controller/IEntityController{TEntity}.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Controller; +namespace KubeOps.Abstractions.Reconciliation.Controller; /// /// Generic entity controller. The controller manages the reconcile loop @@ -37,24 +37,23 @@ namespace KubeOps.Abstractions.Controller; /// } /// /// -public interface IEntityController +public interface IEntityController where TEntity : IKubernetesObject { /// - /// Called for `added` and `modified` events from the watcher. + /// Reconciles the state of the specified entity with the desired state. + /// This method is triggered for `added` and `modified` events from the watcher. /// - /// The entity that fired the reconcile event. - /// The token to monitor for cancellation requests. - /// A task that completes when the reconciliation is done. - Task ReconcileAsync(TEntity entity, CancellationToken cancellationToken); + /// The entity that initiated the reconcile operation. + /// The token used to signal cancellation of the operation. + /// A task that represents the asynchronous operation and contains the result of the reconcile process. + Task> ReconcileAsync(TEntity entity, CancellationToken cancellationToken); /// /// Called for `delete` events for a given entity. /// /// The entity that fired the deleted event. /// The token to monitor for cancellation requests. - /// - /// A task that completes, when the reconciliation is done. - /// - Task DeletedAsync(TEntity entity, CancellationToken cancellationToken); + /// A task that represents the asynchronous operation and contains the result of the reconcile process. + Task> DeletedAsync(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/Finalizer/EntityFinalizerAttacher.cs b/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerAttacher.cs similarity index 97% rename from src/KubeOps.Abstractions/Finalizer/EntityFinalizerAttacher.cs rename to src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerAttacher.cs index 924bfde8..c54275bc 100644 --- a/src/KubeOps.Abstractions/Finalizer/EntityFinalizerAttacher.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerAttacher.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Finalizer; +namespace KubeOps.Abstractions.Reconciliation.Finalizer; /// /// diff --git a/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerExtensions.cs b/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerExtensions.cs new file mode 100644 index 00000000..d2408108 --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerExtensions.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Entities; + +namespace KubeOps.Abstractions.Reconciliation.Finalizer; + +/// +/// Provides extension methods for handling entity finalizers in Kubernetes resources. +/// +public static class EntityFinalizerExtensions +{ + private const byte MaxNameLength = 63; + + /// + /// Generates a unique identifier name for the finalizer of a given Kubernetes entity. + /// The identifier includes the group of the entity and the name of the finalizer, ensuring it conforms to Kubernetes naming conventions. + /// + /// The type of the Kubernetes entity. Must implement . + /// The finalizer implementing for which the identifier is generated. + /// The Kubernetes entity associated with the finalizer. + /// A string representing the unique identifier for the finalizer, truncated if it exceeds the maximum allowed length for Kubernetes names. + public static string GetIdentifierName(this IEntityFinalizer finalizer, TEntity entity) + where TEntity : IKubernetesObject + { + var finalizerName = finalizer.GetType().Name.ToLowerInvariant(); + finalizerName = finalizerName.EndsWith("finalizer") ? finalizerName : $"{finalizerName}finalizer"; + + var entityGroupName = entity.GetKubernetesEntityAttribute()?.Group ?? string.Empty; + var name = $"{entityGroupName}/{finalizerName}".TrimStart('/'); + + return name.Length > MaxNameLength ? name[..MaxNameLength] : name; + } +} diff --git a/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/Finalizer/IEntityFinalizer{TEntity}.cs similarity index 67% rename from src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs rename to src/KubeOps.Abstractions/Reconciliation/Finalizer/IEntityFinalizer{TEntity}.cs index c9fdebc2..f3b38983 100644 --- a/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Finalizer/IEntityFinalizer{TEntity}.cs @@ -5,13 +5,13 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Finalizer; +namespace KubeOps.Abstractions.Reconciliation.Finalizer; /// /// Finalizer for an entity. /// /// The type of the entity. -public interface IEntityFinalizer +public interface IEntityFinalizer where TEntity : IKubernetesObject { /// @@ -19,6 +19,6 @@ public interface IEntityFinalizer /// /// The kubernetes entity that needs to be finalized. /// The token to monitor for cancellation requests. - /// A task that resolves when the operation is done. - Task FinalizeAsync(TEntity entity, CancellationToken cancellationToken); + /// A task that represents the asynchronous operation and contains the result of the reconcile process. + Task> FinalizeAsync(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/Finalizer/IEventFinalizerAttacherFactory.cs b/src/KubeOps.Abstractions/Reconciliation/Finalizer/IEventFinalizerAttacherFactory.cs similarity index 95% rename from src/KubeOps.Abstractions/Finalizer/IEventFinalizerAttacherFactory.cs rename to src/KubeOps.Abstractions/Reconciliation/Finalizer/IEventFinalizerAttacherFactory.cs index 8236f6c8..8235dbfc 100644 --- a/src/KubeOps.Abstractions/Finalizer/IEventFinalizerAttacherFactory.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Finalizer/IEventFinalizerAttacherFactory.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Finalizer; +namespace KubeOps.Abstractions.Reconciliation.Finalizer; /// /// Represents a type used to create for controllers. diff --git a/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs new file mode 100644 index 00000000..15892a9a --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Defines methods for handling reconciliation processes related to Kubernetes resources. +/// This interface provides the necessary functionality for handling the lifecycle events +/// of a resource, such as creation, modification, and deletion. +/// +/// +/// The type of the Kubernetes resource, which must implement . +/// +public interface IReconciler + where TEntity : IKubernetesObject +{ + /// + /// Handles the reconciliation process for a Kubernetes entity. + /// + /// The context containing details of the entity to reconcile. + /// A token to monitor for cancellation requests during the reconciliation process. + /// A task that represents the asynchronous reconciliation operation, returning the result of the reconciliation process. + Task> Reconcile(ReconciliationContext reconciliationContext, CancellationToken cancellationToken); +} diff --git a/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs b/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs new file mode 100644 index 00000000..1d88b9fa --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation.Queue; + +/// +/// Injectable delegate for scheduling an entity to be requeued after a specified amount of time. +/// +/// The type of the Kubernetes entity being requeued. +/// The entity instance that should be requeued. +/// The type of operation for which the reconcile behavior should be performed. +/// The duration to wait before triggering the next reconcile process. +/// A cancellation token to observe while waiting for the requeue duration. +public delegate void EntityRequeue( + TEntity entity, RequeueType type, TimeSpan requeueIn, CancellationToken cancellationToken) + where TEntity : IKubernetesObject; diff --git a/src/KubeOps.Abstractions/Queue/IEntityRequeueFactory.cs b/src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs similarity index 87% rename from src/KubeOps.Abstractions/Queue/IEntityRequeueFactory.cs rename to src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs index 51486eff..8bd89088 100644 --- a/src/KubeOps.Abstractions/Queue/IEntityRequeueFactory.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs @@ -5,10 +5,10 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Queue; +namespace KubeOps.Abstractions.Reconciliation.Queue; /// -/// Represents a type used to create delegates of type for requeueing entities. +/// Represents a type used to create delegates of type for requeuing entities. /// public interface IEntityRequeueFactory { diff --git a/src/KubeOps.Abstractions/Reconciliation/Queue/RequeueType.cs b/src/KubeOps.Abstractions/Reconciliation/Queue/RequeueType.cs new file mode 100644 index 00000000..37346af8 --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/Queue/RequeueType.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Abstractions.Reconciliation.Queue; + +/// +/// Specifies the types of requeue operations that can occur on an entity. +/// +public enum RequeueType +{ + /// + /// Indicates that an entity should be added and is scheduled for requeue. + /// + Added, + + /// + /// Indicates that an entity has been modified and is scheduled for requeue. + /// + Modified, + + /// + /// Indicates that an entity should be deleted and is scheduled for requeue. + /// + Deleted, +} diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationContextExtensions.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContextExtensions.cs new file mode 100644 index 00000000..e8aae9b9 --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContextExtensions.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Provides extension methods for the class +/// to facilitate the identification of reconciliation trigger sources. +/// +public static class ReconciliationContextExtensions +{ + /// + /// Determines if the reconciliation context was triggered by the Kubernetes API server. + /// + /// The type of the Kubernetes resource associated with the reconciliation context. + /// The reconciliation context to check. + /// True if the reconciliation was triggered by the API server; otherwise, false. + public static bool IsTriggeredByApiServer(this ReconciliationContext reconciliationContext) + where TEntity : IKubernetesObject + => reconciliationContext.ReconciliationTriggerSource == ReconciliationTriggerSource.ApiServer; + + /// + /// Determines if the reconciliation context was triggered by the operator. + /// + /// The type of the Kubernetes resource associated with the reconciliation context. + /// The reconciliation context to check. + /// True if the reconciliation was triggered by the operator; otherwise, false. + public static bool IsTriggeredByOperator(this ReconciliationContext reconciliationContext) + where TEntity : IKubernetesObject + => reconciliationContext.ReconciliationTriggerSource == ReconciliationTriggerSource.Operator; +} diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs new file mode 100644 index 00000000..539ed06a --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Represents the context for the reconciliation process. +/// This class contains information about the entity to be reconciled and +/// the source that triggered the reconciliation process. +/// +/// +/// The type of the Kubernetes resource being reconciled. Must implement +/// . +/// +public sealed record ReconciliationContext + where TEntity : IKubernetesObject +{ + private ReconciliationContext(TEntity entity, WatchEventType eventType, ReconciliationTriggerSource reconciliationTriggerSource) + { + Entity = entity; + EventType = eventType; + ReconciliationTriggerSource = reconciliationTriggerSource; + } + + /// + /// Represents the Kubernetes entity involved in the reconciliation process. + /// + public TEntity Entity { get; } + + /// + /// Specifies the type of Kubernetes watch event that triggered the reconciliation process. + /// This property provides information about the nature of the change detected + /// within the Kubernetes resource, such as addition, modification, or deletion. + /// + public WatchEventType EventType { get; } + + /// + /// Specifies the source that initiated the reconciliation process. + /// + public ReconciliationTriggerSource ReconciliationTriggerSource { get; } + + /// + /// Creates a new instance of from an API server event. + /// + /// + /// The Kubernetes entity associated with the reconciliation context. + /// + /// + /// The type of watch event that triggered the context creation. + /// + /// + /// A new instance representing the reconciliation context + /// for the specified entity and event type, triggered by the API server. + /// + public static ReconciliationContext CreateFromApiServerEvent(TEntity entity, WatchEventType eventType) + => new(entity, eventType, ReconciliationTriggerSource.ApiServer); + + /// + /// Creates a new instance of from an operator-driven event. + /// + /// + /// The Kubernetes entity associated with the reconciliation context. + /// + /// + /// The type of watch event that triggered the context creation. + /// + /// + /// A new instance representing the reconciliation context + /// for the specified entity and event type, triggered by the operator. + /// + public static ReconciliationContext CreateFromOperatorEvent(TEntity entity, WatchEventType eventType) + => new(entity, eventType, ReconciliationTriggerSource.Operator); +} diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs new file mode 100644 index 00000000..e018cf15 --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs @@ -0,0 +1,110 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Represents the result of an operation performed on an entity +/// within the context of Kubernetes controllers or finalizers. +/// +/// +/// The type of the Kubernetes entity associated with this result. +/// Must implement where TMetadata is . +/// +public sealed record ReconciliationResult + where TEntity : IKubernetesObject +{ + private ReconciliationResult(TEntity entity, bool isSuccess, string? errorMessage, Exception? error, TimeSpan? requeueAfter) + { + Entity = entity; + IsSuccess = isSuccess; + ErrorMessage = errorMessage; + Error = error; + RequeueAfter = requeueAfter; + } + + /// + /// Represents the Kubernetes entity associated with the result of an operation or reconciliation process. + /// This property contains the entity object of type that was processed, modified, or finalized + /// during an operation. It provides access to the updated state or metadata of the entity after the operation. + /// Typically used for handling further processing, queuing, or logging of the affected entity. + /// + public TEntity Entity { get; } + + /// + /// Indicates whether the operation has completed successfully. + /// Returns true when the operation was successful, and no errors occurred. + /// This property is used to determine the state of the operation + /// and is often checked to decide whether further processing or error handling is necessary. + /// When this property is true, will be null, as there were no errors. + /// + [MemberNotNullWhen(false, nameof(ErrorMessage))] + public bool IsSuccess { get; } + + /// + /// Contains a descriptive message associated with a failure when the operation does not succeed. + /// Used to provide context or details about the failure, assisting in debugging and logging. + /// This property is typically set when is false. + /// It will be null for successful operations. + /// + public string? ErrorMessage { get; } + + /// + /// Represents an exception associated with the operation outcome. + /// If the operation fails, this property may hold the exception that caused the failure, + /// providing additional context about the error for logging or debugging purposes. + /// This is optional and may be null if no exception information is available or applicable. + /// + public Exception? Error { get; } + + /// + /// Specifies the duration to wait before requeuing the entity for reprocessing. + /// If set, the entity will be scheduled for reprocessing after the specified time span. + /// This can be useful in scenarios where the entity needs to be revisited later due to external conditions, + /// such as resource dependencies or transient errors. + /// + public TimeSpan? RequeueAfter { get; set; } + + /// + /// Creates a successful result for the given entity, optionally specifying a requeue duration. + /// + /// + /// The Kubernetes entity that the result is associated with. + /// + /// + /// An optional duration after which the entity should be requeued for processing. Defaults to null. + /// + /// + /// A successful instance containing the provided entity and requeue duration. + /// + public static ReconciliationResult Success(TEntity entity, TimeSpan? requeueAfter = null) + => new(entity, true, null, null, requeueAfter); + + /// + /// Creates a failure result for the given entity, specifying an error message and optionally an exception and requeue duration. + /// + /// + /// The Kubernetes entity that the result is associated with. + /// + /// + /// A detailed message describing the reason for the failure. + /// + /// + /// An optional exception that caused the failure. Defaults to null. + /// + /// + /// An optional duration after which the entity should be requeued for processing. Defaults to null. + /// + /// + /// A failure instance containing the provided entity, error information, and requeue duration. + /// + public static ReconciliationResult Failure( + TEntity entity, string errorMessage, Exception? error = null, TimeSpan? requeueAfter = null) + => new(entity, false, errorMessage, error, requeueAfter); +} diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationTriggerSource.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationTriggerSource.cs new file mode 100644 index 00000000..d5020c39 --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationTriggerSource.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Defines the source that triggered the reconciliation process in the Kubernetes operator. +/// Used to identify which component or mechanism initiated the reconciliation cycle. +/// +public enum ReconciliationTriggerSource +{ + /// + /// Represents a reconciliation trigger initiated by the Kubernetes API server. + /// This source typically implies that the operator has been informed about + /// a resource event (e.g., creation, modification, deletion) via API server + /// notifications or resource watches. + /// + ApiServer, + + /// + /// Represents a reconciliation trigger initiated directly by the operator. + /// This source indicates that the reconciliation process was started internally + /// by the operator, such as during a scheduled task or an operator-specific event. + /// + Operator, +} diff --git a/src/KubeOps.Generator/Generators/OperatorBuilderGenerator.cs b/src/KubeOps.Generator/Generators/OperatorBuilderGenerator.cs index 288cd51c..3acb31de 100644 --- a/src/KubeOps.Generator/Generators/OperatorBuilderGenerator.cs +++ b/src/KubeOps.Generator/Generators/OperatorBuilderGenerator.cs @@ -14,8 +14,10 @@ namespace KubeOps.Generator.Generators; [Generator] -internal class OperatorBuilderGenerator : ISourceGenerator +internal sealed class OperatorBuilderGenerator : ISourceGenerator { + private const string BuilderIdentifier = "builder"; + public void Initialize(GeneratorInitializationContext context) { } @@ -36,7 +38,7 @@ public void Execute(GeneratorExecutionContext context) .WithParameterList(ParameterList( SingletonSeparatedList( Parameter( - Identifier("builder")) + Identifier(BuilderIdentifier)) .WithModifiers( TokenList( Token(SyntaxKind.ThisKeyword))) @@ -47,15 +49,15 @@ public void Execute(GeneratorExecutionContext context) InvocationExpression( MemberAccessExpression( SyntaxKind.SimpleMemberAccessExpression, - IdentifierName("builder"), + IdentifierName(BuilderIdentifier), IdentifierName("RegisterControllers")))), ExpressionStatement( InvocationExpression( MemberAccessExpression( SyntaxKind.SimpleMemberAccessExpression, - IdentifierName("builder"), + IdentifierName(BuilderIdentifier), IdentifierName("RegisterFinalizers")))), - ReturnStatement(IdentifierName("builder"))))))) + ReturnStatement(IdentifierName(BuilderIdentifier))))))) .NormalizeWhitespace(); context.AddSource( diff --git a/src/KubeOps.Generator/SyntaxReceiver/EntityControllerSyntaxReceiver.cs b/src/KubeOps.Generator/SyntaxReceiver/EntityControllerSyntaxReceiver.cs index 686acbe0..08351ab8 100644 --- a/src/KubeOps.Generator/SyntaxReceiver/EntityControllerSyntaxReceiver.cs +++ b/src/KubeOps.Generator/SyntaxReceiver/EntityControllerSyntaxReceiver.cs @@ -9,7 +9,7 @@ namespace KubeOps.Generator.SyntaxReceiver; internal sealed class EntityControllerSyntaxReceiver : ISyntaxContextReceiver { - private const string IEntityControllerMetadataName = "KubeOps.Abstractions.Controller.IEntityController`1"; + private const string IEntityControllerMetadataName = "KubeOps.Abstractions.Reconciliation.Controller.IEntityController`1"; public List<(ClassDeclarationSyntax Controller, string EntityName)> Controllers { get; } = []; diff --git a/src/KubeOps.Generator/SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs b/src/KubeOps.Generator/SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs index ceac128a..9f07e3a8 100644 --- a/src/KubeOps.Generator/SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs +++ b/src/KubeOps.Generator/SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs @@ -9,7 +9,7 @@ namespace KubeOps.Generator.SyntaxReceiver; internal sealed class EntityFinalizerSyntaxReceiver : ISyntaxContextReceiver { - private const string IEntityFinalizerMetadataName = "KubeOps.Abstractions.Finalizer.IEntityFinalizer`1"; + private const string IEntityFinalizerMetadataName = "KubeOps.Abstractions.Reconciliation.Finalizer.IEntityFinalizer`1"; public List<(ClassDeclarationSyntax Finalizer, string EntityName)> Finalizer { get; } = []; diff --git a/src/KubeOps.Generator/SyntaxReceiver/KubernetesEntitySyntaxReceiver.cs b/src/KubeOps.Generator/SyntaxReceiver/KubernetesEntitySyntaxReceiver.cs index f98bcc0d..6d1dd4a2 100644 --- a/src/KubeOps.Generator/SyntaxReceiver/KubernetesEntitySyntaxReceiver.cs +++ b/src/KubeOps.Generator/SyntaxReceiver/KubernetesEntitySyntaxReceiver.cs @@ -28,15 +28,27 @@ public void OnVisitSyntaxNode(GeneratorSyntaxContext context) Entities.Add(new( cls, - GetArgumentValue(attr, KindName) ?? cls.Identifier.ToString(), - GetArgumentValue(attr, VersionName) ?? DefaultVersion, - GetArgumentValue(attr, GroupName), - GetArgumentValue(attr, PluralName))); + GetArgumentValue(context.SemanticModel, attr, KindName) ?? cls.Identifier.ToString(), + GetArgumentValue(context.SemanticModel, attr, VersionName) ?? DefaultVersion, + GetArgumentValue(context.SemanticModel, attr, GroupName), + GetArgumentValue(context.SemanticModel, attr, PluralName))); } - private static string? GetArgumentValue(AttributeSyntax attr, string argName) => - attr.ArgumentList?.Arguments.FirstOrDefault(a => a.NameEquals?.Name.ToString() == argName) is - { Expression: LiteralExpressionSyntax { Token.ValueText: { } value } } + private static string? GetArgumentValue(SemanticModel model, AttributeSyntax attr, string argName) + { + if (attr.ArgumentList?.Arguments.FirstOrDefault(a => a.NameEquals?.Name.ToString() == argName) + is not { Expression: { } expr }) + { + return null; + } + + if (model.GetConstantValue(expr) is { HasValue: true, Value: string s }) + { + return s; + } + + return expr is LiteralExpressionSyntax { Token.ValueText: { } value } ? value : null; + } } diff --git a/src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs b/src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs index d016f16a..7272931a 100644 --- a/src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs +++ b/src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs @@ -13,6 +13,6 @@ namespace KubeOps.Operator.Web.Webhooks.Admission; /// /// A message that is passed to the API. /// A custom status code to provide more detailed information. -public record AdmissionStatus([property: JsonPropertyName("message")] +public sealed record AdmissionStatus([property: JsonPropertyName("message")] string Message, [property: JsonPropertyName("code")] int? Code = StatusCodes.Status200OK); diff --git a/src/KubeOps.Operator.Web/Webhooks/Admission/Mutation/MutationResult.cs b/src/KubeOps.Operator.Web/Webhooks/Admission/Mutation/MutationResult.cs index 100273d3..92a2747f 100644 --- a/src/KubeOps.Operator.Web/Webhooks/Admission/Mutation/MutationResult.cs +++ b/src/KubeOps.Operator.Web/Webhooks/Admission/Mutation/MutationResult.cs @@ -21,7 +21,7 @@ namespace KubeOps.Operator.Web.Webhooks.Admission.Mutation; /// /// The modified entity if any changes are requested. /// The type of the entity. -public record MutationResult(TEntity? ModifiedObject = default) : IActionResult +public sealed record MutationResult(TEntity? ModifiedObject = default) : IActionResult where TEntity : IKubernetesObject { private const string JsonPatch = "JSONPatch"; diff --git a/src/KubeOps.Operator/Builder/OperatorBuilder.cs b/src/KubeOps.Operator/Builder/OperatorBuilder.cs index 1c066729..b7e7be17 100644 --- a/src/KubeOps.Operator/Builder/OperatorBuilder.cs +++ b/src/KubeOps.Operator/Builder/OperatorBuilder.cs @@ -8,18 +8,20 @@ using k8s.Models; using KubeOps.Abstractions.Builder; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Crds; using KubeOps.Abstractions.Entities; using KubeOps.Abstractions.Events; -using KubeOps.Abstractions.Finalizer; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Crds; using KubeOps.Operator.Events; using KubeOps.Operator.Finalizer; using KubeOps.Operator.LeaderElection; using KubeOps.Operator.Queue; +using KubeOps.Operator.Reconciliation; using KubeOps.Operator.Watcher; using Microsoft.Extensions.DependencyInjection; @@ -29,35 +31,44 @@ namespace KubeOps.Operator.Builder; internal sealed class OperatorBuilder : IOperatorBuilder { - private readonly OperatorSettings _settings; - public OperatorBuilder(IServiceCollection services, OperatorSettings settings) { - _settings = settings; + Settings = settings; Services = services; AddOperatorBase(); } public IServiceCollection Services { get; } + public OperatorSettings Settings { get; } + public IOperatorBuilder AddController() where TImplementation : class, IEntityController where TEntity : IKubernetesObject { - Services.AddHostedService>(); Services.TryAddScoped, TImplementation>(); - Services.TryAddSingleton(new TimedEntityQueue()); + Services.TryAddSingleton, Reconciler>(); + + // Requeue Services.TryAddTransient(); Services.TryAddTransient>(services => services.GetRequiredService().Create()); - if (_settings.EnableLeaderElection) + if (Settings.RequeueStrategy == RequeueStrategy.InMemory) { - Services.AddHostedService>(); + Services.TryAddSingleton, TimedEntityQueue>(); + Services.AddHostedService>(); } - else + + // Leader Election + switch (Settings.LeaderElectionType) { - Services.AddHostedService>(); + case LeaderElectionType.None: + Services.AddHostedService>(); + break; + case LeaderElectionType.Single: + Services.AddHostedService>(); + break; } return this; @@ -68,23 +79,9 @@ public IOperatorBuilder AddController( where TEntity : IKubernetesObject where TLabelSelector : class, IEntityLabelSelector { - Services.AddHostedService>(); - Services.TryAddScoped, TImplementation>(); - Services.TryAddSingleton(new TimedEntityQueue()); - Services.TryAddTransient(); - Services.TryAddTransient>(services => - services.GetRequiredService().Create()); + AddController(); Services.TryAddSingleton, TLabelSelector>(); - if (_settings.EnableLeaderElection) - { - Services.AddHostedService>(); - } - else - { - Services.AddHostedService>(); - } - return this; } @@ -112,19 +109,19 @@ public IOperatorBuilder AddCrdInstaller(Action? configure private void AddOperatorBase() { - Services.AddSingleton(_settings); - Services.AddSingleton(new ActivitySource(_settings.Name)); + Services.AddSingleton(Settings); + Services.AddSingleton(new ActivitySource(Settings.Name)); // add and configure resource watcher entity cache - Services.WithResourceWatcherEntityCaching(_settings); + Services.WithResourceWatcherEntityCaching(Settings); // Add the default configuration and the client separately. This allows external users to override either // just the config (e.g. for integration tests) or to replace the whole client, e.g. with a mock. - // We also add the k8s.IKubernetes as a singleton service, in order to allow to access internal services - // and also external users to make use of it's features that might not be implemented in the adapted client. + // We also add the k8s.IKubernetes as a singleton service, in order to allow accessing internal services + // and also external users to make use of its features that might not be implemented in the adapted client. // // Due to a memory leak in the Kubernetes client, it is important that the client is registered with - // with the same lifetime as the KubernetesClientConfiguration. This is tracked in kubernetes/csharp#1446. + // the same lifetime as the KubernetesClientConfiguration. This is tracked in kubernetes/csharp#1446. // https://github.com/kubernetes-client/csharp/issues/1446 // // The missing ability to inject a custom HTTP client and therefore the possibility to use the .AddHttpClient() @@ -140,7 +137,7 @@ private void AddOperatorBase() Services.AddSingleton(typeof(IEntityLabelSelector<>), typeof(DefaultEntityLabelSelector<>)); - if (_settings.EnableLeaderElection) + if (Settings.LeaderElectionType == LeaderElectionType.Single) { Services.AddLeaderElection(); } diff --git a/src/KubeOps.Operator/Finalizer/KubeOpsEventFinalizerAttacherFactory.cs b/src/KubeOps.Operator/Finalizer/KubeOpsEventFinalizerAttacherFactory.cs index 5a2db148..47975b07 100644 --- a/src/KubeOps.Operator/Finalizer/KubeOpsEventFinalizerAttacherFactory.cs +++ b/src/KubeOps.Operator/Finalizer/KubeOpsEventFinalizerAttacherFactory.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation.Finalizer; using KubeOps.KubernetesClient; using Microsoft.Extensions.Logging; diff --git a/src/KubeOps.Operator/KubeOps.Operator.csproj b/src/KubeOps.Operator/KubeOps.Operator.csproj index a0453df3..75593f81 100644 --- a/src/KubeOps.Operator/KubeOps.Operator.csproj +++ b/src/KubeOps.Operator/KubeOps.Operator.csproj @@ -30,11 +30,11 @@ build/ - + <_Parameter1>$(MSBuildProjectName).Web.Test - - + + \ No newline at end of file diff --git a/src/KubeOps.Operator/Logging/EntityLoggingScope.cs b/src/KubeOps.Operator/Logging/EntityLoggingScope.cs index 69e4d22d..3a680129 100644 --- a/src/KubeOps.Operator/Logging/EntityLoggingScope.cs +++ b/src/KubeOps.Operator/Logging/EntityLoggingScope.cs @@ -7,6 +7,9 @@ using k8s; using k8s.Models; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Queue; + namespace KubeOps.Operator.Logging; #pragma warning disable CA1710 @@ -14,7 +17,7 @@ namespace KubeOps.Operator.Logging; /// A logging scope that encapsulates contextual information related to a Kubernetes entity and event type. /// Provides a mechanism for structured logging with key-value pairs corresponding to entity metadata and event type. /// -internal sealed record EntityLoggingScope : IReadOnlyCollection> +public sealed record EntityLoggingScope : IReadOnlyCollection> #pragma warning restore CA1710 { private EntityLoggingScope(IReadOnlyDictionary state) @@ -46,16 +49,27 @@ private EntityLoggingScope(IReadOnlyDictionary state) /// public static EntityLoggingScope CreateFor(WatchEventType eventType, TEntity entity) where TEntity : IKubernetesObject - => new( - new Dictionary - { - { "EventType", eventType }, - { nameof(entity.Kind), entity.Kind }, - { "Namespace", entity.Namespace() }, - { "Name", entity.Name() }, - { "Uid", entity.Uid() }, - { "ResourceVersion", entity.ResourceVersion() }, - }); + => CreateLoggingScope(eventType.ToString(), ReconciliationTriggerSource.ApiServer, entity); + + /// + /// Creates a new instance of for the given Kubernetes entity and requeue event type. + /// + /// + /// The type of the Kubernetes entity. Must implement . + /// + /// + /// The type of the requeue operation for the entity (e.g., Added, Modified, or Deleted). + /// + /// + /// The Kubernetes entity associated with the logging scope. This includes metadata such as Kind, Namespace, Name, UID, and ResourceVersion. + /// + /// + /// A new instance containing contextual key-value pairs + /// related to the requeue event type and the provided Kubernetes entity. + /// + public static EntityLoggingScope CreateFor(RequeueType eventType, TEntity entity) + where TEntity : IKubernetesObject + => CreateLoggingScope(eventType.ToString(), ReconciliationTriggerSource.Operator, entity); /// public IEnumerator> GetEnumerator() @@ -68,4 +82,18 @@ public override string ToString() /// IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + private static EntityLoggingScope CreateLoggingScope(string eventType, ReconciliationTriggerSource triggerSource, TEntity entity) + where TEntity : IKubernetesObject + => new( + new Dictionary + { + { "EventType", eventType }, + { "ReconciliationTriggerSource", triggerSource }, + { nameof(entity.Kind), entity.Kind }, + { "Namespace", entity.Namespace() }, + { "Name", entity.Name() }, + { "Uid", entity.Uid() }, + { "ResourceVersion", entity.ResourceVersion() }, + }); } diff --git a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs index c7580cc3..eab9449d 100644 --- a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs +++ b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs @@ -2,22 +2,32 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. +using System.Diagnostics; + using k8s; using k8s.Models; -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; +using KubeOps.Operator.Logging; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace KubeOps.Operator.Queue; -internal sealed class EntityRequeueBackgroundService( +/// +/// A background service responsible for managing the requeue mechanism of Kubernetes entities. +/// It processes entities from a timed queue and invokes the reconciliation logic for each entity. +/// +/// +/// The type of the Kubernetes entity being managed. This entity must implement the interface. +/// +public class EntityRequeueBackgroundService( + ActivitySource activitySource, IKubernetesClient client, - TimedEntityQueue queue, - IServiceProvider provider, + ITimedEntityQueue queue, + IReconciler reconciler, ILogger> logger) : IHostedService, IDisposable, IAsyncDisposable where TEntity : IKubernetesObject { @@ -53,6 +63,23 @@ public Task StopAsync(CancellationToken cancellationToken) public void Dispose() { + Dispose(true); + GC.SuppressFinalize(this); + } + + public async ValueTask DisposeAsync() + { + await DisposeAsync(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposing) + { + return; + } + _cts.Dispose(); client.Dispose(); queue.Dispose(); @@ -60,13 +87,19 @@ public void Dispose() _disposed = true; } - public async ValueTask DisposeAsync() + protected virtual async ValueTask DisposeAsync(bool disposing) { + if (!disposing) + { + return; + } + await CastAndDispose(_cts); await CastAndDispose(client); await CastAndDispose(queue); _disposed = true; + return; static async ValueTask CastAndDispose(IDisposable resource) { @@ -81,13 +114,35 @@ static async ValueTask CastAndDispose(IDisposable resource) } } + protected virtual async Task ReconcileSingleAsync(RequeueEntry entry, CancellationToken cancellationToken) + { + using var activity = activitySource.StartActivity($"""Processing requeued "{entry.RequeueType}" event""", ActivityKind.Consumer); + using var scope = logger.BeginScope(EntityLoggingScope.CreateFor(entry.RequeueType, entry.Entity)); + + logger.LogTrace("""Executing requested requeued reconciliation for "{Name}".""", entry.Entity.Name()); + + if (await client.GetAsync(entry.Entity.Name(), entry.Entity.Namespace(), cancellationToken) is not + { } entity) + { + logger.LogWarning( + """Requeued entity "{Name}" was not found. Skipping reconciliation.""", entry.Entity.Name()); + return; + } + + await reconciler.Reconcile( + ReconciliationContext.CreateFromOperatorEvent( + entity, + entry.RequeueType.ToWatchEventType()), + cancellationToken); + } + private async Task WatchAsync(CancellationToken cancellationToken) { - await foreach (var entity in queue) + await foreach (var entry in queue) { try { - await ReconcileSingleAsync(entity, cancellationToken); + await ReconcileSingleAsync(entry, cancellationToken); } catch (OperationCanceledException e) when (!cancellationToken.IsCancellationRequested) { @@ -95,8 +150,8 @@ private async Task WatchAsync(CancellationToken cancellationToken) e, """Queued reconciliation for the entity of type {ResourceType} for "{Kind}/{Name}" failed.""", typeof(TEntity).Name, - entity.Kind, - entity.Name()); + entry.Entity.Kind, + entry.Entity.Name()); } catch (Exception e) { @@ -104,26 +159,9 @@ private async Task WatchAsync(CancellationToken cancellationToken) e, """Queued reconciliation for the entity of type {ResourceType} for "{Kind}/{Name}" failed.""", typeof(TEntity).Name, - entity.Kind, - entity.Name()); + entry.Entity.Kind, + entry.Entity.Name()); } } } - - private async Task ReconcileSingleAsync(TEntity queued, CancellationToken cancellationToken) - { - logger.LogTrace("""Execute requested requeued reconciliation for "{Name}".""", queued.Name()); - - if (await client.GetAsync(queued.Name(), queued.Namespace(), cancellationToken) is not - { } entity) - { - logger.LogWarning( - """Requeued entity "{Name}" was not found. Skipping reconciliation.""", queued.Name()); - return; - } - - await using var scope = provider.CreateAsyncScope(); - var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.ReconcileAsync(entity, cancellationToken); - } } diff --git a/src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs b/src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs new file mode 100644 index 00000000..9aa26af0 --- /dev/null +++ b/src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Reconciliation.Queue; + +namespace KubeOps.Operator.Queue; + +/// +/// Represents a timed queue for managing Kubernetes entities of type . +/// This interface provides mechanisms to enqueue entities for later processing and remove entities from the queue. +/// +/// +/// The type of the Kubernetes entity. Must implement . +/// +public interface ITimedEntityQueue : IDisposable, IAsyncEnumerable> + where TEntity : IKubernetesObject +{ + /// + /// Adds the specified entity to the queue for processing after the specified time span has elapsed. + /// + /// The entity to be queued. + /// The type of requeue operation to handle (added, modified, or deleted). + /// The duration to wait before processing the entity. + /// A token that can be used to cancel the asynchronous operation. + /// A task that represents the asynchronous operation of enqueuing the entity. + Task Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn, CancellationToken cancellationToken); + + /// + /// Removes the specified entity from the queue. + /// + /// The entity to be removed from the queue. + /// A token that can be used to cancel the asynchronous operation. + /// A task that represents the asynchronous operation of removing the entity from the queue. + Task Remove(TEntity entity, CancellationToken cancellationToken); +} diff --git a/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs b/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs index c6592fef..f12fe070 100644 --- a/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs +++ b/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation.Queue; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -17,10 +17,10 @@ internal sealed class KubeOpsEntityRequeueFactory(IServiceProvider services) { public EntityRequeue Create() where TEntity : IKubernetesObject => - (entity, timeSpan) => + (entity, type, timeSpan, cancellationToken) => { var logger = services.GetService>>(); - var queue = services.GetRequiredService>(); + var queue = services.GetRequiredService>(); logger?.LogTrace( """Requeue entity "{Kind}/{Name}" in {Milliseconds}ms.""", @@ -28,6 +28,6 @@ public EntityRequeue Create() entity.Name(), timeSpan.TotalMilliseconds); - queue.Enqueue(entity, timeSpan); + queue.Enqueue(entity, type, timeSpan, cancellationToken); }; } diff --git a/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs b/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs new file mode 100644 index 00000000..720e23e6 --- /dev/null +++ b/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using KubeOps.Abstractions.Reconciliation.Queue; + +namespace KubeOps.Operator.Queue; + +/// +/// Represents an entry in a requeue system for managing entities of type . +/// The requeue system facilitates the categorization and reprocessing of entities based on their +/// lifecycle events, such as added, modified, or deleted. +/// +/// +/// The type of the entity associated with the requeue entry. +/// +public readonly record struct RequeueEntry(TEntity Entity, RequeueType RequeueType); diff --git a/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs b/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs new file mode 100644 index 00000000..50a78fd5 --- /dev/null +++ b/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; + +using KubeOps.Abstractions.Reconciliation.Queue; + +namespace KubeOps.Operator.Queue; + +/// +/// Provides extension methods for converting between and . +/// +public static class RequeueTypeExtensions +{ + /// + /// Converts a to its corresponding . + /// + /// The watch event type to be converted. + /// The corresponding for the given . + /// Thrown when the provided is not supported. + public static RequeueType ToRequeueType(this WatchEventType watchEventType) + => watchEventType switch + { + WatchEventType.Added => RequeueType.Added, + WatchEventType.Modified => RequeueType.Modified, + WatchEventType.Deleted => RequeueType.Deleted, + _ => throw new NotSupportedException($"WatchEventType '{watchEventType}' is not supported!"), + }; + + /// + /// Converts a to its corresponding . + /// + /// The requeue type to be converted. + /// The corresponding for the given . + /// Thrown when the provided is not supported. + public static WatchEventType ToWatchEventType(this RequeueType requeueType) + => requeueType switch + { + RequeueType.Added => WatchEventType.Added, + RequeueType.Modified => WatchEventType.Modified, + RequeueType.Deleted => WatchEventType.Deleted, + _ => throw new NotSupportedException($"RequeueType '{requeueType}' is not supported!"), + }; +} diff --git a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs index 3bc228e8..5852df8e 100644 --- a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs +++ b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs @@ -7,6 +7,10 @@ using k8s; using k8s.Models; +using KubeOps.Abstractions.Reconciliation.Queue; + +using Microsoft.Extensions.Logging; + namespace KubeOps.Operator.Queue; /// @@ -14,7 +18,9 @@ namespace KubeOps.Operator.Queue; /// The given enumerable only contains items that should be considered for reconciliations. /// /// The type of the inner entity. -public sealed class TimedEntityQueue : IDisposable +public sealed class TimedEntityQueue( + ILogger> logger) + : ITimedEntityQueue where TEntity : IKubernetesObject { // A shared task factory for all the created tasks. @@ -24,47 +30,58 @@ public sealed class TimedEntityQueue : IDisposable private readonly ConcurrentDictionary> _management = new(); // The actual queue containing all the entries that have to be reconciled. - private readonly BlockingCollection _queue = new(new ConcurrentQueue()); + private readonly BlockingCollection> _queue = new(new ConcurrentQueue>()); internal int Count => _management.Count; - /// - /// Enqueues the given to happen in . - /// If the item already exists, the existing entry is updated. - /// - /// The entity. - /// The time after , where the item is reevaluated again. - public void Enqueue(TEntity entity, TimeSpan requeueIn) + /// + public Task Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn, CancellationToken cancellationToken) { - _management.AddOrUpdate( - TimedEntityQueue.GetKey(entity) ?? throw new InvalidOperationException("Cannot enqueue entities without name."), - key => - { - var entry = new TimedQueueEntry(entity, requeueIn); - _scheduledEntries.StartNew( - async () => - { - await entry.AddAfterDelay(_queue); - _management.TryRemove(key, out _); - }, - entry.Token); - return entry; - }, - (key, oldEntry) => - { - oldEntry.Cancel(); - var entry = new TimedQueueEntry(entity, requeueIn); - _scheduledEntries.StartNew( - async () => - { - await entry.AddAfterDelay(_queue); - _management.TryRemove(key, out _); - }, - entry.Token); - return entry; - }); + _management + .AddOrUpdate( + this.GetKey(entity) ?? throw new InvalidOperationException("Cannot enqueue entities without name."), + key => + { + logger.LogTrace( + """Adding schedule for entity "{Kind}/{Name}" to reconcile in {Seconds}s.""", + entity.Kind, + entity.Name(), + requeueIn.TotalSeconds); + + var entry = new TimedQueueEntry(entity, type, requeueIn); + _scheduledEntries.StartNew( + async () => + { + await entry.AddAfterDelay(_queue); + _management.TryRemove(key, out _); + }, + entry.Token); + return entry; + }, + (key, oldEntry) => + { + logger.LogTrace( + """Updating schedule for entity "{Kind}/{Name}" to reconcile in {Seconds}s.""", + entity.Kind, + entity.Name(), + requeueIn.TotalSeconds); + + oldEntry.Cancel(); + var entry = new TimedQueueEntry(entity, type, requeueIn); + _scheduledEntries.StartNew( + async () => + { + await entry.AddAfterDelay(_queue); + _management.TryRemove(key, out _); + }, + entry.Token); + return entry; + }); + + return Task.CompletedTask; } + /// public void Dispose() { _queue.Dispose(); @@ -74,7 +91,8 @@ public void Dispose() } } - public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + /// + public async IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default) { await Task.Yield(); foreach (var entry in _queue.GetConsumingEnumerable(cancellationToken)) @@ -83,32 +101,20 @@ public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken canc } } - public void Remove(TEntity entity) + /// + public Task Remove(TEntity entity, CancellationToken cancellationToken) { - var key = TimedEntityQueue.GetKey(entity); + var key = this.GetKey(entity); if (key is null) { - return; + return Task.CompletedTask; } if (_management.Remove(key, out var task)) { task.Cancel(); } - } - - private static string? GetKey(TEntity entity) - { - if (entity.Name() is null) - { - return null; - } - - if (entity.Namespace() is null) - { - return entity.Name(); - } - return $"{entity.Namespace()}/{entity.Name()}"; + return Task.CompletedTask; } } diff --git a/src/KubeOps.Operator/Queue/TimedEntityQueueExtensions.cs b/src/KubeOps.Operator/Queue/TimedEntityQueueExtensions.cs new file mode 100644 index 00000000..3ac377d8 --- /dev/null +++ b/src/KubeOps.Operator/Queue/TimedEntityQueueExtensions.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Operator.Queue; + +/// +/// Provides extension methods for the interface. +/// +public static class TimedEntityQueueExtensions +{ + /// + /// Retrieves a unique key for the specified Kubernetes entity. The key is constructed + /// using the entity's namespace and name, if available. If the entity does not have + /// a valid name, the method returns null. + /// + /// + /// The type of the Kubernetes entity. Must implement . + /// + /// + /// The timed entity queue from which the key should be derived. + /// + /// + /// The Kubernetes entity for which the key will be retrieved. + /// + /// + /// A string representing the unique key for the entity, or null if the entity does not have a valid name. + /// + // ReSharper disable once UnusedParameter.Global + public static string? GetKey(this ITimedEntityQueue queue, TEntity entity) + where TEntity : IKubernetesObject + { + if (string.IsNullOrWhiteSpace(entity.Name())) + { + return null; + } + + return string.IsNullOrWhiteSpace(entity.Namespace()) + ? entity.Name() + : $"{entity.Namespace()}/{entity.Name()}"; + } +} diff --git a/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs b/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs index 72ece41e..846fc17e 100644 --- a/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs +++ b/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs @@ -4,6 +4,8 @@ using System.Collections.Concurrent; +using KubeOps.Abstractions.Reconciliation.Queue; + namespace KubeOps.Operator.Queue; internal sealed record TimedQueueEntry : IDisposable @@ -11,11 +13,13 @@ internal sealed record TimedQueueEntry : IDisposable private readonly CancellationTokenSource _cts = new(); private readonly TimeSpan _requeueIn; private readonly TEntity _entity; + private readonly RequeueType _requeueType; - public TimedQueueEntry(TEntity entity, TimeSpan requeueIn) + public TimedQueueEntry(TEntity entity, RequeueType requeueType, TimeSpan requeueIn) { _requeueIn = requeueIn; _entity = entity; + _requeueType = requeueType; } /// @@ -40,7 +44,7 @@ public void Cancel() /// /// The collection to add the entry to. /// A representing the asynchronous operation. - public async Task AddAfterDelay(BlockingCollection collection) + public async Task AddAfterDelay(BlockingCollection> collection) { try { @@ -50,7 +54,7 @@ public async Task AddAfterDelay(BlockingCollection collection) return; } - collection.TryAdd(_entity); + collection.TryAdd(new() { Entity = _entity, RequeueType = _requeueType }); } catch (TaskCanceledException) { diff --git a/src/KubeOps.Operator/README.md b/src/KubeOps.Operator/README.md index d274def0..962c1668 100644 --- a/src/KubeOps.Operator/README.md +++ b/src/KubeOps.Operator/README.md @@ -62,7 +62,7 @@ To create an entity, implement the `IKubernetesObject` interface. ```csharp [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] -public class V1TestEntity : +public sealed class V1TestEntity : CustomKubernetesEntity { public override string ToString() @@ -88,35 +88,30 @@ A controller reconciles a specific entity type. Implement controllers using the Example controller implementation: ```csharp -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Abstractions.Rbac; using KubeOps.KubernetesClient; using Microsoft.Extensions.Logging; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -public class V1TestEntityController : IEntityController +public sealed class V1TestEntityController : IEntityController { private readonly IKubernetesClient _client; - private readonly EntityFinalizerAttacher _finalizer1; private readonly ILogger _logger; public V1TestEntityController( IKubernetesClient client, - EntityFinalizerAttacher finalizer1, ILogger logger) { _client = client; - _finalizer1 = finalizer1; _logger = logger; } - public async Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { _logger.LogInformation("Reconciling entity {Entity}.", entity); - // Attach finalizer and get updated entity - entity = await _finalizer1(entity); - // Update status to indicate reconciliation in progress entity.Status.Status = "Reconciling"; entity = await _client.UpdateStatus(entity); @@ -124,22 +119,23 @@ public class V1TestEntityController : IEntityController // Update status to indicate reconciliation complete entity.Status.Status = "Reconciled"; await _client.UpdateStatus(entity); + + return ReconciliationResult.Success(entity); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { _logger.LogInformation("Entity {Entity} deleted.", entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } ``` This controller: -1. Attaches a finalizer to the entity -2. Updates the entity's status to indicate reconciliation is in progress -3. Updates the status again to indicate reconciliation is complete -4. Implements the required `DeletedAsync` method for handling deletion events +1. Updates the entity's status to indicate reconciliation is in progress +2. Updates the status again to indicate reconciliation is complete +3. Implements the required `DeletedAsync` method for handling deletion events > **CAUTION:** > Always use the returned values from modifying actions of the Kubernetes client. Failure to do so will result in "HTTP CONFLICT" errors due to the resource version field in the entity. @@ -151,17 +147,20 @@ This controller: A [finalizer](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) is a mechanism for asynchronous cleanup in Kubernetes. Implement finalizers using the `IEntityFinalizer` interface. -Finalizers are attached using an `EntityFinalizerAttacher` and are called when the entity is marked for deletion. +KubeOps provides automatic finalizer attachment and detachment to ensure proper resource cleanup. +If you need special handling this automation can be disabled by configuration. +Finalizers then are attached using an `EntityFinalizerAttacher` and are called when the entity is marked for deletion. ```csharp -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Finalizer; -public class FinalizerOne : IEntityFinalizer +public sealed class FinalizerOne : IEntityFinalizer { - public Task FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) { // Implement cleanup logic here - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } ``` @@ -171,4 +170,4 @@ public class FinalizerOne : IEntityFinalizer ## Documentation -For more information, visit the [documentation](https://dotnet.github.io/dotnet-operator-sdk/). +For more information, visit the [documentation](https://dotnet.github.io/dotnet-operator-sdk/). \ No newline at end of file diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs new file mode 100644 index 00000000..15608a34 --- /dev/null +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -0,0 +1,202 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Builder; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; +using KubeOps.KubernetesClient; +using KubeOps.Operator.Constants; +using KubeOps.Operator.Queue; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +using ZiggyCreatures.Caching.Fusion; + +namespace KubeOps.Operator.Reconciliation; + +/// +/// The Reconciler class provides mechanisms for handling creation, modification, and deletion +/// events for Kubernetes objects of the specified entity type. It implements the IReconciler +/// interface and facilitates the reconciliation of desired and actual states of the entity. +/// +/// +/// The type of the Kubernetes entity being reconciled. Must implement IKubernetesObject +/// with V1ObjectMeta. +/// +/// +/// This class leverages logging, caching, and client services to manage and process +/// Kubernetes objects effectively. It also uses internal queuing capabilities for entity +/// processing and requeuing. +/// +internal sealed class Reconciler( + ILogger> logger, + IFusionCacheProvider cacheProvider, + IServiceProvider serviceProvider, + OperatorSettings operatorSettings, + ITimedEntityQueue entityQueue, + IKubernetesClient client) + : IReconciler + where TEntity : IKubernetesObject +{ + private readonly IFusionCache _entityCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); + + public async Task> Reconcile(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) + { + var result = reconciliationContext.EventType switch + { + WatchEventType.Added or WatchEventType.Modified => + await ReconcileModification(reconciliationContext, cancellationToken), + WatchEventType.Deleted => + await ReconcileDeletion(reconciliationContext, cancellationToken), + _ => throw new NotSupportedException($"Reconciliation event type {reconciliationContext.EventType} is not supported!"), + }; + + if (result.RequeueAfter.HasValue) + { + await entityQueue + .Enqueue( + result.Entity, + reconciliationContext.EventType.ToRequeueType(), + result.RequeueAfter.Value, + cancellationToken); + } + + return result; + } + + private async Task> ReconcileModification(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) + { + switch (reconciliationContext.Entity) + { + case { Metadata.DeletionTimestamp: null }: + if (reconciliationContext.IsTriggeredByApiServer()) + { + var cachedGeneration = await _entityCache.TryGetAsync( + reconciliationContext.Entity.Uid(), + token: cancellationToken); + + // Check if entity-spec has changed through "Generation" value increment. Skip reconcile if not changed. + if (cachedGeneration.HasValue && cachedGeneration >= reconciliationContext.Entity.Generation()) + { + logger.LogDebug( + """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", + reconciliationContext.Entity.Kind, + reconciliationContext.Entity.Name()); + + return ReconciliationResult.Success(reconciliationContext.Entity); + } + + // update cached generation since generation now changed + await _entityCache.SetAsync( + reconciliationContext.Entity.Uid(), + reconciliationContext.Entity.Generation() ?? 1, + token: cancellationToken); + } + + return await ReconcileEntity(reconciliationContext.Entity, cancellationToken); + case { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } }: + return await ReconcileFinalizersSequential(reconciliationContext.Entity, cancellationToken); + default: + return ReconciliationResult.Success(reconciliationContext.Entity); + } + } + + private async Task> ReconcileDeletion(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) + { + await entityQueue + .Remove( + reconciliationContext.Entity, + cancellationToken); + + await using var scope = serviceProvider.CreateAsyncScope(); + var controller = scope.ServiceProvider.GetRequiredService>(); + var result = await controller.DeletedAsync(reconciliationContext.Entity, cancellationToken); + + if (result.IsSuccess) + { + await _entityCache.RemoveAsync(reconciliationContext.Entity.Uid(), token: cancellationToken); + } + + return result; + } + + private async Task> ReconcileEntity(TEntity entity, CancellationToken cancellationToken) + { + await entityQueue + .Remove( + entity, + cancellationToken); + + await using var scope = serviceProvider.CreateAsyncScope(); + + if (operatorSettings.AutoAttachFinalizers) + { + var finalizers = scope.ServiceProvider.GetKeyedServices>(KeyedService.AnyKey); + + foreach (var finalizer in finalizers) + { + entity.AddFinalizer(finalizer.GetIdentifierName(entity)); + } + + entity = await client.UpdateAsync(entity, cancellationToken); + } + + var controller = scope.ServiceProvider.GetRequiredService>(); + return await controller.ReconcileAsync(entity, cancellationToken); + } + + private async Task> ReconcileFinalizersSequential(TEntity entity, CancellationToken cancellationToken) + { + await entityQueue + .Remove( + entity, + cancellationToken); + + await using var scope = serviceProvider.CreateAsyncScope(); + + // the condition to call ReconcileFinalizersSequentialAsync is: + // { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } } + // which implies that there is at least a single finalizer + var identifier = entity.Finalizers()[0]; + + if (scope.ServiceProvider.GetKeyedService>(identifier) is not + { } finalizer) + { + logger.LogInformation( + """Entity "{Kind}/{Name}" is finalizing but this operator has no registered finalizers for the identifier {FinalizerIdentifier}.""", + entity.Kind, + entity.Name(), + identifier); + return ReconciliationResult.Success(entity); + } + + var result = await finalizer.FinalizeAsync(entity, cancellationToken); + + if (!result.IsSuccess) + { + return result; + } + + entity = result.Entity; + + if (operatorSettings.AutoDetachFinalizers) + { + entity.RemoveFinalizer(identifier); + entity = await client.UpdateAsync(entity, cancellationToken); + } + + logger.LogInformation( + """Entity "{Kind}/{Name}" finalized with "{Finalizer}".""", + entity.Kind, + entity.Name(), + identifier); + + return ReconciliationResult.Success(entity, result.RequeueAfter); + } +} diff --git a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs index 15f20a72..3cb1b710 100644 --- a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs @@ -10,35 +10,29 @@ using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Entities; +using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; -using KubeOps.Operator.Queue; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using ZiggyCreatures.Caching.Fusion; - namespace KubeOps.Operator.Watcher; internal sealed class LeaderAwareResourceWatcher( ActivitySource activitySource, ILogger> logger, - IServiceProvider provider, - TimedEntityQueue queue, + IReconciler reconciler, OperatorSettings settings, IEntityLabelSelector labelSelector, - IFusionCacheProvider cacheProvider, IKubernetesClient client, IHostApplicationLifetime hostApplicationLifetime, LeaderElector elector) : ResourceWatcher( activitySource, logger, - provider, - queue, + reconciler, settings, labelSelector, - cacheProvider, client) where TEntity : IKubernetesObject { @@ -94,7 +88,7 @@ private void StartedLeading() if (_cts.IsCancellationRequested) { _cts.Dispose(); - _cts = new CancellationTokenSource(); + _cts = new(); } base.StartAsync(_cts.Token); @@ -107,7 +101,7 @@ private void StoppedLeading() logger.LogInformation("This instance stopped leading, stopping watcher."); // Stop the base implementation using the 'ApplicationStopped' cancellation token. - // The cancellation token should only be marked cancelled when the stop should no longer be graceful. + // The cancellation token should only be marked as canceled when the stop should no longer be graceful. base.StopAsync(hostApplicationLifetime.ApplicationStopped).Wait(); } } diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index d88f3b66..bb2e9800 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -11,35 +11,26 @@ using k8s.Models; using KubeOps.Abstractions.Builder; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Entities; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; -using KubeOps.Operator.Constants; using KubeOps.Operator.Logging; -using KubeOps.Operator.Queue; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using ZiggyCreatures.Caching.Fusion; - namespace KubeOps.Operator.Watcher; public class ResourceWatcher( ActivitySource activitySource, ILogger> logger, - IServiceProvider provider, - TimedEntityQueue requeue, + IReconciler reconciler, OperatorSettings settings, IEntityLabelSelector labelSelector, - IFusionCacheProvider cacheProvider, IKubernetesClient client) : IHostedService, IAsyncDisposable, IDisposable where TEntity : IKubernetesObject { - private readonly IFusionCache _entityCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); private CancellationTokenSource _cancellationTokenSource = new(); private uint _watcherReconnectRetries; private Task? _eventWatcher; @@ -54,7 +45,7 @@ public virtual Task StartAsync(CancellationToken cancellationToken) if (_cancellationTokenSource.IsCancellationRequested) { _cancellationTokenSource.Dispose(); - _cancellationTokenSource = new CancellationTokenSource(); + _cancellationTokenSource = new(); } _eventWatcher = WatchClientEventsAsync(_cancellationTokenSource.Token); @@ -102,7 +93,6 @@ protected virtual void Dispose(bool disposing) _cancellationTokenSource.Dispose(); _eventWatcher?.Dispose(); - requeue.Dispose(); client.Dispose(); _disposed = true; @@ -116,7 +106,6 @@ protected virtual async ValueTask DisposeAsyncCore() } await CastAndDispose(_cancellationTokenSource); - await CastAndDispose(requeue); await CastAndDispose(client); _disposed = true; @@ -136,71 +125,15 @@ static async ValueTask CastAndDispose(IDisposable resource) } } - protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) - { - MaybeValue cachedGeneration; - - // Make sure Finalizers are running if Termination has began. - if (type != WatchEventType.Deleted && entity.Metadata.DeletionTimestamp is not null && entity.Metadata.Finalizers.Count > 0) - { - await ReconcileFinalizersSequentialAsync(entity, cancellationToken); - return; - } - - switch (type) - { - case WatchEventType.Added: - cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); - - if (!cachedGeneration.HasValue) - { - // Only perform reconciliation if the entity was not already in the cache. - await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 0, token: cancellationToken); - await ReconcileModificationAsync(entity, cancellationToken); - } - else - { - logger.LogDebug( - """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", - entity.Kind, - entity.Name()); - } - - break; - case WatchEventType.Modified: - cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); - - // Check if entity spec has changed through "Generation" value increment. Skip reconcile if not changed. - if (cachedGeneration.HasValue && cachedGeneration >= entity.Generation()) - { - logger.LogDebug( - """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", - entity.Kind, - entity.Name()); - return; - } - - // update cached generation since generation now changed - await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 1, token: cancellationToken); - await ReconcileModificationAsync(entity, cancellationToken); - - break; - case WatchEventType.Deleted: - await ReconcileDeletionAsync(entity, cancellationToken); - break; - default: - logger.LogWarning( - """Received unsupported event "{EventType}" for "{Kind}/{Name}".""", - type, - entity.Kind, - entity.Name()); - break; - } - } + protected virtual async Task> OnEventAsync(WatchEventType eventType, TEntity entity, CancellationToken cancellationToken) + => await reconciler.Reconcile( + ReconciliationContext.CreateFromApiServerEvent(entity, eventType), + cancellationToken); private async Task WatchClientEventsAsync(CancellationToken stoppingToken) { string? currentVersion = null; + while (!stoppingToken.IsCancellationRequested) { try @@ -212,9 +145,9 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) allowWatchBookmarks: true, cancellationToken: stoppingToken)) { -#pragma warning disable SA1312 - using var __ = activitySource.StartActivity($"""processing "{type}" event""", ActivityKind.Consumer); - using var _ = logger.BeginScope(EntityLoggingScope.CreateFor(type, entity)); + using var activity = activitySource.StartActivity($"""processing "{type}" event""", ActivityKind.Consumer); + using var scope = logger.BeginScope(EntityLoggingScope.CreateFor(type, entity)); + logger.LogInformation( """Received watch event "{EventType}" for "{Kind}/{Name}", last observed resource version: {ResourceVersion}.""", type, @@ -230,7 +163,18 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) try { - await OnEventAsync(type, entity, stoppingToken); + var result = await OnEventAsync(type, entity, stoppingToken); + + if (!result.IsSuccess) + { + logger.LogError( + result.Error, + "Reconciliation of {EventType} for {Kind}/{Name} failed with message '{Message}'.", + type, + entity.Kind, + entity.Name(), + result.ErrorMessage); + } } catch (KubernetesException e) when (e.Status.Code is (int)HttpStatusCode.GatewayTimeout) { @@ -243,14 +187,9 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) throw; } catch (Exception e) - { - LogReconciliationFailed(e); - } - - void LogReconciliationFailed(Exception exception) { logger.LogError( - exception, + e, "Reconciliation of {EventType} for {Kind}/{Name} failed.", type, entity.Kind, @@ -317,55 +256,4 @@ e.InnerException is EndOfStreamException && delay.TotalSeconds); await Task.Delay(delay); } - - private async Task ReconcileDeletionAsync(TEntity entity, CancellationToken cancellationToken) - { - requeue.Remove(entity); - await _entityCache.RemoveAsync(entity.Uid(), token: cancellationToken); - - await using var scope = provider.CreateAsyncScope(); - var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.DeletedAsync(entity, cancellationToken); - } - - private async Task ReconcileFinalizersSequentialAsync(TEntity entity, CancellationToken cancellationToken) - { - requeue.Remove(entity); - await using var scope = provider.CreateAsyncScope(); - - var identifier = entity.Finalizers().FirstOrDefault(); - if (identifier is null) - { - return; - } - - if (scope.ServiceProvider.GetKeyedService>(identifier) is not - { } finalizer) - { - logger.LogDebug( - """Entity "{Kind}/{Name}" is finalizing but this operator has no registered finalizers for the identifier {FinalizerIdentifier}.""", - entity.Kind, - entity.Name(), - identifier); - return; - } - - await finalizer.FinalizeAsync(entity, cancellationToken); - entity.RemoveFinalizer(identifier); - await client.UpdateAsync(entity, cancellationToken); - logger.LogInformation( - """Entity "{Kind}/{Name}" finalized with "{Finalizer}".""", - entity.Kind, - entity.Name(), - identifier); - } - - private async Task ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) - { - // Re-queue should requested in the controller reconcile method. Invalidate any existing queues. - requeue.Remove(entity); - await using var scope = provider.CreateAsyncScope(); - var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.ReconcileAsync(entity, cancellationToken); - } } diff --git a/src/KubeOps.Templates/Templates/Operator.CSharp/Controller/DemoController.cs b/src/KubeOps.Templates/Templates/Operator.CSharp/Controller/DemoController.cs index bc7be892..dbcc72c2 100644 --- a/src/KubeOps.Templates/Templates/Operator.CSharp/Controller/DemoController.cs +++ b/src/KubeOps.Templates/Templates/Operator.CSharp/Controller/DemoController.cs @@ -1,4 +1,5 @@ -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Abstractions.Rbac; using Microsoft.Extensions.Logging; @@ -8,19 +9,19 @@ namespace GeneratedOperatorProject.Controller; [EntityRbac(typeof(V1DemoEntity), Verbs = RbacVerb.All)] -public class DemoController(ILogger logger) : IEntityController +public sealed class DemoController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1DemoEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1DemoEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Reconcile entity {MetadataName}", entity.Metadata.Name); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1DemoEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1DemoEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Deleted entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/src/KubeOps.Templates/Templates/Operator.CSharp/Entities/V1DemoEntity.cs b/src/KubeOps.Templates/Templates/Operator.CSharp/Entities/V1DemoEntity.cs index f5c4f00b..d06b954b 100644 --- a/src/KubeOps.Templates/Templates/Operator.CSharp/Entities/V1DemoEntity.cs +++ b/src/KubeOps.Templates/Templates/Operator.CSharp/Entities/V1DemoEntity.cs @@ -5,7 +5,7 @@ namespace GeneratedOperatorProject.Entities; [KubernetesEntity(Group = "demo.kubeops.dev", ApiVersion = "v1", Kind = "DemoEntity")] -public class V1DemoEntity : CustomKubernetesEntity +public sealed class V1DemoEntity : CustomKubernetesEntity { public class V1DemoEntitySpec { diff --git a/src/KubeOps.Templates/Templates/Operator.CSharp/Finalizer/DemoFinalizer.cs b/src/KubeOps.Templates/Templates/Operator.CSharp/Finalizer/DemoFinalizer.cs index 8d9cd92d..9c3095f6 100644 --- a/src/KubeOps.Templates/Templates/Operator.CSharp/Finalizer/DemoFinalizer.cs +++ b/src/KubeOps.Templates/Templates/Operator.CSharp/Finalizer/DemoFinalizer.cs @@ -1,6 +1,7 @@ using k8s.Models; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Finalizer; using Microsoft.Extensions.Logging; @@ -8,12 +9,12 @@ namespace GeneratedOperatorProject.Finalizer; -public class DemoFinalizer(ILogger logger) : IEntityFinalizer +public sealed class DemoFinalizer(ILogger logger) : IEntityFinalizer { - public Task FinalizeAsync(V1DemoEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1DemoEntity entity, CancellationToken cancellationToken) { logger.LogInformation($"entity {entity.Name()} called {nameof(FinalizeAsync)}."); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Controller/DemoController.cs b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Controller/DemoController.cs index bc7be892..dbcc72c2 100644 --- a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Controller/DemoController.cs +++ b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Controller/DemoController.cs @@ -1,4 +1,5 @@ -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Abstractions.Rbac; using Microsoft.Extensions.Logging; @@ -8,19 +9,19 @@ namespace GeneratedOperatorProject.Controller; [EntityRbac(typeof(V1DemoEntity), Verbs = RbacVerb.All)] -public class DemoController(ILogger logger) : IEntityController +public sealed class DemoController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1DemoEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1DemoEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Reconcile entity {MetadataName}", entity.Metadata.Name); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1DemoEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1DemoEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Deleted entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Entities/V1DemoEntity.cs b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Entities/V1DemoEntity.cs index f5c4f00b..d06b954b 100644 --- a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Entities/V1DemoEntity.cs +++ b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Entities/V1DemoEntity.cs @@ -5,7 +5,7 @@ namespace GeneratedOperatorProject.Entities; [KubernetesEntity(Group = "demo.kubeops.dev", ApiVersion = "v1", Kind = "DemoEntity")] -public class V1DemoEntity : CustomKubernetesEntity +public sealed class V1DemoEntity : CustomKubernetesEntity { public class V1DemoEntitySpec { diff --git a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Finalizer/DemoFinalizer.cs b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Finalizer/DemoFinalizer.cs index 8d9cd92d..9c3095f6 100644 --- a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Finalizer/DemoFinalizer.cs +++ b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Finalizer/DemoFinalizer.cs @@ -1,6 +1,7 @@ using k8s.Models; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Finalizer; using Microsoft.Extensions.Logging; @@ -8,12 +9,12 @@ namespace GeneratedOperatorProject.Finalizer; -public class DemoFinalizer(ILogger logger) : IEntityFinalizer +public sealed class DemoFinalizer(ILogger logger) : IEntityFinalizer { - public Task FinalizeAsync(V1DemoEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1DemoEntity entity, CancellationToken cancellationToken) { logger.LogInformation($"entity {entity.Name()} called {nameof(FinalizeAsync)}."); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj b/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj index 7bc34335..72f8a045 100644 --- a/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj +++ b/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj @@ -2,4 +2,4 @@ - + \ No newline at end of file diff --git a/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Abstractions.Test/Reconciliation/Finalizer/EntityFinalizerExtensions.Test.cs b/test/KubeOps.Abstractions.Test/Reconciliation/Finalizer/EntityFinalizerExtensions.Test.cs new file mode 100644 index 00000000..7f722c1c --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Reconciliation/Finalizer/EntityFinalizerExtensions.Test.cs @@ -0,0 +1,142 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Finalizer; + +namespace KubeOps.Abstractions.Test.Finalizer; + +public sealed class EntityFinalizerExtensions +{ + private const string Group = "finalizer.test"; + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Entity_Group_Has_String_Value() + { + var sut = new EntityWithGroupAsStringValueFinalizer(); + var entity = new EntityWithGroupAsStringValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be("finalizer.test/entitywithgroupasstringvaluefinalizer"); + } + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Entity_Group_Has_Const_Value() + { + var sut = new EntityWithGroupAsConstValueFinalizer(); + var entity = new EntityWithGroupAsConstValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be($"{Group}/entitywithgroupasconstvaluefinalizer"); + } + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Entity_Group_Has_No_Value() + { + var sut = new EntityWithNoGroupFinalizer(); + var entity = new EntityWithNoGroupValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be("entitywithnogroupfinalizer"); + } + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Finalizer_Not_Ending_With_Finalizer() + { + var sut = new EntityFinalizerNotEndingOnFinalizer1(); + var entity = new EntityWithGroupAsConstValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be($"{Group}/entityfinalizernotendingonfinalizer1finalizer"); + } + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Finalizer_Identifier_Would_Be_Greater_Than_63_Characters() + { + var sut = new EntityFinalizerWithATotalIdentifierNameHavingALengthGreaterThan63(); + var entity = new EntityWithGroupAsConstValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be($"{Group}/entityfinalizerwithatotalidentifiernamehavingale"); + identifierName.Length.Should().Be(63); + } + + private sealed class EntityFinalizerWithATotalIdentifierNameHavingALengthGreaterThan63 + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); + } + + private sealed class EntityFinalizerNotEndingOnFinalizer1 + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); + } + + private sealed class EntityWithGroupAsStringValueFinalizer + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityWithGroupAsStringValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); + } + + private sealed class EntityWithGroupAsConstValueFinalizer + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); + } + + private sealed class EntityWithNoGroupFinalizer + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityWithNoGroupValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); + } + + [KubernetesEntity(Group = "finalizer.test", ApiVersion = "v1", Kind = "FinalizerTest")] + private sealed class EntityWithGroupAsStringValue + : IKubernetesObject + { + public string ApiVersion { get; set; } = "finalizer.test/v1"; + + public string Kind { get; set; } = "FinalizerTest"; + + public V1ObjectMeta Metadata { get; set; } = new(); + } + + [KubernetesEntity(Group = Group, ApiVersion = "v1", Kind = "FinalizerTest")] + private sealed class EntityWithGroupAsConstValue + : IKubernetesObject + { + public string ApiVersion { get; set; } = "finalizer.test/v1"; + + public string Kind { get; set; } = "FinalizerTest"; + + public V1ObjectMeta Metadata { get; set; } = new(); + } + + [KubernetesEntity] + private sealed class EntityWithNoGroupValue + : IKubernetesObject + { + public string ApiVersion { get; set; } = "finalizer.test/v1"; + + public string Kind { get; set; } = "FinalizerTest"; + + public V1ObjectMeta Metadata { get; set; } = new(); + } +} diff --git a/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationContext.Test.cs b/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationContext.Test.cs new file mode 100644 index 00000000..41ef7d8f --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationContext.Test.cs @@ -0,0 +1,190 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Reconciliation; + +namespace KubeOps.Abstractions.Test.Reconciliation; + +public sealed class ReconciliationContextTest +{ + [Fact] + public void CreateFromApiServerEvent_Should_Create_Context_With_ApiServer_TriggerSource() + { + var entity = CreateTestEntity(); + const WatchEventType eventType = WatchEventType.Added; + + var context = ReconciliationContext.CreateFromApiServerEvent(entity, eventType); + + context.Entity.Should().Be(entity); + context.EventType.Should().Be(eventType); + context.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.ApiServer); + } + + [Fact] + public void CreateFromOperatorEvent_Should_Create_Context_With_Operator_TriggerSource() + { + var entity = CreateTestEntity(); + const WatchEventType eventType = WatchEventType.Modified; + + var context = ReconciliationContext.CreateFromOperatorEvent(entity, eventType); + + context.Entity.Should().Be(entity); + context.EventType.Should().Be(eventType); + context.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.Operator); + } + + [Theory] + [InlineData(WatchEventType.Added)] + [InlineData(WatchEventType.Modified)] + [InlineData(WatchEventType.Deleted)] + public void CreateFromApiServerEvent_Should_Support_All_WatchEventTypes(WatchEventType eventType) + { + var entity = CreateTestEntity(); + + var context = ReconciliationContext.CreateFromApiServerEvent(entity, eventType); + + context.EventType.Should().Be(eventType); + context.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.ApiServer); + } + + [Theory] + [InlineData(WatchEventType.Added)] + [InlineData(WatchEventType.Modified)] + [InlineData(WatchEventType.Deleted)] + public void CreateFromOperatorEvent_Should_Support_All_WatchEventTypes(WatchEventType eventType) + { + var entity = CreateTestEntity(); + + var context = ReconciliationContext.CreateFromOperatorEvent(entity, eventType); + + context.EventType.Should().Be(eventType); + context.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.Operator); + } + + [Fact] + public void IsTriggeredByApiServer_Should_Return_True_For_ApiServer_Context() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + var isTriggeredByApiServer = context.IsTriggeredByApiServer(); + var isTriggeredByOperator = context.IsTriggeredByOperator(); + + isTriggeredByApiServer.Should().BeTrue(); + isTriggeredByOperator.Should().BeFalse(); + } + + [Fact] + public void IsTriggeredByOperator_Should_Return_True_For_Operator_Context() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromOperatorEvent(entity, WatchEventType.Modified); + + var isTriggeredByOperator = context.IsTriggeredByOperator(); + var isTriggeredByApiServer = context.IsTriggeredByApiServer(); + + isTriggeredByOperator.Should().BeTrue(); + isTriggeredByApiServer.Should().BeFalse(); + } + + [Fact] + public void Record_Equality_Should_Work_For_Same_Values() + { + var entity = CreateTestEntity("test-entity"); + + var context1 = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var context2 = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + context1.Should().NotBeSameAs(context2); + context1.Entity.Should().BeSameAs(context2.Entity); + context1.EventType.Should().Be(context2.EventType); + context1.ReconciliationTriggerSource.Should().Be(context2.ReconciliationTriggerSource); + } + + [Fact] + public void Contexts_With_Different_EventTypes_Should_Have_Different_EventTypes() + { + var entity = CreateTestEntity(); + + var contextAdded = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var contextModified = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + var contextDeleted = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Deleted); + + contextAdded.EventType.Should().Be(WatchEventType.Added); + contextModified.EventType.Should().Be(WatchEventType.Modified); + contextDeleted.EventType.Should().Be(WatchEventType.Deleted); + } + + [Fact] + public void Contexts_With_Different_TriggerSources_Should_Have_Different_TriggerSources() + { + var entity = CreateTestEntity(); + + var apiServerContext = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var operatorContext = ReconciliationContext.CreateFromOperatorEvent(entity, WatchEventType.Added); + + apiServerContext.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.ApiServer); + operatorContext.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.Operator); + apiServerContext.ReconciliationTriggerSource.Should().NotBe(operatorContext.ReconciliationTriggerSource); + } + + [Fact] + public void Context_Should_Contain_Entity_Metadata() + { + var entity = CreateTestEntity("test-configmap", "test-namespace"); + + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + context.Entity.Metadata.Name.Should().Be("test-configmap"); + context.Entity.Metadata.NamespaceProperty.Should().Be("test-namespace"); + } + + [Theory] + [InlineData(ReconciliationTriggerSource.ApiServer, WatchEventType.Added)] + [InlineData(ReconciliationTriggerSource.ApiServer, WatchEventType.Modified)] + [InlineData(ReconciliationTriggerSource.ApiServer, WatchEventType.Deleted)] + [InlineData(ReconciliationTriggerSource.Operator, WatchEventType.Added)] + [InlineData(ReconciliationTriggerSource.Operator, WatchEventType.Modified)] + [InlineData(ReconciliationTriggerSource.Operator, WatchEventType.Deleted)] + public void Context_Should_Support_All_Combinations_Of_TriggerSource_And_EventType( + ReconciliationTriggerSource triggerSource, + WatchEventType eventType) + { + var entity = CreateTestEntity(); + + var context = triggerSource == ReconciliationTriggerSource.ApiServer + ? ReconciliationContext.CreateFromApiServerEvent(entity, eventType) + : ReconciliationContext.CreateFromOperatorEvent(entity, eventType); + + context.ReconciliationTriggerSource.Should().Be(triggerSource); + context.EventType.Should().Be(eventType); + } + + [Fact] + public void Multiple_Contexts_With_Same_Entity_Should_Share_Entity_Reference() + { + var entity = CreateTestEntity(); + + var context1 = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var context2 = ReconciliationContext.CreateFromOperatorEvent(entity, WatchEventType.Modified); + + context1.Entity.Should().BeSameAs(context2.Entity); + } + + private static V1ConfigMap CreateTestEntity(string? name = null, string? ns = null) + => new() + { + Metadata = new() + { + Name = name ?? "test-configmap", + NamespaceProperty = ns ?? "default", + Uid = Guid.NewGuid().ToString(), + }, + }; +} diff --git a/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationResult.Test.cs b/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationResult.Test.cs new file mode 100644 index 00000000..5d90dd42 --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationResult.Test.cs @@ -0,0 +1,258 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s.Models; + +using KubeOps.Abstractions.Reconciliation; + +namespace KubeOps.Abstractions.Test.Reconciliation; + +public sealed class ReconciliationResultTest +{ + [Fact] + public void Success_Should_Create_Successful_Result() + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Success(entity); + + result.IsSuccess.Should().BeTrue(); + result.Entity.Should().Be(entity); + result.ErrorMessage.Should().BeNull(); + result.Error.Should().BeNull(); + result.RequeueAfter.Should().BeNull(); + } + + [Fact] + public void Success_With_RequeueAfter_Should_Set_RequeueAfter() + { + var entity = CreateTestEntity(); + var requeueAfter = TimeSpan.FromMinutes(5); + + var result = ReconciliationResult.Success(entity, requeueAfter); + + result.IsSuccess.Should().BeTrue(); + result.RequeueAfter.Should().Be(requeueAfter); + result.Entity.Should().Be(entity); + } + + [Fact] + public void Success_With_Null_RequeueAfter_Should_Not_Set_RequeueAfter() + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Success(entity, null); + + result.IsSuccess.Should().BeTrue(); + result.RequeueAfter.Should().BeNull(); + } + + [Fact] + public void Failure_Should_Create_Failed_Result_With_ErrorMessage() + { + var entity = CreateTestEntity(); + var errorMessage = "Reconciliation failed due to timeout"; + + var result = ReconciliationResult.Failure(entity, errorMessage); + + result.IsSuccess.Should().BeFalse(); + result.Entity.Should().Be(entity); + result.ErrorMessage.Should().Be(errorMessage); + result.Error.Should().BeNull(); + result.RequeueAfter.Should().BeNull(); + } + + [Fact] + public void Failure_With_Exception_Should_Set_Error() + { + var entity = CreateTestEntity(); + const string errorMessage = "Reconciliation failed"; + var exception = new InvalidOperationException("Invalid state detected"); + + var result = ReconciliationResult.Failure(entity, errorMessage, exception); + + result.IsSuccess.Should().BeFalse(); + result.ErrorMessage.Should().Be(errorMessage); + result.Error.Should().Be(exception); + result.Error.Message.Should().Be("Invalid state detected"); + } + + [Fact] + public void Failure_With_RequeueAfter_Should_Set_RequeueAfter() + { + var entity = CreateTestEntity(); + const string errorMessage = "Transient failure"; + var requeueAfter = TimeSpan.FromSeconds(30); + + var result = ReconciliationResult.Failure( + entity, + errorMessage, + requeueAfter: requeueAfter); + + result.IsSuccess.Should().BeFalse(); + result.ErrorMessage.Should().Be(errorMessage); + result.RequeueAfter.Should().Be(requeueAfter); + } + + [Fact] + public void Failure_With_All_Parameters_Should_Set_All_Properties() + { + var entity = CreateTestEntity(); + const string errorMessage = "Complete failure information"; + var exception = new TimeoutException("Operation timed out"); + var requeueAfter = TimeSpan.FromMinutes(2); + + var result = ReconciliationResult.Failure( + entity, + errorMessage, + exception, + requeueAfter); + + result.IsSuccess.Should().BeFalse(); + result.ErrorMessage.Should().Be(errorMessage); + result.Error.Should().Be(exception); + result.RequeueAfter.Should().Be(requeueAfter); + } + + [Fact] + public void RequeueAfter_Should_Be_Mutable() + { + var entity = CreateTestEntity(); + var result = ReconciliationResult.Success(entity); + + result.RequeueAfter = TimeSpan.FromSeconds(45); + + result.RequeueAfter.Should().Be(TimeSpan.FromSeconds(45)); + } + + [Fact] + public void RequeueAfter_Can_Be_Changed_After_Creation() + { + var entity = CreateTestEntity(); + var initialRequeueAfter = TimeSpan.FromMinutes(1); + var result = ReconciliationResult.Success(entity, initialRequeueAfter); + + result.RequeueAfter = TimeSpan.FromMinutes(5); + + result.RequeueAfter.Should().Be(TimeSpan.FromMinutes(5)); + result.RequeueAfter.Should().NotBe(initialRequeueAfter); + } + + [Fact] + public void RequeueAfter_Can_Be_Set_To_Null() + { + var entity = CreateTestEntity(); + var result = ReconciliationResult.Success(entity, TimeSpan.FromMinutes(1)); + + result.RequeueAfter = null; + + result.RequeueAfter.Should().BeNull(); + } + + [Fact] + public void IsSuccess_And_IsFailure_Should_Be_Mutually_Exclusive() + { + var entity = CreateTestEntity(); + + var successResult = ReconciliationResult.Success(entity); + var failureResult = ReconciliationResult.Failure(entity, "Error"); + + successResult.IsSuccess.Should().BeTrue(); + failureResult.IsSuccess.Should().BeFalse(); + } + + [Fact] + public void Success_Result_ErrorMessage_Should_Be_Null() + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Success(entity); + + if (result.IsSuccess) + { + result.ErrorMessage.Should().BeNull(); + } + } + + [Fact] + public void Failure_Result_ErrorMessage_Should_Not_Be_Null() + { + var entity = CreateTestEntity(); + var errorMessage = "Something went wrong"; + + var result = ReconciliationResult.Failure(entity, errorMessage); + + if (!result.IsSuccess) + { + // This should compile without nullable warning due to MemberNotNullWhen attribute + string message = result.ErrorMessage; + message.Should().NotBeNull(); + message.Should().Be(errorMessage); + } + } + + [Fact] + public void Record_Equality_Should_Work_For_Success_Results() + { + var entity1 = CreateTestEntity("test-entity"); + var entity2 = CreateTestEntity("test-entity"); + + var result1 = ReconciliationResult.Success(entity1); + var result2 = ReconciliationResult.Success(entity2); + + // Records with same values should be equal + result1.Should().NotBeSameAs(result2); + } + + [Fact] + public void Entity_Reference_Should_Be_Preserved() + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Success(entity); + + result.Entity.Should().BeSameAs(entity); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(60)] + [InlineData(3600)] + public void Success_Should_Accept_Various_RequeueAfter_Values(int seconds) + { + var entity = CreateTestEntity(); + var requeueAfter = TimeSpan.FromSeconds(seconds); + + var result = ReconciliationResult.Success(entity, requeueAfter); + + result.RequeueAfter.Should().Be(requeueAfter); + } + + [Theory] + [InlineData("Short error")] + [InlineData("A much longer error message that contains detailed information about what went wrong")] + [InlineData("")] + public void Failure_Should_Accept_Various_ErrorMessage_Lengths(string errorMessage) + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Failure(entity, errorMessage); + + result.ErrorMessage.Should().Be(errorMessage); + } + + private static V1ConfigMap CreateTestEntity(string? name = null) + => new() + { + Metadata = new() + { + Name = name ?? "test-configmap", + NamespaceProperty = "default", + Uid = Guid.NewGuid().ToString(), + }, + }; +} diff --git a/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Generator.Test/ControllerRegistrationGenerator.Test.cs b/test/KubeOps.Generator.Test/ControllerRegistrationGenerator.Test.cs index 6477f6dc..021bb047 100644 --- a/test/KubeOps.Generator.Test/ControllerRegistrationGenerator.Test.cs +++ b/test/KubeOps.Generator.Test/ControllerRegistrationGenerator.Test.cs @@ -35,7 +35,7 @@ public static IOperatorBuilder RegisterControllers(this IOperatorBuilder builder [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Controller; + using KubeOps.Abstractions.Reconciliation.Controller; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public class V1TestEntity : IKubernetesObject @@ -65,7 +65,7 @@ public static IOperatorBuilder RegisterControllers(this IOperatorBuilder builder [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Controller; + using KubeOps.Abstractions.Reconciliation.Controller; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject diff --git a/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs b/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs index 6fa73317..fca9af3c 100644 --- a/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs +++ b/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs @@ -13,7 +13,7 @@ namespace KubeOps.Generator.Test; -public class FinalizerRegistrationGeneratorTest +public sealed class FinalizerRegistrationGeneratorTest { [Theory] [InlineData("", """ @@ -35,7 +35,45 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; + + public static class Constants + { + public const string Group = "testing.dev"; + public const string ApiVersion = "v1"; + public const string Kind = "TestEntity"; + } + + [KubernetesEntity(Group = Constants.Group, ApiVersion = Constants.ApiVersion, Kind = Constants.Kind)] + public sealed class V1TestEntity : IKubernetesObject + { + } + + public sealed class V1TestEntityFinalizer : IEntityFinalizer + { + } + """, """ + // + // This code was generated by a tool. + // Changes to this file may cause incorrect behavior and will be lost if the code is regenerated. + // + #pragma warning disable CS1591 + using KubeOps.Abstractions.Builder; + + public static class FinalizerRegistrations + { + public const string V1TestEntityFinalizerIdentifier = "testing.dev/v1testentityfinalizer"; + public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) + { + builder.AddFinalizer(V1TestEntityFinalizerIdentifier); + return builder; + } + } + """)] + [InlineData(""" + using k8s; + using k8s.Models; + using KubeOps.Abstractions.Reconciliation.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject @@ -66,7 +104,7 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject @@ -102,7 +140,7 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject @@ -139,7 +177,7 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject diff --git a/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Generator.Test/TestHelperExtensions.cs b/test/KubeOps.Generator.Test/TestHelperExtensions.cs index 8dc9edca..2f40cdec 100644 --- a/test/KubeOps.Generator.Test/TestHelperExtensions.cs +++ b/test/KubeOps.Generator.Test/TestHelperExtensions.cs @@ -19,7 +19,7 @@ public static Compilation CreateCompilation(this string source) ], [ MetadataReference.CreateFromFile(typeof(Binder).GetTypeInfo().Assembly.Location), - MetadataReference.CreateFromFile(typeof(Abstractions.Controller.IEntityController<>).GetTypeInfo().Assembly.Location), + MetadataReference.CreateFromFile(typeof(Abstractions.Reconciliation.Controller.IEntityController<>).GetTypeInfo().Assembly.Location), MetadataReference.CreateFromFile(typeof(k8s.IKubernetesObject<>).GetTypeInfo().Assembly.Location), ], new(OutputKind.DynamicallyLinkedLibrary)); diff --git a/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs index 34cf751a..105b024c 100644 --- a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs +++ b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs @@ -5,14 +5,14 @@ using FluentAssertions; using KubeOps.Abstractions.Builder; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Entities; using KubeOps.Abstractions.Events; -using KubeOps.Abstractions.Finalizer; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient.LabelSelectors; using KubeOps.Operator.Builder; -using KubeOps.Operator.Finalizer; using KubeOps.Operator.Queue; using KubeOps.Operator.Test.TestEntities; using KubeOps.Operator.Watcher; @@ -23,7 +23,7 @@ namespace KubeOps.Operator.Test.Builder; -public class OperatorBuilderTest +public sealed class OperatorBuilderTest { private readonly IOperatorBuilder _builder = new OperatorBuilder(new ServiceCollection(), new()); @@ -45,7 +45,6 @@ public void Should_Add_Default_Resources() [Fact] public void Should_Use_Specific_EntityLabelSelector_Implementation() { - // Arrange var services = new ServiceCollection(); // Register the default and specific implementations @@ -54,10 +53,8 @@ public void Should_Use_Specific_EntityLabelSelector_Implementation() var serviceProvider = services.BuildServiceProvider(); - // Act var resolvedService = serviceProvider.GetRequiredService>(); - // Assert Assert.IsType(resolvedService); } @@ -75,7 +72,7 @@ public void Should_Add_Controller_Resources() s.ImplementationType == typeof(ResourceWatcher) && s.Lifetime == ServiceLifetime.Singleton); _builder.Services.Should().Contain(s => - s.ServiceType == typeof(TimedEntityQueue) && + s.ServiceType == typeof(ITimedEntityQueue) && s.Lifetime == ServiceLifetime.Singleton); _builder.Services.Should().Contain(s => s.ServiceType == typeof(EntityRequeue) && @@ -96,7 +93,7 @@ public void Should_Add_Controller_Resources_With_Label_Selector() s.ImplementationType == typeof(ResourceWatcher) && s.Lifetime == ServiceLifetime.Singleton); _builder.Services.Should().Contain(s => - s.ServiceType == typeof(TimedEntityQueue) && + s.ServiceType == typeof(ITimedEntityQueue) && s.Lifetime == ServiceLifetime.Singleton); _builder.Services.Should().Contain(s => s.ServiceType == typeof(EntityRequeue) && @@ -124,7 +121,7 @@ public void Should_Add_Finalizer_Resources() [Fact] public void Should_Add_Leader_Elector() { - var builder = new OperatorBuilder(new ServiceCollection(), new() { EnableLeaderElection = true }); + var builder = new OperatorBuilder(new ServiceCollection(), new() { LeaderElectionType = LeaderElectionType.Single }); builder.Services.Should().Contain(s => s.ServiceType == typeof(k8s.LeaderElection.LeaderElector) && s.Lifetime == ServiceLifetime.Singleton); @@ -133,7 +130,7 @@ public void Should_Add_Leader_Elector() [Fact] public void Should_Add_LeaderAwareResourceWatcher() { - var builder = new OperatorBuilder(new ServiceCollection(), new() { EnableLeaderElection = true }); + var builder = new OperatorBuilder(new ServiceCollection(), new() { LeaderElectionType = LeaderElectionType.Single }); builder.AddController(); builder.Services.Should().Contain(s => @@ -146,22 +143,22 @@ public void Should_Add_LeaderAwareResourceWatcher() s.Lifetime == ServiceLifetime.Singleton); } - private class TestController : IEntityController + private sealed class TestController : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(ReconciliationResult.Success(entity)); - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(ReconciliationResult.Success(entity)); } - private class TestFinalizer : IEntityFinalizer + private sealed class TestFinalizer : IEntityFinalizer { - public Task FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(ReconciliationResult.Success(entity)); } - private class TestLabelSelector : IEntityLabelSelector + private sealed class TestLabelSelector : IEntityLabelSelector { public ValueTask GetLabelSelectorAsync(CancellationToken cancellationToken) { diff --git a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs index 2381ffc7..a6adf42c 100644 --- a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs @@ -4,8 +4,9 @@ using FluentAssertions; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Queue; using KubeOps.Operator.Test.TestEntities; @@ -15,7 +16,7 @@ namespace KubeOps.Operator.Test.Controller; -public class CancelEntityRequeueIntegrationTest : IntegrationTestBase +public sealed class CancelEntityRequeueIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -35,7 +36,10 @@ public async Task Should_Cancel_Requeue_If_New_Event_Fires() await _mock.WaitForInvocations; _mock.Invocations.Count.Should().Be(2); - Services.GetRequiredService>().Count.Should().Be(0); + var timedEntityQueue = Services.GetRequiredService>(); + timedEntityQueue.Should().NotBeNull(); + timedEntityQueue.Should().BeOfType>(); + timedEntityQueue.As>().Count.Should().Be(0); } [Fact] @@ -48,8 +52,11 @@ public async Task Should_Not_Affect_Queues_If_Only_Status_Updated() await _mock.WaitForInvocations; _mock.Invocations.Count.Should().Be(1); - Services.GetRequiredService>().Count.Should().Be(1); + var timedEntityQueue = Services.GetRequiredService>(); + timedEntityQueue.Should().NotBeNull(); + timedEntityQueue.Should().BeOfType>(); + timedEntityQueue.As>().Count.Should().Be(1); } public override async Task InitializeAsync() @@ -77,20 +84,20 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (svc.Invocations.Count < 2) { - requeue(entity, TimeSpan.FromMilliseconds(1000)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1000), CancellationToken.None); } - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs index 27e86443..c6021993 100644 --- a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs @@ -4,8 +4,9 @@ using FluentAssertions; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Queue; using KubeOps.Operator.Test.TestEntities; @@ -15,7 +16,7 @@ namespace KubeOps.Operator.Test.Controller; -public class DeletedEntityRequeueIntegrationTest : IntegrationTestBase +public sealed class DeletedEntityRequeueIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -31,7 +32,10 @@ public async Task Should_Cancel_Requeue_If_Entity_Is_Deleted() await _mock.WaitForInvocations; _mock.Invocations.Count.Should().Be(2); - Services.GetRequiredService>().Count.Should().Be(0); + var timedEntityQueue = Services.GetRequiredService>(); + timedEntityQueue.Should().NotBeNull(); + timedEntityQueue.Should().BeOfType>(); + timedEntityQueue.As>().Count.Should().Be(0); } public override async Task InitializeAsync() @@ -59,17 +63,17 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - requeue(entity, TimeSpan.FromMilliseconds(1000)); - return Task.CompletedTask; + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1000), CancellationToken.None); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs index 6f0921db..fee650d9 100644 --- a/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs @@ -6,7 +6,8 @@ using k8s.Models; -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -120,16 +121,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs index e600f280..2101286b 100644 --- a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs @@ -4,8 +4,9 @@ using FluentAssertions; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -14,7 +15,7 @@ namespace KubeOps.Operator.Test.Controller; -public class EntityRequeueIntegrationTest : IntegrationTestBase +public sealed class EntityRequeueIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -88,18 +89,18 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (svc.Invocations.Count <= svc.TargetInvocationCount) { - requeue(entity, TimeSpan.FromMilliseconds(1)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1), CancellationToken.None); } - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs index 0e5239fc..60d4334b 100644 --- a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs @@ -9,9 +9,10 @@ using k8s.Models; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Events; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -20,7 +21,7 @@ namespace KubeOps.Operator.Test.Events; -public class EventPublisherIntegrationTest : IntegrationTestBase +public sealed class EventPublisherIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -90,20 +91,20 @@ private class TestController( EventPublisher eventPublisher) : IEntityController { - public async Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { await eventPublisher(entity, "REASON", "message", cancellationToken: cancellationToken); svc.Invocation(entity); if (svc.Invocations.Count < svc.TargetInvocationCount) { - requeue(entity, TimeSpan.FromMilliseconds(10)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(10), CancellationToken.None); } - } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) - { - return Task.CompletedTask; + return ReconciliationResult.Success(entity); } + + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs index 36f979f8..54574859 100644 --- a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs @@ -6,8 +6,9 @@ using k8s.Models; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -204,7 +205,12 @@ protected override void ConfigureHost(HostApplicationBuilder builder) { builder.Services .AddSingleton(_mock) - .AddKubernetesOperator(s => s.Namespace = _ns.Namespace) + .AddKubernetesOperator(s => + { + s.Namespace = _ns.Namespace; + s.AutoAttachFinalizers = false; + s.AutoDetachFinalizers = true; + }) .AddController() .AddFinalizer("first") .AddFinalizer("second"); @@ -215,42 +221,44 @@ private class TestController(InvocationCounter EntityFinalizerAttacher second) : IEntityController { - public async Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (entity.Name().Contains("first")) { - entity = await first(entity); + entity = await first(entity, cancellationToken); } if (entity.Name().Contains("second")) { - await second(entity); + entity = await second(entity, cancellationToken); } + + return ReconciliationResult.Success(entity); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } private class FirstFinalizer(InvocationCounter svc) : IEntityFinalizer { - public Task FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } private class SecondFinalizer(InvocationCounter svc) : IEntityFinalizer { - public Task FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs b/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs index a86ef250..baf0f5c2 100644 --- a/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs @@ -2,28 +2,30 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Builder; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Operator.Test.TestEntities; using Microsoft.Extensions.Hosting; namespace KubeOps.Operator.Test.HostedServices; -public class LeaderAwareHostedServiceDisposeIntegrationTest : HostedServiceDisposeIntegrationTest +public sealed class LeaderAwareHostedServiceDisposeIntegrationTest : HostedServiceDisposeIntegrationTest { protected override void ConfigureHost(HostApplicationBuilder builder) { builder.Services - .AddKubernetesOperator(op => op.EnableLeaderElection = true) + .AddKubernetesOperator(op => op.LeaderElectionType = LeaderElectionType.Single) .AddController(); } - private class TestController : IEntityController + private sealed class TestController : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs b/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs index 1a45c3ff..d4b58d00 100644 --- a/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs @@ -2,7 +2,8 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Operator.Test.TestEntities; using Microsoft.Extensions.DependencyInjection; @@ -49,10 +50,10 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Operator.Test/KubeOps.Operator.Test.csproj b/test/KubeOps.Operator.Test/KubeOps.Operator.Test.csproj index 4fa0d4a5..1c04d7d5 100644 --- a/test/KubeOps.Operator.Test/KubeOps.Operator.Test.csproj +++ b/test/KubeOps.Operator.Test/KubeOps.Operator.Test.csproj @@ -14,4 +14,4 @@ - + \ No newline at end of file diff --git a/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs b/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs index 3de6889d..ce541f6e 100644 --- a/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs @@ -6,7 +6,9 @@ using k8s.Models; -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Builder; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -49,22 +51,22 @@ protected override void ConfigureHost(HostApplicationBuilder builder) { builder.Services .AddSingleton(_mock) - .AddKubernetesOperator(s => s.EnableLeaderElection = true) + .AddKubernetesOperator(s => s.LeaderElectionType = LeaderElectionType.Single) .AddController(); } private class TestController(InvocationCounter svc) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs index 936fe057..1116a667 100644 --- a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs @@ -7,7 +7,8 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -86,16 +87,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Operator.Test/Queue/RequeueTypeExtensions.Test.cs b/test/KubeOps.Operator.Test/Queue/RequeueTypeExtensions.Test.cs new file mode 100644 index 00000000..d59a87cc --- /dev/null +++ b/test/KubeOps.Operator.Test/Queue/RequeueTypeExtensions.Test.cs @@ -0,0 +1,173 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s; + +using KubeOps.Abstractions.Reconciliation.Queue; +using KubeOps.Operator.Queue; + +namespace KubeOps.Operator.Test.Queue; + +public sealed class RequeueTypeExtensionsTest +{ + [Theory] + [InlineData(WatchEventType.Added, RequeueType.Added)] + [InlineData(WatchEventType.Modified, RequeueType.Modified)] + [InlineData(WatchEventType.Deleted, RequeueType.Deleted)] + public void ToRequeueType_Should_Convert_WatchEventType_Correctly( + WatchEventType watchEventType, + RequeueType expectedRequeueType) + { + var result = watchEventType.ToRequeueType(); + + result.Should().Be(expectedRequeueType); + } + + [Theory] + [InlineData(RequeueType.Added, WatchEventType.Added)] + [InlineData(RequeueType.Modified, WatchEventType.Modified)] + [InlineData(RequeueType.Deleted, WatchEventType.Deleted)] + public void ToWatchEventType_Should_Convert_RequeueType_Correctly( + RequeueType requeueType, + WatchEventType expectedWatchEventType) + { + var result = requeueType.ToWatchEventType(); + + result.Should().Be(expectedWatchEventType); + } + + [Fact] + public void ToRequeueType_Should_Throw_For_Unsupported_WatchEventType() + { + var unsupportedType = (WatchEventType)999; + + Action act = () => unsupportedType.ToRequeueType(); + + act.Should().Throw() + .WithMessage("*WatchEventType*999*not supported*"); + } + + [Fact] + public void ToWatchEventType_Should_Throw_For_Unsupported_RequeueType() + { + var unsupportedType = (RequeueType)999; + + Action act = () => unsupportedType.ToWatchEventType(); + + act.Should().Throw() + .WithMessage("*RequeueType*999*not supported*"); + } + + [Theory] + [InlineData(WatchEventType.Added)] + [InlineData(WatchEventType.Modified)] + [InlineData(WatchEventType.Deleted)] + public void Bidirectional_Conversion_Should_Be_Reversible_From_WatchEventType(WatchEventType original) + { + var requeueType = original.ToRequeueType(); + var converted = requeueType.ToWatchEventType(); + + converted.Should().Be(original); + } + + [Theory] + [InlineData(RequeueType.Added)] + [InlineData(RequeueType.Modified)] + [InlineData(RequeueType.Deleted)] + public void Bidirectional_Conversion_Should_Be_Reversible_From_RequeueType(RequeueType original) + { + var watchEventType = original.ToWatchEventType(); + var converted = watchEventType.ToRequeueType(); + + converted.Should().Be(original); + } + + [Fact] + public void ToRequeueType_Should_Handle_Added_EventType() + { + var eventType = WatchEventType.Added; + + var result = eventType.ToRequeueType(); + + result.Should().Be(RequeueType.Added); + } + + [Fact] + public void ToRequeueType_Should_Handle_Modified_EventType() + { + var eventType = WatchEventType.Modified; + + var result = eventType.ToRequeueType(); + + result.Should().Be(RequeueType.Modified); + } + + [Fact] + public void ToRequeueType_Should_Handle_Deleted_EventType() + { + var eventType = WatchEventType.Deleted; + + var result = eventType.ToRequeueType(); + + result.Should().Be(RequeueType.Deleted); + } + + [Fact] + public void ToWatchEventType_Should_Handle_Added_RequeueType() + { + var requeueType = RequeueType.Added; + + var result = requeueType.ToWatchEventType(); + + result.Should().Be(WatchEventType.Added); + } + + [Fact] + public void ToWatchEventType_Should_Handle_Modified_RequeueType() + { + var requeueType = RequeueType.Modified; + + var result = requeueType.ToWatchEventType(); + + result.Should().Be(WatchEventType.Modified); + } + + [Fact] + public void ToWatchEventType_Should_Handle_Deleted_RequeueType() + { + var requeueType = RequeueType.Deleted; + + var result = requeueType.ToWatchEventType(); + + result.Should().Be(WatchEventType.Deleted); + } + + [Fact] + public void All_RequeueTypes_Should_Have_Corresponding_WatchEventType() + { + var allRequeueTypes = Enum.GetValues(); + + // Act & Assert + foreach (var requeueType in allRequeueTypes) + { + Action act = () => requeueType.ToWatchEventType(); + act.Should().NotThrow($"RequeueType.{requeueType} should have a corresponding WatchEventType"); + } + } + + [Fact] + public void Supported_WatchEventTypes_Should_Have_Corresponding_RequeueType() + { + var supportedWatchEventTypes = new[] { WatchEventType.Added, WatchEventType.Modified, WatchEventType.Deleted }; + + // Act & Assert + foreach (var watchEventType in supportedWatchEventTypes) + { + Action act = () => watchEventType.ToRequeueType(); + act.Should().NotThrow($"WatchEventType.{watchEventType} should have a corresponding RequeueType"); + } + } +} diff --git a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs index 5d0a7c78..8f275244 100644 --- a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs +++ b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs @@ -4,19 +4,24 @@ using k8s.Models; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.Operator.Queue; +using Microsoft.Extensions.Logging; + +using Moq; + namespace KubeOps.Operator.Test.Queue; -public class TimedEntityQueueTest +public sealed class TimedEntityQueueTest { [Fact] public async Task Can_Enqueue_Multiple_Entities_With_Same_Name() { - var queue = new TimedEntityQueue(); + var queue = new TimedEntityQueue(Mock.Of>>()); - queue.Enqueue(CreateSecret("app-ns1", "secret-name"), TimeSpan.FromSeconds(1)); - queue.Enqueue(CreateSecret("app-ns2", "secret-name"), TimeSpan.FromSeconds(1)); + await queue.Enqueue(CreateSecret("app-ns1", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1), CancellationToken.None); + await queue.Enqueue(CreateSecret("app-ns2", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1), CancellationToken.None); var items = new List(); @@ -29,7 +34,7 @@ public async Task Can_Enqueue_Multiple_Entities_With_Same_Name() { while (await enumerator.MoveNextAsync()) { - items.Add(enumerator.Current); + items.Add(enumerator.Current.Entity); } } catch (OperationCanceledException) @@ -40,7 +45,7 @@ public async Task Can_Enqueue_Multiple_Entities_With_Same_Name() Assert.Equal(2, items.Count); } - private V1Secret CreateSecret(string secretNamespace, string secretName) + private static V1Secret CreateSecret(string secretNamespace, string secretName) { var secret = new V1Secret(); secret.EnsureMetadata(); diff --git a/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs b/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs new file mode 100644 index 00000000..9b6eab5e --- /dev/null +++ b/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs @@ -0,0 +1,536 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Builder; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; +using KubeOps.Abstractions.Reconciliation.Queue; +using KubeOps.KubernetesClient; +using KubeOps.Operator.Queue; +using KubeOps.Operator.Reconciliation; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +using Moq; + +using ZiggyCreatures.Caching.Fusion; + +namespace KubeOps.Operator.Test.Reconciliation; + +public sealed class ReconcilerTest +{ + private readonly Mock>> _mockLogger; + private readonly Mock _mockCacheProvider; + private readonly Mock _mockCache; + private readonly Mock _mockServiceProvider; + private readonly Mock _mockClient; + private readonly Mock> _mockQueue; + private readonly OperatorSettings _settings; + + public ReconcilerTest() + { + _mockLogger = new(); + _mockCacheProvider = new(); + _mockCache = new(); + _mockServiceProvider = new(); + _mockClient = new(); + _mockQueue = new(); + _settings = new() { AutoAttachFinalizers = false, AutoDetachFinalizers = false }; + + _mockCacheProvider + .Setup(p => p.GetCache(It.IsAny())) + .Returns(_mockCache.Object); + } + + [Fact] + public async Task Reconcile_Should_Remove_Entity_From_Queue_Before_Processing() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var controller = CreateMockController(); + + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Remove(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Enqueue_Entity_When_Result_Has_RequeueAfter() + { + var entity = CreateTestEntity(); + var requeueAfter = TimeSpan.FromMinutes(5); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + var controller = CreateMockController( + reconcileResult: ReconciliationResult.Success(entity, requeueAfter)); + + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Enqueue( + entity, + RequeueType.Added, + requeueAfter, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Not_Enqueue_Entity_When_Result_Has_No_RequeueAfter() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + var controller = CreateMockController( + reconcileResult: ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Enqueue( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Reconcile_Should_Skip_On_Cached_Generation() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var mockController = new Mock>(); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + _mockCache + .Setup(c => c.TryGetAsync( + It.Is(s => s == entity.Uid()), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(MaybeValue.FromValue(entity.Generation())); + + var reconciler = CreateReconcilerForController(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockLogger.Verify(logger => logger.Log( + It.Is(logLevel => logLevel == LogLevel.Debug), + It.Is(eventId => eventId.Id == 0), + It.Is((@object, type) => @object.ToString() == $"""Entity "{entity.Kind}/{entity.Name()}" modification did not modify generation. Skip event.""" && type.Name == "FormattedLogValues"), + It.IsAny(), + It.IsAny>()!), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Call_ReconcileAsync_For_Added_Event() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var mockController = new Mock>(); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + mockController.Verify( + c => c.ReconcileAsync(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Call_ReconcileAsync_For_Modified_Event_With_No_Deletion_Timestamp() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + var mockController = new Mock>(); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + mockController.Verify( + c => c.ReconcileAsync(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Call_FinalizeAsync_For_Modified_Event_With_Deletion_Timestamp() + { + const string finalizerName = "test-finalizer"; + var entity = CreateTestEntityForFinalization(deletionTimestamp: DateTime.UtcNow, finalizer: finalizerName); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + + var mockFinalizer = new Mock>(); + + mockFinalizer + .Setup(c => c.FinalizeAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForFinalizer(mockFinalizer.Object, finalizerName); + + await reconciler.Reconcile(context, CancellationToken.None); + + mockFinalizer.Verify( + c => c.FinalizeAsync(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Call_DeletedAsync_For_Deleted_Event() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Deleted); + var mockController = new Mock>(); + + mockController + .Setup(c => c.DeletedAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + mockController.Verify( + c => c.DeletedAsync(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Remove_From_Cache_After_Successful_Deletion() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Deleted); + + var controller = CreateMockController( + deletedResult: ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockCache.Verify( + c => c.RemoveAsync(entity.Uid(), It.IsAny(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Not_Remove_From_Cache_After_Failed_Deletion() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Deleted); + + var controller = CreateMockController( + deletedResult: ReconciliationResult.Failure(entity, "Deletion failed")); + + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockCache.Verify( + c => c.RemoveAsync(It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Theory] + [InlineData(RequeueType.Added)] + [InlineData(RequeueType.Modified)] + [InlineData(RequeueType.Deleted)] + public async Task Reconcile_Should_Use_Correct_RequeueType_For_EventType(RequeueType expectedRequeueType) + { + var entity = CreateTestEntity(); + var requeueAfter = TimeSpan.FromSeconds(30); + var watchEventType = expectedRequeueType switch + { + RequeueType.Added => WatchEventType.Added, + RequeueType.Modified => WatchEventType.Modified, + RequeueType.Deleted => WatchEventType.Deleted, + _ => throw new ArgumentException("Invalid RequeueType"), + }; + + var context = ReconciliationContext.CreateFromApiServerEvent(entity, watchEventType); + var result = ReconciliationResult.Success(entity, requeueAfter); + + var controller = watchEventType == WatchEventType.Deleted + ? CreateMockController(deletedResult: result) + : CreateMockController(reconcileResult: result); + + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Enqueue( + entity, + expectedRequeueType, + requeueAfter, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Return_Result_From_Controller() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var expectedResult = ReconciliationResult.Success(entity, TimeSpan.FromMinutes(1)); + + var controller = CreateMockController(reconcileResult: expectedResult); + var reconciler = CreateReconcilerForController(controller); + + var result = await reconciler.Reconcile(context, CancellationToken.None); + + result.Should().Be(expectedResult); + result.Entity.Should().Be(entity); + result.RequeueAfter.Should().Be(TimeSpan.FromMinutes(1)); + } + + [Fact] + public async Task Reconcile_Should_Handle_Failure_Result() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + const string errorMessage = "Test error"; + var failureResult = ReconciliationResult.Failure(entity, errorMessage); + + var controller = CreateMockController(reconcileResult: failureResult); + var reconciler = CreateReconcilerForController(controller); + + var result = await reconciler.Reconcile(context, CancellationToken.None); + + result.IsSuccess.Should().BeFalse(); + result.ErrorMessage.Should().Be(errorMessage); + } + + [Fact] + public async Task Reconcile_Should_Enqueue_Failed_Result_If_RequeueAfter_Set() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var requeueAfter = TimeSpan.FromSeconds(30); + var failureResult = ReconciliationResult.Failure( + entity, + "Temporary failure", + requeueAfter: requeueAfter); + + var controller = CreateMockController(reconcileResult: failureResult); + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Enqueue( + entity, + RequeueType.Added, + requeueAfter, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_When_Auto_Attach_Finalizers_Is_Enabled_Should_Attach_Finalizer() + { + _settings.AutoAttachFinalizers = true; + + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + var mockController = new Mock>(); + var mockFinalizer = new Mock>(); + + _mockClient + .Setup(c => c.UpdateAsync(It.Is( + e => e == entity), + It.IsAny())) + .ReturnsAsync(entity); + + _mockServiceProvider + .Setup(p => p.GetRequiredKeyedService( + It.Is(t => t == typeof(IEnumerable>)), + It.Is(o => ReferenceEquals(o, KeyedService.AnyKey)))) + .Returns(new List> { mockFinalizer.Object }); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockClient.Verify( + c => c.UpdateAsync(It.Is(cm => cm.HasFinalizer("ientityfinalizer`1proxyfinalizer")), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_When_Auto_Detach_Finalizers_Is_Enabled_Should_Detach_Finalizer() + { + _settings.AutoDetachFinalizers = true; + + const string finalizerName = "test-finalizer"; + var entity = CreateTestEntityForFinalization(deletionTimestamp: DateTime.UtcNow, finalizer: finalizerName); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + + var mockFinalizer = new Mock>(); + + _mockClient + .Setup(c => c.UpdateAsync(It.Is( + e => e == entity), + It.IsAny())) + .ReturnsAsync(entity); + + mockFinalizer + .Setup(c => c.FinalizeAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForFinalizer(mockFinalizer.Object, finalizerName); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockClient.Verify( + c => c.UpdateAsync(It.Is(cm => !cm.HasFinalizer(finalizerName)), + It.IsAny()), + Times.Once); + } + + private Reconciler CreateReconcilerForController(IEntityController controller) + { + var mockScope = new Mock(); + var mockScopeFactory = new Mock(); + + mockScope + .Setup(s => s.ServiceProvider) + .Returns(_mockServiceProvider.Object); + + mockScopeFactory + .Setup(s => s.CreateScope()) + .Returns(mockScope.Object); + + _mockServiceProvider + .Setup(p => p.GetService(typeof(IServiceScopeFactory))) + .Returns(mockScopeFactory.Object); + + _mockServiceProvider + .Setup(p => p.GetService(typeof(IEntityController))) + .Returns(controller); + + return new( + _mockLogger.Object, + _mockCacheProvider.Object, + _mockServiceProvider.Object, + _settings, + _mockQueue.Object, + _mockClient.Object); + } + + private Reconciler CreateReconcilerForFinalizer(IEntityFinalizer? finalizer, string finalizerName) + { + var mockScope = new Mock(); + var mockScopeFactory = new Mock(); + + mockScope + .Setup(s => s.ServiceProvider) + .Returns(_mockServiceProvider.Object); + + mockScopeFactory + .Setup(s => s.CreateScope()) + .Returns(mockScope.Object); + + _mockServiceProvider + .Setup(p => p.GetService(typeof(IServiceScopeFactory))) + .Returns(mockScopeFactory.Object); + + _mockServiceProvider + .Setup(p => p.GetKeyedService( + It.Is(t => t == typeof(IEntityFinalizer)), + It.Is(s => s == finalizerName))) + .Returns(finalizer); + + return new( + _mockLogger.Object, + _mockCacheProvider.Object, + _mockServiceProvider.Object, + _settings, + _mockQueue.Object, + _mockClient.Object); + } + + private static IEntityController CreateMockController( + ReconciliationResult? reconcileResult = null, + ReconciliationResult? deletedResult = null) + { + var mockController = new Mock>(); + var entity = CreateTestEntity(); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(reconcileResult ?? ReconciliationResult.Success(entity)); + + mockController + .Setup(c => c.DeletedAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(deletedResult ?? ReconciliationResult.Success(entity)); + + return mockController.Object; + } + + private static V1ConfigMap CreateTestEntity(string? name = null) + { + return new() + { + Metadata = new() + { + Name = name ?? "test-configmap", + NamespaceProperty = "default", + Uid = Guid.NewGuid().ToString(), + Generation = 1, + }, + Kind = V1ConfigMap.KubeKind, + }; + } + + private static V1ConfigMap CreateTestEntityForFinalization(string? name = null, DateTime? deletionTimestamp = null, string? finalizer = null) + { + return new() + { + Metadata = new() + { + Name = name ?? "test-configmap", + NamespaceProperty = "default", + Uid = Guid.NewGuid().ToString(), + Generation = 1, + DeletionTimestamp = deletionTimestamp, + Finalizers = !string.IsNullOrEmpty(finalizer) ? new List { finalizer } : new(), + }, + Kind = V1ConfigMap.KubeKind, + }; + } +} diff --git a/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs b/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs index 590494e9..85724ffb 100644 --- a/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs +++ b/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs @@ -10,17 +10,14 @@ using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Entities; +using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; -using KubeOps.Operator.Constants; -using KubeOps.Operator.Queue; using KubeOps.Operator.Watcher; using Microsoft.Extensions.Logging; using Moq; -using ZiggyCreatures.Caching.Fusion; - namespace KubeOps.Operator.Test.Watcher; public sealed class ResourceWatcherTest @@ -31,30 +28,21 @@ public async Task Restarting_Watcher_Should_Trigger_New_Watch() // Arrange. var activitySource = new ActivitySource("unit-test"); var logger = Mock.Of>>(); - var serviceProvider = Mock.Of(); - var timedEntityQueue = new TimedEntityQueue(); + var reconciler = Mock.Of>(); var operatorSettings = new OperatorSettings { Namespace = "unit-test" }; var kubernetesClient = Mock.Of(); - var cache = Mock.Of(); - var cacheProvider = Mock.Of(); var labelSelector = new DefaultEntityLabelSelector(); Mock.Get(kubernetesClient) .Setup(client => client.WatchAsync("unit-test", null, null, true, It.IsAny())) .Returns((_, _, _, _, cancellationToken) => WaitForCancellationAsync<(WatchEventType, V1Pod)>(cancellationToken)); - Mock.Get(cacheProvider) - .Setup(cp => cp.GetCache(It.Is(s => s == CacheConstants.CacheNames.ResourceWatcher))) - .Returns(() => cache); - var resourceWatcher = new ResourceWatcher( activitySource, logger, - serviceProvider, - timedEntityQueue, + reconciler, operatorSettings, labelSelector, - cacheProvider, kubernetesClient); // Act. diff --git a/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Transpiler.Test/KubeOps.Transpiler.Test.csproj b/test/KubeOps.Transpiler.Test/KubeOps.Transpiler.Test.csproj index db5dde7f..46d28996 100644 --- a/test/KubeOps.Transpiler.Test/KubeOps.Transpiler.Test.csproj +++ b/test/KubeOps.Transpiler.Test/KubeOps.Transpiler.Test.csproj @@ -14,4 +14,4 @@ - + \ No newline at end of file diff --git a/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage]