diff --git a/.gitignore b/.gitignore index a5eddaf2..da9560b0 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ bld/ coverage.json coverage.info .vs +[Tt]est[Rr]esults/ # Docs _site diff --git a/docs/docs/operator/advanced-configuration.mdx b/docs/docs/operator/advanced-configuration.mdx new file mode 100644 index 00000000..d6469a63 --- /dev/null +++ b/docs/docs/operator/advanced-configuration.mdx @@ -0,0 +1,591 @@ +--- +title: Advanced Configuration +description: Advanced Operator Configuration Options +sidebar_position: 8 +--- + +# Advanced Configuration + +This guide covers advanced configuration options for KubeOps operators, including finalizer management, custom leader election, and durable requeue mechanisms. + +## Finalizer Management + +KubeOps provides automatic finalizer attachment and detachment to ensure proper resource cleanup. These features can be configured through `OperatorSettings`. + +### Auto-Attach Finalizers + +By default, KubeOps automatically attaches finalizers to entities during reconciliation. This ensures that cleanup operations are performed before resources are deleted. + +```csharp +builder.Services + .AddKubernetesOperator(settings => + { + // Enable automatic finalizer attachment (default: true) + settings.AutoAttachFinalizers = true; + }); +``` + +When `AutoAttachFinalizers` is enabled: +- Finalizers are automatically added to entities during reconciliation +- You don't need to manually call the `EntityFinalizerAttacher` delegate +- All registered finalizers for an entity type are automatically attached + +When disabled: +```csharp +settings.AutoAttachFinalizers = false; +``` + +You must manually attach finalizers in your controller: + +```csharp +public class V1DemoEntityController( + ILogger logger, + EntityFinalizerAttacher finalizerAttacher) + : IEntityController +{ + public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) + { + // Manually attach finalizer + entity = await finalizerAttacher(entity, cancellationToken); + + // Continue with reconciliation logic + logger.LogInformation("Reconciling entity {Entity}", entity); + return ReconciliationResult.Success(entity); + } + + public Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) + { + return Task.FromResult(ReconciliationResult.Success(entity)); + } +} +``` + +### Auto-Detach Finalizers + +KubeOps automatically removes finalizers after successful finalization. This can also be configured: + +```csharp +builder.Services + .AddKubernetesOperator(settings => + { + // Enable automatic finalizer removal (default: true) + settings.AutoDetachFinalizers = true; + }); +``` + +When `AutoDetachFinalizers` is enabled: +- Finalizers are automatically removed when `FinalizeAsync` returns success + +When disabled: +```csharp +settings.AutoDetachFinalizers = false; +``` + +You must manually manage finalizer removal, which is typically not recommended unless you have specific requirements. + +### Use Cases + +**Keep defaults enabled** when: +- You want standard finalizer behavior +- Your finalizers follow the typical pattern +- You don't need fine-grained control + +**Disable auto-attach** when: +- You need conditional finalizer attachment +- Different instances should have different finalizers +- You want to attach finalizers based on specific conditions + +**Disable auto-detach** when: +- You need custom finalizer removal logic +- You want to coordinate multiple finalizers manually +- You have external systems that need to confirm cleanup + +## Custom Leader Election + +KubeOps supports different leader election mechanisms through the `LeaderElectionType` setting. This allows you to control how multiple operator instances coordinate in a cluster. + +### Leader Election Types + +```csharp +public enum LeaderElectionType +{ + None = 0, // No leader election - all instances process events + Single = 1, // Single leader election - only one instance processes events + Custom = 2 // Custom implementation - user-defined coordination +} +``` + +### Configuration + +```csharp +builder.Services + .AddKubernetesOperator(settings => + { + settings.LeaderElectionType = LeaderElectionType.Single; + settings.LeaderElectionLeaseDuration = TimeSpan.FromSeconds(15); + settings.LeaderElectionRenewDeadline = TimeSpan.FromSeconds(10); + settings.LeaderElectionRetryPeriod = TimeSpan.FromSeconds(2); + }); +``` + +### Custom Leader Election + +The `Custom` leader election type allows you to implement your own coordination logic, such as namespace-based leader election. + +#### Example: Namespace-Based Leader Election + +In some scenarios, you may want different operator instances to handle different namespaces. This enables horizontal scaling while maintaining isolation. + +**Step 1: Implement a custom ResourceWatcher** + +```csharp +public sealed class NamespacedLeaderElectionResourceWatcher( + ActivitySource activitySource, + ILogger> logger, + IReconciler reconciler, + OperatorSettings settings, + IEntityLabelSelector labelSelector, + IKubernetesClient client, + INamespaceLeadershipManager namespaceLeadershipManager) + : ResourceWatcher( + activitySource, + logger, + reconciler, + settings, + labelSelector, + client) + where TEntity : IKubernetesObject +{ + protected override async Task> OnEventAsync( + WatchEventType eventType, + TEntity entity, + CancellationToken cancellationToken) + { + // Check if this instance is responsible for the entity's namespace + if (!await namespaceLeadershipManager.IsResponsibleForNamespace( + entity.Namespace(), + cancellationToken)) + { + // Skip processing - another instance handles this namespace + return ReconciliationResult.Success(entity); + } + + // Process the event + return await base.OnEventAsync(eventType, entity, cancellationToken); + } +} +``` + +**Step 2: Implement the leadership manager** + +```csharp +public interface INamespaceLeadershipManager +{ + Task IsResponsibleForNamespace(string @namespace, CancellationToken cancellationToken); +} + +public class NamespaceLeadershipManager : INamespaceLeadershipManager +{ + private readonly ILeaderElector _leaderElector; + private readonly ConcurrentDictionary _namespaceResponsibility = new(); + + public async Task IsResponsibleForNamespace( + string @namespace, + CancellationToken cancellationToken) + { + // Implement your logic here: + // - Consistent hashing of namespace names + // - Lease-based namespace assignment + // - External coordination service (e.g., etcd, Consul) + + return _namespaceResponsibility.GetOrAdd( + @namespace, + ns => CalculateResponsibility(ns)); + } + + private bool CalculateResponsibility(string @namespace) + { + // Example: Simple hash-based distribution + var instanceId = Environment.GetEnvironmentVariable("POD_NAME") ?? "instance-0"; + var instanceCount = int.Parse( + Environment.GetEnvironmentVariable("REPLICA_COUNT") ?? "1"); + + var namespaceHash = @namespace.GetHashCode(); + var assignedInstance = Math.Abs(namespaceHash % instanceCount); + var currentInstance = int.Parse(instanceId.Split('-').Last()); + + return assignedInstance == currentInstance; + } +} +``` + +**Step 3: Register the custom watcher** + +```csharp +builder.Services + .AddKubernetesOperator(settings => + { + settings.LeaderElectionType = LeaderElectionType.Custom; + }) + .AddSingleton() + .AddHostedService>(); +``` + +### Benefits of Custom Leader Election + +- **Horizontal Scaling**: Multiple instances can process different subsets of resources +- **Namespace Isolation**: Different teams or environments can have dedicated operator instances +- **Geographic Distribution**: Route requests to instances in specific regions +- **Load Balancing**: Distribute work across multiple instances + +## Custom Requeue Mechanism + +By default, KubeOps uses an in-memory queue for requeuing entities. This queue is volatile and does not survive operator restarts. For production scenarios, you may want to implement a durable queue. + +### Default Behavior + +The default `ITimedEntityQueue` implementation: +- Stores requeue entries in memory +- Processes them after the specified delay +- Loses pending requeues on operator restart + +### Implementing a Durable Queue + +You can implement `ITimedEntityQueue` to use external queue systems like Azure Service Bus, RabbitMQ, or AWS SQS. + +#### Example: Azure Service Bus Integration + +**Step 1: Implement ITimedEntityQueue** + +```csharp +public sealed class DurableTimedEntityQueue( + ServiceBusClient serviceBusClient, + IEntityRequeueQueueNameProvider queueNameProvider, + TimeProvider timeProvider) + : ITimedEntityQueue + where TEntity : IKubernetesObject +{ + private readonly ServiceBusSender _sender = serviceBusClient.CreateSender( + queueNameProvider.GetRequeueQueueName()); + + public async Task Enqueue( + TEntity entity, + RequeueType type, + TimeSpan requeueIn, + CancellationToken cancellationToken) + { + var entry = RequeueEntry.CreateFor(entity, type); + var message = new ServiceBusMessage(BinaryData.FromObjectAsJson(entry)); + + // Schedule the message for future delivery + await _sender.ScheduleMessageAsync( + message, + timeProvider.GetUtcNow().Add(requeueIn), + cancellationToken); + } + + public Task Remove(TEntity entity, CancellationToken cancellationToken) + { + // will be automatically removed when the message is processed + return Task.CompletedTask; + } + + public async IAsyncEnumerator> GetAsyncEnumerator( + CancellationToken cancellationToken = default) + { + // Not used with external queue - processing happens via the processor + await Task.CompletedTask; + yield break; + } + + public void Dispose() + { + _sender.DisposeAsync().AsTask().GetAwaiter().GetResult(); + } +} +``` + +**Step 2: Implement a background service to process messages** + +```csharp +public sealed class DurableEntityRequeueBackgroundService( + ServiceBusClient serviceBusClient, + IKubernetesClient kubernetesClient, + IReconciler reconciler, + IEntityRequeueQueueNameProvider queueNameProvider, + ILogger> logger) + : BackgroundService + where TEntity : IKubernetesObject +{ + private readonly ServiceBusProcessor _processor = serviceBusClient.CreateProcessor( + queueNameProvider.GetRequeueQueueName(), + new ServiceBusProcessorOptions + { + MaxConcurrentCalls = 1, + AutoCompleteMessages = false + }); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _processor.ProcessMessageAsync += ProcessMessageAsync; + _processor.ProcessErrorAsync += ProcessErrorAsync; + + await _processor.StartProcessingAsync(stoppingToken); + + try + { + await Task.Delay(Timeout.Infinite, stoppingToken); + } + catch (OperationCanceledException) + { + // Expected during shutdown + } + } + + private async Task ProcessMessageAsync(ProcessMessageEventArgs args) + { + var entry = args.Message.Body.ToObjectFromJson>(); + + // Verify entity still exists + var entity = await kubernetesClient.GetAsync( + entry.Entity.Name(), + entry.Entity.Namespace(), + args.CancellationToken); + + if (entity == null) + { + logger.LogInformation( + "Skipping reconciliation for deleted entity {Name}", + entry.Entity.Name()); + await args.CompleteMessageAsync(args.Message, args.CancellationToken); + return; + } + + // Complete message before reconciliation to avoid reprocessing + await args.CompleteMessageAsync(args.Message, args.CancellationToken); + + // Trigger reconciliation + await reconciler.Reconcile( + ReconciliationContext.CreateFromOperatorEvent( + entity, + entry.RequeueType.ToWatchEventType()), + args.CancellationToken); + } + + private Task ProcessErrorAsync(ProcessErrorEventArgs args) + { + logger.LogError(args.Exception, "Error processing requeue message"); + return Task.CompletedTask; + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + await _processor.StopProcessingAsync(cancellationToken); + _processor.ProcessMessageAsync -= ProcessMessageAsync; + _processor.ProcessErrorAsync -= ProcessErrorAsync; + await _processor.DisposeAsync(); + await base.StopAsync(cancellationToken); + } +} +``` + +**Step 3: Register the durable queue** + +```csharp +builder.Services + .AddSingleton(sp => + new ServiceBusClient(configuration["ServiceBus:ConnectionString"])) + .AddSingleton() + .AddKubernetesOperator() + .RegisterComponents(); + +// Replace the default queue with the durable implementation +builder.Services.Replace(ServiceDescriptor.Singleton( + typeof(ITimedEntityQueue<>), + typeof(DurableTimedEntityQueue<>))); + +// Add the background service to process messages +builder.Services.AddHostedService>(); +``` + +**Step 4: Create the queue name provider** + +```csharp +public interface IEntityRequeueQueueNameProvider +{ + string GetRequeueQueueName() where TEntity : IKubernetesObject; +} + +public class EntityRequeueQueueNameProvider : IEntityRequeueQueueNameProvider +{ + public string GetRequeueQueueName() + where TEntity : IKubernetesObject + { + return $"operator-requeue-{typeof(TEntity).Name.ToLowerInvariant()}"; + } +} +``` + +### Benefits of Durable Requeues + +- **Persistence**: Requeue requests survive operator restarts +- **Reliability**: Messages are not lost during failures +- **Scalability**: External queue systems can handle high volumes +- **Observability**: Queue metrics provide insights into requeue patterns +- **Coordination**: Multiple operator instances can share the same queue + +### Combining Custom Leader Election with Durable Queues + +For advanced scenarios, you can combine namespace-based leader election with durable queues: + +```csharp +public sealed class NamespacedLeaderElectionEntityRequeueBackgroundService( + ServiceBusClient serviceBusClient, + IKubernetesClient kubernetesClient, + IReconciler reconciler, + IEntityRequeueQueueNameProvider queueNameProvider, + INamespaceLeadershipManager namespaceLeadershipManager, + ILogger> logger) + : BackgroundService + where TEntity : IKubernetesObject +{ + private readonly ServiceBusProcessor _processor = serviceBusClient.CreateProcessor( + queueNameProvider.GetRequeueQueueName(), + new ServiceBusProcessorOptions + { + MaxConcurrentCalls = 1, + AutoCompleteMessages = false + }); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _processor.ProcessMessageAsync += ProcessMessageAsync; + _processor.ProcessErrorAsync += ProcessErrorAsync; + await _processor.StartProcessingAsync(stoppingToken); + + try + { + await Task.Delay(Timeout.Infinite, stoppingToken); + } + catch (OperationCanceledException) + { + // Expected during shutdown + } + } + + private async Task ProcessMessageAsync(ProcessMessageEventArgs args) + { + var entry = args.Message.Body.ToObjectFromJson>(); + + // Verify entity still exists + var entity = await kubernetesClient.GetAsync( + entry.Entity.Name(), + entry.Entity.Namespace(), + args.CancellationToken); + + if (entity == null) + { + logger.LogInformation("Entity no longer exists, completing message"); + await args.CompleteMessageAsync(args.Message, args.CancellationToken); + return; + } + + // Check if this instance is responsible for the namespace + if (!await namespaceLeadershipManager.IsResponsibleForNamespace( + entity.Namespace(), + args.CancellationToken)) + { + logger.LogInformation( + "Not responsible for namespace {Namespace}, abandoning message", + entity.Namespace()); + + // Abandon the message so another instance can process it + await args.AbandonMessageAsync(args.Message, cancellationToken: args.CancellationToken); + return; + } + + // Complete message and trigger reconciliation + await args.CompleteMessageAsync(args.Message, args.CancellationToken); + + await reconciler.Reconcile( + ReconciliationContext.CreateFromOperatorEvent( + entity, + entry.RequeueType.ToWatchEventType()), + args.CancellationToken); + } + + private Task ProcessErrorAsync(ProcessErrorEventArgs args) + { + logger.LogError(args.Exception, "Error processing requeue message"); + return Task.CompletedTask; + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + await _processor.StopProcessingAsync(cancellationToken); + _processor.ProcessMessageAsync -= ProcessMessageAsync; + _processor.ProcessErrorAsync -= ProcessErrorAsync; + await _processor.DisposeAsync(); + await base.StopAsync(cancellationToken); + } +} +``` + +This combination provides: +- **Namespace-based work distribution** across operator instances +- **Durable requeue persistence** for reliability +- **Message abandonment** when an instance is not responsible, allowing proper routing + +## Best Practices + +### Finalizer Management + +1. **Keep defaults enabled** for most use cases +2. **Monitor finalizer attachment** in your logs +3. **Test finalizer behavior** in development before production +4. **Handle finalizer failures** gracefully with proper error messages + +### Leader Election + +1. **Start with `Single`** for simple deployments +2. **Use `Custom`** only when you need advanced coordination +3. **Test failover scenarios** to ensure seamless transitions +4. **Monitor leader election** status in your logs +5. **Set appropriate lease durations** based on your workload + +### Requeue Mechanisms + +1. **Use in-memory queue** for development and simple operators +2. **Implement durable queues** for production workloads +3. **Monitor queue depth** to detect processing issues +4. **Set appropriate requeue delays** to avoid overwhelming your system +5. **Handle message deduplication** to prevent duplicate processing +6. **Test restart scenarios** to ensure requeues survive failures + +## Troubleshooting + +### Finalizers Not Attaching + +- Check `settings.AutoAttachFinalizers` is `true` +- Verify finalizer is registered with `AddFinalizer()` +- Check logs for finalizer attachment errors + +### Leader Election Issues + +- Verify RBAC permissions for lease resources +- Check network connectivity between instances +- Review lease duration settings +- Monitor logs for leader election events +- Ensure cluster time and local time are synchronized (time drift can cause lease issues) + +### Requeue Problems + +- Verify queue connection (for durable queues) +- Check queue permissions and quotas +- Monitor message processing errors +- Review requeue delay settings +- Ensure entities still exist before reprocessing diff --git a/docs/docs/operator/building-blocks/controllers.mdx b/docs/docs/operator/building-blocks/controllers.mdx index 1aba0b3b..79493a33 100644 --- a/docs/docs/operator/building-blocks/controllers.mdx +++ b/docs/docs/operator/building-blocks/controllers.mdx @@ -17,16 +17,22 @@ public class V1DemoEntityController( ILogger logger, IKubernetesClient client) : IEntityController { - public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) + public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); // Implement your reconciliation logic here + return ReconciliationResult.Success(entity); } - public async Task DeletedAsync(V1DemoEntity entity, CancellationToken token) + public async Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { logger.LogInformation("Deleting entity {Entity}.", entity); // Implement your cleanup logic here + return ReconciliationResult.Success(entity); } } ``` @@ -52,41 +58,48 @@ This method is called when: - The operator starts up and discovers existing resources ```csharp -public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Check if required resources exist var deployment = await client.GetAsync( entity.Spec.DeploymentName, - entity.Namespace()); + entity.Namespace(), + cancellationToken); if (deployment == null) { // Create the deployment if it doesn't exist - await client.CreateAsync(new V1Deployment - { - Metadata = new V1ObjectMeta + await client.CreateAsync( + new V1Deployment { - Name = entity.Spec.DeploymentName, - NamespaceProperty = entity.Namespace() + Metadata = new V1ObjectMeta + { + Name = entity.Spec.DeploymentName, + NamespaceProperty = entity.Namespace() + }, + Spec = new V1DeploymentSpec + { + Replicas = entity.Spec.Replicas, + // ... other deployment configuration + } }, - Spec = new V1DeploymentSpec - { - Replicas = entity.Spec.Replicas, - // ... other deployment configuration - } - }); + cancellationToken); } // Update status to reflect current state entity.Status.LastReconciled = DateTime.UtcNow; - await client.UpdateStatusAsync(entity); + await client.UpdateStatusAsync(entity, cancellationToken); + + return ReconciliationResult.Success(entity); } ``` ### DeletedAsync :::warning Important -The `DeletedAsync` method is purely informative and "fire and forget". It is called when a resource is deleted, but it cannot guarantee proper cleanup of resources. For reliable resource cleanup, you must use [Finalizers](./finalizer). +The `DeletedAsync` method is informational only and executes asynchronously without guarantees. While it is called when a resource is deleted, it cannot ensure proper cleanup. For reliable resource cleanup, use [finalizers](./finalizer). ::: This method is called when a resource is deleted, but should only be used for: @@ -96,13 +109,130 @@ This method is called when a resource is deleted, but should only be used for: - Updating external systems about the deletion ```csharp -public async Task DeletedAsync(V1DemoEntity entity, CancellationToken token) +public async Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Log the deletion event logger.LogInformation("Entity {Entity} was deleted.", entity); // Update external systems if needed - await NotifyExternalSystem(entity); + await NotifyExternalSystem(entity, cancellationToken); + + return ReconciliationResult.Success(entity); +} +``` + +## Reconciliation Results + +All reconciliation methods must return a `ReconciliationResult`. This provides a standardized way to communicate the outcome of reconciliation operations. + +### Success Results + +Return a success result when reconciliation completes without errors: + +```csharp +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) +{ + // Perform reconciliation + await ApplyDesiredState(entity, cancellationToken); + + // Return success + return ReconciliationResult.Success(entity); +} +``` + +### Failure Results + +Return a failure result when reconciliation encounters an error: + +```csharp +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) +{ + try + { + await ApplyDesiredState(entity, cancellationToken); + return ReconciliationResult.Success(entity); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to reconcile entity {Name}", entity.Name()); + return ReconciliationResult.Failure( + entity, + "Failed to apply desired state", + ex); + } +} +``` + +### Requeuing Entities + +You can request automatic requeuing by specifying a `requeueAfter` parameter: + +```csharp +// Requeue after 5 minutes +return ReconciliationResult.Success(entity, TimeSpan.FromMinutes(5)); + +// Or set it after creation +var result = ReconciliationResult.Success(entity); +result.RequeueAfter = TimeSpan.FromSeconds(30); +return result; +``` + +This is useful for: +- Polling external resources +- Implementing retry logic with backoff +- Periodic status checks +- Waiting for external dependencies + +:::info Durable Requeue Mechanisms +By default, requeue requests are stored in memory and will be lost on operator restart. For production scenarios requiring persistence, see [Advanced Configuration - Custom Requeue Mechanism](../advanced-configuration#custom-requeue-mechanism) to learn how to implement durable queues using Azure Service Bus, RabbitMQ, or other messaging systems. +::: + +### Error Handling with Results + +The `ReconciliationResult` provides structured error handling: + +```csharp +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) +{ + if (!await ValidateConfiguration(entity)) + { + return ReconciliationResult.Failure( + entity, + "Configuration validation failed: Required field 'DeploymentName' is empty", + requeueAfter: TimeSpan.FromMinutes(1)); + } + + try + { + await ReconcileInternal(entity, cancellationToken); + return ReconciliationResult.Success(entity); + } + catch (KubernetesException ex) when (ex.Status.Code == 409) + { + // Conflict - retry after short delay + return ReconciliationResult.Failure( + entity, + "Resource conflict detected", + ex, + requeueAfter: TimeSpan.FromSeconds(5)); + } + catch (Exception ex) + { + logger.LogError(ex, "Unexpected error during reconciliation"); + return ReconciliationResult.Failure( + entity, + $"Reconciliation failed: {ex.Message}", + ex, + requeueAfter: TimeSpan.FromMinutes(5)); + } } ``` @@ -145,38 +275,47 @@ For more details about RBAC configuration, see the [RBAC documentation](../rbac) - Always check the current state before making changes ```csharp -public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Check if required resources exist - if (await IsDesiredState(entity)) + if (await IsDesiredState(entity, cancellationToken)) { - return; + return ReconciliationResult.Success(entity); } // Only make changes if needed - await ApplyDesiredState(entity); + await ApplyDesiredState(entity, cancellationToken); + return ReconciliationResult.Success(entity); } ``` ### Error Handling -- Handle errors gracefully -- Log errors with appropriate context -- Consider implementing retry logic for transient failures +- Use `ReconciliationResult.Failure()` for errors +- Include meaningful error messages +- Use the `requeueAfter` parameter for retry logic +- Preserve exception information ```csharp -public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { try { - await ReconcileInternal(entity, token); + await ReconcileInternal(entity, cancellationToken); + return ReconciliationResult.Success(entity); } catch (Exception ex) { - logger.LogError(ex, "Error reconciling entity {Entity}", entity); - // Update status to reflect the error - entity.Status.Error = ex.Message; - await client.UpdateStatusAsync(entity); + logger.LogError(ex, "Error reconciling entity {Name}", entity.Name()); + return ReconciliationResult.Failure( + entity, + $"Reconciliation failed: {ex.Message}", + ex, + requeueAfter: TimeSpan.FromMinutes(1)); } } ``` @@ -198,5 +337,5 @@ public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) 1. **Infinite Loops**: Avoid creating reconciliation loops that trigger themselves 2. **Missing Error Handling**: Always handle potential errors 3. **Resource Leaks**: Ensure proper cleanup of resources -4. **Missing RBAC**: Configure appropriate permissions +4. **Missing RBAC Configuration**: Configure appropriate permissions 5. **Status Updates**: Remember that status updates don't trigger reconciliation diff --git a/docs/docs/operator/building-blocks/finalizer.mdx b/docs/docs/operator/building-blocks/finalizer.mdx index fe349c86..d59052b9 100644 --- a/docs/docs/operator/building-blocks/finalizer.mdx +++ b/docs/docs/operator/building-blocks/finalizer.mdx @@ -25,10 +25,10 @@ When a resource is marked for deletion: 3. Each finalizer must explicitly remove itself after completing its cleanup 4. Only when all finalizers are removed is the resource actually deleted -This mechanism ensures that cleanup operations are: +This mechanism guarantees that cleanup operations are: - Guaranteed to run -- Run in a controlled manner +- Executed in a controlled sequence - Completed before resource deletion ## Implementing Finalizers @@ -40,19 +40,26 @@ public class DemoEntityFinalizer( ILogger logger, IKubernetesClient client) : IEntityFinalizer { - public async Task FinalizeAsync(V1DemoEntity entity, CancellationToken token) + public async Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { logger.LogInformation("Finalizing entity {Entity}", entity); try { // Clean up resources - await CleanupResources(entity); + await CleanupResources(entity, cancellationToken); + return ReconciliationResult.Success(entity); } catch (Exception ex) { logger.LogError(ex, "Error finalizing entity {Entity}", entity); - throw; // Re-throw to prevent finalizer removal + // Return failure to prevent finalizer removal + return ReconciliationResult.Failure( + entity, + $"Finalization failed: {ex.Message}", + ex); } } } @@ -60,7 +67,9 @@ public class DemoEntityFinalizer( ## Using Finalizers -Finalizers are automatically attached to entities using the `EntityFinalizerAttacher` delegate. This delegate is injected into your controller and handles the finalizer attachment: +By default, KubeOps automatically attaches finalizers to entities during reconciliation. This behavior can be configured through `OperatorSettings`. See [Advanced Configuration](../advanced-configuration#finalizer-management) for details on controlling automatic finalizer attachment. + +If you need manual control, you can use the `EntityFinalizerAttacher` delegate, which is injected into your controller: ```csharp [EntityRbac(typeof(V1DemoEntity), Verbs = RbacVerb.All)] @@ -69,18 +78,25 @@ public class V1DemoEntityController( EntityFinalizerAttacher finalizer) : IEntityController { - public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) + public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Attach the finalizer to the entity - entity = await finalizer(entity, token); + entity = await finalizer(entity, cancellationToken); // Continue with reconciliation logic logger.LogInformation("Reconciling entity {Entity}", entity); + + return ReconciliationResult.Success(entity); } - public async Task DeletedAsync(V1DemoEntity entity, CancellationToken token) + public async Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { logger.LogInformation("Entity {Entity} was deleted", entity); + return ReconciliationResult.Success(entity); } } ``` @@ -94,39 +110,50 @@ public class V1DemoEntityController( - Check resource existence before attempting cleanup ```csharp -public async Task FinalizeAsync(V1DemoEntity entity, CancellationToken token) +public async Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Check if resources still exist before cleanup - var resources = await GetResources(entity); + var resources = await GetResources(entity, cancellationToken); if (!resources.Any()) { // Resources already cleaned up - return; + return ReconciliationResult.Success(entity); } // Perform cleanup - await CleanupResources(resources); + await CleanupResources(resources, cancellationToken); + return ReconciliationResult.Success(entity); } ``` ### 2. Error Handling -- Handle errors gracefully +- Use `ReconciliationResult.Failure()` for errors +- Return failure results to prevent finalizer removal - Log errors with appropriate context -- Consider implementing retry logic for transient failures +- Include retry delays when appropriate ```csharp -public async Task FinalizeAsync(V1DemoEntity entity, CancellationToken token) +public async Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { try { - await FinalizeInternal(entity, token); + await FinalizeInternal(entity, cancellationToken); + return ReconciliationResult.Success(entity); } catch (Exception ex) { logger.LogError(ex, "Error finalizing entity {Entity}", entity); - // Re-throw to prevent finalizer removal - throw; + // Return failure to prevent finalizer removal + return ReconciliationResult.Failure( + entity, + $"Finalization failed: {ex.Message}", + ex, + requeueAfter: TimeSpan.FromMinutes(1)); } } ``` @@ -137,6 +164,50 @@ public async Task FinalizeAsync(V1DemoEntity entity, CancellationToken token) - Handle dependencies between resources - Consider cleanup order (e.g., delete pods before services) +### 4. Finalizer Results + +The finalizer must return a `ReconciliationResult`: + +- **Success**: The finalizer is removed, and the entity is deleted +- **Failure**: The finalizer remains, and the entity is requeued for retry + +```csharp +public async Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) +{ + // Check if external resources exist + var externalResource = await GetExternalResource(entity.Spec.ResourceId); + + if (externalResource == null) + { + // Already cleaned up + return ReconciliationResult.Success(entity); + } + + try + { + await DeleteExternalResource(externalResource, cancellationToken); + return ReconciliationResult.Success(entity); + } + catch (Exception ex) when (IsRetryable(ex)) + { + // Transient error - retry after delay + return ReconciliationResult.Failure( + entity, + "Failed to delete external resource", + ex, + requeueAfter: TimeSpan.FromSeconds(30)); + } + catch (Exception ex) + { + // Permanent error - log and succeed to prevent stuck resource + logger.LogError(ex, "Permanent error during finalization, allowing deletion"); + return ReconciliationResult.Success(entity); + } +} +``` + ## Common Pitfalls ### 1. Stuck Resources @@ -145,13 +216,15 @@ If a finalizer fails to complete: - The resource will remain in the cluster - It will be marked for deletion but never actually deleted -- Manual intervention may be required +- The finalizer will be retried based on the `requeueAfter` value +- Manual intervention may be required for permanent failures To fix stuck resources: 1. Identify the failing finalizer 2. Fix the underlying issue -3. Manually remove the finalizer only if necessary: +3. Check if the finalizer is being retried +4. Manually remove the finalizer only if necessary: ```bash kubectl patch -p '{"metadata":{"finalizers":[]}}' --type=merge ``` diff --git a/docs/docs/operator/deployment.mdx b/docs/docs/operator/deployment.mdx index 8dfd83cb..04df353c 100644 --- a/docs/docs/operator/deployment.mdx +++ b/docs/docs/operator/deployment.mdx @@ -1,7 +1,7 @@ --- title: Deployment description: Deploying your KubeOps Operator -sidebar_position: 8 +sidebar_position: 9 --- # Deployment @@ -177,7 +177,14 @@ kubectl apply -f https://github.com/your-org/your-operator/releases/download/v1. - Configure resource limits - Implement logging and metrics -4. **Updates**: +4. **High Availability**: + + - Run multiple replicas for redundancy + - Configure leader election for coordinated operation (see [Advanced Configuration - Leader Election](./advanced-configuration#custom-leader-election)) + - Consider namespace-based distribution for horizontal scaling + - Implement durable requeue mechanisms for reliability (see [Advanced Configuration - Custom Requeue Mechanism](./advanced-configuration#custom-requeue-mechanism)) + +5. **Updates**: - Document upgrade procedures - Test upgrades in staging - Provide rollback instructions diff --git a/docs/docs/operator/events.mdx b/docs/docs/operator/events.mdx index 1e3cd645..252e0496 100644 --- a/docs/docs/operator/events.mdx +++ b/docs/docs/operator/events.mdx @@ -64,7 +64,9 @@ KubeOps provides an `EventPublisher` to create and update events for your custom ```csharp public class DemoController(EventPublisher eventPublisher) : IEntityController { - public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) + public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { try { @@ -74,7 +76,9 @@ public class DemoController(EventPublisher eventPublisher) : IEntityController.Success(entity); } catch (Exception ex) { @@ -83,9 +87,21 @@ public class DemoController(EventPublisher eventPublisher) : IEntityController.Failure( + entity, + "Reconciliation failed", + ex); } } + + public Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) + { + return Task.FromResult(ReconciliationResult.Success(entity)); + } } ``` @@ -100,8 +116,8 @@ KubeOps provides two event types: 1. **Event Naming**: - - Use consistent, machine-readable reasons - - Make messages human-readable and descriptive + - Use consistent, machine-readable reason codes + - Write human-readable, descriptive messages - Include relevant details in the message 2. **Event Frequency**: diff --git a/docs/docs/operator/getting-started.mdx b/docs/docs/operator/getting-started.mdx index 1221c77e..35f86599 100644 --- a/docs/docs/operator/getting-started.mdx +++ b/docs/docs/operator/getting-started.mdx @@ -16,7 +16,7 @@ Before you begin, ensure you have the following installed: - A local Kubernetes cluster (like [kind](https://kind.sigs.k8s.io/) or [minikube](https://minikube.sigs.k8s.io/)) :::warning Development Environment -For local development, we recommend using `kind` or Docker Desktop as it provides a lightweight Kubernetes cluster that's perfect for operator development. Make sure your cluster is running before proceeding with the installation steps. +For local development, we recommend using `kind` or Docker Desktop, which provide lightweight Kubernetes clusters ideal for operator development. Ensure your cluster is running before proceeding with the installation steps. ::: ## Installing KubeOps Templates @@ -150,10 +150,20 @@ Controllers implement the reconciliation logic for your custom resources: [EntityRbac(typeof(V1DemoEntity), Verbs = RbacVerb.All)] public class DemoController : IEntityController { - public Task ReconcileAsync(V1DemoEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Implement your reconciliation logic here - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); + } + + public Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) + { + // Handle deletion event + return Task.FromResult(ReconciliationResult.Success(entity)); } } ``` @@ -165,10 +175,12 @@ Finalizers handle cleanup when resources are deleted: ```csharp public class DemoFinalizer : IEntityFinalizer { - public Task FinalizeAsync(V1DemoEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Implement your cleanup logic here - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } ``` @@ -181,6 +193,7 @@ The following sections will dive deeper into: - [Controllers](./building-blocks/controllers) - Implementing reconciliation logic - [Finalizers](./building-blocks/finalizer) - Handling resource cleanup - [Webhooks](./building-blocks/webhooks) - Implementing validation and mutation webhooks +- [Advanced Configuration](./advanced-configuration) - Leader election, durable queues, and finalizer management Make sure to read these sections before deploying your operator to production. ::: diff --git a/docs/docs/operator/testing/_category_.json b/docs/docs/operator/testing/_category_.json index 3ea28e1c..ccb9a78a 100644 --- a/docs/docs/operator/testing/_category_.json +++ b/docs/docs/operator/testing/_category_.json @@ -1,5 +1,5 @@ { - "position": 9, + "position": 10, "label": "Testing", "collapsible": true, "collapsed": true diff --git a/docs/docs/operator/testing/integration-tests.mdx b/docs/docs/operator/testing/integration-tests.mdx index 2724f57b..a0f4c5d8 100644 --- a/docs/docs/operator/testing/integration-tests.mdx +++ b/docs/docs/operator/testing/integration-tests.mdx @@ -238,10 +238,19 @@ public class EntityControllerIntegrationTest : IntegrationTestBase _svc = svc; } - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync( + V1OperatorIntegrationTestEntity entity, + CancellationToken cancellationToken) { _svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); + } + + public Task> DeletedAsync( + V1OperatorIntegrationTestEntity entity, + CancellationToken cancellationToken) + { + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/docs/docs/operator/utilities.mdx b/docs/docs/operator/utilities.mdx index 3cdfa14e..3e05b3d0 100644 --- a/docs/docs/operator/utilities.mdx +++ b/docs/docs/operator/utilities.mdx @@ -1,7 +1,7 @@ --- title: Utilities description: Utilities for your Operator and Development -sidebar_position: 10 +sidebar_position: 11 --- # Development and Operator Utilities diff --git a/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs b/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs index 9916626c..08b05179 100644 --- a/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs +++ b/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs @@ -4,23 +4,26 @@ using ConversionWebhookOperator.Entities; -using KubeOps.Abstractions.Controller; +using k8s.Models; + using KubeOps.Abstractions.Rbac; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; namespace ConversionWebhookOperator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] public class V1TestEntityController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.CompletedTask; + logger.LogInformation("Reconciling entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Deleted entity {Entity}.", entity); - return Task.CompletedTask; + logger.LogInformation("Deleted entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/examples/Operator/Controller/V1TestEntityController.cs b/examples/Operator/Controller/V1TestEntityController.cs index 7527ad6f..a593e872 100644 --- a/examples/Operator/Controller/V1TestEntityController.cs +++ b/examples/Operator/Controller/V1TestEntityController.cs @@ -2,10 +2,11 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Events; -using KubeOps.Abstractions.Queue; +using k8s.Models; + using KubeOps.Abstractions.Rbac; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using Microsoft.Extensions.Logging; @@ -14,18 +15,20 @@ namespace Operator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -public class V1TestEntityController(ILogger logger) +public sealed class V1TestEntityController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync( + V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.CompletedTask; + logger.LogInformation("Reconciling entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync( + V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Deleting entity {Entity}.", entity); - return Task.CompletedTask; + logger.LogInformation("Deleted entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/examples/Operator/Finalizer/FinalizerOne.cs b/examples/Operator/Finalizer/FinalizerOne.cs index c7179900..f80b8537 100644 --- a/examples/Operator/Finalizer/FinalizerOne.cs +++ b/examples/Operator/Finalizer/FinalizerOne.cs @@ -2,16 +2,15 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Finalizer; using Operator.Entities; namespace Operator.Finalizer; -public class FinalizerOne : IEntityFinalizer +public sealed class FinalizerOne : IEntityFinalizer { - public Task FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) - { - return Task.CompletedTask; - } + public Task> FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } diff --git a/examples/WebhookOperator/Controller/V1TestEntityController.cs b/examples/WebhookOperator/Controller/V1TestEntityController.cs index ac96eba4..72bae63d 100644 --- a/examples/WebhookOperator/Controller/V1TestEntityController.cs +++ b/examples/WebhookOperator/Controller/V1TestEntityController.cs @@ -2,25 +2,28 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Controller; +using k8s.Models; + using KubeOps.Abstractions.Rbac; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using WebhookOperator.Entities; namespace WebhookOperator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -public class V1TestEntityController(ILogger logger) : IEntityController +public sealed class V1TestEntityController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.CompletedTask; + logger.LogInformation("Reconciling entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Deleted entity {Entity}.", entity); - return Task.CompletedTask; + logger.LogInformation("Deleted entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs b/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs index fbc6ad57..2cc630f2 100644 --- a/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs +++ b/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs @@ -5,10 +5,10 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Crds; using KubeOps.Abstractions.Entities; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; using Microsoft.Extensions.DependencyInjection; @@ -24,6 +24,11 @@ public interface IOperatorBuilder /// IServiceCollection Services { get; } + /// + /// Configuration settings for the operator. + /// + OperatorSettings Settings { get; } + /// /// Add a controller implementation for a specific entity to the operator. /// The metadata for the entity must be added as well. diff --git a/src/KubeOps.Abstractions/Builder/LeaderElectionType.cs b/src/KubeOps.Abstractions/Builder/LeaderElectionType.cs new file mode 100644 index 00000000..7e68a7aa --- /dev/null +++ b/src/KubeOps.Abstractions/Builder/LeaderElectionType.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Abstractions.Builder; + +/// +/// Specifies the types of leader election mechanisms to be used in distributed systems or workloads. +/// +public enum LeaderElectionType +{ + /// + /// Represents the absence of a leader election mechanism. + /// This option is used when no leader election is required, and all instances + /// are expected to operate without coordination or exclusivity. + /// + None = 0, + + /// + /// Represents the leader election mechanism where only a single instance of the application + /// assumes the leader role at any given time. This is used to coordinate operations + /// that require exclusivity or to manage shared resources in distributed systems. + /// + Single = 1, + + /// + /// Represents a custom leader election mechanism determined by the user. + /// This option allows the integration of user-defined logic for handling + /// leader election, enabling tailored coordination strategies beyond the + /// provided defaults. + /// + Custom = 2, +} diff --git a/src/KubeOps.Abstractions/Builder/OperatorSettings.cs b/src/KubeOps.Abstractions/Builder/OperatorSettings.cs index d02b642e..522ab1a9 100644 --- a/src/KubeOps.Abstractions/Builder/OperatorSettings.cs +++ b/src/KubeOps.Abstractions/Builder/OperatorSettings.cs @@ -36,19 +36,11 @@ public sealed partial class OperatorSettings public string? Namespace { get; set; } /// - /// - /// Whether the leader elector should run. You should enable - /// this if you plan to run the operator redundantly. - /// - /// - /// If this is disabled and an operator runs in multiple instances - /// (in the same namespace), it can lead to a "split brain" problem. - /// - /// - /// Defaults to `false`. - /// + /// Defines the type of leader election mechanism to be used by the operator. + /// Determines how resources and controllers are coordinated in a distributed environment. + /// Defaults to indicating no leader election is configured. /// - public bool EnableLeaderElection { get; set; } = false; + public LeaderElectionType LeaderElectionType { get; set; } = LeaderElectionType.None; /// /// Defines how long one lease is valid for any leader. @@ -73,6 +65,19 @@ public sealed partial class OperatorSettings /// public Action? ConfigureResourceWatcherEntityCache { get; set; } + /// + /// Indicates whether finalizers should be automatically attached to Kubernetes entities during reconciliation. + /// When enabled, the operator will ensure that all defined finalizers for the entity are added if they are not already present. + /// Defaults to true. + /// + public bool AutoAttachFinalizers { get; set; } = true; + + /// + /// Indicates whether finalizers should be automatically removed from Kubernetes resources + /// upon successful completion of their finalization process. Defaults to true. + /// + public bool AutoDetachFinalizers { get; set; } = true; + [GeneratedRegex(@"(\W|_)", RegexOptions.CultureInvariant)] private static partial Regex OperatorNameRegex(); } diff --git a/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs b/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs index 408db1a5..2f26f16b 100644 --- a/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs +++ b/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. +using System.Reflection; + using k8s; using k8s.Models; @@ -91,6 +93,14 @@ public static V1OwnerReference MakeOwnerReference(this IKubernetesObject + /// Retrieves the applied to the specified Kubernetes object type, if it exists. + /// + /// The Kubernetes object to inspect for the attribute. + /// The if found; otherwise, null. + public static KubernetesEntityAttribute? GetKubernetesEntityAttribute(this IKubernetesObject entity) + => entity.GetType().GetCustomAttribute(true); + private static IList EnsureOwnerReferences(this V1ObjectMeta meta) => meta.OwnerReferences ??= new List(); } diff --git a/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj b/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj index 84762526..cc1b20f3 100644 --- a/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj +++ b/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj @@ -20,4 +20,4 @@ - + \ No newline at end of file diff --git a/src/KubeOps.Abstractions/Queue/EntityRequeue.cs b/src/KubeOps.Abstractions/Queue/EntityRequeue.cs deleted file mode 100644 index b3518138..00000000 --- a/src/KubeOps.Abstractions/Queue/EntityRequeue.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information. - -using k8s; -using k8s.Models; - -namespace KubeOps.Abstractions.Queue; - -/// -/// Injectable delegate for requeueing entities. -/// -/// Use this delegate when you need to pro-actively reconcile an entity after a -/// certain amount of time. This is useful, if you want to check your entities -/// periodically. -/// -/// -/// After the timeout is reached, the entity is fetched -/// from the API and passed to the controller for reconciliation. -/// If the entity was deleted in the meantime, the controller will not be called. -/// -/// -/// If the entity gets modified while the timeout is running, the timer -/// is canceled and restarted, if another requeue is requested. -/// -/// -/// The type of the entity. -/// The instance of the entity that should be requeued. -/// The time to wait before another reconcile loop is fired. -/// -/// Use the requeue delegate to repeatedly reconcile an entity after 5 seconds. -/// -/// [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -/// public class V1TestEntityController : IEntityController<V1TestEntity> -/// { -/// private readonly EntityRequeue<V1TestEntity> _requeue; -/// -/// public V1TestEntityController(EntityRequeue<V1TestEntity> requeue) -/// { -/// _requeue = requeue; -/// } -/// -/// public async Task ReconcileAsync(V1TestEntity entity, CancellationToken token) -/// { -/// _requeue(entity, TimeSpan.FromSeconds(5)); -/// } -/// } -/// -/// -public delegate void EntityRequeue(TEntity entity, TimeSpan requeueIn) - where TEntity : IKubernetesObject; diff --git a/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/Controller/IEntityController{TEntity}.cs similarity index 63% rename from src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs rename to src/KubeOps.Abstractions/Reconciliation/Controller/IEntityController{TEntity}.cs index fc6e23ab..e650b76a 100644 --- a/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Controller/IEntityController{TEntity}.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Controller; +namespace KubeOps.Abstractions.Reconciliation.Controller; /// /// Generic entity controller. The controller manages the reconcile loop @@ -37,24 +37,23 @@ namespace KubeOps.Abstractions.Controller; /// } /// /// -public interface IEntityController +public interface IEntityController where TEntity : IKubernetesObject { /// - /// Called for `added` and `modified` events from the watcher. + /// Reconciles the state of the specified entity with the desired state. + /// This method is triggered for `added` and `modified` events from the watcher. /// - /// The entity that fired the reconcile event. - /// The token to monitor for cancellation requests. - /// A task that completes when the reconciliation is done. - Task ReconcileAsync(TEntity entity, CancellationToken cancellationToken); + /// The entity that initiated the reconcile operation. + /// The token used to signal cancellation of the operation. + /// A task that represents the asynchronous operation and contains the result of the reconcile process. + Task> ReconcileAsync(TEntity entity, CancellationToken cancellationToken); /// /// Called for `delete` events for a given entity. /// /// The entity that fired the deleted event. /// The token to monitor for cancellation requests. - /// - /// A task that completes, when the reconciliation is done. - /// - Task DeletedAsync(TEntity entity, CancellationToken cancellationToken); + /// A task that represents the asynchronous operation and contains the result of the reconcile process. + Task> DeletedAsync(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/Finalizer/EntityFinalizerAttacher.cs b/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerAttacher.cs similarity index 97% rename from src/KubeOps.Abstractions/Finalizer/EntityFinalizerAttacher.cs rename to src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerAttacher.cs index 924bfde8..c54275bc 100644 --- a/src/KubeOps.Abstractions/Finalizer/EntityFinalizerAttacher.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerAttacher.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Finalizer; +namespace KubeOps.Abstractions.Reconciliation.Finalizer; /// /// diff --git a/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerExtensions.cs b/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerExtensions.cs new file mode 100644 index 00000000..d2408108 --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerExtensions.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Entities; + +namespace KubeOps.Abstractions.Reconciliation.Finalizer; + +/// +/// Provides extension methods for handling entity finalizers in Kubernetes resources. +/// +public static class EntityFinalizerExtensions +{ + private const byte MaxNameLength = 63; + + /// + /// Generates a unique identifier name for the finalizer of a given Kubernetes entity. + /// The identifier includes the group of the entity and the name of the finalizer, ensuring it conforms to Kubernetes naming conventions. + /// + /// The type of the Kubernetes entity. Must implement . + /// The finalizer implementing for which the identifier is generated. + /// The Kubernetes entity associated with the finalizer. + /// A string representing the unique identifier for the finalizer, truncated if it exceeds the maximum allowed length for Kubernetes names. + public static string GetIdentifierName(this IEntityFinalizer finalizer, TEntity entity) + where TEntity : IKubernetesObject + { + var finalizerName = finalizer.GetType().Name.ToLowerInvariant(); + finalizerName = finalizerName.EndsWith("finalizer") ? finalizerName : $"{finalizerName}finalizer"; + + var entityGroupName = entity.GetKubernetesEntityAttribute()?.Group ?? string.Empty; + var name = $"{entityGroupName}/{finalizerName}".TrimStart('/'); + + return name.Length > MaxNameLength ? name[..MaxNameLength] : name; + } +} diff --git a/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/Finalizer/IEntityFinalizer{TEntity}.cs similarity index 67% rename from src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs rename to src/KubeOps.Abstractions/Reconciliation/Finalizer/IEntityFinalizer{TEntity}.cs index c9fdebc2..f3b38983 100644 --- a/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Finalizer/IEntityFinalizer{TEntity}.cs @@ -5,13 +5,13 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Finalizer; +namespace KubeOps.Abstractions.Reconciliation.Finalizer; /// /// Finalizer for an entity. /// /// The type of the entity. -public interface IEntityFinalizer +public interface IEntityFinalizer where TEntity : IKubernetesObject { /// @@ -19,6 +19,6 @@ public interface IEntityFinalizer /// /// The kubernetes entity that needs to be finalized. /// The token to monitor for cancellation requests. - /// A task that resolves when the operation is done. - Task FinalizeAsync(TEntity entity, CancellationToken cancellationToken); + /// A task that represents the asynchronous operation and contains the result of the reconcile process. + Task> FinalizeAsync(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/Finalizer/IEventFinalizerAttacherFactory.cs b/src/KubeOps.Abstractions/Reconciliation/Finalizer/IEventFinalizerAttacherFactory.cs similarity index 95% rename from src/KubeOps.Abstractions/Finalizer/IEventFinalizerAttacherFactory.cs rename to src/KubeOps.Abstractions/Reconciliation/Finalizer/IEventFinalizerAttacherFactory.cs index 8236f6c8..8235dbfc 100644 --- a/src/KubeOps.Abstractions/Finalizer/IEventFinalizerAttacherFactory.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Finalizer/IEventFinalizerAttacherFactory.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Finalizer; +namespace KubeOps.Abstractions.Reconciliation.Finalizer; /// /// Represents a type used to create for controllers. diff --git a/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs new file mode 100644 index 00000000..15892a9a --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Defines methods for handling reconciliation processes related to Kubernetes resources. +/// This interface provides the necessary functionality for handling the lifecycle events +/// of a resource, such as creation, modification, and deletion. +/// +/// +/// The type of the Kubernetes resource, which must implement . +/// +public interface IReconciler + where TEntity : IKubernetesObject +{ + /// + /// Handles the reconciliation process for a Kubernetes entity. + /// + /// The context containing details of the entity to reconcile. + /// A token to monitor for cancellation requests during the reconciliation process. + /// A task that represents the asynchronous reconciliation operation, returning the result of the reconciliation process. + Task> Reconcile(ReconciliationContext reconciliationContext, CancellationToken cancellationToken); +} diff --git a/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs b/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs new file mode 100644 index 00000000..1d88b9fa --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation.Queue; + +/// +/// Injectable delegate for scheduling an entity to be requeued after a specified amount of time. +/// +/// The type of the Kubernetes entity being requeued. +/// The entity instance that should be requeued. +/// The type of operation for which the reconcile behavior should be performed. +/// The duration to wait before triggering the next reconcile process. +/// A cancellation token to observe while waiting for the requeue duration. +public delegate void EntityRequeue( + TEntity entity, RequeueType type, TimeSpan requeueIn, CancellationToken cancellationToken) + where TEntity : IKubernetesObject; diff --git a/src/KubeOps.Abstractions/Queue/IEntityRequeueFactory.cs b/src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs similarity index 87% rename from src/KubeOps.Abstractions/Queue/IEntityRequeueFactory.cs rename to src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs index 51486eff..8bd89088 100644 --- a/src/KubeOps.Abstractions/Queue/IEntityRequeueFactory.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs @@ -5,10 +5,10 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Queue; +namespace KubeOps.Abstractions.Reconciliation.Queue; /// -/// Represents a type used to create delegates of type for requeueing entities. +/// Represents a type used to create delegates of type for requeuing entities. /// public interface IEntityRequeueFactory { diff --git a/src/KubeOps.Abstractions/Reconciliation/Queue/RequeueType.cs b/src/KubeOps.Abstractions/Reconciliation/Queue/RequeueType.cs new file mode 100644 index 00000000..37346af8 --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/Queue/RequeueType.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Abstractions.Reconciliation.Queue; + +/// +/// Specifies the types of requeue operations that can occur on an entity. +/// +public enum RequeueType +{ + /// + /// Indicates that an entity should be added and is scheduled for requeue. + /// + Added, + + /// + /// Indicates that an entity has been modified and is scheduled for requeue. + /// + Modified, + + /// + /// Indicates that an entity should be deleted and is scheduled for requeue. + /// + Deleted, +} diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationContextExtensions.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContextExtensions.cs new file mode 100644 index 00000000..e8aae9b9 --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContextExtensions.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Provides extension methods for the class +/// to facilitate the identification of reconciliation trigger sources. +/// +public static class ReconciliationContextExtensions +{ + /// + /// Determines if the reconciliation context was triggered by the Kubernetes API server. + /// + /// The type of the Kubernetes resource associated with the reconciliation context. + /// The reconciliation context to check. + /// True if the reconciliation was triggered by the API server; otherwise, false. + public static bool IsTriggeredByApiServer(this ReconciliationContext reconciliationContext) + where TEntity : IKubernetesObject + => reconciliationContext.ReconciliationTriggerSource == ReconciliationTriggerSource.ApiServer; + + /// + /// Determines if the reconciliation context was triggered by the operator. + /// + /// The type of the Kubernetes resource associated with the reconciliation context. + /// The reconciliation context to check. + /// True if the reconciliation was triggered by the operator; otherwise, false. + public static bool IsTriggeredByOperator(this ReconciliationContext reconciliationContext) + where TEntity : IKubernetesObject + => reconciliationContext.ReconciliationTriggerSource == ReconciliationTriggerSource.Operator; +} diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs new file mode 100644 index 00000000..539ed06a --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Represents the context for the reconciliation process. +/// This class contains information about the entity to be reconciled and +/// the source that triggered the reconciliation process. +/// +/// +/// The type of the Kubernetes resource being reconciled. Must implement +/// . +/// +public sealed record ReconciliationContext + where TEntity : IKubernetesObject +{ + private ReconciliationContext(TEntity entity, WatchEventType eventType, ReconciliationTriggerSource reconciliationTriggerSource) + { + Entity = entity; + EventType = eventType; + ReconciliationTriggerSource = reconciliationTriggerSource; + } + + /// + /// Represents the Kubernetes entity involved in the reconciliation process. + /// + public TEntity Entity { get; } + + /// + /// Specifies the type of Kubernetes watch event that triggered the reconciliation process. + /// This property provides information about the nature of the change detected + /// within the Kubernetes resource, such as addition, modification, or deletion. + /// + public WatchEventType EventType { get; } + + /// + /// Specifies the source that initiated the reconciliation process. + /// + public ReconciliationTriggerSource ReconciliationTriggerSource { get; } + + /// + /// Creates a new instance of from an API server event. + /// + /// + /// The Kubernetes entity associated with the reconciliation context. + /// + /// + /// The type of watch event that triggered the context creation. + /// + /// + /// A new instance representing the reconciliation context + /// for the specified entity and event type, triggered by the API server. + /// + public static ReconciliationContext CreateFromApiServerEvent(TEntity entity, WatchEventType eventType) + => new(entity, eventType, ReconciliationTriggerSource.ApiServer); + + /// + /// Creates a new instance of from an operator-driven event. + /// + /// + /// The Kubernetes entity associated with the reconciliation context. + /// + /// + /// The type of watch event that triggered the context creation. + /// + /// + /// A new instance representing the reconciliation context + /// for the specified entity and event type, triggered by the operator. + /// + public static ReconciliationContext CreateFromOperatorEvent(TEntity entity, WatchEventType eventType) + => new(entity, eventType, ReconciliationTriggerSource.Operator); +} diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs new file mode 100644 index 00000000..db4fa98f --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs @@ -0,0 +1,119 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Represents the result of an operation performed on an entity +/// within the context of Kubernetes controllers or finalizers. +/// +/// +/// The type of the Kubernetes entity associated with this result. +/// Must implement where TMetadata is . +/// +public sealed record ReconciliationResult + where TEntity : IKubernetesObject +{ + private ReconciliationResult(TEntity entity, bool isSuccess, string? errorMessage, Exception? error, TimeSpan? requeueAfter) + { + Entity = entity; + IsSuccess = isSuccess; + ErrorMessage = errorMessage; + Error = error; + RequeueAfter = requeueAfter; + } + + /// + /// Represents the Kubernetes entity associated with the result of an operation or reconciliation process. + /// This property contains the entity object of type that was processed, modified, or finalized + /// during an operation. It provides access to the updated state or metadata of the entity after the operation. + /// Typically used for handling further processing, queuing, or logging of the affected entity. + /// + public TEntity Entity { get; } + + /// + /// Indicates whether the operation has completed successfully. + /// Returns true when the operation was successful, and no errors occurred. + /// This property is used to determine the state of the operation + /// and is often checked to decide whether further processing or error handling is necessary. + /// When this property is true, will be null, as there were no errors. + /// + [MemberNotNullWhen(false, nameof(ErrorMessage))] + public bool IsSuccess { get; } + + /// + /// Indicates whether the operation has failed. + /// Returns true when the operation was unsuccessful, as represented by being false. + /// Typically used to distinguish failure states and to conditionally handle error scenarios such as logging or retries. + /// When this property is true, will be non-null, providing additional details about the error. + /// + [MemberNotNullWhen(true, nameof(ErrorMessage))] + public bool IsFailure => !IsSuccess; + + /// + /// Contains a descriptive message associated with a failure when the operation does not succeed. + /// Used to provide context or details about the failure, assisting in debugging and logging. + /// This property is typically set when is false. + /// It will be null for successful operations. + /// + public string? ErrorMessage { get; } + + /// + /// Represents an exception associated with the operation outcome. + /// If the operation fails, this property may hold the exception that caused the failure, + /// providing additional context about the error for logging or debugging purposes. + /// This is optional and may be null if no exception information is available or applicable. + /// + public Exception? Error { get; } + + /// + /// Specifies the duration to wait before requeuing the entity for reprocessing. + /// If set, the entity will be scheduled for reprocessing after the specified time span. + /// This can be useful in scenarios where the entity needs to be revisited later due to external conditions, + /// such as resource dependencies or transient errors. + /// + public TimeSpan? RequeueAfter { get; set; } + + /// + /// Creates a successful result for the given entity, optionally specifying a requeue duration. + /// + /// + /// The Kubernetes entity that the result is associated with. + /// + /// + /// An optional duration after which the entity should be requeued for processing. Defaults to null. + /// + /// + /// A successful instance containing the provided entity and requeue duration. + /// + public static ReconciliationResult Success(TEntity entity, TimeSpan? requeueAfter = null) + => new(entity, true, null, null, requeueAfter); + + /// + /// Creates a failure result for the given entity, specifying an error message and optionally an exception and requeue duration. + /// + /// + /// The Kubernetes entity that the result is associated with. + /// + /// + /// A detailed message describing the reason for the failure. + /// + /// + /// An optional exception that caused the failure. Defaults to null. + /// + /// + /// An optional duration after which the entity should be requeued for processing. Defaults to null. + /// + /// + /// A failure instance containing the provided entity, error information, and requeue duration. + /// + public static ReconciliationResult Failure( + TEntity entity, string errorMessage, Exception? error = null, TimeSpan? requeueAfter = null) + => new(entity, false, errorMessage, error, requeueAfter); +} diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationTriggerSource.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationTriggerSource.cs new file mode 100644 index 00000000..d5020c39 --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationTriggerSource.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Defines the source that triggered the reconciliation process in the Kubernetes operator. +/// Used to identify which component or mechanism initiated the reconciliation cycle. +/// +public enum ReconciliationTriggerSource +{ + /// + /// Represents a reconciliation trigger initiated by the Kubernetes API server. + /// This source typically implies that the operator has been informed about + /// a resource event (e.g., creation, modification, deletion) via API server + /// notifications or resource watches. + /// + ApiServer, + + /// + /// Represents a reconciliation trigger initiated directly by the operator. + /// This source indicates that the reconciliation process was started internally + /// by the operator, such as during a scheduled task or an operator-specific event. + /// + Operator, +} diff --git a/src/KubeOps.Generator/Generators/OperatorBuilderGenerator.cs b/src/KubeOps.Generator/Generators/OperatorBuilderGenerator.cs index 288cd51c..3acb31de 100644 --- a/src/KubeOps.Generator/Generators/OperatorBuilderGenerator.cs +++ b/src/KubeOps.Generator/Generators/OperatorBuilderGenerator.cs @@ -14,8 +14,10 @@ namespace KubeOps.Generator.Generators; [Generator] -internal class OperatorBuilderGenerator : ISourceGenerator +internal sealed class OperatorBuilderGenerator : ISourceGenerator { + private const string BuilderIdentifier = "builder"; + public void Initialize(GeneratorInitializationContext context) { } @@ -36,7 +38,7 @@ public void Execute(GeneratorExecutionContext context) .WithParameterList(ParameterList( SingletonSeparatedList( Parameter( - Identifier("builder")) + Identifier(BuilderIdentifier)) .WithModifiers( TokenList( Token(SyntaxKind.ThisKeyword))) @@ -47,15 +49,15 @@ public void Execute(GeneratorExecutionContext context) InvocationExpression( MemberAccessExpression( SyntaxKind.SimpleMemberAccessExpression, - IdentifierName("builder"), + IdentifierName(BuilderIdentifier), IdentifierName("RegisterControllers")))), ExpressionStatement( InvocationExpression( MemberAccessExpression( SyntaxKind.SimpleMemberAccessExpression, - IdentifierName("builder"), + IdentifierName(BuilderIdentifier), IdentifierName("RegisterFinalizers")))), - ReturnStatement(IdentifierName("builder"))))))) + ReturnStatement(IdentifierName(BuilderIdentifier))))))) .NormalizeWhitespace(); context.AddSource( diff --git a/src/KubeOps.Generator/SyntaxReceiver/EntityControllerSyntaxReceiver.cs b/src/KubeOps.Generator/SyntaxReceiver/EntityControllerSyntaxReceiver.cs index 686acbe0..08351ab8 100644 --- a/src/KubeOps.Generator/SyntaxReceiver/EntityControllerSyntaxReceiver.cs +++ b/src/KubeOps.Generator/SyntaxReceiver/EntityControllerSyntaxReceiver.cs @@ -9,7 +9,7 @@ namespace KubeOps.Generator.SyntaxReceiver; internal sealed class EntityControllerSyntaxReceiver : ISyntaxContextReceiver { - private const string IEntityControllerMetadataName = "KubeOps.Abstractions.Controller.IEntityController`1"; + private const string IEntityControllerMetadataName = "KubeOps.Abstractions.Reconciliation.Controller.IEntityController`1"; public List<(ClassDeclarationSyntax Controller, string EntityName)> Controllers { get; } = []; diff --git a/src/KubeOps.Generator/SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs b/src/KubeOps.Generator/SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs index ceac128a..9f07e3a8 100644 --- a/src/KubeOps.Generator/SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs +++ b/src/KubeOps.Generator/SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs @@ -9,7 +9,7 @@ namespace KubeOps.Generator.SyntaxReceiver; internal sealed class EntityFinalizerSyntaxReceiver : ISyntaxContextReceiver { - private const string IEntityFinalizerMetadataName = "KubeOps.Abstractions.Finalizer.IEntityFinalizer`1"; + private const string IEntityFinalizerMetadataName = "KubeOps.Abstractions.Reconciliation.Finalizer.IEntityFinalizer`1"; public List<(ClassDeclarationSyntax Finalizer, string EntityName)> Finalizer { get; } = []; diff --git a/src/KubeOps.Generator/SyntaxReceiver/KubernetesEntitySyntaxReceiver.cs b/src/KubeOps.Generator/SyntaxReceiver/KubernetesEntitySyntaxReceiver.cs index f98bcc0d..6d1dd4a2 100644 --- a/src/KubeOps.Generator/SyntaxReceiver/KubernetesEntitySyntaxReceiver.cs +++ b/src/KubeOps.Generator/SyntaxReceiver/KubernetesEntitySyntaxReceiver.cs @@ -28,15 +28,27 @@ public void OnVisitSyntaxNode(GeneratorSyntaxContext context) Entities.Add(new( cls, - GetArgumentValue(attr, KindName) ?? cls.Identifier.ToString(), - GetArgumentValue(attr, VersionName) ?? DefaultVersion, - GetArgumentValue(attr, GroupName), - GetArgumentValue(attr, PluralName))); + GetArgumentValue(context.SemanticModel, attr, KindName) ?? cls.Identifier.ToString(), + GetArgumentValue(context.SemanticModel, attr, VersionName) ?? DefaultVersion, + GetArgumentValue(context.SemanticModel, attr, GroupName), + GetArgumentValue(context.SemanticModel, attr, PluralName))); } - private static string? GetArgumentValue(AttributeSyntax attr, string argName) => - attr.ArgumentList?.Arguments.FirstOrDefault(a => a.NameEquals?.Name.ToString() == argName) is - { Expression: LiteralExpressionSyntax { Token.ValueText: { } value } } + private static string? GetArgumentValue(SemanticModel model, AttributeSyntax attr, string argName) + { + if (attr.ArgumentList?.Arguments.FirstOrDefault(a => a.NameEquals?.Name.ToString() == argName) + is not { Expression: { } expr }) + { + return null; + } + + if (model.GetConstantValue(expr) is { HasValue: true, Value: string s }) + { + return s; + } + + return expr is LiteralExpressionSyntax { Token.ValueText: { } value } ? value : null; + } } diff --git a/src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs b/src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs index d016f16a..7272931a 100644 --- a/src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs +++ b/src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs @@ -13,6 +13,6 @@ namespace KubeOps.Operator.Web.Webhooks.Admission; /// /// A message that is passed to the API. /// A custom status code to provide more detailed information. -public record AdmissionStatus([property: JsonPropertyName("message")] +public sealed record AdmissionStatus([property: JsonPropertyName("message")] string Message, [property: JsonPropertyName("code")] int? Code = StatusCodes.Status200OK); diff --git a/src/KubeOps.Operator.Web/Webhooks/Admission/Mutation/MutationResult.cs b/src/KubeOps.Operator.Web/Webhooks/Admission/Mutation/MutationResult.cs index 100273d3..92a2747f 100644 --- a/src/KubeOps.Operator.Web/Webhooks/Admission/Mutation/MutationResult.cs +++ b/src/KubeOps.Operator.Web/Webhooks/Admission/Mutation/MutationResult.cs @@ -21,7 +21,7 @@ namespace KubeOps.Operator.Web.Webhooks.Admission.Mutation; /// /// The modified entity if any changes are requested. /// The type of the entity. -public record MutationResult(TEntity? ModifiedObject = default) : IActionResult +public sealed record MutationResult(TEntity? ModifiedObject = default) : IActionResult where TEntity : IKubernetesObject { private const string JsonPatch = "JSONPatch"; diff --git a/src/KubeOps.Operator/Builder/OperatorBuilder.cs b/src/KubeOps.Operator/Builder/OperatorBuilder.cs index 1c066729..dd78cd4b 100644 --- a/src/KubeOps.Operator/Builder/OperatorBuilder.cs +++ b/src/KubeOps.Operator/Builder/OperatorBuilder.cs @@ -8,18 +8,20 @@ using k8s.Models; using KubeOps.Abstractions.Builder; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Crds; using KubeOps.Abstractions.Entities; using KubeOps.Abstractions.Events; -using KubeOps.Abstractions.Finalizer; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Crds; using KubeOps.Operator.Events; using KubeOps.Operator.Finalizer; using KubeOps.Operator.LeaderElection; using KubeOps.Operator.Queue; +using KubeOps.Operator.Reconciliation; using KubeOps.Operator.Watcher; using Microsoft.Extensions.DependencyInjection; @@ -29,35 +31,38 @@ namespace KubeOps.Operator.Builder; internal sealed class OperatorBuilder : IOperatorBuilder { - private readonly OperatorSettings _settings; - public OperatorBuilder(IServiceCollection services, OperatorSettings settings) { - _settings = settings; + Settings = settings; Services = services; AddOperatorBase(); } public IServiceCollection Services { get; } + public OperatorSettings Settings { get; } + public IOperatorBuilder AddController() where TImplementation : class, IEntityController where TEntity : IKubernetesObject { - Services.AddHostedService>(); Services.TryAddScoped, TImplementation>(); - Services.TryAddSingleton(new TimedEntityQueue()); + Services.TryAddSingleton, TimedEntityQueue>(); + Services.TryAddSingleton, Reconciler>(); Services.TryAddTransient(); Services.TryAddTransient>(services => services.GetRequiredService().Create()); - if (_settings.EnableLeaderElection) - { - Services.AddHostedService>(); - } - else + switch (Settings.LeaderElectionType) { - Services.AddHostedService>(); + case LeaderElectionType.None: + Services.AddHostedService>(); + Services.AddHostedService>(); + break; + case LeaderElectionType.Single: + Services.AddHostedService>(); + Services.AddHostedService>(); + break; } return this; @@ -68,23 +73,9 @@ public IOperatorBuilder AddController( where TEntity : IKubernetesObject where TLabelSelector : class, IEntityLabelSelector { - Services.AddHostedService>(); - Services.TryAddScoped, TImplementation>(); - Services.TryAddSingleton(new TimedEntityQueue()); - Services.TryAddTransient(); - Services.TryAddTransient>(services => - services.GetRequiredService().Create()); + AddController(); Services.TryAddSingleton, TLabelSelector>(); - if (_settings.EnableLeaderElection) - { - Services.AddHostedService>(); - } - else - { - Services.AddHostedService>(); - } - return this; } @@ -112,19 +103,19 @@ public IOperatorBuilder AddCrdInstaller(Action? configure private void AddOperatorBase() { - Services.AddSingleton(_settings); - Services.AddSingleton(new ActivitySource(_settings.Name)); + Services.AddSingleton(Settings); + Services.AddSingleton(new ActivitySource(Settings.Name)); // add and configure resource watcher entity cache - Services.WithResourceWatcherEntityCaching(_settings); + Services.WithResourceWatcherEntityCaching(Settings); // Add the default configuration and the client separately. This allows external users to override either // just the config (e.g. for integration tests) or to replace the whole client, e.g. with a mock. - // We also add the k8s.IKubernetes as a singleton service, in order to allow to access internal services - // and also external users to make use of it's features that might not be implemented in the adapted client. + // We also add the k8s.IKubernetes as a singleton service, in order to allow accessing internal services + // and also external users to make use of its features that might not be implemented in the adapted client. // // Due to a memory leak in the Kubernetes client, it is important that the client is registered with - // with the same lifetime as the KubernetesClientConfiguration. This is tracked in kubernetes/csharp#1446. + // the same lifetime as the KubernetesClientConfiguration. This is tracked in kubernetes/csharp#1446. // https://github.com/kubernetes-client/csharp/issues/1446 // // The missing ability to inject a custom HTTP client and therefore the possibility to use the .AddHttpClient() @@ -140,7 +131,7 @@ private void AddOperatorBase() Services.AddSingleton(typeof(IEntityLabelSelector<>), typeof(DefaultEntityLabelSelector<>)); - if (_settings.EnableLeaderElection) + if (Settings.LeaderElectionType == LeaderElectionType.Single) { Services.AddLeaderElection(); } diff --git a/src/KubeOps.Operator/Finalizer/KubeOpsEventFinalizerAttacherFactory.cs b/src/KubeOps.Operator/Finalizer/KubeOpsEventFinalizerAttacherFactory.cs index 5a2db148..47975b07 100644 --- a/src/KubeOps.Operator/Finalizer/KubeOpsEventFinalizerAttacherFactory.cs +++ b/src/KubeOps.Operator/Finalizer/KubeOpsEventFinalizerAttacherFactory.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation.Finalizer; using KubeOps.KubernetesClient; using Microsoft.Extensions.Logging; diff --git a/src/KubeOps.Operator/KubeOps.Operator.csproj b/src/KubeOps.Operator/KubeOps.Operator.csproj index b230512d..88de0ee1 100644 --- a/src/KubeOps.Operator/KubeOps.Operator.csproj +++ b/src/KubeOps.Operator/KubeOps.Operator.csproj @@ -30,11 +30,11 @@ build/ - + <_Parameter1>$(MSBuildProjectName).Web.Test - - + + \ No newline at end of file diff --git a/src/KubeOps.Operator/Properties/AssemblyInfo.cs b/src/KubeOps.Operator/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..09d683dd --- /dev/null +++ b/src/KubeOps.Operator/Properties/AssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("KubeOps.Operator.Tests")] diff --git a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs index c7580cc3..e59c896b 100644 --- a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs +++ b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs @@ -5,19 +5,19 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace KubeOps.Operator.Queue; -internal sealed class EntityRequeueBackgroundService( +public class EntityRequeueBackgroundService( IKubernetesClient client, - TimedEntityQueue queue, - IServiceProvider provider, + ITimedEntityQueue queue, + IReconciler reconciler, ILogger> logger) : IHostedService, IDisposable, IAsyncDisposable where TEntity : IKubernetesObject { @@ -53,6 +53,23 @@ public Task StopAsync(CancellationToken cancellationToken) public void Dispose() { + Dispose(true); + GC.SuppressFinalize(this); + } + + public async ValueTask DisposeAsync() + { + await DisposeAsync(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposing) + { + return; + } + _cts.Dispose(); client.Dispose(); queue.Dispose(); @@ -60,13 +77,19 @@ public void Dispose() _disposed = true; } - public async ValueTask DisposeAsync() + protected virtual async ValueTask DisposeAsync(bool disposing) { + if (!disposing) + { + return; + } + await CastAndDispose(_cts); await CastAndDispose(client); await CastAndDispose(queue); _disposed = true; + return; static async ValueTask CastAndDispose(IDisposable resource) { @@ -81,13 +104,32 @@ static async ValueTask CastAndDispose(IDisposable resource) } } + protected virtual async Task ReconcileSingleAsync(RequeueEntry entry, CancellationToken cancellationToken) + { + logger.LogTrace("""Execute requested requeued reconciliation for "{Name}".""", entry.Entity.Name()); + + if (await client.GetAsync(entry.Entity.Name(), entry.Entity.Namespace(), cancellationToken) is not + { } entity) + { + logger.LogWarning( + """Requeued entity "{Name}" was not found. Skipping reconciliation.""", entry.Entity.Name()); + return; + } + + await reconciler.Reconcile( + ReconciliationContext.CreateFromOperatorEvent( + entity, + entry.RequeueType.ToWatchEventType()), + cancellationToken); + } + private async Task WatchAsync(CancellationToken cancellationToken) { - await foreach (var entity in queue) + await foreach (var entry in queue) { try { - await ReconcileSingleAsync(entity, cancellationToken); + await ReconcileSingleAsync(entry, cancellationToken); } catch (OperationCanceledException e) when (!cancellationToken.IsCancellationRequested) { @@ -95,8 +137,8 @@ private async Task WatchAsync(CancellationToken cancellationToken) e, """Queued reconciliation for the entity of type {ResourceType} for "{Kind}/{Name}" failed.""", typeof(TEntity).Name, - entity.Kind, - entity.Name()); + entry.Entity.Kind, + entry.Entity.Name()); } catch (Exception e) { @@ -104,26 +146,9 @@ private async Task WatchAsync(CancellationToken cancellationToken) e, """Queued reconciliation for the entity of type {ResourceType} for "{Kind}/{Name}" failed.""", typeof(TEntity).Name, - entity.Kind, - entity.Name()); + entry.Entity.Kind, + entry.Entity.Name()); } } } - - private async Task ReconcileSingleAsync(TEntity queued, CancellationToken cancellationToken) - { - logger.LogTrace("""Execute requested requeued reconciliation for "{Name}".""", queued.Name()); - - if (await client.GetAsync(queued.Name(), queued.Namespace(), cancellationToken) is not - { } entity) - { - logger.LogWarning( - """Requeued entity "{Name}" was not found. Skipping reconciliation.""", queued.Name()); - return; - } - - await using var scope = provider.CreateAsyncScope(); - var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.ReconcileAsync(entity, cancellationToken); - } } diff --git a/src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs b/src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs new file mode 100644 index 00000000..9aa26af0 --- /dev/null +++ b/src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Reconciliation.Queue; + +namespace KubeOps.Operator.Queue; + +/// +/// Represents a timed queue for managing Kubernetes entities of type . +/// This interface provides mechanisms to enqueue entities for later processing and remove entities from the queue. +/// +/// +/// The type of the Kubernetes entity. Must implement . +/// +public interface ITimedEntityQueue : IDisposable, IAsyncEnumerable> + where TEntity : IKubernetesObject +{ + /// + /// Adds the specified entity to the queue for processing after the specified time span has elapsed. + /// + /// The entity to be queued. + /// The type of requeue operation to handle (added, modified, or deleted). + /// The duration to wait before processing the entity. + /// A token that can be used to cancel the asynchronous operation. + /// A task that represents the asynchronous operation of enqueuing the entity. + Task Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn, CancellationToken cancellationToken); + + /// + /// Removes the specified entity from the queue. + /// + /// The entity to be removed from the queue. + /// A token that can be used to cancel the asynchronous operation. + /// A task that represents the asynchronous operation of removing the entity from the queue. + Task Remove(TEntity entity, CancellationToken cancellationToken); +} diff --git a/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs b/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs index c6592fef..f12fe070 100644 --- a/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs +++ b/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation.Queue; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -17,10 +17,10 @@ internal sealed class KubeOpsEntityRequeueFactory(IServiceProvider services) { public EntityRequeue Create() where TEntity : IKubernetesObject => - (entity, timeSpan) => + (entity, type, timeSpan, cancellationToken) => { var logger = services.GetService>>(); - var queue = services.GetRequiredService>(); + var queue = services.GetRequiredService>(); logger?.LogTrace( """Requeue entity "{Kind}/{Name}" in {Milliseconds}ms.""", @@ -28,6 +28,6 @@ public EntityRequeue Create() entity.Name(), timeSpan.TotalMilliseconds); - queue.Enqueue(entity, timeSpan); + queue.Enqueue(entity, type, timeSpan, cancellationToken); }; } diff --git a/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs b/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs new file mode 100644 index 00000000..53e852bb --- /dev/null +++ b/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Text.Json.Serialization; + +using KubeOps.Abstractions.Reconciliation.Queue; + +namespace KubeOps.Operator.Queue; + +public sealed record RequeueEntry +{ + [JsonConstructor] + private RequeueEntry(TEntity entity, RequeueType requeueType) + { + Entity = entity; + RequeueType = requeueType; + } + + public TEntity Entity { get; } + + public RequeueType RequeueType { get; } + + public static RequeueEntry CreateFor(TEntity entity, RequeueType requeueType) + => new(entity, requeueType); +} diff --git a/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs b/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs new file mode 100644 index 00000000..50a78fd5 --- /dev/null +++ b/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; + +using KubeOps.Abstractions.Reconciliation.Queue; + +namespace KubeOps.Operator.Queue; + +/// +/// Provides extension methods for converting between and . +/// +public static class RequeueTypeExtensions +{ + /// + /// Converts a to its corresponding . + /// + /// The watch event type to be converted. + /// The corresponding for the given . + /// Thrown when the provided is not supported. + public static RequeueType ToRequeueType(this WatchEventType watchEventType) + => watchEventType switch + { + WatchEventType.Added => RequeueType.Added, + WatchEventType.Modified => RequeueType.Modified, + WatchEventType.Deleted => RequeueType.Deleted, + _ => throw new NotSupportedException($"WatchEventType '{watchEventType}' is not supported!"), + }; + + /// + /// Converts a to its corresponding . + /// + /// The requeue type to be converted. + /// The corresponding for the given . + /// Thrown when the provided is not supported. + public static WatchEventType ToWatchEventType(this RequeueType requeueType) + => requeueType switch + { + RequeueType.Added => WatchEventType.Added, + RequeueType.Modified => WatchEventType.Modified, + RequeueType.Deleted => WatchEventType.Deleted, + _ => throw new NotSupportedException($"RequeueType '{requeueType}' is not supported!"), + }; +} diff --git a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs index 3bc228e8..5852df8e 100644 --- a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs +++ b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs @@ -7,6 +7,10 @@ using k8s; using k8s.Models; +using KubeOps.Abstractions.Reconciliation.Queue; + +using Microsoft.Extensions.Logging; + namespace KubeOps.Operator.Queue; /// @@ -14,7 +18,9 @@ namespace KubeOps.Operator.Queue; /// The given enumerable only contains items that should be considered for reconciliations. /// /// The type of the inner entity. -public sealed class TimedEntityQueue : IDisposable +public sealed class TimedEntityQueue( + ILogger> logger) + : ITimedEntityQueue where TEntity : IKubernetesObject { // A shared task factory for all the created tasks. @@ -24,47 +30,58 @@ public sealed class TimedEntityQueue : IDisposable private readonly ConcurrentDictionary> _management = new(); // The actual queue containing all the entries that have to be reconciled. - private readonly BlockingCollection _queue = new(new ConcurrentQueue()); + private readonly BlockingCollection> _queue = new(new ConcurrentQueue>()); internal int Count => _management.Count; - /// - /// Enqueues the given to happen in . - /// If the item already exists, the existing entry is updated. - /// - /// The entity. - /// The time after , where the item is reevaluated again. - public void Enqueue(TEntity entity, TimeSpan requeueIn) + /// + public Task Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn, CancellationToken cancellationToken) { - _management.AddOrUpdate( - TimedEntityQueue.GetKey(entity) ?? throw new InvalidOperationException("Cannot enqueue entities without name."), - key => - { - var entry = new TimedQueueEntry(entity, requeueIn); - _scheduledEntries.StartNew( - async () => - { - await entry.AddAfterDelay(_queue); - _management.TryRemove(key, out _); - }, - entry.Token); - return entry; - }, - (key, oldEntry) => - { - oldEntry.Cancel(); - var entry = new TimedQueueEntry(entity, requeueIn); - _scheduledEntries.StartNew( - async () => - { - await entry.AddAfterDelay(_queue); - _management.TryRemove(key, out _); - }, - entry.Token); - return entry; - }); + _management + .AddOrUpdate( + this.GetKey(entity) ?? throw new InvalidOperationException("Cannot enqueue entities without name."), + key => + { + logger.LogTrace( + """Adding schedule for entity "{Kind}/{Name}" to reconcile in {Seconds}s.""", + entity.Kind, + entity.Name(), + requeueIn.TotalSeconds); + + var entry = new TimedQueueEntry(entity, type, requeueIn); + _scheduledEntries.StartNew( + async () => + { + await entry.AddAfterDelay(_queue); + _management.TryRemove(key, out _); + }, + entry.Token); + return entry; + }, + (key, oldEntry) => + { + logger.LogTrace( + """Updating schedule for entity "{Kind}/{Name}" to reconcile in {Seconds}s.""", + entity.Kind, + entity.Name(), + requeueIn.TotalSeconds); + + oldEntry.Cancel(); + var entry = new TimedQueueEntry(entity, type, requeueIn); + _scheduledEntries.StartNew( + async () => + { + await entry.AddAfterDelay(_queue); + _management.TryRemove(key, out _); + }, + entry.Token); + return entry; + }); + + return Task.CompletedTask; } + /// public void Dispose() { _queue.Dispose(); @@ -74,7 +91,8 @@ public void Dispose() } } - public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + /// + public async IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default) { await Task.Yield(); foreach (var entry in _queue.GetConsumingEnumerable(cancellationToken)) @@ -83,32 +101,20 @@ public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken canc } } - public void Remove(TEntity entity) + /// + public Task Remove(TEntity entity, CancellationToken cancellationToken) { - var key = TimedEntityQueue.GetKey(entity); + var key = this.GetKey(entity); if (key is null) { - return; + return Task.CompletedTask; } if (_management.Remove(key, out var task)) { task.Cancel(); } - } - - private static string? GetKey(TEntity entity) - { - if (entity.Name() is null) - { - return null; - } - - if (entity.Namespace() is null) - { - return entity.Name(); - } - return $"{entity.Namespace()}/{entity.Name()}"; + return Task.CompletedTask; } } diff --git a/src/KubeOps.Operator/Queue/TimedEntityQueueExtensions.cs b/src/KubeOps.Operator/Queue/TimedEntityQueueExtensions.cs new file mode 100644 index 00000000..3ac377d8 --- /dev/null +++ b/src/KubeOps.Operator/Queue/TimedEntityQueueExtensions.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Operator.Queue; + +/// +/// Provides extension methods for the interface. +/// +public static class TimedEntityQueueExtensions +{ + /// + /// Retrieves a unique key for the specified Kubernetes entity. The key is constructed + /// using the entity's namespace and name, if available. If the entity does not have + /// a valid name, the method returns null. + /// + /// + /// The type of the Kubernetes entity. Must implement . + /// + /// + /// The timed entity queue from which the key should be derived. + /// + /// + /// The Kubernetes entity for which the key will be retrieved. + /// + /// + /// A string representing the unique key for the entity, or null if the entity does not have a valid name. + /// + // ReSharper disable once UnusedParameter.Global + public static string? GetKey(this ITimedEntityQueue queue, TEntity entity) + where TEntity : IKubernetesObject + { + if (string.IsNullOrWhiteSpace(entity.Name())) + { + return null; + } + + return string.IsNullOrWhiteSpace(entity.Namespace()) + ? entity.Name() + : $"{entity.Namespace()}/{entity.Name()}"; + } +} diff --git a/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs b/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs index 72ece41e..65aabf0e 100644 --- a/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs +++ b/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs @@ -4,6 +4,8 @@ using System.Collections.Concurrent; +using KubeOps.Abstractions.Reconciliation.Queue; + namespace KubeOps.Operator.Queue; internal sealed record TimedQueueEntry : IDisposable @@ -11,11 +13,13 @@ internal sealed record TimedQueueEntry : IDisposable private readonly CancellationTokenSource _cts = new(); private readonly TimeSpan _requeueIn; private readonly TEntity _entity; + private readonly RequeueType _requeueType; - public TimedQueueEntry(TEntity entity, TimeSpan requeueIn) + public TimedQueueEntry(TEntity entity, RequeueType requeueType, TimeSpan requeueIn) { _requeueIn = requeueIn; _entity = entity; + _requeueType = requeueType; } /// @@ -40,7 +44,7 @@ public void Cancel() /// /// The collection to add the entry to. /// A representing the asynchronous operation. - public async Task AddAfterDelay(BlockingCollection collection) + public async Task AddAfterDelay(BlockingCollection> collection) { try { @@ -50,7 +54,7 @@ public async Task AddAfterDelay(BlockingCollection collection) return; } - collection.TryAdd(_entity); + collection.TryAdd(RequeueEntry.CreateFor(_entity, _requeueType)); } catch (TaskCanceledException) { diff --git a/src/KubeOps.Operator/README.md b/src/KubeOps.Operator/README.md index d274def0..f3ad39ac 100644 --- a/src/KubeOps.Operator/README.md +++ b/src/KubeOps.Operator/README.md @@ -88,7 +88,7 @@ A controller reconciles a specific entity type. Implement controllers using the Example controller implementation: ```csharp -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Abstractions.Rbac; using KubeOps.KubernetesClient; using Microsoft.Extensions.Logging; @@ -154,7 +154,7 @@ A [finalizer](https://kubernetes.io/docs/concepts/overview/working-with-objects/ Finalizers are attached using an `EntityFinalizerAttacher` and are called when the entity is marked for deletion. ```csharp -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation.Finalizer; public class FinalizerOne : IEntityFinalizer { @@ -171,4 +171,4 @@ public class FinalizerOne : IEntityFinalizer ## Documentation -For more information, visit the [documentation](https://dotnet.github.io/dotnet-operator-sdk/). +For more information, visit the [documentation](https://dotnet.github.io/dotnet-operator-sdk/). \ No newline at end of file diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs new file mode 100644 index 00000000..dfcfd08a --- /dev/null +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -0,0 +1,202 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Builder; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; +using KubeOps.KubernetesClient; +using KubeOps.Operator.Constants; +using KubeOps.Operator.Queue; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +using ZiggyCreatures.Caching.Fusion; + +namespace KubeOps.Operator.Reconciliation; + +/// +/// The Reconciler class provides mechanisms for handling creation, modification, and deletion +/// events for Kubernetes objects of the specified entity type. It implements the IReconciler +/// interface and facilitates the reconciliation of desired and actual states of the entity. +/// +/// +/// The type of the Kubernetes entity being reconciled. Must implement IKubernetesObject +/// with V1ObjectMeta. +/// +/// +/// This class leverages logging, caching, and client services to manage and process +/// Kubernetes objects effectively. It also uses internal queuing capabilities for entity +/// processing and requeuing. +/// +internal sealed class Reconciler( + ILogger> logger, + IFusionCacheProvider cacheProvider, + IServiceProvider serviceProvider, + OperatorSettings operatorSettings, + ITimedEntityQueue entityQueue, + IKubernetesClient client) + : IReconciler + where TEntity : IKubernetesObject +{ + private readonly IFusionCache _entityCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); + + public async Task> Reconcile(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) + { + var result = reconciliationContext.EventType switch + { + WatchEventType.Added or WatchEventType.Modified => + await ReconcileModification(reconciliationContext, cancellationToken), + WatchEventType.Deleted => + await ReconcileDeletion(reconciliationContext, cancellationToken), + _ => throw new NotSupportedException($"Reconciliation event type {reconciliationContext.EventType} is not supported!"), + }; + + if (result.RequeueAfter.HasValue) + { + await entityQueue + .Enqueue( + result.Entity, + reconciliationContext.EventType.ToRequeueType(), + result.RequeueAfter.Value, + cancellationToken); + } + + return result; + } + + private async Task> ReconcileModification(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) + { + switch (reconciliationContext.Entity) + { + case { Metadata.DeletionTimestamp: null }: + if (reconciliationContext.IsTriggeredByApiServer()) + { + var cachedGeneration = await _entityCache.TryGetAsync( + reconciliationContext.Entity.Uid(), + token: cancellationToken); + + // Check if entity-spec has changed through "Generation" value increment. Skip reconcile if not changed. + if (cachedGeneration.HasValue && cachedGeneration >= reconciliationContext.Entity.Generation()) + { + logger.LogDebug( + """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", + reconciliationContext.Entity.Kind, + reconciliationContext.Entity.Name()); + + return ReconciliationResult.Success(reconciliationContext.Entity); + } + + // update cached generation since generation now changed + await _entityCache.SetAsync( + reconciliationContext.Entity.Uid(), + reconciliationContext.Entity.Generation() ?? 1, + token: cancellationToken); + } + + return await ReconcileEntity(reconciliationContext.Entity, cancellationToken); + case { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } }: + return await ReconcileFinalizersSequential(reconciliationContext.Entity, cancellationToken); + default: + return ReconciliationResult.Success(reconciliationContext.Entity); + } + } + + private async Task> ReconcileDeletion(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) + { + await entityQueue + .Remove( + reconciliationContext.Entity, + cancellationToken); + + await using var scope = serviceProvider.CreateAsyncScope(); + var controller = scope.ServiceProvider.GetRequiredService>(); + var result = await controller.DeletedAsync(reconciliationContext.Entity, cancellationToken); + + if (result.IsSuccess) + { + await _entityCache.RemoveAsync(reconciliationContext.Entity.Uid(), token: cancellationToken); + } + + return result; + } + + private async Task> ReconcileEntity(TEntity entity, CancellationToken cancellationToken) + { + await entityQueue + .Remove( + entity, + cancellationToken); + + await using var scope = serviceProvider.CreateAsyncScope(); + + if (operatorSettings.AutoAttachFinalizers) + { + var finalizers = scope.ServiceProvider.GetKeyedServices>(KeyedService.AnyKey); + + foreach (var finalizer in finalizers) + { + entity.AddFinalizer(finalizer.GetIdentifierName(entity)); + } + + entity = await client.UpdateAsync(entity, cancellationToken); + } + + var controller = scope.ServiceProvider.GetRequiredService>(); + return await controller.ReconcileAsync(entity, cancellationToken); + } + + private async Task> ReconcileFinalizersSequential(TEntity entity, CancellationToken cancellationToken) + { + await entityQueue + .Remove( + entity, + cancellationToken); + + await using var scope = serviceProvider.CreateAsyncScope(); + + // the condition to call ReconcileFinalizersSequentialAsync is: + // { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } } + // which implies that there is at least a single finalizer + var identifier = entity.Finalizers()[0]; + + if (scope.ServiceProvider.GetKeyedService>(identifier) is not + { } finalizer) + { + logger.LogDebug( + """Entity "{Kind}/{Name}" is finalizing but this operator has no registered finalizers for the identifier {FinalizerIdentifier}.""", + entity.Kind, + entity.Name(), + identifier); + return ReconciliationResult.Success(entity); + } + + var result = await finalizer.FinalizeAsync(entity, cancellationToken); + + if (!result.IsSuccess) + { + return result; + } + + entity = result.Entity; + + if (operatorSettings.AutoDetachFinalizers) + { + entity.RemoveFinalizer(identifier); + entity = await client.UpdateAsync(entity, cancellationToken); + } + + logger.LogInformation( + """Entity "{Kind}/{Name}" finalized with "{Finalizer}".""", + entity.Kind, + entity.Name(), + identifier); + + return ReconciliationResult.Success(entity, result.RequeueAfter); + } +} diff --git a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs index 15f20a72..3cb1b710 100644 --- a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs @@ -10,35 +10,29 @@ using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Entities; +using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; -using KubeOps.Operator.Queue; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using ZiggyCreatures.Caching.Fusion; - namespace KubeOps.Operator.Watcher; internal sealed class LeaderAwareResourceWatcher( ActivitySource activitySource, ILogger> logger, - IServiceProvider provider, - TimedEntityQueue queue, + IReconciler reconciler, OperatorSettings settings, IEntityLabelSelector labelSelector, - IFusionCacheProvider cacheProvider, IKubernetesClient client, IHostApplicationLifetime hostApplicationLifetime, LeaderElector elector) : ResourceWatcher( activitySource, logger, - provider, - queue, + reconciler, settings, labelSelector, - cacheProvider, client) where TEntity : IKubernetesObject { @@ -94,7 +88,7 @@ private void StartedLeading() if (_cts.IsCancellationRequested) { _cts.Dispose(); - _cts = new CancellationTokenSource(); + _cts = new(); } base.StartAsync(_cts.Token); @@ -107,7 +101,7 @@ private void StoppedLeading() logger.LogInformation("This instance stopped leading, stopping watcher."); // Stop the base implementation using the 'ApplicationStopped' cancellation token. - // The cancellation token should only be marked cancelled when the stop should no longer be graceful. + // The cancellation token should only be marked as canceled when the stop should no longer be graceful. base.StopAsync(hostApplicationLifetime.ApplicationStopped).Wait(); } } diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index d88f3b66..85e9b895 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -11,35 +11,26 @@ using k8s.Models; using KubeOps.Abstractions.Builder; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Entities; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; -using KubeOps.Operator.Constants; using KubeOps.Operator.Logging; -using KubeOps.Operator.Queue; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using ZiggyCreatures.Caching.Fusion; - namespace KubeOps.Operator.Watcher; public class ResourceWatcher( ActivitySource activitySource, ILogger> logger, - IServiceProvider provider, - TimedEntityQueue requeue, + IReconciler reconciler, OperatorSettings settings, IEntityLabelSelector labelSelector, - IFusionCacheProvider cacheProvider, IKubernetesClient client) : IHostedService, IAsyncDisposable, IDisposable where TEntity : IKubernetesObject { - private readonly IFusionCache _entityCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); private CancellationTokenSource _cancellationTokenSource = new(); private uint _watcherReconnectRetries; private Task? _eventWatcher; @@ -54,7 +45,7 @@ public virtual Task StartAsync(CancellationToken cancellationToken) if (_cancellationTokenSource.IsCancellationRequested) { _cancellationTokenSource.Dispose(); - _cancellationTokenSource = new CancellationTokenSource(); + _cancellationTokenSource = new(); } _eventWatcher = WatchClientEventsAsync(_cancellationTokenSource.Token); @@ -102,7 +93,6 @@ protected virtual void Dispose(bool disposing) _cancellationTokenSource.Dispose(); _eventWatcher?.Dispose(); - requeue.Dispose(); client.Dispose(); _disposed = true; @@ -116,7 +106,6 @@ protected virtual async ValueTask DisposeAsyncCore() } await CastAndDispose(_cancellationTokenSource); - await CastAndDispose(requeue); await CastAndDispose(client); _disposed = true; @@ -136,71 +125,15 @@ static async ValueTask CastAndDispose(IDisposable resource) } } - protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) - { - MaybeValue cachedGeneration; - - // Make sure Finalizers are running if Termination has began. - if (type != WatchEventType.Deleted && entity.Metadata.DeletionTimestamp is not null && entity.Metadata.Finalizers.Count > 0) - { - await ReconcileFinalizersSequentialAsync(entity, cancellationToken); - return; - } - - switch (type) - { - case WatchEventType.Added: - cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); - - if (!cachedGeneration.HasValue) - { - // Only perform reconciliation if the entity was not already in the cache. - await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 0, token: cancellationToken); - await ReconcileModificationAsync(entity, cancellationToken); - } - else - { - logger.LogDebug( - """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", - entity.Kind, - entity.Name()); - } - - break; - case WatchEventType.Modified: - cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); - - // Check if entity spec has changed through "Generation" value increment. Skip reconcile if not changed. - if (cachedGeneration.HasValue && cachedGeneration >= entity.Generation()) - { - logger.LogDebug( - """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", - entity.Kind, - entity.Name()); - return; - } - - // update cached generation since generation now changed - await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 1, token: cancellationToken); - await ReconcileModificationAsync(entity, cancellationToken); - - break; - case WatchEventType.Deleted: - await ReconcileDeletionAsync(entity, cancellationToken); - break; - default: - logger.LogWarning( - """Received unsupported event "{EventType}" for "{Kind}/{Name}".""", - type, - entity.Kind, - entity.Name()); - break; - } - } + protected virtual async Task> OnEventAsync(WatchEventType eventType, TEntity entity, CancellationToken cancellationToken) + => await reconciler.Reconcile( + ReconciliationContext.CreateFromApiServerEvent(entity, eventType), + cancellationToken); private async Task WatchClientEventsAsync(CancellationToken stoppingToken) { string? currentVersion = null; + while (!stoppingToken.IsCancellationRequested) { try @@ -230,7 +163,18 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) try { - await OnEventAsync(type, entity, stoppingToken); + var result = await OnEventAsync(type, entity, stoppingToken); + + if (result.IsFailure) + { + logger.LogError( + result.Error, + "Reconciliation of {EventType} for {Kind}/{Name} failed with message '{Message}'.", + type, + entity.Kind, + entity.Name(), + result.ErrorMessage); + } } catch (KubernetesException e) when (e.Status.Code is (int)HttpStatusCode.GatewayTimeout) { @@ -243,14 +187,9 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) throw; } catch (Exception e) - { - LogReconciliationFailed(e); - } - - void LogReconciliationFailed(Exception exception) { logger.LogError( - exception, + e, "Reconciliation of {EventType} for {Kind}/{Name} failed.", type, entity.Kind, @@ -317,55 +256,4 @@ e.InnerException is EndOfStreamException && delay.TotalSeconds); await Task.Delay(delay); } - - private async Task ReconcileDeletionAsync(TEntity entity, CancellationToken cancellationToken) - { - requeue.Remove(entity); - await _entityCache.RemoveAsync(entity.Uid(), token: cancellationToken); - - await using var scope = provider.CreateAsyncScope(); - var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.DeletedAsync(entity, cancellationToken); - } - - private async Task ReconcileFinalizersSequentialAsync(TEntity entity, CancellationToken cancellationToken) - { - requeue.Remove(entity); - await using var scope = provider.CreateAsyncScope(); - - var identifier = entity.Finalizers().FirstOrDefault(); - if (identifier is null) - { - return; - } - - if (scope.ServiceProvider.GetKeyedService>(identifier) is not - { } finalizer) - { - logger.LogDebug( - """Entity "{Kind}/{Name}" is finalizing but this operator has no registered finalizers for the identifier {FinalizerIdentifier}.""", - entity.Kind, - entity.Name(), - identifier); - return; - } - - await finalizer.FinalizeAsync(entity, cancellationToken); - entity.RemoveFinalizer(identifier); - await client.UpdateAsync(entity, cancellationToken); - logger.LogInformation( - """Entity "{Kind}/{Name}" finalized with "{Finalizer}".""", - entity.Kind, - entity.Name(), - identifier); - } - - private async Task ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) - { - // Re-queue should requested in the controller reconcile method. Invalidate any existing queues. - requeue.Remove(entity); - await using var scope = provider.CreateAsyncScope(); - var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.ReconcileAsync(entity, cancellationToken); - } } diff --git a/src/KubeOps.Templates/Templates/Operator.CSharp/Controller/DemoController.cs b/src/KubeOps.Templates/Templates/Operator.CSharp/Controller/DemoController.cs index bc7be892..f6af3968 100644 --- a/src/KubeOps.Templates/Templates/Operator.CSharp/Controller/DemoController.cs +++ b/src/KubeOps.Templates/Templates/Operator.CSharp/Controller/DemoController.cs @@ -1,4 +1,4 @@ -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Abstractions.Rbac; using Microsoft.Extensions.Logging; diff --git a/src/KubeOps.Templates/Templates/Operator.CSharp/Finalizer/DemoFinalizer.cs b/src/KubeOps.Templates/Templates/Operator.CSharp/Finalizer/DemoFinalizer.cs index 8d9cd92d..225f2186 100644 --- a/src/KubeOps.Templates/Templates/Operator.CSharp/Finalizer/DemoFinalizer.cs +++ b/src/KubeOps.Templates/Templates/Operator.CSharp/Finalizer/DemoFinalizer.cs @@ -1,6 +1,6 @@ using k8s.Models; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation.Finalizer; using Microsoft.Extensions.Logging; diff --git a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Controller/DemoController.cs b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Controller/DemoController.cs index bc7be892..f6af3968 100644 --- a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Controller/DemoController.cs +++ b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Controller/DemoController.cs @@ -1,4 +1,4 @@ -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Abstractions.Rbac; using Microsoft.Extensions.Logging; diff --git a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Finalizer/DemoFinalizer.cs b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Finalizer/DemoFinalizer.cs index 8d9cd92d..225f2186 100644 --- a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Finalizer/DemoFinalizer.cs +++ b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Finalizer/DemoFinalizer.cs @@ -1,6 +1,6 @@ using k8s.Models; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation.Finalizer; using Microsoft.Extensions.Logging; diff --git a/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj b/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj index 7bc34335..72f8a045 100644 --- a/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj +++ b/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj @@ -2,4 +2,4 @@ - + \ No newline at end of file diff --git a/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Abstractions.Test/Reconciliation/Finalizer/EntityFinalizerExtensions.Test.cs b/test/KubeOps.Abstractions.Test/Reconciliation/Finalizer/EntityFinalizerExtensions.Test.cs new file mode 100644 index 00000000..7f722c1c --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Reconciliation/Finalizer/EntityFinalizerExtensions.Test.cs @@ -0,0 +1,142 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Finalizer; + +namespace KubeOps.Abstractions.Test.Finalizer; + +public sealed class EntityFinalizerExtensions +{ + private const string Group = "finalizer.test"; + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Entity_Group_Has_String_Value() + { + var sut = new EntityWithGroupAsStringValueFinalizer(); + var entity = new EntityWithGroupAsStringValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be("finalizer.test/entitywithgroupasstringvaluefinalizer"); + } + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Entity_Group_Has_Const_Value() + { + var sut = new EntityWithGroupAsConstValueFinalizer(); + var entity = new EntityWithGroupAsConstValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be($"{Group}/entitywithgroupasconstvaluefinalizer"); + } + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Entity_Group_Has_No_Value() + { + var sut = new EntityWithNoGroupFinalizer(); + var entity = new EntityWithNoGroupValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be("entitywithnogroupfinalizer"); + } + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Finalizer_Not_Ending_With_Finalizer() + { + var sut = new EntityFinalizerNotEndingOnFinalizer1(); + var entity = new EntityWithGroupAsConstValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be($"{Group}/entityfinalizernotendingonfinalizer1finalizer"); + } + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Finalizer_Identifier_Would_Be_Greater_Than_63_Characters() + { + var sut = new EntityFinalizerWithATotalIdentifierNameHavingALengthGreaterThan63(); + var entity = new EntityWithGroupAsConstValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be($"{Group}/entityfinalizerwithatotalidentifiernamehavingale"); + identifierName.Length.Should().Be(63); + } + + private sealed class EntityFinalizerWithATotalIdentifierNameHavingALengthGreaterThan63 + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); + } + + private sealed class EntityFinalizerNotEndingOnFinalizer1 + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); + } + + private sealed class EntityWithGroupAsStringValueFinalizer + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityWithGroupAsStringValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); + } + + private sealed class EntityWithGroupAsConstValueFinalizer + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); + } + + private sealed class EntityWithNoGroupFinalizer + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityWithNoGroupValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); + } + + [KubernetesEntity(Group = "finalizer.test", ApiVersion = "v1", Kind = "FinalizerTest")] + private sealed class EntityWithGroupAsStringValue + : IKubernetesObject + { + public string ApiVersion { get; set; } = "finalizer.test/v1"; + + public string Kind { get; set; } = "FinalizerTest"; + + public V1ObjectMeta Metadata { get; set; } = new(); + } + + [KubernetesEntity(Group = Group, ApiVersion = "v1", Kind = "FinalizerTest")] + private sealed class EntityWithGroupAsConstValue + : IKubernetesObject + { + public string ApiVersion { get; set; } = "finalizer.test/v1"; + + public string Kind { get; set; } = "FinalizerTest"; + + public V1ObjectMeta Metadata { get; set; } = new(); + } + + [KubernetesEntity] + private sealed class EntityWithNoGroupValue + : IKubernetesObject + { + public string ApiVersion { get; set; } = "finalizer.test/v1"; + + public string Kind { get; set; } = "FinalizerTest"; + + public V1ObjectMeta Metadata { get; set; } = new(); + } +} diff --git a/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationContext.Test.cs b/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationContext.Test.cs new file mode 100644 index 00000000..41ef7d8f --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationContext.Test.cs @@ -0,0 +1,190 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Reconciliation; + +namespace KubeOps.Abstractions.Test.Reconciliation; + +public sealed class ReconciliationContextTest +{ + [Fact] + public void CreateFromApiServerEvent_Should_Create_Context_With_ApiServer_TriggerSource() + { + var entity = CreateTestEntity(); + const WatchEventType eventType = WatchEventType.Added; + + var context = ReconciliationContext.CreateFromApiServerEvent(entity, eventType); + + context.Entity.Should().Be(entity); + context.EventType.Should().Be(eventType); + context.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.ApiServer); + } + + [Fact] + public void CreateFromOperatorEvent_Should_Create_Context_With_Operator_TriggerSource() + { + var entity = CreateTestEntity(); + const WatchEventType eventType = WatchEventType.Modified; + + var context = ReconciliationContext.CreateFromOperatorEvent(entity, eventType); + + context.Entity.Should().Be(entity); + context.EventType.Should().Be(eventType); + context.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.Operator); + } + + [Theory] + [InlineData(WatchEventType.Added)] + [InlineData(WatchEventType.Modified)] + [InlineData(WatchEventType.Deleted)] + public void CreateFromApiServerEvent_Should_Support_All_WatchEventTypes(WatchEventType eventType) + { + var entity = CreateTestEntity(); + + var context = ReconciliationContext.CreateFromApiServerEvent(entity, eventType); + + context.EventType.Should().Be(eventType); + context.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.ApiServer); + } + + [Theory] + [InlineData(WatchEventType.Added)] + [InlineData(WatchEventType.Modified)] + [InlineData(WatchEventType.Deleted)] + public void CreateFromOperatorEvent_Should_Support_All_WatchEventTypes(WatchEventType eventType) + { + var entity = CreateTestEntity(); + + var context = ReconciliationContext.CreateFromOperatorEvent(entity, eventType); + + context.EventType.Should().Be(eventType); + context.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.Operator); + } + + [Fact] + public void IsTriggeredByApiServer_Should_Return_True_For_ApiServer_Context() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + var isTriggeredByApiServer = context.IsTriggeredByApiServer(); + var isTriggeredByOperator = context.IsTriggeredByOperator(); + + isTriggeredByApiServer.Should().BeTrue(); + isTriggeredByOperator.Should().BeFalse(); + } + + [Fact] + public void IsTriggeredByOperator_Should_Return_True_For_Operator_Context() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromOperatorEvent(entity, WatchEventType.Modified); + + var isTriggeredByOperator = context.IsTriggeredByOperator(); + var isTriggeredByApiServer = context.IsTriggeredByApiServer(); + + isTriggeredByOperator.Should().BeTrue(); + isTriggeredByApiServer.Should().BeFalse(); + } + + [Fact] + public void Record_Equality_Should_Work_For_Same_Values() + { + var entity = CreateTestEntity("test-entity"); + + var context1 = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var context2 = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + context1.Should().NotBeSameAs(context2); + context1.Entity.Should().BeSameAs(context2.Entity); + context1.EventType.Should().Be(context2.EventType); + context1.ReconciliationTriggerSource.Should().Be(context2.ReconciliationTriggerSource); + } + + [Fact] + public void Contexts_With_Different_EventTypes_Should_Have_Different_EventTypes() + { + var entity = CreateTestEntity(); + + var contextAdded = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var contextModified = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + var contextDeleted = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Deleted); + + contextAdded.EventType.Should().Be(WatchEventType.Added); + contextModified.EventType.Should().Be(WatchEventType.Modified); + contextDeleted.EventType.Should().Be(WatchEventType.Deleted); + } + + [Fact] + public void Contexts_With_Different_TriggerSources_Should_Have_Different_TriggerSources() + { + var entity = CreateTestEntity(); + + var apiServerContext = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var operatorContext = ReconciliationContext.CreateFromOperatorEvent(entity, WatchEventType.Added); + + apiServerContext.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.ApiServer); + operatorContext.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.Operator); + apiServerContext.ReconciliationTriggerSource.Should().NotBe(operatorContext.ReconciliationTriggerSource); + } + + [Fact] + public void Context_Should_Contain_Entity_Metadata() + { + var entity = CreateTestEntity("test-configmap", "test-namespace"); + + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + context.Entity.Metadata.Name.Should().Be("test-configmap"); + context.Entity.Metadata.NamespaceProperty.Should().Be("test-namespace"); + } + + [Theory] + [InlineData(ReconciliationTriggerSource.ApiServer, WatchEventType.Added)] + [InlineData(ReconciliationTriggerSource.ApiServer, WatchEventType.Modified)] + [InlineData(ReconciliationTriggerSource.ApiServer, WatchEventType.Deleted)] + [InlineData(ReconciliationTriggerSource.Operator, WatchEventType.Added)] + [InlineData(ReconciliationTriggerSource.Operator, WatchEventType.Modified)] + [InlineData(ReconciliationTriggerSource.Operator, WatchEventType.Deleted)] + public void Context_Should_Support_All_Combinations_Of_TriggerSource_And_EventType( + ReconciliationTriggerSource triggerSource, + WatchEventType eventType) + { + var entity = CreateTestEntity(); + + var context = triggerSource == ReconciliationTriggerSource.ApiServer + ? ReconciliationContext.CreateFromApiServerEvent(entity, eventType) + : ReconciliationContext.CreateFromOperatorEvent(entity, eventType); + + context.ReconciliationTriggerSource.Should().Be(triggerSource); + context.EventType.Should().Be(eventType); + } + + [Fact] + public void Multiple_Contexts_With_Same_Entity_Should_Share_Entity_Reference() + { + var entity = CreateTestEntity(); + + var context1 = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var context2 = ReconciliationContext.CreateFromOperatorEvent(entity, WatchEventType.Modified); + + context1.Entity.Should().BeSameAs(context2.Entity); + } + + private static V1ConfigMap CreateTestEntity(string? name = null, string? ns = null) + => new() + { + Metadata = new() + { + Name = name ?? "test-configmap", + NamespaceProperty = ns ?? "default", + Uid = Guid.NewGuid().ToString(), + }, + }; +} diff --git a/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationResult.Test.cs b/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationResult.Test.cs new file mode 100644 index 00000000..34c48538 --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationResult.Test.cs @@ -0,0 +1,263 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s.Models; + +using KubeOps.Abstractions.Reconciliation; + +namespace KubeOps.Abstractions.Test.Reconciliation; + +public sealed class ReconciliationResultTest +{ + [Fact] + public void Success_Should_Create_Successful_Result() + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Success(entity); + + result.IsSuccess.Should().BeTrue(); + result.IsFailure.Should().BeFalse(); + result.Entity.Should().Be(entity); + result.ErrorMessage.Should().BeNull(); + result.Error.Should().BeNull(); + result.RequeueAfter.Should().BeNull(); + } + + [Fact] + public void Success_With_RequeueAfter_Should_Set_RequeueAfter() + { + var entity = CreateTestEntity(); + var requeueAfter = TimeSpan.FromMinutes(5); + + var result = ReconciliationResult.Success(entity, requeueAfter); + + result.IsSuccess.Should().BeTrue(); + result.RequeueAfter.Should().Be(requeueAfter); + result.Entity.Should().Be(entity); + } + + [Fact] + public void Success_With_Null_RequeueAfter_Should_Not_Set_RequeueAfter() + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Success(entity, null); + + result.IsSuccess.Should().BeTrue(); + result.RequeueAfter.Should().BeNull(); + } + + [Fact] + public void Failure_Should_Create_Failed_Result_With_ErrorMessage() + { + var entity = CreateTestEntity(); + var errorMessage = "Reconciliation failed due to timeout"; + + var result = ReconciliationResult.Failure(entity, errorMessage); + + result.IsSuccess.Should().BeFalse(); + result.IsFailure.Should().BeTrue(); + result.Entity.Should().Be(entity); + result.ErrorMessage.Should().Be(errorMessage); + result.Error.Should().BeNull(); + result.RequeueAfter.Should().BeNull(); + } + + [Fact] + public void Failure_With_Exception_Should_Set_Error() + { + var entity = CreateTestEntity(); + const string errorMessage = "Reconciliation failed"; + var exception = new InvalidOperationException("Invalid state detected"); + + var result = ReconciliationResult.Failure(entity, errorMessage, exception); + + result.IsFailure.Should().BeTrue(); + result.ErrorMessage.Should().Be(errorMessage); + result.Error.Should().Be(exception); + result.Error.Message.Should().Be("Invalid state detected"); + } + + [Fact] + public void Failure_With_RequeueAfter_Should_Set_RequeueAfter() + { + var entity = CreateTestEntity(); + const string errorMessage = "Transient failure"; + var requeueAfter = TimeSpan.FromSeconds(30); + + var result = ReconciliationResult.Failure( + entity, + errorMessage, + requeueAfter: requeueAfter); + + result.IsFailure.Should().BeTrue(); + result.ErrorMessage.Should().Be(errorMessage); + result.RequeueAfter.Should().Be(requeueAfter); + } + + [Fact] + public void Failure_With_All_Parameters_Should_Set_All_Properties() + { + var entity = CreateTestEntity(); + const string errorMessage = "Complete failure information"; + var exception = new TimeoutException("Operation timed out"); + var requeueAfter = TimeSpan.FromMinutes(2); + + var result = ReconciliationResult.Failure( + entity, + errorMessage, + exception, + requeueAfter); + + result.IsFailure.Should().BeTrue(); + result.ErrorMessage.Should().Be(errorMessage); + result.Error.Should().Be(exception); + result.RequeueAfter.Should().Be(requeueAfter); + } + + [Fact] + public void RequeueAfter_Should_Be_Mutable() + { + var entity = CreateTestEntity(); + var result = ReconciliationResult.Success(entity); + + result.RequeueAfter = TimeSpan.FromSeconds(45); + + result.RequeueAfter.Should().Be(TimeSpan.FromSeconds(45)); + } + + [Fact] + public void RequeueAfter_Can_Be_Changed_After_Creation() + { + var entity = CreateTestEntity(); + var initialRequeueAfter = TimeSpan.FromMinutes(1); + var result = ReconciliationResult.Success(entity, initialRequeueAfter); + + result.RequeueAfter = TimeSpan.FromMinutes(5); + + result.RequeueAfter.Should().Be(TimeSpan.FromMinutes(5)); + result.RequeueAfter.Should().NotBe(initialRequeueAfter); + } + + [Fact] + public void RequeueAfter_Can_Be_Set_To_Null() + { + var entity = CreateTestEntity(); + var result = ReconciliationResult.Success(entity, TimeSpan.FromMinutes(1)); + + result.RequeueAfter = null; + + result.RequeueAfter.Should().BeNull(); + } + + [Fact] + public void IsSuccess_And_IsFailure_Should_Be_Mutually_Exclusive() + { + var entity = CreateTestEntity(); + + var successResult = ReconciliationResult.Success(entity); + var failureResult = ReconciliationResult.Failure(entity, "Error"); + + successResult.IsSuccess.Should().BeTrue(); + successResult.IsFailure.Should().BeFalse(); + + failureResult.IsSuccess.Should().BeFalse(); + failureResult.IsFailure.Should().BeTrue(); + } + + [Fact] + public void Success_Result_ErrorMessage_Should_Be_Null() + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Success(entity); + + if (result.IsSuccess) + { + result.ErrorMessage.Should().BeNull(); + } + } + + [Fact] + public void Failure_Result_ErrorMessage_Should_Not_Be_Null() + { + var entity = CreateTestEntity(); + var errorMessage = "Something went wrong"; + + var result = ReconciliationResult.Failure(entity, errorMessage); + + if (result.IsFailure) + { + // This should compile without nullable warning due to MemberNotNullWhen attribute + string message = result.ErrorMessage; + message.Should().NotBeNull(); + message.Should().Be(errorMessage); + } + } + + [Fact] + public void Record_Equality_Should_Work_For_Success_Results() + { + var entity1 = CreateTestEntity("test-entity"); + var entity2 = CreateTestEntity("test-entity"); + + var result1 = ReconciliationResult.Success(entity1); + var result2 = ReconciliationResult.Success(entity2); + + // Records with same values should be equal + result1.Should().NotBeSameAs(result2); + } + + [Fact] + public void Entity_Reference_Should_Be_Preserved() + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Success(entity); + + result.Entity.Should().BeSameAs(entity); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(60)] + [InlineData(3600)] + public void Success_Should_Accept_Various_RequeueAfter_Values(int seconds) + { + var entity = CreateTestEntity(); + var requeueAfter = TimeSpan.FromSeconds(seconds); + + var result = ReconciliationResult.Success(entity, requeueAfter); + + result.RequeueAfter.Should().Be(requeueAfter); + } + + [Theory] + [InlineData("Short error")] + [InlineData("A much longer error message that contains detailed information about what went wrong")] + [InlineData("")] + public void Failure_Should_Accept_Various_ErrorMessage_Lengths(string errorMessage) + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Failure(entity, errorMessage); + + result.ErrorMessage.Should().Be(errorMessage); + } + + private static V1ConfigMap CreateTestEntity(string? name = null) + => new() + { + Metadata = new() + { + Name = name ?? "test-configmap", + NamespaceProperty = "default", + Uid = Guid.NewGuid().ToString(), + }, + }; +} diff --git a/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Generator.Test/ControllerRegistrationGenerator.Test.cs b/test/KubeOps.Generator.Test/ControllerRegistrationGenerator.Test.cs index 6477f6dc..021bb047 100644 --- a/test/KubeOps.Generator.Test/ControllerRegistrationGenerator.Test.cs +++ b/test/KubeOps.Generator.Test/ControllerRegistrationGenerator.Test.cs @@ -35,7 +35,7 @@ public static IOperatorBuilder RegisterControllers(this IOperatorBuilder builder [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Controller; + using KubeOps.Abstractions.Reconciliation.Controller; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public class V1TestEntity : IKubernetesObject @@ -65,7 +65,7 @@ public static IOperatorBuilder RegisterControllers(this IOperatorBuilder builder [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Controller; + using KubeOps.Abstractions.Reconciliation.Controller; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject diff --git a/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs b/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs index 6fa73317..fca9af3c 100644 --- a/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs +++ b/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs @@ -13,7 +13,7 @@ namespace KubeOps.Generator.Test; -public class FinalizerRegistrationGeneratorTest +public sealed class FinalizerRegistrationGeneratorTest { [Theory] [InlineData("", """ @@ -35,7 +35,45 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; + + public static class Constants + { + public const string Group = "testing.dev"; + public const string ApiVersion = "v1"; + public const string Kind = "TestEntity"; + } + + [KubernetesEntity(Group = Constants.Group, ApiVersion = Constants.ApiVersion, Kind = Constants.Kind)] + public sealed class V1TestEntity : IKubernetesObject + { + } + + public sealed class V1TestEntityFinalizer : IEntityFinalizer + { + } + """, """ + // + // This code was generated by a tool. + // Changes to this file may cause incorrect behavior and will be lost if the code is regenerated. + // + #pragma warning disable CS1591 + using KubeOps.Abstractions.Builder; + + public static class FinalizerRegistrations + { + public const string V1TestEntityFinalizerIdentifier = "testing.dev/v1testentityfinalizer"; + public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) + { + builder.AddFinalizer(V1TestEntityFinalizerIdentifier); + return builder; + } + } + """)] + [InlineData(""" + using k8s; + using k8s.Models; + using KubeOps.Abstractions.Reconciliation.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject @@ -66,7 +104,7 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject @@ -102,7 +140,7 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject @@ -139,7 +177,7 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject diff --git a/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Generator.Test/TestHelperExtensions.cs b/test/KubeOps.Generator.Test/TestHelperExtensions.cs index 8dc9edca..2f40cdec 100644 --- a/test/KubeOps.Generator.Test/TestHelperExtensions.cs +++ b/test/KubeOps.Generator.Test/TestHelperExtensions.cs @@ -19,7 +19,7 @@ public static Compilation CreateCompilation(this string source) ], [ MetadataReference.CreateFromFile(typeof(Binder).GetTypeInfo().Assembly.Location), - MetadataReference.CreateFromFile(typeof(Abstractions.Controller.IEntityController<>).GetTypeInfo().Assembly.Location), + MetadataReference.CreateFromFile(typeof(Abstractions.Reconciliation.Controller.IEntityController<>).GetTypeInfo().Assembly.Location), MetadataReference.CreateFromFile(typeof(k8s.IKubernetesObject<>).GetTypeInfo().Assembly.Location), ], new(OutputKind.DynamicallyLinkedLibrary)); diff --git a/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs index 34cf751a..105b024c 100644 --- a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs +++ b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs @@ -5,14 +5,14 @@ using FluentAssertions; using KubeOps.Abstractions.Builder; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Entities; using KubeOps.Abstractions.Events; -using KubeOps.Abstractions.Finalizer; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient.LabelSelectors; using KubeOps.Operator.Builder; -using KubeOps.Operator.Finalizer; using KubeOps.Operator.Queue; using KubeOps.Operator.Test.TestEntities; using KubeOps.Operator.Watcher; @@ -23,7 +23,7 @@ namespace KubeOps.Operator.Test.Builder; -public class OperatorBuilderTest +public sealed class OperatorBuilderTest { private readonly IOperatorBuilder _builder = new OperatorBuilder(new ServiceCollection(), new()); @@ -45,7 +45,6 @@ public void Should_Add_Default_Resources() [Fact] public void Should_Use_Specific_EntityLabelSelector_Implementation() { - // Arrange var services = new ServiceCollection(); // Register the default and specific implementations @@ -54,10 +53,8 @@ public void Should_Use_Specific_EntityLabelSelector_Implementation() var serviceProvider = services.BuildServiceProvider(); - // Act var resolvedService = serviceProvider.GetRequiredService>(); - // Assert Assert.IsType(resolvedService); } @@ -75,7 +72,7 @@ public void Should_Add_Controller_Resources() s.ImplementationType == typeof(ResourceWatcher) && s.Lifetime == ServiceLifetime.Singleton); _builder.Services.Should().Contain(s => - s.ServiceType == typeof(TimedEntityQueue) && + s.ServiceType == typeof(ITimedEntityQueue) && s.Lifetime == ServiceLifetime.Singleton); _builder.Services.Should().Contain(s => s.ServiceType == typeof(EntityRequeue) && @@ -96,7 +93,7 @@ public void Should_Add_Controller_Resources_With_Label_Selector() s.ImplementationType == typeof(ResourceWatcher) && s.Lifetime == ServiceLifetime.Singleton); _builder.Services.Should().Contain(s => - s.ServiceType == typeof(TimedEntityQueue) && + s.ServiceType == typeof(ITimedEntityQueue) && s.Lifetime == ServiceLifetime.Singleton); _builder.Services.Should().Contain(s => s.ServiceType == typeof(EntityRequeue) && @@ -124,7 +121,7 @@ public void Should_Add_Finalizer_Resources() [Fact] public void Should_Add_Leader_Elector() { - var builder = new OperatorBuilder(new ServiceCollection(), new() { EnableLeaderElection = true }); + var builder = new OperatorBuilder(new ServiceCollection(), new() { LeaderElectionType = LeaderElectionType.Single }); builder.Services.Should().Contain(s => s.ServiceType == typeof(k8s.LeaderElection.LeaderElector) && s.Lifetime == ServiceLifetime.Singleton); @@ -133,7 +130,7 @@ public void Should_Add_Leader_Elector() [Fact] public void Should_Add_LeaderAwareResourceWatcher() { - var builder = new OperatorBuilder(new ServiceCollection(), new() { EnableLeaderElection = true }); + var builder = new OperatorBuilder(new ServiceCollection(), new() { LeaderElectionType = LeaderElectionType.Single }); builder.AddController(); builder.Services.Should().Contain(s => @@ -146,22 +143,22 @@ public void Should_Add_LeaderAwareResourceWatcher() s.Lifetime == ServiceLifetime.Singleton); } - private class TestController : IEntityController + private sealed class TestController : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(ReconciliationResult.Success(entity)); - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(ReconciliationResult.Success(entity)); } - private class TestFinalizer : IEntityFinalizer + private sealed class TestFinalizer : IEntityFinalizer { - public Task FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(ReconciliationResult.Success(entity)); } - private class TestLabelSelector : IEntityLabelSelector + private sealed class TestLabelSelector : IEntityLabelSelector { public ValueTask GetLabelSelectorAsync(CancellationToken cancellationToken) { diff --git a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs index 2381ffc7..a6adf42c 100644 --- a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs @@ -4,8 +4,9 @@ using FluentAssertions; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Queue; using KubeOps.Operator.Test.TestEntities; @@ -15,7 +16,7 @@ namespace KubeOps.Operator.Test.Controller; -public class CancelEntityRequeueIntegrationTest : IntegrationTestBase +public sealed class CancelEntityRequeueIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -35,7 +36,10 @@ public async Task Should_Cancel_Requeue_If_New_Event_Fires() await _mock.WaitForInvocations; _mock.Invocations.Count.Should().Be(2); - Services.GetRequiredService>().Count.Should().Be(0); + var timedEntityQueue = Services.GetRequiredService>(); + timedEntityQueue.Should().NotBeNull(); + timedEntityQueue.Should().BeOfType>(); + timedEntityQueue.As>().Count.Should().Be(0); } [Fact] @@ -48,8 +52,11 @@ public async Task Should_Not_Affect_Queues_If_Only_Status_Updated() await _mock.WaitForInvocations; _mock.Invocations.Count.Should().Be(1); - Services.GetRequiredService>().Count.Should().Be(1); + var timedEntityQueue = Services.GetRequiredService>(); + timedEntityQueue.Should().NotBeNull(); + timedEntityQueue.Should().BeOfType>(); + timedEntityQueue.As>().Count.Should().Be(1); } public override async Task InitializeAsync() @@ -77,20 +84,20 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (svc.Invocations.Count < 2) { - requeue(entity, TimeSpan.FromMilliseconds(1000)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1000), CancellationToken.None); } - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs index 27e86443..c6021993 100644 --- a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs @@ -4,8 +4,9 @@ using FluentAssertions; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Queue; using KubeOps.Operator.Test.TestEntities; @@ -15,7 +16,7 @@ namespace KubeOps.Operator.Test.Controller; -public class DeletedEntityRequeueIntegrationTest : IntegrationTestBase +public sealed class DeletedEntityRequeueIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -31,7 +32,10 @@ public async Task Should_Cancel_Requeue_If_Entity_Is_Deleted() await _mock.WaitForInvocations; _mock.Invocations.Count.Should().Be(2); - Services.GetRequiredService>().Count.Should().Be(0); + var timedEntityQueue = Services.GetRequiredService>(); + timedEntityQueue.Should().NotBeNull(); + timedEntityQueue.Should().BeOfType>(); + timedEntityQueue.As>().Count.Should().Be(0); } public override async Task InitializeAsync() @@ -59,17 +63,17 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - requeue(entity, TimeSpan.FromMilliseconds(1000)); - return Task.CompletedTask; + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1000), CancellationToken.None); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs index 6f0921db..fee650d9 100644 --- a/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs @@ -6,7 +6,8 @@ using k8s.Models; -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -120,16 +121,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs index e600f280..2101286b 100644 --- a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs @@ -4,8 +4,9 @@ using FluentAssertions; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -14,7 +15,7 @@ namespace KubeOps.Operator.Test.Controller; -public class EntityRequeueIntegrationTest : IntegrationTestBase +public sealed class EntityRequeueIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -88,18 +89,18 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (svc.Invocations.Count <= svc.TargetInvocationCount) { - requeue(entity, TimeSpan.FromMilliseconds(1)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1), CancellationToken.None); } - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs index 0e5239fc..60d4334b 100644 --- a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs @@ -9,9 +9,10 @@ using k8s.Models; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Events; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -20,7 +21,7 @@ namespace KubeOps.Operator.Test.Events; -public class EventPublisherIntegrationTest : IntegrationTestBase +public sealed class EventPublisherIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -90,20 +91,20 @@ private class TestController( EventPublisher eventPublisher) : IEntityController { - public async Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { await eventPublisher(entity, "REASON", "message", cancellationToken: cancellationToken); svc.Invocation(entity); if (svc.Invocations.Count < svc.TargetInvocationCount) { - requeue(entity, TimeSpan.FromMilliseconds(10)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(10), CancellationToken.None); } - } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) - { - return Task.CompletedTask; + return ReconciliationResult.Success(entity); } + + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs index 36f979f8..54574859 100644 --- a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs @@ -6,8 +6,9 @@ using k8s.Models; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -204,7 +205,12 @@ protected override void ConfigureHost(HostApplicationBuilder builder) { builder.Services .AddSingleton(_mock) - .AddKubernetesOperator(s => s.Namespace = _ns.Namespace) + .AddKubernetesOperator(s => + { + s.Namespace = _ns.Namespace; + s.AutoAttachFinalizers = false; + s.AutoDetachFinalizers = true; + }) .AddController() .AddFinalizer("first") .AddFinalizer("second"); @@ -215,42 +221,44 @@ private class TestController(InvocationCounter EntityFinalizerAttacher second) : IEntityController { - public async Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (entity.Name().Contains("first")) { - entity = await first(entity); + entity = await first(entity, cancellationToken); } if (entity.Name().Contains("second")) { - await second(entity); + entity = await second(entity, cancellationToken); } + + return ReconciliationResult.Success(entity); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } private class FirstFinalizer(InvocationCounter svc) : IEntityFinalizer { - public Task FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } private class SecondFinalizer(InvocationCounter svc) : IEntityFinalizer { - public Task FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs b/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs index a86ef250..baf0f5c2 100644 --- a/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs @@ -2,28 +2,30 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Builder; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Operator.Test.TestEntities; using Microsoft.Extensions.Hosting; namespace KubeOps.Operator.Test.HostedServices; -public class LeaderAwareHostedServiceDisposeIntegrationTest : HostedServiceDisposeIntegrationTest +public sealed class LeaderAwareHostedServiceDisposeIntegrationTest : HostedServiceDisposeIntegrationTest { protected override void ConfigureHost(HostApplicationBuilder builder) { builder.Services - .AddKubernetesOperator(op => op.EnableLeaderElection = true) + .AddKubernetesOperator(op => op.LeaderElectionType = LeaderElectionType.Single) .AddController(); } - private class TestController : IEntityController + private sealed class TestController : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs b/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs index 1a45c3ff..d4b58d00 100644 --- a/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs @@ -2,7 +2,8 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Operator.Test.TestEntities; using Microsoft.Extensions.DependencyInjection; @@ -49,10 +50,10 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Operator.Test/KubeOps.Operator.Test.csproj b/test/KubeOps.Operator.Test/KubeOps.Operator.Test.csproj index 4fa0d4a5..1c04d7d5 100644 --- a/test/KubeOps.Operator.Test/KubeOps.Operator.Test.csproj +++ b/test/KubeOps.Operator.Test/KubeOps.Operator.Test.csproj @@ -14,4 +14,4 @@ - + \ No newline at end of file diff --git a/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs b/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs index 3de6889d..ce541f6e 100644 --- a/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs @@ -6,7 +6,9 @@ using k8s.Models; -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Builder; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -49,22 +51,22 @@ protected override void ConfigureHost(HostApplicationBuilder builder) { builder.Services .AddSingleton(_mock) - .AddKubernetesOperator(s => s.EnableLeaderElection = true) + .AddKubernetesOperator(s => s.LeaderElectionType = LeaderElectionType.Single) .AddController(); } private class TestController(InvocationCounter svc) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs index 936fe057..1116a667 100644 --- a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs @@ -7,7 +7,8 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -86,16 +87,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Operator.Test/Queue/RequeueTypeExtensions.Test.cs b/test/KubeOps.Operator.Test/Queue/RequeueTypeExtensions.Test.cs new file mode 100644 index 00000000..d59a87cc --- /dev/null +++ b/test/KubeOps.Operator.Test/Queue/RequeueTypeExtensions.Test.cs @@ -0,0 +1,173 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s; + +using KubeOps.Abstractions.Reconciliation.Queue; +using KubeOps.Operator.Queue; + +namespace KubeOps.Operator.Test.Queue; + +public sealed class RequeueTypeExtensionsTest +{ + [Theory] + [InlineData(WatchEventType.Added, RequeueType.Added)] + [InlineData(WatchEventType.Modified, RequeueType.Modified)] + [InlineData(WatchEventType.Deleted, RequeueType.Deleted)] + public void ToRequeueType_Should_Convert_WatchEventType_Correctly( + WatchEventType watchEventType, + RequeueType expectedRequeueType) + { + var result = watchEventType.ToRequeueType(); + + result.Should().Be(expectedRequeueType); + } + + [Theory] + [InlineData(RequeueType.Added, WatchEventType.Added)] + [InlineData(RequeueType.Modified, WatchEventType.Modified)] + [InlineData(RequeueType.Deleted, WatchEventType.Deleted)] + public void ToWatchEventType_Should_Convert_RequeueType_Correctly( + RequeueType requeueType, + WatchEventType expectedWatchEventType) + { + var result = requeueType.ToWatchEventType(); + + result.Should().Be(expectedWatchEventType); + } + + [Fact] + public void ToRequeueType_Should_Throw_For_Unsupported_WatchEventType() + { + var unsupportedType = (WatchEventType)999; + + Action act = () => unsupportedType.ToRequeueType(); + + act.Should().Throw() + .WithMessage("*WatchEventType*999*not supported*"); + } + + [Fact] + public void ToWatchEventType_Should_Throw_For_Unsupported_RequeueType() + { + var unsupportedType = (RequeueType)999; + + Action act = () => unsupportedType.ToWatchEventType(); + + act.Should().Throw() + .WithMessage("*RequeueType*999*not supported*"); + } + + [Theory] + [InlineData(WatchEventType.Added)] + [InlineData(WatchEventType.Modified)] + [InlineData(WatchEventType.Deleted)] + public void Bidirectional_Conversion_Should_Be_Reversible_From_WatchEventType(WatchEventType original) + { + var requeueType = original.ToRequeueType(); + var converted = requeueType.ToWatchEventType(); + + converted.Should().Be(original); + } + + [Theory] + [InlineData(RequeueType.Added)] + [InlineData(RequeueType.Modified)] + [InlineData(RequeueType.Deleted)] + public void Bidirectional_Conversion_Should_Be_Reversible_From_RequeueType(RequeueType original) + { + var watchEventType = original.ToWatchEventType(); + var converted = watchEventType.ToRequeueType(); + + converted.Should().Be(original); + } + + [Fact] + public void ToRequeueType_Should_Handle_Added_EventType() + { + var eventType = WatchEventType.Added; + + var result = eventType.ToRequeueType(); + + result.Should().Be(RequeueType.Added); + } + + [Fact] + public void ToRequeueType_Should_Handle_Modified_EventType() + { + var eventType = WatchEventType.Modified; + + var result = eventType.ToRequeueType(); + + result.Should().Be(RequeueType.Modified); + } + + [Fact] + public void ToRequeueType_Should_Handle_Deleted_EventType() + { + var eventType = WatchEventType.Deleted; + + var result = eventType.ToRequeueType(); + + result.Should().Be(RequeueType.Deleted); + } + + [Fact] + public void ToWatchEventType_Should_Handle_Added_RequeueType() + { + var requeueType = RequeueType.Added; + + var result = requeueType.ToWatchEventType(); + + result.Should().Be(WatchEventType.Added); + } + + [Fact] + public void ToWatchEventType_Should_Handle_Modified_RequeueType() + { + var requeueType = RequeueType.Modified; + + var result = requeueType.ToWatchEventType(); + + result.Should().Be(WatchEventType.Modified); + } + + [Fact] + public void ToWatchEventType_Should_Handle_Deleted_RequeueType() + { + var requeueType = RequeueType.Deleted; + + var result = requeueType.ToWatchEventType(); + + result.Should().Be(WatchEventType.Deleted); + } + + [Fact] + public void All_RequeueTypes_Should_Have_Corresponding_WatchEventType() + { + var allRequeueTypes = Enum.GetValues(); + + // Act & Assert + foreach (var requeueType in allRequeueTypes) + { + Action act = () => requeueType.ToWatchEventType(); + act.Should().NotThrow($"RequeueType.{requeueType} should have a corresponding WatchEventType"); + } + } + + [Fact] + public void Supported_WatchEventTypes_Should_Have_Corresponding_RequeueType() + { + var supportedWatchEventTypes = new[] { WatchEventType.Added, WatchEventType.Modified, WatchEventType.Deleted }; + + // Act & Assert + foreach (var watchEventType in supportedWatchEventTypes) + { + Action act = () => watchEventType.ToRequeueType(); + act.Should().NotThrow($"WatchEventType.{watchEventType} should have a corresponding RequeueType"); + } + } +} diff --git a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs index 5d0a7c78..8f275244 100644 --- a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs +++ b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs @@ -4,19 +4,24 @@ using k8s.Models; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.Operator.Queue; +using Microsoft.Extensions.Logging; + +using Moq; + namespace KubeOps.Operator.Test.Queue; -public class TimedEntityQueueTest +public sealed class TimedEntityQueueTest { [Fact] public async Task Can_Enqueue_Multiple_Entities_With_Same_Name() { - var queue = new TimedEntityQueue(); + var queue = new TimedEntityQueue(Mock.Of>>()); - queue.Enqueue(CreateSecret("app-ns1", "secret-name"), TimeSpan.FromSeconds(1)); - queue.Enqueue(CreateSecret("app-ns2", "secret-name"), TimeSpan.FromSeconds(1)); + await queue.Enqueue(CreateSecret("app-ns1", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1), CancellationToken.None); + await queue.Enqueue(CreateSecret("app-ns2", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1), CancellationToken.None); var items = new List(); @@ -29,7 +34,7 @@ public async Task Can_Enqueue_Multiple_Entities_With_Same_Name() { while (await enumerator.MoveNextAsync()) { - items.Add(enumerator.Current); + items.Add(enumerator.Current.Entity); } } catch (OperationCanceledException) @@ -40,7 +45,7 @@ public async Task Can_Enqueue_Multiple_Entities_With_Same_Name() Assert.Equal(2, items.Count); } - private V1Secret CreateSecret(string secretNamespace, string secretName) + private static V1Secret CreateSecret(string secretNamespace, string secretName) { var secret = new V1Secret(); secret.EnsureMetadata(); diff --git a/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs b/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs new file mode 100644 index 00000000..ac4d88c6 --- /dev/null +++ b/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs @@ -0,0 +1,536 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Builder; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; +using KubeOps.Abstractions.Reconciliation.Queue; +using KubeOps.KubernetesClient; +using KubeOps.Operator.Queue; +using KubeOps.Operator.Reconciliation; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +using Moq; + +using ZiggyCreatures.Caching.Fusion; + +namespace KubeOps.Operator.Test.Reconciliation; + +public sealed class ReconcilerTest +{ + private readonly Mock>> _mockLogger; + private readonly Mock _mockCacheProvider; + private readonly Mock _mockCache; + private readonly Mock _mockServiceProvider; + private readonly Mock _mockClient; + private readonly Mock> _mockQueue; + private readonly OperatorSettings _settings; + + public ReconcilerTest() + { + _mockLogger = new(); + _mockCacheProvider = new(); + _mockCache = new(); + _mockServiceProvider = new(); + _mockClient = new(); + _mockQueue = new(); + _settings = new() { AutoAttachFinalizers = false, AutoDetachFinalizers = false }; + + _mockCacheProvider + .Setup(p => p.GetCache(It.IsAny())) + .Returns(_mockCache.Object); + } + + [Fact] + public async Task Reconcile_Should_Remove_Entity_From_Queue_Before_Processing() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var controller = CreateMockController(); + + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Remove(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Enqueue_Entity_When_Result_Has_RequeueAfter() + { + var entity = CreateTestEntity(); + var requeueAfter = TimeSpan.FromMinutes(5); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + var controller = CreateMockController( + reconcileResult: ReconciliationResult.Success(entity, requeueAfter)); + + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Enqueue( + entity, + RequeueType.Added, + requeueAfter, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Not_Enqueue_Entity_When_Result_Has_No_RequeueAfter() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + var controller = CreateMockController( + reconcileResult: ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Enqueue( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Reconcile_Should_Skip_On_Cached_Generation() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var mockController = new Mock>(); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + _mockCache + .Setup(c => c.TryGetAsync( + It.Is(s => s == entity.Uid()), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(MaybeValue.FromValue(entity.Generation())); + + var reconciler = CreateReconcilerForController(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockLogger.Verify(logger => logger.Log( + It.Is(logLevel => logLevel == LogLevel.Debug), + It.Is(eventId => eventId.Id == 0), + It.Is((@object, type) => @object.ToString() == $"""Entity "{entity.Kind}/{entity.Name()}" modification did not modify generation. Skip event.""" && type.Name == "FormattedLogValues"), + It.IsAny(), + It.IsAny>()!), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Call_ReconcileAsync_For_Added_Event() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var mockController = new Mock>(); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + mockController.Verify( + c => c.ReconcileAsync(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Call_ReconcileAsync_For_Modified_Event_With_No_Deletion_Timestamp() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + var mockController = new Mock>(); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + mockController.Verify( + c => c.ReconcileAsync(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Call_FinalizeAsync_For_Modified_Event_With_Deletion_Timestamp() + { + const string finalizerName = "test-finalizer"; + var entity = CreateTestEntityForFinalization(deletionTimestamp: DateTime.UtcNow, finalizer: finalizerName); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + + var mockFinalizer = new Mock>(); + + mockFinalizer + .Setup(c => c.FinalizeAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForFinalizer(mockFinalizer.Object, finalizerName); + + await reconciler.Reconcile(context, CancellationToken.None); + + mockFinalizer.Verify( + c => c.FinalizeAsync(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Call_DeletedAsync_For_Deleted_Event() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Deleted); + var mockController = new Mock>(); + + mockController + .Setup(c => c.DeletedAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + mockController.Verify( + c => c.DeletedAsync(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Remove_From_Cache_After_Successful_Deletion() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Deleted); + + var controller = CreateMockController( + deletedResult: ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockCache.Verify( + c => c.RemoveAsync(entity.Uid(), It.IsAny(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Not_Remove_From_Cache_After_Failed_Deletion() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Deleted); + + var controller = CreateMockController( + deletedResult: ReconciliationResult.Failure(entity, "Deletion failed")); + + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockCache.Verify( + c => c.RemoveAsync(It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Theory] + [InlineData(RequeueType.Added)] + [InlineData(RequeueType.Modified)] + [InlineData(RequeueType.Deleted)] + public async Task Reconcile_Should_Use_Correct_RequeueType_For_EventType(RequeueType expectedRequeueType) + { + var entity = CreateTestEntity(); + var requeueAfter = TimeSpan.FromSeconds(30); + var watchEventType = expectedRequeueType switch + { + RequeueType.Added => WatchEventType.Added, + RequeueType.Modified => WatchEventType.Modified, + RequeueType.Deleted => WatchEventType.Deleted, + _ => throw new ArgumentException("Invalid RequeueType"), + }; + + var context = ReconciliationContext.CreateFromApiServerEvent(entity, watchEventType); + var result = ReconciliationResult.Success(entity, requeueAfter); + + var controller = watchEventType == WatchEventType.Deleted + ? CreateMockController(deletedResult: result) + : CreateMockController(reconcileResult: result); + + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Enqueue( + entity, + expectedRequeueType, + requeueAfter, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Return_Result_From_Controller() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var expectedResult = ReconciliationResult.Success(entity, TimeSpan.FromMinutes(1)); + + var controller = CreateMockController(reconcileResult: expectedResult); + var reconciler = CreateReconcilerForController(controller); + + var result = await reconciler.Reconcile(context, CancellationToken.None); + + result.Should().Be(expectedResult); + result.Entity.Should().Be(entity); + result.RequeueAfter.Should().Be(TimeSpan.FromMinutes(1)); + } + + [Fact] + public async Task Reconcile_Should_Handle_Failure_Result() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + const string errorMessage = "Test error"; + var failureResult = ReconciliationResult.Failure(entity, errorMessage); + + var controller = CreateMockController(reconcileResult: failureResult); + var reconciler = CreateReconcilerForController(controller); + + var result = await reconciler.Reconcile(context, CancellationToken.None); + + result.IsFailure.Should().BeTrue(); + result.ErrorMessage.Should().Be(errorMessage); + } + + [Fact] + public async Task Reconcile_Should_Enqueue_Failed_Result_If_RequeueAfter_Set() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var requeueAfter = TimeSpan.FromSeconds(30); + var failureResult = ReconciliationResult.Failure( + entity, + "Temporary failure", + requeueAfter: requeueAfter); + + var controller = CreateMockController(reconcileResult: failureResult); + var reconciler = CreateReconcilerForController(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Enqueue( + entity, + RequeueType.Added, + requeueAfter, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_When_Auto_Attach_Finalizers_Is_Enabled_Should_Attach_Finalizer() + { + _settings.AutoAttachFinalizers = true; + + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + var mockController = new Mock>(); + var mockFinalizer = new Mock>(); + + _mockClient + .Setup(c => c.UpdateAsync(It.Is( + e => e == entity), + It.IsAny())) + .ReturnsAsync(entity); + + _mockServiceProvider + .Setup(p => p.GetRequiredKeyedService( + It.Is(t => t == typeof(IEnumerable>)), + It.Is(o => ReferenceEquals(o, KeyedService.AnyKey)))) + .Returns(new List> { mockFinalizer.Object }); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockClient.Verify( + c => c.UpdateAsync(It.Is(cm => cm.HasFinalizer("ientityfinalizer`1proxyfinalizer")), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_When_Auto_Detach_Finalizers_Is_Enabled_Should_Detach_Finalizer() + { + _settings.AutoDetachFinalizers = true; + + const string finalizerName = "test-finalizer"; + var entity = CreateTestEntityForFinalization(deletionTimestamp: DateTime.UtcNow, finalizer: finalizerName); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + + var mockFinalizer = new Mock>(); + + _mockClient + .Setup(c => c.UpdateAsync(It.Is( + e => e == entity), + It.IsAny())) + .ReturnsAsync(entity); + + mockFinalizer + .Setup(c => c.FinalizeAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForFinalizer(mockFinalizer.Object, finalizerName); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockClient.Verify( + c => c.UpdateAsync(It.Is(cm => !cm.HasFinalizer(finalizerName)), + It.IsAny()), + Times.Once); + } + + private Reconciler CreateReconcilerForController(IEntityController controller) + { + var mockScope = new Mock(); + var mockScopeFactory = new Mock(); + + mockScope + .Setup(s => s.ServiceProvider) + .Returns(_mockServiceProvider.Object); + + mockScopeFactory + .Setup(s => s.CreateScope()) + .Returns(mockScope.Object); + + _mockServiceProvider + .Setup(p => p.GetService(typeof(IServiceScopeFactory))) + .Returns(mockScopeFactory.Object); + + _mockServiceProvider + .Setup(p => p.GetService(typeof(IEntityController))) + .Returns(controller); + + return new( + _mockLogger.Object, + _mockCacheProvider.Object, + _mockServiceProvider.Object, + _settings, + _mockQueue.Object, + _mockClient.Object); + } + + private Reconciler CreateReconcilerForFinalizer(IEntityFinalizer? finalizer, string finalizerName) + { + var mockScope = new Mock(); + var mockScopeFactory = new Mock(); + + mockScope + .Setup(s => s.ServiceProvider) + .Returns(_mockServiceProvider.Object); + + mockScopeFactory + .Setup(s => s.CreateScope()) + .Returns(mockScope.Object); + + _mockServiceProvider + .Setup(p => p.GetService(typeof(IServiceScopeFactory))) + .Returns(mockScopeFactory.Object); + + _mockServiceProvider + .Setup(p => p.GetKeyedService( + It.Is(t => t == typeof(IEntityFinalizer)), + It.Is(s => s == finalizerName))) + .Returns(finalizer); + + return new( + _mockLogger.Object, + _mockCacheProvider.Object, + _mockServiceProvider.Object, + _settings, + _mockQueue.Object, + _mockClient.Object); + } + + private static IEntityController CreateMockController( + ReconciliationResult? reconcileResult = null, + ReconciliationResult? deletedResult = null) + { + var mockController = new Mock>(); + var entity = CreateTestEntity(); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(reconcileResult ?? ReconciliationResult.Success(entity)); + + mockController + .Setup(c => c.DeletedAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(deletedResult ?? ReconciliationResult.Success(entity)); + + return mockController.Object; + } + + private static V1ConfigMap CreateTestEntity(string? name = null) + { + return new() + { + Metadata = new() + { + Name = name ?? "test-configmap", + NamespaceProperty = "default", + Uid = Guid.NewGuid().ToString(), + Generation = 1, + }, + Kind = V1ConfigMap.KubeKind, + }; + } + + private static V1ConfigMap CreateTestEntityForFinalization(string? name = null, DateTime? deletionTimestamp = null, string? finalizer = null) + { + return new() + { + Metadata = new() + { + Name = name ?? "test-configmap", + NamespaceProperty = "default", + Uid = Guid.NewGuid().ToString(), + Generation = 1, + DeletionTimestamp = deletionTimestamp, + Finalizers = !string.IsNullOrEmpty(finalizer) ? new List { finalizer } : new(), + }, + Kind = V1ConfigMap.KubeKind, + }; + } +} diff --git a/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs b/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs index 590494e9..85724ffb 100644 --- a/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs +++ b/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs @@ -10,17 +10,14 @@ using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Entities; +using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; -using KubeOps.Operator.Constants; -using KubeOps.Operator.Queue; using KubeOps.Operator.Watcher; using Microsoft.Extensions.Logging; using Moq; -using ZiggyCreatures.Caching.Fusion; - namespace KubeOps.Operator.Test.Watcher; public sealed class ResourceWatcherTest @@ -31,30 +28,21 @@ public async Task Restarting_Watcher_Should_Trigger_New_Watch() // Arrange. var activitySource = new ActivitySource("unit-test"); var logger = Mock.Of>>(); - var serviceProvider = Mock.Of(); - var timedEntityQueue = new TimedEntityQueue(); + var reconciler = Mock.Of>(); var operatorSettings = new OperatorSettings { Namespace = "unit-test" }; var kubernetesClient = Mock.Of(); - var cache = Mock.Of(); - var cacheProvider = Mock.Of(); var labelSelector = new DefaultEntityLabelSelector(); Mock.Get(kubernetesClient) .Setup(client => client.WatchAsync("unit-test", null, null, true, It.IsAny())) .Returns((_, _, _, _, cancellationToken) => WaitForCancellationAsync<(WatchEventType, V1Pod)>(cancellationToken)); - Mock.Get(cacheProvider) - .Setup(cp => cp.GetCache(It.Is(s => s == CacheConstants.CacheNames.ResourceWatcher))) - .Returns(() => cache); - var resourceWatcher = new ResourceWatcher( activitySource, logger, - serviceProvider, - timedEntityQueue, + reconciler, operatorSettings, labelSelector, - cacheProvider, kubernetesClient); // Act. diff --git a/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Transpiler.Test/KubeOps.Transpiler.Test.csproj b/test/KubeOps.Transpiler.Test/KubeOps.Transpiler.Test.csproj index db5dde7f..46d28996 100644 --- a/test/KubeOps.Transpiler.Test/KubeOps.Transpiler.Test.csproj +++ b/test/KubeOps.Transpiler.Test/KubeOps.Transpiler.Test.csproj @@ -14,4 +14,4 @@ - + \ No newline at end of file diff --git a/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..791c4aff --- /dev/null +++ b/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage]