From 34ace2611766507378d17fcbd2634864ac4ca72d Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Wed, 4 Jun 2025 07:35:02 +0200 Subject: [PATCH 01/58] refactor: update controller and finalizer interfaces to return `Result` --- .../Controller/V1TestEntityController.cs | 8 +- .../Controller/V1TestEntityController.cs | 10 +- examples/Operator/Finalizer/FinalizerOne.cs | 11 +- .../Controller/V1TestEntityController.cs | 10 +- .../Controller/IEntityController{TEntity}.cs | 19 ++-- src/KubeOps.Abstractions/Controller/Result.cs | 43 ++++++++ .../Finalizer/IEntityFinalizer{TEntity}.cs | 8 +- .../KubeOps.Abstractions.csproj | 1 + .../Queue/EntityRequeue.cs | 2 +- .../Queue/EntityRequeueBackgroundService.cs | 20 +++- .../Watcher/ResourceWatcher{TEntity}.cs | 104 +++++++++++------- .../Builder/OperatorBuilder.Test.cs | 21 ++-- .../CancelEntityRequeue.Integration.Test.cs | 8 +- .../DeletedEntityRequeue.Integration.Test.cs | 8 +- .../EntityController.Integration.Test.cs | 8 +- .../EntityRequeue.Integration.Test.cs | 8 +- .../Events/EventPublisher.Integration.Test.cs | 10 +- .../EntityFinalizer.Integration.Test.cs | 16 +-- .../LeaderResourceWatcher.Integration.Test.cs | 12 +- .../ResourceWatcher.Integration.Test.cs | 8 +- .../LeaderAwareness.Integration.Test.cs | 8 +- .../NamespacedOperator.Integration.Test.cs | 8 +- 22 files changed, 215 insertions(+), 136 deletions(-) create mode 100644 src/KubeOps.Abstractions/Controller/Result.cs diff --git a/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs b/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs index 7746041a..fcb01b56 100644 --- a/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs +++ b/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs @@ -8,15 +8,15 @@ namespace ConversionWebhookOperator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] public class V1TestEntityController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Deleted entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/examples/Operator/Controller/V1TestEntityController.cs b/examples/Operator/Controller/V1TestEntityController.cs index 461d3a3c..6a9a6526 100644 --- a/examples/Operator/Controller/V1TestEntityController.cs +++ b/examples/Operator/Controller/V1TestEntityController.cs @@ -10,18 +10,18 @@ namespace Operator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -public class V1TestEntityController(ILogger logger) +public sealed class V1TestEntityController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Deleting entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/examples/Operator/Finalizer/FinalizerOne.cs b/examples/Operator/Finalizer/FinalizerOne.cs index 319bbabf..8c7eba23 100644 --- a/examples/Operator/Finalizer/FinalizerOne.cs +++ b/examples/Operator/Finalizer/FinalizerOne.cs @@ -1,13 +1,12 @@ -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Finalizer; using Operator.Entities; namespace Operator.Finalizer; -public class FinalizerOne : IEntityFinalizer +public sealed class FinalizerOne : IEntityFinalizer { - public Task FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) - { - return Task.CompletedTask; - } + public Task> FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } diff --git a/examples/WebhookOperator/Controller/V1TestEntityController.cs b/examples/WebhookOperator/Controller/V1TestEntityController.cs index c511b7a7..9ee202f2 100644 --- a/examples/WebhookOperator/Controller/V1TestEntityController.cs +++ b/examples/WebhookOperator/Controller/V1TestEntityController.cs @@ -6,17 +6,17 @@ namespace WebhookOperator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -public class V1TestEntityController(ILogger logger) : IEntityController +public sealed class V1TestEntityController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Deleted entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs b/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs index 87856087..cb4e5e4d 100644 --- a/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs +++ b/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs @@ -33,24 +33,23 @@ namespace KubeOps.Abstractions.Controller; /// } /// /// -public interface IEntityController +public interface IEntityController where TEntity : IKubernetesObject { /// - /// Called for `added` and `modified` events from the watcher. + /// Reconciles the state of the specified entity with the desired state. + /// This method is triggered for `added` and `modified` events from the watcher. /// - /// The entity that fired the reconcile event. - /// The token to monitor for cancellation requests. - /// A task that completes when the reconciliation is done. - Task ReconcileAsync(TEntity entity, CancellationToken cancellationToken); + /// The entity that initiated the reconcile operation. + /// The token used to signal cancellation of the operation. + /// A task that represents the asynchronous operation and contains the result of the reconcile process. + Task> ReconcileAsync(TEntity entity, CancellationToken cancellationToken); /// /// Called for `delete` events for a given entity. /// /// The entity that fired the deleted event. /// The token to monitor for cancellation requests. - /// - /// A task that completes, when the reconciliation is done. - /// - Task DeletedAsync(TEntity entity, CancellationToken cancellationToken); + /// A task that represents the asynchronous operation and contains the result of the reconcile process. + Task> DeletedAsync(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/Controller/Result.cs b/src/KubeOps.Abstractions/Controller/Result.cs new file mode 100644 index 00000000..dab05298 --- /dev/null +++ b/src/KubeOps.Abstractions/Controller/Result.cs @@ -0,0 +1,43 @@ +using System.Diagnostics.CodeAnalysis; + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Controller; + +public sealed record Result + where TEntity : IKubernetesObject +{ + private Result(TEntity entity, bool isSuccess, string? errorMessage, Exception? error, TimeSpan? requeueAfter) + { + Entity = entity; + IsSuccess = isSuccess; + ErrorMessage = errorMessage; + Error = error; + RequeueAfter = requeueAfter; + } + + public TEntity Entity { get; } + + [MemberNotNullWhen(false, nameof(ErrorMessage))] + public bool IsSuccess { get; } + + [MemberNotNullWhen(true, nameof(ErrorMessage))] + public bool IsFailure => !IsSuccess; + + public string? ErrorMessage { get; set; } + + public Exception? Error { get; } + + public TimeSpan? RequeueAfter { get; } + + public static Result ForSuccess(TEntity entity, TimeSpan? requeueAfter = null) + { + return new(entity, true, null, null, requeueAfter); + } + + public static Result ForFailure(TEntity entity, string errorMessage, Exception? error = null, TimeSpan? requeueAfter = null) + { + return new(entity, false, errorMessage, error, requeueAfter); + } +} diff --git a/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs b/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs index d0af39fc..bd598f9c 100644 --- a/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs +++ b/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs @@ -1,13 +1,15 @@ using k8s; using k8s.Models; +using KubeOps.Abstractions.Controller; + namespace KubeOps.Abstractions.Finalizer; /// /// Finalizer for an entity. /// /// The type of the entity. -public interface IEntityFinalizer +public interface IEntityFinalizer where TEntity : IKubernetesObject { /// @@ -15,6 +17,6 @@ public interface IEntityFinalizer /// /// The kubernetes entity that needs to be finalized. /// The token to monitor for cancellation requests. - /// A task that resolves when the operation is done. - Task FinalizeAsync(TEntity entity, CancellationToken cancellationToken); + /// A task that represents the asynchronous operation and contains the result of the reconcile process. + Task> FinalizeAsync(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj b/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj index f2fa3237..99f8c914 100644 --- a/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj +++ b/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj @@ -15,6 +15,7 @@ + diff --git a/src/KubeOps.Abstractions/Queue/EntityRequeue.cs b/src/KubeOps.Abstractions/Queue/EntityRequeue.cs index ce951440..016ffa41 100644 --- a/src/KubeOps.Abstractions/Queue/EntityRequeue.cs +++ b/src/KubeOps.Abstractions/Queue/EntityRequeue.cs @@ -43,5 +43,5 @@ namespace KubeOps.Abstractions.Queue; /// } /// /// -public delegate void EntityRequeue(TEntity entity, TimeSpan requeueIn) +public delegate void EntityRequeue(TEntity entity, TimeSpan requeueIn) where TEntity : IKubernetesObject; diff --git a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs index dcfff6c9..0b891a1e 100644 --- a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs +++ b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs @@ -2,6 +2,7 @@ using k8s.Models; using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Finalizer; using KubeOps.KubernetesClient; using Microsoft.Extensions.DependencyInjection; @@ -118,8 +119,21 @@ private async Task ReconcileSingleAsync(TEntity queued, CancellationToken cancel return; } - await using var scope = provider.CreateAsyncScope(); - var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.ReconcileAsync(entity, cancellationToken); + if (entity.DeletionTimestamp() is not null) + { + if (entity.Finalizers()?.Count > 0) + { + var identifier = entity.Finalizers()[0]; + await using var scope = provider.CreateAsyncScope(); + var finalizer = scope.ServiceProvider.GetRequiredKeyedService>(identifier); + await finalizer.FinalizeAsync(entity, cancellationToken); + } + } + else + { + await using var scope = provider.CreateAsyncScope(); + var controller = scope.ServiceProvider.GetRequiredService>(); + await controller.ReconcileAsync(entity, cancellationToken); + } } } diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index 65847a00..f9ff9043 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -127,7 +127,7 @@ static async ValueTask CastAndDispose(IDisposable resource) } } - protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) + protected virtual async Task> OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) { switch (type) { @@ -135,16 +135,14 @@ protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, C if (_entityCache.TryAdd(entity.Uid(), entity.Generation() ?? 0)) { // Only perform reconciliation if the entity was not already in the cache. - await ReconcileModificationAsync(entity, cancellationToken); - } - else - { - logger.LogDebug( - """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", - entity.Kind, - entity.Name()); + return await ReconcileModificationAsync(entity, cancellationToken); } + logger.LogDebug( + """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", + entity.Kind, + entity.Name()); + break; case WatchEventType.Modified: switch (entity) @@ -159,22 +157,20 @@ protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, C """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", entity.Kind, entity.Name()); - return; + return Result.ForSuccess(entity); } // update cached generation since generation now changed _entityCache.TryUpdate(entity.Uid(), entity.Generation() ?? 1, cachedGeneration); - await ReconcileModificationAsync(entity, cancellationToken); - break; + return await ReconcileModificationAsync(entity, cancellationToken); case { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } }: - await ReconcileFinalizersSequentialAsync(entity, cancellationToken); - break; + return await ReconcileFinalizersSequentialAsync(entity, cancellationToken); } break; case WatchEventType.Deleted: - await ReconcileDeletionAsync(entity, cancellationToken); - break; + return await ReconcileDeletionAsync(entity, cancellationToken); + default: logger.LogWarning( """Received unsupported event "{EventType}" for "{Kind}/{Name}".""", @@ -183,6 +179,8 @@ protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, C entity.Name()); break; } + + return Result.ForSuccess(entity); } private async Task WatchClientEventsAsync(CancellationToken stoppingToken) @@ -226,7 +224,24 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) try { - await OnEventAsync(type, entity, stoppingToken); + requeue.Remove(entity); + var result = await OnEventAsync(type, entity, stoppingToken); + + if (result.RequeueAfter.HasValue) + { + requeue.Enqueue(result.Entity, result.RequeueAfter.Value); + } + + if (result.IsFailure) + { + logger.LogError( + result.Error, + "Reconciliation of {EventType} for {Kind}/{Name} failed with message '{Message}'.", + type, + entity.Kind, + entity.Name(), + result.ErrorMessage); + } } catch (KubernetesException e) when (e.Status.Code is (int)HttpStatusCode.GatewayTimeout) { @@ -239,14 +254,9 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) throw; } catch (Exception e) - { - LogReconciliationFailed(e); - } - - void LogReconciliationFailed(Exception exception) { logger.LogError( - exception, + e, "Reconciliation of {EventType} for {Kind}/{Name} failed.", type, entity.Kind, @@ -314,26 +324,28 @@ e.InnerException is EndOfStreamException && await Task.Delay(delay); } - private async Task ReconcileDeletionAsync(TEntity entity, CancellationToken cancellationToken) + private async Task> ReconcileDeletionAsync(TEntity entity, CancellationToken cancellationToken) { - requeue.Remove(entity); - _entityCache.TryRemove(entity.Uid(), out _); - await using var scope = provider.CreateAsyncScope(); var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.DeletedAsync(entity, cancellationToken); + var result = await controller.DeletedAsync(entity, cancellationToken); + + if (result.IsSuccess) + { + _entityCache.TryRemove(result.Entity.Uid(), out _); + } + + return result; } - private async Task ReconcileFinalizersSequentialAsync(TEntity entity, CancellationToken cancellationToken) + private async Task> ReconcileFinalizersSequentialAsync(TEntity entity, CancellationToken cancellationToken) { - requeue.Remove(entity); await using var scope = provider.CreateAsyncScope(); - var identifier = entity.Finalizers().FirstOrDefault(); - if (identifier is null) - { - return; - } + // condition to call ReconcileFinalizersSequentialAsync is: + // { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } } + // which implies that there is at least a single finalizer + var identifier = entity.Finalizers()[0]; if (scope.ServiceProvider.GetKeyedService>(identifier) is not { } finalizer) @@ -343,25 +355,33 @@ private async Task ReconcileFinalizersSequentialAsync(TEntity entity, Cancellati entity.Kind, entity.Name(), identifier); - return; + return Result.ForSuccess(entity); } - await finalizer.FinalizeAsync(entity, cancellationToken); + var result = await finalizer.FinalizeAsync(entity, cancellationToken); + + if (!result.IsSuccess) + { + return result; + } + + entity = result.Entity; entity.RemoveFinalizer(identifier); - await client.UpdateAsync(entity, cancellationToken); + entity = await client.UpdateAsync(entity, cancellationToken); + logger.LogInformation( """Entity "{Kind}/{Name}" finalized with "{Finalizer}".""", entity.Kind, entity.Name(), identifier); + + return Result.ForSuccess(entity, result.RequeueAfter); } - private async Task ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) + private async Task> ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) { - // Re-queue should requested in the controller reconcile method. Invalidate any existing queues. - requeue.Remove(entity); await using var scope = provider.CreateAsyncScope(); var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.ReconcileAsync(entity, cancellationToken); + return await controller.ReconcileAsync(entity, cancellationToken); } } diff --git a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs index c8b514c0..de9f3723 100644 --- a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs +++ b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs @@ -8,7 +8,6 @@ using KubeOps.Abstractions.Queue; using KubeOps.KubernetesClient.LabelSelectors; using KubeOps.Operator.Builder; -using KubeOps.Operator.Finalizer; using KubeOps.Operator.Queue; using KubeOps.Operator.Test.TestEntities; using KubeOps.Operator.Watcher; @@ -19,7 +18,7 @@ namespace KubeOps.Operator.Test.Builder; -public class OperatorBuilderTest +public sealed class OperatorBuilderTest { private readonly IOperatorBuilder _builder = new OperatorBuilder(new ServiceCollection(), new()); @@ -142,22 +141,22 @@ public void Should_Add_LeaderAwareResourceWatcher() s.Lifetime == ServiceLifetime.Singleton); } - private class TestController : IEntityController + private sealed class TestController : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(Result.ForSuccess(entity)); - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(Result.ForSuccess(entity)); } - private class TestFinalizer : IEntityFinalizer + private sealed class TestFinalizer : IEntityFinalizer { - public Task FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(Result.ForSuccess(entity)); } - private class TestLabelSelector : IEntityLabelSelector + private sealed class TestLabelSelector : IEntityLabelSelector { public ValueTask GetLabelSelectorAsync(CancellationToken cancellationToken) { diff --git a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs index 75b75f5d..5beca5de 100644 --- a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs @@ -73,7 +73,7 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (svc.Invocations.Count < 2) @@ -81,12 +81,12 @@ public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationT requeue(entity, TimeSpan.FromMilliseconds(1000)); } - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs index e6635c2d..fe2546c0 100644 --- a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs @@ -55,17 +55,17 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); requeue(entity, TimeSpan.FromMilliseconds(1000)); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs index f753af12..79d05778 100644 --- a/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs @@ -116,16 +116,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs index 98bc69a7..a5e489ae 100644 --- a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs @@ -84,7 +84,7 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (svc.Invocations.Count <= svc.TargetInvocationCount) @@ -92,10 +92,10 @@ public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationT requeue(entity, TimeSpan.FromMilliseconds(1)); } - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs index 26bc1eb4..b81a98cf 100644 --- a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs @@ -86,7 +86,7 @@ private class TestController( EventPublisher eventPublisher) : IEntityController { - public async Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { await eventPublisher(entity, "REASON", "message", cancellationToken: cancellationToken); svc.Invocation(entity); @@ -95,11 +95,11 @@ public async Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, Cancell { requeue(entity, TimeSpan.FromMilliseconds(10)); } - } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) - { - return Task.CompletedTask; + return Result.ForSuccess(entity); } + + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs index bcf1a687..5f27e6ae 100644 --- a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs @@ -209,7 +209,7 @@ private class TestController(InvocationCounter EntityFinalizerAttacher second) : IEntityController { - public async Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (entity.Name().Contains("first")) @@ -221,30 +221,32 @@ public async Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, Cancell { await second(entity); } + + return Result.ForSuccess(entity); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } private class FirstFinalizer(InvocationCounter svc) : IEntityFinalizer { - public Task FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } private class SecondFinalizer(InvocationCounter svc) : IEntityFinalizer { - public Task FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } } diff --git a/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs b/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs index d4994428..ecf0bc26 100644 --- a/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs @@ -5,7 +5,7 @@ namespace KubeOps.Operator.Test.HostedServices; -public class LeaderAwareHostedServiceDisposeIntegrationTest : HostedServiceDisposeIntegrationTest +public sealed class LeaderAwareHostedServiceDisposeIntegrationTest : HostedServiceDisposeIntegrationTest { protected override void ConfigureHost(HostApplicationBuilder builder) { @@ -14,12 +14,12 @@ protected override void ConfigureHost(HostApplicationBuilder builder) .AddController(); } - private class TestController : IEntityController + private sealed class TestController : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs b/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs index a30a15a9..4eb85781 100644 --- a/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs @@ -45,10 +45,10 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs b/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs index 85c1fafa..d923d3b2 100644 --- a/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs @@ -51,16 +51,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } } diff --git a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs index 265c995e..7dd0a4df 100644 --- a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs @@ -77,16 +77,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } } From ac36c0489beabd56b2e98494c9db9e746361cf23 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Wed, 4 Jun 2025 07:35:02 +0200 Subject: [PATCH 02/58] refactor: update controller and finalizer interfaces to return `Result` --- .../Controller/V1TestEntityController.cs | 8 +- .../Controller/V1TestEntityController.cs | 10 +- examples/Operator/Finalizer/FinalizerOne.cs | 11 +- .../Controller/V1TestEntityController.cs | 10 +- .../Controller/IEntityController{TEntity}.cs | 19 ++-- src/KubeOps.Abstractions/Controller/Result.cs | 43 ++++++++ .../Finalizer/IEntityFinalizer{TEntity}.cs | 8 +- .../KubeOps.Abstractions.csproj | 3 +- .../Queue/EntityRequeue.cs | 2 +- .../Queue/EntityRequeueBackgroundService.cs | 20 +++- .../Watcher/ResourceWatcher{TEntity}.cs | 104 +++++++++++------- .../Builder/OperatorBuilder.Test.cs | 21 ++-- .../CancelEntityRequeue.Integration.Test.cs | 8 +- .../DeletedEntityRequeue.Integration.Test.cs | 8 +- .../EntityController.Integration.Test.cs | 8 +- .../EntityRequeue.Integration.Test.cs | 8 +- .../Events/EventPublisher.Integration.Test.cs | 10 +- .../EntityFinalizer.Integration.Test.cs | 16 +-- .../LeaderResourceWatcher.Integration.Test.cs | 12 +- .../ResourceWatcher.Integration.Test.cs | 8 +- .../LeaderAwareness.Integration.Test.cs | 8 +- .../NamespacedOperator.Integration.Test.cs | 8 +- 22 files changed, 216 insertions(+), 137 deletions(-) create mode 100644 src/KubeOps.Abstractions/Controller/Result.cs diff --git a/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs b/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs index 7746041a..fcb01b56 100644 --- a/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs +++ b/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs @@ -8,15 +8,15 @@ namespace ConversionWebhookOperator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] public class V1TestEntityController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Deleted entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/examples/Operator/Controller/V1TestEntityController.cs b/examples/Operator/Controller/V1TestEntityController.cs index 461d3a3c..6a9a6526 100644 --- a/examples/Operator/Controller/V1TestEntityController.cs +++ b/examples/Operator/Controller/V1TestEntityController.cs @@ -10,18 +10,18 @@ namespace Operator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -public class V1TestEntityController(ILogger logger) +public sealed class V1TestEntityController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Deleting entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/examples/Operator/Finalizer/FinalizerOne.cs b/examples/Operator/Finalizer/FinalizerOne.cs index 319bbabf..8c7eba23 100644 --- a/examples/Operator/Finalizer/FinalizerOne.cs +++ b/examples/Operator/Finalizer/FinalizerOne.cs @@ -1,13 +1,12 @@ -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Finalizer; using Operator.Entities; namespace Operator.Finalizer; -public class FinalizerOne : IEntityFinalizer +public sealed class FinalizerOne : IEntityFinalizer { - public Task FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) - { - return Task.CompletedTask; - } + public Task> FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } diff --git a/examples/WebhookOperator/Controller/V1TestEntityController.cs b/examples/WebhookOperator/Controller/V1TestEntityController.cs index c511b7a7..9ee202f2 100644 --- a/examples/WebhookOperator/Controller/V1TestEntityController.cs +++ b/examples/WebhookOperator/Controller/V1TestEntityController.cs @@ -6,17 +6,17 @@ namespace WebhookOperator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -public class V1TestEntityController(ILogger logger) : IEntityController +public sealed class V1TestEntityController(ILogger logger) : IEntityController { - public Task ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Deleted entity {Entity}.", entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs b/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs index 87856087..cb4e5e4d 100644 --- a/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs +++ b/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs @@ -33,24 +33,23 @@ namespace KubeOps.Abstractions.Controller; /// } /// /// -public interface IEntityController +public interface IEntityController where TEntity : IKubernetesObject { /// - /// Called for `added` and `modified` events from the watcher. + /// Reconciles the state of the specified entity with the desired state. + /// This method is triggered for `added` and `modified` events from the watcher. /// - /// The entity that fired the reconcile event. - /// The token to monitor for cancellation requests. - /// A task that completes when the reconciliation is done. - Task ReconcileAsync(TEntity entity, CancellationToken cancellationToken); + /// The entity that initiated the reconcile operation. + /// The token used to signal cancellation of the operation. + /// A task that represents the asynchronous operation and contains the result of the reconcile process. + Task> ReconcileAsync(TEntity entity, CancellationToken cancellationToken); /// /// Called for `delete` events for a given entity. /// /// The entity that fired the deleted event. /// The token to monitor for cancellation requests. - /// - /// A task that completes, when the reconciliation is done. - /// - Task DeletedAsync(TEntity entity, CancellationToken cancellationToken); + /// A task that represents the asynchronous operation and contains the result of the reconcile process. + Task> DeletedAsync(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/Controller/Result.cs b/src/KubeOps.Abstractions/Controller/Result.cs new file mode 100644 index 00000000..dab05298 --- /dev/null +++ b/src/KubeOps.Abstractions/Controller/Result.cs @@ -0,0 +1,43 @@ +using System.Diagnostics.CodeAnalysis; + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Controller; + +public sealed record Result + where TEntity : IKubernetesObject +{ + private Result(TEntity entity, bool isSuccess, string? errorMessage, Exception? error, TimeSpan? requeueAfter) + { + Entity = entity; + IsSuccess = isSuccess; + ErrorMessage = errorMessage; + Error = error; + RequeueAfter = requeueAfter; + } + + public TEntity Entity { get; } + + [MemberNotNullWhen(false, nameof(ErrorMessage))] + public bool IsSuccess { get; } + + [MemberNotNullWhen(true, nameof(ErrorMessage))] + public bool IsFailure => !IsSuccess; + + public string? ErrorMessage { get; set; } + + public Exception? Error { get; } + + public TimeSpan? RequeueAfter { get; } + + public static Result ForSuccess(TEntity entity, TimeSpan? requeueAfter = null) + { + return new(entity, true, null, null, requeueAfter); + } + + public static Result ForFailure(TEntity entity, string errorMessage, Exception? error = null, TimeSpan? requeueAfter = null) + { + return new(entity, false, errorMessage, error, requeueAfter); + } +} diff --git a/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs b/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs index d0af39fc..bd598f9c 100644 --- a/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs +++ b/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs @@ -1,13 +1,15 @@ using k8s; using k8s.Models; +using KubeOps.Abstractions.Controller; + namespace KubeOps.Abstractions.Finalizer; /// /// Finalizer for an entity. /// /// The type of the entity. -public interface IEntityFinalizer +public interface IEntityFinalizer where TEntity : IKubernetesObject { /// @@ -15,6 +17,6 @@ public interface IEntityFinalizer /// /// The kubernetes entity that needs to be finalized. /// The token to monitor for cancellation requests. - /// A task that resolves when the operation is done. - Task FinalizeAsync(TEntity entity, CancellationToken cancellationToken); + /// A task that represents the asynchronous operation and contains the result of the reconcile process. + Task> FinalizeAsync(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj b/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj index c88647c5..8ec18eac 100644 --- a/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj +++ b/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj @@ -15,7 +15,8 @@ + - + \ No newline at end of file diff --git a/src/KubeOps.Abstractions/Queue/EntityRequeue.cs b/src/KubeOps.Abstractions/Queue/EntityRequeue.cs index ce951440..016ffa41 100644 --- a/src/KubeOps.Abstractions/Queue/EntityRequeue.cs +++ b/src/KubeOps.Abstractions/Queue/EntityRequeue.cs @@ -43,5 +43,5 @@ namespace KubeOps.Abstractions.Queue; /// } /// /// -public delegate void EntityRequeue(TEntity entity, TimeSpan requeueIn) +public delegate void EntityRequeue(TEntity entity, TimeSpan requeueIn) where TEntity : IKubernetesObject; diff --git a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs index dcfff6c9..0b891a1e 100644 --- a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs +++ b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs @@ -2,6 +2,7 @@ using k8s.Models; using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Finalizer; using KubeOps.KubernetesClient; using Microsoft.Extensions.DependencyInjection; @@ -118,8 +119,21 @@ private async Task ReconcileSingleAsync(TEntity queued, CancellationToken cancel return; } - await using var scope = provider.CreateAsyncScope(); - var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.ReconcileAsync(entity, cancellationToken); + if (entity.DeletionTimestamp() is not null) + { + if (entity.Finalizers()?.Count > 0) + { + var identifier = entity.Finalizers()[0]; + await using var scope = provider.CreateAsyncScope(); + var finalizer = scope.ServiceProvider.GetRequiredKeyedService>(identifier); + await finalizer.FinalizeAsync(entity, cancellationToken); + } + } + else + { + await using var scope = provider.CreateAsyncScope(); + var controller = scope.ServiceProvider.GetRequiredService>(); + await controller.ReconcileAsync(entity, cancellationToken); + } } } diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index 8d74ba6b..b0c7bb26 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -130,7 +130,7 @@ static async ValueTask CastAndDispose(IDisposable resource) } } - protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) + protected virtual async Task> OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) { switch (type) { @@ -138,16 +138,14 @@ protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, C if (_entityCache.TryAdd(entity.Uid(), entity.Generation() ?? 0)) { // Only perform reconciliation if the entity was not already in the cache. - await ReconcileModificationAsync(entity, cancellationToken); - } - else - { - logger.LogDebug( - """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", - entity.Kind, - entity.Name()); + return await ReconcileModificationAsync(entity, cancellationToken); } + logger.LogDebug( + """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", + entity.Kind, + entity.Name()); + break; case WatchEventType.Modified: switch (entity) @@ -162,22 +160,20 @@ protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, C """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", entity.Kind, entity.Name()); - return; + return Result.ForSuccess(entity); } // update cached generation since generation now changed _entityCache.TryUpdate(entity.Uid(), entity.Generation() ?? 1, cachedGeneration); - await ReconcileModificationAsync(entity, cancellationToken); - break; + return await ReconcileModificationAsync(entity, cancellationToken); case { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } }: - await ReconcileFinalizersSequentialAsync(entity, cancellationToken); - break; + return await ReconcileFinalizersSequentialAsync(entity, cancellationToken); } break; case WatchEventType.Deleted: - await ReconcileDeletionAsync(entity, cancellationToken); - break; + return await ReconcileDeletionAsync(entity, cancellationToken); + default: logger.LogWarning( """Received unsupported event "{EventType}" for "{Kind}/{Name}".""", @@ -186,6 +182,8 @@ protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, C entity.Name()); break; } + + return Result.ForSuccess(entity); } private async Task WatchClientEventsAsync(CancellationToken stoppingToken) @@ -220,7 +218,24 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) try { - await OnEventAsync(type, entity, stoppingToken); + requeue.Remove(entity); + var result = await OnEventAsync(type, entity, stoppingToken); + + if (result.RequeueAfter.HasValue) + { + requeue.Enqueue(result.Entity, result.RequeueAfter.Value); + } + + if (result.IsFailure) + { + logger.LogError( + result.Error, + "Reconciliation of {EventType} for {Kind}/{Name} failed with message '{Message}'.", + type, + entity.Kind, + entity.Name(), + result.ErrorMessage); + } } catch (KubernetesException e) when (e.Status.Code is (int)HttpStatusCode.GatewayTimeout) { @@ -233,14 +248,9 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) throw; } catch (Exception e) - { - LogReconciliationFailed(e); - } - - void LogReconciliationFailed(Exception exception) { logger.LogError( - exception, + e, "Reconciliation of {EventType} for {Kind}/{Name} failed.", type, entity.Kind, @@ -308,26 +318,28 @@ e.InnerException is EndOfStreamException && await Task.Delay(delay); } - private async Task ReconcileDeletionAsync(TEntity entity, CancellationToken cancellationToken) + private async Task> ReconcileDeletionAsync(TEntity entity, CancellationToken cancellationToken) { - requeue.Remove(entity); - _entityCache.TryRemove(entity.Uid(), out _); - await using var scope = provider.CreateAsyncScope(); var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.DeletedAsync(entity, cancellationToken); + var result = await controller.DeletedAsync(entity, cancellationToken); + + if (result.IsSuccess) + { + _entityCache.TryRemove(result.Entity.Uid(), out _); + } + + return result; } - private async Task ReconcileFinalizersSequentialAsync(TEntity entity, CancellationToken cancellationToken) + private async Task> ReconcileFinalizersSequentialAsync(TEntity entity, CancellationToken cancellationToken) { - requeue.Remove(entity); await using var scope = provider.CreateAsyncScope(); - var identifier = entity.Finalizers().FirstOrDefault(); - if (identifier is null) - { - return; - } + // condition to call ReconcileFinalizersSequentialAsync is: + // { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } } + // which implies that there is at least a single finalizer + var identifier = entity.Finalizers()[0]; if (scope.ServiceProvider.GetKeyedService>(identifier) is not { } finalizer) @@ -337,25 +349,33 @@ private async Task ReconcileFinalizersSequentialAsync(TEntity entity, Cancellati entity.Kind, entity.Name(), identifier); - return; + return Result.ForSuccess(entity); } - await finalizer.FinalizeAsync(entity, cancellationToken); + var result = await finalizer.FinalizeAsync(entity, cancellationToken); + + if (!result.IsSuccess) + { + return result; + } + + entity = result.Entity; entity.RemoveFinalizer(identifier); - await client.UpdateAsync(entity, cancellationToken); + entity = await client.UpdateAsync(entity, cancellationToken); + logger.LogInformation( """Entity "{Kind}/{Name}" finalized with "{Finalizer}".""", entity.Kind, entity.Name(), identifier); + + return Result.ForSuccess(entity, result.RequeueAfter); } - private async Task ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) + private async Task> ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) { - // Re-queue should requested in the controller reconcile method. Invalidate any existing queues. - requeue.Remove(entity); await using var scope = provider.CreateAsyncScope(); var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.ReconcileAsync(entity, cancellationToken); + return await controller.ReconcileAsync(entity, cancellationToken); } } diff --git a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs index c8b514c0..de9f3723 100644 --- a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs +++ b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs @@ -8,7 +8,6 @@ using KubeOps.Abstractions.Queue; using KubeOps.KubernetesClient.LabelSelectors; using KubeOps.Operator.Builder; -using KubeOps.Operator.Finalizer; using KubeOps.Operator.Queue; using KubeOps.Operator.Test.TestEntities; using KubeOps.Operator.Watcher; @@ -19,7 +18,7 @@ namespace KubeOps.Operator.Test.Builder; -public class OperatorBuilderTest +public sealed class OperatorBuilderTest { private readonly IOperatorBuilder _builder = new OperatorBuilder(new ServiceCollection(), new()); @@ -142,22 +141,22 @@ public void Should_Add_LeaderAwareResourceWatcher() s.Lifetime == ServiceLifetime.Singleton); } - private class TestController : IEntityController + private sealed class TestController : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(Result.ForSuccess(entity)); - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(Result.ForSuccess(entity)); } - private class TestFinalizer : IEntityFinalizer + private sealed class TestFinalizer : IEntityFinalizer { - public Task FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(Result.ForSuccess(entity)); } - private class TestLabelSelector : IEntityLabelSelector + private sealed class TestLabelSelector : IEntityLabelSelector { public ValueTask GetLabelSelectorAsync(CancellationToken cancellationToken) { diff --git a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs index 75b75f5d..5beca5de 100644 --- a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs @@ -73,7 +73,7 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (svc.Invocations.Count < 2) @@ -81,12 +81,12 @@ public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationT requeue(entity, TimeSpan.FromMilliseconds(1000)); } - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs index e6635c2d..fe2546c0 100644 --- a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs @@ -55,17 +55,17 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); requeue(entity, TimeSpan.FromMilliseconds(1000)); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs index f753af12..79d05778 100644 --- a/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs @@ -116,16 +116,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs index 98bc69a7..a5e489ae 100644 --- a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs @@ -84,7 +84,7 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (svc.Invocations.Count <= svc.TargetInvocationCount) @@ -92,10 +92,10 @@ public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationT requeue(entity, TimeSpan.FromMilliseconds(1)); } - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs index 26bc1eb4..b81a98cf 100644 --- a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs @@ -86,7 +86,7 @@ private class TestController( EventPublisher eventPublisher) : IEntityController { - public async Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { await eventPublisher(entity, "REASON", "message", cancellationToken: cancellationToken); svc.Invocation(entity); @@ -95,11 +95,11 @@ public async Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, Cancell { requeue(entity, TimeSpan.FromMilliseconds(10)); } - } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) - { - return Task.CompletedTask; + return Result.ForSuccess(entity); } + + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs index bcf1a687..5f27e6ae 100644 --- a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs @@ -209,7 +209,7 @@ private class TestController(InvocationCounter EntityFinalizerAttacher second) : IEntityController { - public async Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (entity.Name().Contains("first")) @@ -221,30 +221,32 @@ public async Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, Cancell { await second(entity); } + + return Result.ForSuccess(entity); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } private class FirstFinalizer(InvocationCounter svc) : IEntityFinalizer { - public Task FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } private class SecondFinalizer(InvocationCounter svc) : IEntityFinalizer { - public Task FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } } diff --git a/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs b/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs index d4994428..ecf0bc26 100644 --- a/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs @@ -5,7 +5,7 @@ namespace KubeOps.Operator.Test.HostedServices; -public class LeaderAwareHostedServiceDisposeIntegrationTest : HostedServiceDisposeIntegrationTest +public sealed class LeaderAwareHostedServiceDisposeIntegrationTest : HostedServiceDisposeIntegrationTest { protected override void ConfigureHost(HostApplicationBuilder builder) { @@ -14,12 +14,12 @@ protected override void ConfigureHost(HostApplicationBuilder builder) .AddController(); } - private class TestController : IEntityController + private sealed class TestController : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs b/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs index a30a15a9..4eb85781 100644 --- a/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs @@ -45,10 +45,10 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.CompletedTask; + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } } diff --git a/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs b/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs index 85c1fafa..d923d3b2 100644 --- a/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs @@ -51,16 +51,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } } diff --git a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs index 265c995e..7dd0a4df 100644 --- a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs @@ -77,16 +77,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } - public Task DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(Result.ForSuccess(entity)); } } } From d31a132f8efa4e3ded8983c64e75be847d9011aa Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 27 Jun 2025 10:22:38 +0200 Subject: [PATCH 03/58] refactor: mark `OperatorBuilderGenerator` as `sealed` and use constant for builder identifier --- .../Generators/OperatorBuilderGenerator.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/KubeOps.Generator/Generators/OperatorBuilderGenerator.cs b/src/KubeOps.Generator/Generators/OperatorBuilderGenerator.cs index 4dbb044d..b7f9f01d 100644 --- a/src/KubeOps.Generator/Generators/OperatorBuilderGenerator.cs +++ b/src/KubeOps.Generator/Generators/OperatorBuilderGenerator.cs @@ -10,8 +10,10 @@ namespace KubeOps.Generator.Generators; [Generator] -internal class OperatorBuilderGenerator : ISourceGenerator +internal sealed class OperatorBuilderGenerator : ISourceGenerator { + private const string BuilderIdentifier = "builder"; + public void Initialize(GeneratorInitializationContext context) { } @@ -32,7 +34,7 @@ public void Execute(GeneratorExecutionContext context) .WithParameterList(ParameterList( SingletonSeparatedList( Parameter( - Identifier("builder")) + Identifier(BuilderIdentifier)) .WithModifiers( TokenList( Token(SyntaxKind.ThisKeyword))) @@ -43,15 +45,15 @@ public void Execute(GeneratorExecutionContext context) InvocationExpression( MemberAccessExpression( SyntaxKind.SimpleMemberAccessExpression, - IdentifierName("builder"), + IdentifierName(BuilderIdentifier), IdentifierName("RegisterControllers")))), ExpressionStatement( InvocationExpression( MemberAccessExpression( SyntaxKind.SimpleMemberAccessExpression, - IdentifierName("builder"), + IdentifierName(BuilderIdentifier), IdentifierName("RegisterFinalizers")))), - ReturnStatement(IdentifierName("builder"))))))) + ReturnStatement(IdentifierName(BuilderIdentifier))))))) .NormalizeWhitespace(); context.AddSource( From 5e00fa3fd04d6b26526dba6b95ecab6c09f18a79 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 27 Jun 2025 13:56:28 +0200 Subject: [PATCH 04/58] feat: add FusionCache for resource watcher to enable L1 and L2 caching (hybrid cache) - Integrated FusionCache for robust caching in resource watchers. - Enhanced default configuration with extensible settings in `OperatorSettings`. - Improved concurrency handling using `SemaphoreSlim` for entity events. - Updated tests and dependencies to reflect caching changes. --- .../Builder/OperatorSettings.cs | 16 +++- .../KubeOps.Abstractions.csproj | 1 + .../Builder/OperatorBuilder.cs | 15 ++++ .../Constants/CacheConstants.cs | 19 +++++ src/KubeOps.Operator/KubeOps.Operator.csproj | 1 + .../LeaderAwareResourceWatcher{TEntity}.cs | 4 + .../Watcher/ResourceWatcher{TEntity}.cs | 73 +++++++++++++------ .../Watcher/ResourceWatcher{TEntity}.Test.cs | 19 ++++- 8 files changed, 124 insertions(+), 24 deletions(-) create mode 100644 src/KubeOps.Operator/Constants/CacheConstants.cs diff --git a/src/KubeOps.Abstractions/Builder/OperatorSettings.cs b/src/KubeOps.Abstractions/Builder/OperatorSettings.cs index 2faed930..d7609d6a 100644 --- a/src/KubeOps.Abstractions/Builder/OperatorSettings.cs +++ b/src/KubeOps.Abstractions/Builder/OperatorSettings.cs @@ -1,11 +1,13 @@ using System.Text.RegularExpressions; +using ZiggyCreatures.Caching.Fusion; + namespace KubeOps.Abstractions.Builder; /// /// Operator settings. /// -public sealed class OperatorSettings +public sealed partial class OperatorSettings { private const string DefaultOperatorName = "KubernetesOperator"; private const string NonCharReplacement = "-"; @@ -15,7 +17,7 @@ public sealed class OperatorSettings /// Defaults to "kubernetesoperator" when not set. /// public string Name { get; set; } = - new Regex(@"(\W|_)", RegexOptions.CultureInvariant).Replace( + OperatorNameRegex().Replace( DefaultOperatorName, NonCharReplacement) .ToLowerInvariant(); @@ -59,4 +61,14 @@ public sealed class OperatorSettings /// The wait timeout if the lease cannot be acquired. /// public TimeSpan LeaderElectionRetryPeriod { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// Allows configuration of the FusionCache settings for resource watcher entity caching. + /// This property is optional and can be used to customize caching behavior for resource watcher entities. + /// If not set, a default cache configuration is applied. + /// + public Action? ConfigureResourceWatcherEntityCache { get; set; } + + [GeneratedRegex(@"(\W|_)", RegexOptions.CultureInvariant)] + private static partial Regex OperatorNameRegex(); } diff --git a/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj b/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj index c88647c5..93e54942 100644 --- a/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj +++ b/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj @@ -16,6 +16,7 @@ + diff --git a/src/KubeOps.Operator/Builder/OperatorBuilder.cs b/src/KubeOps.Operator/Builder/OperatorBuilder.cs index f1dc4381..69349836 100644 --- a/src/KubeOps.Operator/Builder/OperatorBuilder.cs +++ b/src/KubeOps.Operator/Builder/OperatorBuilder.cs @@ -21,6 +21,8 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using ZiggyCreatures.Caching.Fusion; + namespace KubeOps.Operator.Builder; internal sealed class OperatorBuilder : IOperatorBuilder @@ -36,6 +38,13 @@ public OperatorBuilder(IServiceCollection services, OperatorSettings settings) public IServiceCollection Services { get; } + private static Action DefaultCacheConfiguration + => options => + { + options.DefaultEntryOptions + .SetDuration(Timeout.InfiniteTimeSpan); + }; + public IOperatorBuilder AddController() where TImplementation : class, IEntityController where TEntity : IKubernetesObject @@ -111,6 +120,12 @@ private void AddOperatorBase() Services.AddSingleton(_settings); Services.AddSingleton(new ActivitySource(_settings.Name)); + // add and configure resource watcher entity cache + Services + .AddFusionCache(CacheConstants.CacheNames.ResourceWatcher) + .WithOptions( + options => (_settings.ConfigureResourceWatcherEntityCache ?? DefaultCacheConfiguration).Invoke(options)); + // Add the default configuration and the client separately. This allows external users to override either // just the config (e.g. for integration tests) or to replace the whole client, e.g. with a mock. // We also add the k8s.IKubernetes as a singleton service, in order to allow to access internal services diff --git a/src/KubeOps.Operator/Constants/CacheConstants.cs b/src/KubeOps.Operator/Constants/CacheConstants.cs new file mode 100644 index 00000000..1b6faf1e --- /dev/null +++ b/src/KubeOps.Operator/Constants/CacheConstants.cs @@ -0,0 +1,19 @@ +namespace KubeOps.Operator.Constants; + +/// +/// Provides constant values used for caching purposes within the operator. +/// +public static class CacheConstants +{ + /// + /// Contains constant values representing names used within the operator's caching mechanisms. + /// + public static class CacheNames + { + /// + /// Represents a constant string used as a name for the resource watcher + /// in the operator's caching mechanisms. + /// + public const string ResourceWatcher = "ResourceWatcher"; + } +} diff --git a/src/KubeOps.Operator/KubeOps.Operator.csproj b/src/KubeOps.Operator/KubeOps.Operator.csproj index 5361c6e6..a120b9ba 100644 --- a/src/KubeOps.Operator/KubeOps.Operator.csproj +++ b/src/KubeOps.Operator/KubeOps.Operator.csproj @@ -17,6 +17,7 @@ + diff --git a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs index add231f7..8fce8c01 100644 --- a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs @@ -12,6 +12,8 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using ZiggyCreatures.Caching.Fusion; + namespace KubeOps.Operator.Watcher; internal sealed class LeaderAwareResourceWatcher( @@ -21,6 +23,7 @@ internal sealed class LeaderAwareResourceWatcher( TimedEntityQueue queue, OperatorSettings settings, IEntityLabelSelector labelSelector, + IFusionCacheProvider cacheProvider, IKubernetesClient client, IHostApplicationLifetime hostApplicationLifetime, LeaderElector elector) @@ -31,6 +34,7 @@ internal sealed class LeaderAwareResourceWatcher( queue, settings, labelSelector, + cacheProvider, client) where TEntity : IKubernetesObject { diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index 8d74ba6b..d22c3fc5 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -12,6 +12,7 @@ using KubeOps.Abstractions.Entities; using KubeOps.Abstractions.Finalizer; using KubeOps.KubernetesClient; +using KubeOps.Operator.Constants; using KubeOps.Operator.Logging; using KubeOps.Operator.Queue; @@ -19,6 +20,8 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using ZiggyCreatures.Caching.Fusion; + namespace KubeOps.Operator.Watcher; public class ResourceWatcher( @@ -28,12 +31,14 @@ public class ResourceWatcher( TimedEntityQueue requeue, OperatorSettings settings, IEntityLabelSelector labelSelector, + IFusionCacheProvider cacheProvider, IKubernetesClient client) : IHostedService, IAsyncDisposable, IDisposable where TEntity : IKubernetesObject { - private readonly ConcurrentDictionary _entityCache = new(); + private readonly ConcurrentDictionary _entityLocks = new(); + private readonly IFusionCache _entityCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); private CancellationTokenSource _cancellationTokenSource = new(); private uint _watcherReconnectRetries; private Task? _eventWatcher; @@ -132,20 +137,35 @@ static async ValueTask CastAndDispose(IDisposable resource) protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) { + SemaphoreSlim? semaphore; + switch (type) { case WatchEventType.Added: - if (_entityCache.TryAdd(entity.Uid(), entity.Generation() ?? 0)) + semaphore = _entityLocks.GetOrAdd(entity.Uid(), _ => new(1, 1)); + await semaphore.WaitAsync(cancellationToken); + + try { - // Only perform reconciliation if the entity was not already in the cache. - await ReconcileModificationAsync(entity, cancellationToken); + var cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); + + if (!cachedGeneration.HasValue) + { + // Only perform reconciliation if the entity was not already in the cache. + await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 0, token: cancellationToken); + await ReconcileModificationAsync(entity, cancellationToken); + } + else + { + logger.LogDebug( + """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", + entity.Kind, + entity.Name()); + } } - else + finally { - logger.LogDebug( - """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", - entity.Kind, - entity.Name()); + semaphore.Release(); } break; @@ -153,21 +173,32 @@ protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, C switch (entity) { case { Metadata.DeletionTimestamp: null }: - _entityCache.TryGetValue(entity.Uid(), out var cachedGeneration); + semaphore = _entityLocks.GetOrAdd(entity.Uid(), _ => new(1, 1)); + await semaphore.WaitAsync(cancellationToken); - // Check if entity spec has changed through "Generation" value increment. Skip reconcile if not changed. - if (entity.Generation() <= cachedGeneration) + try { - logger.LogDebug( - """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", - entity.Kind, - entity.Name()); - return; + var cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); + + // Check if entity spec has changed through "Generation" value increment. Skip reconcile if not changed. + if (cachedGeneration.HasValue && cachedGeneration >= entity.Generation()) + { + logger.LogDebug( + """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", + entity.Kind, + entity.Name()); + return; + } + + // update cached generation since generation now changed + await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 1, token: cancellationToken); + await ReconcileModificationAsync(entity, cancellationToken); + } + finally + { + semaphore.Release(); } - // update cached generation since generation now changed - _entityCache.TryUpdate(entity.Uid(), entity.Generation() ?? 1, cachedGeneration); - await ReconcileModificationAsync(entity, cancellationToken); break; case { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } }: await ReconcileFinalizersSequentialAsync(entity, cancellationToken); @@ -311,7 +342,7 @@ e.InnerException is EndOfStreamException && private async Task ReconcileDeletionAsync(TEntity entity, CancellationToken cancellationToken) { requeue.Remove(entity); - _entityCache.TryRemove(entity.Uid(), out _); + await _entityCache.RemoveAsync(entity.Uid(), token: cancellationToken); await using var scope = provider.CreateAsyncScope(); var controller = scope.ServiceProvider.GetRequiredService>(); diff --git a/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs b/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs index 9b23e331..10a8f65c 100644 --- a/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs +++ b/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs @@ -7,6 +7,7 @@ using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Entities; using KubeOps.KubernetesClient; +using KubeOps.Operator.Constants; using KubeOps.Operator.Queue; using KubeOps.Operator.Watcher; @@ -14,6 +15,8 @@ using Moq; +using ZiggyCreatures.Caching.Fusion; + namespace KubeOps.Operator.Test.Watcher; public sealed class ResourceWatcherTest @@ -28,13 +31,27 @@ public async Task Restarting_Watcher_Should_Trigger_New_Watch() var timedEntityQueue = new TimedEntityQueue(); var operatorSettings = new OperatorSettings { Namespace = "unit-test" }; var kubernetesClient = Mock.Of(); + var cache = Mock.Of(); + var cacheProvider = Mock.Of(); var labelSelector = new DefaultEntityLabelSelector(); Mock.Get(kubernetesClient) .Setup(client => client.WatchAsync("unit-test", null, null, true, It.IsAny())) .Returns((_, _, _, _, cancellationToken) => WaitForCancellationAsync<(WatchEventType, V1Pod)>(cancellationToken)); - var resourceWatcher = new ResourceWatcher(activitySource, logger, serviceProvider, timedEntityQueue, operatorSettings, labelSelector, kubernetesClient); + Mock.Get(cacheProvider) + .Setup(cp => cp.GetCache(It.Is(s => s == CacheConstants.CacheNames.ResourceWatcher))) + .Returns(() => cache); + + var resourceWatcher = new ResourceWatcher( + activitySource, + logger, + serviceProvider, + timedEntityQueue, + operatorSettings, + labelSelector, + cacheProvider, + kubernetesClient); // Act. // Start and stop the watcher. From 6d8def683f5511e997f1e4bd691d57e35d6bdad7 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Mon, 30 Jun 2025 12:33:29 +0200 Subject: [PATCH 05/58] refactor: optimize resource watcher cache handling and remove redundant entity locks - Renamed `DefaultCacheConfiguration` to `DefaultResourceWatcherCacheConfiguration` for clarity. - Introduced cache key prefix to improve cache segmentation. - Removed `ConcurrentDictionary` for entity locks to simplify concurrency management. - Refactored event handling logic for "added" and "modified" events to streamline codebase. --- .../Builder/OperatorBuilder.cs | 7 +- .../Watcher/ResourceWatcher{TEntity}.cs | 69 +++++++------------ 2 files changed, 27 insertions(+), 49 deletions(-) diff --git a/src/KubeOps.Operator/Builder/OperatorBuilder.cs b/src/KubeOps.Operator/Builder/OperatorBuilder.cs index 69349836..0c59d4e3 100644 --- a/src/KubeOps.Operator/Builder/OperatorBuilder.cs +++ b/src/KubeOps.Operator/Builder/OperatorBuilder.cs @@ -38,11 +38,12 @@ public OperatorBuilder(IServiceCollection services, OperatorSettings settings) public IServiceCollection Services { get; } - private static Action DefaultCacheConfiguration + private static Action DefaultResourceWatcherCacheConfiguration => options => { + options.CacheKeyPrefix = "rw-"; options.DefaultEntryOptions - .SetDuration(Timeout.InfiniteTimeSpan); + .SetDuration(TimeSpan.MaxValue); }; public IOperatorBuilder AddController() @@ -124,7 +125,7 @@ private void AddOperatorBase() Services .AddFusionCache(CacheConstants.CacheNames.ResourceWatcher) .WithOptions( - options => (_settings.ConfigureResourceWatcherEntityCache ?? DefaultCacheConfiguration).Invoke(options)); + options => (_settings.ConfigureResourceWatcherEntityCache ?? DefaultResourceWatcherCacheConfiguration).Invoke(options)); // Add the default configuration and the client separately. This allows external users to override either // just the config (e.g. for integration tests) or to replace the whole client, e.g. with a mock. diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index d22c3fc5..cd73ba6d 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -1,4 +1,3 @@ -using System.Collections.Concurrent; using System.Diagnostics; using System.Net; using System.Runtime.Serialization; @@ -36,8 +35,6 @@ public class ResourceWatcher( : IHostedService, IAsyncDisposable, IDisposable where TEntity : IKubernetesObject { - private readonly ConcurrentDictionary _entityLocks = new(); - private readonly IFusionCache _entityCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); private CancellationTokenSource _cancellationTokenSource = new(); private uint _watcherReconnectRetries; @@ -137,35 +134,25 @@ static async ValueTask CastAndDispose(IDisposable resource) protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) { - SemaphoreSlim? semaphore; + MaybeValue cachedGeneration; switch (type) { case WatchEventType.Added: - semaphore = _entityLocks.GetOrAdd(entity.Uid(), _ => new(1, 1)); - await semaphore.WaitAsync(cancellationToken); + cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); - try + if (!cachedGeneration.HasValue) { - var cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); - - if (!cachedGeneration.HasValue) - { - // Only perform reconciliation if the entity was not already in the cache. - await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 0, token: cancellationToken); - await ReconcileModificationAsync(entity, cancellationToken); - } - else - { - logger.LogDebug( - """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", - entity.Kind, - entity.Name()); - } + // Only perform reconciliation if the entity was not already in the cache. + await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 0, token: cancellationToken); + await ReconcileModificationAsync(entity, cancellationToken); } - finally + else { - semaphore.Release(); + logger.LogDebug( + """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", + entity.Kind, + entity.Name()); } break; @@ -173,32 +160,22 @@ protected virtual async Task OnEventAsync(WatchEventType type, TEntity entity, C switch (entity) { case { Metadata.DeletionTimestamp: null }: - semaphore = _entityLocks.GetOrAdd(entity.Uid(), _ => new(1, 1)); - await semaphore.WaitAsync(cancellationToken); + cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); - try + // Check if entity spec has changed through "Generation" value increment. Skip reconcile if not changed. + if (cachedGeneration.HasValue && cachedGeneration >= entity.Generation()) { - var cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); - - // Check if entity spec has changed through "Generation" value increment. Skip reconcile if not changed. - if (cachedGeneration.HasValue && cachedGeneration >= entity.Generation()) - { - logger.LogDebug( - """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", - entity.Kind, - entity.Name()); - return; - } - - // update cached generation since generation now changed - await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 1, token: cancellationToken); - await ReconcileModificationAsync(entity, cancellationToken); - } - finally - { - semaphore.Release(); + logger.LogDebug( + """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", + entity.Kind, + entity.Name()); + return; } + // update cached generation since generation now changed + await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 1, token: cancellationToken); + await ReconcileModificationAsync(entity, cancellationToken); + break; case { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } }: await ReconcileFinalizersSequentialAsync(entity, cancellationToken); From 3d2e60781edce713657a7d1d51d5fb16b04aa019 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Tue, 1 Jul 2025 08:18:17 +0200 Subject: [PATCH 06/58] refactor: enhance resource watcher cache configuration and logging scope - Updated `ConfigureResourceWatcherEntityCache` to use `IFusionCacheBuilder` for extensibility. - Moved resource watcher cache setup logic to `WithResourceWatcherCaching` extension. - Added detailed XML comments for `EntityLoggingScope` to improve documentation. - Removed redundant `DefaultResourceWatcherCacheConfiguration`. --- .../Builder/OperatorSettings.cs | 2 +- .../Builder/CacheExtensions.cs | 46 +++++++++++++++++++ .../Builder/OperatorBuilder.cs | 13 +----- .../Logging/EntityLoggingScope.cs | 24 ++++++++++ 4 files changed, 72 insertions(+), 13 deletions(-) create mode 100644 src/KubeOps.Operator/Builder/CacheExtensions.cs diff --git a/src/KubeOps.Abstractions/Builder/OperatorSettings.cs b/src/KubeOps.Abstractions/Builder/OperatorSettings.cs index d7609d6a..fa8902aa 100644 --- a/src/KubeOps.Abstractions/Builder/OperatorSettings.cs +++ b/src/KubeOps.Abstractions/Builder/OperatorSettings.cs @@ -67,7 +67,7 @@ public sealed partial class OperatorSettings /// This property is optional and can be used to customize caching behavior for resource watcher entities. /// If not set, a default cache configuration is applied. /// - public Action? ConfigureResourceWatcherEntityCache { get; set; } + public Action? ConfigureResourceWatcherEntityCache { get; set; } [GeneratedRegex(@"(\W|_)", RegexOptions.CultureInvariant)] private static partial Regex OperatorNameRegex(); diff --git a/src/KubeOps.Operator/Builder/CacheExtensions.cs b/src/KubeOps.Operator/Builder/CacheExtensions.cs new file mode 100644 index 00000000..33e0ede9 --- /dev/null +++ b/src/KubeOps.Operator/Builder/CacheExtensions.cs @@ -0,0 +1,46 @@ +using KubeOps.Abstractions.Builder; +using KubeOps.Operator.Constants; + +using Microsoft.Extensions.DependencyInjection; + +using ZiggyCreatures.Caching.Fusion; + +namespace KubeOps.Operator.Builder; + +/// +/// Provides extension methods for configuring caching related to the operator. +/// +public static class CacheExtensions +{ + /// + /// Configures resource watcher caching for the given service collection. + /// Adds a FusionCache instance for resource watchers and applies custom or default cache configuration. + /// + /// The service collection to add the resource watcher caching to. + /// + /// The operator settings that optionally provide a custom configuration for the resource watcher entity cache. + /// + /// The modified service collection with resource watcher caching configured. + public static IServiceCollection WithResourceWatcherCaching(this IServiceCollection services, OperatorSettings settings) + { + var cacheBuilder = services + .AddFusionCache(CacheConstants.CacheNames.ResourceWatcher); + + if (settings.ConfigureResourceWatcherEntityCache != default) + { + settings.ConfigureResourceWatcherEntityCache(cacheBuilder); + } + else + { + cacheBuilder + .WithOptions(options => + { + options.CacheKeyPrefix = "rw-"; + options.DefaultEntryOptions + .SetDuration(TimeSpan.MaxValue); + }); + } + + return services; + } +} diff --git a/src/KubeOps.Operator/Builder/OperatorBuilder.cs b/src/KubeOps.Operator/Builder/OperatorBuilder.cs index 0c59d4e3..a46b46a5 100644 --- a/src/KubeOps.Operator/Builder/OperatorBuilder.cs +++ b/src/KubeOps.Operator/Builder/OperatorBuilder.cs @@ -38,14 +38,6 @@ public OperatorBuilder(IServiceCollection services, OperatorSettings settings) public IServiceCollection Services { get; } - private static Action DefaultResourceWatcherCacheConfiguration - => options => - { - options.CacheKeyPrefix = "rw-"; - options.DefaultEntryOptions - .SetDuration(TimeSpan.MaxValue); - }; - public IOperatorBuilder AddController() where TImplementation : class, IEntityController where TEntity : IKubernetesObject @@ -122,10 +114,7 @@ private void AddOperatorBase() Services.AddSingleton(new ActivitySource(_settings.Name)); // add and configure resource watcher entity cache - Services - .AddFusionCache(CacheConstants.CacheNames.ResourceWatcher) - .WithOptions( - options => (_settings.ConfigureResourceWatcherEntityCache ?? DefaultResourceWatcherCacheConfiguration).Invoke(options)); + Services.WithResourceWatcherCaching(_settings); // Add the default configuration and the client separately. This allows external users to override either // just the config (e.g. for integration tests) or to replace the whole client, e.g. with a mock. diff --git a/src/KubeOps.Operator/Logging/EntityLoggingScope.cs b/src/KubeOps.Operator/Logging/EntityLoggingScope.cs index 3a254ce1..19622cbe 100644 --- a/src/KubeOps.Operator/Logging/EntityLoggingScope.cs +++ b/src/KubeOps.Operator/Logging/EntityLoggingScope.cs @@ -6,6 +6,10 @@ namespace KubeOps.Operator.Logging; #pragma warning disable CA1710 +/// +/// A logging scope that encapsulates contextual information related to a Kubernetes entity and event type. +/// Provides a mechanism for structured logging with key-value pairs corresponding to entity metadata and event type. +/// internal sealed record EntityLoggingScope : IReadOnlyCollection> #pragma warning restore CA1710 { @@ -20,6 +24,22 @@ private EntityLoggingScope(IReadOnlyDictionary state) private IReadOnlyDictionary Values { get; } + /// + /// Creates a new instance of for the provided Kubernetes entity and event type. + /// + /// + /// The type of the Kubernetes entity. Must implement . + /// + /// + /// The type of the watch event for the entity (e.g., Added, Modified, Deleted, or Bookmark). + /// + /// + /// The Kubernetes entity associated with the logging scope. This includes metadata such as Kind, Namespace, Name, UID, and ResourceVersion. + /// + /// + /// A new instance containing contextual key-value pairs + /// related to the event type and the provided Kubernetes entity. + /// public static EntityLoggingScope CreateFor(WatchEventType eventType, TEntity entity) where TEntity : IKubernetesObject => new( @@ -29,15 +49,19 @@ public static EntityLoggingScope CreateFor(WatchEventType eventType, TE { nameof(entity.Kind), entity.Kind }, { "Namespace", entity.Namespace() }, { "Name", entity.Name() }, + { "Uid", entity.Uid() }, { "ResourceVersion", entity.ResourceVersion() }, }); + /// public IEnumerator> GetEnumerator() => Values.GetEnumerator(); + /// public override string ToString() => CachedFormattedString ??= $"{{ {string.Join(", ", Values.Select(kvp => $"{kvp.Key} = {kvp.Value}"))} }}"; + /// IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); } From f1840483caeb74952f10831641b3a885c6b4a324 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Tue, 1 Jul 2025 08:38:32 +0200 Subject: [PATCH 07/58] refactor: rename cache extension methods and adjust visibility - Renamed `WithResourceWatcherCaching` to `WithResourceWatcherEntityCaching` for clarity. - Updated `CacheExtensions` to be `internal` to limit scope. - Removed unused dependency on `ZiggyCreatures.Caching.Fusion`. --- src/KubeOps.Operator/Builder/CacheExtensions.cs | 4 ++-- src/KubeOps.Operator/Builder/OperatorBuilder.cs | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/KubeOps.Operator/Builder/CacheExtensions.cs b/src/KubeOps.Operator/Builder/CacheExtensions.cs index 33e0ede9..40367fe1 100644 --- a/src/KubeOps.Operator/Builder/CacheExtensions.cs +++ b/src/KubeOps.Operator/Builder/CacheExtensions.cs @@ -10,7 +10,7 @@ namespace KubeOps.Operator.Builder; /// /// Provides extension methods for configuring caching related to the operator. /// -public static class CacheExtensions +internal static class CacheExtensions { /// /// Configures resource watcher caching for the given service collection. @@ -21,7 +21,7 @@ public static class CacheExtensions /// The operator settings that optionally provide a custom configuration for the resource watcher entity cache. /// /// The modified service collection with resource watcher caching configured. - public static IServiceCollection WithResourceWatcherCaching(this IServiceCollection services, OperatorSettings settings) + internal static IServiceCollection WithResourceWatcherEntityCaching(this IServiceCollection services, OperatorSettings settings) { var cacheBuilder = services .AddFusionCache(CacheConstants.CacheNames.ResourceWatcher); diff --git a/src/KubeOps.Operator/Builder/OperatorBuilder.cs b/src/KubeOps.Operator/Builder/OperatorBuilder.cs index a46b46a5..7c9fc676 100644 --- a/src/KubeOps.Operator/Builder/OperatorBuilder.cs +++ b/src/KubeOps.Operator/Builder/OperatorBuilder.cs @@ -21,8 +21,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -using ZiggyCreatures.Caching.Fusion; - namespace KubeOps.Operator.Builder; internal sealed class OperatorBuilder : IOperatorBuilder @@ -114,7 +112,7 @@ private void AddOperatorBase() Services.AddSingleton(new ActivitySource(_settings.Name)); // add and configure resource watcher entity cache - Services.WithResourceWatcherCaching(_settings); + Services.WithResourceWatcherEntityCaching(_settings); // Add the default configuration and the client separately. This allows external users to override either // just the config (e.g. for integration tests) or to replace the whole client, e.g. with a mock. From 5e4de87ab249f11fc4770cbe7b21775929645566 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Mon, 7 Jul 2025 14:11:31 +0200 Subject: [PATCH 08/58] refactor: update cache key prefix in `CacheExtensions` to use `CacheConstants` for consistency --- src/KubeOps.Operator/Builder/CacheExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/KubeOps.Operator/Builder/CacheExtensions.cs b/src/KubeOps.Operator/Builder/CacheExtensions.cs index 40367fe1..a75a26b9 100644 --- a/src/KubeOps.Operator/Builder/CacheExtensions.cs +++ b/src/KubeOps.Operator/Builder/CacheExtensions.cs @@ -35,7 +35,7 @@ internal static IServiceCollection WithResourceWatcherEntityCaching(this IServic cacheBuilder .WithOptions(options => { - options.CacheKeyPrefix = "rw-"; + options.CacheKeyPrefix = $"{CacheConstants.CacheNames.ResourceWatcher}:"; options.DefaultEntryOptions .SetDuration(TimeSpan.MaxValue); }); From 8ce68a84eeba50607c54004a96ffa606115b7642 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Mon, 7 Jul 2025 15:48:49 +0200 Subject: [PATCH 09/58] docs: add caching documentation and adjust sidebar positions - Added a new `Caching` documentation page explaining resource watcher caching with FusionCache and configuration options (in-memory and distributed). - Updated sidebar positions for `Deployment`, `Utilities`, and `Testing` to accommodate the new `Caching` page. --- docs/docs/operator/caching.mdx | 50 ++++++++++++++++++++++ docs/docs/operator/deployment.mdx | 2 +- docs/docs/operator/testing/_category_.json | 2 +- docs/docs/operator/utilities.mdx | 2 +- 4 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 docs/docs/operator/caching.mdx diff --git a/docs/docs/operator/caching.mdx b/docs/docs/operator/caching.mdx new file mode 100644 index 00000000..4fb5b239 --- /dev/null +++ b/docs/docs/operator/caching.mdx @@ -0,0 +1,50 @@ +--- +title: Caching +description: Caching - Memory and Distributed +sidebar_position: 7 +--- + +## ResourceWatcher + +The `ResourceWatcher` uses an instance of `IFusionCache` to store the `.metadata.generation` value of each observed resource. The key for the cache entry is the resource's unique ID (`metadata.uid`). +The primary purpose of the cache is to skip reconciliation cycles for events that do not represent an actual change to a resource's specification (`.spec`). + +1. **`MODIFIED` Event Type**: + - Kubernetes only increments the `.metadata.generation` value of a resource when its specification (`.spec`) changes. Updates to status fields (`.status`), while also triggering a `MODIFIED` event, do not increase the `generation`. + - When a `MODIFIED` event arrives, the `ResourceWatcher` compares the `generation` of the incoming resource with the value stored in `FusionCache`. + - If the new `generation` is not greater than the cached one, the reconciliation is skipped. This is a critical optimization, as status updates can occur very frequently (e.g., from other controllers) and typically do not require action from your operator. + - Only when the `generation` has increased is the resource forwarded for reconciliation, and the new `generation` value is stored in the cache. + +2. **`ADDED` Event Type**: + - On an `ADDED` event, the watcher checks if the resource is already present in the cache. + - This prevents resources that the operator already knows about (e.g., after a watcher restart) from being incorrectly treated as "new" and reconciled again. + +3. **`DELETED` Event Type**: + - When a resource is deleted, the watcher removes the corresponding entry from the cache to keep the memory clean. + +### Default Configuration: In-Memory (L1) Cache + +By default, and without any extra configuration, `KubeOps` uses a simple in-memory cache for `FusionCache`. + +- **Advantages**: + - Requires zero configuration. + - Very fast, as all data is held in the operator pod's memory. + +- **Disadvantages**: + - The cache is volatile. If the pod restarts, all stored `generation` values are lost, leading to a full reconciliation of all observed resources. + +### Advanced Configuration: Distributed (L2) Cache + +For robust use in production or HA environments, it is essential to extend `FusionCache` with a distributed L2 cache and a backplane. This ensures that all operator instances share a consistent state. +A common setup for this involves using **Redis**. + +**Steps to Configure a Distributed Cache with Redis:** + +1. **Add the necessary NuGet packages to your project:** + - `ZiggyCreatures.FusionCache.Serialization.SystemTextJson` (or another serializer) + - `ZiggyCreatures.FusionCache.Backplane.StackExchangeRedis` + - `Microsoft.Extensions.Caching.StackExchangeRedis` + +2. **Configure the services in your `Program.cs`:** +You need to configure `FusionCache` with a serializer, the distributed cache (L2), and a backplane. The backplane (e.g., via Redis Pub/Sub) ensures that cache invalidations (like a `SET` or `REMOVE`) are immediately propagated to all other operator pods, keeping their L1 caches in sync. + diff --git a/docs/docs/operator/deployment.mdx b/docs/docs/operator/deployment.mdx index 10267813..8dfd83cb 100644 --- a/docs/docs/operator/deployment.mdx +++ b/docs/docs/operator/deployment.mdx @@ -1,7 +1,7 @@ --- title: Deployment description: Deploying your KubeOps Operator -sidebar_position: 7 +sidebar_position: 8 --- # Deployment diff --git a/docs/docs/operator/testing/_category_.json b/docs/docs/operator/testing/_category_.json index 56f722ea..3ea28e1c 100644 --- a/docs/docs/operator/testing/_category_.json +++ b/docs/docs/operator/testing/_category_.json @@ -1,5 +1,5 @@ { - "position": 8, + "position": 9, "label": "Testing", "collapsible": true, "collapsed": true diff --git a/docs/docs/operator/utilities.mdx b/docs/docs/operator/utilities.mdx index ef2656db..3cdfa14e 100644 --- a/docs/docs/operator/utilities.mdx +++ b/docs/docs/operator/utilities.mdx @@ -1,7 +1,7 @@ --- title: Utilities description: Utilities for your Operator and Development -sidebar_position: 9 +sidebar_position: 10 --- # Development and Operator Utilities From 2a6f4fc3ea7039d828d034dcbcef1788377ec53e Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Mon, 7 Jul 2025 16:42:46 +0200 Subject: [PATCH 10/58] docs: enhance caching documentation with configuration examples and FusionCache details - Improved explanations for in-memory and distributed caching setups. - Added example code for customizing resource watcher cache with FusionCache. - Included references to FusionCache and Redis documentation for further guidance. --- docs/docs/operator/caching.mdx | 51 ++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/docs/docs/operator/caching.mdx b/docs/docs/operator/caching.mdx index 4fb5b239..ce90dec5 100644 --- a/docs/docs/operator/caching.mdx +++ b/docs/docs/operator/caching.mdx @@ -6,13 +6,17 @@ sidebar_position: 7 ## ResourceWatcher -The `ResourceWatcher` uses an instance of `IFusionCache` to store the `.metadata.generation` value of each observed resource. The key for the cache entry is the resource's unique ID (`metadata.uid`). +The `ResourceWatcher` uses a cache instance to store the `.metadata.generation` value of each observed resource. +The key for the cache entry is the resource's unique ID (`metadata.uid`). + The primary purpose of the cache is to skip reconciliation cycles for events that do not represent an actual change to a resource's specification (`.spec`). 1. **`MODIFIED` Event Type**: - - Kubernetes only increments the `.metadata.generation` value of a resource when its specification (`.spec`) changes. Updates to status fields (`.status`), while also triggering a `MODIFIED` event, do not increase the `generation`. - - When a `MODIFIED` event arrives, the `ResourceWatcher` compares the `generation` of the incoming resource with the value stored in `FusionCache`. - - If the new `generation` is not greater than the cached one, the reconciliation is skipped. This is a critical optimization, as status updates can occur very frequently (e.g., from other controllers) and typically do not require action from your operator. + - Kubernetes only increments the `.metadata.generation` value of a resource when its specification (`.spec`) changes. + Updates to status fields (`.status`), while also triggering a `MODIFIED` event, do not increase the `generation`. + - When a `MODIFIED` event arrives, the `ResourceWatcher` compares the `generation` of the incoming resource with the value stored in the cache. + - If the new `generation` is not greater than the cached one, the reconciliation is skipped. + This is a critical optimization, as status updates can occur very frequently (e.g., from other controllers) and typically do not require action from your operator. - Only when the `generation` has increased is the resource forwarded for reconciliation, and the new `generation` value is stored in the cache. 2. **`ADDED` Event Type**: @@ -24,7 +28,7 @@ The primary purpose of the cache is to skip reconciliation cycles for events tha ### Default Configuration: In-Memory (L1) Cache -By default, and without any extra configuration, `KubeOps` uses a simple in-memory cache for `FusionCache`. +By default, and without any extra configuration, `KubeOps` uses a simple in-memory cache. - **Advantages**: - Requires zero configuration. @@ -35,16 +39,35 @@ By default, and without any extra configuration, `KubeOps` uses a simple in-memo ### Advanced Configuration: Distributed (L2) Cache -For robust use in production or HA environments, it is essential to extend `FusionCache` with a distributed L2 cache and a backplane. This ensures that all operator instances share a consistent state. -A common setup for this involves using **Redis**. +For robust use in production or HA environments, it could be essential to extend cache with a distributed L2 cache and a backplane. +This ensures that all operator instances share a consistent state. +A common setup for this involves using [**Redis**](https://github.com/redis/redis). + +### FusionCache + +KubeOps utilizes [`FusionCache`](https://github.com/ZiggyCreatures/FusionCache/blob/main/docs/AGentleIntroduction.md) for seamless support of an L1/L2 cache. +Via `OperatorSettings.ConfigureResourceWatcherEntityCache`, an `Action` is provided that allows extending the standard configuration or +overwriting it with a customized version. -**Steps to Configure a Distributed Cache with Redis:** +Here is an example of what a customized configuration with an L2 cache could look like: -1. **Add the necessary NuGet packages to your project:** - - `ZiggyCreatures.FusionCache.Serialization.SystemTextJson` (or another serializer) - - `ZiggyCreatures.FusionCache.Backplane.StackExchangeRedis` - - `Microsoft.Extensions.Caching.StackExchangeRedis` +```csharp +builder + .Services + .AddKubernetesOperator(settings => + { + settings.Name = OperatorName; + settings.ConfigureResourceWatcherEntityCache = + cacheBuilder => + cacheBuilder + .WithCacheKeyPrefix($"{CacheConstants.CacheNames.ResourceWatcher}:") + .WithSerializer(_ => new FusionCacheSystemTextJsonSerializer()) + .WithRegisteredDistributedCache() + .WithDefaultEntryOptions(options => + options.Duration = TimeSpan.MaxValue); + }) +``` -2. **Configure the services in your `Program.cs`:** -You need to configure `FusionCache` with a serializer, the distributed cache (L2), and a backplane. The backplane (e.g., via Redis Pub/Sub) ensures that cache invalidations (like a `SET` or `REMOVE`) are immediately propagated to all other operator pods, keeping their L1 caches in sync. +For an overview of all of FusionCache's features, we refer you to the corresponding documentation: +https://github.com/ZiggyCreatures/FusionCache/blob/main/docs/CacheLevels.md \ No newline at end of file From 1ad141cd5281c00719b00199522c7f0084bd4b86 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 5 Sep 2025 15:50:22 +0200 Subject: [PATCH 11/58] refactor(watcher): streamline deletion logic and update FusionCache dependency - Removed redundant requeue logic and optimized entity cache operations during deletion in `ResourceWatcher`. - Upgraded `ZiggyCreatures.FusionCache` to version `2.4.0`. --- src/KubeOps.Operator/KubeOps.Operator.csproj | 2 +- src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/KubeOps.Operator/KubeOps.Operator.csproj b/src/KubeOps.Operator/KubeOps.Operator.csproj index 7be6ea18..6c3a952f 100644 --- a/src/KubeOps.Operator/KubeOps.Operator.csproj +++ b/src/KubeOps.Operator/KubeOps.Operator.csproj @@ -17,7 +17,7 @@ - + diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index 3dc716fb..4f55e60b 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -331,16 +331,13 @@ e.InnerException is EndOfStreamException && private async Task> ReconcileDeletionAsync(TEntity entity, CancellationToken cancellationToken) { - requeue.Remove(entity); - await _entityCache.RemoveAsync(entity.Uid(), token: cancellationToken); - await using var scope = provider.CreateAsyncScope(); var controller = scope.ServiceProvider.GetRequiredService>(); var result = await controller.DeletedAsync(entity, cancellationToken); if (result.IsSuccess) { - _entityCache.TryRemove(result.Entity.Uid(), out _); + await _entityCache.RemoveAsync(entity.Uid(), token: cancellationToken); } return result; From 6ded18f77844d03efbcb281b1ead7eecbaad7e67 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 5 Sep 2025 15:55:49 +0200 Subject: [PATCH 12/58] refactor(operator): remove unused FusionCache dependency from project file --- src/KubeOps.Operator/KubeOps.Operator.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/KubeOps.Operator/KubeOps.Operator.csproj b/src/KubeOps.Operator/KubeOps.Operator.csproj index 6c3a952f..1046c85e 100644 --- a/src/KubeOps.Operator/KubeOps.Operator.csproj +++ b/src/KubeOps.Operator/KubeOps.Operator.csproj @@ -17,7 +17,6 @@ - From 9dce9dfead8ffdadcb6b96ebc68bec47f66488f9 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Mon, 8 Sep 2025 12:57:07 +0200 Subject: [PATCH 13/58] feat(queue): add `RequeueType` and enhance requeue handling - Introduced `RequeueType` enumeration to specify requeue operation types (`Added`, `Modified`, `Deleted`). - Implemented `RequeueTypeExtensions` for mapping `WatchEventType` to `RequeueType`. - Updated requeue mechanism to include `RequeueType` in `EntityRequeue` and related methods. - Refactored `TimedEntityQueue` and related classes to support `RequeueEntry` containing both the entity and its requeue type. - Adjusted tests to incorporate `RequeueType` into entity requeue logic. --- .../Queue/EntityRequeue.cs | 3 ++- src/KubeOps.Abstractions/Queue/RequeueType.cs | 26 +++++++++++++++++++ .../Queue/EntityRequeueBackgroundService.cs | 12 ++++----- .../Queue/KubeOpsEntityRequeueFactory.cs | 4 +-- .../Queue/RequeueEntry{TEntity}.cs | 23 ++++++++++++++++ .../Queue/RequeueTypeExtensions.cs | 21 +++++++++++++++ .../Queue/TimedEntityQueue.cs | 13 ++++++---- .../Queue/TimedQueueEntry{TEntity}.cs | 10 ++++--- .../Watcher/ResourceWatcher{TEntity}.cs | 2 +- .../CancelEntityRequeue.Integration.Test.cs | 3 +-- .../DeletedEntityRequeue.Integration.Test.cs | 2 +- .../EntityRequeue.Integration.Test.cs | 2 +- .../Events/EventPublisher.Integration.Test.cs | 2 +- .../Queue/TimedEntityQueue.Test.cs | 11 ++++---- 14 files changed, 106 insertions(+), 28 deletions(-) create mode 100644 src/KubeOps.Abstractions/Queue/RequeueType.cs create mode 100644 src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs create mode 100644 src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs diff --git a/src/KubeOps.Abstractions/Queue/EntityRequeue.cs b/src/KubeOps.Abstractions/Queue/EntityRequeue.cs index 1aeae1b6..5354a73e 100644 --- a/src/KubeOps.Abstractions/Queue/EntityRequeue.cs +++ b/src/KubeOps.Abstractions/Queue/EntityRequeue.cs @@ -26,6 +26,7 @@ namespace KubeOps.Abstractions.Queue; /// /// The type of the entity. /// The instance of the entity that should be requeued. +/// The type of which the reconcile operation should be executed. /// The time to wait before another reconcile loop is fired. /// /// Use the requeue delegate to repeatedly reconcile an entity after 5 seconds. @@ -47,5 +48,5 @@ namespace KubeOps.Abstractions.Queue; /// } /// /// -public delegate void EntityRequeue(TEntity entity, TimeSpan requeueIn) +public delegate void EntityRequeue(TEntity entity, RequeueType type, TimeSpan requeueIn) where TEntity : IKubernetesObject; diff --git a/src/KubeOps.Abstractions/Queue/RequeueType.cs b/src/KubeOps.Abstractions/Queue/RequeueType.cs new file mode 100644 index 00000000..4bce1f14 --- /dev/null +++ b/src/KubeOps.Abstractions/Queue/RequeueType.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Abstractions.Queue; + +/// +/// Specifies the types of requeue operations that can occur on an entity. +/// +public enum RequeueType +{ + /// + /// Indicates that an entity should be added and is scheduled for requeue. + /// + Added, + + /// + /// Indicates that an entity has been modified and is scheduled for requeue. + /// + Modified, + + /// + /// Indicates that an entity should be deleted and is scheduled for requeue. + /// + Deleted, +} diff --git a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs index d296fc55..8fba86d7 100644 --- a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs +++ b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs @@ -84,11 +84,11 @@ static async ValueTask CastAndDispose(IDisposable resource) private async Task WatchAsync(CancellationToken cancellationToken) { - await foreach (var entity in queue) + await foreach (var entry in queue) { try { - await ReconcileSingleAsync(entity, cancellationToken); + await ReconcileSingleAsync(entry.Entity, cancellationToken); } catch (OperationCanceledException e) when (!cancellationToken.IsCancellationRequested) { @@ -96,8 +96,8 @@ private async Task WatchAsync(CancellationToken cancellationToken) e, """Queued reconciliation for the entity of type {ResourceType} for "{Kind}/{Name}" failed.""", typeof(TEntity).Name, - entity.Kind, - entity.Name()); + entry.Entity.Kind, + entry.Entity.Name()); } catch (Exception e) { @@ -105,8 +105,8 @@ private async Task WatchAsync(CancellationToken cancellationToken) e, """Queued reconciliation for the entity of type {ResourceType} for "{Kind}/{Name}" failed.""", typeof(TEntity).Name, - entity.Kind, - entity.Name()); + entry.Entity.Kind, + entry.Entity.Name()); } } } diff --git a/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs b/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs index c6592fef..b729e233 100644 --- a/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs +++ b/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs @@ -17,7 +17,7 @@ internal sealed class KubeOpsEntityRequeueFactory(IServiceProvider services) { public EntityRequeue Create() where TEntity : IKubernetesObject => - (entity, timeSpan) => + (entity, type, timeSpan) => { var logger = services.GetService>>(); var queue = services.GetRequiredService>(); @@ -28,6 +28,6 @@ public EntityRequeue Create() entity.Name(), timeSpan.TotalMilliseconds); - queue.Enqueue(entity, timeSpan); + queue.Enqueue(entity, type, timeSpan); }; } diff --git a/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs b/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs new file mode 100644 index 00000000..d8541d91 --- /dev/null +++ b/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using KubeOps.Abstractions.Queue; + +namespace KubeOps.Operator.Queue; + +public sealed record RequeueEntry +{ + private RequeueEntry(TEntity entity, RequeueType requeueType) + { + Entity = entity; + RequeueType = requeueType; + } + + public TEntity Entity { get; } + + public RequeueType RequeueType { get; } + + public static RequeueEntry CreateFor(TEntity entity, RequeueType requeueType) + => new(entity, requeueType); +} diff --git a/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs b/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs new file mode 100644 index 00000000..01ed97e6 --- /dev/null +++ b/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; + +using KubeOps.Abstractions.Queue; + +namespace KubeOps.Operator.Queue; + +public static class RequeueTypeExtensions +{ + public static RequeueType ToRequeueType(this WatchEventType watchEventType) + => watchEventType switch + { + WatchEventType.Added => RequeueType.Added, + WatchEventType.Modified => RequeueType.Modified, + WatchEventType.Deleted => RequeueType.Deleted, + _ => throw new NotSupportedException($"WatchEventType '{watchEventType}' is not supported!"), + }; +} diff --git a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs index 3bc228e8..1ca2b67c 100644 --- a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs +++ b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs @@ -7,6 +7,8 @@ using k8s; using k8s.Models; +using KubeOps.Abstractions.Queue; + namespace KubeOps.Operator.Queue; /// @@ -24,7 +26,7 @@ public sealed class TimedEntityQueue : IDisposable private readonly ConcurrentDictionary> _management = new(); // The actual queue containing all the entries that have to be reconciled. - private readonly BlockingCollection _queue = new(new ConcurrentQueue()); + private readonly BlockingCollection> _queue = new(new ConcurrentQueue>()); internal int Count => _management.Count; @@ -33,14 +35,15 @@ public sealed class TimedEntityQueue : IDisposable /// If the item already exists, the existing entry is updated. /// /// The entity. + /// The type of which the reconcile operation should be executed. /// The time after , where the item is reevaluated again. - public void Enqueue(TEntity entity, TimeSpan requeueIn) + public void Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn) { _management.AddOrUpdate( TimedEntityQueue.GetKey(entity) ?? throw new InvalidOperationException("Cannot enqueue entities without name."), key => { - var entry = new TimedQueueEntry(entity, requeueIn); + var entry = new TimedQueueEntry(entity, type, requeueIn); _scheduledEntries.StartNew( async () => { @@ -53,7 +56,7 @@ public void Enqueue(TEntity entity, TimeSpan requeueIn) (key, oldEntry) => { oldEntry.Cancel(); - var entry = new TimedQueueEntry(entity, requeueIn); + var entry = new TimedQueueEntry(entity, type, requeueIn); _scheduledEntries.StartNew( async () => { @@ -74,7 +77,7 @@ public void Dispose() } } - public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + public async IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default) { await Task.Yield(); foreach (var entry in _queue.GetConsumingEnumerable(cancellationToken)) diff --git a/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs b/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs index 72ece41e..bdf105f7 100644 --- a/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs +++ b/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs @@ -4,6 +4,8 @@ using System.Collections.Concurrent; +using KubeOps.Abstractions.Queue; + namespace KubeOps.Operator.Queue; internal sealed record TimedQueueEntry : IDisposable @@ -11,11 +13,13 @@ internal sealed record TimedQueueEntry : IDisposable private readonly CancellationTokenSource _cts = new(); private readonly TimeSpan _requeueIn; private readonly TEntity _entity; + private readonly RequeueType _requeueType; - public TimedQueueEntry(TEntity entity, TimeSpan requeueIn) + public TimedQueueEntry(TEntity entity, RequeueType requeueType, TimeSpan requeueIn) { _requeueIn = requeueIn; _entity = entity; + _requeueType = requeueType; } /// @@ -40,7 +44,7 @@ public void Cancel() /// /// The collection to add the entry to. /// A representing the asynchronous operation. - public async Task AddAfterDelay(BlockingCollection collection) + public async Task AddAfterDelay(BlockingCollection> collection) { try { @@ -50,7 +54,7 @@ public async Task AddAfterDelay(BlockingCollection collection) return; } - collection.TryAdd(_entity); + collection.TryAdd(RequeueEntry.CreateFor(_entity, _requeueType)); } catch (TaskCanceledException) { diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index 4f55e60b..ff79c118 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -234,7 +234,7 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) if (result.RequeueAfter.HasValue) { - requeue.Enqueue(result.Entity, result.RequeueAfter.Value); + requeue.Enqueue(result.Entity, type.ToRequeueType(), result.RequeueAfter.Value); } if (result.IsFailure) diff --git a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs index 414e8a38..bdd2d3d8 100644 --- a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs @@ -49,7 +49,6 @@ public async Task Should_Not_Affect_Queues_If_Only_Status_Updated() _mock.Invocations.Count.Should().Be(1); Services.GetRequiredService>().Count.Should().Be(1); - } public override async Task InitializeAsync() @@ -82,7 +81,7 @@ public Task> ReconcileAsync(V1OperatorIn svc.Invocation(entity); if (svc.Invocations.Count < 2) { - requeue(entity, TimeSpan.FromMilliseconds(1000)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1000)); } return Task.FromResult(Result.ForSuccess(entity)); diff --git a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs index a31d42f2..5e00edcd 100644 --- a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs @@ -62,7 +62,7 @@ private class TestController(InvocationCounter public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - requeue(entity, TimeSpan.FromMilliseconds(1000)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1000)); return Task.FromResult(Result.ForSuccess(entity)); } diff --git a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs index c0e14a3f..49cb6b2c 100644 --- a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs @@ -93,7 +93,7 @@ public Task> ReconcileAsync(V1OperatorIn svc.Invocation(entity); if (svc.Invocations.Count <= svc.TargetInvocationCount) { - requeue(entity, TimeSpan.FromMilliseconds(1)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1)); } return Task.FromResult(Result.ForSuccess(entity)); diff --git a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs index a714cab2..a6fc4843 100644 --- a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs @@ -97,7 +97,7 @@ public async Task> ReconcileAsync(V1Oper if (svc.Invocations.Count < svc.TargetInvocationCount) { - requeue(entity, TimeSpan.FromMilliseconds(10)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(10)); } return Result.ForSuccess(entity); diff --git a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs index 5d0a7c78..0e54b19a 100644 --- a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs +++ b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs @@ -4,19 +4,20 @@ using k8s.Models; +using KubeOps.Abstractions.Queue; using KubeOps.Operator.Queue; namespace KubeOps.Operator.Test.Queue; -public class TimedEntityQueueTest +public sealed class TimedEntityQueueTest { [Fact] public async Task Can_Enqueue_Multiple_Entities_With_Same_Name() { var queue = new TimedEntityQueue(); - queue.Enqueue(CreateSecret("app-ns1", "secret-name"), TimeSpan.FromSeconds(1)); - queue.Enqueue(CreateSecret("app-ns2", "secret-name"), TimeSpan.FromSeconds(1)); + queue.Enqueue(CreateSecret("app-ns1", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1)); + queue.Enqueue(CreateSecret("app-ns2", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1)); var items = new List(); @@ -29,7 +30,7 @@ public async Task Can_Enqueue_Multiple_Entities_With_Same_Name() { while (await enumerator.MoveNextAsync()) { - items.Add(enumerator.Current); + items.Add(enumerator.Current.Entity); } } catch (OperationCanceledException) @@ -40,7 +41,7 @@ public async Task Can_Enqueue_Multiple_Entities_With_Same_Name() Assert.Equal(2, items.Count); } - private V1Secret CreateSecret(string secretNamespace, string secretName) + private static V1Secret CreateSecret(string secretNamespace, string secretName) { var secret = new V1Secret(); secret.EnsureMetadata(); From 12aeda4d7a9e9e8ae7f5201a4abad16a72f975a2 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Wed, 10 Sep 2025 15:56:55 +0200 Subject: [PATCH 14/58] refactor(reconciliation): introduce `Reconciler` to centralize entity reconciliation logic - Created `IReconciler` interface and its implementation to handle entity creation, modification, and deletion. - Updated `ResourceWatcher` and `EntityRequeueBackgroundService` to use `Reconciler` for reconciliation operations. - Removed redundant FusionCache dependency from `ResourceWatcher` and related classes. - Streamlined requeue mechanics and replaced entity finalization logic with `Reconciler` integration. --- .../Queue/EntityRequeueBackgroundService.cs | 43 ++--- .../Reconciliation/IReconciler{TEntity}.cs | 20 ++ .../Reconciliation/Reconciler.cs | 182 ++++++++++++++++++ .../LeaderAwareResourceWatcher{TEntity}.cs | 14 +- .../Watcher/ResourceWatcher{TEntity}.cs | 123 +----------- .../Watcher/ResourceWatcher{TEntity}.Test.cs | 18 +- 6 files changed, 234 insertions(+), 166 deletions(-) create mode 100644 src/KubeOps.Operator/Reconciliation/IReconciler{TEntity}.cs create mode 100644 src/KubeOps.Operator/Reconciliation/Reconciler.cs diff --git a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs index 8fba86d7..c6b5c953 100644 --- a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs +++ b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs @@ -5,11 +5,10 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Queue; using KubeOps.KubernetesClient; +using KubeOps.Operator.Reconciliation; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -18,7 +17,7 @@ namespace KubeOps.Operator.Queue; internal sealed class EntityRequeueBackgroundService( IKubernetesClient client, TimedEntityQueue queue, - IServiceProvider provider, + IReconciler reconciler, ILogger> logger) : IHostedService, IDisposable, IAsyncDisposable where TEntity : IKubernetesObject { @@ -88,7 +87,7 @@ private async Task WatchAsync(CancellationToken cancellationToken) { try { - await ReconcileSingleAsync(entry.Entity, cancellationToken); + await ReconcileSingleAsync(entry, cancellationToken); } catch (OperationCanceledException e) when (!cancellationToken.IsCancellationRequested) { @@ -111,33 +110,31 @@ private async Task WatchAsync(CancellationToken cancellationToken) } } - private async Task ReconcileSingleAsync(TEntity queued, CancellationToken cancellationToken) + private async Task ReconcileSingleAsync(RequeueEntry queuedEntry, CancellationToken cancellationToken) { - logger.LogTrace("""Execute requested requeued reconciliation for "{Name}".""", queued.Name()); + logger.LogTrace("""Execute requested requeued reconciliation for "{Name}".""", queuedEntry.Entity.Name()); - if (await client.GetAsync(queued.Name(), queued.Namespace(), cancellationToken) is not + if (await client.GetAsync(queuedEntry.Entity.Name(), queuedEntry.Entity.Namespace(), cancellationToken) is not { } entity) { logger.LogWarning( - """Requeued entity "{Name}" was not found. Skipping reconciliation.""", queued.Name()); + """Requeued entity "{Name}" was not found. Skipping reconciliation.""", queuedEntry.Entity.Name()); return; } - if (entity.DeletionTimestamp() is not null) + switch (queuedEntry.RequeueType) { - if (entity.Finalizers()?.Count > 0) - { - var identifier = entity.Finalizers()[0]; - await using var scope = provider.CreateAsyncScope(); - var finalizer = scope.ServiceProvider.GetRequiredKeyedService>(identifier); - await finalizer.FinalizeAsync(entity, cancellationToken); - } - } - else - { - await using var scope = provider.CreateAsyncScope(); - var controller = scope.ServiceProvider.GetRequiredService>(); - await controller.ReconcileAsync(entity, cancellationToken); + case RequeueType.Added: + await reconciler.ReconcileCreation(entity, cancellationToken); + break; + case RequeueType.Modified: + await reconciler.ReconcileModification(entity, cancellationToken); + break; + case RequeueType.Deleted: + await reconciler.ReconcileDeletion(entity, cancellationToken); + break; + default: + throw new NotSupportedException($"RequeueType '{queuedEntry.RequeueType}' is not supported!"); } } } diff --git a/src/KubeOps.Operator/Reconciliation/IReconciler{TEntity}.cs b/src/KubeOps.Operator/Reconciliation/IReconciler{TEntity}.cs new file mode 100644 index 00000000..196d6213 --- /dev/null +++ b/src/KubeOps.Operator/Reconciliation/IReconciler{TEntity}.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Controller; + +namespace KubeOps.Operator.Reconciliation; + +public interface IReconciler + where TEntity : IKubernetesObject +{ + Task> ReconcileCreation(TEntity entity, CancellationToken cancellationToken); + + Task> ReconcileModification(TEntity entity, CancellationToken cancellationToken); + + Task> ReconcileDeletion(TEntity entity, CancellationToken cancellationToken); +} diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs new file mode 100644 index 00000000..1fcf5d13 --- /dev/null +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -0,0 +1,182 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Queue; +using KubeOps.KubernetesClient; +using KubeOps.Operator.Constants; +using KubeOps.Operator.Queue; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +using ZiggyCreatures.Caching.Fusion; + +namespace KubeOps.Operator.Reconciliation; + +internal sealed class Reconciler( + ILogger> logger, + IFusionCacheProvider cacheProvider, + IServiceProvider provider, + TimedEntityQueue requeue, + IKubernetesClient client) + : IReconciler + where TEntity : IKubernetesObject +{ + private readonly IFusionCache _entityCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); + + public async Task> ReconcileCreation(TEntity entity, CancellationToken cancellationToken) + { + requeue.Remove(entity); + var cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); + + if (!cachedGeneration.HasValue) + { + // Only perform reconciliation if the entity was not already in the cache. + var result = await ReconcileModificationAsync(entity, cancellationToken); + + if (result.IsSuccess) + { + await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 0, token: cancellationToken); + } + + if (result.RequeueAfter.HasValue) + { + requeue.Enqueue( + result.Entity, + result.IsSuccess ? RequeueType.Modified : RequeueType.Added, + result.RequeueAfter.Value); + } + + return result; + } + + logger.LogDebug( + """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", + entity.Kind, + entity.Name()); + + return Result.ForSuccess(entity); + } + + public async Task> ReconcileModification(TEntity entity, CancellationToken cancellationToken) + { + requeue.Remove(entity); + + Result result; + + switch (entity) + { + case { Metadata.DeletionTimestamp: null }: + var cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); + + // Check if entity spec has changed through "Generation" value increment. Skip reconcile if not changed. + if (cachedGeneration.HasValue && cachedGeneration >= entity.Generation()) + { + logger.LogDebug( + """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", + entity.Kind, + entity.Name()); + + return Result.ForSuccess(entity); + } + + // update cached generation since generation now changed + await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 1, token: cancellationToken); + result = await ReconcileModificationAsync(entity, cancellationToken); + + break; + case { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } }: + result = await ReconcileFinalizersSequentialAsync(entity, cancellationToken); + + break; + default: + result = Result.ForSuccess(entity); + + break; + } + + if (result.RequeueAfter.HasValue) + { + requeue.Enqueue(result.Entity, RequeueType.Modified, result.RequeueAfter.Value); + } + + return result; + } + + public async Task> ReconcileDeletion(TEntity entity, CancellationToken cancellationToken) + { + requeue.Remove(entity); + + await using var scope = provider.CreateAsyncScope(); + var controller = scope.ServiceProvider.GetRequiredService>(); + var result = await controller.DeletedAsync(entity, cancellationToken); + + if (result.IsSuccess) + { + await _entityCache.RemoveAsync(entity.Uid(), token: cancellationToken); + } + + if (result.RequeueAfter.HasValue) + { + requeue.Enqueue( + result.Entity, + RequeueType.Deleted, + result.RequeueAfter.Value); + } + + return result; + } + + private async Task> ReconcileFinalizersSequentialAsync(TEntity entity, CancellationToken cancellationToken) + { + await using var scope = provider.CreateAsyncScope(); + + // condition to call ReconcileFinalizersSequentialAsync is: + // { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } } + // which implies that there is at least a single finalizer + var identifier = entity.Finalizers()[0]; + + if (scope.ServiceProvider.GetKeyedService>(identifier) is not + { } finalizer) + { + logger.LogDebug( + """Entity "{Kind}/{Name}" is finalizing but this operator has no registered finalizers for the identifier {FinalizerIdentifier}.""", + entity.Kind, + entity.Name(), + identifier); + return Result.ForSuccess(entity); + } + + var result = await finalizer.FinalizeAsync(entity, cancellationToken); + + if (!result.IsSuccess) + { + return result; + } + + entity = result.Entity; + entity.RemoveFinalizer(identifier); + entity = await client.UpdateAsync(entity, cancellationToken); + + logger.LogInformation( + """Entity "{Kind}/{Name}" finalized with "{Finalizer}".""", + entity.Kind, + entity.Name(), + identifier); + + return Result.ForSuccess(entity, result.RequeueAfter); + } + + private async Task> ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) + { + await using var scope = provider.CreateAsyncScope(); + var controller = scope.ServiceProvider.GetRequiredService>(); + return await controller.ReconcileAsync(entity, cancellationToken); + } +} diff --git a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs index 15f20a72..4861147d 100644 --- a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs @@ -11,34 +11,28 @@ using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Entities; using KubeOps.KubernetesClient; -using KubeOps.Operator.Queue; +using KubeOps.Operator.Reconciliation; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using ZiggyCreatures.Caching.Fusion; - namespace KubeOps.Operator.Watcher; internal sealed class LeaderAwareResourceWatcher( ActivitySource activitySource, ILogger> logger, - IServiceProvider provider, - TimedEntityQueue queue, + IReconciler reconciler, OperatorSettings settings, IEntityLabelSelector labelSelector, - IFusionCacheProvider cacheProvider, IKubernetesClient client, IHostApplicationLifetime hostApplicationLifetime, LeaderElector elector) : ResourceWatcher( activitySource, logger, - provider, - queue, + reconciler, settings, labelSelector, - cacheProvider, client) where TEntity : IKubernetesObject { @@ -94,7 +88,7 @@ private void StartedLeading() if (_cts.IsCancellationRequested) { _cts.Dispose(); - _cts = new CancellationTokenSource(); + _cts = new(); } base.StartAsync(_cts.Token); diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index ff79c118..6d241d34 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -13,33 +13,25 @@ using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Entities; -using KubeOps.Abstractions.Finalizer; using KubeOps.KubernetesClient; -using KubeOps.Operator.Constants; using KubeOps.Operator.Logging; -using KubeOps.Operator.Queue; +using KubeOps.Operator.Reconciliation; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using ZiggyCreatures.Caching.Fusion; - namespace KubeOps.Operator.Watcher; public class ResourceWatcher( ActivitySource activitySource, ILogger> logger, - IServiceProvider provider, - TimedEntityQueue requeue, + IReconciler reconciler, OperatorSettings settings, IEntityLabelSelector labelSelector, - IFusionCacheProvider cacheProvider, IKubernetesClient client) : IHostedService, IAsyncDisposable, IDisposable where TEntity : IKubernetesObject { - private readonly IFusionCache _entityCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); private CancellationTokenSource _cancellationTokenSource = new(); private uint _watcherReconnectRetries; private Task? _eventWatcher; @@ -102,7 +94,6 @@ protected virtual void Dispose(bool disposing) _cancellationTokenSource.Dispose(); _eventWatcher?.Dispose(); - requeue.Dispose(); client.Dispose(); _disposed = true; @@ -116,7 +107,6 @@ protected virtual async ValueTask DisposeAsyncCore() } await CastAndDispose(_cancellationTokenSource); - await CastAndDispose(requeue); await CastAndDispose(client); _disposed = true; @@ -138,52 +128,16 @@ static async ValueTask CastAndDispose(IDisposable resource) protected virtual async Task> OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) { - MaybeValue cachedGeneration; - switch (type) { case WatchEventType.Added: - cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); - - if (!cachedGeneration.HasValue) - { - // Only perform reconciliation if the entity was not already in the cache. - await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 0, token: cancellationToken); - return await ReconcileModificationAsync(entity, cancellationToken); - } - - logger.LogDebug( - """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", - entity.Kind, - entity.Name()); + return await reconciler.ReconcileCreation(entity, cancellationToken); - break; case WatchEventType.Modified: - switch (entity) - { - case { Metadata.DeletionTimestamp: null }: - cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); + return await reconciler.ReconcileModification(entity, cancellationToken); - // Check if entity spec has changed through "Generation" value increment. Skip reconcile if not changed. - if (cachedGeneration.HasValue && cachedGeneration >= entity.Generation()) - { - logger.LogDebug( - """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", - entity.Kind, - entity.Name()); - return Result.ForSuccess(entity); - } - - // update cached generation since generation now changed - await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 1, token: cancellationToken); - return await ReconcileModificationAsync(entity, cancellationToken); - case { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } }: - return await ReconcileFinalizersSequentialAsync(entity, cancellationToken); - } - - break; case WatchEventType.Deleted: - return await ReconcileDeletionAsync(entity, cancellationToken); + return await reconciler.ReconcileDeletion(entity, cancellationToken); default: logger.LogWarning( @@ -229,14 +183,8 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) try { - requeue.Remove(entity); var result = await OnEventAsync(type, entity, stoppingToken); - if (result.RequeueAfter.HasValue) - { - requeue.Enqueue(result.Entity, type.ToRequeueType(), result.RequeueAfter.Value); - } - if (result.IsFailure) { logger.LogError( @@ -328,65 +276,4 @@ e.InnerException is EndOfStreamException && delay.TotalSeconds); await Task.Delay(delay); } - - private async Task> ReconcileDeletionAsync(TEntity entity, CancellationToken cancellationToken) - { - await using var scope = provider.CreateAsyncScope(); - var controller = scope.ServiceProvider.GetRequiredService>(); - var result = await controller.DeletedAsync(entity, cancellationToken); - - if (result.IsSuccess) - { - await _entityCache.RemoveAsync(entity.Uid(), token: cancellationToken); - } - - return result; - } - - private async Task> ReconcileFinalizersSequentialAsync(TEntity entity, CancellationToken cancellationToken) - { - await using var scope = provider.CreateAsyncScope(); - - // condition to call ReconcileFinalizersSequentialAsync is: - // { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } } - // which implies that there is at least a single finalizer - var identifier = entity.Finalizers()[0]; - - if (scope.ServiceProvider.GetKeyedService>(identifier) is not - { } finalizer) - { - logger.LogDebug( - """Entity "{Kind}/{Name}" is finalizing but this operator has no registered finalizers for the identifier {FinalizerIdentifier}.""", - entity.Kind, - entity.Name(), - identifier); - return Result.ForSuccess(entity); - } - - var result = await finalizer.FinalizeAsync(entity, cancellationToken); - - if (!result.IsSuccess) - { - return result; - } - - entity = result.Entity; - entity.RemoveFinalizer(identifier); - entity = await client.UpdateAsync(entity, cancellationToken); - - logger.LogInformation( - """Entity "{Kind}/{Name}" finalized with "{Finalizer}".""", - entity.Kind, - entity.Name(), - identifier); - - return Result.ForSuccess(entity, result.RequeueAfter); - } - - private async Task> ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) - { - await using var scope = provider.CreateAsyncScope(); - var controller = scope.ServiceProvider.GetRequiredService>(); - return await controller.ReconcileAsync(entity, cancellationToken); - } } diff --git a/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs b/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs index 590494e9..3960e67f 100644 --- a/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs +++ b/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs @@ -11,16 +11,13 @@ using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Entities; using KubeOps.KubernetesClient; -using KubeOps.Operator.Constants; -using KubeOps.Operator.Queue; +using KubeOps.Operator.Reconciliation; using KubeOps.Operator.Watcher; using Microsoft.Extensions.Logging; using Moq; -using ZiggyCreatures.Caching.Fusion; - namespace KubeOps.Operator.Test.Watcher; public sealed class ResourceWatcherTest @@ -31,30 +28,21 @@ public async Task Restarting_Watcher_Should_Trigger_New_Watch() // Arrange. var activitySource = new ActivitySource("unit-test"); var logger = Mock.Of>>(); - var serviceProvider = Mock.Of(); - var timedEntityQueue = new TimedEntityQueue(); + var reconciler = Mock.Of>(); var operatorSettings = new OperatorSettings { Namespace = "unit-test" }; var kubernetesClient = Mock.Of(); - var cache = Mock.Of(); - var cacheProvider = Mock.Of(); var labelSelector = new DefaultEntityLabelSelector(); Mock.Get(kubernetesClient) .Setup(client => client.WatchAsync("unit-test", null, null, true, It.IsAny())) .Returns((_, _, _, _, cancellationToken) => WaitForCancellationAsync<(WatchEventType, V1Pod)>(cancellationToken)); - Mock.Get(cacheProvider) - .Setup(cp => cp.GetCache(It.Is(s => s == CacheConstants.CacheNames.ResourceWatcher))) - .Returns(() => cache); - var resourceWatcher = new ResourceWatcher( activitySource, logger, - serviceProvider, - timedEntityQueue, + reconciler, operatorSettings, labelSelector, - cacheProvider, kubernetesClient); // Act. From f1d20f665245aad2b33150f13f747e10b66211d3 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Wed, 10 Sep 2025 16:19:25 +0200 Subject: [PATCH 15/58] feat(operator): add `IReconciler` registration in `OperatorBuilder` - Registered `IReconciler` and its implementation `Reconciler` in the service container. - Ensured proper integration with existing requeue and entity processing workflows. --- src/KubeOps.Operator/Builder/OperatorBuilder.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/KubeOps.Operator/Builder/OperatorBuilder.cs b/src/KubeOps.Operator/Builder/OperatorBuilder.cs index 1c066729..2e4a7dca 100644 --- a/src/KubeOps.Operator/Builder/OperatorBuilder.cs +++ b/src/KubeOps.Operator/Builder/OperatorBuilder.cs @@ -20,6 +20,7 @@ using KubeOps.Operator.Finalizer; using KubeOps.Operator.LeaderElection; using KubeOps.Operator.Queue; +using KubeOps.Operator.Reconciliation; using KubeOps.Operator.Watcher; using Microsoft.Extensions.DependencyInjection; @@ -47,6 +48,7 @@ public IOperatorBuilder AddController() Services.AddHostedService>(); Services.TryAddScoped, TImplementation>(); Services.TryAddSingleton(new TimedEntityQueue()); + Services.TryAddSingleton, Reconciler>(); Services.TryAddTransient(); Services.TryAddTransient>(services => services.GetRequiredService().Create()); @@ -71,6 +73,7 @@ public IOperatorBuilder AddController( Services.AddHostedService>(); Services.TryAddScoped, TImplementation>(); Services.TryAddSingleton(new TimedEntityQueue()); + Services.TryAddSingleton, Reconciler>(); Services.TryAddTransient(); Services.TryAddTransient>(services => services.GetRequiredService().Create()); From 4fc59ebfb0a23bc1f775c324e07121cecc0e264d Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Thu, 11 Sep 2025 16:41:34 +0200 Subject: [PATCH 16/58] feat(reconciler): enhance finalizer management with configurable auto-attach/detach options - Added `AutoAttachFinalizers` and `AutoDetachFinalizers` settings in `OperatorSettings`, enabling automatic management of entity finalizers during reconciliation. - Extended `Reconciler` to respect these settings for adding and removing finalizers. - Introduced `EntityFinalizerExtensions` for streamlined finalizer handling and identifier generation. - Updated relevant interfaces and documentation for improved clarity and usability. --- .../Builder/OperatorSettings.cs | 13 +++++++ .../Entities/KubernetesExtensions.cs | 10 +++++ .../Finalizer/EntityFinalizerExtensions.cs | 38 +++++++++++++++++++ .../Reconciliation/IReconciler{TEntity}.cs | 26 +++++++++++++ .../Reconciliation/Reconciler.cs | 37 +++++++++++++++++- 5 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 src/KubeOps.Abstractions/Finalizer/EntityFinalizerExtensions.cs diff --git a/src/KubeOps.Abstractions/Builder/OperatorSettings.cs b/src/KubeOps.Abstractions/Builder/OperatorSettings.cs index d02b642e..5f3f98b5 100644 --- a/src/KubeOps.Abstractions/Builder/OperatorSettings.cs +++ b/src/KubeOps.Abstractions/Builder/OperatorSettings.cs @@ -73,6 +73,19 @@ public sealed partial class OperatorSettings /// public Action? ConfigureResourceWatcherEntityCache { get; set; } + /// + /// Indicates whether finalizers should be automatically attached to Kubernetes entities during reconciliation. + /// When enabled, the operator will ensure that all defined finalizers for the entity are added if they are not already present. + /// Defaults to true. + /// + public bool AutoAttachFinalizers { get; set; } = true; + + /// + /// Indicates whether finalizers should be automatically removed from Kubernetes resources + /// upon successful completion of their finalization process. Defaults to true. + /// + public bool AutoDetachFinalizers { get; set; } = true; + [GeneratedRegex(@"(\W|_)", RegexOptions.CultureInvariant)] private static partial Regex OperatorNameRegex(); } diff --git a/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs b/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs index f9304aab..6199fc50 100644 --- a/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs +++ b/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. +using System.Reflection; + using k8s; using k8s.Models; @@ -89,6 +91,14 @@ public static V1OwnerReference MakeOwnerReference(this IKubernetesObject + /// Retrieves the applied to the specified Kubernetes object type, if it exists. + /// + /// The Kubernetes object to inspect for the attribute. + /// The if found; otherwise, null. + public static KubernetesEntityAttribute? GetKubernetesEntityAttribute(this IKubernetesObject entity) + => entity.GetType().GetCustomAttribute(true); + private static IList EnsureOwnerReferences(this V1ObjectMeta meta) => meta.OwnerReferences ??= new List(); } diff --git a/src/KubeOps.Abstractions/Finalizer/EntityFinalizerExtensions.cs b/src/KubeOps.Abstractions/Finalizer/EntityFinalizerExtensions.cs new file mode 100644 index 00000000..6c24d1c2 --- /dev/null +++ b/src/KubeOps.Abstractions/Finalizer/EntityFinalizerExtensions.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Entities; + +namespace KubeOps.Abstractions.Finalizer; + +/// +/// Provides extension methods for handling entity finalizers in Kubernetes resources. +/// +public static class EntityFinalizerExtensions +{ + private const byte MaxNameLength = 63; + + /// + /// Generates a unique identifier name for the finalizer of a given Kubernetes entity. + /// The identifier includes the group of the entity and the name of the finalizer, ensuring it conforms to Kubernetes naming conventions. + /// + /// The type of the Kubernetes entity. Must implement . + /// The finalizer implementing for which the identifier is generated. + /// The Kubernetes entity associated with the finalizer. + /// A string representing the unique identifier for the finalizer, truncated if it exceeds the maximum allowed length for Kubernetes names. + public static string GetIdentifierName(this IEntityFinalizer finalizer, TEntity entity) + where TEntity : IKubernetesObject + { + var finalizerName = finalizer.GetType().Name.ToLowerInvariant(); + finalizerName = finalizerName.EndsWith("finalizer") ? string.Empty : "finalizer"; + + var entityGroupName = entity.GetKubernetesEntityAttribute()?.Group ?? string.Empty; + var name = $"{entityGroupName}/{finalizerName}".TrimStart('/'); + + return name.Length > MaxNameLength ? name[..MaxNameLength] : name; + } +} diff --git a/src/KubeOps.Operator/Reconciliation/IReconciler{TEntity}.cs b/src/KubeOps.Operator/Reconciliation/IReconciler{TEntity}.cs index 196d6213..6660d2a2 100644 --- a/src/KubeOps.Operator/Reconciliation/IReconciler{TEntity}.cs +++ b/src/KubeOps.Operator/Reconciliation/IReconciler{TEntity}.cs @@ -9,12 +9,38 @@ namespace KubeOps.Operator.Reconciliation; +/// +/// Defines methods for handling reconciliation processes related to Kubernetes resources. +/// This interface provides the necessary functionality for handling the lifecycle events +/// of a resource, such as creation, modification, and deletion. +/// +/// +/// The type of the Kubernetes resource, which must implement . +/// public interface IReconciler where TEntity : IKubernetesObject { + /// + /// Handles the reconciliation process when a new entity is created. + /// + /// The entity to reconcile during its creation. + /// A token to monitor for cancellation requests. + /// A task representing the asynchronous operation, with a result of the reconciliation process. Task> ReconcileCreation(TEntity entity, CancellationToken cancellationToken); + /// + /// Handles the reconciliation process when an existing entity is modified. + /// + /// The entity to reconcile after modification. + /// A token to monitor for cancellation requests. + /// A task representing the asynchronous operation, with a result of the reconciliation process. Task> ReconcileModification(TEntity entity, CancellationToken cancellationToken); + /// + /// Handles the reconciliation process when an entity is deleted. + /// + /// The entity to reconcile during its deletion. + /// A token to monitor for cancellation requests. + /// A task representing the asynchronous operation, with a result of the reconciliation process. Task> ReconcileDeletion(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index 1fcf5d13..1303592d 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -5,6 +5,7 @@ using k8s; using k8s.Models; +using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Finalizer; using KubeOps.Abstractions.Queue; @@ -19,10 +20,25 @@ namespace KubeOps.Operator.Reconciliation; +/// +/// The Reconciler class provides mechanisms for handling creation, modification, and deletion +/// events for Kubernetes objects of the specified entity type. It implements the IReconciler +/// interface and facilitates the reconciliation of desired and actual states of the entity. +/// +/// +/// The type of the Kubernetes entity being reconciled. Must implement IKubernetesObject +/// with V1ObjectMeta. +/// +/// +/// This class leverages logging, caching, and client services to manage and process +/// Kubernetes objects effectively. It also uses internal queuing capabilities for entity +/// processing and requeuing. +/// internal sealed class Reconciler( ILogger> logger, IFusionCacheProvider cacheProvider, IServiceProvider provider, + OperatorSettings settings, TimedEntityQueue requeue, IKubernetesClient client) : IReconciler @@ -161,8 +177,12 @@ private async Task> ReconcileFinalizersSequentialAsync(TEntity e } entity = result.Entity; - entity.RemoveFinalizer(identifier); - entity = await client.UpdateAsync(entity, cancellationToken); + + if (settings.AutoDetachFinalizers) + { + entity.RemoveFinalizer(identifier); + entity = await client.UpdateAsync(entity, cancellationToken); + } logger.LogInformation( """Entity "{Kind}/{Name}" finalized with "{Finalizer}".""", @@ -176,6 +196,19 @@ private async Task> ReconcileFinalizersSequentialAsync(TEntity e private async Task> ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) { await using var scope = provider.CreateAsyncScope(); + + if (settings.AutoAttachFinalizers) + { + var finalizers = scope.ServiceProvider.GetService>>() ?? []; + + foreach (var finalizer in finalizers) + { + entity.AddFinalizer(finalizer.GetIdentifierName(entity)); + } + + entity = await client.UpdateAsync(entity, cancellationToken); + } + var controller = scope.ServiceProvider.GetRequiredService>(); return await controller.ReconcileAsync(entity, cancellationToken); } From fda45f21965718634a203eb112b60208d353026a Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 12 Sep 2025 11:11:23 +0200 Subject: [PATCH 17/58] refactor(generator): enhance syntax model resolution to support constant values - Update `KubernetesEntitySyntaxReceiver` to utilize `SemanticModel` for attribute argument resolution, ensuring accurate value retrieval. --- .../KubernetesEntitySyntaxReceiver.cs | 26 ++++++++---- .../FinalizerRegistrationGenerator.Test.cs | 40 ++++++++++++++++++- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/src/KubeOps.Generator/SyntaxReceiver/KubernetesEntitySyntaxReceiver.cs b/src/KubeOps.Generator/SyntaxReceiver/KubernetesEntitySyntaxReceiver.cs index f98bcc0d..6d1dd4a2 100644 --- a/src/KubeOps.Generator/SyntaxReceiver/KubernetesEntitySyntaxReceiver.cs +++ b/src/KubeOps.Generator/SyntaxReceiver/KubernetesEntitySyntaxReceiver.cs @@ -28,15 +28,27 @@ public void OnVisitSyntaxNode(GeneratorSyntaxContext context) Entities.Add(new( cls, - GetArgumentValue(attr, KindName) ?? cls.Identifier.ToString(), - GetArgumentValue(attr, VersionName) ?? DefaultVersion, - GetArgumentValue(attr, GroupName), - GetArgumentValue(attr, PluralName))); + GetArgumentValue(context.SemanticModel, attr, KindName) ?? cls.Identifier.ToString(), + GetArgumentValue(context.SemanticModel, attr, VersionName) ?? DefaultVersion, + GetArgumentValue(context.SemanticModel, attr, GroupName), + GetArgumentValue(context.SemanticModel, attr, PluralName))); } - private static string? GetArgumentValue(AttributeSyntax attr, string argName) => - attr.ArgumentList?.Arguments.FirstOrDefault(a => a.NameEquals?.Name.ToString() == argName) is - { Expression: LiteralExpressionSyntax { Token.ValueText: { } value } } + private static string? GetArgumentValue(SemanticModel model, AttributeSyntax attr, string argName) + { + if (attr.ArgumentList?.Arguments.FirstOrDefault(a => a.NameEquals?.Name.ToString() == argName) + is not { Expression: { } expr }) + { + return null; + } + + if (model.GetConstantValue(expr) is { HasValue: true, Value: string s }) + { + return s; + } + + return expr is LiteralExpressionSyntax { Token.ValueText: { } value } ? value : null; + } } diff --git a/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs b/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs index 6fa73317..7d00f7fe 100644 --- a/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs +++ b/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs @@ -13,7 +13,7 @@ namespace KubeOps.Generator.Test; -public class FinalizerRegistrationGeneratorTest +public sealed class FinalizerRegistrationGeneratorTest { [Theory] [InlineData("", """ @@ -36,6 +36,44 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) using k8s; using k8s.Models; using KubeOps.Abstractions.Finalizer; + + public static class Constants + { + public const string Group = "testing.dev"; + public const string ApiVersion = "v1"; + public const string Kind = "TestEntity"; + } + + [KubernetesEntity(Group = Constants.Group, ApiVersion = Constants.ApiVersion, Kind = Constants.Kind)] + public sealed class V1TestEntity : IKubernetesObject + { + } + + public sealed class V1TestEntityFinalizer : IEntityFinalizer + { + } + """, """ + // + // This code was generated by a tool. + // Changes to this file may cause incorrect behavior and will be lost if the code is regenerated. + // + #pragma warning disable CS1591 + using KubeOps.Abstractions.Builder; + + public static class FinalizerRegistrations + { + public const string V1TestEntityFinalizerIdentifier = "testing.dev/v1testentityfinalizer"; + public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) + { + builder.AddFinalizer(V1TestEntityFinalizerIdentifier); + return builder; + } + } + """)] + [InlineData(""" + using k8s; + using k8s.Models; + using KubeOps.Abstractions.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject From e875b837432b8deb9ec2b5168ae058799dfcecd1 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 12 Sep 2025 12:39:48 +0200 Subject: [PATCH 18/58] feat(finalizer): fix identifier generation and add unit tests - Updated `EntityFinalizerExtensions` to correctly append "finalizer" when missing from the name. - Added unit tests to validate finalizer identifier generation, including cases for length limits and naming consistency. --- .../Finalizer/EntityFinalizerExtensions.cs | 2 +- .../EntityFinalizerExtensions.Test.cs | 113 ++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 test/KubeOps.Abstractions.Test/Finalizer/EntityFinalizerExtensions.Test.cs diff --git a/src/KubeOps.Abstractions/Finalizer/EntityFinalizerExtensions.cs b/src/KubeOps.Abstractions/Finalizer/EntityFinalizerExtensions.cs index 6c24d1c2..2f71af7e 100644 --- a/src/KubeOps.Abstractions/Finalizer/EntityFinalizerExtensions.cs +++ b/src/KubeOps.Abstractions/Finalizer/EntityFinalizerExtensions.cs @@ -28,7 +28,7 @@ public static string GetIdentifierName(this IEntityFinalizer f where TEntity : IKubernetesObject { var finalizerName = finalizer.GetType().Name.ToLowerInvariant(); - finalizerName = finalizerName.EndsWith("finalizer") ? string.Empty : "finalizer"; + finalizerName = finalizerName.EndsWith("finalizer") ? finalizerName : $"{finalizerName}finalizer"; var entityGroupName = entity.GetKubernetesEntityAttribute()?.Group ?? string.Empty; var name = $"{entityGroupName}/{finalizerName}".TrimStart('/'); diff --git a/test/KubeOps.Abstractions.Test/Finalizer/EntityFinalizerExtensions.Test.cs b/test/KubeOps.Abstractions.Test/Finalizer/EntityFinalizerExtensions.Test.cs new file mode 100644 index 00000000..5d374c9e --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Finalizer/EntityFinalizerExtensions.Test.cs @@ -0,0 +1,113 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Finalizer; + +namespace KubeOps.Abstractions.Test.Finalizer; + +public sealed class EntityFinalizerExtensions +{ + private const string Group = "finalizer.test"; + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Entity_Group_Has_String_Value() + { + var sut = new EntityWithStringValueFinalizer(); + var entity = new EntityFinalizerTestEntityWithStringValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be("finalizer.test/entitywithstringvaluefinalizer"); + } + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Entity_Group_Has_Const_Value() + { + var sut = new EntityWithConstValueFinalizer(); + var entity = new EntityFinalizerTestEntityWithConstValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be($"{Group}/entitywithconstvaluefinalizer"); + } + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Finalizer_Not_Ending_With_Finalizer() + { + var sut = new EntityFinalizerNotEndingOnFinalizerTest(); + var entity = new EntityFinalizerTestEntityWithConstValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be($"{Group}/entityfinalizernotendingonfinalizertestfinalizer"); + } + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Finalizer_Identifier_Would_Be_Greater_Than_63_Characters() + { + var sut = new EntityFinalizerWithATotalIdentifierNameHavingALengthGreaterThan63(); + var entity = new EntityFinalizerTestEntityWithConstValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be($"{Group}/entityfinalizerwithatotalidentifiernamehavingale"); + identifierName.Length.Should().Be(63); + } + + private sealed class EntityFinalizerWithATotalIdentifierNameHavingALengthGreaterThan63 + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityFinalizerTestEntityWithConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); + } + + private sealed class EntityFinalizerNotEndingOnFinalizerTest + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityFinalizerTestEntityWithConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); + } + + private sealed class EntityWithStringValueFinalizer + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityFinalizerTestEntityWithStringValue entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); + } + + private sealed class EntityWithConstValueFinalizer + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityFinalizerTestEntityWithConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); + } + + [KubernetesEntity(Group = "finalizer.test", ApiVersion = "v1", Kind = "FinalizerTest")] + private sealed class EntityFinalizerTestEntityWithStringValue + : IKubernetesObject + { + public string ApiVersion { get; set; } = "finalizer.test/v1"; + + public string Kind { get; set; } = "FinalizerTest"; + + public V1ObjectMeta Metadata { get; set; } = new(); + } + + [KubernetesEntity(Group = Group, ApiVersion = "v1", Kind = "FinalizerTest")] + private sealed class EntityFinalizerTestEntityWithConstValue + : IKubernetesObject + { + public string ApiVersion { get; set; } = "finalizer.test/v1"; + + public string Kind { get; set; } = "FinalizerTest"; + + public V1ObjectMeta Metadata { get; set; } = new(); + } +} From 84d97efbd98670719d8e446abe0634d63d0282b3 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 12 Sep 2025 12:55:01 +0200 Subject: [PATCH 19/58] test(finalizer): update and expand unit tests for entity finalizers - Renamed test cases and entities for improved clarity and consistency. - Added new tests for entities with no group values and entities with varying group definitions. - Adjusted expected --- .gitignore | 1 + .../EntityFinalizerExtensions.Test.cs | 83 +++++++++++++------ 2 files changed, 57 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index a5eddaf2..da9560b0 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ bld/ coverage.json coverage.info .vs +[Tt]est[Rr]esults/ # Docs _site diff --git a/test/KubeOps.Abstractions.Test/Finalizer/EntityFinalizerExtensions.Test.cs b/test/KubeOps.Abstractions.Test/Finalizer/EntityFinalizerExtensions.Test.cs index 5d374c9e..8f9e148b 100644 --- a/test/KubeOps.Abstractions.Test/Finalizer/EntityFinalizerExtensions.Test.cs +++ b/test/KubeOps.Abstractions.Test/Finalizer/EntityFinalizerExtensions.Test.cs @@ -19,41 +19,52 @@ public sealed class EntityFinalizerExtensions [Fact] public void GetIdentifierName_Should_Return_Correct_Name_When_Entity_Group_Has_String_Value() { - var sut = new EntityWithStringValueFinalizer(); - var entity = new EntityFinalizerTestEntityWithStringValue(); + var sut = new EntityWithGroupAsStringValueFinalizer(); + var entity = new EntityWithGroupAsStringValue(); var identifierName = sut.GetIdentifierName(entity); - identifierName.Should().Be("finalizer.test/entitywithstringvaluefinalizer"); + identifierName.Should().Be("finalizer.test/entitywithgroupasstringvaluefinalizer"); } [Fact] public void GetIdentifierName_Should_Return_Correct_Name_When_Entity_Group_Has_Const_Value() { - var sut = new EntityWithConstValueFinalizer(); - var entity = new EntityFinalizerTestEntityWithConstValue(); + var sut = new EntityWithGroupAsConstValueFinalizer(); + var entity = new EntityWithGroupAsConstValue(); var identifierName = sut.GetIdentifierName(entity); - identifierName.Should().Be($"{Group}/entitywithconstvaluefinalizer"); + identifierName.Should().Be($"{Group}/entitywithgroupasconstvaluefinalizer"); + } + + [Fact] + public void GetIdentifierName_Should_Return_Correct_Name_When_Entity_Group_Has_No_Value() + { + var sut = new EntityWithNoGroupFinalizer(); + var entity = new EntityWithNoGroupValue(); + + var identifierName = sut.GetIdentifierName(entity); + + identifierName.Should().Be("entitywithnogroupfinalizer"); } [Fact] public void GetIdentifierName_Should_Return_Correct_Name_When_Finalizer_Not_Ending_With_Finalizer() { - var sut = new EntityFinalizerNotEndingOnFinalizerTest(); - var entity = new EntityFinalizerTestEntityWithConstValue(); + var sut = new EntityFinalizerNotEndingOnFinalizer1(); + var entity = new EntityWithGroupAsConstValue(); var identifierName = sut.GetIdentifierName(entity); - identifierName.Should().Be($"{Group}/entityfinalizernotendingonfinalizertestfinalizer"); + identifierName.Should().Be($"{Group}/entityfinalizernotendingonfinalizer1finalizer"); } [Fact] public void GetIdentifierName_Should_Return_Correct_Name_When_Finalizer_Identifier_Would_Be_Greater_Than_63_Characters() { var sut = new EntityFinalizerWithATotalIdentifierNameHavingALengthGreaterThan63(); - var entity = new EntityFinalizerTestEntityWithConstValue(); + var entity = new EntityWithGroupAsConstValue(); var identifierName = sut.GetIdentifierName(entity); @@ -62,35 +73,42 @@ public void GetIdentifierName_Should_Return_Correct_Name_When_Finalizer_Identifi } private sealed class EntityFinalizerWithATotalIdentifierNameHavingALengthGreaterThan63 - : IEntityFinalizer + : IEntityFinalizer + { + public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); + } + + private sealed class EntityFinalizerNotEndingOnFinalizer1 + : IEntityFinalizer { - public Task> FinalizeAsync(EntityFinalizerTestEntityWithConstValue entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } - private sealed class EntityFinalizerNotEndingOnFinalizerTest - : IEntityFinalizer + private sealed class EntityWithGroupAsStringValueFinalizer + : IEntityFinalizer { - public Task> FinalizeAsync(EntityFinalizerTestEntityWithConstValue entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> FinalizeAsync(EntityWithGroupAsStringValue entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } - private sealed class EntityWithStringValueFinalizer - : IEntityFinalizer + private sealed class EntityWithGroupAsConstValueFinalizer + : IEntityFinalizer { - public Task> FinalizeAsync(EntityFinalizerTestEntityWithStringValue entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } - private sealed class EntityWithConstValueFinalizer - : IEntityFinalizer + private sealed class EntityWithNoGroupFinalizer + : IEntityFinalizer { - public Task> FinalizeAsync(EntityFinalizerTestEntityWithConstValue entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> FinalizeAsync(EntityWithNoGroupValue entity, CancellationToken cancellationToken) + => Task.FromResult(Result.ForSuccess(entity)); } [KubernetesEntity(Group = "finalizer.test", ApiVersion = "v1", Kind = "FinalizerTest")] - private sealed class EntityFinalizerTestEntityWithStringValue + private sealed class EntityWithGroupAsStringValue : IKubernetesObject { public string ApiVersion { get; set; } = "finalizer.test/v1"; @@ -101,7 +119,18 @@ private sealed class EntityFinalizerTestEntityWithStringValue } [KubernetesEntity(Group = Group, ApiVersion = "v1", Kind = "FinalizerTest")] - private sealed class EntityFinalizerTestEntityWithConstValue + private sealed class EntityWithGroupAsConstValue + : IKubernetesObject + { + public string ApiVersion { get; set; } = "finalizer.test/v1"; + + public string Kind { get; set; } = "FinalizerTest"; + + public V1ObjectMeta Metadata { get; set; } = new(); + } + + [KubernetesEntity] + private sealed class EntityWithNoGroupValue : IKubernetesObject { public string ApiVersion { get; set; } = "finalizer.test/v1"; From d6d4b340ba2193a8b9ec9a7219bcf1babbc1d903 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Mon, 15 Sep 2025 12:44:19 +0200 Subject: [PATCH 20/58] refactor(result): make `ErrorMessage` readonly and allow setting `RequeueAfter` --- src/KubeOps.Abstractions/Controller/Result.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/KubeOps.Abstractions/Controller/Result.cs b/src/KubeOps.Abstractions/Controller/Result.cs index dab05298..8f9d72e6 100644 --- a/src/KubeOps.Abstractions/Controller/Result.cs +++ b/src/KubeOps.Abstractions/Controller/Result.cs @@ -25,11 +25,11 @@ private Result(TEntity entity, bool isSuccess, string? errorMessage, Exception? [MemberNotNullWhen(true, nameof(ErrorMessage))] public bool IsFailure => !IsSuccess; - public string? ErrorMessage { get; set; } + public string? ErrorMessage { get; } public Exception? Error { get; } - public TimeSpan? RequeueAfter { get; } + public TimeSpan? RequeueAfter { get; set; } public static Result ForSuccess(TEntity entity, TimeSpan? requeueAfter = null) { From 5c935aa4666a9272fad4b2c19d4db7d2599089d2 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Mon, 15 Sep 2025 13:08:18 +0200 Subject: [PATCH 21/58] refactor(queue): replace `TimedEntityQueue` with `ITimedEntityQueue` interface for improved flexibility - Extracted `ITimedEntityQueue` interface from `TimedEntityQueue` implementation. - Updated all usages, including services and tests, to rely on the interface. - Added extension methods for requeue key management. - Improved code consistency and maintainability across the queue system. --- .../Builder/OperatorBuilder.cs | 4 +- .../Queue/EntityRequeueBackgroundService.cs | 3 +- .../Queue/ITimedEntityQueue.cs | 35 +++++++++++++++ .../Queue/KubeOpsEntityRequeueFactory.cs | 2 +- .../Queue/TimedEntityQueue.cs | 32 +++---------- .../Queue/TimedEntityQueueExtensions.cs | 45 +++++++++++++++++++ .../Reconciliation/Reconciler.cs | 4 +- .../Builder/OperatorBuilder.Test.cs | 4 +- 8 files changed, 96 insertions(+), 33 deletions(-) create mode 100644 src/KubeOps.Operator/Queue/ITimedEntityQueue.cs create mode 100644 src/KubeOps.Operator/Queue/TimedEntityQueueExtensions.cs diff --git a/src/KubeOps.Operator/Builder/OperatorBuilder.cs b/src/KubeOps.Operator/Builder/OperatorBuilder.cs index 2e4a7dca..fdb224ca 100644 --- a/src/KubeOps.Operator/Builder/OperatorBuilder.cs +++ b/src/KubeOps.Operator/Builder/OperatorBuilder.cs @@ -47,7 +47,7 @@ public IOperatorBuilder AddController() { Services.AddHostedService>(); Services.TryAddScoped, TImplementation>(); - Services.TryAddSingleton(new TimedEntityQueue()); + Services.TryAddSingleton, TimedEntityQueue>(); Services.TryAddSingleton, Reconciler>(); Services.TryAddTransient(); Services.TryAddTransient>(services => @@ -72,7 +72,7 @@ public IOperatorBuilder AddController( { Services.AddHostedService>(); Services.TryAddScoped, TImplementation>(); - Services.TryAddSingleton(new TimedEntityQueue()); + Services.TryAddSingleton, TimedEntityQueue>(); Services.TryAddSingleton, Reconciler>(); Services.TryAddTransient(); Services.TryAddTransient>(services => diff --git a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs index c6b5c953..0339db13 100644 --- a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs +++ b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs @@ -16,7 +16,7 @@ namespace KubeOps.Operator.Queue; internal sealed class EntityRequeueBackgroundService( IKubernetesClient client, - TimedEntityQueue queue, + ITimedEntityQueue queue, IReconciler reconciler, ILogger> logger) : IHostedService, IDisposable, IAsyncDisposable where TEntity : IKubernetesObject @@ -67,6 +67,7 @@ public async ValueTask DisposeAsync() await CastAndDispose(queue); _disposed = true; + return; static async ValueTask CastAndDispose(IDisposable resource) { diff --git a/src/KubeOps.Operator/Queue/ITimedEntityQueue.cs b/src/KubeOps.Operator/Queue/ITimedEntityQueue.cs new file mode 100644 index 00000000..c74fe96d --- /dev/null +++ b/src/KubeOps.Operator/Queue/ITimedEntityQueue.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Queue; + +namespace KubeOps.Operator.Queue; + +/// +/// Represents a timed queue for managing Kubernetes entities of type . +/// This interface provides mechanisms to enqueue entities for later processing and remove entities from the queue. +/// +/// +/// The type of the Kubernetes entity. Must implement . +/// +public interface ITimedEntityQueue : IDisposable, IAsyncEnumerable> + where TEntity : IKubernetesObject +{ + /// + /// Adds the specified entity to the queue for processing after the specified time span has expired. + /// + /// The entity to be added to the queue. + /// The type of requeue operation to be performed. + /// The duration after which the entity should be requeued. + void Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn); + + /// + /// Removes the specified from the queue. + /// + /// The entity to be removed from the queue. + void Remove(TEntity entity); +} diff --git a/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs b/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs index b729e233..ec53d551 100644 --- a/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs +++ b/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs @@ -20,7 +20,7 @@ public EntityRequeue Create() (entity, type, timeSpan) => { var logger = services.GetService>>(); - var queue = services.GetRequiredService>(); + var queue = services.GetRequiredService>(); logger?.LogTrace( """Requeue entity "{Kind}/{Name}" in {Milliseconds}ms.""", diff --git a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs index 1ca2b67c..e1f151ad 100644 --- a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs +++ b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs @@ -16,7 +16,7 @@ namespace KubeOps.Operator.Queue; /// The given enumerable only contains items that should be considered for reconciliations. /// /// The type of the inner entity. -public sealed class TimedEntityQueue : IDisposable +public sealed class TimedEntityQueue : ITimedEntityQueue where TEntity : IKubernetesObject { // A shared task factory for all the created tasks. @@ -30,17 +30,11 @@ public sealed class TimedEntityQueue : IDisposable internal int Count => _management.Count; - /// - /// Enqueues the given to happen in . - /// If the item already exists, the existing entry is updated. - /// - /// The entity. - /// The type of which the reconcile operation should be executed. - /// The time after , where the item is reevaluated again. + /// public void Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn) { _management.AddOrUpdate( - TimedEntityQueue.GetKey(entity) ?? throw new InvalidOperationException("Cannot enqueue entities without name."), + this.GetKey(entity) ?? throw new InvalidOperationException("Cannot enqueue entities without name."), key => { var entry = new TimedQueueEntry(entity, type, requeueIn); @@ -68,6 +62,7 @@ public void Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn) }); } + /// public void Dispose() { _queue.Dispose(); @@ -77,6 +72,7 @@ public void Dispose() } } + /// public async IAsyncEnumerator> GetAsyncEnumerator(CancellationToken cancellationToken = default) { await Task.Yield(); @@ -86,9 +82,10 @@ public async IAsyncEnumerator> GetAsyncEnumerator(Cancella } } + /// public void Remove(TEntity entity) { - var key = TimedEntityQueue.GetKey(entity); + var key = this.GetKey(entity); if (key is null) { return; @@ -99,19 +96,4 @@ public void Remove(TEntity entity) task.Cancel(); } } - - private static string? GetKey(TEntity entity) - { - if (entity.Name() is null) - { - return null; - } - - if (entity.Namespace() is null) - { - return entity.Name(); - } - - return $"{entity.Namespace()}/{entity.Name()}"; - } } diff --git a/src/KubeOps.Operator/Queue/TimedEntityQueueExtensions.cs b/src/KubeOps.Operator/Queue/TimedEntityQueueExtensions.cs new file mode 100644 index 00000000..3ac377d8 --- /dev/null +++ b/src/KubeOps.Operator/Queue/TimedEntityQueueExtensions.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Operator.Queue; + +/// +/// Provides extension methods for the interface. +/// +public static class TimedEntityQueueExtensions +{ + /// + /// Retrieves a unique key for the specified Kubernetes entity. The key is constructed + /// using the entity's namespace and name, if available. If the entity does not have + /// a valid name, the method returns null. + /// + /// + /// The type of the Kubernetes entity. Must implement . + /// + /// + /// The timed entity queue from which the key should be derived. + /// + /// + /// The Kubernetes entity for which the key will be retrieved. + /// + /// + /// A string representing the unique key for the entity, or null if the entity does not have a valid name. + /// + // ReSharper disable once UnusedParameter.Global + public static string? GetKey(this ITimedEntityQueue queue, TEntity entity) + where TEntity : IKubernetesObject + { + if (string.IsNullOrWhiteSpace(entity.Name())) + { + return null; + } + + return string.IsNullOrWhiteSpace(entity.Namespace()) + ? entity.Name() + : $"{entity.Namespace()}/{entity.Name()}"; + } +} diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index 1303592d..34a7a5cc 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -39,7 +39,7 @@ internal sealed class Reconciler( IFusionCacheProvider cacheProvider, IServiceProvider provider, OperatorSettings settings, - TimedEntityQueue requeue, + ITimedEntityQueue requeue, IKubernetesClient client) : IReconciler where TEntity : IKubernetesObject @@ -91,7 +91,7 @@ public async Task> ReconcileModification(TEntity entity, Cancell case { Metadata.DeletionTimestamp: null }: var cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); - // Check if entity spec has changed through "Generation" value increment. Skip reconcile if not changed. + // Check if entity-spec has changed through "Generation" value increment. Skip reconcile if not changed. if (cachedGeneration.HasValue && cachedGeneration >= entity.Generation()) { logger.LogDebug( diff --git a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs index 93875215..67935e66 100644 --- a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs +++ b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs @@ -74,7 +74,7 @@ public void Should_Add_Controller_Resources() s.ImplementationType == typeof(ResourceWatcher) && s.Lifetime == ServiceLifetime.Singleton); _builder.Services.Should().Contain(s => - s.ServiceType == typeof(TimedEntityQueue) && + s.ServiceType == typeof(ITimedEntityQueue) && s.Lifetime == ServiceLifetime.Singleton); _builder.Services.Should().Contain(s => s.ServiceType == typeof(EntityRequeue) && @@ -95,7 +95,7 @@ public void Should_Add_Controller_Resources_With_Label_Selector() s.ImplementationType == typeof(ResourceWatcher) && s.Lifetime == ServiceLifetime.Singleton); _builder.Services.Should().Contain(s => - s.ServiceType == typeof(TimedEntityQueue) && + s.ServiceType == typeof(ITimedEntityQueue) && s.Lifetime == ServiceLifetime.Singleton); _builder.Services.Should().Contain(s => s.ServiceType == typeof(EntityRequeue) && From b222cac21dc2074c83aa1c08f84ed27796491762 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Mon, 15 Sep 2025 15:31:18 +0200 Subject: [PATCH 22/58] feat(operator): introduce `LeaderElectionType` for configurable leader election - Replaced `EnableLeaderElection` with `LeaderElectionType` in `OperatorSettings` for enhanced configurability. - Added `LeaderElectionType` enum with options: None, Single, and Custom. - Updated `OperatorBuilder` to handle leader election logic based on `LeaderElectionType`. - Modified `EntityRequeueBackgroundService` to public visibility and implemented proper `Dispose` logic. - Adjusted tests to reflect new leader election mechanism. - Improved code maintainability and alignment with distributed system requirements. --- .../Builder/LeaderElectionType.cs | 33 ++++++++ .../Builder/OperatorSettings.cs | 16 +--- .../Builder/OperatorBuilder.cs | 35 +++----- .../Queue/EntityRequeueBackgroundService.cs | 82 ++++++++++++------- .../Builder/OperatorBuilder.Test.cs | 4 +- .../LeaderResourceWatcher.Integration.Test.cs | 3 +- .../LeaderAwareness.Integration.Test.cs | 3 +- 7 files changed, 106 insertions(+), 70 deletions(-) create mode 100644 src/KubeOps.Abstractions/Builder/LeaderElectionType.cs diff --git a/src/KubeOps.Abstractions/Builder/LeaderElectionType.cs b/src/KubeOps.Abstractions/Builder/LeaderElectionType.cs new file mode 100644 index 00000000..7e68a7aa --- /dev/null +++ b/src/KubeOps.Abstractions/Builder/LeaderElectionType.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Abstractions.Builder; + +/// +/// Specifies the types of leader election mechanisms to be used in distributed systems or workloads. +/// +public enum LeaderElectionType +{ + /// + /// Represents the absence of a leader election mechanism. + /// This option is used when no leader election is required, and all instances + /// are expected to operate without coordination or exclusivity. + /// + None = 0, + + /// + /// Represents the leader election mechanism where only a single instance of the application + /// assumes the leader role at any given time. This is used to coordinate operations + /// that require exclusivity or to manage shared resources in distributed systems. + /// + Single = 1, + + /// + /// Represents a custom leader election mechanism determined by the user. + /// This option allows the integration of user-defined logic for handling + /// leader election, enabling tailored coordination strategies beyond the + /// provided defaults. + /// + Custom = 2, +} diff --git a/src/KubeOps.Abstractions/Builder/OperatorSettings.cs b/src/KubeOps.Abstractions/Builder/OperatorSettings.cs index 5f3f98b5..522ab1a9 100644 --- a/src/KubeOps.Abstractions/Builder/OperatorSettings.cs +++ b/src/KubeOps.Abstractions/Builder/OperatorSettings.cs @@ -36,19 +36,11 @@ public sealed partial class OperatorSettings public string? Namespace { get; set; } /// - /// - /// Whether the leader elector should run. You should enable - /// this if you plan to run the operator redundantly. - /// - /// - /// If this is disabled and an operator runs in multiple instances - /// (in the same namespace), it can lead to a "split brain" problem. - /// - /// - /// Defaults to `false`. - /// + /// Defines the type of leader election mechanism to be used by the operator. + /// Determines how resources and controllers are coordinated in a distributed environment. + /// Defaults to indicating no leader election is configured. /// - public bool EnableLeaderElection { get; set; } = false; + public LeaderElectionType LeaderElectionType { get; set; } = LeaderElectionType.None; /// /// Defines how long one lease is valid for any leader. diff --git a/src/KubeOps.Operator/Builder/OperatorBuilder.cs b/src/KubeOps.Operator/Builder/OperatorBuilder.cs index fdb224ca..0e1df1e0 100644 --- a/src/KubeOps.Operator/Builder/OperatorBuilder.cs +++ b/src/KubeOps.Operator/Builder/OperatorBuilder.cs @@ -45,7 +45,6 @@ public IOperatorBuilder AddController() where TImplementation : class, IEntityController where TEntity : IKubernetesObject { - Services.AddHostedService>(); Services.TryAddScoped, TImplementation>(); Services.TryAddSingleton, TimedEntityQueue>(); Services.TryAddSingleton, Reconciler>(); @@ -53,13 +52,16 @@ public IOperatorBuilder AddController() Services.TryAddTransient>(services => services.GetRequiredService().Create()); - if (_settings.EnableLeaderElection) + switch (_settings.LeaderElectionType) { - Services.AddHostedService>(); - } - else - { - Services.AddHostedService>(); + case LeaderElectionType.None: + Services.AddHostedService>(); + Services.AddHostedService>(); + break; + case LeaderElectionType.Single: + Services.AddHostedService>(); + Services.AddHostedService>(); + break; } return this; @@ -70,24 +72,9 @@ public IOperatorBuilder AddController( where TEntity : IKubernetesObject where TLabelSelector : class, IEntityLabelSelector { - Services.AddHostedService>(); - Services.TryAddScoped, TImplementation>(); - Services.TryAddSingleton, TimedEntityQueue>(); - Services.TryAddSingleton, Reconciler>(); - Services.TryAddTransient(); - Services.TryAddTransient>(services => - services.GetRequiredService().Create()); + AddController(); Services.TryAddSingleton, TLabelSelector>(); - if (_settings.EnableLeaderElection) - { - Services.AddHostedService>(); - } - else - { - Services.AddHostedService>(); - } - return this; } @@ -143,7 +130,7 @@ private void AddOperatorBase() Services.AddSingleton(typeof(IEntityLabelSelector<>), typeof(DefaultEntityLabelSelector<>)); - if (_settings.EnableLeaderElection) + if (_settings.LeaderElectionType == LeaderElectionType.Single) { Services.AddLeaderElection(); } diff --git a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs index 0339db13..1e1e1639 100644 --- a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs +++ b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs @@ -14,7 +14,7 @@ namespace KubeOps.Operator.Queue; -internal sealed class EntityRequeueBackgroundService( +public class EntityRequeueBackgroundService( IKubernetesClient client, ITimedEntityQueue queue, IReconciler reconciler, @@ -53,6 +53,23 @@ public Task StopAsync(CancellationToken cancellationToken) public void Dispose() { + Dispose(true); + GC.SuppressFinalize(this); + } + + public async ValueTask DisposeAsync() + { + await DisposeAsync(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposing) + { + return; + } + _cts.Dispose(); client.Dispose(); queue.Dispose(); @@ -60,8 +77,13 @@ public void Dispose() _disposed = true; } - public async ValueTask DisposeAsync() + protected virtual async ValueTask DisposeAsync(bool disposing) { + if (!disposing) + { + return; + } + await CastAndDispose(_cts); await CastAndDispose(client); await CastAndDispose(queue); @@ -82,6 +104,34 @@ static async ValueTask CastAndDispose(IDisposable resource) } } + protected virtual async Task ReconcileSingleAsync(RequeueEntry queuedEntry, CancellationToken cancellationToken) + { + logger.LogTrace("""Execute requested requeued reconciliation for "{Name}".""", queuedEntry.Entity.Name()); + + if (await client.GetAsync(queuedEntry.Entity.Name(), queuedEntry.Entity.Namespace(), cancellationToken) is not + { } entity) + { + logger.LogWarning( + """Requeued entity "{Name}" was not found. Skipping reconciliation.""", queuedEntry.Entity.Name()); + return; + } + + switch (queuedEntry.RequeueType) + { + case RequeueType.Added: + await reconciler.ReconcileCreation(entity, cancellationToken); + break; + case RequeueType.Modified: + await reconciler.ReconcileModification(entity, cancellationToken); + break; + case RequeueType.Deleted: + await reconciler.ReconcileDeletion(entity, cancellationToken); + break; + default: + throw new NotSupportedException($"RequeueType '{queuedEntry.RequeueType}' is not supported!"); + } + } + private async Task WatchAsync(CancellationToken cancellationToken) { await foreach (var entry in queue) @@ -110,32 +160,4 @@ private async Task WatchAsync(CancellationToken cancellationToken) } } } - - private async Task ReconcileSingleAsync(RequeueEntry queuedEntry, CancellationToken cancellationToken) - { - logger.LogTrace("""Execute requested requeued reconciliation for "{Name}".""", queuedEntry.Entity.Name()); - - if (await client.GetAsync(queuedEntry.Entity.Name(), queuedEntry.Entity.Namespace(), cancellationToken) is not - { } entity) - { - logger.LogWarning( - """Requeued entity "{Name}" was not found. Skipping reconciliation.""", queuedEntry.Entity.Name()); - return; - } - - switch (queuedEntry.RequeueType) - { - case RequeueType.Added: - await reconciler.ReconcileCreation(entity, cancellationToken); - break; - case RequeueType.Modified: - await reconciler.ReconcileModification(entity, cancellationToken); - break; - case RequeueType.Deleted: - await reconciler.ReconcileDeletion(entity, cancellationToken); - break; - default: - throw new NotSupportedException($"RequeueType '{queuedEntry.RequeueType}' is not supported!"); - } - } } diff --git a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs index 67935e66..0ee2adbb 100644 --- a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs +++ b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs @@ -123,7 +123,7 @@ public void Should_Add_Finalizer_Resources() [Fact] public void Should_Add_Leader_Elector() { - var builder = new OperatorBuilder(new ServiceCollection(), new() { EnableLeaderElection = true }); + var builder = new OperatorBuilder(new ServiceCollection(), new() { LeaderElectionType = LeaderElectionType.Single }); builder.Services.Should().Contain(s => s.ServiceType == typeof(k8s.LeaderElection.LeaderElector) && s.Lifetime == ServiceLifetime.Singleton); @@ -132,7 +132,7 @@ public void Should_Add_Leader_Elector() [Fact] public void Should_Add_LeaderAwareResourceWatcher() { - var builder = new OperatorBuilder(new ServiceCollection(), new() { EnableLeaderElection = true }); + var builder = new OperatorBuilder(new ServiceCollection(), new() { LeaderElectionType = LeaderElectionType.Single }); builder.AddController(); builder.Services.Should().Contain(s => diff --git a/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs b/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs index 16ff8aba..5b48552e 100644 --- a/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. +using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Controller; using KubeOps.Operator.Test.TestEntities; @@ -14,7 +15,7 @@ public sealed class LeaderAwareHostedServiceDisposeIntegrationTest : HostedServi protected override void ConfigureHost(HostApplicationBuilder builder) { builder.Services - .AddKubernetesOperator(op => op.EnableLeaderElection = true) + .AddKubernetesOperator(op => op.LeaderElectionType = LeaderElectionType.Single) .AddController(); } diff --git a/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs b/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs index e51a3ea5..076889f8 100644 --- a/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs @@ -6,6 +6,7 @@ using k8s.Models; +using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Controller; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -49,7 +50,7 @@ protected override void ConfigureHost(HostApplicationBuilder builder) { builder.Services .AddSingleton(_mock) - .AddKubernetesOperator(s => s.EnableLeaderElection = true) + .AddKubernetesOperator(s => s.LeaderElectionType = LeaderElectionType.Single) .AddController(); } From 88180edc2d9f45a440acd38bade6e57e5aaffc41 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Tue, 16 Sep 2025 10:23:13 +0200 Subject: [PATCH 23/58] refactor(operator): expose `Settings` in `OperatorBuilder` and update its references - Made `OperatorSettings` accessible through a public property in `IOperatorBuilder` and `OperatorBuilder`. - Replaced private field `_settings` with the new `Settings` property throughout the codebase for improved readability and maintainability. - Updated relevant comments and documentation for consistency. --- .../Builder/IOperatorBuilder.cs | 5 +++++ .../Builder/OperatorBuilder.cs | 22 +++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs b/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs index fbc6ad57..d0f7ca47 100644 --- a/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs +++ b/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs @@ -24,6 +24,11 @@ public interface IOperatorBuilder /// IServiceCollection Services { get; } + /// + /// Configuration settings for the operator. + /// + OperatorSettings Settings { get; } + /// /// Add a controller implementation for a specific entity to the operator. /// The metadata for the entity must be added as well. diff --git a/src/KubeOps.Operator/Builder/OperatorBuilder.cs b/src/KubeOps.Operator/Builder/OperatorBuilder.cs index 0e1df1e0..bb089919 100644 --- a/src/KubeOps.Operator/Builder/OperatorBuilder.cs +++ b/src/KubeOps.Operator/Builder/OperatorBuilder.cs @@ -30,17 +30,17 @@ namespace KubeOps.Operator.Builder; internal sealed class OperatorBuilder : IOperatorBuilder { - private readonly OperatorSettings _settings; - public OperatorBuilder(IServiceCollection services, OperatorSettings settings) { - _settings = settings; + Settings = settings; Services = services; AddOperatorBase(); } public IServiceCollection Services { get; } + public OperatorSettings Settings { get; } + public IOperatorBuilder AddController() where TImplementation : class, IEntityController where TEntity : IKubernetesObject @@ -52,7 +52,7 @@ public IOperatorBuilder AddController() Services.TryAddTransient>(services => services.GetRequiredService().Create()); - switch (_settings.LeaderElectionType) + switch (Settings.LeaderElectionType) { case LeaderElectionType.None: Services.AddHostedService>(); @@ -102,19 +102,19 @@ public IOperatorBuilder AddCrdInstaller(Action? configure private void AddOperatorBase() { - Services.AddSingleton(_settings); - Services.AddSingleton(new ActivitySource(_settings.Name)); + Services.AddSingleton(Settings); + Services.AddSingleton(new ActivitySource(Settings.Name)); // add and configure resource watcher entity cache - Services.WithResourceWatcherEntityCaching(_settings); + Services.WithResourceWatcherEntityCaching(Settings); // Add the default configuration and the client separately. This allows external users to override either // just the config (e.g. for integration tests) or to replace the whole client, e.g. with a mock. - // We also add the k8s.IKubernetes as a singleton service, in order to allow to access internal services - // and also external users to make use of it's features that might not be implemented in the adapted client. + // We also add the k8s.IKubernetes as a singleton service, in order to allow accessing internal services + // and also external users to make use of its features that might not be implemented in the adapted client. // // Due to a memory leak in the Kubernetes client, it is important that the client is registered with - // with the same lifetime as the KubernetesClientConfiguration. This is tracked in kubernetes/csharp#1446. + // the same lifetime as the KubernetesClientConfiguration. This is tracked in kubernetes/csharp#1446. // https://github.com/kubernetes-client/csharp/issues/1446 // // The missing ability to inject a custom HTTP client and therefore the possibility to use the .AddHttpClient() @@ -130,7 +130,7 @@ private void AddOperatorBase() Services.AddSingleton(typeof(IEntityLabelSelector<>), typeof(DefaultEntityLabelSelector<>)); - if (_settings.LeaderElectionType == LeaderElectionType.Single) + if (Settings.LeaderElectionType == LeaderElectionType.Single) { Services.AddLeaderElection(); } From 9bee002e11bac3ff2722309d5b6e9fed25ae22f8 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Tue, 16 Sep 2025 12:46:00 +0200 Subject: [PATCH 24/58] feat(queue): add logging to `TimedEntityQueue` and update test implementation - Introduced `ILogger` dependency to `TimedEntityQueue` for improved debugging and observability. - Added trace logs for scheduling and updating entities in the queue. - Adjusted tests to provide mock `ILogger` dependency. - Simplified `_cancellationTokenSource` initialization in `ResourceWatcher`. --- .../Queue/TimedEntityQueue.cs | 73 ++++++++++++------- .../Watcher/ResourceWatcher{TEntity}.cs | 2 +- .../Queue/TimedEntityQueue.Test.cs | 6 +- 3 files changed, 51 insertions(+), 30 deletions(-) diff --git a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs index e1f151ad..864d8312 100644 --- a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs +++ b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs @@ -9,6 +9,8 @@ using KubeOps.Abstractions.Queue; +using Microsoft.Extensions.Logging; + namespace KubeOps.Operator.Queue; /// @@ -16,7 +18,9 @@ namespace KubeOps.Operator.Queue; /// The given enumerable only contains items that should be considered for reconciliations. /// /// The type of the inner entity. -public sealed class TimedEntityQueue : ITimedEntityQueue +public sealed class TimedEntityQueue( + ILogger> logger) + : ITimedEntityQueue where TEntity : IKubernetesObject { // A shared task factory for all the created tasks. @@ -33,33 +37,46 @@ public sealed class TimedEntityQueue : ITimedEntityQueue /// public void Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn) { - _management.AddOrUpdate( - this.GetKey(entity) ?? throw new InvalidOperationException("Cannot enqueue entities without name."), - key => - { - var entry = new TimedQueueEntry(entity, type, requeueIn); - _scheduledEntries.StartNew( - async () => - { - await entry.AddAfterDelay(_queue); - _management.TryRemove(key, out _); - }, - entry.Token); - return entry; - }, - (key, oldEntry) => - { - oldEntry.Cancel(); - var entry = new TimedQueueEntry(entity, type, requeueIn); - _scheduledEntries.StartNew( - async () => - { - await entry.AddAfterDelay(_queue); - _management.TryRemove(key, out _); - }, - entry.Token); - return entry; - }); + _management + .AddOrUpdate( + this.GetKey(entity) ?? throw new InvalidOperationException("Cannot enqueue entities without name."), + key => + { + logger.LogTrace( + """Adding schedule for entity "{Kind}/{Name}" to reconcile in {Seconds}s.""", + entity.Kind, + entity.Name(), + requeueIn.TotalSeconds); + + var entry = new TimedQueueEntry(entity, type, requeueIn); + _scheduledEntries.StartNew( + async () => + { + await entry.AddAfterDelay(_queue); + _management.TryRemove(key, out _); + }, + entry.Token); + return entry; + }, + (key, oldEntry) => + { + logger.LogTrace( + """Updating schedule for entity "{Kind}/{Name}" to reconcile in {Seconds}s.""", + entity.Kind, + entity.Name(), + requeueIn.TotalSeconds); + + oldEntry.Cancel(); + var entry = new TimedQueueEntry(entity, type, requeueIn); + _scheduledEntries.StartNew( + async () => + { + await entry.AddAfterDelay(_queue); + _management.TryRemove(key, out _); + }, + entry.Token); + return entry; + }); } /// diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index 6d241d34..7184675d 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -46,7 +46,7 @@ public virtual Task StartAsync(CancellationToken cancellationToken) if (_cancellationTokenSource.IsCancellationRequested) { _cancellationTokenSource.Dispose(); - _cancellationTokenSource = new CancellationTokenSource(); + _cancellationTokenSource = new(); } _eventWatcher = WatchClientEventsAsync(_cancellationTokenSource.Token); diff --git a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs index 0e54b19a..a7fc3975 100644 --- a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs +++ b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs @@ -7,6 +7,10 @@ using KubeOps.Abstractions.Queue; using KubeOps.Operator.Queue; +using Microsoft.Extensions.Logging; + +using Moq; + namespace KubeOps.Operator.Test.Queue; public sealed class TimedEntityQueueTest @@ -14,7 +18,7 @@ public sealed class TimedEntityQueueTest [Fact] public async Task Can_Enqueue_Multiple_Entities_With_Same_Name() { - var queue = new TimedEntityQueue(); + var queue = new TimedEntityQueue(Mock.Of>>()); queue.Enqueue(CreateSecret("app-ns1", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1)); queue.Enqueue(CreateSecret("app-ns2", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1)); From 54c59b36f7a5cde4342626e0bd5c6049bae32919 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Tue, 23 Sep 2025 16:49:57 +0200 Subject: [PATCH 25/58] refactor(admission): seal `AdmissionStatus` and `MutationResult`; update finalizer resolution in reconciler - Changed `AdmissionStatus` and `MutationResult` to `sealed` records for improved immutability. - Updated reconciler to use `GetKeyedServices` for resolving entity finalizers, enhancing flexibility and service resolution clarity. --- src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs | 2 +- .../Webhooks/Admission/Mutation/MutationResult.cs | 2 +- src/KubeOps.Operator/Reconciliation/Reconciler.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs b/src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs index d016f16a..7272931a 100644 --- a/src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs +++ b/src/KubeOps.Operator.Web/Webhooks/Admission/AdmissionStatus.cs @@ -13,6 +13,6 @@ namespace KubeOps.Operator.Web.Webhooks.Admission; /// /// A message that is passed to the API. /// A custom status code to provide more detailed information. -public record AdmissionStatus([property: JsonPropertyName("message")] +public sealed record AdmissionStatus([property: JsonPropertyName("message")] string Message, [property: JsonPropertyName("code")] int? Code = StatusCodes.Status200OK); diff --git a/src/KubeOps.Operator.Web/Webhooks/Admission/Mutation/MutationResult.cs b/src/KubeOps.Operator.Web/Webhooks/Admission/Mutation/MutationResult.cs index 100273d3..92a2747f 100644 --- a/src/KubeOps.Operator.Web/Webhooks/Admission/Mutation/MutationResult.cs +++ b/src/KubeOps.Operator.Web/Webhooks/Admission/Mutation/MutationResult.cs @@ -21,7 +21,7 @@ namespace KubeOps.Operator.Web.Webhooks.Admission.Mutation; /// /// The modified entity if any changes are requested. /// The type of the entity. -public record MutationResult(TEntity? ModifiedObject = default) : IActionResult +public sealed record MutationResult(TEntity? ModifiedObject = default) : IActionResult where TEntity : IKubernetesObject { private const string JsonPatch = "JSONPatch"; diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index 34a7a5cc..e1040223 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -199,7 +199,7 @@ private async Task> ReconcileModificationAsync(TEntity entity, C if (settings.AutoAttachFinalizers) { - var finalizers = scope.ServiceProvider.GetService>>() ?? []; + var finalizers = scope.ServiceProvider.GetKeyedServices>(KeyedService.AnyKey); foreach (var finalizer in finalizers) { From d48c2a537664fb4e2591b5e4df283559c3d27966 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Wed, 24 Sep 2025 11:49:53 +0200 Subject: [PATCH 26/58] refactor(reconciliation): migrate `Result` to `ReconciliationResult` and reorganize namespaces - Replaced `Result` with `ReconciliationResult` for enhanced naming clarity and contextual usage. - Moved reconciliation-related components and interfaces under `KubeOps.Abstractions.Reconciliation` namespace for better logical separation. - Updated tests and examples to use new types and namespaces. - Removed the obsolete `Result` class. --- .../Controller/V1TestEntityController.cs | 11 +- .../Controller/V1TestEntityController.cs | 12 +- examples/Operator/Finalizer/FinalizerOne.cs | 8 +- .../Controller/V1TestEntityController.cs | 11 +- .../Builder/IOperatorBuilder.cs | 4 +- src/KubeOps.Abstractions/Controller/Result.cs | 43 ------- .../Controller/IEntityController{TEntity}.cs | 6 +- .../Finalizer/EntityFinalizerAttacher.cs | 2 +- .../Finalizer/EntityFinalizerExtensions.cs | 2 +- .../Finalizer/IEntityFinalizer{TEntity}.cs | 6 +- .../IEventFinalizerAttacherFactory.cs | 2 +- .../Reconciliation/IReconciler{TEntity}.cs | 10 +- .../Queue/EntityRequeue.cs | 2 +- .../Queue/IEntityRequeueFactory.cs | 2 +- .../{ => Reconciliation}/Queue/RequeueType.cs | 2 +- .../ReconciliationResult{TEntity}.cs | 115 ++++++++++++++++++ .../Builder/OperatorBuilder.cs | 7 +- .../KubeOpsEventFinalizerAttacherFactory.cs | 2 +- .../Queue/EntityRequeueBackgroundService.cs | 3 +- .../Queue/ITimedEntityQueue.cs | 2 +- .../Queue/KubeOpsEntityRequeueFactory.cs | 2 +- .../Queue/RequeueEntry{TEntity}.cs | 2 +- .../Queue/RequeueTypeExtensions.cs | 2 +- .../Queue/TimedEntityQueue.cs | 2 +- .../Queue/TimedQueueEntry{TEntity}.cs | 2 +- .../Reconciliation/Reconciler.cs | 39 +++--- .../LeaderAwareResourceWatcher{TEntity}.cs | 1 + .../Watcher/ResourceWatcher{TEntity}.cs | 6 +- .../KubeOps.Abstractions.Test.csproj | 3 + .../EntityFinalizerExtensions.Test.cs | 24 ++-- .../TestHelperExtensions.cs | 2 +- .../Builder/OperatorBuilder.Test.cs | 19 +-- .../CancelEntityRequeue.Integration.Test.cs | 13 +- .../DeletedEntityRequeue.Integration.Test.cs | 13 +- .../EntityController.Integration.Test.cs | 11 +- .../EntityRequeue.Integration.Test.cs | 13 +- .../Events/EventPublisher.Integration.Test.cs | 13 +- .../EntityFinalizer.Integration.Test.cs | 21 ++-- .../LeaderResourceWatcher.Integration.Test.cs | 11 +- .../ResourceWatcher.Integration.Test.cs | 11 +- .../LeaderAwareness.Integration.Test.cs | 11 +- .../NamespacedOperator.Integration.Test.cs | 11 +- .../Queue/TimedEntityQueue.Test.cs | 2 +- .../Watcher/ResourceWatcher{TEntity}.Test.cs | 2 +- 44 files changed, 288 insertions(+), 200 deletions(-) delete mode 100644 src/KubeOps.Abstractions/Controller/Result.cs rename src/KubeOps.Abstractions/{ => Reconciliation}/Controller/IEntityController{TEntity}.cs (89%) rename src/KubeOps.Abstractions/{ => Reconciliation}/Finalizer/EntityFinalizerAttacher.cs (97%) rename src/KubeOps.Abstractions/{ => Reconciliation}/Finalizer/EntityFinalizerExtensions.cs (97%) rename src/KubeOps.Abstractions/{ => Reconciliation}/Finalizer/IEntityFinalizer{TEntity}.cs (83%) rename src/KubeOps.Abstractions/{ => Reconciliation}/Finalizer/IEventFinalizerAttacherFactory.cs (95%) rename src/{KubeOps.Operator => KubeOps.Abstractions}/Reconciliation/IReconciler{TEntity}.cs (83%) rename src/KubeOps.Abstractions/{ => Reconciliation}/Queue/EntityRequeue.cs (97%) rename src/KubeOps.Abstractions/{ => Reconciliation}/Queue/IEntityRequeueFactory.cs (93%) rename src/KubeOps.Abstractions/{ => Reconciliation}/Queue/RequeueType.cs (93%) create mode 100644 src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs rename test/KubeOps.Abstractions.Test/{ => Reconciliation}/Finalizer/EntityFinalizerExtensions.Test.cs (75%) diff --git a/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs b/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs index c574cc12..406a176c 100644 --- a/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs +++ b/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs @@ -4,23 +4,24 @@ using ConversionWebhookOperator.Entities; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Rbac; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; namespace ConversionWebhookOperator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] public class V1TestEntityController(ILogger logger) : IEntityController { - public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Deleted entity {Entity}.", entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/examples/Operator/Controller/V1TestEntityController.cs b/examples/Operator/Controller/V1TestEntityController.cs index a7d97657..aaf4b572 100644 --- a/examples/Operator/Controller/V1TestEntityController.cs +++ b/examples/Operator/Controller/V1TestEntityController.cs @@ -2,10 +2,10 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Events; -using KubeOps.Abstractions.Queue; using KubeOps.Abstractions.Rbac; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using Microsoft.Extensions.Logging; @@ -17,15 +17,15 @@ namespace Operator.Controller; public sealed class V1TestEntityController(ILogger logger) : IEntityController { - public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Deleting entity {Entity}.", entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/examples/Operator/Finalizer/FinalizerOne.cs b/examples/Operator/Finalizer/FinalizerOne.cs index d43ce286..f80b8537 100644 --- a/examples/Operator/Finalizer/FinalizerOne.cs +++ b/examples/Operator/Finalizer/FinalizerOne.cs @@ -2,8 +2,8 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Finalizer; using Operator.Entities; @@ -11,6 +11,6 @@ namespace Operator.Finalizer; public sealed class FinalizerOne : IEntityFinalizer { - public Task> FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> FinalizeAsync(V1TestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } diff --git a/examples/WebhookOperator/Controller/V1TestEntityController.cs b/examples/WebhookOperator/Controller/V1TestEntityController.cs index 984cb00d..24b9a59f 100644 --- a/examples/WebhookOperator/Controller/V1TestEntityController.cs +++ b/examples/WebhookOperator/Controller/V1TestEntityController.cs @@ -2,8 +2,9 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Rbac; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using WebhookOperator.Entities; @@ -12,15 +13,15 @@ namespace WebhookOperator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] public sealed class V1TestEntityController(ILogger logger) : IEntityController { - public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Deleted entity {Entity}.", entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs b/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs index d0f7ca47..2cc630f2 100644 --- a/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs +++ b/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs @@ -5,10 +5,10 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Crds; using KubeOps.Abstractions.Entities; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; using Microsoft.Extensions.DependencyInjection; diff --git a/src/KubeOps.Abstractions/Controller/Result.cs b/src/KubeOps.Abstractions/Controller/Result.cs deleted file mode 100644 index 8f9d72e6..00000000 --- a/src/KubeOps.Abstractions/Controller/Result.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System.Diagnostics.CodeAnalysis; - -using k8s; -using k8s.Models; - -namespace KubeOps.Abstractions.Controller; - -public sealed record Result - where TEntity : IKubernetesObject -{ - private Result(TEntity entity, bool isSuccess, string? errorMessage, Exception? error, TimeSpan? requeueAfter) - { - Entity = entity; - IsSuccess = isSuccess; - ErrorMessage = errorMessage; - Error = error; - RequeueAfter = requeueAfter; - } - - public TEntity Entity { get; } - - [MemberNotNullWhen(false, nameof(ErrorMessage))] - public bool IsSuccess { get; } - - [MemberNotNullWhen(true, nameof(ErrorMessage))] - public bool IsFailure => !IsSuccess; - - public string? ErrorMessage { get; } - - public Exception? Error { get; } - - public TimeSpan? RequeueAfter { get; set; } - - public static Result ForSuccess(TEntity entity, TimeSpan? requeueAfter = null) - { - return new(entity, true, null, null, requeueAfter); - } - - public static Result ForFailure(TEntity entity, string errorMessage, Exception? error = null, TimeSpan? requeueAfter = null) - { - return new(entity, false, errorMessage, error, requeueAfter); - } -} diff --git a/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/Controller/IEntityController{TEntity}.cs similarity index 89% rename from src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs rename to src/KubeOps.Abstractions/Reconciliation/Controller/IEntityController{TEntity}.cs index 2d82012c..e650b76a 100644 --- a/src/KubeOps.Abstractions/Controller/IEntityController{TEntity}.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Controller/IEntityController{TEntity}.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Controller; +namespace KubeOps.Abstractions.Reconciliation.Controller; /// /// Generic entity controller. The controller manages the reconcile loop @@ -47,7 +47,7 @@ public interface IEntityController /// The entity that initiated the reconcile operation. /// The token used to signal cancellation of the operation. /// A task that represents the asynchronous operation and contains the result of the reconcile process. - Task> ReconcileAsync(TEntity entity, CancellationToken cancellationToken); + Task> ReconcileAsync(TEntity entity, CancellationToken cancellationToken); /// /// Called for `delete` events for a given entity. @@ -55,5 +55,5 @@ public interface IEntityController /// The entity that fired the deleted event. /// The token to monitor for cancellation requests. /// A task that represents the asynchronous operation and contains the result of the reconcile process. - Task> DeletedAsync(TEntity entity, CancellationToken cancellationToken); + Task> DeletedAsync(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/Finalizer/EntityFinalizerAttacher.cs b/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerAttacher.cs similarity index 97% rename from src/KubeOps.Abstractions/Finalizer/EntityFinalizerAttacher.cs rename to src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerAttacher.cs index 924bfde8..c54275bc 100644 --- a/src/KubeOps.Abstractions/Finalizer/EntityFinalizerAttacher.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerAttacher.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Finalizer; +namespace KubeOps.Abstractions.Reconciliation.Finalizer; /// /// diff --git a/src/KubeOps.Abstractions/Finalizer/EntityFinalizerExtensions.cs b/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerExtensions.cs similarity index 97% rename from src/KubeOps.Abstractions/Finalizer/EntityFinalizerExtensions.cs rename to src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerExtensions.cs index 2f71af7e..d2408108 100644 --- a/src/KubeOps.Abstractions/Finalizer/EntityFinalizerExtensions.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Finalizer/EntityFinalizerExtensions.cs @@ -7,7 +7,7 @@ using KubeOps.Abstractions.Entities; -namespace KubeOps.Abstractions.Finalizer; +namespace KubeOps.Abstractions.Reconciliation.Finalizer; /// /// Provides extension methods for handling entity finalizers in Kubernetes resources. diff --git a/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/Finalizer/IEntityFinalizer{TEntity}.cs similarity index 83% rename from src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs rename to src/KubeOps.Abstractions/Reconciliation/Finalizer/IEntityFinalizer{TEntity}.cs index a4aa14b6..f3b38983 100644 --- a/src/KubeOps.Abstractions/Finalizer/IEntityFinalizer{TEntity}.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Finalizer/IEntityFinalizer{TEntity}.cs @@ -5,9 +5,7 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Controller; - -namespace KubeOps.Abstractions.Finalizer; +namespace KubeOps.Abstractions.Reconciliation.Finalizer; /// /// Finalizer for an entity. @@ -22,5 +20,5 @@ public interface IEntityFinalizer /// The kubernetes entity that needs to be finalized. /// The token to monitor for cancellation requests. /// A task that represents the asynchronous operation and contains the result of the reconcile process. - Task> FinalizeAsync(TEntity entity, CancellationToken cancellationToken); + Task> FinalizeAsync(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/Finalizer/IEventFinalizerAttacherFactory.cs b/src/KubeOps.Abstractions/Reconciliation/Finalizer/IEventFinalizerAttacherFactory.cs similarity index 95% rename from src/KubeOps.Abstractions/Finalizer/IEventFinalizerAttacherFactory.cs rename to src/KubeOps.Abstractions/Reconciliation/Finalizer/IEventFinalizerAttacherFactory.cs index 8236f6c8..8235dbfc 100644 --- a/src/KubeOps.Abstractions/Finalizer/IEventFinalizerAttacherFactory.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Finalizer/IEventFinalizerAttacherFactory.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Finalizer; +namespace KubeOps.Abstractions.Reconciliation.Finalizer; /// /// Represents a type used to create for controllers. diff --git a/src/KubeOps.Operator/Reconciliation/IReconciler{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs similarity index 83% rename from src/KubeOps.Operator/Reconciliation/IReconciler{TEntity}.cs rename to src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs index 6660d2a2..5e3290aa 100644 --- a/src/KubeOps.Operator/Reconciliation/IReconciler{TEntity}.cs +++ b/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs @@ -5,9 +5,7 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Controller; - -namespace KubeOps.Operator.Reconciliation; +namespace KubeOps.Abstractions.Reconciliation; /// /// Defines methods for handling reconciliation processes related to Kubernetes resources. @@ -26,7 +24,7 @@ public interface IReconciler /// The entity to reconcile during its creation. /// A token to monitor for cancellation requests. /// A task representing the asynchronous operation, with a result of the reconciliation process. - Task> ReconcileCreation(TEntity entity, CancellationToken cancellationToken); + Task> ReconcileCreation(TEntity entity, CancellationToken cancellationToken); /// /// Handles the reconciliation process when an existing entity is modified. @@ -34,7 +32,7 @@ public interface IReconciler /// The entity to reconcile after modification. /// A token to monitor for cancellation requests. /// A task representing the asynchronous operation, with a result of the reconciliation process. - Task> ReconcileModification(TEntity entity, CancellationToken cancellationToken); + Task> ReconcileModification(TEntity entity, CancellationToken cancellationToken); /// /// Handles the reconciliation process when an entity is deleted. @@ -42,5 +40,5 @@ public interface IReconciler /// The entity to reconcile during its deletion. /// A token to monitor for cancellation requests. /// A task representing the asynchronous operation, with a result of the reconciliation process. - Task> ReconcileDeletion(TEntity entity, CancellationToken cancellationToken); + Task> ReconcileDeletion(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/Queue/EntityRequeue.cs b/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs similarity index 97% rename from src/KubeOps.Abstractions/Queue/EntityRequeue.cs rename to src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs index 5354a73e..7ac634c8 100644 --- a/src/KubeOps.Abstractions/Queue/EntityRequeue.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Queue; +namespace KubeOps.Abstractions.Reconciliation.Queue; /// /// Injectable delegate for requeueing entities. diff --git a/src/KubeOps.Abstractions/Queue/IEntityRequeueFactory.cs b/src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs similarity index 93% rename from src/KubeOps.Abstractions/Queue/IEntityRequeueFactory.cs rename to src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs index 51486eff..8e4faf71 100644 --- a/src/KubeOps.Abstractions/Queue/IEntityRequeueFactory.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -namespace KubeOps.Abstractions.Queue; +namespace KubeOps.Abstractions.Reconciliation.Queue; /// /// Represents a type used to create delegates of type for requeueing entities. diff --git a/src/KubeOps.Abstractions/Queue/RequeueType.cs b/src/KubeOps.Abstractions/Reconciliation/Queue/RequeueType.cs similarity index 93% rename from src/KubeOps.Abstractions/Queue/RequeueType.cs rename to src/KubeOps.Abstractions/Reconciliation/Queue/RequeueType.cs index 4bce1f14..37346af8 100644 --- a/src/KubeOps.Abstractions/Queue/RequeueType.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Queue/RequeueType.cs @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -namespace KubeOps.Abstractions.Queue; +namespace KubeOps.Abstractions.Reconciliation.Queue; /// /// Specifies the types of requeue operations that can occur on an entity. diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs new file mode 100644 index 00000000..bfb32824 --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs @@ -0,0 +1,115 @@ +using System.Diagnostics.CodeAnalysis; + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Represents the result of an operation performed on an entity +/// within the context of Kubernetes controllers or finalizers. +/// +/// +/// The type of the Kubernetes entity associated with this result. +/// Must implement where TMetadata is . +/// +public sealed record ReconciliationResult + where TEntity : IKubernetesObject +{ + private ReconciliationResult(TEntity entity, bool isSuccess, string? errorMessage, Exception? error, TimeSpan? requeueAfter) + { + Entity = entity; + IsSuccess = isSuccess; + ErrorMessage = errorMessage; + Error = error; + RequeueAfter = requeueAfter; + } + + /// + /// Represents the Kubernetes entity associated with the result of an operation or reconciliation process. + /// This property contains the entity object of type that was processed, modified, or finalized + /// during an operation. It provides access to the updated state or metadata of the entity after the operation. + /// Typically used for handling further processing, queuing, or logging of the affected entity. + /// + public TEntity Entity { get; } + + /// + /// Indicates whether the operation has completed successfully. + /// Returns true when the operation was successful, and no errors occurred. + /// This property is used to determine the state of the operation + /// and is often checked to decide whether further processing or error handling is necessary. + /// When this property is true, will be null, as there were no errors. + /// + [MemberNotNullWhen(false, nameof(ErrorMessage))] + public bool IsSuccess { get; } + + /// + /// Indicates whether the operation has failed. + /// Returns true when the operation was unsuccessful, as represented by being false. + /// Typically used to distinguish failure states and to conditionally handle error scenarios such as logging or retries. + /// When this property is true, will be non-null, providing additional details about the error. + /// + [MemberNotNullWhen(true, nameof(ErrorMessage))] + public bool IsFailure => !IsSuccess; + + /// + /// Contains a descriptive message associated with a failure when the operation does not succeed. + /// Used to provide context or details about the failure, assisting in debugging and logging. + /// This property is typically set when is false. + /// It will be null for successful operations. + /// + public string? ErrorMessage { get; } + + /// + /// Represents an exception associated with the operation outcome. + /// If the operation fails, this property may hold the exception that caused the failure, + /// providing additional context about the error for logging or debugging purposes. + /// This is optional and may be null if no exception information is available or applicable. + /// + public Exception? Error { get; } + + /// + /// Specifies the duration to wait before requeuing the entity for reprocessing. + /// If set, the entity will be scheduled for reprocessing after the specified time span. + /// This can be useful in scenarios where the entity needs to be revisited later due to external conditions, + /// such as resource dependencies or transient errors. + /// + public TimeSpan? RequeueAfter { get; set; } + + /// + /// Creates a successful result for the given entity, optionally specifying a requeue duration. + /// + /// + /// The Kubernetes entity that the result is associated with. + /// + /// + /// An optional duration after which the entity should be requeued for processing. Defaults to null. + /// + /// + /// A successful instance containing the provided entity and requeue duration. + /// + public static ReconciliationResult Success(TEntity entity, TimeSpan? requeueAfter = null) + => new(entity, true, null, null, requeueAfter); + + /// + /// Creates a failure result for the given entity, specifying an error message and optionally an exception and requeue duration. + /// + /// + /// The Kubernetes entity that the result is associated with. + /// + /// + /// A detailed message describing the reason for the failure. + /// + /// + /// An optional exception that caused the failure. Defaults to null. + /// + /// + /// An optional duration after which the entity should be requeued for processing. Defaults to null. + /// + /// + /// A failure instance containing the provided entity, error information, and requeue duration. + /// + public static ReconciliationResult Failure( + TEntity entity, string errorMessage, Exception? error = null, TimeSpan? requeueAfter = null) + => new(entity, false, errorMessage, error, requeueAfter); +} diff --git a/src/KubeOps.Operator/Builder/OperatorBuilder.cs b/src/KubeOps.Operator/Builder/OperatorBuilder.cs index bb089919..dd78cd4b 100644 --- a/src/KubeOps.Operator/Builder/OperatorBuilder.cs +++ b/src/KubeOps.Operator/Builder/OperatorBuilder.cs @@ -8,12 +8,13 @@ using k8s.Models; using KubeOps.Abstractions.Builder; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Crds; using KubeOps.Abstractions.Entities; using KubeOps.Abstractions.Events; -using KubeOps.Abstractions.Finalizer; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Crds; using KubeOps.Operator.Events; diff --git a/src/KubeOps.Operator/Finalizer/KubeOpsEventFinalizerAttacherFactory.cs b/src/KubeOps.Operator/Finalizer/KubeOpsEventFinalizerAttacherFactory.cs index 5a2db148..47975b07 100644 --- a/src/KubeOps.Operator/Finalizer/KubeOpsEventFinalizerAttacherFactory.cs +++ b/src/KubeOps.Operator/Finalizer/KubeOpsEventFinalizerAttacherFactory.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation.Finalizer; using KubeOps.KubernetesClient; using Microsoft.Extensions.Logging; diff --git a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs index 1e1e1639..b1f4db78 100644 --- a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs +++ b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs @@ -5,7 +5,8 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Reconciliation; diff --git a/src/KubeOps.Operator/Queue/ITimedEntityQueue.cs b/src/KubeOps.Operator/Queue/ITimedEntityQueue.cs index c74fe96d..5842a12e 100644 --- a/src/KubeOps.Operator/Queue/ITimedEntityQueue.cs +++ b/src/KubeOps.Operator/Queue/ITimedEntityQueue.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation.Queue; namespace KubeOps.Operator.Queue; diff --git a/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs b/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs index ec53d551..d7792489 100644 --- a/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs +++ b/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs @@ -5,7 +5,7 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation.Queue; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; diff --git a/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs b/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs index d8541d91..f21f2a4e 100644 --- a/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs +++ b/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation.Queue; namespace KubeOps.Operator.Queue; diff --git a/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs b/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs index 01ed97e6..630cc9a4 100644 --- a/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs +++ b/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs @@ -4,7 +4,7 @@ using k8s; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation.Queue; namespace KubeOps.Operator.Queue; diff --git a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs index 864d8312..ef1756af 100644 --- a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs +++ b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs @@ -7,7 +7,7 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation.Queue; using Microsoft.Extensions.Logging; diff --git a/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs b/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs index bdf105f7..65aabf0e 100644 --- a/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs +++ b/src/KubeOps.Operator/Queue/TimedQueueEntry{TEntity}.cs @@ -4,7 +4,7 @@ using System.Collections.Concurrent; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation.Queue; namespace KubeOps.Operator.Queue; diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index e1040223..962041e3 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -6,9 +6,10 @@ using k8s.Models; using KubeOps.Abstractions.Builder; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Finalizer; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Constants; using KubeOps.Operator.Queue; @@ -46,7 +47,7 @@ internal sealed class Reconciler( { private readonly IFusionCache _entityCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); - public async Task> ReconcileCreation(TEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileCreation(TEntity entity, CancellationToken cancellationToken) { requeue.Remove(entity); var cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); @@ -77,14 +78,14 @@ public async Task> ReconcileCreation(TEntity entity, Cancellatio entity.Kind, entity.Name()); - return Result.ForSuccess(entity); + return ReconciliationResult.Success(entity); } - public async Task> ReconcileModification(TEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileModification(TEntity entity, CancellationToken cancellationToken) { requeue.Remove(entity); - Result result; + ReconciliationResult reconciliationResult; switch (entity) { @@ -99,33 +100,33 @@ public async Task> ReconcileModification(TEntity entity, Cancell entity.Kind, entity.Name()); - return Result.ForSuccess(entity); + return ReconciliationResult.Success(entity); } // update cached generation since generation now changed await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 1, token: cancellationToken); - result = await ReconcileModificationAsync(entity, cancellationToken); + reconciliationResult = await ReconcileModificationAsync(entity, cancellationToken); break; case { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } }: - result = await ReconcileFinalizersSequentialAsync(entity, cancellationToken); + reconciliationResult = await ReconcileFinalizersSequentialAsync(entity, cancellationToken); break; default: - result = Result.ForSuccess(entity); + reconciliationResult = ReconciliationResult.Success(entity); break; } - if (result.RequeueAfter.HasValue) + if (reconciliationResult.RequeueAfter.HasValue) { - requeue.Enqueue(result.Entity, RequeueType.Modified, result.RequeueAfter.Value); + requeue.Enqueue(reconciliationResult.Entity, RequeueType.Modified, reconciliationResult.RequeueAfter.Value); } - return result; + return reconciliationResult; } - public async Task> ReconcileDeletion(TEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileDeletion(TEntity entity, CancellationToken cancellationToken) { requeue.Remove(entity); @@ -149,7 +150,7 @@ public async Task> ReconcileDeletion(TEntity entity, Cancellatio return result; } - private async Task> ReconcileFinalizersSequentialAsync(TEntity entity, CancellationToken cancellationToken) + private async Task> ReconcileFinalizersSequentialAsync(TEntity entity, CancellationToken cancellationToken) { await using var scope = provider.CreateAsyncScope(); @@ -166,7 +167,7 @@ private async Task> ReconcileFinalizersSequentialAsync(TEntity e entity.Kind, entity.Name(), identifier); - return Result.ForSuccess(entity); + return ReconciliationResult.Success(entity); } var result = await finalizer.FinalizeAsync(entity, cancellationToken); @@ -190,10 +191,10 @@ private async Task> ReconcileFinalizersSequentialAsync(TEntity e entity.Name(), identifier); - return Result.ForSuccess(entity, result.RequeueAfter); + return ReconciliationResult.Success(entity, result.RequeueAfter); } - private async Task> ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) + private async Task> ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) { await using var scope = provider.CreateAsyncScope(); diff --git a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs index 4861147d..0f3c2eb0 100644 --- a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs @@ -10,6 +10,7 @@ using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Entities; +using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; using KubeOps.Operator.Reconciliation; diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index 7184675d..811439f2 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -11,8 +11,8 @@ using k8s.Models; using KubeOps.Abstractions.Builder; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Entities; +using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; using KubeOps.Operator.Logging; using KubeOps.Operator.Reconciliation; @@ -126,7 +126,7 @@ static async ValueTask CastAndDispose(IDisposable resource) } } - protected virtual async Task> OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) + protected virtual async Task> OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) { switch (type) { @@ -148,7 +148,7 @@ protected virtual async Task> OnEventAsync(WatchEventType type, break; } - return Result.ForSuccess(entity); + return ReconciliationResult.Success(entity); } private async Task WatchClientEventsAsync(CancellationToken stoppingToken) diff --git a/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj b/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj index 7bc34335..425967cf 100644 --- a/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj +++ b/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj @@ -2,4 +2,7 @@ + + + diff --git a/test/KubeOps.Abstractions.Test/Finalizer/EntityFinalizerExtensions.Test.cs b/test/KubeOps.Abstractions.Test/Reconciliation/Finalizer/EntityFinalizerExtensions.Test.cs similarity index 75% rename from test/KubeOps.Abstractions.Test/Finalizer/EntityFinalizerExtensions.Test.cs rename to test/KubeOps.Abstractions.Test/Reconciliation/Finalizer/EntityFinalizerExtensions.Test.cs index 8f9e148b..7f722c1c 100644 --- a/test/KubeOps.Abstractions.Test/Finalizer/EntityFinalizerExtensions.Test.cs +++ b/test/KubeOps.Abstractions.Test/Reconciliation/Finalizer/EntityFinalizerExtensions.Test.cs @@ -7,8 +7,8 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Finalizer; namespace KubeOps.Abstractions.Test.Finalizer; @@ -75,36 +75,36 @@ public void GetIdentifierName_Should_Return_Correct_Name_When_Finalizer_Identifi private sealed class EntityFinalizerWithATotalIdentifierNameHavingALengthGreaterThan63 : IEntityFinalizer { - public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } private sealed class EntityFinalizerNotEndingOnFinalizer1 : IEntityFinalizer { - public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } private sealed class EntityWithGroupAsStringValueFinalizer : IEntityFinalizer { - public Task> FinalizeAsync(EntityWithGroupAsStringValue entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> FinalizeAsync(EntityWithGroupAsStringValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } private sealed class EntityWithGroupAsConstValueFinalizer : IEntityFinalizer { - public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> FinalizeAsync(EntityWithGroupAsConstValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } private sealed class EntityWithNoGroupFinalizer : IEntityFinalizer { - public Task> FinalizeAsync(EntityWithNoGroupValue entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> FinalizeAsync(EntityWithNoGroupValue entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } [KubernetesEntity(Group = "finalizer.test", ApiVersion = "v1", Kind = "FinalizerTest")] diff --git a/test/KubeOps.Generator.Test/TestHelperExtensions.cs b/test/KubeOps.Generator.Test/TestHelperExtensions.cs index 8dc9edca..2f40cdec 100644 --- a/test/KubeOps.Generator.Test/TestHelperExtensions.cs +++ b/test/KubeOps.Generator.Test/TestHelperExtensions.cs @@ -19,7 +19,7 @@ public static Compilation CreateCompilation(this string source) ], [ MetadataReference.CreateFromFile(typeof(Binder).GetTypeInfo().Assembly.Location), - MetadataReference.CreateFromFile(typeof(Abstractions.Controller.IEntityController<>).GetTypeInfo().Assembly.Location), + MetadataReference.CreateFromFile(typeof(Abstractions.Reconciliation.Controller.IEntityController<>).GetTypeInfo().Assembly.Location), MetadataReference.CreateFromFile(typeof(k8s.IKubernetesObject<>).GetTypeInfo().Assembly.Location), ], new(OutputKind.DynamicallyLinkedLibrary)); diff --git a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs index 0ee2adbb..29bd34c2 100644 --- a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs +++ b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs @@ -5,11 +5,12 @@ using FluentAssertions; using KubeOps.Abstractions.Builder; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Entities; using KubeOps.Abstractions.Events; -using KubeOps.Abstractions.Finalizer; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient.LabelSelectors; using KubeOps.Operator.Builder; using KubeOps.Operator.Queue; @@ -147,17 +148,17 @@ public void Should_Add_LeaderAwareResourceWatcher() private sealed class TestController : IEntityController { - public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.FromResult(Result.ForSuccess(entity)); + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(ReconciliationResult.Success(entity)); - public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.FromResult(Result.ForSuccess(entity)); + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(ReconciliationResult.Success(entity)); } private sealed class TestFinalizer : IEntityFinalizer { - public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => - Task.FromResult(Result.ForSuccess(entity)); + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => + Task.FromResult(ReconciliationResult.Success(entity)); } private sealed class TestLabelSelector : IEntityLabelSelector diff --git a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs index bdd2d3d8..4f732a6b 100644 --- a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs @@ -4,8 +4,9 @@ using FluentAssertions; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Queue; using KubeOps.Operator.Test.TestEntities; @@ -76,7 +77,7 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (svc.Invocations.Count < 2) @@ -84,12 +85,12 @@ public Task> ReconcileAsync(V1OperatorIn requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1000)); } - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs index 5e00edcd..952a5346 100644 --- a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs @@ -4,8 +4,9 @@ using FluentAssertions; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Queue; using KubeOps.Operator.Test.TestEntities; @@ -59,17 +60,17 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1000)); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs index 3b17c007..fee650d9 100644 --- a/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/EntityController.Integration.Test.cs @@ -6,7 +6,8 @@ using k8s.Models; -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -120,16 +121,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs index 49cb6b2c..5ce6f3a1 100644 --- a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs @@ -4,8 +4,9 @@ using FluentAssertions; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -88,7 +89,7 @@ private class TestController(InvocationCounter EntityRequeue requeue) : IEntityController { - public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (svc.Invocations.Count <= svc.TargetInvocationCount) @@ -96,10 +97,10 @@ public Task> ReconcileAsync(V1OperatorIn requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1)); } - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs index a6fc4843..e41ff4e8 100644 --- a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs @@ -9,9 +9,10 @@ using k8s.Models; -using KubeOps.Abstractions.Controller; using KubeOps.Abstractions.Events; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -90,7 +91,7 @@ private class TestController( EventPublisher eventPublisher) : IEntityController { - public async Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { await eventPublisher(entity, "REASON", "message", cancellationToken: cancellationToken); svc.Invocation(entity); @@ -100,10 +101,10 @@ public async Task> ReconcileAsync(V1Oper requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(10)); } - return Result.ForSuccess(entity); + return ReconciliationResult.Success(entity); } - public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs index ab31d7b6..f298c3ab 100644 --- a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs @@ -6,8 +6,9 @@ using k8s.Models; -using KubeOps.Abstractions.Controller; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -213,7 +214,7 @@ private class TestController(InvocationCounter EntityFinalizerAttacher second) : IEntityController { - public async Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); if (entity.Name().Contains("first")) @@ -226,31 +227,31 @@ public async Task> ReconcileAsync(V1Oper await second(entity); } - return Result.ForSuccess(entity); + return ReconciliationResult.Success(entity); } - public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } } private class FirstFinalizer(InvocationCounter svc) : IEntityFinalizer { - public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } } private class SecondFinalizer(InvocationCounter svc) : IEntityFinalizer { - public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs b/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs index 5b48552e..baf0f5c2 100644 --- a/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/HostedServices/LeaderResourceWatcher.Integration.Test.cs @@ -3,7 +3,8 @@ // See the LICENSE file in the project root for more information. using KubeOps.Abstractions.Builder; -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Operator.Test.TestEntities; using Microsoft.Extensions.Hosting; @@ -21,10 +22,10 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private sealed class TestController : IEntityController { - public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); - public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs b/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs index 989ceddb..d4b58d00 100644 --- a/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/HostedServices/ResourceWatcher.Integration.Test.cs @@ -2,7 +2,8 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Operator.Test.TestEntities; using Microsoft.Extensions.DependencyInjection; @@ -49,10 +50,10 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController : IEntityController { - public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); - public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) - => Task.FromResult(Result.ForSuccess(entity)); + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + => Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs b/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs index 076889f8..ce541f6e 100644 --- a/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/LeaderElector/LeaderAwareness.Integration.Test.cs @@ -7,7 +7,8 @@ using k8s.Models; using KubeOps.Abstractions.Builder; -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -56,16 +57,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs index aeeae790..ea50202d 100644 --- a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs @@ -7,7 +7,8 @@ using k8s; using k8s.Models; -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.KubernetesClient; using KubeOps.Operator.Test.TestEntities; @@ -81,16 +82,16 @@ protected override void ConfigureHost(HostApplicationBuilder builder) private class TestController(InvocationCounter svc) : IEntityController { - public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - return Task.FromResult(Result.ForSuccess(entity)); + return Task.FromResult(ReconciliationResult.Success(entity)); } } } diff --git a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs index a7fc3975..6cf9ce65 100644 --- a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs +++ b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs @@ -4,7 +4,7 @@ using k8s.Models; -using KubeOps.Abstractions.Queue; +using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.Operator.Queue; using Microsoft.Extensions.Logging; diff --git a/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs b/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs index 3960e67f..85724ffb 100644 --- a/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs +++ b/test/KubeOps.Operator.Test/Watcher/ResourceWatcher{TEntity}.Test.cs @@ -10,8 +10,8 @@ using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Entities; +using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; -using KubeOps.Operator.Reconciliation; using KubeOps.Operator.Watcher; using Microsoft.Extensions.Logging; From 415473ce2a8dd741153144e3efd6939188768da6 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Wed, 24 Sep 2025 13:49:57 +0200 Subject: [PATCH 27/58] refactor(syntax-receiver): update metadata names to reflect new reconciliation namespace - Adjusted `IEntityFinalizerMetadataName` and `IEntityControllerMetadataName` constants to match updated namespace structure under `KubeOps.Abstractions.Reconciliation`. --- .../SyntaxReceiver/EntityControllerSyntaxReceiver.cs | 2 +- .../SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/KubeOps.Generator/SyntaxReceiver/EntityControllerSyntaxReceiver.cs b/src/KubeOps.Generator/SyntaxReceiver/EntityControllerSyntaxReceiver.cs index 686acbe0..08351ab8 100644 --- a/src/KubeOps.Generator/SyntaxReceiver/EntityControllerSyntaxReceiver.cs +++ b/src/KubeOps.Generator/SyntaxReceiver/EntityControllerSyntaxReceiver.cs @@ -9,7 +9,7 @@ namespace KubeOps.Generator.SyntaxReceiver; internal sealed class EntityControllerSyntaxReceiver : ISyntaxContextReceiver { - private const string IEntityControllerMetadataName = "KubeOps.Abstractions.Controller.IEntityController`1"; + private const string IEntityControllerMetadataName = "KubeOps.Abstractions.Reconciliation.Controller.IEntityController`1"; public List<(ClassDeclarationSyntax Controller, string EntityName)> Controllers { get; } = []; diff --git a/src/KubeOps.Generator/SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs b/src/KubeOps.Generator/SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs index ceac128a..9f07e3a8 100644 --- a/src/KubeOps.Generator/SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs +++ b/src/KubeOps.Generator/SyntaxReceiver/EntityFinalizerSyntaxReceiver.cs @@ -9,7 +9,7 @@ namespace KubeOps.Generator.SyntaxReceiver; internal sealed class EntityFinalizerSyntaxReceiver : ISyntaxContextReceiver { - private const string IEntityFinalizerMetadataName = "KubeOps.Abstractions.Finalizer.IEntityFinalizer`1"; + private const string IEntityFinalizerMetadataName = "KubeOps.Abstractions.Reconciliation.Finalizer.IEntityFinalizer`1"; public List<(ClassDeclarationSyntax Finalizer, string EntityName)> Finalizer { get; } = []; From 9df73fc45b566b635b6d25963763c15b33690d1f Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Wed, 24 Sep 2025 14:03:22 +0200 Subject: [PATCH 28/58] refactor(reconciliation): update imports to reflect new `Reconciliation` namespace structure - Updated imports in controllers, finalizers, templates, tests, and documentation to use the `KubeOps.Abstractions.Reconciliation` namespace. - Ensured consistency across all examples, templates, and test cases. --- src/KubeOps.Operator/README.md | 6 +++--- .../Operator.CSharp/Controller/DemoController.cs | 2 +- .../Operator.CSharp/Finalizer/DemoFinalizer.cs | 2 +- .../WebOperator.CSharp/Controller/DemoController.cs | 2 +- .../WebOperator.CSharp/Finalizer/DemoFinalizer.cs | 2 +- .../ControllerRegistrationGenerator.Test.cs | 4 ++-- .../FinalizerRegistrationGenerator.Test.cs | 10 +++++----- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/KubeOps.Operator/README.md b/src/KubeOps.Operator/README.md index d274def0..f3ad39ac 100644 --- a/src/KubeOps.Operator/README.md +++ b/src/KubeOps.Operator/README.md @@ -88,7 +88,7 @@ A controller reconciles a specific entity type. Implement controllers using the Example controller implementation: ```csharp -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Abstractions.Rbac; using KubeOps.KubernetesClient; using Microsoft.Extensions.Logging; @@ -154,7 +154,7 @@ A [finalizer](https://kubernetes.io/docs/concepts/overview/working-with-objects/ Finalizers are attached using an `EntityFinalizerAttacher` and are called when the entity is marked for deletion. ```csharp -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation.Finalizer; public class FinalizerOne : IEntityFinalizer { @@ -171,4 +171,4 @@ public class FinalizerOne : IEntityFinalizer ## Documentation -For more information, visit the [documentation](https://dotnet.github.io/dotnet-operator-sdk/). +For more information, visit the [documentation](https://dotnet.github.io/dotnet-operator-sdk/). \ No newline at end of file diff --git a/src/KubeOps.Templates/Templates/Operator.CSharp/Controller/DemoController.cs b/src/KubeOps.Templates/Templates/Operator.CSharp/Controller/DemoController.cs index bc7be892..f6af3968 100644 --- a/src/KubeOps.Templates/Templates/Operator.CSharp/Controller/DemoController.cs +++ b/src/KubeOps.Templates/Templates/Operator.CSharp/Controller/DemoController.cs @@ -1,4 +1,4 @@ -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Abstractions.Rbac; using Microsoft.Extensions.Logging; diff --git a/src/KubeOps.Templates/Templates/Operator.CSharp/Finalizer/DemoFinalizer.cs b/src/KubeOps.Templates/Templates/Operator.CSharp/Finalizer/DemoFinalizer.cs index 8d9cd92d..225f2186 100644 --- a/src/KubeOps.Templates/Templates/Operator.CSharp/Finalizer/DemoFinalizer.cs +++ b/src/KubeOps.Templates/Templates/Operator.CSharp/Finalizer/DemoFinalizer.cs @@ -1,6 +1,6 @@ using k8s.Models; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation.Finalizer; using Microsoft.Extensions.Logging; diff --git a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Controller/DemoController.cs b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Controller/DemoController.cs index bc7be892..f6af3968 100644 --- a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Controller/DemoController.cs +++ b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Controller/DemoController.cs @@ -1,4 +1,4 @@ -using KubeOps.Abstractions.Controller; +using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Abstractions.Rbac; using Microsoft.Extensions.Logging; diff --git a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Finalizer/DemoFinalizer.cs b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Finalizer/DemoFinalizer.cs index 8d9cd92d..225f2186 100644 --- a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Finalizer/DemoFinalizer.cs +++ b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Finalizer/DemoFinalizer.cs @@ -1,6 +1,6 @@ using k8s.Models; -using KubeOps.Abstractions.Finalizer; +using KubeOps.Abstractions.Reconciliation.Finalizer; using Microsoft.Extensions.Logging; diff --git a/test/KubeOps.Generator.Test/ControllerRegistrationGenerator.Test.cs b/test/KubeOps.Generator.Test/ControllerRegistrationGenerator.Test.cs index 6477f6dc..021bb047 100644 --- a/test/KubeOps.Generator.Test/ControllerRegistrationGenerator.Test.cs +++ b/test/KubeOps.Generator.Test/ControllerRegistrationGenerator.Test.cs @@ -35,7 +35,7 @@ public static IOperatorBuilder RegisterControllers(this IOperatorBuilder builder [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Controller; + using KubeOps.Abstractions.Reconciliation.Controller; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public class V1TestEntity : IKubernetesObject @@ -65,7 +65,7 @@ public static IOperatorBuilder RegisterControllers(this IOperatorBuilder builder [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Controller; + using KubeOps.Abstractions.Reconciliation.Controller; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject diff --git a/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs b/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs index 7d00f7fe..fca9af3c 100644 --- a/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs +++ b/test/KubeOps.Generator.Test/FinalizerRegistrationGenerator.Test.cs @@ -35,7 +35,7 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; public static class Constants { @@ -73,7 +73,7 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject @@ -104,7 +104,7 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject @@ -140,7 +140,7 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject @@ -177,7 +177,7 @@ public static IOperatorBuilder RegisterFinalizers(this IOperatorBuilder builder) [InlineData(""" using k8s; using k8s.Models; - using KubeOps.Abstractions.Finalizer; + using KubeOps.Abstractions.Reconciliation.Finalizer; [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] public sealed class V1TestEntity : IKubernetesObject From 4f000e9e828fdf1b80ed967fc7e8186f683c56b4 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Wed, 24 Sep 2025 19:33:46 +0200 Subject: [PATCH 29/58] fix(reconciliation): ensure finalizers are executed for entities marked for deletion - Updated `Reconciler` to handle scenarios where entities with deletion timestamps may have missed events. - Executes finalizers sequentially for such entities to ensure proper cleanup. --- src/KubeOps.Operator/Reconciliation/Reconciler.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index 962041e3..7eb17b80 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -52,10 +52,14 @@ public async Task> ReconcileCreation(TEntity entit requeue.Remove(entity); var cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); + // Only perform reconciliation if the entity was not already in the cache. if (!cachedGeneration.HasValue) { - // Only perform reconciliation if the entity was not already in the cache. - var result = await ReconcileModificationAsync(entity, cancellationToken); + // in case we missed an event (operator was not running) + // and the entity was marked for deletion - execute finalizers + var result = (entity.Metadata.DeletionTimestamp is null) + ? await ReconcileModificationAsync(entity, cancellationToken) + : await ReconcileFinalizersSequentialAsync(entity, cancellationToken); if (result.IsSuccess) { From d21719e074057c9b52c15693b840ce4078fc84a7 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Thu, 25 Sep 2025 09:24:53 +0200 Subject: [PATCH 30/58] feat(watcher): integrate `FusionCache` for bookmark version caching in resource watchers - Introduced `FusionCache` to store and retrieve resource bookmark versions for `ResourceWatcher` and `LeaderAwareResourceWatcher`. - Added caching logic to persist and fetch bookmark versions, improving resilience during restarts. - Logged skipped reconciliation for entities with deletion timestamps to enhance debugging clarity. --- .../Reconciliation/Reconciler.cs | 16 +++++++++++----- .../LeaderAwareResourceWatcher{TEntity}.cs | 7 +++++-- .../Watcher/ResourceWatcher{TEntity}.cs | 14 ++++++++++++-- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index 7eb17b80..149c789b 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -55,11 +55,17 @@ public async Task> ReconcileCreation(TEntity entit // Only perform reconciliation if the entity was not already in the cache. if (!cachedGeneration.HasValue) { - // in case we missed an event (operator was not running) - // and the entity was marked for deletion - execute finalizers - var result = (entity.Metadata.DeletionTimestamp is null) - ? await ReconcileModificationAsync(entity, cancellationToken) - : await ReconcileFinalizersSequentialAsync(entity, cancellationToken); + if (entity.Metadata.DeletionTimestamp is not null) + { + logger.LogDebug( + """Received ADDED event for entity "{Kind}/{Name}" which already has a deletion timestamp "{DeletionTimestamp}". Skip event.""", + entity.Kind, + entity.Name(), + entity.Metadata.DeletionTimestamp.Value.ToString("O")); + return ReconciliationResult.Success(entity); + } + + var result = await ReconcileModificationAsync(entity, cancellationToken); if (result.IsSuccess) { diff --git a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs index 0f3c2eb0..c57fd240 100644 --- a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs @@ -12,17 +12,19 @@ using KubeOps.Abstractions.Entities; using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; -using KubeOps.Operator.Reconciliation; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using ZiggyCreatures.Caching.Fusion; + namespace KubeOps.Operator.Watcher; internal sealed class LeaderAwareResourceWatcher( ActivitySource activitySource, ILogger> logger, IReconciler reconciler, + IFusionCacheProvider cacheProvider, OperatorSettings settings, IEntityLabelSelector labelSelector, IKubernetesClient client, @@ -32,6 +34,7 @@ internal sealed class LeaderAwareResourceWatcher( activitySource, logger, reconciler, + cacheProvider, settings, labelSelector, client) @@ -102,7 +105,7 @@ private void StoppedLeading() logger.LogInformation("This instance stopped leading, stopping watcher."); // Stop the base implementation using the 'ApplicationStopped' cancellation token. - // The cancellation token should only be marked cancelled when the stop should no longer be graceful. + // The cancellation token should only be marked as canceled when the stop should no longer be graceful. base.StopAsync(hostApplicationLifetime.ApplicationStopped).Wait(); } } diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index 811439f2..3efbafaa 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -14,24 +14,28 @@ using KubeOps.Abstractions.Entities; using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; +using KubeOps.Operator.Constants; using KubeOps.Operator.Logging; -using KubeOps.Operator.Reconciliation; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using ZiggyCreatures.Caching.Fusion; + namespace KubeOps.Operator.Watcher; public class ResourceWatcher( ActivitySource activitySource, ILogger> logger, IReconciler reconciler, + IFusionCacheProvider cacheProvider, OperatorSettings settings, IEntityLabelSelector labelSelector, IKubernetesClient client) : IHostedService, IAsyncDisposable, IDisposable where TEntity : IKubernetesObject { + private readonly IFusionCache _bookmarkVersionCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); private CancellationTokenSource _cancellationTokenSource = new(); private uint _watcherReconnectRetries; private Task? _eventWatcher; @@ -151,9 +155,14 @@ protected virtual async Task> OnEventAsync(WatchEv return ReconciliationResult.Success(entity); } + private static string GetBookmarkCacheKey() + => $"bookmark:{typeof(TEntity).Name}"; + private async Task WatchClientEventsAsync(CancellationToken stoppingToken) { - string? currentVersion = null; + var cachedBookmarkVersion = await _bookmarkVersionCache.TryGetAsync(GetBookmarkCacheKey(), token: stoppingToken); + var currentVersion = cachedBookmarkVersion.HasValue ? cachedBookmarkVersion.Value : null; + while (!stoppingToken.IsCancellationRequested) { try @@ -177,6 +186,7 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) if (type == WatchEventType.Bookmark) { + await _bookmarkVersionCache.SetAsync(GetBookmarkCacheKey(), entity.ResourceVersion(), token: stoppingToken); currentVersion = entity.ResourceVersion(); continue; } From c3b3af0d8bd67bbe87e77974105765990cdf67f6 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Thu, 25 Sep 2025 10:34:30 +0200 Subject: [PATCH 31/58] refactor(watcher): remove `FusionCache` usage for bookmark version handling - Eliminated `FusionCacheProvider` and related caching logic from `ResourceWatcher`. - Simplified bookmark version handling by directly assigning `ResourceVersion` during watch events. --- .../Watcher/ResourceWatcher{TEntity}.cs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index 3efbafaa..2fd5ecf0 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -14,28 +14,23 @@ using KubeOps.Abstractions.Entities; using KubeOps.Abstractions.Reconciliation; using KubeOps.KubernetesClient; -using KubeOps.Operator.Constants; using KubeOps.Operator.Logging; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using ZiggyCreatures.Caching.Fusion; - namespace KubeOps.Operator.Watcher; public class ResourceWatcher( ActivitySource activitySource, ILogger> logger, IReconciler reconciler, - IFusionCacheProvider cacheProvider, OperatorSettings settings, IEntityLabelSelector labelSelector, IKubernetesClient client) : IHostedService, IAsyncDisposable, IDisposable where TEntity : IKubernetesObject { - private readonly IFusionCache _bookmarkVersionCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); private CancellationTokenSource _cancellationTokenSource = new(); private uint _watcherReconnectRetries; private Task? _eventWatcher; @@ -155,13 +150,9 @@ protected virtual async Task> OnEventAsync(WatchEv return ReconciliationResult.Success(entity); } - private static string GetBookmarkCacheKey() - => $"bookmark:{typeof(TEntity).Name}"; - private async Task WatchClientEventsAsync(CancellationToken stoppingToken) { - var cachedBookmarkVersion = await _bookmarkVersionCache.TryGetAsync(GetBookmarkCacheKey(), token: stoppingToken); - var currentVersion = cachedBookmarkVersion.HasValue ? cachedBookmarkVersion.Value : null; + string? currentVersion = null; while (!stoppingToken.IsCancellationRequested) { @@ -186,7 +177,6 @@ private async Task WatchClientEventsAsync(CancellationToken stoppingToken) if (type == WatchEventType.Bookmark) { - await _bookmarkVersionCache.SetAsync(GetBookmarkCacheKey(), entity.ResourceVersion(), token: stoppingToken); currentVersion = entity.ResourceVersion(); continue; } From 2e21cb8ffa5cc592cd9463599bf81bed5160cc3d Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Thu, 25 Sep 2025 10:37:54 +0200 Subject: [PATCH 32/58] refactor(watcher): remove `FusionCache` dependency in `LeaderAwareResourceWatcher` - Dropped `FusionCacheProvider` from `LeaderAwareResourceWatcher` dependencies. - Simplified constructor by removing unused caching logic. --- .../Watcher/LeaderAwareResourceWatcher{TEntity}.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs index c57fd240..3cb1b710 100644 --- a/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/LeaderAwareResourceWatcher{TEntity}.cs @@ -16,15 +16,12 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using ZiggyCreatures.Caching.Fusion; - namespace KubeOps.Operator.Watcher; internal sealed class LeaderAwareResourceWatcher( ActivitySource activitySource, ILogger> logger, IReconciler reconciler, - IFusionCacheProvider cacheProvider, OperatorSettings settings, IEntityLabelSelector labelSelector, IKubernetesClient client, @@ -34,7 +31,6 @@ internal sealed class LeaderAwareResourceWatcher( activitySource, logger, reconciler, - cacheProvider, settings, labelSelector, client) From 5f5f3e8f9e9f96dc8b2c70e9e5e54607780a90fd Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Thu, 2 Oct 2025 15:44:55 +0200 Subject: [PATCH 33/58] feat(reconciliation): introduce `ReconciliationContext` and trigger sources - Added `ReconciliationContext` to encapsulate entity context and trigger source during reconciliation. - Introduced `ReconciliationTriggerSource` enum to differentiate between API server and operator-initiated events. - Updated `Reconciler` and related methods to leverage `ReconciliationContext`. - Improved clarity by replacing entity parameters with reconciliation context across operator logic. --- .../Reconciliation/IReconciler{TEntity}.cs | 14 +- .../Reconciliation/Queue/EntityRequeue.cs | 2 +- .../Queue/IEntityRequeueFactory.cs | 2 +- .../ReconciliationContextExtensions.cs | 35 +++++ .../ReconciliationContext{TEntity}.cs | 67 ++++++++++ .../ReconciliationTriggerSource.cs | 27 ++++ .../Queue/EntityRequeueBackgroundService.cs | 8 +- .../Reconciliation/Reconciler.cs | 125 ++++++++++-------- .../Watcher/ResourceWatcher{TEntity}.cs | 8 +- 9 files changed, 220 insertions(+), 68 deletions(-) create mode 100644 src/KubeOps.Abstractions/Reconciliation/ReconciliationContextExtensions.cs create mode 100644 src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs create mode 100644 src/KubeOps.Abstractions/Reconciliation/ReconciliationTriggerSource.cs diff --git a/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs index 5e3290aa..63d22594 100644 --- a/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs +++ b/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs @@ -21,24 +21,24 @@ public interface IReconciler /// /// Handles the reconciliation process when a new entity is created. /// - /// The entity to reconcile during its creation. + /// The context of the entity to reconcile during its creation. /// A token to monitor for cancellation requests. - /// A task representing the asynchronous operation, with a result of the reconciliation process. - Task> ReconcileCreation(TEntity entity, CancellationToken cancellationToken); + /// A task representing the asynchronous operation, yielding the result of the reconciliation process. + Task> ReconcileCreation(ReconciliationContext reconciliationContext, CancellationToken cancellationToken); /// /// Handles the reconciliation process when an existing entity is modified. /// - /// The entity to reconcile after modification. + /// The context of the entity to reconcile during its creation. /// A token to monitor for cancellation requests. /// A task representing the asynchronous operation, with a result of the reconciliation process. - Task> ReconcileModification(TEntity entity, CancellationToken cancellationToken); + Task> ReconcileModification(ReconciliationContext reconciliationContext, CancellationToken cancellationToken); /// /// Handles the reconciliation process when an entity is deleted. /// - /// The entity to reconcile during its deletion. + /// The context of the entity to reconcile during its creation. /// A token to monitor for cancellation requests. /// A task representing the asynchronous operation, with a result of the reconciliation process. - Task> ReconcileDeletion(TEntity entity, CancellationToken cancellationToken); + Task> ReconcileDeletion(ReconciliationContext reconciliationContext, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs b/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs index 7ac634c8..34af3cc2 100644 --- a/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs @@ -8,7 +8,7 @@ namespace KubeOps.Abstractions.Reconciliation.Queue; /// -/// Injectable delegate for requeueing entities. +/// Injectable delegate for requeuing entities. /// /// Use this delegate when you need to pro-actively reconcile an entity after a /// certain amount of time. This is useful, if you want to check your entities diff --git a/src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs b/src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs index 8e4faf71..8bd89088 100644 --- a/src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Queue/IEntityRequeueFactory.cs @@ -8,7 +8,7 @@ namespace KubeOps.Abstractions.Reconciliation.Queue; /// -/// Represents a type used to create delegates of type for requeueing entities. +/// Represents a type used to create delegates of type for requeuing entities. /// public interface IEntityRequeueFactory { diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationContextExtensions.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContextExtensions.cs new file mode 100644 index 00000000..e8aae9b9 --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContextExtensions.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Provides extension methods for the class +/// to facilitate the identification of reconciliation trigger sources. +/// +public static class ReconciliationContextExtensions +{ + /// + /// Determines if the reconciliation context was triggered by the Kubernetes API server. + /// + /// The type of the Kubernetes resource associated with the reconciliation context. + /// The reconciliation context to check. + /// True if the reconciliation was triggered by the API server; otherwise, false. + public static bool IsTriggeredByApiServer(this ReconciliationContext reconciliationContext) + where TEntity : IKubernetesObject + => reconciliationContext.ReconciliationTriggerSource == ReconciliationTriggerSource.ApiServer; + + /// + /// Determines if the reconciliation context was triggered by the operator. + /// + /// The type of the Kubernetes resource associated with the reconciliation context. + /// The reconciliation context to check. + /// True if the reconciliation was triggered by the operator; otherwise, false. + public static bool IsTriggeredByOperator(this ReconciliationContext reconciliationContext) + where TEntity : IKubernetesObject + => reconciliationContext.ReconciliationTriggerSource == ReconciliationTriggerSource.Operator; +} diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs new file mode 100644 index 00000000..e63f8f9e --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using k8s; +using k8s.Models; + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Represents the context for the reconciliation process. +/// This class contains information about the entity to be reconciled and +/// the source that triggered the reconciliation process. +/// +/// +/// The type of the Kubernetes resource being reconciled. Must implement +/// . +/// +public sealed record ReconciliationContext + where TEntity : IKubernetesObject +{ + private ReconciliationContext(TEntity entity, ReconciliationTriggerSource reconciliationTriggerSource) + { + Entity = entity; + ReconciliationTriggerSource = reconciliationTriggerSource; + } + + /// + /// Represents the Kubernetes entity involved in the reconciliation process. + /// + public TEntity Entity { get; } + + /// + /// Specifies the source that initiated the reconciliation process. + /// + public ReconciliationTriggerSource ReconciliationTriggerSource { get; } + + /// + /// Creates a new instance of the class + /// using an event triggered by the Kubernetes API server as the source of reconciliation. + /// + /// + /// The Kubernetes resource instance that is being reconciled. This must implement + /// . + /// + /// + /// A new instance of with + /// as the trigger source. + /// + public static ReconciliationContext CreateFromApiServerEvent(TEntity entity) + => new(entity, ReconciliationTriggerSource.ApiServer); + + /// + /// Creates a new instance of the class + /// using an event triggered by the operator as the source of reconciliation. + /// + /// + /// The Kubernetes resource instance that is being reconciled. This must implement + /// . + /// + /// + /// A new instance of with + /// as the trigger source. + /// + public static ReconciliationContext CreateFromOperatorEvent(TEntity entity) + => new(entity, ReconciliationTriggerSource.Operator); +} diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationTriggerSource.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationTriggerSource.cs new file mode 100644 index 00000000..d5020c39 --- /dev/null +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationTriggerSource.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Abstractions.Reconciliation; + +/// +/// Defines the source that triggered the reconciliation process in the Kubernetes operator. +/// Used to identify which component or mechanism initiated the reconciliation cycle. +/// +public enum ReconciliationTriggerSource +{ + /// + /// Represents a reconciliation trigger initiated by the Kubernetes API server. + /// This source typically implies that the operator has been informed about + /// a resource event (e.g., creation, modification, deletion) via API server + /// notifications or resource watches. + /// + ApiServer, + + /// + /// Represents a reconciliation trigger initiated directly by the operator. + /// This source indicates that the reconciliation process was started internally + /// by the operator, such as during a scheduled task or an operator-specific event. + /// + Operator, +} diff --git a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs index b1f4db78..21a6dce3 100644 --- a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs +++ b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs @@ -117,16 +117,18 @@ protected virtual async Task ReconcileSingleAsync(RequeueEntry queuedEn return; } + var reconciliationContext = ReconciliationContext.CreateFromOperatorEvent(entity); + switch (queuedEntry.RequeueType) { case RequeueType.Added: - await reconciler.ReconcileCreation(entity, cancellationToken); + await reconciler.ReconcileCreation(reconciliationContext, cancellationToken); break; case RequeueType.Modified: - await reconciler.ReconcileModification(entity, cancellationToken); + await reconciler.ReconcileModification(reconciliationContext, cancellationToken); break; case RequeueType.Deleted: - await reconciler.ReconcileDeletion(entity, cancellationToken); + await reconciler.ReconcileDeletion(reconciliationContext, cancellationToken); break; default: throw new NotSupportedException($"RequeueType '{queuedEntry.RequeueType}' is not supported!"); diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index 149c789b..22bab4bf 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -47,106 +47,125 @@ internal sealed class Reconciler( { private readonly IFusionCache _entityCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); - public async Task> ReconcileCreation(TEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileCreation(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - requeue.Remove(entity); - var cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); + requeue.Remove(reconciliationContext.Entity); - // Only perform reconciliation if the entity was not already in the cache. - if (!cachedGeneration.HasValue) + if (reconciliationContext.Entity.Metadata.DeletionTimestamp is not null) { - if (entity.Metadata.DeletionTimestamp is not null) - { - logger.LogDebug( - """Received ADDED event for entity "{Kind}/{Name}" which already has a deletion timestamp "{DeletionTimestamp}". Skip event.""", - entity.Kind, - entity.Name(), - entity.Metadata.DeletionTimestamp.Value.ToString("O")); - return ReconciliationResult.Success(entity); - } + logger.LogDebug( + """Received ADDED event for entity "{Kind}/{Name}" which already has a deletion timestamp "{DeletionTimestamp}". Skip event.""", + reconciliationContext.Entity.Kind, + reconciliationContext.Entity.Name(), + reconciliationContext.Entity.Metadata.DeletionTimestamp.Value.ToString("O")); - var result = await ReconcileModificationAsync(entity, cancellationToken); + return ReconciliationResult.Success(reconciliationContext.Entity); + } - if (result.IsSuccess) - { - await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 0, token: cancellationToken); - } + if (reconciliationContext.IsTriggeredByApiServer()) + { + var cachedGeneration = + await _entityCache.TryGetAsync(reconciliationContext.Entity.Uid(), token: cancellationToken); - if (result.RequeueAfter.HasValue) + // Only perform reconciliation if the entity was not already in the cache. + if (cachedGeneration.HasValue) { - requeue.Enqueue( - result.Entity, - result.IsSuccess ? RequeueType.Modified : RequeueType.Added, - result.RequeueAfter.Value); + logger.LogDebug( + """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", + reconciliationContext.Entity.Kind, + reconciliationContext.Entity.Name()); + + return ReconciliationResult.Success(reconciliationContext.Entity); } - return result; + await _entityCache.SetAsync( + reconciliationContext.Entity.Uid(), + reconciliationContext.Entity.Generation() ?? 0, + token: cancellationToken); } - logger.LogDebug( - """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", - entity.Kind, - entity.Name()); + var result = await ReconcileModificationAsync(reconciliationContext.Entity, cancellationToken); - return ReconciliationResult.Success(entity); + if (result.RequeueAfter.HasValue) + { + requeue.Enqueue( + result.Entity, + result.IsSuccess ? RequeueType.Modified : RequeueType.Added, + result.RequeueAfter.Value); + } + + return result; } - public async Task> ReconcileModification(TEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileModification(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - requeue.Remove(entity); + requeue.Remove(reconciliationContext.Entity); ReconciliationResult reconciliationResult; - switch (entity) + switch (reconciliationContext.Entity) { case { Metadata.DeletionTimestamp: null }: - var cachedGeneration = await _entityCache.TryGetAsync(entity.Uid(), token: cancellationToken); - - // Check if entity-spec has changed through "Generation" value increment. Skip reconcile if not changed. - if (cachedGeneration.HasValue && cachedGeneration >= entity.Generation()) + if (reconciliationContext.IsTriggeredByApiServer()) { - logger.LogDebug( - """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", - entity.Kind, - entity.Name()); - - return ReconciliationResult.Success(entity); + var cachedGeneration = await _entityCache.TryGetAsync( + reconciliationContext.Entity.Uid(), + token: cancellationToken); + + // Check if entity-spec has changed through "Generation" value increment. Skip reconcile if not changed. + if (cachedGeneration.HasValue && cachedGeneration >= reconciliationContext.Entity.Generation()) + { + logger.LogDebug( + """Entity "{Kind}/{Name}" modification did not modify generation. Skip event.""", + reconciliationContext.Entity.Kind, + reconciliationContext.Entity.Name()); + + return ReconciliationResult.Success(reconciliationContext.Entity); + } + + // update cached generation since generation now changed + await _entityCache.SetAsync( + reconciliationContext.Entity.Uid(), + reconciliationContext.Entity.Generation() ?? 1, + token: cancellationToken); } - // update cached generation since generation now changed - await _entityCache.SetAsync(entity.Uid(), entity.Generation() ?? 1, token: cancellationToken); - reconciliationResult = await ReconcileModificationAsync(entity, cancellationToken); + reconciliationResult = await ReconcileModificationAsync(reconciliationContext.Entity, cancellationToken); break; case { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } }: - reconciliationResult = await ReconcileFinalizersSequentialAsync(entity, cancellationToken); + reconciliationResult = await ReconcileFinalizersSequentialAsync(reconciliationContext.Entity, cancellationToken); break; default: - reconciliationResult = ReconciliationResult.Success(entity); + reconciliationResult = ReconciliationResult.Success(reconciliationContext.Entity); break; } if (reconciliationResult.RequeueAfter.HasValue) { - requeue.Enqueue(reconciliationResult.Entity, RequeueType.Modified, reconciliationResult.RequeueAfter.Value); + requeue + .Enqueue( + reconciliationResult.Entity, + RequeueType.Modified, + reconciliationResult.RequeueAfter.Value); } return reconciliationResult; } - public async Task> ReconcileDeletion(TEntity entity, CancellationToken cancellationToken) + public async Task> ReconcileDeletion(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - requeue.Remove(entity); + requeue.Remove(reconciliationContext.Entity); await using var scope = provider.CreateAsyncScope(); var controller = scope.ServiceProvider.GetRequiredService>(); - var result = await controller.DeletedAsync(entity, cancellationToken); + var result = await controller.DeletedAsync(reconciliationContext.Entity, cancellationToken); if (result.IsSuccess) { - await _entityCache.RemoveAsync(entity.Uid(), token: cancellationToken); + await _entityCache.RemoveAsync(reconciliationContext.Entity.Uid(), token: cancellationToken); } if (result.RequeueAfter.HasValue) diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index 2fd5ecf0..52789c2a 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -127,16 +127,18 @@ static async ValueTask CastAndDispose(IDisposable resource) protected virtual async Task> OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) { + var reconciliationContext = ReconciliationContext.CreateFromApiServerEvent(entity); + switch (type) { case WatchEventType.Added: - return await reconciler.ReconcileCreation(entity, cancellationToken); + return await reconciler.ReconcileCreation(reconciliationContext, cancellationToken); case WatchEventType.Modified: - return await reconciler.ReconcileModification(entity, cancellationToken); + return await reconciler.ReconcileModification(reconciliationContext, cancellationToken); case WatchEventType.Deleted: - return await reconciler.ReconcileDeletion(entity, cancellationToken); + return await reconciler.ReconcileDeletion(reconciliationContext, cancellationToken); default: logger.LogWarning( From efa9a91fcdc214c8609c9f4d62ee2dc04b677680 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 10 Oct 2025 15:08:05 +0200 Subject: [PATCH 34/58] refactor(queue): make `Enqueue` and `Remove` methods asynchronous - Updated the `ITimedEntityQueue` interface to return `Task` for `Enqueue` and `Remove` methods. - Adjusted implementations in `TimedEntityQueue` to comply with async method signatures. - Updated relevant tests and reconciler logic to use `await` with `Enqueue` and `Remove` calls. --- ...Queue.cs => ITimedEntityQueue{TEntity}.cs} | 16 +++++++++------- .../Queue/TimedEntityQueue.cs | 10 +++++++--- .../Reconciliation/Reconciler.cs | 19 ++++++++++--------- .../Queue/TimedEntityQueue.Test.cs | 4 ++-- 4 files changed, 28 insertions(+), 21 deletions(-) rename src/KubeOps.Operator/Queue/{ITimedEntityQueue.cs => ITimedEntityQueue{TEntity}.cs} (62%) diff --git a/src/KubeOps.Operator/Queue/ITimedEntityQueue.cs b/src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs similarity index 62% rename from src/KubeOps.Operator/Queue/ITimedEntityQueue.cs rename to src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs index 5842a12e..4daf3c0e 100644 --- a/src/KubeOps.Operator/Queue/ITimedEntityQueue.cs +++ b/src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs @@ -20,16 +20,18 @@ public interface ITimedEntityQueue : IDisposable, IAsyncEnumerable { /// - /// Adds the specified entity to the queue for processing after the specified time span has expired. + /// Adds the specified entity to the queue for processing after the specified time span has elapsed. /// - /// The entity to be added to the queue. - /// The type of requeue operation to be performed. - /// The duration after which the entity should be requeued. - void Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn); + /// The entity to be queued. + /// The type of requeue operation to handle (added, modified, or deleted). + /// The duration to wait before processing the entity. + /// A task that represents the asynchronous operation of enqueuing the entity. + Task Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn); /// - /// Removes the specified from the queue. + /// Removes the specified entity from the queue. /// /// The entity to be removed from the queue. - void Remove(TEntity entity); + /// A task that represents the asynchronous operation of removing the entity from the queue. + Task Remove(TEntity entity); } diff --git a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs index ef1756af..a45db52d 100644 --- a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs +++ b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs @@ -35,7 +35,7 @@ public sealed class TimedEntityQueue( internal int Count => _management.Count; /// - public void Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn) + public Task Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn) { _management .AddOrUpdate( @@ -77,6 +77,8 @@ public void Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn) entry.Token); return entry; }); + + return Task.CompletedTask; } /// @@ -100,17 +102,19 @@ public async IAsyncEnumerator> GetAsyncEnumerator(Cancella } /// - public void Remove(TEntity entity) + public Task Remove(TEntity entity) { var key = this.GetKey(entity); if (key is null) { - return; + return Task.CompletedTask; } if (_management.Remove(key, out var task)) { task.Cancel(); } + + return Task.CompletedTask; } } diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index 22bab4bf..378beb3a 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -49,7 +49,7 @@ internal sealed class Reconciler( public async Task> ReconcileCreation(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - requeue.Remove(reconciliationContext.Entity); + await requeue.Remove(reconciliationContext.Entity); if (reconciliationContext.Entity.Metadata.DeletionTimestamp is not null) { @@ -88,7 +88,7 @@ await _entityCache.SetAsync( if (result.RequeueAfter.HasValue) { - requeue.Enqueue( + await requeue.Enqueue( result.Entity, result.IsSuccess ? RequeueType.Modified : RequeueType.Added, result.RequeueAfter.Value); @@ -99,7 +99,7 @@ await _entityCache.SetAsync( public async Task> ReconcileModification(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - requeue.Remove(reconciliationContext.Entity); + await requeue.Remove(reconciliationContext.Entity); ReconciliationResult reconciliationResult; @@ -145,7 +145,7 @@ await _entityCache.SetAsync( if (reconciliationResult.RequeueAfter.HasValue) { - requeue + await requeue .Enqueue( reconciliationResult.Entity, RequeueType.Modified, @@ -157,7 +157,7 @@ await _entityCache.SetAsync( public async Task> ReconcileDeletion(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - requeue.Remove(reconciliationContext.Entity); + await requeue.Remove(reconciliationContext.Entity); await using var scope = provider.CreateAsyncScope(); var controller = scope.ServiceProvider.GetRequiredService>(); @@ -170,10 +170,11 @@ public async Task> ReconcileDeletion(Reconciliatio if (result.RequeueAfter.HasValue) { - requeue.Enqueue( - result.Entity, - RequeueType.Deleted, - result.RequeueAfter.Value); + await requeue + .Enqueue( + result.Entity, + RequeueType.Deleted, + result.RequeueAfter.Value); } return result; diff --git a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs index 6cf9ce65..4f6cbe43 100644 --- a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs +++ b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs @@ -20,8 +20,8 @@ public async Task Can_Enqueue_Multiple_Entities_With_Same_Name() { var queue = new TimedEntityQueue(Mock.Of>>()); - queue.Enqueue(CreateSecret("app-ns1", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1)); - queue.Enqueue(CreateSecret("app-ns2", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1)); + await queue.Enqueue(CreateSecret("app-ns1", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1)); + await queue.Enqueue(CreateSecret("app-ns2", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1)); var items = new List(); From b215401a81fdcc5cc989d3ff9bd9dff84c23c35a Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 10 Oct 2025 15:18:43 +0200 Subject: [PATCH 35/58] refactor(queue): add `CancellationToken` support for `Enqueue` and `Remove` - Updated `EntityRequeue` delegate and `ITimedEntityQueue` interface to include `CancellationToken` parameters. - Adjusted implementations in `TimedEntityQueue` and `Reconciler` to pass and handle `CancellationToken`. - Updated tests to reflect the new method signatures and ensure cancellation token usage. --- .../Reconciliation/Queue/EntityRequeue.cs | 48 ++++--------------- .../Queue/ITimedEntityQueue{TEntity}.cs | 6 ++- .../Queue/KubeOpsEntityRequeueFactory.cs | 4 +- .../Queue/TimedEntityQueue.cs | 4 +- .../Reconciliation/Reconciler.cs | 40 +++++++++++----- .../CancelEntityRequeue.Integration.Test.cs | 4 +- .../DeletedEntityRequeue.Integration.Test.cs | 4 +- .../EntityRequeue.Integration.Test.cs | 4 +- .../Events/EventPublisher.Integration.Test.cs | 4 +- .../Queue/TimedEntityQueue.Test.cs | 4 +- 10 files changed, 53 insertions(+), 69 deletions(-) diff --git a/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs b/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs index 34af3cc2..1d88b9fa 100644 --- a/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs +++ b/src/KubeOps.Abstractions/Reconciliation/Queue/EntityRequeue.cs @@ -8,45 +8,13 @@ namespace KubeOps.Abstractions.Reconciliation.Queue; /// -/// Injectable delegate for requeuing entities. -/// -/// Use this delegate when you need to pro-actively reconcile an entity after a -/// certain amount of time. This is useful, if you want to check your entities -/// periodically. -/// -/// -/// After the timeout is reached, the entity is fetched -/// from the API and passed to the controller for reconciliation. -/// If the entity was deleted in the meantime, the controller will not be called. -/// -/// -/// If the entity gets modified while the timeout is running, the timer -/// is canceled and restarted, if another requeue is requested. -/// +/// Injectable delegate for scheduling an entity to be requeued after a specified amount of time. /// -/// The type of the entity. -/// The instance of the entity that should be requeued. -/// The type of which the reconcile operation should be executed. -/// The time to wait before another reconcile loop is fired. -/// -/// Use the requeue delegate to repeatedly reconcile an entity after 5 seconds. -/// -/// [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -/// public class V1TestEntityController : IEntityController<V1TestEntity> -/// { -/// private readonly EntityRequeue<V1TestEntity> _requeue; -/// -/// public V1TestEntityController(EntityRequeue<V1TestEntity> requeue) -/// { -/// _requeue = requeue; -/// } -/// -/// public async Task ReconcileAsync(V1TestEntity entity, CancellationToken token) -/// { -/// _requeue(entity, TimeSpan.FromSeconds(5)); -/// } -/// } -/// -/// -public delegate void EntityRequeue(TEntity entity, RequeueType type, TimeSpan requeueIn) +/// The type of the Kubernetes entity being requeued. +/// The entity instance that should be requeued. +/// The type of operation for which the reconcile behavior should be performed. +/// The duration to wait before triggering the next reconcile process. +/// A cancellation token to observe while waiting for the requeue duration. +public delegate void EntityRequeue( + TEntity entity, RequeueType type, TimeSpan requeueIn, CancellationToken cancellationToken) where TEntity : IKubernetesObject; diff --git a/src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs b/src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs index 4daf3c0e..9aa26af0 100644 --- a/src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs +++ b/src/KubeOps.Operator/Queue/ITimedEntityQueue{TEntity}.cs @@ -25,13 +25,15 @@ public interface ITimedEntityQueue : IDisposable, IAsyncEnumerableThe entity to be queued. /// The type of requeue operation to handle (added, modified, or deleted). /// The duration to wait before processing the entity. + /// A token that can be used to cancel the asynchronous operation. /// A task that represents the asynchronous operation of enqueuing the entity. - Task Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn); + Task Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn, CancellationToken cancellationToken); /// /// Removes the specified entity from the queue. /// /// The entity to be removed from the queue. + /// A token that can be used to cancel the asynchronous operation. /// A task that represents the asynchronous operation of removing the entity from the queue. - Task Remove(TEntity entity); + Task Remove(TEntity entity, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs b/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs index d7792489..f12fe070 100644 --- a/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs +++ b/src/KubeOps.Operator/Queue/KubeOpsEntityRequeueFactory.cs @@ -17,7 +17,7 @@ internal sealed class KubeOpsEntityRequeueFactory(IServiceProvider services) { public EntityRequeue Create() where TEntity : IKubernetesObject => - (entity, type, timeSpan) => + (entity, type, timeSpan, cancellationToken) => { var logger = services.GetService>>(); var queue = services.GetRequiredService>(); @@ -28,6 +28,6 @@ public EntityRequeue Create() entity.Name(), timeSpan.TotalMilliseconds); - queue.Enqueue(entity, type, timeSpan); + queue.Enqueue(entity, type, timeSpan, cancellationToken); }; } diff --git a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs index a45db52d..5852df8e 100644 --- a/src/KubeOps.Operator/Queue/TimedEntityQueue.cs +++ b/src/KubeOps.Operator/Queue/TimedEntityQueue.cs @@ -35,7 +35,7 @@ public sealed class TimedEntityQueue( internal int Count => _management.Count; /// - public Task Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn) + public Task Enqueue(TEntity entity, RequeueType type, TimeSpan requeueIn, CancellationToken cancellationToken) { _management .AddOrUpdate( @@ -102,7 +102,7 @@ public async IAsyncEnumerator> GetAsyncEnumerator(Cancella } /// - public Task Remove(TEntity entity) + public Task Remove(TEntity entity, CancellationToken cancellationToken) { var key = this.GetKey(entity); if (key is null) diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index 378beb3a..fd5b8805 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -49,7 +49,10 @@ internal sealed class Reconciler( public async Task> ReconcileCreation(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - await requeue.Remove(reconciliationContext.Entity); + await requeue + .Remove( + reconciliationContext.Entity, + cancellationToken); if (reconciliationContext.Entity.Metadata.DeletionTimestamp is not null) { @@ -78,20 +81,23 @@ public async Task> ReconcileCreation(Reconciliatio return ReconciliationResult.Success(reconciliationContext.Entity); } - await _entityCache.SetAsync( - reconciliationContext.Entity.Uid(), - reconciliationContext.Entity.Generation() ?? 0, - token: cancellationToken); + await _entityCache + .SetAsync( + reconciliationContext.Entity.Uid(), + reconciliationContext.Entity.Generation() ?? 0, + token: cancellationToken); } var result = await ReconcileModificationAsync(reconciliationContext.Entity, cancellationToken); if (result.RequeueAfter.HasValue) { - await requeue.Enqueue( - result.Entity, - result.IsSuccess ? RequeueType.Modified : RequeueType.Added, - result.RequeueAfter.Value); + await requeue + .Enqueue( + result.Entity, + result.IsSuccess ? RequeueType.Modified : RequeueType.Added, + result.RequeueAfter.Value, + cancellationToken); } return result; @@ -99,7 +105,10 @@ await requeue.Enqueue( public async Task> ReconcileModification(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - await requeue.Remove(reconciliationContext.Entity); + await requeue + .Remove( + reconciliationContext.Entity, + cancellationToken); ReconciliationResult reconciliationResult; @@ -149,7 +158,8 @@ await requeue .Enqueue( reconciliationResult.Entity, RequeueType.Modified, - reconciliationResult.RequeueAfter.Value); + reconciliationResult.RequeueAfter.Value, + cancellationToken); } return reconciliationResult; @@ -157,7 +167,10 @@ await requeue public async Task> ReconcileDeletion(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - await requeue.Remove(reconciliationContext.Entity); + await requeue + .Remove( + reconciliationContext.Entity, + cancellationToken); await using var scope = provider.CreateAsyncScope(); var controller = scope.ServiceProvider.GetRequiredService>(); @@ -174,7 +187,8 @@ await requeue .Enqueue( result.Entity, RequeueType.Deleted, - result.RequeueAfter.Value); + result.RequeueAfter.Value, + cancellationToken); } return result; diff --git a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs index 4f732a6b..007811f3 100644 --- a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs @@ -16,7 +16,7 @@ namespace KubeOps.Operator.Test.Controller; -public class CancelEntityRequeueIntegrationTest : IntegrationTestBase +public sealed class CancelEntityRequeueIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -82,7 +82,7 @@ public Task> ReconcileAsyn svc.Invocation(entity); if (svc.Invocations.Count < 2) { - requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1000)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1000), CancellationToken.None); } return Task.FromResult(ReconciliationResult.Success(entity)); diff --git a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs index 952a5346..d4471f2f 100644 --- a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs @@ -16,7 +16,7 @@ namespace KubeOps.Operator.Test.Controller; -public class DeletedEntityRequeueIntegrationTest : IntegrationTestBase +public sealed class DeletedEntityRequeueIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -63,7 +63,7 @@ private class TestController(InvocationCounter public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { svc.Invocation(entity); - requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1000)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1000), CancellationToken.None); return Task.FromResult(ReconciliationResult.Success(entity)); } diff --git a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs index 5ce6f3a1..2101286b 100644 --- a/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/EntityRequeue.Integration.Test.cs @@ -15,7 +15,7 @@ namespace KubeOps.Operator.Test.Controller; -public class EntityRequeueIntegrationTest : IntegrationTestBase +public sealed class EntityRequeueIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -94,7 +94,7 @@ public Task> ReconcileAsyn svc.Invocation(entity); if (svc.Invocations.Count <= svc.TargetInvocationCount) { - requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(1), CancellationToken.None); } return Task.FromResult(ReconciliationResult.Success(entity)); diff --git a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs index e41ff4e8..60d4334b 100644 --- a/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Events/EventPublisher.Integration.Test.cs @@ -21,7 +21,7 @@ namespace KubeOps.Operator.Test.Events; -public class EventPublisherIntegrationTest : IntegrationTestBase +public sealed class EventPublisherIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -98,7 +98,7 @@ public async Task> Reconci if (svc.Invocations.Count < svc.TargetInvocationCount) { - requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(10)); + requeue(entity, RequeueType.Modified, TimeSpan.FromMilliseconds(10), CancellationToken.None); } return ReconciliationResult.Success(entity); diff --git a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs index 4f6cbe43..8f275244 100644 --- a/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs +++ b/test/KubeOps.Operator.Test/Queue/TimedEntityQueue.Test.cs @@ -20,8 +20,8 @@ public async Task Can_Enqueue_Multiple_Entities_With_Same_Name() { var queue = new TimedEntityQueue(Mock.Of>>()); - await queue.Enqueue(CreateSecret("app-ns1", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1)); - await queue.Enqueue(CreateSecret("app-ns2", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1)); + await queue.Enqueue(CreateSecret("app-ns1", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1), CancellationToken.None); + await queue.Enqueue(CreateSecret("app-ns2", "secret-name"), RequeueType.Modified, TimeSpan.FromSeconds(1), CancellationToken.None); var items = new List(); From daeba95337c410771bc3ab8e3ab2aa433f960389 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Tue, 21 Oct 2025 11:40:13 +0200 Subject: [PATCH 36/58] refactor(queue): remove unused `Reconciliation` import in `EntityRequeueBackgroundService` --- src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs index 21a6dce3..eb2c1177 100644 --- a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs +++ b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs @@ -8,7 +8,6 @@ using KubeOps.Abstractions.Reconciliation; using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; -using KubeOps.Operator.Reconciliation; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; From 24f2ca73c15dbe6f23921a1c3b098819e1560be5 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Tue, 21 Oct 2025 14:02:32 +0200 Subject: [PATCH 37/58] refactor(queue): add `JsonConstructor` to `RequeueEntry` --- src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs b/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs index f21f2a4e..53e852bb 100644 --- a/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs +++ b/src/KubeOps.Operator/Queue/RequeueEntry{TEntity}.cs @@ -2,12 +2,15 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. +using System.Text.Json.Serialization; + using KubeOps.Abstractions.Reconciliation.Queue; namespace KubeOps.Operator.Queue; public sealed record RequeueEntry { + [JsonConstructor] private RequeueEntry(TEntity entity, RequeueType requeueType) { Entity = entity; From 5b0fc66f25fd6589b0c1e57c219e01df1e86e881 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Thu, 30 Oct 2025 11:03:24 +0100 Subject: [PATCH 38/58] refactor(reconciliation): consolidate event-specific reconciliation methods into unified `Reconcile` - Replaced separate `ReconcileCreation`, `ReconcileModification`, and `ReconcileDeletion` methods with a single `Reconcile` method. - Enhanced `ReconciliationContext` to include `WatchEventType` for event context, improving flexibility and code clarity. - Updated `RequeueTypeExtensions` to support conversions between `WatchEventType` and `RequeueType`. - Simplified `OnEventAsync` logic in `ResourceWatcher` to leverage the unified `Reconcile` method. - Adjusted all relevant interfaces and calls to align with the refactored reconciliation approach. --- .../Reconciliation/IReconciler{TEntity}.cs | 26 +--- .../ReconciliationContext{TEntity}.cs | 44 +++--- .../Queue/EntityRequeueBackgroundService.cs | 29 ++-- .../Queue/RequeueTypeExtensions.cs | 24 ++++ .../Reconciliation/Reconciler.cs | 132 +++++------------- .../Watcher/ResourceWatcher{TEntity}.cs | 30 +--- 6 files changed, 101 insertions(+), 184 deletions(-) diff --git a/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs index 63d22594..15892a9a 100644 --- a/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs +++ b/src/KubeOps.Abstractions/Reconciliation/IReconciler{TEntity}.cs @@ -19,26 +19,10 @@ public interface IReconciler where TEntity : IKubernetesObject { /// - /// Handles the reconciliation process when a new entity is created. + /// Handles the reconciliation process for a Kubernetes entity. /// - /// The context of the entity to reconcile during its creation. - /// A token to monitor for cancellation requests. - /// A task representing the asynchronous operation, yielding the result of the reconciliation process. - Task> ReconcileCreation(ReconciliationContext reconciliationContext, CancellationToken cancellationToken); - - /// - /// Handles the reconciliation process when an existing entity is modified. - /// - /// The context of the entity to reconcile during its creation. - /// A token to monitor for cancellation requests. - /// A task representing the asynchronous operation, with a result of the reconciliation process. - Task> ReconcileModification(ReconciliationContext reconciliationContext, CancellationToken cancellationToken); - - /// - /// Handles the reconciliation process when an entity is deleted. - /// - /// The context of the entity to reconcile during its creation. - /// A token to monitor for cancellation requests. - /// A task representing the asynchronous operation, with a result of the reconciliation process. - Task> ReconcileDeletion(ReconciliationContext reconciliationContext, CancellationToken cancellationToken); + /// The context containing details of the entity to reconcile. + /// A token to monitor for cancellation requests during the reconciliation process. + /// A task that represents the asynchronous reconciliation operation, returning the result of the reconciliation process. + Task> Reconcile(ReconciliationContext reconciliationContext, CancellationToken cancellationToken); } diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs index e63f8f9e..539ed06a 100644 --- a/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationContext{TEntity}.cs @@ -19,9 +19,10 @@ namespace KubeOps.Abstractions.Reconciliation; public sealed record ReconciliationContext where TEntity : IKubernetesObject { - private ReconciliationContext(TEntity entity, ReconciliationTriggerSource reconciliationTriggerSource) + private ReconciliationContext(TEntity entity, WatchEventType eventType, ReconciliationTriggerSource reconciliationTriggerSource) { Entity = entity; + EventType = eventType; ReconciliationTriggerSource = reconciliationTriggerSource; } @@ -30,38 +31,47 @@ private ReconciliationContext(TEntity entity, ReconciliationTriggerSource reconc /// public TEntity Entity { get; } + /// + /// Specifies the type of Kubernetes watch event that triggered the reconciliation process. + /// This property provides information about the nature of the change detected + /// within the Kubernetes resource, such as addition, modification, or deletion. + /// + public WatchEventType EventType { get; } + /// /// Specifies the source that initiated the reconciliation process. /// public ReconciliationTriggerSource ReconciliationTriggerSource { get; } /// - /// Creates a new instance of the class - /// using an event triggered by the Kubernetes API server as the source of reconciliation. + /// Creates a new instance of from an API server event. /// /// - /// The Kubernetes resource instance that is being reconciled. This must implement - /// . + /// The Kubernetes entity associated with the reconciliation context. + /// + /// + /// The type of watch event that triggered the context creation. /// /// - /// A new instance of with - /// as the trigger source. + /// A new instance representing the reconciliation context + /// for the specified entity and event type, triggered by the API server. /// - public static ReconciliationContext CreateFromApiServerEvent(TEntity entity) - => new(entity, ReconciliationTriggerSource.ApiServer); + public static ReconciliationContext CreateFromApiServerEvent(TEntity entity, WatchEventType eventType) + => new(entity, eventType, ReconciliationTriggerSource.ApiServer); /// - /// Creates a new instance of the class - /// using an event triggered by the operator as the source of reconciliation. + /// Creates a new instance of from an operator-driven event. /// /// - /// The Kubernetes resource instance that is being reconciled. This must implement - /// . + /// The Kubernetes entity associated with the reconciliation context. + /// + /// + /// The type of watch event that triggered the context creation. /// /// - /// A new instance of with - /// as the trigger source. + /// A new instance representing the reconciliation context + /// for the specified entity and event type, triggered by the operator. /// - public static ReconciliationContext CreateFromOperatorEvent(TEntity entity) - => new(entity, ReconciliationTriggerSource.Operator); + public static ReconciliationContext CreateFromOperatorEvent(TEntity entity, WatchEventType eventType) + => new(entity, eventType, ReconciliationTriggerSource.Operator); } diff --git a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs index eb2c1177..e59c896b 100644 --- a/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs +++ b/src/KubeOps.Operator/Queue/EntityRequeueBackgroundService.cs @@ -104,34 +104,23 @@ static async ValueTask CastAndDispose(IDisposable resource) } } - protected virtual async Task ReconcileSingleAsync(RequeueEntry queuedEntry, CancellationToken cancellationToken) + protected virtual async Task ReconcileSingleAsync(RequeueEntry entry, CancellationToken cancellationToken) { - logger.LogTrace("""Execute requested requeued reconciliation for "{Name}".""", queuedEntry.Entity.Name()); + logger.LogTrace("""Execute requested requeued reconciliation for "{Name}".""", entry.Entity.Name()); - if (await client.GetAsync(queuedEntry.Entity.Name(), queuedEntry.Entity.Namespace(), cancellationToken) is not + if (await client.GetAsync(entry.Entity.Name(), entry.Entity.Namespace(), cancellationToken) is not { } entity) { logger.LogWarning( - """Requeued entity "{Name}" was not found. Skipping reconciliation.""", queuedEntry.Entity.Name()); + """Requeued entity "{Name}" was not found. Skipping reconciliation.""", entry.Entity.Name()); return; } - var reconciliationContext = ReconciliationContext.CreateFromOperatorEvent(entity); - - switch (queuedEntry.RequeueType) - { - case RequeueType.Added: - await reconciler.ReconcileCreation(reconciliationContext, cancellationToken); - break; - case RequeueType.Modified: - await reconciler.ReconcileModification(reconciliationContext, cancellationToken); - break; - case RequeueType.Deleted: - await reconciler.ReconcileDeletion(reconciliationContext, cancellationToken); - break; - default: - throw new NotSupportedException($"RequeueType '{queuedEntry.RequeueType}' is not supported!"); - } + await reconciler.Reconcile( + ReconciliationContext.CreateFromOperatorEvent( + entity, + entry.RequeueType.ToWatchEventType()), + cancellationToken); } private async Task WatchAsync(CancellationToken cancellationToken) diff --git a/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs b/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs index 630cc9a4..50a78fd5 100644 --- a/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs +++ b/src/KubeOps.Operator/Queue/RequeueTypeExtensions.cs @@ -8,8 +8,17 @@ namespace KubeOps.Operator.Queue; +/// +/// Provides extension methods for converting between and . +/// public static class RequeueTypeExtensions { + /// + /// Converts a to its corresponding . + /// + /// The watch event type to be converted. + /// The corresponding for the given . + /// Thrown when the provided is not supported. public static RequeueType ToRequeueType(this WatchEventType watchEventType) => watchEventType switch { @@ -18,4 +27,19 @@ public static RequeueType ToRequeueType(this WatchEventType watchEventType) WatchEventType.Deleted => RequeueType.Deleted, _ => throw new NotSupportedException($"WatchEventType '{watchEventType}' is not supported!"), }; + + /// + /// Converts a to its corresponding . + /// + /// The requeue type to be converted. + /// The corresponding for the given . + /// Thrown when the provided is not supported. + public static WatchEventType ToWatchEventType(this RequeueType requeueType) + => requeueType switch + { + RequeueType.Added => WatchEventType.Added, + RequeueType.Modified => WatchEventType.Modified, + RequeueType.Deleted => WatchEventType.Deleted, + _ => throw new NotSupportedException($"RequeueType '{requeueType}' is not supported!"), + }; } diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index fd5b8805..1679041b 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -9,7 +9,6 @@ using KubeOps.Abstractions.Reconciliation; using KubeOps.Abstractions.Reconciliation.Controller; using KubeOps.Abstractions.Reconciliation.Finalizer; -using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Constants; using KubeOps.Operator.Queue; @@ -47,55 +46,28 @@ internal sealed class Reconciler( { private readonly IFusionCache _entityCache = cacheProvider.GetCache(CacheConstants.CacheNames.ResourceWatcher); - public async Task> ReconcileCreation(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) + public async Task> Reconcile(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { await requeue .Remove( reconciliationContext.Entity, cancellationToken); - if (reconciliationContext.Entity.Metadata.DeletionTimestamp is not null) + var result = reconciliationContext.EventType switch { - logger.LogDebug( - """Received ADDED event for entity "{Kind}/{Name}" which already has a deletion timestamp "{DeletionTimestamp}". Skip event.""", - reconciliationContext.Entity.Kind, - reconciliationContext.Entity.Name(), - reconciliationContext.Entity.Metadata.DeletionTimestamp.Value.ToString("O")); - - return ReconciliationResult.Success(reconciliationContext.Entity); - } - - if (reconciliationContext.IsTriggeredByApiServer()) - { - var cachedGeneration = - await _entityCache.TryGetAsync(reconciliationContext.Entity.Uid(), token: cancellationToken); - - // Only perform reconciliation if the entity was not already in the cache. - if (cachedGeneration.HasValue) - { - logger.LogDebug( - """Received ADDED event for entity "{Kind}/{Name}" which was already in the cache. Skip event.""", - reconciliationContext.Entity.Kind, - reconciliationContext.Entity.Name()); - - return ReconciliationResult.Success(reconciliationContext.Entity); - } - - await _entityCache - .SetAsync( - reconciliationContext.Entity.Uid(), - reconciliationContext.Entity.Generation() ?? 0, - token: cancellationToken); - } - - var result = await ReconcileModificationAsync(reconciliationContext.Entity, cancellationToken); + WatchEventType.Added or WatchEventType.Modified => + await ReconcileModification(reconciliationContext, cancellationToken), + WatchEventType.Deleted => + await ReconcileDeletion(reconciliationContext, cancellationToken), + _ => throw new NotSupportedException($"Reconciliation event type {reconciliationContext.EventType} is not supported!"), + }; if (result.RequeueAfter.HasValue) { await requeue .Enqueue( result.Entity, - result.IsSuccess ? RequeueType.Modified : RequeueType.Added, + reconciliationContext.EventType.ToRequeueType(), result.RequeueAfter.Value, cancellationToken); } @@ -105,13 +77,6 @@ await requeue public async Task> ReconcileModification(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - await requeue - .Remove( - reconciliationContext.Entity, - cancellationToken); - - ReconciliationResult reconciliationResult; - switch (reconciliationContext.Entity) { case { Metadata.DeletionTimestamp: null }: @@ -139,39 +104,16 @@ await _entityCache.SetAsync( token: cancellationToken); } - reconciliationResult = await ReconcileModificationAsync(reconciliationContext.Entity, cancellationToken); - - break; + return await ReconcileModificationAsync(reconciliationContext.Entity, cancellationToken); case { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } }: - reconciliationResult = await ReconcileFinalizersSequentialAsync(reconciliationContext.Entity, cancellationToken); - - break; + return await ReconcileFinalizersSequentialAsync(reconciliationContext.Entity, cancellationToken); default: - reconciliationResult = ReconciliationResult.Success(reconciliationContext.Entity); - - break; - } - - if (reconciliationResult.RequeueAfter.HasValue) - { - await requeue - .Enqueue( - reconciliationResult.Entity, - RequeueType.Modified, - reconciliationResult.RequeueAfter.Value, - cancellationToken); + return ReconciliationResult.Success(reconciliationContext.Entity); } - - return reconciliationResult; } - public async Task> ReconcileDeletion(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) + private async Task> ReconcileDeletion(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - await requeue - .Remove( - reconciliationContext.Entity, - cancellationToken); - await using var scope = provider.CreateAsyncScope(); var controller = scope.ServiceProvider.GetRequiredService>(); var result = await controller.DeletedAsync(reconciliationContext.Entity, cancellationToken); @@ -181,24 +123,34 @@ await requeue await _entityCache.RemoveAsync(reconciliationContext.Entity.Uid(), token: cancellationToken); } - if (result.RequeueAfter.HasValue) + return result; + } + + private async Task> ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) + { + await using var scope = provider.CreateAsyncScope(); + + if (settings.AutoAttachFinalizers) { - await requeue - .Enqueue( - result.Entity, - RequeueType.Deleted, - result.RequeueAfter.Value, - cancellationToken); + var finalizers = scope.ServiceProvider.GetKeyedServices>(KeyedService.AnyKey); + + foreach (var finalizer in finalizers) + { + entity.AddFinalizer(finalizer.GetIdentifierName(entity)); + } + + entity = await client.UpdateAsync(entity, cancellationToken); } - return result; + var controller = scope.ServiceProvider.GetRequiredService>(); + return await controller.ReconcileAsync(entity, cancellationToken); } private async Task> ReconcileFinalizersSequentialAsync(TEntity entity, CancellationToken cancellationToken) { await using var scope = provider.CreateAsyncScope(); - // condition to call ReconcileFinalizersSequentialAsync is: + // the condition to call ReconcileFinalizersSequentialAsync is: // { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } } // which implies that there is at least a single finalizer var identifier = entity.Finalizers()[0]; @@ -237,24 +189,4 @@ private async Task> ReconcileFinalizersSequentialA return ReconciliationResult.Success(entity, result.RequeueAfter); } - - private async Task> ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) - { - await using var scope = provider.CreateAsyncScope(); - - if (settings.AutoAttachFinalizers) - { - var finalizers = scope.ServiceProvider.GetKeyedServices>(KeyedService.AnyKey); - - foreach (var finalizer in finalizers) - { - entity.AddFinalizer(finalizer.GetIdentifierName(entity)); - } - - entity = await client.UpdateAsync(entity, cancellationToken); - } - - var controller = scope.ServiceProvider.GetRequiredService>(); - return await controller.ReconcileAsync(entity, cancellationToken); - } } diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index 52789c2a..85e9b895 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -125,32 +125,10 @@ static async ValueTask CastAndDispose(IDisposable resource) } } - protected virtual async Task> OnEventAsync(WatchEventType type, TEntity entity, CancellationToken cancellationToken) - { - var reconciliationContext = ReconciliationContext.CreateFromApiServerEvent(entity); - - switch (type) - { - case WatchEventType.Added: - return await reconciler.ReconcileCreation(reconciliationContext, cancellationToken); - - case WatchEventType.Modified: - return await reconciler.ReconcileModification(reconciliationContext, cancellationToken); - - case WatchEventType.Deleted: - return await reconciler.ReconcileDeletion(reconciliationContext, cancellationToken); - - default: - logger.LogWarning( - """Received unsupported event "{EventType}" for "{Kind}/{Name}".""", - type, - entity.Kind, - entity.Name()); - break; - } - - return ReconciliationResult.Success(entity); - } + protected virtual async Task> OnEventAsync(WatchEventType eventType, TEntity entity, CancellationToken cancellationToken) + => await reconciler.Reconcile( + ReconciliationContext.CreateFromApiServerEvent(entity, eventType), + cancellationToken); private async Task WatchClientEventsAsync(CancellationToken stoppingToken) { From 254b646c4c469e917f4a264bc9ca5474e0ea0434 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Thu, 30 Oct 2025 11:19:05 +0100 Subject: [PATCH 39/58] refactor(tests): remove unused `Reconciliation` folder entry from `.csproj` --- .../KubeOps.Abstractions.Test.csproj | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj b/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj index 425967cf..72f8a045 100644 --- a/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj +++ b/test/KubeOps.Abstractions.Test/KubeOps.Abstractions.Test.csproj @@ -2,7 +2,4 @@ - - - - + \ No newline at end of file From a6e01ba8442720e717443ab8366e4ff9964cc1c6 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Thu, 30 Oct 2025 12:36:09 +0100 Subject: [PATCH 40/58] refactor(docs): update reconciliation methods and examples to use `ReconciliationResult` - Replaced `Task` return type with `ReconciliationResult` across reconciliation examples. - Expanded documentation to incorporate success and failure patterns with `ReconciliationResult`. - Updated `DeletedAsync` examples to align with new return structure and error handling. - Enhanced clarity around `RequeueAfter` usage and structured error handling in finalizers and controllers. --- .../operator/building-blocks/controllers.mdx | 201 +++++++++++++++--- .../operator/building-blocks/finalizer.mdx | 111 ++++++++-- docs/docs/operator/events.mdx | 26 ++- docs/docs/operator/getting-started.mdx | 22 +- .../operator/testing/integration-tests.mdx | 13 +- 5 files changed, 308 insertions(+), 65 deletions(-) diff --git a/docs/docs/operator/building-blocks/controllers.mdx b/docs/docs/operator/building-blocks/controllers.mdx index 1aba0b3b..3c060e2c 100644 --- a/docs/docs/operator/building-blocks/controllers.mdx +++ b/docs/docs/operator/building-blocks/controllers.mdx @@ -17,16 +17,22 @@ public class V1DemoEntityController( ILogger logger, IKubernetesClient client) : IEntityController { - public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) + public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); // Implement your reconciliation logic here + return ReconciliationResult.Success(entity); } - public async Task DeletedAsync(V1DemoEntity entity, CancellationToken token) + public async Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { logger.LogInformation("Deleting entity {Entity}.", entity); // Implement your cleanup logic here + return ReconciliationResult.Success(entity); } } ``` @@ -52,41 +58,48 @@ This method is called when: - The operator starts up and discovers existing resources ```csharp -public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Check if required resources exist var deployment = await client.GetAsync( entity.Spec.DeploymentName, - entity.Namespace()); + entity.Namespace(), + cancellationToken); if (deployment == null) { // Create the deployment if it doesn't exist - await client.CreateAsync(new V1Deployment - { - Metadata = new V1ObjectMeta + await client.CreateAsync( + new V1Deployment { - Name = entity.Spec.DeploymentName, - NamespaceProperty = entity.Namespace() + Metadata = new V1ObjectMeta + { + Name = entity.Spec.DeploymentName, + NamespaceProperty = entity.Namespace() + }, + Spec = new V1DeploymentSpec + { + Replicas = entity.Spec.Replicas, + // ... other deployment configuration + } }, - Spec = new V1DeploymentSpec - { - Replicas = entity.Spec.Replicas, - // ... other deployment configuration - } - }); + cancellationToken); } // Update status to reflect current state entity.Status.LastReconciled = DateTime.UtcNow; - await client.UpdateStatusAsync(entity); + await client.UpdateStatusAsync(entity, cancellationToken); + + return ReconciliationResult.Success(entity); } ``` ### DeletedAsync :::warning Important -The `DeletedAsync` method is purely informative and "fire and forget". It is called when a resource is deleted, but it cannot guarantee proper cleanup of resources. For reliable resource cleanup, you must use [Finalizers](./finalizer). +The `DeletedAsync` method is informational only and executes asynchronously without guarantees. While it is called when a resource is deleted, it cannot ensure proper cleanup. For reliable resource cleanup, use [finalizers](./finalizer). ::: This method is called when a resource is deleted, but should only be used for: @@ -96,13 +109,126 @@ This method is called when a resource is deleted, but should only be used for: - Updating external systems about the deletion ```csharp -public async Task DeletedAsync(V1DemoEntity entity, CancellationToken token) +public async Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Log the deletion event logger.LogInformation("Entity {Entity} was deleted.", entity); // Update external systems if needed - await NotifyExternalSystem(entity); + await NotifyExternalSystem(entity, cancellationToken); + + return ReconciliationResult.Success(entity); +} +``` + +## Reconciliation Results + +All reconciliation methods must return a `ReconciliationResult`. This provides a standardized way to communicate the outcome of reconciliation operations. + +### Success Results + +Return a success result when reconciliation completes without errors: + +```csharp +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) +{ + // Perform reconciliation + await ApplyDesiredState(entity, cancellationToken); + + // Return success + return ReconciliationResult.Success(entity); +} +``` + +### Failure Results + +Return a failure result when reconciliation encounters an error: + +```csharp +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) +{ + try + { + await ApplyDesiredState(entity, cancellationToken); + return ReconciliationResult.Success(entity); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to reconcile entity {Name}", entity.Name()); + return ReconciliationResult.Failure( + entity, + "Failed to apply desired state", + ex); + } +} +``` + +### Requeuing Entities + +You can request automatic requeuing by specifying a `requeueAfter` parameter: + +```csharp +// Requeue after 5 minutes +return ReconciliationResult.Success(entity, TimeSpan.FromMinutes(5)); + +// Or set it after creation +var result = ReconciliationResult.Success(entity); +result.RequeueAfter = TimeSpan.FromSeconds(30); +return result; +``` + +This is useful for: +- Polling external resources +- Implementing retry logic with backoff +- Periodic status checks +- Waiting for external dependencies + +### Error Handling with Results + +The `ReconciliationResult` provides structured error handling: + +```csharp +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) +{ + if (!await ValidateConfiguration(entity)) + { + return ReconciliationResult.Failure( + entity, + "Configuration validation failed: Required field 'DeploymentName' is empty", + requeueAfter: TimeSpan.FromMinutes(1)); + } + + try + { + await ReconcileInternal(entity, cancellationToken); + return ReconciliationResult.Success(entity); + } + catch (KubernetesException ex) when (ex.Status.Code == 409) + { + // Conflict - retry after short delay + return ReconciliationResult.Failure( + entity, + "Resource conflict detected", + ex, + requeueAfter: TimeSpan.FromSeconds(5)); + } + catch (Exception ex) + { + logger.LogError(ex, "Unexpected error during reconciliation"); + return ReconciliationResult.Failure( + entity, + $"Reconciliation failed: {ex.Message}", + ex, + requeueAfter: TimeSpan.FromMinutes(5)); + } } ``` @@ -145,38 +271,47 @@ For more details about RBAC configuration, see the [RBAC documentation](../rbac) - Always check the current state before making changes ```csharp -public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Check if required resources exist - if (await IsDesiredState(entity)) + if (await IsDesiredState(entity, cancellationToken)) { - return; + return ReconciliationResult.Success(entity); } // Only make changes if needed - await ApplyDesiredState(entity); + await ApplyDesiredState(entity, cancellationToken); + return ReconciliationResult.Success(entity); } ``` ### Error Handling -- Handle errors gracefully -- Log errors with appropriate context -- Consider implementing retry logic for transient failures +- Use `ReconciliationResult.Failure()` for errors +- Include meaningful error messages +- Use the `requeueAfter` parameter for retry logic +- Preserve exception information ```csharp -public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) +public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { try { - await ReconcileInternal(entity, token); + await ReconcileInternal(entity, cancellationToken); + return ReconciliationResult.Success(entity); } catch (Exception ex) { - logger.LogError(ex, "Error reconciling entity {Entity}", entity); - // Update status to reflect the error - entity.Status.Error = ex.Message; - await client.UpdateStatusAsync(entity); + logger.LogError(ex, "Error reconciling entity {Name}", entity.Name()); + return ReconciliationResult.Failure( + entity, + $"Reconciliation failed: {ex.Message}", + ex, + requeueAfter: TimeSpan.FromMinutes(1)); } } ``` @@ -198,5 +333,5 @@ public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) 1. **Infinite Loops**: Avoid creating reconciliation loops that trigger themselves 2. **Missing Error Handling**: Always handle potential errors 3. **Resource Leaks**: Ensure proper cleanup of resources -4. **Missing RBAC**: Configure appropriate permissions +4. **Missing RBAC Configuration**: Configure appropriate permissions 5. **Status Updates**: Remember that status updates don't trigger reconciliation diff --git a/docs/docs/operator/building-blocks/finalizer.mdx b/docs/docs/operator/building-blocks/finalizer.mdx index fe349c86..04722990 100644 --- a/docs/docs/operator/building-blocks/finalizer.mdx +++ b/docs/docs/operator/building-blocks/finalizer.mdx @@ -25,10 +25,10 @@ When a resource is marked for deletion: 3. Each finalizer must explicitly remove itself after completing its cleanup 4. Only when all finalizers are removed is the resource actually deleted -This mechanism ensures that cleanup operations are: +This mechanism guarantees that cleanup operations are: - Guaranteed to run -- Run in a controlled manner +- Executed in a controlled sequence - Completed before resource deletion ## Implementing Finalizers @@ -40,19 +40,26 @@ public class DemoEntityFinalizer( ILogger logger, IKubernetesClient client) : IEntityFinalizer { - public async Task FinalizeAsync(V1DemoEntity entity, CancellationToken token) + public async Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { logger.LogInformation("Finalizing entity {Entity}", entity); try { // Clean up resources - await CleanupResources(entity); + await CleanupResources(entity, cancellationToken); + return ReconciliationResult.Success(entity); } catch (Exception ex) { logger.LogError(ex, "Error finalizing entity {Entity}", entity); - throw; // Re-throw to prevent finalizer removal + // Return failure to prevent finalizer removal + return ReconciliationResult.Failure( + entity, + $"Finalization failed: {ex.Message}", + ex); } } } @@ -69,18 +76,25 @@ public class V1DemoEntityController( EntityFinalizerAttacher finalizer) : IEntityController { - public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) + public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Attach the finalizer to the entity - entity = await finalizer(entity, token); + entity = await finalizer(entity, cancellationToken); // Continue with reconciliation logic logger.LogInformation("Reconciling entity {Entity}", entity); + + return ReconciliationResult.Success(entity); } - public async Task DeletedAsync(V1DemoEntity entity, CancellationToken token) + public async Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { logger.LogInformation("Entity {Entity} was deleted", entity); + return ReconciliationResult.Success(entity); } } ``` @@ -94,39 +108,50 @@ public class V1DemoEntityController( - Check resource existence before attempting cleanup ```csharp -public async Task FinalizeAsync(V1DemoEntity entity, CancellationToken token) +public async Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Check if resources still exist before cleanup - var resources = await GetResources(entity); + var resources = await GetResources(entity, cancellationToken); if (!resources.Any()) { // Resources already cleaned up - return; + return ReconciliationResult.Success(entity); } // Perform cleanup - await CleanupResources(resources); + await CleanupResources(resources, cancellationToken); + return ReconciliationResult.Success(entity); } ``` ### 2. Error Handling -- Handle errors gracefully +- Use `ReconciliationResult.Failure()` for errors +- Return failure results to prevent finalizer removal - Log errors with appropriate context -- Consider implementing retry logic for transient failures +- Include retry delays when appropriate ```csharp -public async Task FinalizeAsync(V1DemoEntity entity, CancellationToken token) +public async Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { try { - await FinalizeInternal(entity, token); + await FinalizeInternal(entity, cancellationToken); + return ReconciliationResult.Success(entity); } catch (Exception ex) { logger.LogError(ex, "Error finalizing entity {Entity}", entity); - // Re-throw to prevent finalizer removal - throw; + // Return failure to prevent finalizer removal + return ReconciliationResult.Failure( + entity, + $"Finalization failed: {ex.Message}", + ex, + requeueAfter: TimeSpan.FromMinutes(1)); } } ``` @@ -137,6 +162,50 @@ public async Task FinalizeAsync(V1DemoEntity entity, CancellationToken token) - Handle dependencies between resources - Consider cleanup order (e.g., delete pods before services) +### 4. Finalizer Results + +The finalizer must return a `ReconciliationResult`: + +- **Success**: The finalizer is removed, and the entity is deleted +- **Failure**: The finalizer remains, and the entity is requeued for retry + +```csharp +public async Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) +{ + // Check if external resources exist + var externalResource = await GetExternalResource(entity.Spec.ResourceId); + + if (externalResource == null) + { + // Already cleaned up + return ReconciliationResult.Success(entity); + } + + try + { + await DeleteExternalResource(externalResource, cancellationToken); + return ReconciliationResult.Success(entity); + } + catch (Exception ex) when (IsRetryable(ex)) + { + // Transient error - retry after delay + return ReconciliationResult.Failure( + entity, + "Failed to delete external resource", + ex, + requeueAfter: TimeSpan.FromSeconds(30)); + } + catch (Exception ex) + { + // Permanent error - log and succeed to prevent stuck resource + logger.LogError(ex, "Permanent error during finalization, allowing deletion"); + return ReconciliationResult.Success(entity); + } +} +``` + ## Common Pitfalls ### 1. Stuck Resources @@ -145,13 +214,15 @@ If a finalizer fails to complete: - The resource will remain in the cluster - It will be marked for deletion but never actually deleted -- Manual intervention may be required +- The finalizer will be retried based on the `requeueAfter` value +- Manual intervention may be required for permanent failures To fix stuck resources: 1. Identify the failing finalizer 2. Fix the underlying issue -3. Manually remove the finalizer only if necessary: +3. Check if the finalizer is being retried +4. Manually remove the finalizer only if necessary: ```bash kubectl patch -p '{"metadata":{"finalizers":[]}}' --type=merge ``` diff --git a/docs/docs/operator/events.mdx b/docs/docs/operator/events.mdx index 1e3cd645..252e0496 100644 --- a/docs/docs/operator/events.mdx +++ b/docs/docs/operator/events.mdx @@ -64,7 +64,9 @@ KubeOps provides an `EventPublisher` to create and update events for your custom ```csharp public class DemoController(EventPublisher eventPublisher) : IEntityController { - public async Task ReconcileAsync(V1DemoEntity entity, CancellationToken token) + public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { try { @@ -74,7 +76,9 @@ public class DemoController(EventPublisher eventPublisher) : IEntityController.Success(entity); } catch (Exception ex) { @@ -83,9 +87,21 @@ public class DemoController(EventPublisher eventPublisher) : IEntityController.Failure( + entity, + "Reconciliation failed", + ex); } } + + public Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) + { + return Task.FromResult(ReconciliationResult.Success(entity)); + } } ``` @@ -100,8 +116,8 @@ KubeOps provides two event types: 1. **Event Naming**: - - Use consistent, machine-readable reasons - - Make messages human-readable and descriptive + - Use consistent, machine-readable reason codes + - Write human-readable, descriptive messages - Include relevant details in the message 2. **Event Frequency**: diff --git a/docs/docs/operator/getting-started.mdx b/docs/docs/operator/getting-started.mdx index 1221c77e..e9dcab68 100644 --- a/docs/docs/operator/getting-started.mdx +++ b/docs/docs/operator/getting-started.mdx @@ -16,7 +16,7 @@ Before you begin, ensure you have the following installed: - A local Kubernetes cluster (like [kind](https://kind.sigs.k8s.io/) or [minikube](https://minikube.sigs.k8s.io/)) :::warning Development Environment -For local development, we recommend using `kind` or Docker Desktop as it provides a lightweight Kubernetes cluster that's perfect for operator development. Make sure your cluster is running before proceeding with the installation steps. +For local development, we recommend using `kind` or Docker Desktop, which provide lightweight Kubernetes clusters ideal for operator development. Ensure your cluster is running before proceeding with the installation steps. ::: ## Installing KubeOps Templates @@ -150,10 +150,20 @@ Controllers implement the reconciliation logic for your custom resources: [EntityRbac(typeof(V1DemoEntity), Verbs = RbacVerb.All)] public class DemoController : IEntityController { - public Task ReconcileAsync(V1DemoEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Implement your reconciliation logic here - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); + } + + public Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) + { + // Handle deletion event + return Task.FromResult(ReconciliationResult.Success(entity)); } } ``` @@ -165,10 +175,12 @@ Finalizers handle cleanup when resources are deleted: ```csharp public class DemoFinalizer : IEntityFinalizer { - public Task FinalizeAsync(V1DemoEntity entity, CancellationToken cancellationToken) + public Task> FinalizeAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) { // Implement your cleanup logic here - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); } } ``` diff --git a/docs/docs/operator/testing/integration-tests.mdx b/docs/docs/operator/testing/integration-tests.mdx index 2724f57b..a0f4c5d8 100644 --- a/docs/docs/operator/testing/integration-tests.mdx +++ b/docs/docs/operator/testing/integration-tests.mdx @@ -238,10 +238,19 @@ public class EntityControllerIntegrationTest : IntegrationTestBase _svc = svc; } - public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync( + V1OperatorIntegrationTestEntity entity, + CancellationToken cancellationToken) { _svc.Invocation(entity); - return Task.CompletedTask; + return Task.FromResult(ReconciliationResult.Success(entity)); + } + + public Task> DeletedAsync( + V1OperatorIntegrationTestEntity entity, + CancellationToken cancellationToken) + { + return Task.FromResult(ReconciliationResult.Success(entity)); } } } From 53b25139eec7a0e10227ba8d47c87ab4dd657207 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Thu, 30 Oct 2025 12:59:29 +0100 Subject: [PATCH 41/58] docs(operator): add advanced configuration guide and update related sections - Introduced a new "Advanced Configuration" guide covering leader election, durable queues, and finalizer management. - Updated "Deployment" and "Getting Started" guides to reference advanced configuration options. - Provided documentation for custom leader election and durable queue implementations. - Enhanced finalizer documentation to include automated and manual management options with cross-references. - Adjusted sidebar positions to reflect new content structure. --- docs/docs/operator/advanced-configuration.mdx | 592 ++++++++++++++++++ .../operator/building-blocks/controllers.mdx | 4 + .../operator/building-blocks/finalizer.mdx | 4 +- docs/docs/operator/deployment.mdx | 11 +- docs/docs/operator/getting-started.mdx | 1 + docs/docs/operator/testing/_category_.json | 2 +- docs/docs/operator/utilities.mdx | 2 +- 7 files changed, 611 insertions(+), 5 deletions(-) create mode 100644 docs/docs/operator/advanced-configuration.mdx diff --git a/docs/docs/operator/advanced-configuration.mdx b/docs/docs/operator/advanced-configuration.mdx new file mode 100644 index 00000000..76a412c2 --- /dev/null +++ b/docs/docs/operator/advanced-configuration.mdx @@ -0,0 +1,592 @@ +--- +title: Advanced Configuration +description: Advanced Operator Configuration Options +sidebar_position: 8 +--- + +# Advanced Configuration + +This guide covers advanced configuration options for KubeOps operators, including finalizer management, custom leader election, and durable requeue mechanisms. + +## Finalizer Management + +KubeOps provides automatic finalizer attachment and detachment to ensure proper resource cleanup. These features can be configured through `OperatorSettings`. + +### Auto-Attach Finalizers + +By default, KubeOps automatically attaches finalizers to entities during reconciliation. This ensures that cleanup operations are performed before resources are deleted. + +```csharp +builder.Services + .AddKubernetesOperator(settings => + { + // Enable automatic finalizer attachment (default: true) + settings.AutoAttachFinalizers = true; + }); +``` + +When `AutoAttachFinalizers` is enabled: +- Finalizers are automatically added to entities during reconciliation +- You don't need to manually call the `EntityFinalizerAttacher` delegate +- All registered finalizers for an entity type are automatically attached + +When disabled: +```csharp +settings.AutoAttachFinalizers = false; +``` + +You must manually attach finalizers in your controller: + +```csharp +public class V1DemoEntityController( + ILogger logger, + EntityFinalizerAttacher finalizerAttacher) + : IEntityController +{ + public async Task> ReconcileAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) + { + // Manually attach finalizer + entity = await finalizerAttacher(entity, cancellationToken); + + // Continue with reconciliation logic + logger.LogInformation("Reconciling entity {Entity}", entity); + return ReconciliationResult.Success(entity); + } + + public Task> DeletedAsync( + V1DemoEntity entity, + CancellationToken cancellationToken) + { + return Task.FromResult(ReconciliationResult.Success(entity)); + } +} +``` + +### Auto-Detach Finalizers + +KubeOps automatically removes finalizers after successful finalization. This can also be configured: + +```csharp +builder.Services + .AddKubernetesOperator(settings => + { + // Enable automatic finalizer removal (default: true) + settings.AutoDetachFinalizers = true; + }); +``` + +When `AutoDetachFinalizers` is enabled: +- Finalizers are automatically removed when `FinalizeAsync` returns success +- The entity is then allowed to be deleted by Kubernetes + +When disabled: +```csharp +settings.AutoDetachFinalizers = false; +``` + +You must manually manage finalizer removal, which is typically not recommended unless you have specific requirements. + +### Use Cases + +**Keep defaults enabled** when: +- You want standard finalizer behavior +- Your finalizers follow the typical pattern +- You don't need fine-grained control + +**Disable auto-attach** when: +- You need conditional finalizer attachment +- Different instances should have different finalizers +- You want to attach finalizers based on specific conditions + +**Disable auto-detach** when: +- You need custom finalizer removal logic +- You want to coordinate multiple finalizers manually +- You have external systems that need to confirm cleanup + +## Custom Leader Election + +KubeOps supports different leader election mechanisms through the `LeaderElectionType` setting. This allows you to control how multiple operator instances coordinate in a cluster. + +### Leader Election Types + +```csharp +public enum LeaderElectionType +{ + None = 0, // No leader election - all instances process events + Single = 1, // Single leader election - only one instance processes events + Custom = 2 // Custom implementation - user-defined coordination +} +``` + +### Configuration + +```csharp +builder.Services + .AddKubernetesOperator(settings => + { + settings.LeaderElectionType = LeaderElectionType.Single; + settings.LeaderElectionLeaseDuration = TimeSpan.FromSeconds(15); + settings.LeaderElectionRenewDeadline = TimeSpan.FromSeconds(10); + settings.LeaderElectionRetryPeriod = TimeSpan.FromSeconds(2); + }); +``` + +### Custom Leader Election + +The `Custom` leader election type allows you to implement your own coordination logic, such as namespace-based leader election. + +#### Example: Namespace-Based Leader Election + +In some scenarios, you may want different operator instances to handle different namespaces. This enables horizontal scaling while maintaining isolation. + +**Step 1: Implement a custom ResourceWatcher** + +```csharp +public sealed class NamespacedLeaderElectionResourceWatcher( + ActivitySource activitySource, + ILogger> logger, + IReconciler reconciler, + OperatorSettings settings, + IEntityLabelSelector labelSelector, + IKubernetesClient client, + INamespaceLeadershipManager namespaceLeadershipManager) + : ResourceWatcher( + activitySource, + logger, + reconciler, + settings, + labelSelector, + client) + where TEntity : IKubernetesObject +{ + protected override async Task> OnEventAsync( + WatchEventType eventType, + TEntity entity, + CancellationToken cancellationToken) + { + // Check if this instance is responsible for the entity's namespace + if (!await namespaceLeadershipManager.IsResponsibleForNamespace( + entity.Namespace(), + cancellationToken)) + { + // Skip processing - another instance handles this namespace + return ReconciliationResult.Success(entity); + } + + // Process the event + return await base.OnEventAsync(eventType, entity, cancellationToken); + } +} +``` + +**Step 2: Implement the leadership manager** + +```csharp +public interface INamespaceLeadershipManager +{ + Task IsResponsibleForNamespace(string @namespace, CancellationToken cancellationToken); +} + +public class NamespaceLeadershipManager : INamespaceLeadershipManager +{ + private readonly ILeaderElector _leaderElector; + private readonly ConcurrentDictionary _namespaceResponsibility = new(); + + public async Task IsResponsibleForNamespace( + string @namespace, + CancellationToken cancellationToken) + { + // Implement your logic here: + // - Consistent hashing of namespace names + // - Lease-based namespace assignment + // - External coordination service (e.g., etcd, Consul) + + return _namespaceResponsibility.GetOrAdd( + @namespace, + ns => CalculateResponsibility(ns)); + } + + private bool CalculateResponsibility(string @namespace) + { + // Example: Simple hash-based distribution + var instanceId = Environment.GetEnvironmentVariable("POD_NAME") ?? "instance-0"; + var instanceCount = int.Parse( + Environment.GetEnvironmentVariable("REPLICA_COUNT") ?? "1"); + + var namespaceHash = @namespace.GetHashCode(); + var assignedInstance = Math.Abs(namespaceHash % instanceCount); + var currentInstance = int.Parse(instanceId.Split('-').Last()); + + return assignedInstance == currentInstance; + } +} +``` + +**Step 3: Register the custom watcher** + +```csharp +builder.Services + .AddKubernetesOperator(settings => + { + settings.LeaderElectionType = LeaderElectionType.Custom; + }) + .AddSingleton() + .AddHostedService>(); +``` + +### Benefits of Custom Leader Election + +- **Horizontal Scaling**: Multiple instances can process different subsets of resources +- **Namespace Isolation**: Different teams or environments can have dedicated operator instances +- **Geographic Distribution**: Route requests to instances in specific regions +- **Load Balancing**: Distribute work across multiple instances + +## Custom Requeue Mechanism + +By default, KubeOps uses an in-memory queue for requeuing entities. This queue is volatile and does not survive operator restarts. For production scenarios, you may want to implement a durable queue. + +### Default Behavior + +The default `ITimedEntityQueue` implementation: +- Stores requeue entries in memory +- Processes them after the specified delay +- Loses pending requeues on operator restart + +### Implementing a Durable Queue + +You can implement `ITimedEntityQueue` to use external queue systems like Azure Service Bus, RabbitMQ, or AWS SQS. + +#### Example: Azure Service Bus Integration + +**Step 1: Implement ITimedEntityQueue** + +```csharp +public sealed class DurableTimedEntityQueue( + ServiceBusClient serviceBusClient, + IEntityRequeueQueueNameProvider queueNameProvider, + TimeProvider timeProvider) + : ITimedEntityQueue + where TEntity : IKubernetesObject +{ + private readonly ServiceBusSender _sender = serviceBusClient.CreateSender( + queueNameProvider.GetRequeueQueueName()); + + public async Task Enqueue( + TEntity entity, + RequeueType type, + TimeSpan requeueIn, + CancellationToken cancellationToken) + { + var entry = RequeueEntry.CreateFor(entity, type); + var message = new ServiceBusMessage(BinaryData.FromObjectAsJson(entry)); + + // Schedule the message for future delivery + await _sender.ScheduleMessageAsync( + message, + timeProvider.GetUtcNow().Add(requeueIn), + cancellationToken); + } + + public Task Remove(TEntity entity, CancellationToken cancellationToken) + { + // Azure Service Bus doesn't support removing scheduled messages + // Consider using message deduplication or message properties to skip processing + return Task.CompletedTask; + } + + public async IAsyncEnumerator> GetAsyncEnumerator( + CancellationToken cancellationToken = default) + { + // Not used with external queue - processing happens via the processor + await Task.CompletedTask; + yield break; + } + + public void Dispose() + { + _sender.DisposeAsync().AsTask().GetAwaiter().GetResult(); + } +} +``` + +**Step 2: Implement a background service to process messages** + +```csharp +public sealed class DurableEntityRequeueBackgroundService( + ServiceBusClient serviceBusClient, + IKubernetesClient kubernetesClient, + IReconciler reconciler, + IEntityRequeueQueueNameProvider queueNameProvider, + ILogger> logger) + : BackgroundService + where TEntity : IKubernetesObject +{ + private readonly ServiceBusProcessor _processor = serviceBusClient.CreateProcessor( + queueNameProvider.GetRequeueQueueName(), + new ServiceBusProcessorOptions + { + MaxConcurrentCalls = 1, + AutoCompleteMessages = false + }); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _processor.ProcessMessageAsync += ProcessMessageAsync; + _processor.ProcessErrorAsync += ProcessErrorAsync; + + await _processor.StartProcessingAsync(stoppingToken); + + try + { + await Task.Delay(Timeout.Infinite, stoppingToken); + } + catch (OperationCanceledException) + { + // Expected during shutdown + } + } + + private async Task ProcessMessageAsync(ProcessMessageEventArgs args) + { + var entry = args.Message.Body.ToObjectFromJson>(); + + // Verify entity still exists + var entity = await kubernetesClient.GetAsync( + entry.Entity.Name(), + entry.Entity.Namespace(), + args.CancellationToken); + + if (entity == null) + { + logger.LogInformation( + "Skipping reconciliation for deleted entity {Name}", + entry.Entity.Name()); + await args.CompleteMessageAsync(args.Message, args.CancellationToken); + return; + } + + // Complete message before reconciliation to avoid reprocessing + await args.CompleteMessageAsync(args.Message, args.CancellationToken); + + // Trigger reconciliation + await reconciler.Reconcile( + ReconciliationContext.CreateFromOperatorEvent( + entity, + entry.RequeueType.ToWatchEventType()), + args.CancellationToken); + } + + private Task ProcessErrorAsync(ProcessErrorEventArgs args) + { + logger.LogError(args.Exception, "Error processing requeue message"); + return Task.CompletedTask; + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + await _processor.StopProcessingAsync(cancellationToken); + _processor.ProcessMessageAsync -= ProcessMessageAsync; + _processor.ProcessErrorAsync -= ProcessErrorAsync; + await _processor.DisposeAsync(); + await base.StopAsync(cancellationToken); + } +} +``` + +**Step 3: Register the durable queue** + +```csharp +builder.Services + .AddSingleton(sp => + new ServiceBusClient(configuration["ServiceBus:ConnectionString"])) + .AddSingleton() + .AddKubernetesOperator() + .RegisterComponents(); + +// Replace the default queue with the durable implementation +builder.Services.Replace(ServiceDescriptor.Singleton( + typeof(ITimedEntityQueue<>), + typeof(DurableTimedEntityQueue<>))); + +// Add the background service to process messages +builder.Services.AddHostedService>(); +``` + +**Step 4: Create the queue name provider** + +```csharp +public interface IEntityRequeueQueueNameProvider +{ + string GetRequeueQueueName() where TEntity : IKubernetesObject; +} + +public class EntityRequeueQueueNameProvider : IEntityRequeueQueueNameProvider +{ + public string GetRequeueQueueName() + where TEntity : IKubernetesObject + { + return $"operator-requeue-{typeof(TEntity).Name.ToLowerInvariant()}"; + } +} +``` + +### Benefits of Durable Requeues + +- **Persistence**: Requeue requests survive operator restarts +- **Reliability**: Messages are not lost during failures +- **Scalability**: External queue systems can handle high volumes +- **Observability**: Queue metrics provide insights into requeue patterns +- **Coordination**: Multiple operator instances can share the same queue + +### Combining Custom Leader Election with Durable Queues + +For advanced scenarios, you can combine namespace-based leader election with durable queues: + +```csharp +public sealed class NamespacedLeaderElectionEntityRequeueBackgroundService( + ServiceBusClient serviceBusClient, + IKubernetesClient kubernetesClient, + IReconciler reconciler, + IEntityRequeueQueueNameProvider queueNameProvider, + INamespaceLeadershipManager namespaceLeadershipManager, + ILogger> logger) + : BackgroundService + where TEntity : IKubernetesObject +{ + private readonly ServiceBusProcessor _processor = serviceBusClient.CreateProcessor( + queueNameProvider.GetRequeueQueueName(), + new ServiceBusProcessorOptions + { + MaxConcurrentCalls = 1, + AutoCompleteMessages = false + }); + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _processor.ProcessMessageAsync += ProcessMessageAsync; + _processor.ProcessErrorAsync += ProcessErrorAsync; + await _processor.StartProcessingAsync(stoppingToken); + + try + { + await Task.Delay(Timeout.Infinite, stoppingToken); + } + catch (OperationCanceledException) + { + // Expected during shutdown + } + } + + private async Task ProcessMessageAsync(ProcessMessageEventArgs args) + { + var entry = args.Message.Body.ToObjectFromJson>(); + + // Verify entity still exists + var entity = await kubernetesClient.GetAsync( + entry.Entity.Name(), + entry.Entity.Namespace(), + args.CancellationToken); + + if (entity == null) + { + logger.LogInformation("Entity no longer exists, completing message"); + await args.CompleteMessageAsync(args.Message, args.CancellationToken); + return; + } + + // Check if this instance is responsible for the namespace + if (!await namespaceLeadershipManager.IsResponsibleForNamespace( + entity.Namespace(), + args.CancellationToken)) + { + logger.LogInformation( + "Not responsible for namespace {Namespace}, abandoning message", + entity.Namespace()); + + // Abandon the message so another instance can process it + await args.AbandonMessageAsync(args.Message, cancellationToken: args.CancellationToken); + return; + } + + // Complete message and trigger reconciliation + await args.CompleteMessageAsync(args.Message, args.CancellationToken); + + await reconciler.Reconcile( + ReconciliationContext.CreateFromOperatorEvent( + entity, + entry.RequeueType.ToWatchEventType()), + args.CancellationToken); + } + + private Task ProcessErrorAsync(ProcessErrorEventArgs args) + { + logger.LogError(args.Exception, "Error processing requeue message"); + return Task.CompletedTask; + } + + public override async Task StopAsync(CancellationToken cancellationToken) + { + await _processor.StopProcessingAsync(cancellationToken); + _processor.ProcessMessageAsync -= ProcessMessageAsync; + _processor.ProcessErrorAsync -= ProcessErrorAsync; + await _processor.DisposeAsync(); + await base.StopAsync(cancellationToken); + } +} +``` + +This combination provides: +- **Namespace-based work distribution** across operator instances +- **Durable requeue persistence** for reliability +- **Message abandonment** when an instance is not responsible, allowing proper routing + +## Best Practices + +### Finalizer Management + +1. **Keep defaults enabled** for most use cases +2. **Monitor finalizer attachment** in your logs +3. **Test finalizer behavior** in development before production +4. **Handle finalizer failures** gracefully with proper error messages + +### Leader Election + +1. **Start with `Single`** for simple deployments +2. **Use `Custom`** only when you need advanced coordination +3. **Test failover scenarios** to ensure seamless transitions +4. **Monitor leader election** status in your logs +5. **Set appropriate lease durations** based on your workload + +### Requeue Mechanisms + +1. **Use in-memory queue** for development and simple operators +2. **Implement durable queues** for production workloads +3. **Monitor queue depth** to detect processing issues +4. **Set appropriate requeue delays** to avoid overwhelming your system +5. **Handle message deduplication** to prevent duplicate processing +6. **Test restart scenarios** to ensure requeues survive failures + +## Troubleshooting + +### Finalizers Not Attaching + +- Check `settings.AutoAttachFinalizers` is `true` +- Verify finalizer is registered with `AddFinalizer()` +- Check logs for finalizer attachment errors + +### Leader Election Issues + +- Verify RBAC permissions for lease resources +- Check network connectivity between instances +- Review lease duration settings +- Monitor logs for leader election events + +### Requeue Problems + +- Verify queue connection (for durable queues) +- Check queue permissions and quotas +- Monitor message processing errors +- Review requeue delay settings +- Ensure entities still exist before reprocessing diff --git a/docs/docs/operator/building-blocks/controllers.mdx b/docs/docs/operator/building-blocks/controllers.mdx index 3c060e2c..79493a33 100644 --- a/docs/docs/operator/building-blocks/controllers.mdx +++ b/docs/docs/operator/building-blocks/controllers.mdx @@ -189,6 +189,10 @@ This is useful for: - Periodic status checks - Waiting for external dependencies +:::info Durable Requeue Mechanisms +By default, requeue requests are stored in memory and will be lost on operator restart. For production scenarios requiring persistence, see [Advanced Configuration - Custom Requeue Mechanism](../advanced-configuration#custom-requeue-mechanism) to learn how to implement durable queues using Azure Service Bus, RabbitMQ, or other messaging systems. +::: + ### Error Handling with Results The `ReconciliationResult` provides structured error handling: diff --git a/docs/docs/operator/building-blocks/finalizer.mdx b/docs/docs/operator/building-blocks/finalizer.mdx index 04722990..d59052b9 100644 --- a/docs/docs/operator/building-blocks/finalizer.mdx +++ b/docs/docs/operator/building-blocks/finalizer.mdx @@ -67,7 +67,9 @@ public class DemoEntityFinalizer( ## Using Finalizers -Finalizers are automatically attached to entities using the `EntityFinalizerAttacher` delegate. This delegate is injected into your controller and handles the finalizer attachment: +By default, KubeOps automatically attaches finalizers to entities during reconciliation. This behavior can be configured through `OperatorSettings`. See [Advanced Configuration](../advanced-configuration#finalizer-management) for details on controlling automatic finalizer attachment. + +If you need manual control, you can use the `EntityFinalizerAttacher` delegate, which is injected into your controller: ```csharp [EntityRbac(typeof(V1DemoEntity), Verbs = RbacVerb.All)] diff --git a/docs/docs/operator/deployment.mdx b/docs/docs/operator/deployment.mdx index 8dfd83cb..04df353c 100644 --- a/docs/docs/operator/deployment.mdx +++ b/docs/docs/operator/deployment.mdx @@ -1,7 +1,7 @@ --- title: Deployment description: Deploying your KubeOps Operator -sidebar_position: 8 +sidebar_position: 9 --- # Deployment @@ -177,7 +177,14 @@ kubectl apply -f https://github.com/your-org/your-operator/releases/download/v1. - Configure resource limits - Implement logging and metrics -4. **Updates**: +4. **High Availability**: + + - Run multiple replicas for redundancy + - Configure leader election for coordinated operation (see [Advanced Configuration - Leader Election](./advanced-configuration#custom-leader-election)) + - Consider namespace-based distribution for horizontal scaling + - Implement durable requeue mechanisms for reliability (see [Advanced Configuration - Custom Requeue Mechanism](./advanced-configuration#custom-requeue-mechanism)) + +5. **Updates**: - Document upgrade procedures - Test upgrades in staging - Provide rollback instructions diff --git a/docs/docs/operator/getting-started.mdx b/docs/docs/operator/getting-started.mdx index e9dcab68..35f86599 100644 --- a/docs/docs/operator/getting-started.mdx +++ b/docs/docs/operator/getting-started.mdx @@ -193,6 +193,7 @@ The following sections will dive deeper into: - [Controllers](./building-blocks/controllers) - Implementing reconciliation logic - [Finalizers](./building-blocks/finalizer) - Handling resource cleanup - [Webhooks](./building-blocks/webhooks) - Implementing validation and mutation webhooks +- [Advanced Configuration](./advanced-configuration) - Leader election, durable queues, and finalizer management Make sure to read these sections before deploying your operator to production. ::: diff --git a/docs/docs/operator/testing/_category_.json b/docs/docs/operator/testing/_category_.json index 3ea28e1c..ccb9a78a 100644 --- a/docs/docs/operator/testing/_category_.json +++ b/docs/docs/operator/testing/_category_.json @@ -1,5 +1,5 @@ { - "position": 9, + "position": 10, "label": "Testing", "collapsible": true, "collapsed": true diff --git a/docs/docs/operator/utilities.mdx b/docs/docs/operator/utilities.mdx index 3cdfa14e..3e05b3d0 100644 --- a/docs/docs/operator/utilities.mdx +++ b/docs/docs/operator/utilities.mdx @@ -1,7 +1,7 @@ --- title: Utilities description: Utilities for your Operator and Development -sidebar_position: 10 +sidebar_position: 11 --- # Development and Operator Utilities From 515d4cd1876dddfd1f18d39c821eac901b7e64e9 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Thu, 30 Oct 2025 14:36:50 +0100 Subject: [PATCH 42/58] test: add comprehensive unit tests for reconciliation and queue logic - Added unit tests for `ReconciliationContext`, covering trigger source differentiation, entity metadata, and event type handling. - Introduced tests for `RequeueTypeExtensions` to ensure correct conversions between `WatchEventType` and `RequeueType`. - Implemented extensive tests for `ReconciliationResult` to validate success, failure, error handling, and `RequeueAfter` behavior. - Added `Reconciler` tests focusing on queue interaction, cache updates, requeue logic, and event-driven reconciliation methods. --- .../Properties/AssemblyInfo.cs | 3 + .../Reconciliation/Reconciler.cs | 2 +- .../ReconciliationContext.Test.cs | 190 +++++++++ .../ReconciliationResult.Test.cs | 263 +++++++++++++ .../Builder/OperatorBuilder.Test.cs | 3 - .../Queue/RequeueTypeExtensions.Test.cs | 173 +++++++++ .../Reconciliation/Reconciler.Test.cs | 364 ++++++++++++++++++ 7 files changed, 994 insertions(+), 4 deletions(-) create mode 100644 src/KubeOps.Operator/Properties/AssemblyInfo.cs create mode 100644 test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationContext.Test.cs create mode 100644 test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationResult.Test.cs create mode 100644 test/KubeOps.Operator.Test/Queue/RequeueTypeExtensions.Test.cs create mode 100644 test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs diff --git a/src/KubeOps.Operator/Properties/AssemblyInfo.cs b/src/KubeOps.Operator/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..ba834f24 --- /dev/null +++ b/src/KubeOps.Operator/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("KubeOps.Operator.Tests")] diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index 1679041b..5513dc23 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -75,7 +75,7 @@ await requeue return result; } - public async Task> ReconcileModification(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) + private async Task> ReconcileModification(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { switch (reconciliationContext.Entity) { diff --git a/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationContext.Test.cs b/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationContext.Test.cs new file mode 100644 index 00000000..41ef7d8f --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationContext.Test.cs @@ -0,0 +1,190 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Reconciliation; + +namespace KubeOps.Abstractions.Test.Reconciliation; + +public sealed class ReconciliationContextTest +{ + [Fact] + public void CreateFromApiServerEvent_Should_Create_Context_With_ApiServer_TriggerSource() + { + var entity = CreateTestEntity(); + const WatchEventType eventType = WatchEventType.Added; + + var context = ReconciliationContext.CreateFromApiServerEvent(entity, eventType); + + context.Entity.Should().Be(entity); + context.EventType.Should().Be(eventType); + context.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.ApiServer); + } + + [Fact] + public void CreateFromOperatorEvent_Should_Create_Context_With_Operator_TriggerSource() + { + var entity = CreateTestEntity(); + const WatchEventType eventType = WatchEventType.Modified; + + var context = ReconciliationContext.CreateFromOperatorEvent(entity, eventType); + + context.Entity.Should().Be(entity); + context.EventType.Should().Be(eventType); + context.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.Operator); + } + + [Theory] + [InlineData(WatchEventType.Added)] + [InlineData(WatchEventType.Modified)] + [InlineData(WatchEventType.Deleted)] + public void CreateFromApiServerEvent_Should_Support_All_WatchEventTypes(WatchEventType eventType) + { + var entity = CreateTestEntity(); + + var context = ReconciliationContext.CreateFromApiServerEvent(entity, eventType); + + context.EventType.Should().Be(eventType); + context.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.ApiServer); + } + + [Theory] + [InlineData(WatchEventType.Added)] + [InlineData(WatchEventType.Modified)] + [InlineData(WatchEventType.Deleted)] + public void CreateFromOperatorEvent_Should_Support_All_WatchEventTypes(WatchEventType eventType) + { + var entity = CreateTestEntity(); + + var context = ReconciliationContext.CreateFromOperatorEvent(entity, eventType); + + context.EventType.Should().Be(eventType); + context.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.Operator); + } + + [Fact] + public void IsTriggeredByApiServer_Should_Return_True_For_ApiServer_Context() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + var isTriggeredByApiServer = context.IsTriggeredByApiServer(); + var isTriggeredByOperator = context.IsTriggeredByOperator(); + + isTriggeredByApiServer.Should().BeTrue(); + isTriggeredByOperator.Should().BeFalse(); + } + + [Fact] + public void IsTriggeredByOperator_Should_Return_True_For_Operator_Context() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromOperatorEvent(entity, WatchEventType.Modified); + + var isTriggeredByOperator = context.IsTriggeredByOperator(); + var isTriggeredByApiServer = context.IsTriggeredByApiServer(); + + isTriggeredByOperator.Should().BeTrue(); + isTriggeredByApiServer.Should().BeFalse(); + } + + [Fact] + public void Record_Equality_Should_Work_For_Same_Values() + { + var entity = CreateTestEntity("test-entity"); + + var context1 = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var context2 = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + context1.Should().NotBeSameAs(context2); + context1.Entity.Should().BeSameAs(context2.Entity); + context1.EventType.Should().Be(context2.EventType); + context1.ReconciliationTriggerSource.Should().Be(context2.ReconciliationTriggerSource); + } + + [Fact] + public void Contexts_With_Different_EventTypes_Should_Have_Different_EventTypes() + { + var entity = CreateTestEntity(); + + var contextAdded = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var contextModified = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + var contextDeleted = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Deleted); + + contextAdded.EventType.Should().Be(WatchEventType.Added); + contextModified.EventType.Should().Be(WatchEventType.Modified); + contextDeleted.EventType.Should().Be(WatchEventType.Deleted); + } + + [Fact] + public void Contexts_With_Different_TriggerSources_Should_Have_Different_TriggerSources() + { + var entity = CreateTestEntity(); + + var apiServerContext = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var operatorContext = ReconciliationContext.CreateFromOperatorEvent(entity, WatchEventType.Added); + + apiServerContext.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.ApiServer); + operatorContext.ReconciliationTriggerSource.Should().Be(ReconciliationTriggerSource.Operator); + apiServerContext.ReconciliationTriggerSource.Should().NotBe(operatorContext.ReconciliationTriggerSource); + } + + [Fact] + public void Context_Should_Contain_Entity_Metadata() + { + var entity = CreateTestEntity("test-configmap", "test-namespace"); + + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + context.Entity.Metadata.Name.Should().Be("test-configmap"); + context.Entity.Metadata.NamespaceProperty.Should().Be("test-namespace"); + } + + [Theory] + [InlineData(ReconciliationTriggerSource.ApiServer, WatchEventType.Added)] + [InlineData(ReconciliationTriggerSource.ApiServer, WatchEventType.Modified)] + [InlineData(ReconciliationTriggerSource.ApiServer, WatchEventType.Deleted)] + [InlineData(ReconciliationTriggerSource.Operator, WatchEventType.Added)] + [InlineData(ReconciliationTriggerSource.Operator, WatchEventType.Modified)] + [InlineData(ReconciliationTriggerSource.Operator, WatchEventType.Deleted)] + public void Context_Should_Support_All_Combinations_Of_TriggerSource_And_EventType( + ReconciliationTriggerSource triggerSource, + WatchEventType eventType) + { + var entity = CreateTestEntity(); + + var context = triggerSource == ReconciliationTriggerSource.ApiServer + ? ReconciliationContext.CreateFromApiServerEvent(entity, eventType) + : ReconciliationContext.CreateFromOperatorEvent(entity, eventType); + + context.ReconciliationTriggerSource.Should().Be(triggerSource); + context.EventType.Should().Be(eventType); + } + + [Fact] + public void Multiple_Contexts_With_Same_Entity_Should_Share_Entity_Reference() + { + var entity = CreateTestEntity(); + + var context1 = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var context2 = ReconciliationContext.CreateFromOperatorEvent(entity, WatchEventType.Modified); + + context1.Entity.Should().BeSameAs(context2.Entity); + } + + private static V1ConfigMap CreateTestEntity(string? name = null, string? ns = null) + => new() + { + Metadata = new() + { + Name = name ?? "test-configmap", + NamespaceProperty = ns ?? "default", + Uid = Guid.NewGuid().ToString(), + }, + }; +} diff --git a/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationResult.Test.cs b/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationResult.Test.cs new file mode 100644 index 00000000..34c48538 --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Reconciliation/ReconciliationResult.Test.cs @@ -0,0 +1,263 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s.Models; + +using KubeOps.Abstractions.Reconciliation; + +namespace KubeOps.Abstractions.Test.Reconciliation; + +public sealed class ReconciliationResultTest +{ + [Fact] + public void Success_Should_Create_Successful_Result() + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Success(entity); + + result.IsSuccess.Should().BeTrue(); + result.IsFailure.Should().BeFalse(); + result.Entity.Should().Be(entity); + result.ErrorMessage.Should().BeNull(); + result.Error.Should().BeNull(); + result.RequeueAfter.Should().BeNull(); + } + + [Fact] + public void Success_With_RequeueAfter_Should_Set_RequeueAfter() + { + var entity = CreateTestEntity(); + var requeueAfter = TimeSpan.FromMinutes(5); + + var result = ReconciliationResult.Success(entity, requeueAfter); + + result.IsSuccess.Should().BeTrue(); + result.RequeueAfter.Should().Be(requeueAfter); + result.Entity.Should().Be(entity); + } + + [Fact] + public void Success_With_Null_RequeueAfter_Should_Not_Set_RequeueAfter() + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Success(entity, null); + + result.IsSuccess.Should().BeTrue(); + result.RequeueAfter.Should().BeNull(); + } + + [Fact] + public void Failure_Should_Create_Failed_Result_With_ErrorMessage() + { + var entity = CreateTestEntity(); + var errorMessage = "Reconciliation failed due to timeout"; + + var result = ReconciliationResult.Failure(entity, errorMessage); + + result.IsSuccess.Should().BeFalse(); + result.IsFailure.Should().BeTrue(); + result.Entity.Should().Be(entity); + result.ErrorMessage.Should().Be(errorMessage); + result.Error.Should().BeNull(); + result.RequeueAfter.Should().BeNull(); + } + + [Fact] + public void Failure_With_Exception_Should_Set_Error() + { + var entity = CreateTestEntity(); + const string errorMessage = "Reconciliation failed"; + var exception = new InvalidOperationException("Invalid state detected"); + + var result = ReconciliationResult.Failure(entity, errorMessage, exception); + + result.IsFailure.Should().BeTrue(); + result.ErrorMessage.Should().Be(errorMessage); + result.Error.Should().Be(exception); + result.Error.Message.Should().Be("Invalid state detected"); + } + + [Fact] + public void Failure_With_RequeueAfter_Should_Set_RequeueAfter() + { + var entity = CreateTestEntity(); + const string errorMessage = "Transient failure"; + var requeueAfter = TimeSpan.FromSeconds(30); + + var result = ReconciliationResult.Failure( + entity, + errorMessage, + requeueAfter: requeueAfter); + + result.IsFailure.Should().BeTrue(); + result.ErrorMessage.Should().Be(errorMessage); + result.RequeueAfter.Should().Be(requeueAfter); + } + + [Fact] + public void Failure_With_All_Parameters_Should_Set_All_Properties() + { + var entity = CreateTestEntity(); + const string errorMessage = "Complete failure information"; + var exception = new TimeoutException("Operation timed out"); + var requeueAfter = TimeSpan.FromMinutes(2); + + var result = ReconciliationResult.Failure( + entity, + errorMessage, + exception, + requeueAfter); + + result.IsFailure.Should().BeTrue(); + result.ErrorMessage.Should().Be(errorMessage); + result.Error.Should().Be(exception); + result.RequeueAfter.Should().Be(requeueAfter); + } + + [Fact] + public void RequeueAfter_Should_Be_Mutable() + { + var entity = CreateTestEntity(); + var result = ReconciliationResult.Success(entity); + + result.RequeueAfter = TimeSpan.FromSeconds(45); + + result.RequeueAfter.Should().Be(TimeSpan.FromSeconds(45)); + } + + [Fact] + public void RequeueAfter_Can_Be_Changed_After_Creation() + { + var entity = CreateTestEntity(); + var initialRequeueAfter = TimeSpan.FromMinutes(1); + var result = ReconciliationResult.Success(entity, initialRequeueAfter); + + result.RequeueAfter = TimeSpan.FromMinutes(5); + + result.RequeueAfter.Should().Be(TimeSpan.FromMinutes(5)); + result.RequeueAfter.Should().NotBe(initialRequeueAfter); + } + + [Fact] + public void RequeueAfter_Can_Be_Set_To_Null() + { + var entity = CreateTestEntity(); + var result = ReconciliationResult.Success(entity, TimeSpan.FromMinutes(1)); + + result.RequeueAfter = null; + + result.RequeueAfter.Should().BeNull(); + } + + [Fact] + public void IsSuccess_And_IsFailure_Should_Be_Mutually_Exclusive() + { + var entity = CreateTestEntity(); + + var successResult = ReconciliationResult.Success(entity); + var failureResult = ReconciliationResult.Failure(entity, "Error"); + + successResult.IsSuccess.Should().BeTrue(); + successResult.IsFailure.Should().BeFalse(); + + failureResult.IsSuccess.Should().BeFalse(); + failureResult.IsFailure.Should().BeTrue(); + } + + [Fact] + public void Success_Result_ErrorMessage_Should_Be_Null() + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Success(entity); + + if (result.IsSuccess) + { + result.ErrorMessage.Should().BeNull(); + } + } + + [Fact] + public void Failure_Result_ErrorMessage_Should_Not_Be_Null() + { + var entity = CreateTestEntity(); + var errorMessage = "Something went wrong"; + + var result = ReconciliationResult.Failure(entity, errorMessage); + + if (result.IsFailure) + { + // This should compile without nullable warning due to MemberNotNullWhen attribute + string message = result.ErrorMessage; + message.Should().NotBeNull(); + message.Should().Be(errorMessage); + } + } + + [Fact] + public void Record_Equality_Should_Work_For_Success_Results() + { + var entity1 = CreateTestEntity("test-entity"); + var entity2 = CreateTestEntity("test-entity"); + + var result1 = ReconciliationResult.Success(entity1); + var result2 = ReconciliationResult.Success(entity2); + + // Records with same values should be equal + result1.Should().NotBeSameAs(result2); + } + + [Fact] + public void Entity_Reference_Should_Be_Preserved() + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Success(entity); + + result.Entity.Should().BeSameAs(entity); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(60)] + [InlineData(3600)] + public void Success_Should_Accept_Various_RequeueAfter_Values(int seconds) + { + var entity = CreateTestEntity(); + var requeueAfter = TimeSpan.FromSeconds(seconds); + + var result = ReconciliationResult.Success(entity, requeueAfter); + + result.RequeueAfter.Should().Be(requeueAfter); + } + + [Theory] + [InlineData("Short error")] + [InlineData("A much longer error message that contains detailed information about what went wrong")] + [InlineData("")] + public void Failure_Should_Accept_Various_ErrorMessage_Lengths(string errorMessage) + { + var entity = CreateTestEntity(); + + var result = ReconciliationResult.Failure(entity, errorMessage); + + result.ErrorMessage.Should().Be(errorMessage); + } + + private static V1ConfigMap CreateTestEntity(string? name = null) + => new() + { + Metadata = new() + { + Name = name ?? "test-configmap", + NamespaceProperty = "default", + Uid = Guid.NewGuid().ToString(), + }, + }; +} diff --git a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs index 29bd34c2..105b024c 100644 --- a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs +++ b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs @@ -45,7 +45,6 @@ public void Should_Add_Default_Resources() [Fact] public void Should_Use_Specific_EntityLabelSelector_Implementation() { - // Arrange var services = new ServiceCollection(); // Register the default and specific implementations @@ -54,10 +53,8 @@ public void Should_Use_Specific_EntityLabelSelector_Implementation() var serviceProvider = services.BuildServiceProvider(); - // Act var resolvedService = serviceProvider.GetRequiredService>(); - // Assert Assert.IsType(resolvedService); } diff --git a/test/KubeOps.Operator.Test/Queue/RequeueTypeExtensions.Test.cs b/test/KubeOps.Operator.Test/Queue/RequeueTypeExtensions.Test.cs new file mode 100644 index 00000000..d59a87cc --- /dev/null +++ b/test/KubeOps.Operator.Test/Queue/RequeueTypeExtensions.Test.cs @@ -0,0 +1,173 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s; + +using KubeOps.Abstractions.Reconciliation.Queue; +using KubeOps.Operator.Queue; + +namespace KubeOps.Operator.Test.Queue; + +public sealed class RequeueTypeExtensionsTest +{ + [Theory] + [InlineData(WatchEventType.Added, RequeueType.Added)] + [InlineData(WatchEventType.Modified, RequeueType.Modified)] + [InlineData(WatchEventType.Deleted, RequeueType.Deleted)] + public void ToRequeueType_Should_Convert_WatchEventType_Correctly( + WatchEventType watchEventType, + RequeueType expectedRequeueType) + { + var result = watchEventType.ToRequeueType(); + + result.Should().Be(expectedRequeueType); + } + + [Theory] + [InlineData(RequeueType.Added, WatchEventType.Added)] + [InlineData(RequeueType.Modified, WatchEventType.Modified)] + [InlineData(RequeueType.Deleted, WatchEventType.Deleted)] + public void ToWatchEventType_Should_Convert_RequeueType_Correctly( + RequeueType requeueType, + WatchEventType expectedWatchEventType) + { + var result = requeueType.ToWatchEventType(); + + result.Should().Be(expectedWatchEventType); + } + + [Fact] + public void ToRequeueType_Should_Throw_For_Unsupported_WatchEventType() + { + var unsupportedType = (WatchEventType)999; + + Action act = () => unsupportedType.ToRequeueType(); + + act.Should().Throw() + .WithMessage("*WatchEventType*999*not supported*"); + } + + [Fact] + public void ToWatchEventType_Should_Throw_For_Unsupported_RequeueType() + { + var unsupportedType = (RequeueType)999; + + Action act = () => unsupportedType.ToWatchEventType(); + + act.Should().Throw() + .WithMessage("*RequeueType*999*not supported*"); + } + + [Theory] + [InlineData(WatchEventType.Added)] + [InlineData(WatchEventType.Modified)] + [InlineData(WatchEventType.Deleted)] + public void Bidirectional_Conversion_Should_Be_Reversible_From_WatchEventType(WatchEventType original) + { + var requeueType = original.ToRequeueType(); + var converted = requeueType.ToWatchEventType(); + + converted.Should().Be(original); + } + + [Theory] + [InlineData(RequeueType.Added)] + [InlineData(RequeueType.Modified)] + [InlineData(RequeueType.Deleted)] + public void Bidirectional_Conversion_Should_Be_Reversible_From_RequeueType(RequeueType original) + { + var watchEventType = original.ToWatchEventType(); + var converted = watchEventType.ToRequeueType(); + + converted.Should().Be(original); + } + + [Fact] + public void ToRequeueType_Should_Handle_Added_EventType() + { + var eventType = WatchEventType.Added; + + var result = eventType.ToRequeueType(); + + result.Should().Be(RequeueType.Added); + } + + [Fact] + public void ToRequeueType_Should_Handle_Modified_EventType() + { + var eventType = WatchEventType.Modified; + + var result = eventType.ToRequeueType(); + + result.Should().Be(RequeueType.Modified); + } + + [Fact] + public void ToRequeueType_Should_Handle_Deleted_EventType() + { + var eventType = WatchEventType.Deleted; + + var result = eventType.ToRequeueType(); + + result.Should().Be(RequeueType.Deleted); + } + + [Fact] + public void ToWatchEventType_Should_Handle_Added_RequeueType() + { + var requeueType = RequeueType.Added; + + var result = requeueType.ToWatchEventType(); + + result.Should().Be(WatchEventType.Added); + } + + [Fact] + public void ToWatchEventType_Should_Handle_Modified_RequeueType() + { + var requeueType = RequeueType.Modified; + + var result = requeueType.ToWatchEventType(); + + result.Should().Be(WatchEventType.Modified); + } + + [Fact] + public void ToWatchEventType_Should_Handle_Deleted_RequeueType() + { + var requeueType = RequeueType.Deleted; + + var result = requeueType.ToWatchEventType(); + + result.Should().Be(WatchEventType.Deleted); + } + + [Fact] + public void All_RequeueTypes_Should_Have_Corresponding_WatchEventType() + { + var allRequeueTypes = Enum.GetValues(); + + // Act & Assert + foreach (var requeueType in allRequeueTypes) + { + Action act = () => requeueType.ToWatchEventType(); + act.Should().NotThrow($"RequeueType.{requeueType} should have a corresponding WatchEventType"); + } + } + + [Fact] + public void Supported_WatchEventTypes_Should_Have_Corresponding_RequeueType() + { + var supportedWatchEventTypes = new[] { WatchEventType.Added, WatchEventType.Modified, WatchEventType.Deleted }; + + // Act & Assert + foreach (var watchEventType in supportedWatchEventTypes) + { + Action act = () => watchEventType.ToRequeueType(); + act.Should().NotThrow($"WatchEventType.{watchEventType} should have a corresponding RequeueType"); + } + } +} diff --git a/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs b/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs new file mode 100644 index 00000000..e6208160 --- /dev/null +++ b/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs @@ -0,0 +1,364 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Builder; +using KubeOps.Abstractions.Reconciliation; +using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Queue; +using KubeOps.KubernetesClient; +using KubeOps.Operator.Queue; +using KubeOps.Operator.Reconciliation; + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +using Moq; + +using ZiggyCreatures.Caching.Fusion; + +namespace KubeOps.Operator.Test.Reconciliation; + +public sealed class ReconcilerTest +{ + private readonly Mock>> _mockLogger; + private readonly Mock _mockCacheProvider; + private readonly Mock _mockCache; + private readonly Mock _mockServiceProvider; + private readonly Mock _mockClient; + private readonly Mock> _mockQueue; + private readonly OperatorSettings _settings; + + public ReconcilerTest() + { + _mockLogger = new(); + _mockCacheProvider = new(); + _mockCache = new(); + _mockServiceProvider = new(); + _mockClient = new(); + _mockQueue = new(); + _settings = new() { AutoAttachFinalizers = false, AutoDetachFinalizers = false }; + + _mockCacheProvider + .Setup(p => p.GetCache(It.IsAny())) + .Returns(_mockCache.Object); + } + + [Fact] + public async Task Reconcile_Should_Remove_Entity_From_Queue_Before_Processing() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var controller = CreateMockController(); + + var reconciler = CreateReconciler(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Remove(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Enqueue_Entity_When_Result_Has_RequeueAfter() + { + var entity = CreateTestEntity(); + var requeueAfter = TimeSpan.FromMinutes(5); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + var controller = CreateMockController( + reconcileResult: ReconciliationResult.Success(entity, requeueAfter)); + + var reconciler = CreateReconciler(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Enqueue( + entity, + RequeueType.Added, + requeueAfter, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Not_Enqueue_Entity_When_Result_Has_No_RequeueAfter() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + + var controller = CreateMockController( + reconcileResult: ReconciliationResult.Success(entity)); + + var reconciler = CreateReconciler(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Enqueue( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Reconcile_Should_Call_ReconcileAsync_For_Added_Event() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var mockController = new Mock>(); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconciler(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + mockController.Verify( + c => c.ReconcileAsync(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Call_ReconcileAsync_For_Modified_Event() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + var mockController = new Mock>(); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconciler(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + mockController.Verify( + c => c.ReconcileAsync(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Call_DeletedAsync_For_Deleted_Event() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Deleted); + var mockController = new Mock>(); + + mockController + .Setup(c => c.DeletedAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconciler(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + mockController.Verify( + c => c.DeletedAsync(entity, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Remove_From_Cache_After_Successful_Deletion() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Deleted); + + var controller = CreateMockController( + deletedResult: ReconciliationResult.Success(entity)); + + var reconciler = CreateReconciler(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockCache.Verify( + c => c.RemoveAsync(entity.Uid(), It.IsAny(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Not_Remove_From_Cache_After_Failed_Deletion() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Deleted); + + var controller = CreateMockController( + deletedResult: ReconciliationResult.Failure(entity, "Deletion failed")); + + var reconciler = CreateReconciler(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockCache.Verify( + c => c.RemoveAsync(It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Theory] + [InlineData(RequeueType.Added)] + [InlineData(RequeueType.Modified)] + [InlineData(RequeueType.Deleted)] + public async Task Reconcile_Should_Use_Correct_RequeueType_For_EventType(RequeueType expectedRequeueType) + { + var entity = CreateTestEntity(); + var requeueAfter = TimeSpan.FromSeconds(30); + var watchEventType = expectedRequeueType switch + { + RequeueType.Added => WatchEventType.Added, + RequeueType.Modified => WatchEventType.Modified, + RequeueType.Deleted => WatchEventType.Deleted, + _ => throw new ArgumentException("Invalid RequeueType"), + }; + + var context = ReconciliationContext.CreateFromApiServerEvent(entity, watchEventType); + var result = ReconciliationResult.Success(entity, requeueAfter); + + var controller = watchEventType == WatchEventType.Deleted + ? CreateMockController(deletedResult: result) + : CreateMockController(reconcileResult: result); + + var reconciler = CreateReconciler(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Enqueue( + entity, + expectedRequeueType, + requeueAfter, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_Should_Return_Result_From_Controller() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var expectedResult = ReconciliationResult.Success(entity, TimeSpan.FromMinutes(1)); + + var controller = CreateMockController(reconcileResult: expectedResult); + var reconciler = CreateReconciler(controller); + + var result = await reconciler.Reconcile(context, CancellationToken.None); + + result.Should().Be(expectedResult); + result.Entity.Should().Be(entity); + result.RequeueAfter.Should().Be(TimeSpan.FromMinutes(1)); + } + + [Fact] + public async Task Reconcile_Should_Handle_Failure_Result() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + const string errorMessage = "Test error"; + var failureResult = ReconciliationResult.Failure(entity, errorMessage); + + var controller = CreateMockController(reconcileResult: failureResult); + var reconciler = CreateReconciler(controller); + + var result = await reconciler.Reconcile(context, CancellationToken.None); + + result.IsFailure.Should().BeTrue(); + result.ErrorMessage.Should().Be(errorMessage); + } + + [Fact] + public async Task Reconcile_Should_Enqueue_Failed_Result_If_RequeueAfter_Set() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var requeueAfter = TimeSpan.FromSeconds(30); + var failureResult = ReconciliationResult.Failure( + entity, + "Temporary failure", + requeueAfter: requeueAfter); + + var controller = CreateMockController(reconcileResult: failureResult); + var reconciler = CreateReconciler(controller); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockQueue.Verify( + q => q.Enqueue( + entity, + RequeueType.Added, + requeueAfter, + It.IsAny()), + Times.Once); + } + + private Reconciler CreateReconciler(IEntityController controller) + { + var mockScope = new Mock(); + var mockScopeFactory = new Mock(); + + mockScope + .Setup(s => s.ServiceProvider) + .Returns(_mockServiceProvider.Object); + + mockScopeFactory + .Setup(s => s.CreateScope()) + .Returns(mockScope.Object); + + _mockServiceProvider + .Setup(p => p.GetService(typeof(IServiceScopeFactory))) + .Returns(mockScopeFactory.Object); + + _mockServiceProvider + .Setup(p => p.GetService(typeof(IEntityController))) + .Returns(controller); + + return new( + _mockLogger.Object, + _mockCacheProvider.Object, + _mockServiceProvider.Object, + _settings, + _mockQueue.Object, + _mockClient.Object); + } + + private static IEntityController CreateMockController( + ReconciliationResult? reconcileResult = null, + ReconciliationResult? deletedResult = null) + { + var mockController = new Mock>(); + var entity = CreateTestEntity(); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(reconcileResult ?? ReconciliationResult.Success(entity)); + + mockController + .Setup(c => c.DeletedAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(deletedResult ?? ReconciliationResult.Success(entity)); + + return mockController.Object; + } + + private static V1ConfigMap CreateTestEntity(string? name = null) + { + return new() + { + Metadata = new() + { + Name = name ?? "test-configmap", + NamespaceProperty = "default", + Uid = Guid.NewGuid().ToString(), + }, + }; + } +} From 8bd859b6005db5a2a4e403a338b0e61f773e53d8 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Thu, 30 Oct 2025 16:13:59 +0100 Subject: [PATCH 43/58] test: extend reconciliation tests with finalizer handling and caching logic - Added unit tests for finalizer attachment/detachment with auto-attach/detach settings. - Introduced tests to skip reconciliation for unchanged entity generations using caching. - Updated existing tests to leverage `CreateReconcilerForController` and `CreateReconcilerForFinalizer` methods. --- .../Reconciliation/Reconciler.Test.cs | 202 ++++++++++++++++-- 1 file changed, 187 insertions(+), 15 deletions(-) diff --git a/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs b/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs index e6208160..ac4d88c6 100644 --- a/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs +++ b/test/KubeOps.Operator.Test/Reconciliation/Reconciler.Test.cs @@ -10,6 +10,7 @@ using KubeOps.Abstractions.Builder; using KubeOps.Abstractions.Reconciliation; using KubeOps.Abstractions.Reconciliation.Controller; +using KubeOps.Abstractions.Reconciliation.Finalizer; using KubeOps.Abstractions.Reconciliation.Queue; using KubeOps.KubernetesClient; using KubeOps.Operator.Queue; @@ -29,7 +30,7 @@ public sealed class ReconcilerTest private readonly Mock>> _mockLogger; private readonly Mock _mockCacheProvider; private readonly Mock _mockCache; - private readonly Mock _mockServiceProvider; + private readonly Mock _mockServiceProvider; private readonly Mock _mockClient; private readonly Mock> _mockQueue; private readonly OperatorSettings _settings; @@ -56,7 +57,7 @@ public async Task Reconcile_Should_Remove_Entity_From_Queue_Before_Processing() var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); var controller = CreateMockController(); - var reconciler = CreateReconciler(controller); + var reconciler = CreateReconcilerForController(controller); await reconciler.Reconcile(context, CancellationToken.None); @@ -75,7 +76,7 @@ public async Task Reconcile_Should_Enqueue_Entity_When_Result_Has_RequeueAfter() var controller = CreateMockController( reconcileResult: ReconciliationResult.Success(entity, requeueAfter)); - var reconciler = CreateReconciler(controller); + var reconciler = CreateReconcilerForController(controller); await reconciler.Reconcile(context, CancellationToken.None); @@ -97,7 +98,7 @@ public async Task Reconcile_Should_Not_Enqueue_Entity_When_Result_Has_No_Requeue var controller = CreateMockController( reconcileResult: ReconciliationResult.Success(entity)); - var reconciler = CreateReconciler(controller); + var reconciler = CreateReconcilerForController(controller); await reconciler.Reconcile(context, CancellationToken.None); @@ -110,6 +111,37 @@ public async Task Reconcile_Should_Not_Enqueue_Entity_When_Result_Has_No_Requeue Times.Never); } + [Fact] + public async Task Reconcile_Should_Skip_On_Cached_Generation() + { + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Added); + var mockController = new Mock>(); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + _mockCache + .Setup(c => c.TryGetAsync( + It.Is(s => s == entity.Uid()), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(MaybeValue.FromValue(entity.Generation())); + + var reconciler = CreateReconcilerForController(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockLogger.Verify(logger => logger.Log( + It.Is(logLevel => logLevel == LogLevel.Debug), + It.Is(eventId => eventId.Id == 0), + It.Is((@object, type) => @object.ToString() == $"""Entity "{entity.Kind}/{entity.Name()}" modification did not modify generation. Skip event.""" && type.Name == "FormattedLogValues"), + It.IsAny(), + It.IsAny>()!), + Times.Once); + } + [Fact] public async Task Reconcile_Should_Call_ReconcileAsync_For_Added_Event() { @@ -121,7 +153,7 @@ public async Task Reconcile_Should_Call_ReconcileAsync_For_Added_Event() .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(ReconciliationResult.Success(entity)); - var reconciler = CreateReconciler(mockController.Object); + var reconciler = CreateReconcilerForController(mockController.Object); await reconciler.Reconcile(context, CancellationToken.None); @@ -131,7 +163,7 @@ public async Task Reconcile_Should_Call_ReconcileAsync_For_Added_Event() } [Fact] - public async Task Reconcile_Should_Call_ReconcileAsync_For_Modified_Event() + public async Task Reconcile_Should_Call_ReconcileAsync_For_Modified_Event_With_No_Deletion_Timestamp() { var entity = CreateTestEntity(); var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); @@ -141,7 +173,7 @@ public async Task Reconcile_Should_Call_ReconcileAsync_For_Modified_Event() .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(ReconciliationResult.Success(entity)); - var reconciler = CreateReconciler(mockController.Object); + var reconciler = CreateReconcilerForController(mockController.Object); await reconciler.Reconcile(context, CancellationToken.None); @@ -150,6 +182,28 @@ public async Task Reconcile_Should_Call_ReconcileAsync_For_Modified_Event() Times.Once); } + [Fact] + public async Task Reconcile_Should_Call_FinalizeAsync_For_Modified_Event_With_Deletion_Timestamp() + { + const string finalizerName = "test-finalizer"; + var entity = CreateTestEntityForFinalization(deletionTimestamp: DateTime.UtcNow, finalizer: finalizerName); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + + var mockFinalizer = new Mock>(); + + mockFinalizer + .Setup(c => c.FinalizeAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForFinalizer(mockFinalizer.Object, finalizerName); + + await reconciler.Reconcile(context, CancellationToken.None); + + mockFinalizer.Verify( + c => c.FinalizeAsync(entity, It.IsAny()), + Times.Once); + } + [Fact] public async Task Reconcile_Should_Call_DeletedAsync_For_Deleted_Event() { @@ -161,7 +215,7 @@ public async Task Reconcile_Should_Call_DeletedAsync_For_Deleted_Event() .Setup(c => c.DeletedAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(ReconciliationResult.Success(entity)); - var reconciler = CreateReconciler(mockController.Object); + var reconciler = CreateReconcilerForController(mockController.Object); await reconciler.Reconcile(context, CancellationToken.None); @@ -179,7 +233,7 @@ public async Task Reconcile_Should_Remove_From_Cache_After_Successful_Deletion() var controller = CreateMockController( deletedResult: ReconciliationResult.Success(entity)); - var reconciler = CreateReconciler(controller); + var reconciler = CreateReconcilerForController(controller); await reconciler.Reconcile(context, CancellationToken.None); @@ -197,7 +251,7 @@ public async Task Reconcile_Should_Not_Remove_From_Cache_After_Failed_Deletion() var controller = CreateMockController( deletedResult: ReconciliationResult.Failure(entity, "Deletion failed")); - var reconciler = CreateReconciler(controller); + var reconciler = CreateReconcilerForController(controller); await reconciler.Reconcile(context, CancellationToken.None); @@ -229,7 +283,7 @@ public async Task Reconcile_Should_Use_Correct_RequeueType_For_EventType(Requeue ? CreateMockController(deletedResult: result) : CreateMockController(reconcileResult: result); - var reconciler = CreateReconciler(controller); + var reconciler = CreateReconcilerForController(controller); await reconciler.Reconcile(context, CancellationToken.None); @@ -250,7 +304,7 @@ public async Task Reconcile_Should_Return_Result_From_Controller() var expectedResult = ReconciliationResult.Success(entity, TimeSpan.FromMinutes(1)); var controller = CreateMockController(reconcileResult: expectedResult); - var reconciler = CreateReconciler(controller); + var reconciler = CreateReconcilerForController(controller); var result = await reconciler.Reconcile(context, CancellationToken.None); @@ -268,7 +322,7 @@ public async Task Reconcile_Should_Handle_Failure_Result() var failureResult = ReconciliationResult.Failure(entity, errorMessage); var controller = CreateMockController(reconcileResult: failureResult); - var reconciler = CreateReconciler(controller); + var reconciler = CreateReconcilerForController(controller); var result = await reconciler.Reconcile(context, CancellationToken.None); @@ -288,7 +342,7 @@ public async Task Reconcile_Should_Enqueue_Failed_Result_If_RequeueAfter_Set() requeueAfter: requeueAfter); var controller = CreateMockController(reconcileResult: failureResult); - var reconciler = CreateReconciler(controller); + var reconciler = CreateReconcilerForController(controller); await reconciler.Reconcile(context, CancellationToken.None); @@ -301,7 +355,74 @@ public async Task Reconcile_Should_Enqueue_Failed_Result_If_RequeueAfter_Set() Times.Once); } - private Reconciler CreateReconciler(IEntityController controller) + [Fact] + public async Task Reconcile_When_Auto_Attach_Finalizers_Is_Enabled_Should_Attach_Finalizer() + { + _settings.AutoAttachFinalizers = true; + + var entity = CreateTestEntity(); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + var mockController = new Mock>(); + var mockFinalizer = new Mock>(); + + _mockClient + .Setup(c => c.UpdateAsync(It.Is( + e => e == entity), + It.IsAny())) + .ReturnsAsync(entity); + + _mockServiceProvider + .Setup(p => p.GetRequiredKeyedService( + It.Is(t => t == typeof(IEnumerable>)), + It.Is(o => ReferenceEquals(o, KeyedService.AnyKey)))) + .Returns(new List> { mockFinalizer.Object }); + + mockController + .Setup(c => c.ReconcileAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForController(mockController.Object); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockClient.Verify( + c => c.UpdateAsync(It.Is(cm => cm.HasFinalizer("ientityfinalizer`1proxyfinalizer")), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Reconcile_When_Auto_Detach_Finalizers_Is_Enabled_Should_Detach_Finalizer() + { + _settings.AutoDetachFinalizers = true; + + const string finalizerName = "test-finalizer"; + var entity = CreateTestEntityForFinalization(deletionTimestamp: DateTime.UtcNow, finalizer: finalizerName); + var context = ReconciliationContext.CreateFromApiServerEvent(entity, WatchEventType.Modified); + + var mockFinalizer = new Mock>(); + + _mockClient + .Setup(c => c.UpdateAsync(It.Is( + e => e == entity), + It.IsAny())) + .ReturnsAsync(entity); + + mockFinalizer + .Setup(c => c.FinalizeAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(ReconciliationResult.Success(entity)); + + var reconciler = CreateReconcilerForFinalizer(mockFinalizer.Object, finalizerName); + + await reconciler.Reconcile(context, CancellationToken.None); + + _mockClient.Verify( + c => c.UpdateAsync(It.Is(cm => !cm.HasFinalizer(finalizerName)), + It.IsAny()), + Times.Once); + } + + private Reconciler CreateReconcilerForController(IEntityController controller) { var mockScope = new Mock(); var mockScopeFactory = new Mock(); @@ -331,6 +452,38 @@ private Reconciler CreateReconciler(IEntityController _mockClient.Object); } + private Reconciler CreateReconcilerForFinalizer(IEntityFinalizer? finalizer, string finalizerName) + { + var mockScope = new Mock(); + var mockScopeFactory = new Mock(); + + mockScope + .Setup(s => s.ServiceProvider) + .Returns(_mockServiceProvider.Object); + + mockScopeFactory + .Setup(s => s.CreateScope()) + .Returns(mockScope.Object); + + _mockServiceProvider + .Setup(p => p.GetService(typeof(IServiceScopeFactory))) + .Returns(mockScopeFactory.Object); + + _mockServiceProvider + .Setup(p => p.GetKeyedService( + It.Is(t => t == typeof(IEntityFinalizer)), + It.Is(s => s == finalizerName))) + .Returns(finalizer); + + return new( + _mockLogger.Object, + _mockCacheProvider.Object, + _mockServiceProvider.Object, + _settings, + _mockQueue.Object, + _mockClient.Object); + } + private static IEntityController CreateMockController( ReconciliationResult? reconcileResult = null, ReconciliationResult? deletedResult = null) @@ -358,7 +511,26 @@ private static V1ConfigMap CreateTestEntity(string? name = null) Name = name ?? "test-configmap", NamespaceProperty = "default", Uid = Guid.NewGuid().ToString(), + Generation = 1, + }, + Kind = V1ConfigMap.KubeKind, + }; + } + + private static V1ConfigMap CreateTestEntityForFinalization(string? name = null, DateTime? deletionTimestamp = null, string? finalizer = null) + { + return new() + { + Metadata = new() + { + Name = name ?? "test-configmap", + NamespaceProperty = "default", + Uid = Guid.NewGuid().ToString(), + Generation = 1, + DeletionTimestamp = deletionTimestamp, + Finalizers = !string.IsNullOrEmpty(finalizer) ? new List { finalizer } : new(), }, + Kind = V1ConfigMap.KubeKind, }; } } From 1ceb11668a530676f7c637f42ef85196210dab7a Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Thu, 30 Oct 2025 16:27:37 +0100 Subject: [PATCH 44/58] test: exclude test projects from code coverage analysis - Added `[ExcludeFromCodeCoverage]` to `GlobalAssemblyInfo.cs` in `KubeOps.Abstractions.Test` and `KubeOps.Operator.Test` projects. --- .../KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs | 3 +++ test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs | 3 +++ test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs | 3 +++ .../Properties/GlobalAssemblyInfo.cs | 3 +++ test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs | 3 +++ .../KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs | 3 +++ test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs | 3 +++ 7 files changed, 21 insertions(+) create mode 100644 test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs create mode 100644 test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs create mode 100644 test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs create mode 100644 test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs create mode 100644 test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs create mode 100644 test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs create mode 100644 test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs diff --git a/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..5e80f0d0 --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..5e80f0d0 --- /dev/null +++ b/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..5e80f0d0 --- /dev/null +++ b/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..5e80f0d0 --- /dev/null +++ b/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..5e80f0d0 --- /dev/null +++ b/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..5e80f0d0 --- /dev/null +++ b/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs new file mode 100644 index 00000000..5e80f0d0 --- /dev/null +++ b/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Diagnostics.CodeAnalysis; + +[assembly: ExcludeFromCodeCoverage] From 7a65fed003b63919b3d342b6a6d6f4980bb1a198 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Wed, 29 Oct 2025 16:52:32 +0100 Subject: [PATCH 45/58] chore(deps): upgrade `KubernetesClient` to version `18.0.5` - Updated project files to use the latest version of `KubernetesClient` across the solution. - Adjusted `Crds` logic to align with the updated library API. --- .../KubeOps.Abstractions.csproj | 2 +- src/KubeOps.Transpiler/Crds.cs | 37 +++++++++++-------- test/KubeOps.Cli.Test/KubeOps.Cli.Test.csproj | 2 +- .../KubeOps.Operator.Test.csproj | 2 +- .../KubeOps.Operator.Web.Test.csproj | 2 +- .../KubeOps.Transpiler.Test.csproj | 2 +- 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj b/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj index 996b4f2a..84762526 100644 --- a/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj +++ b/src/KubeOps.Abstractions/KubeOps.Abstractions.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/KubeOps.Transpiler/Crds.cs b/src/KubeOps.Transpiler/Crds.cs index 104f67b2..dff72d60 100644 --- a/src/KubeOps.Transpiler/Crds.cs +++ b/src/KubeOps.Transpiler/Crds.cs @@ -48,13 +48,13 @@ public static V1CustomResourceDefinition Transpile(this MetadataLoadContext cont { type = context.GetContextType(type); var (meta, scope) = context.ToEntityMetadata(type); - var crd = new V1CustomResourceDefinition(new()).Initialize(); + var crd = new V1CustomResourceDefinition { Spec = new() }.Initialize(); crd.Metadata.Name = $"{meta.PluralName}.{meta.Group}"; crd.Spec.Group = meta.Group; crd.Spec.Names = - new V1CustomResourceDefinitionNames + new() { Kind = meta.Kind, ListKind = meta.ListKind, @@ -68,25 +68,32 @@ public static V1CustomResourceDefinition Transpile(this MetadataLoadContext cont crd.Spec.Names.ShortNames = shortNames.Select(a => a.Value?.ToString()).ToList(); } - var version = new V1CustomResourceDefinitionVersion(meta.Version, true, true); + var version = new V1CustomResourceDefinitionVersion { Name = meta.Version, Served = true, Storage = true }; if (type.GetProperty("Status") != null || type.GetProperty("status") != null) { - version.Subresources = new V1CustomResourceSubresources(null, new object()); + version.Subresources = new() + { + Scale = null, + Status = new(), + }; } - version.Schema = new V1CustomResourceValidation(new V1JSONSchemaProps - { - Type = Object, - Description = - type.GetCustomAttributeData()?.GetCustomAttributeCtorArg(context, 0), - Properties = type.GetProperties() - .Where(p => !IgnoredToplevelProperties.Contains(p.Name.ToLowerInvariant()) - && p.GetCustomAttributeData() == null) - .Select(p => (Name: p.GetPropertyName(context), Schema: context.Map(p))) - .ToDictionary(t => t.Name, t => t.Schema), - }); + version.Schema = new() + { + OpenAPIV3Schema = new() + { + Type = Object, + Description = + type.GetCustomAttributeData()?.GetCustomAttributeCtorArg(context, 0), + Properties = type.GetProperties() + .Where(p => !IgnoredToplevelProperties.Contains(p.Name.ToLowerInvariant()) + && p.GetCustomAttributeData() == null) + .Select(p => (Name: p.GetPropertyName(context), Schema: context.Map(p))) + .ToDictionary(t => t.Name, t => t.Schema), + }, + }; version.AdditionalPrinterColumns = context.MapPrinterColumns(type).ToList() switch { diff --git a/test/KubeOps.Cli.Test/KubeOps.Cli.Test.csproj b/test/KubeOps.Cli.Test/KubeOps.Cli.Test.csproj index 8357c7f7..e14f730b 100644 --- a/test/KubeOps.Cli.Test/KubeOps.Cli.Test.csproj +++ b/test/KubeOps.Cli.Test/KubeOps.Cli.Test.csproj @@ -6,7 +6,7 @@ - + diff --git a/test/KubeOps.Operator.Test/KubeOps.Operator.Test.csproj b/test/KubeOps.Operator.Test/KubeOps.Operator.Test.csproj index 1b545ac7..8f9449e9 100644 --- a/test/KubeOps.Operator.Test/KubeOps.Operator.Test.csproj +++ b/test/KubeOps.Operator.Test/KubeOps.Operator.Test.csproj @@ -1,6 +1,6 @@ - + diff --git a/test/KubeOps.Operator.Web.Test/KubeOps.Operator.Web.Test.csproj b/test/KubeOps.Operator.Web.Test/KubeOps.Operator.Web.Test.csproj index 5ac72a08..c9b338e3 100644 --- a/test/KubeOps.Operator.Web.Test/KubeOps.Operator.Web.Test.csproj +++ b/test/KubeOps.Operator.Web.Test/KubeOps.Operator.Web.Test.csproj @@ -1,6 +1,6 @@ - + diff --git a/test/KubeOps.Transpiler.Test/KubeOps.Transpiler.Test.csproj b/test/KubeOps.Transpiler.Test/KubeOps.Transpiler.Test.csproj index 6a1b39c8..fdafcc13 100644 --- a/test/KubeOps.Transpiler.Test/KubeOps.Transpiler.Test.csproj +++ b/test/KubeOps.Transpiler.Test/KubeOps.Transpiler.Test.csproj @@ -1,7 +1,7 @@ - + From 014a335ce9bd292cb47015b83c02c74f094c8609 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 31 Oct 2025 07:46:53 +0100 Subject: [PATCH 46/58] refactor: update object initialization to use object initializer shorthand across tests and core classes - Refactored entity `Metadata` and other class initializations to use object initializer shorthand consistently. - Marked test classes as `sealed` for improved optimization and design clarity. - Adjusted tests and core logic to align with the updated initialization style and coding standards. --- .../CustomKubernetesEntity{TSpec,TStatus}.cs | 2 +- .../Entities/CustomKubernetesEntity{TSpec}.cs | 2 +- .../Entities/KubernetesExtensions.cs | 12 +-- .../Commands/Generator/OperatorGenerator.cs | 5 +- .../Webhooks/WebhookServiceBase.cs | 35 +++++---- src/KubeOps.Transpiler/Crds.cs | 70 +++++++++--------- .../KubernetesClient.Test.cs | 68 ++++++++++++++--- .../KubernetesClientAsync.Test.cs | 74 +++++++++++++++---- .../IntegrationTestCollection.cs | 5 +- .../NamespacedOperator.Integration.Test.cs | 7 +- test/KubeOps.Transpiler.Test/Crds.Mlc.Test.cs | 2 +- 11 files changed, 194 insertions(+), 88 deletions(-) diff --git a/src/KubeOps.Abstractions/Entities/CustomKubernetesEntity{TSpec,TStatus}.cs b/src/KubeOps.Abstractions/Entities/CustomKubernetesEntity{TSpec,TStatus}.cs index c8cfa990..3b3eb3ad 100644 --- a/src/KubeOps.Abstractions/Entities/CustomKubernetesEntity{TSpec,TStatus}.cs +++ b/src/KubeOps.Abstractions/Entities/CustomKubernetesEntity{TSpec,TStatus}.cs @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using k8s; +using k8s.Models; namespace KubeOps.Abstractions.Entities; diff --git a/src/KubeOps.Abstractions/Entities/CustomKubernetesEntity{TSpec}.cs b/src/KubeOps.Abstractions/Entities/CustomKubernetesEntity{TSpec}.cs index fe2f6369..7ffac91d 100644 --- a/src/KubeOps.Abstractions/Entities/CustomKubernetesEntity{TSpec}.cs +++ b/src/KubeOps.Abstractions/Entities/CustomKubernetesEntity{TSpec}.cs @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using k8s; +using k8s.Models; namespace KubeOps.Abstractions.Entities; diff --git a/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs b/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs index f9304aab..408db1a5 100644 --- a/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs +++ b/src/KubeOps.Abstractions/Entities/KubernetesExtensions.cs @@ -83,11 +83,13 @@ public static TEntity WithOwnerReference( /// The object that should be translated. /// The created . public static V1OwnerReference MakeOwnerReference(this IKubernetesObject kubernetesObject) - => new( - kubernetesObject.ApiVersion, - kubernetesObject.Kind, - kubernetesObject.Metadata.Name, - kubernetesObject.Metadata.Uid); + => new() + { + ApiVersion = kubernetesObject.ApiVersion, + Kind = kubernetesObject.Kind, + Name = kubernetesObject.Metadata.Name, + Uid = kubernetesObject.Metadata.Uid, + }; private static IList EnsureOwnerReferences(this V1ObjectMeta meta) => meta.OwnerReferences ??= new List(); diff --git a/src/KubeOps.Cli/Commands/Generator/OperatorGenerator.cs b/src/KubeOps.Cli/Commands/Generator/OperatorGenerator.cs index 227eb389..5c94fb81 100644 --- a/src/KubeOps.Cli/Commands/Generator/OperatorGenerator.cs +++ b/src/KubeOps.Cli/Commands/Generator/OperatorGenerator.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information. using System.CommandLine; -using System.CommandLine.Invocation; using System.Text; using k8s; @@ -116,7 +115,7 @@ internal static async Task Handler(IAnsiConsole console, ParseResult parseR result.Add( $"namespace.{format.GetFileExtension()}", - new V1Namespace(metadata: new(name: "system")).Initialize()); + new V1Namespace { Metadata = new() { Name = "system" } }.Initialize()); result.Add( $"kustomization.{format.GetFileExtension()}", @@ -124,7 +123,7 @@ internal static async Task Handler(IAnsiConsole console, ParseResult parseR { NamePrefix = $"{name}-", Namespace = $"{name}-system", - Labels = [new KustomizationCommonLabels(new Dictionary { { "operator", name }, })], + Labels = [new(new Dictionary { { "operator", name }, })], Resources = result.DefaultFormatFiles.ToList(), Images = new List diff --git a/src/KubeOps.Operator.Web/Webhooks/WebhookServiceBase.cs b/src/KubeOps.Operator.Web/Webhooks/WebhookServiceBase.cs index 4948cb19..5d8911c3 100644 --- a/src/KubeOps.Operator.Web/Webhooks/WebhookServiceBase.cs +++ b/src/KubeOps.Operator.Web/Webhooks/WebhookServiceBase.cs @@ -50,12 +50,13 @@ internal async Task RegisterConverters() } var whUrl = $"{Uri}convert/{metadata.Group}/{metadata.PluralName}"; - crd.Spec.Conversion = new V1CustomResourceConversion("Webhook") + crd.Spec.Conversion = new() { - Webhook = new V1WebhookConversion + Strategy = "Webhook", + Webhook = new() { ConversionReviewVersions = new[] { "v1" }, - ClientConfig = new Apiextensionsv1WebhookClientConfig + ClientConfig = new() { Url = whUrl, CaBundle = CaBundle, @@ -89,16 +90,18 @@ internal async Task RegisterMutators() ApiVersions = new[] { hook.Metadata.Version }, }, }, - ClientConfig = new Admissionregistrationv1WebhookClientConfig + ClientConfig = new() { Url = $"{Uri}mutate/{hook.HookTypeName}", CaBundle = CaBundle, }, }); - var mutatorConfig = new V1MutatingWebhookConfiguration( - metadata: new V1ObjectMeta(name: "dev-mutators"), - webhooks: mutationWebhooks.ToList()).Initialize(); + var mutatorConfig = new V1MutatingWebhookConfiguration() + { + Metadata = new() { Name = "dev-mutators" }, + Webhooks = mutationWebhooks.ToList(), + }.Initialize(); if (mutatorConfig.Webhooks.Any()) { @@ -106,7 +109,10 @@ internal async Task RegisterMutators() } } - internal async Task RegisterValidators() + private static string Defaulted(string? value, string defaultValue) => + string.IsNullOrWhiteSpace(value) ? defaultValue : value; + + private async Task RegisterValidators() { var validationWebhooks = loader .ValidationWebhooks @@ -128,23 +134,22 @@ internal async Task RegisterValidators() ApiVersions = new[] { hook.Metadata.Version }, }, }, - ClientConfig = new Admissionregistrationv1WebhookClientConfig + ClientConfig = new() { Url = $"{Uri}validate/{hook.HookTypeName}", CaBundle = CaBundle, }, }); - var validatorConfig = new V1ValidatingWebhookConfiguration( - metadata: new V1ObjectMeta(name: "dev-validators"), - webhooks: validationWebhooks.ToList()).Initialize(); + var validatorConfig = new V1ValidatingWebhookConfiguration() + { + Metadata = new() { Name = "dev-validators" }, + Webhooks = validationWebhooks.ToList(), + }.Initialize(); if (validatorConfig.Webhooks.Any()) { await Client.SaveAsync(validatorConfig); } } - - private static string Defaulted(string? value, string defaultValue) => - string.IsNullOrWhiteSpace(value) ? defaultValue : value; } diff --git a/src/KubeOps.Transpiler/Crds.cs b/src/KubeOps.Transpiler/Crds.cs index dff72d60..9329a920 100644 --- a/src/KubeOps.Transpiler/Crds.cs +++ b/src/KubeOps.Transpiler/Crds.cs @@ -5,7 +5,6 @@ using System.Collections; using System.Collections.ObjectModel; using System.Reflection; -using System.Runtime.Serialization; using System.Text.Json.Serialization; using k8s; @@ -101,7 +100,6 @@ public static V1CustomResourceDefinition Transpile(this MetadataLoadContext cont _ => null, }; crd.Spec.Versions = new List { version }; - crd.Validate(); return crd; } @@ -184,7 +182,7 @@ private static IEnumerable MapPrinterColumns( } var mapped = context.Map(prop); - yield return new V1CustomResourceColumnDefinition + yield return new() { Name = attr.GetCustomAttributeCtorArg(context, 1) ?? prop.GetPropertyName(context), JsonPath = $"{path}.{prop.GetPropertyName(context)}", @@ -201,7 +199,7 @@ private static IEnumerable MapPrinterColumns( foreach (var attr in type.GetCustomAttributesData()) { - yield return new V1CustomResourceColumnDefinition + yield return new() { Name = attr.GetCustomAttributeCtorArg(context, 1), JsonPath = attr.GetCustomAttributeCtorArg(context, 0), @@ -232,9 +230,11 @@ private static V1JSONSchemaProps Map(this MetadataLoadContext context, PropertyI if (prop.GetCustomAttributeData() is { } extDocs) { - props.ExternalDocs = new V1ExternalDocumentation( - extDocs.GetCustomAttributeCtorArg(context, 0), - extDocs.GetCustomAttributeCtorArg(context, 1)); + props.ExternalDocs = new() + { + Url = extDocs.GetCustomAttributeCtorArg(context, 0), + Description = extDocs.GetCustomAttributeCtorArg(context, 1), + }; } if (prop.GetCustomAttributeData() is { } items) @@ -289,12 +289,14 @@ private static V1JSONSchemaProps Map(this MetadataLoadContext context, PropertyI if (prop.GetCustomAttributesData().ToArray() is { Length: > 0 } validations) { props.XKubernetesValidations = validations - .Select(validation => new V1ValidationRule( - validation.GetCustomAttributeCtorArg(context, 0), - fieldPath: validation.GetCustomAttributeCtorArg(context, 1), - message: validation.GetCustomAttributeCtorArg(context, 2), - messageExpression: validation.GetCustomAttributeCtorArg(context, 3), - reason: validation.GetCustomAttributeCtorArg(context, 4))) + .Select(validation => new V1ValidationRule() + { + Rule = validation.GetCustomAttributeCtorArg(context, 0), + FieldPath = validation.GetCustomAttributeCtorArg(context, 1), + Message = validation.GetCustomAttributeCtorArg(context, 2), + MessageExpression = validation.GetCustomAttributeCtorArg(context, 3), + Reason = validation.GetCustomAttributeCtorArg(context, 4), + }) .ToList(); } @@ -305,12 +307,12 @@ private static V1JSONSchemaProps Map(this MetadataLoadContext context, Type type { if (type.FullName == "System.String") { - return new V1JSONSchemaProps { Type = String }; + return new() { Type = String }; } if (type.FullName == "System.Object") { - return new V1JSONSchemaProps { Type = Object, XKubernetesPreserveUnknownFields = true }; + return new() { Type = Object, XKubernetesPreserveUnknownFields = true }; } if (type.Name == typeof(Nullable<>).Name && type.GenericTypeArguments.Length == 1) @@ -336,12 +338,12 @@ private static V1JSONSchemaProps Map(this MetadataLoadContext context, Type type && i.GetGenericTypeDefinition().FullName == typeof(IDictionary<,>).FullName); var additionalProperties = context.Map(dictionaryImpl.GenericTypeArguments[1]); - return new V1JSONSchemaProps { Type = Object, AdditionalProperties = additionalProperties, }; + return new() { Type = Object, AdditionalProperties = additionalProperties, }; } if (interfaceNames.Contains(typeof(IDictionary).FullName)) { - return new V1JSONSchemaProps { Type = Object, XKubernetesPreserveUnknownFields = true }; + return new() { Type = Object, XKubernetesPreserveUnknownFields = true }; } if (interfaceNames.Contains(typeof(IEnumerable<>).FullName)) @@ -381,7 +383,7 @@ static Type GetRootBaseType(Type type) { "System.Object" => context.MapObjectType(type), "System.ValueType" => context.MapValueType(type), - "System.Enum" => new V1JSONSchemaProps { Type = String, EnumProperty = GetEnumNames(context, type), }, + "System.Enum" => new() { Type = String, EnumProperty = GetEnumNames(context, type), }, _ => throw InvalidType(type), }; } @@ -426,17 +428,17 @@ private static V1JSONSchemaProps MapObjectType(this MetadataLoadContext context, { case "k8s.Models.ResourceQuantity": // Quantities are serialized as strings in CRDs (e.g., "500m", "2Gi") - return new V1JSONSchemaProps { Type = String }; + return new() { Type = String }; case "k8s.Models.V1ObjectMeta": - return new V1JSONSchemaProps { Type = Object }; - case "k8s.Models.IntstrIntOrString": - return new V1JSONSchemaProps { XKubernetesIntOrString = true }; + return new() { Type = Object }; + case "k8s.Models.IntOrString": + return new() { XKubernetesIntOrString = true }; default: if (context.GetContextType().IsAssignableFrom(type) && type is { IsAbstract: false, IsInterface: false } && type.Assembly == context.GetContextType().Assembly) { - return new V1JSONSchemaProps + return new() { Type = Object, Properties = null, @@ -445,7 +447,7 @@ private static V1JSONSchemaProps MapObjectType(this MetadataLoadContext context, }; } - return new V1JSONSchemaProps + return new() { Type = Object, Description = @@ -490,24 +492,24 @@ private static V1JSONSchemaProps MapEnumerationType( if (listType.IsGenericType && listType.GetGenericTypeDefinition().FullName == typeof(KeyValuePair<,>).FullName) { var additionalProperties = context.Map(listType.GenericTypeArguments[1]); - return new V1JSONSchemaProps { Type = Object, AdditionalProperties = additionalProperties, }; + return new() { Type = Object, AdditionalProperties = additionalProperties, }; } var items = context.Map(listType); - return new V1JSONSchemaProps { Type = Array, Items = items }; + return new() { Type = Array, Items = items }; } private static V1JSONSchemaProps MapValueType(this MetadataLoadContext _, Type type) => type.FullName switch { - "System.Int32" => new V1JSONSchemaProps { Type = Integer, Format = Int32 }, - "System.Int64" => new V1JSONSchemaProps { Type = Integer, Format = Int64 }, - "System.Single" => new V1JSONSchemaProps { Type = Number, Format = Float }, - "System.Double" => new V1JSONSchemaProps { Type = Number, Format = Double }, - "System.Decimal" => new V1JSONSchemaProps { Type = Number, Format = Decimal }, - "System.Boolean" => new V1JSONSchemaProps { Type = Boolean }, - "System.DateTime" => new V1JSONSchemaProps { Type = String, Format = DateTime }, - "System.DateTimeOffset" => new V1JSONSchemaProps { Type = String, Format = DateTime }, + "System.Int32" => new() { Type = Integer, Format = Int32 }, + "System.Int64" => new() { Type = Integer, Format = Int64 }, + "System.Single" => new() { Type = Number, Format = Float }, + "System.Double" => new() { Type = Number, Format = Double }, + "System.Decimal" => new() { Type = Number, Format = Decimal }, + "System.Boolean" => new() { Type = Boolean }, + "System.DateTime" => new() { Type = String, Format = DateTime }, + "System.DateTimeOffset" => new() { Type = String, Format = DateTime }, _ => throw InvalidType(type), }; diff --git a/test/KubeOps.KubernetesClient.Test/KubernetesClient.Test.cs b/test/KubeOps.KubernetesClient.Test/KubernetesClient.Test.cs index aaa88540..ac162447 100644 --- a/test/KubeOps.KubernetesClient.Test/KubernetesClient.Test.cs +++ b/test/KubeOps.KubernetesClient.Test/KubernetesClient.Test.cs @@ -8,7 +8,7 @@ namespace KubeOps.KubernetesClient.Test; -public class KubernetesClientTest : IntegrationTestBase, IDisposable +public sealed class KubernetesClientTest : IntegrationTestBase, IDisposable { private readonly IKubernetesClient _client = new KubernetesClient(); @@ -30,7 +30,11 @@ public void Should_Create_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); @@ -48,7 +52,11 @@ public void Should_Get_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); @@ -58,7 +66,11 @@ public void Should_Get_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, })); _objects.Add(_client.Create( @@ -66,7 +78,11 @@ public void Should_Get_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, })); @@ -82,7 +98,11 @@ public void Should_Update_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new V1ObjectMeta(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); var r1 = config.Metadata.ResourceVersion; @@ -103,7 +123,11 @@ public void Should_List_Some_Objects() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); var config2 = _client.Create( @@ -111,7 +135,11 @@ public void Should_List_Some_Objects() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); @@ -132,7 +160,11 @@ public void Should_Delete_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); var config2 = _client.Create( @@ -140,7 +172,11 @@ public void Should_Delete_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); _objects.Add(config1); @@ -161,7 +197,11 @@ public void Should_Not_Throw_On_Not_Found_Delete() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }; _client.Delete(config); @@ -176,7 +216,11 @@ public void Should_Patch_ConfigMap_Sync() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "foo", "bar" } }, }); _objects.Add(config); diff --git a/test/KubeOps.KubernetesClient.Test/KubernetesClientAsync.Test.cs b/test/KubeOps.KubernetesClient.Test/KubernetesClientAsync.Test.cs index 3a844a9a..cc14e0ab 100644 --- a/test/KubeOps.KubernetesClient.Test/KubernetesClientAsync.Test.cs +++ b/test/KubeOps.KubernetesClient.Test/KubernetesClientAsync.Test.cs @@ -8,7 +8,7 @@ namespace KubeOps.KubernetesClient.Test; -public class KubernetesClientAsyncTest : IntegrationTestBase, IDisposable +public sealed class KubernetesClientAsyncTest : IntegrationTestBase, IDisposable { private readonly IKubernetesClient _client = new KubernetesClient(); @@ -30,7 +30,11 @@ public async Task Should_Create_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); @@ -48,7 +52,11 @@ public async Task Should_Get_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); @@ -58,7 +66,11 @@ public async Task Should_Get_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, })); _objects.Add(await _client.CreateAsync( @@ -66,7 +78,11 @@ public async Task Should_Get_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, })); @@ -82,7 +98,11 @@ public async Task Should_Update_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new V1ObjectMeta(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); var r1 = config.Metadata.ResourceVersion; @@ -103,7 +123,11 @@ public async Task Should_List_Some_Objects() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); var config2 = await _client.CreateAsync( @@ -111,7 +135,11 @@ public async Task Should_List_Some_Objects() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); @@ -132,7 +160,11 @@ public async Task Should_Delete_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); var config2 = await _client.CreateAsync( @@ -140,7 +172,11 @@ public async Task Should_Delete_Some_Object() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }); _objects.Add(config1); @@ -161,7 +197,11 @@ public async Task Should_Not_Throw_On_Not_Found_Delete() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }; await _client.DeleteAsync(config); @@ -176,7 +216,11 @@ public async Task Should_Patch_ConfigMap_Async() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "foo", "bar" } }, }); _objects.Add(config); @@ -213,7 +257,11 @@ public async Task Should_Patch_ConfigMap_With_Stale_Base_Async() { Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, - Metadata = new(name: RandomName(), namespaceProperty: "default"), + Metadata = new() + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "foo", "bar" } }, }); _objects.Add(original); diff --git a/test/KubeOps.Operator.Test/IntegrationTestCollection.cs b/test/KubeOps.Operator.Test/IntegrationTestCollection.cs index 35ea16be..fba46a9d 100644 --- a/test/KubeOps.Operator.Test/IntegrationTestCollection.cs +++ b/test/KubeOps.Operator.Test/IntegrationTestCollection.cs @@ -64,7 +64,10 @@ public sealed class TestNamespaceProvider : IAsyncLifetime public async Task InitializeAsync() { _namespace = - await _client.CreateAsync(new V1Namespace(metadata: new V1ObjectMeta(name: Namespace)).Initialize()); + await _client.CreateAsync(new V1Namespace() + { + Metadata = new() { Name = Namespace }, + }.Initialize()); } public async Task DisposeAsync() diff --git a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs index 4b754878..9b7296e7 100644 --- a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs @@ -16,7 +16,7 @@ namespace KubeOps.Operator.Test; -public class NamespacedOperatorIntegrationTest : IntegrationTestBase +public sealed class NamespacedOperatorIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -58,7 +58,10 @@ public override async Task InitializeAsync() { await base.InitializeAsync(); _otherNamespace = - await _client.CreateAsync(new V1Namespace(metadata: new(name: Guid.NewGuid().ToString().ToLower())) + await _client.CreateAsync(new V1Namespace + { + Metadata = new() { Name = Guid.NewGuid().ToString().ToLower() }, + } .Initialize()); await _ns.InitializeAsync(); } diff --git a/test/KubeOps.Transpiler.Test/Crds.Mlc.Test.cs b/test/KubeOps.Transpiler.Test/Crds.Mlc.Test.cs index db2e217f..4bb2cc56 100644 --- a/test/KubeOps.Transpiler.Test/Crds.Mlc.Test.cs +++ b/test/KubeOps.Transpiler.Test/Crds.Mlc.Test.cs @@ -769,7 +769,7 @@ private class EnumerableKeyPairsEntity : CustomKubernetesEntity [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] private class IntstrOrStringEntity : CustomKubernetesEntity { - public IntstrIntOrString Property { get; set; } = null!; + public IntOrString Property { get; set; } = null!; } [KubernetesEntity(Group = "testing.dev", ApiVersion = "v1", Kind = "TestEntity")] From 464730ad8ab746716bfca2c87490e97892001c59 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 31 Oct 2025 08:05:05 +0100 Subject: [PATCH 47/58] refactor: apply object initializer shorthand and seal generator classes - Updated object initializations across generator classes to use initializer shorthand for consistency. - Marked generator classes as `sealed` for optimization and better design practices. - Improved readability and alignment with modern C# coding standards. --- .../Generators/DeploymentGenerator.cs | 45 ++++++----- .../Generators/MutationWebhookGenerator.cs | 14 ++-- src/KubeOps.Cli/Generators/RbacGenerator.cs | 26 +++++-- .../Generators/ValidationWebhookGenerator.cs | 16 ++-- .../Generators/WebhookDeploymentGenerator.cs | 78 +++++++++++-------- 5 files changed, 110 insertions(+), 69 deletions(-) diff --git a/src/KubeOps.Cli/Generators/DeploymentGenerator.cs b/src/KubeOps.Cli/Generators/DeploymentGenerator.cs index 04669be4..b8e31627 100644 --- a/src/KubeOps.Cli/Generators/DeploymentGenerator.cs +++ b/src/KubeOps.Cli/Generators/DeploymentGenerator.cs @@ -9,24 +9,33 @@ namespace KubeOps.Cli.Generators; -internal class DeploymentGenerator(OutputFormat format) : IConfigGenerator +internal sealed class DeploymentGenerator(OutputFormat format) : IConfigGenerator { public void Generate(ResultOutput output) { - var deployment = new V1Deployment(metadata: new V1ObjectMeta( - labels: new Dictionary { { "operator-deployment", "kubernetes-operator" } }, - name: "operator")).Initialize(); - deployment.Spec = new V1DeploymentSpec + var deployment = new V1Deployment + { + Metadata = new() + { + Name = "operator", + Labels = new Dictionary { { "operator-deployment", "kubernetes-operator" } }, + }, + }.Initialize(); + deployment.Spec = new() { Replicas = 1, RevisionHistoryLimit = 0, - Selector = new V1LabelSelector( - matchLabels: new Dictionary { { "operator-deployment", "kubernetes-operator" } }), - Template = new V1PodTemplateSpec + Selector = new() { - Metadata = new V1ObjectMeta( - labels: new Dictionary { { "operator-deployment", "kubernetes-operator" } }), - Spec = new V1PodSpec + MatchLabels = new Dictionary { { "operator-deployment", "kubernetes-operator" } }, + }, + Template = new() + { + Metadata = new() + { + Labels = new Dictionary { { "operator-deployment", "kubernetes-operator" } }, + }, + Spec = new() { TerminationGracePeriodSeconds = 10, Containers = new List @@ -41,26 +50,26 @@ public void Generate(ResultOutput output) { Name = "POD_NAMESPACE", ValueFrom = - new V1EnvVarSource + new() { - FieldRef = new V1ObjectFieldSelector + FieldRef = new() { FieldPath = "metadata.namespace", }, }, }, }, - Resources = new V1ResourceRequirements + Resources = new() { Requests = new Dictionary { - { "cpu", new ResourceQuantity("100m") }, - { "memory", new ResourceQuantity("64Mi") }, + { "cpu", new("100m") }, + { "memory", new("64Mi") }, }, Limits = new Dictionary { - { "cpu", new ResourceQuantity("100m") }, - { "memory", new ResourceQuantity("128Mi") }, + { "cpu", new("100m") }, + { "memory", new("128Mi") }, }, }, }, diff --git a/src/KubeOps.Cli/Generators/MutationWebhookGenerator.cs b/src/KubeOps.Cli/Generators/MutationWebhookGenerator.cs index 1e49a9fa..bf904e75 100644 --- a/src/KubeOps.Cli/Generators/MutationWebhookGenerator.cs +++ b/src/KubeOps.Cli/Generators/MutationWebhookGenerator.cs @@ -20,13 +20,15 @@ public void Generate(ResultOutput output) return; } - var mutatorConfig = new V1MutatingWebhookConfiguration( - metadata: new V1ObjectMeta(name: "mutators"), - webhooks: new List()).Initialize(); + var mutatorConfig = new V1MutatingWebhookConfiguration + { + Metadata = new() { Name = "mutators" }, + Webhooks = new List(), + }.Initialize(); foreach (var hook in webhooks) { - mutatorConfig.Webhooks.Add(new V1MutatingWebhook + mutatorConfig.Webhooks.Add(new() { Name = $"mutate.{hook.Metadata.SingularName}.{hook.Metadata.Group}.{hook.Metadata.Version}", MatchPolicy = "Exact", @@ -42,10 +44,10 @@ public void Generate(ResultOutput output) ApiVersions = new[] { hook.Metadata.Version }, }, }, - ClientConfig = new Admissionregistrationv1WebhookClientConfig + ClientConfig = new() { CaBundle = caBundle, - Service = new Admissionregistrationv1ServiceReference + Service = new() { Name = "operator", Path = hook.WebhookPath, diff --git a/src/KubeOps.Cli/Generators/RbacGenerator.cs b/src/KubeOps.Cli/Generators/RbacGenerator.cs index a1001516..c419fc1e 100644 --- a/src/KubeOps.Cli/Generators/RbacGenerator.cs +++ b/src/KubeOps.Cli/Generators/RbacGenerator.cs @@ -14,7 +14,7 @@ namespace KubeOps.Cli.Generators; -internal class RbacGenerator(MetadataLoadContext parser, +internal sealed class RbacGenerator(MetadataLoadContext parser, OutputFormat outputFormat) : IConfigGenerator { public void Generate(ResultOutput output) @@ -24,16 +24,28 @@ public void Generate(ResultOutput output) .Concat(parser.GetContextType().GetCustomAttributesData()) .ToList(); - var role = new V1ClusterRole(rules: parser.Transpile(attributes).ToList()).Initialize(); + var role = new V1ClusterRole { Rules = parser.Transpile(attributes).ToList() }.Initialize(); role.Metadata.Name = "operator-role"; output.Add($"operator-role.{outputFormat.GetFileExtension()}", role); - var roleBinding = new V1ClusterRoleBinding( - roleRef: new V1RoleRef(V1ClusterRole.KubeGroup, V1ClusterRole.KubeKind, "operator-role"), - subjects: new List + var roleBinding = new V1ClusterRoleBinding + { + RoleRef = new() { - new(V1ServiceAccount.KubeKind, "default", namespaceProperty: "system"), - }) + ApiGroup = V1ClusterRole.KubeGroup, + Kind = V1ClusterRole.KubeKind, + Name = "operator-role", + }, + Subjects = new List + { + new() + { + Kind = V1ServiceAccount.KubeKind, + Name = "default", + NamespaceProperty = "system", + }, + }, + } .Initialize(); roleBinding.Metadata.Name = "operator-role-binding"; output.Add($"operator-role-binding.{outputFormat.GetFileExtension()}", roleBinding); diff --git a/src/KubeOps.Cli/Generators/ValidationWebhookGenerator.cs b/src/KubeOps.Cli/Generators/ValidationWebhookGenerator.cs index bb70a3a4..0b1faec1 100644 --- a/src/KubeOps.Cli/Generators/ValidationWebhookGenerator.cs +++ b/src/KubeOps.Cli/Generators/ValidationWebhookGenerator.cs @@ -10,7 +10,7 @@ namespace KubeOps.Cli.Generators; -internal class ValidationWebhookGenerator +internal sealed class ValidationWebhookGenerator (List webhooks, byte[] caBundle, OutputFormat format) : IConfigGenerator { public void Generate(ResultOutput output) @@ -20,13 +20,15 @@ public void Generate(ResultOutput output) return; } - var validatorConfig = new V1ValidatingWebhookConfiguration( - metadata: new V1ObjectMeta(name: "validators"), - webhooks: new List()).Initialize(); + var validatorConfig = new V1ValidatingWebhookConfiguration + { + Metadata = new() { Name = "validators" }, + Webhooks = new List(), + }.Initialize(); foreach (var hook in webhooks) { - validatorConfig.Webhooks.Add(new V1ValidatingWebhook + validatorConfig.Webhooks.Add(new() { Name = $"validate.{hook.Metadata.SingularName}.{hook.Metadata.Group}.{hook.Metadata.Version}", MatchPolicy = "Exact", @@ -42,10 +44,10 @@ public void Generate(ResultOutput output) ApiVersions = new[] { hook.Metadata.Version }, }, }, - ClientConfig = new Admissionregistrationv1WebhookClientConfig + ClientConfig = new() { CaBundle = caBundle, - Service = new Admissionregistrationv1ServiceReference + Service = new() { Name = "operator", Path = hook.WebhookPath, diff --git a/src/KubeOps.Cli/Generators/WebhookDeploymentGenerator.cs b/src/KubeOps.Cli/Generators/WebhookDeploymentGenerator.cs index 6f4f449a..4ef06745 100644 --- a/src/KubeOps.Cli/Generators/WebhookDeploymentGenerator.cs +++ b/src/KubeOps.Cli/Generators/WebhookDeploymentGenerator.cs @@ -2,37 +2,40 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using System.Reflection; - using k8s; using k8s.Models; using KubeOps.Cli.Output; -using KubeOps.Cli.Transpilation; -using KubeOps.Transpiler; namespace KubeOps.Cli.Generators; -internal class WebhookDeploymentGenerator(OutputFormat format) : IConfigGenerator +internal sealed class WebhookDeploymentGenerator(OutputFormat format) : IConfigGenerator { public void Generate(ResultOutput output) { - var deployment = new V1Deployment(metadata: new V1ObjectMeta( - labels: new Dictionary { { "operator-deployment", "kubernetes-operator" } }, - name: "operator")).Initialize(); - deployment.Spec = new V1DeploymentSpec + var deployment = new V1Deployment + { + Metadata = new() + { + Name = "operator", + Labels = new Dictionary { { "operator-deployment", "kubernetes-operator" } }, + }, + }.Initialize(); + deployment.Spec = new() { Replicas = 1, RevisionHistoryLimit = 0, - Selector = new V1LabelSelector( - matchLabels: - new Dictionary { { "operator-deployment", "kubernetes-operator" } }), - Template = new V1PodTemplateSpec + Selector = new() { - Metadata = new V1ObjectMeta( - labels: - new Dictionary { { "operator-deployment", "kubernetes-operator" }, }), - Spec = new V1PodSpec + MatchLabels = new Dictionary { { "operator-deployment", "kubernetes-operator" } }, + }, + Template = new() + { + Metadata = new() + { + Labels = new Dictionary { { "operator-deployment", "kubernetes-operator" } }, + }, + Spec = new() { TerminationGracePeriodSeconds = 10, Volumes = new List @@ -57,9 +60,9 @@ public void Generate(ResultOutput output) { Name = "POD_NAMESPACE", ValueFrom = - new V1EnvVarSource + new() { - FieldRef = new V1ObjectFieldSelector + FieldRef = new() { FieldPath = "metadata.namespace", }, @@ -71,18 +74,18 @@ public void Generate(ResultOutput output) { new() { ConfigMapRef = new() { Name = "webhook-config" } }, }, - Ports = new List { new(5001, name: "https"), }, - Resources = new V1ResourceRequirements + Ports = new List { new() { HostPort = 5001, Name = "https" } }, + Resources = new() { Requests = new Dictionary { - { "cpu", new ResourceQuantity("100m") }, - { "memory", new ResourceQuantity("64Mi") }, + { "cpu", new("100m") }, + { "memory", new("64Mi") }, }, Limits = new Dictionary { - { "cpu", new ResourceQuantity("100m") }, - { "memory", new ResourceQuantity("128Mi") }, + { "cpu", new("100m") }, + { "memory", new("128Mi") }, }, }, }, @@ -94,13 +97,26 @@ public void Generate(ResultOutput output) output.Add( $"service.{format.GetFileExtension()}", - new V1Service( - metadata: new V1ObjectMeta(name: "operator"), - spec: new V1ServiceSpec + new V1Service + { + Metadata = new() { Name = "operator" }, + Spec = new() { Ports = - new List { new() { Name = "https", TargetPort = "https", Port = 443, }, }, - Selector = new Dictionary { { "operator-deployment", "kubernetes-operator" }, }, - }).Initialize()); + new List + { + new() + { + Name = "https", + TargetPort = "https", + Port = 443, + }, + }, + Selector = new Dictionary + { + { "operator-deployment", "kubernetes-operator" }, + }, + }, + }.Initialize()); } } From a8a1366b391702fef6a6e4494dae7ee82f50d356 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 31 Oct 2025 08:34:02 +0100 Subject: [PATCH 48/58] refactor: mark entities and tests as `sealed`, add null checks in finalizer tests - Marked entities (`V1OperatorIntegrationTestEntity` and its sub-classes) and test classes as `sealed` for better design and optimization. - Added null checks in finalizer integration tests to improve safety and adherence to modern C# standards. - Disabled obsolete warning in `KubernetesClient` with a TODO for clarification. --- src/KubeOps.KubernetesClient/KubernetesClient.cs | 2 ++ .../Finalizer/EntityFinalizer.Integration.Test.cs | 6 ++++-- .../TestEntities/V1OperatorIntegrationTestEntity.cs | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/KubeOps.KubernetesClient/KubernetesClient.cs b/src/KubeOps.KubernetesClient/KubernetesClient.cs index 95383aab..18853eca 100644 --- a/src/KubeOps.KubernetesClient/KubernetesClient.cs +++ b/src/KubeOps.KubernetesClient/KubernetesClient.cs @@ -16,6 +16,8 @@ namespace KubeOps.KubernetesClient; +#pragma warning disable CS0618 // Type or member is obsolete - TODO: clarify with k8s team + /// public class KubernetesClient : IKubernetesClient { diff --git a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs index 2c046943..36f979f8 100644 --- a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs @@ -16,7 +16,7 @@ namespace KubeOps.Operator.Test.Finalizer; -public class EntityFinalizerIntegrationTest : IntegrationTestBase +public sealed class EntityFinalizerIntegrationTest : IntegrationTestBase { private readonly InvocationCounter _mock = new(); private readonly IKubernetesClient _client = new KubernetesClient.KubernetesClient(); @@ -91,7 +91,9 @@ public async Task Should_Attach_Multiple_Finalizer_On_Entity() await watcherCounter.WaitForInvocations; var result = await _client.GetAsync("first-second", _ns.Namespace); - result!.Metadata.Finalizers.Should().Contain("first"); + result.Should().NotBeNull(); + result.Metadata.Should().NotBeNull(); + result.Metadata.Finalizers.Should().Contain("first"); result.Metadata.Finalizers.Should().Contain("second"); } diff --git a/test/KubeOps.Operator.Test/TestEntities/V1OperatorIntegrationTestEntity.cs b/test/KubeOps.Operator.Test/TestEntities/V1OperatorIntegrationTestEntity.cs index 39cb0652..3e9547c4 100644 --- a/test/KubeOps.Operator.Test/TestEntities/V1OperatorIntegrationTestEntity.cs +++ b/test/KubeOps.Operator.Test/TestEntities/V1OperatorIntegrationTestEntity.cs @@ -9,7 +9,7 @@ namespace KubeOps.Operator.Test.TestEntities; [KubernetesEntity(Group = "operator.test", ApiVersion = "v1", Kind = "OperatorIntegrationTest")] -public class V1OperatorIntegrationTestEntity : CustomKubernetesEntity { public V1OperatorIntegrationTestEntity() @@ -27,12 +27,12 @@ public V1OperatorIntegrationTestEntity(string name, string username, string ns) public override string ToString() => $"Test Entity ({Metadata.Name}): {Spec.Username}"; - public class EntitySpec + public sealed class EntitySpec { public string Username { get; set; } = string.Empty; } - public class EntityStatus + public sealed class EntityStatus { public string Status { get; set; } = string.Empty; } From 812252caa178b4995e8466f5bfdc50ec4a3ccb99 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 31 Oct 2025 08:57:12 +0100 Subject: [PATCH 49/58] refactor: adjust formatting for consistency and readability across tests and core classes - Updated object and dictionary initializations to improve readability. - Applied consistent formatting to multiline constructs and private methods. - Improved alignment with modern C# coding style. --- src/KubeOps.Cli/Generators/RbacGenerator.cs | 3 ++- .../KubernetesClient.Test.cs | 17 +++++++++++------ .../KubernetesClientAsync.Test.cs | 17 +++++++++++------ .../NamespacedOperator.Integration.Test.cs | 3 ++- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/KubeOps.Cli/Generators/RbacGenerator.cs b/src/KubeOps.Cli/Generators/RbacGenerator.cs index c419fc1e..767e2e0e 100644 --- a/src/KubeOps.Cli/Generators/RbacGenerator.cs +++ b/src/KubeOps.Cli/Generators/RbacGenerator.cs @@ -14,7 +14,8 @@ namespace KubeOps.Cli.Generators; -internal sealed class RbacGenerator(MetadataLoadContext parser, +internal sealed class RbacGenerator( + MetadataLoadContext parser, OutputFormat outputFormat) : IConfigGenerator { public void Generate(ResultOutput output) diff --git a/test/KubeOps.KubernetesClient.Test/KubernetesClient.Test.cs b/test/KubeOps.KubernetesClient.Test/KubernetesClient.Test.cs index ac162447..6f5b81ec 100644 --- a/test/KubeOps.KubernetesClient.Test/KubernetesClient.Test.cs +++ b/test/KubeOps.KubernetesClient.Test/KubernetesClient.Test.cs @@ -198,10 +198,10 @@ public void Should_Not_Throw_On_Not_Found_Delete() Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, Metadata = new() - { - Name = RandomName(), - NamespaceProperty = "default", - }, + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }; _client.Delete(config); @@ -237,7 +237,11 @@ public void Should_Patch_ConfigMap_Sync() Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, Metadata = from.Metadata, - Data = new Dictionary { { "foo", "baz" }, { "hello", "world" } }, + Data = new Dictionary + { + { "foo", "baz" }, + { "hello", "world" } + }, }; config = _client.Patch(from, to); config.Data["foo"].Should().Be("baz"); @@ -253,5 +257,6 @@ public void Dispose() _client.Delete(_objects); } - private static string RandomName() => "cm-" + Guid.NewGuid().ToString().ToLower(); + private static string RandomName() + => "cm-" + Guid.NewGuid().ToString().ToLower(); } diff --git a/test/KubeOps.KubernetesClient.Test/KubernetesClientAsync.Test.cs b/test/KubeOps.KubernetesClient.Test/KubernetesClientAsync.Test.cs index cc14e0ab..bc74000b 100644 --- a/test/KubeOps.KubernetesClient.Test/KubernetesClientAsync.Test.cs +++ b/test/KubeOps.KubernetesClient.Test/KubernetesClientAsync.Test.cs @@ -198,10 +198,10 @@ public async Task Should_Not_Throw_On_Not_Found_Delete() Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, Metadata = new() - { - Name = RandomName(), - NamespaceProperty = "default", - }, + { + Name = RandomName(), + NamespaceProperty = "default", + }, Data = new Dictionary { { "Hello", "World" } }, }; await _client.DeleteAsync(config); @@ -237,7 +237,11 @@ public async Task Should_Patch_ConfigMap_Async() Kind = V1ConfigMap.KubeKind, ApiVersion = V1ConfigMap.KubeApiVersion, Metadata = from.Metadata, - Data = new Dictionary { { "foo", "baz" }, { "hello", "world" } }, + Data = new Dictionary + { + { "foo", "baz" }, + { "hello", "world" } + }, }; config = await _client.PatchAsync(from, to); config.Data["foo"].Should().Be("baz"); @@ -283,5 +287,6 @@ public void Dispose() _client.Delete(_objects); } - private static string RandomName() => "cm-" + Guid.NewGuid().ToString().ToLower(); + private static string RandomName() + => "cm-" + Guid.NewGuid().ToString().ToLower(); } diff --git a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs index 9b7296e7..e8e8a790 100644 --- a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs @@ -82,7 +82,8 @@ protected override void ConfigureHost(HostApplicationBuilder builder) .AddController(); } - private class TestController(InvocationCounter svc) : IEntityController + private class TestController(InvocationCounter svc) + : IEntityController { public Task ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) { From ed5c1402e55e3554884c5220a91c857068a49936 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 31 Oct 2025 09:00:48 +0100 Subject: [PATCH 50/58] refactor: fixed whitespace formatting - Adjusted formatting in `RbacGenerator` and `NamespacedOperator.Integration.Test` to enhance consistency and readability. --- src/KubeOps.Cli/Generators/RbacGenerator.cs | 30 +++++++++---------- .../NamespacedOperator.Integration.Test.cs | 3 +- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/KubeOps.Cli/Generators/RbacGenerator.cs b/src/KubeOps.Cli/Generators/RbacGenerator.cs index 767e2e0e..8975a1de 100644 --- a/src/KubeOps.Cli/Generators/RbacGenerator.cs +++ b/src/KubeOps.Cli/Generators/RbacGenerator.cs @@ -30,24 +30,24 @@ public void Generate(ResultOutput output) output.Add($"operator-role.{outputFormat.GetFileExtension()}", role); var roleBinding = new V1ClusterRoleBinding + { + RoleRef = new() { - RoleRef = new() - { - ApiGroup = V1ClusterRole.KubeGroup, - Kind = V1ClusterRole.KubeKind, - Name = "operator-role", - }, - Subjects = new List + ApiGroup = V1ClusterRole.KubeGroup, + Kind = V1ClusterRole.KubeKind, + Name = "operator-role", + }, + Subjects = new List + { + new() { - new() - { - Kind = V1ServiceAccount.KubeKind, - Name = "default", - NamespaceProperty = "system", - }, + Kind = V1ServiceAccount.KubeKind, + Name = "default", + NamespaceProperty = "system", }, - } - .Initialize(); + }, + } + .Initialize(); roleBinding.Metadata.Name = "operator-role-binding"; output.Add($"operator-role-binding.{outputFormat.GetFileExtension()}", roleBinding); } diff --git a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs index e8e8a790..936fe057 100644 --- a/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/NamespacedOperator.Integration.Test.cs @@ -58,7 +58,8 @@ public override async Task InitializeAsync() { await base.InitializeAsync(); _otherNamespace = - await _client.CreateAsync(new V1Namespace + await _client.CreateAsync( + new V1Namespace { Metadata = new() { Name = Guid.NewGuid().ToString().ToLower() }, } From 58ea5280e4f5f1602cf6343fa84b7ed9795b4bf5 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 31 Oct 2025 09:12:12 +0100 Subject: [PATCH 51/58] chore: add Apache 2.0 license headers and improve readability - Added license headers to multiple source and test files to align with .NET Foundation standards. - Adjusted method formatting in `V1TestEntityController` for better readability and consistency. --- examples/Operator/Controller/V1TestEntityController.cs | 9 +++++---- .../Reconciliation/ReconciliationResult{TEntity}.cs | 4 ++++ src/KubeOps.Operator/Properties/AssemblyInfo.cs | 4 ++++ .../Properties/GlobalAssemblyInfo.cs | 4 ++++ test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs | 4 ++++ .../Properties/GlobalAssemblyInfo.cs | 4 ++++ .../Properties/GlobalAssemblyInfo.cs | 4 ++++ .../Properties/GlobalAssemblyInfo.cs | 4 ++++ .../Properties/GlobalAssemblyInfo.cs | 4 ++++ .../Properties/GlobalAssemblyInfo.cs | 4 ++++ 10 files changed, 41 insertions(+), 4 deletions(-) diff --git a/examples/Operator/Controller/V1TestEntityController.cs b/examples/Operator/Controller/V1TestEntityController.cs index aaf4b572..d6a53340 100644 --- a/examples/Operator/Controller/V1TestEntityController.cs +++ b/examples/Operator/Controller/V1TestEntityController.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using KubeOps.Abstractions.Events; using KubeOps.Abstractions.Rbac; using KubeOps.Abstractions.Reconciliation; using KubeOps.Abstractions.Reconciliation.Controller; @@ -14,16 +13,18 @@ namespace Operator.Controller; [EntityRbac(typeof(V1TestEntity), Verbs = RbacVerb.All)] -public sealed class V1TestEntityController(ILogger logger) +public sealed class V1TestEntityController(ILogger logger) : IEntityController { - public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> ReconcileAsync( + V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Reconciling entity {Entity}.", entity); return Task.FromResult(ReconciliationResult.Success(entity)); } - public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) + public Task> DeletedAsync( + V1TestEntity entity, CancellationToken cancellationToken) { logger.LogInformation("Deleting entity {Entity}.", entity); return Task.FromResult(ReconciliationResult.Success(entity)); diff --git a/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs b/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs index bfb32824..db4fa98f 100644 --- a/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs +++ b/src/KubeOps.Abstractions/Reconciliation/ReconciliationResult{TEntity}.cs @@ -1,3 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + using System.Diagnostics.CodeAnalysis; using k8s; diff --git a/src/KubeOps.Operator/Properties/AssemblyInfo.cs b/src/KubeOps.Operator/Properties/AssemblyInfo.cs index ba834f24..09d683dd 100644 --- a/src/KubeOps.Operator/Properties/AssemblyInfo.cs +++ b/src/KubeOps.Operator/Properties/AssemblyInfo.cs @@ -1,3 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("KubeOps.Operator.Tests")] diff --git a/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs index 5e80f0d0..791c4aff 100644 --- a/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs +++ b/test/KubeOps.Abstractions.Test/Properties/GlobalAssemblyInfo.cs @@ -1,3 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + using System.Diagnostics.CodeAnalysis; [assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs index 5e80f0d0..791c4aff 100644 --- a/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs +++ b/test/KubeOps.Cli.Test/Properties/GlobalAssemblyInfo.cs @@ -1,3 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + using System.Diagnostics.CodeAnalysis; [assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs index 5e80f0d0..791c4aff 100644 --- a/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs +++ b/test/KubeOps.Generator.Test/Properties/GlobalAssemblyInfo.cs @@ -1,3 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + using System.Diagnostics.CodeAnalysis; [assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs index 5e80f0d0..791c4aff 100644 --- a/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs +++ b/test/KubeOps.KubernetesClient.Test/Properties/GlobalAssemblyInfo.cs @@ -1,3 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + using System.Diagnostics.CodeAnalysis; [assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs index 5e80f0d0..791c4aff 100644 --- a/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs +++ b/test/KubeOps.Operator.Test/Properties/GlobalAssemblyInfo.cs @@ -1,3 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + using System.Diagnostics.CodeAnalysis; [assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs index 5e80f0d0..791c4aff 100644 --- a/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs +++ b/test/KubeOps.Operator.Web.Test/Properties/GlobalAssemblyInfo.cs @@ -1,3 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + using System.Diagnostics.CodeAnalysis; [assembly: ExcludeFromCodeCoverage] diff --git a/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs b/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs index 5e80f0d0..791c4aff 100644 --- a/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs +++ b/test/KubeOps.Transpiler.Test/Properties/GlobalAssemblyInfo.cs @@ -1,3 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + using System.Diagnostics.CodeAnalysis; [assembly: ExcludeFromCodeCoverage] From 1e6380456b255f928a56d2d9b6a0386be251cc9b Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 31 Oct 2025 15:27:18 +0100 Subject: [PATCH 52/58] docs: remove details in finalizer configuration guide - Simplified explanation of `AutoDetachFinalizers` behavior. - Updated code documentation to clarify handling of processed messages. --- docs/docs/operator/advanced-configuration.mdx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/docs/operator/advanced-configuration.mdx b/docs/docs/operator/advanced-configuration.mdx index 76a412c2..fba7af1c 100644 --- a/docs/docs/operator/advanced-configuration.mdx +++ b/docs/docs/operator/advanced-configuration.mdx @@ -79,7 +79,6 @@ builder.Services When `AutoDetachFinalizers` is enabled: - Finalizers are automatically removed when `FinalizeAsync` returns success -- The entity is then allowed to be deleted by Kubernetes When disabled: ```csharp @@ -291,8 +290,7 @@ public sealed class DurableTimedEntityQueue( public Task Remove(TEntity entity, CancellationToken cancellationToken) { - // Azure Service Bus doesn't support removing scheduled messages - // Consider using message deduplication or message properties to skip processing + // will be automatically removed when the message is processed return Task.CompletedTask; } From 18450f62c8c5369c04ad8b882c194f10a9c599ba Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Fri, 31 Oct 2025 15:39:30 +0100 Subject: [PATCH 53/58] docs: update advanced configuration guide with time synchronization tip - Added a note about ensuring cluster and local time synchronization to address potential leader election issues caused by time drift. --- docs/docs/operator/advanced-configuration.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/operator/advanced-configuration.mdx b/docs/docs/operator/advanced-configuration.mdx index fba7af1c..d6469a63 100644 --- a/docs/docs/operator/advanced-configuration.mdx +++ b/docs/docs/operator/advanced-configuration.mdx @@ -580,6 +580,7 @@ This combination provides: - Check network connectivity between instances - Review lease duration settings - Monitor logs for leader election events +- Ensure cluster time and local time are synchronized (time drift can cause lease issues) ### Requeue Problems From 42ea86121200b598d2c1009550ffa9034fb7f526 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Mon, 3 Nov 2025 14:40:25 +0100 Subject: [PATCH 54/58] refactor: improve naming consistency - Updated parameter and method names for clarity and better alignment with coding conventions. - Replaced `provider` with `serviceProvider` and `settings` with `operatorSettings`. - Enhanced readability and alignment with modern C# standards in reconciliation methods. --- .../Reconciliation/Reconciler.cs | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index 5513dc23..e007d58b 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -37,9 +37,9 @@ namespace KubeOps.Operator.Reconciliation; internal sealed class Reconciler( ILogger> logger, IFusionCacheProvider cacheProvider, - IServiceProvider provider, - OperatorSettings settings, - ITimedEntityQueue requeue, + IServiceProvider serviceProvider, + OperatorSettings operatorSettings, + ITimedEntityQueue entityQueue, IKubernetesClient client) : IReconciler where TEntity : IKubernetesObject @@ -48,7 +48,7 @@ internal sealed class Reconciler( public async Task> Reconcile(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - await requeue + await entityQueue .Remove( reconciliationContext.Entity, cancellationToken); @@ -64,7 +64,7 @@ await ReconcileDeletion(reconciliationContext, cancellationToken), if (result.RequeueAfter.HasValue) { - await requeue + await entityQueue .Enqueue( result.Entity, reconciliationContext.EventType.ToRequeueType(), @@ -104,9 +104,9 @@ await _entityCache.SetAsync( token: cancellationToken); } - return await ReconcileModificationAsync(reconciliationContext.Entity, cancellationToken); + return await ReconcileEntity(reconciliationContext.Entity, cancellationToken); case { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } }: - return await ReconcileFinalizersSequentialAsync(reconciliationContext.Entity, cancellationToken); + return await ReconcileFinalizersSequential(reconciliationContext.Entity, cancellationToken); default: return ReconciliationResult.Success(reconciliationContext.Entity); } @@ -114,7 +114,7 @@ await _entityCache.SetAsync( private async Task> ReconcileDeletion(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - await using var scope = provider.CreateAsyncScope(); + await using var scope = serviceProvider.CreateAsyncScope(); var controller = scope.ServiceProvider.GetRequiredService>(); var result = await controller.DeletedAsync(reconciliationContext.Entity, cancellationToken); @@ -126,11 +126,11 @@ private async Task> ReconcileDeletion(Reconciliati return result; } - private async Task> ReconcileModificationAsync(TEntity entity, CancellationToken cancellationToken) + private async Task> ReconcileEntity(TEntity entity, CancellationToken cancellationToken) { - await using var scope = provider.CreateAsyncScope(); + await using var scope = serviceProvider.CreateAsyncScope(); - if (settings.AutoAttachFinalizers) + if (operatorSettings.AutoAttachFinalizers) { var finalizers = scope.ServiceProvider.GetKeyedServices>(KeyedService.AnyKey); @@ -146,9 +146,9 @@ private async Task> ReconcileModificationAsync(TEn return await controller.ReconcileAsync(entity, cancellationToken); } - private async Task> ReconcileFinalizersSequentialAsync(TEntity entity, CancellationToken cancellationToken) + private async Task> ReconcileFinalizersSequential(TEntity entity, CancellationToken cancellationToken) { - await using var scope = provider.CreateAsyncScope(); + await using var scope = serviceProvider.CreateAsyncScope(); // the condition to call ReconcileFinalizersSequentialAsync is: // { Metadata: { DeletionTimestamp: not null, Finalizers.Count: > 0 } } @@ -175,7 +175,7 @@ private async Task> ReconcileFinalizersSequentialA entity = result.Entity; - if (settings.AutoDetachFinalizers) + if (operatorSettings.AutoDetachFinalizers) { entity.RemoveFinalizer(identifier); entity = await client.UpdateAsync(entity, cancellationToken); From 067a773eba9a910bc3dee11e71bea6b1ff1b9380 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Tue, 4 Nov 2025 12:36:18 +0100 Subject: [PATCH 55/58] try to fix CodeQL recommendations: https://codeql.github.com/codeql-query-help/csharp/cs-log-forging/ --- .../Controller/V1TestEntityController.cs | 6 ++++-- examples/Operator/Controller/V1TestEntityController.cs | 6 ++++-- .../WebhookOperator/Controller/V1TestEntityController.cs | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs b/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs index 406a176c..08b05179 100644 --- a/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs +++ b/examples/ConversionWebhookOperator/Controller/V1TestEntityController.cs @@ -4,6 +4,8 @@ using ConversionWebhookOperator.Entities; +using k8s.Models; + using KubeOps.Abstractions.Rbac; using KubeOps.Abstractions.Reconciliation; using KubeOps.Abstractions.Reconciliation.Controller; @@ -15,13 +17,13 @@ public class V1TestEntityController(ILogger logger) : IE { public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Reconciling entity {Entity}.", entity); + logger.LogInformation("Reconciling entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); return Task.FromResult(ReconciliationResult.Success(entity)); } public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Deleted entity {Entity}.", entity); + logger.LogInformation("Deleted entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/examples/Operator/Controller/V1TestEntityController.cs b/examples/Operator/Controller/V1TestEntityController.cs index d6a53340..a593e872 100644 --- a/examples/Operator/Controller/V1TestEntityController.cs +++ b/examples/Operator/Controller/V1TestEntityController.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. +using k8s.Models; + using KubeOps.Abstractions.Rbac; using KubeOps.Abstractions.Reconciliation; using KubeOps.Abstractions.Reconciliation.Controller; @@ -19,14 +21,14 @@ public sealed class V1TestEntityController(ILogger logge public Task> ReconcileAsync( V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Reconciling entity {Entity}.", entity); + logger.LogInformation("Reconciling entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); return Task.FromResult(ReconciliationResult.Success(entity)); } public Task> DeletedAsync( V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Deleting entity {Entity}.", entity); + logger.LogInformation("Deleted entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); return Task.FromResult(ReconciliationResult.Success(entity)); } } diff --git a/examples/WebhookOperator/Controller/V1TestEntityController.cs b/examples/WebhookOperator/Controller/V1TestEntityController.cs index 24b9a59f..72bae63d 100644 --- a/examples/WebhookOperator/Controller/V1TestEntityController.cs +++ b/examples/WebhookOperator/Controller/V1TestEntityController.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. +using k8s.Models; + using KubeOps.Abstractions.Rbac; using KubeOps.Abstractions.Reconciliation; using KubeOps.Abstractions.Reconciliation.Controller; @@ -15,13 +17,13 @@ public sealed class V1TestEntityController(ILogger logge { public Task> ReconcileAsync(V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Reconciling entity {Entity}.", entity); + logger.LogInformation("Reconciling entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); return Task.FromResult(ReconciliationResult.Success(entity)); } public Task> DeletedAsync(V1TestEntity entity, CancellationToken cancellationToken) { - logger.LogInformation("Deleted entity {Entity}.", entity); + logger.LogInformation("Deleted entity {Namespace}/{Name}.", entity.Namespace(), entity.Name()); return Task.FromResult(ReconciliationResult.Success(entity)); } } From 17944e99bb10227ce526282abf0f5672b5c8c204 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Tue, 4 Nov 2025 13:06:35 +0100 Subject: [PATCH 56/58] restore finalizer integration tests with new configuration options - Added `AutoAttachFinalizers` and `AutoDetachFinalizers` configurations in integration tests. - Updated finalizer method signatures to include cancellation tokens for improved handling. --- .../Finalizer/EntityFinalizer.Integration.Test.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs index ab13f4a2..54574859 100644 --- a/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Finalizer/EntityFinalizer.Integration.Test.cs @@ -205,7 +205,12 @@ protected override void ConfigureHost(HostApplicationBuilder builder) { builder.Services .AddSingleton(_mock) - .AddKubernetesOperator(s => s.Namespace = _ns.Namespace) + .AddKubernetesOperator(s => + { + s.Namespace = _ns.Namespace; + s.AutoAttachFinalizers = false; + s.AutoDetachFinalizers = true; + }) .AddController() .AddFinalizer("first") .AddFinalizer("second"); @@ -221,12 +226,12 @@ public async Task> Reconci svc.Invocation(entity); if (entity.Name().Contains("first")) { - entity = await first(entity); + entity = await first(entity, cancellationToken); } if (entity.Name().Contains("second")) { - await second(entity); + entity = await second(entity, cancellationToken); } return ReconciliationResult.Success(entity); From 8bc4345bb3f99073285cb2a0606e45d53d63b861 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Tue, 4 Nov 2025 13:21:01 +0100 Subject: [PATCH 57/58] fixed `No service for type` test issues --- .../CancelEntityRequeue.Integration.Test.cs | 11 +++++++++-- .../DeletedEntityRequeue.Integration.Test.cs | 5 ++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs index 007811f3..a6adf42c 100644 --- a/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/CancelEntityRequeue.Integration.Test.cs @@ -36,7 +36,10 @@ public async Task Should_Cancel_Requeue_If_New_Event_Fires() await _mock.WaitForInvocations; _mock.Invocations.Count.Should().Be(2); - Services.GetRequiredService>().Count.Should().Be(0); + var timedEntityQueue = Services.GetRequiredService>(); + timedEntityQueue.Should().NotBeNull(); + timedEntityQueue.Should().BeOfType>(); + timedEntityQueue.As>().Count.Should().Be(0); } [Fact] @@ -49,7 +52,11 @@ public async Task Should_Not_Affect_Queues_If_Only_Status_Updated() await _mock.WaitForInvocations; _mock.Invocations.Count.Should().Be(1); - Services.GetRequiredService>().Count.Should().Be(1); + + var timedEntityQueue = Services.GetRequiredService>(); + timedEntityQueue.Should().NotBeNull(); + timedEntityQueue.Should().BeOfType>(); + timedEntityQueue.As>().Count.Should().Be(1); } public override async Task InitializeAsync() diff --git a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs index d4471f2f..c6021993 100644 --- a/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs +++ b/test/KubeOps.Operator.Test/Controller/DeletedEntityRequeue.Integration.Test.cs @@ -32,7 +32,10 @@ public async Task Should_Cancel_Requeue_If_Entity_Is_Deleted() await _mock.WaitForInvocations; _mock.Invocations.Count.Should().Be(2); - Services.GetRequiredService>().Count.Should().Be(0); + var timedEntityQueue = Services.GetRequiredService>(); + timedEntityQueue.Should().NotBeNull(); + timedEntityQueue.Should().BeOfType>(); + timedEntityQueue.As>().Count.Should().Be(0); } public override async Task InitializeAsync() From 1289fa7e60ebd66fa0d4231a3c3af34aa2fcbd72 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Tue, 4 Nov 2025 13:39:38 +0100 Subject: [PATCH 58/58] reconcile: move `Remove` calls to specific reconciliation methods --- .../Reconciliation/Reconciler.cs | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/KubeOps.Operator/Reconciliation/Reconciler.cs b/src/KubeOps.Operator/Reconciliation/Reconciler.cs index e007d58b..dfcfd08a 100644 --- a/src/KubeOps.Operator/Reconciliation/Reconciler.cs +++ b/src/KubeOps.Operator/Reconciliation/Reconciler.cs @@ -48,11 +48,6 @@ internal sealed class Reconciler( public async Task> Reconcile(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { - await entityQueue - .Remove( - reconciliationContext.Entity, - cancellationToken); - var result = reconciliationContext.EventType switch { WatchEventType.Added or WatchEventType.Modified => @@ -114,6 +109,11 @@ await _entityCache.SetAsync( private async Task> ReconcileDeletion(ReconciliationContext reconciliationContext, CancellationToken cancellationToken) { + await entityQueue + .Remove( + reconciliationContext.Entity, + cancellationToken); + await using var scope = serviceProvider.CreateAsyncScope(); var controller = scope.ServiceProvider.GetRequiredService>(); var result = await controller.DeletedAsync(reconciliationContext.Entity, cancellationToken); @@ -128,6 +128,11 @@ private async Task> ReconcileDeletion(Reconciliati private async Task> ReconcileEntity(TEntity entity, CancellationToken cancellationToken) { + await entityQueue + .Remove( + entity, + cancellationToken); + await using var scope = serviceProvider.CreateAsyncScope(); if (operatorSettings.AutoAttachFinalizers) @@ -148,6 +153,11 @@ private async Task> ReconcileEntity(TEntity entity private async Task> ReconcileFinalizersSequential(TEntity entity, CancellationToken cancellationToken) { + await entityQueue + .Remove( + entity, + cancellationToken); + await using var scope = serviceProvider.CreateAsyncScope(); // the condition to call ReconcileFinalizersSequentialAsync is: