implement remove (#17)

Author: bitfaster

Lightweight.Caching.UnitTests/ClassicLruTests.cs

@@ -1,4 +1,5 @@
 using FluentAssertions;
+using Lightweight.Caching.UnitTests.Lru;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -62,6 +63,16 @@ public void WhenKeyIsRequestedItIsCreatedAndCached()
 			result1.Should().Be(result2);
 		}
 
+		[Fact]
+		public async Task WhenKeyIsRequesteItIsCreatedAndCachedAsync()
+		{
+			var result1 = await lru.GetOrAddAsync(1, valueFactory.CreateAsync).ConfigureAwait(false);
+			var result2 = await lru.GetOrAddAsync(1, valueFactory.CreateAsync).ConfigureAwait(false);
+
+			valueFactory.timesCalled.Should().Be(1);
+			result1.Should().Be(result2);
+		}
+
 		[Fact]
 		public void WhenDifferentKeysAreRequestedValueIsCreatedForEach()
 		{
@@ -74,6 +85,18 @@ public void WhenDifferentKeysAreRequestedValueIsCreatedForEach()
 			result2.Should().Be("2");
 		}
 
+		[Fact]
+		public async Task WhenDifferentKeysAreRequesteValueIsCreatedForEachAsync()
+		{
+			var result1 = await lru.GetOrAddAsync(1, valueFactory.CreateAsync).ConfigureAwait(false);
+			var result2 = await lru.GetOrAddAsync(2, valueFactory.CreateAsync).ConfigureAwait(false);
+
+			valueFactory.timesCalled.Should().Be(2);
+
+			result1.Should().Be("1");
+			result2.Should().Be("2");
+		}
+
 		[Fact]
 		public void WhenMoreKeysRequestedThanCapacityCountDoesNotIncrease()
 		{
@@ -86,6 +109,18 @@ public void WhenMoreKeysRequestedThanCapacityCountDoesNotIncrease()
 			valueFactory.timesCalled.Should().Be(capacity + 1);
 		}
 
+		[Fact]
+		public async Task WhenMoreKeysRequestedThanCapacityCountDoesNotIncreaseAsync()
+		{
+			for (int i = 0; i < capacity + 1; i++)
+			{
+				await lru.GetOrAddAsync(i, valueFactory.CreateAsync);
+			}
+
+			lru.Count.Should().Be(capacity);
+			valueFactory.timesCalled.Should().Be(capacity + 1);
+		}
+
 		[Fact]
 		public void WhenMoreKeysRequestedThanCapacityOldestItemIsEvicted()
 		{
@@ -132,15 +167,21 @@ public void WhenKeyExistsTryGetReturnsTrueAndOutValueIsCorrect()
 			value.Should().Be("1");
 		}
 
-		private class ValueFactory
+		[Fact]
+		public void WhenKeyExistsTryRemoveRemovesItemAndReturnsTrue()
+        {
+			lru.GetOrAdd(1, valueFactory.Create);
+
+			lru.TryRemove(1).Should().BeTrue();
+			lru.TryGet(1, out var value).Should().BeFalse();
+		}
+
+		[Fact]
+		public void WhenKeyDoesNotExistTryRemoveReturnsFalse()
 		{
-			public int timesCalled;
+			lru.GetOrAdd(1, valueFactory.Create);
 
-			public string Create(int key)
-			{
-				timesCalled++;
-				return key.ToString();
-			}
+			lru.TryRemove(2).Should().BeFalse();
 		}
 	}
 }

Lightweight.Caching.UnitTests/Lru/ConcurrentLruTests.cs

@@ -299,6 +299,35 @@ public void WhenValueExpiresItIsDisposed()
 			disposableValueFactory.Items[1].IsDisposed.Should().BeFalse();
 		}
 
+		[Fact]
+		public void WhenKeyExistsTryRemoveRemovesItemAndReturnsTrue()
+		{
+			lru.GetOrAdd(1, valueFactory.Create);
+
+			lru.TryRemove(1).Should().BeTrue();
+			lru.TryGet(1, out var value).Should().BeFalse();
+		}
+
+		[Fact]
+		public void WhenItemIsRemovedItIsDisposed()
+		{
+			var lruOfDisposable = new ConcurrentLru<int, DisposableItem>(1, 6, EqualityComparer<int>.Default);
+			var disposableValueFactory = new DisposableValueFactory();
+
+			lruOfDisposable.GetOrAdd(1, disposableValueFactory.Create);
+			lruOfDisposable.TryRemove(1);
+
+			disposableValueFactory.Items[1].IsDisposed.Should().BeTrue();
+		}
+
+		[Fact]
+		public void WhenKeyDoesNotExistTryRemoveReturnsFalse()
+		{
+			lru.GetOrAdd(1, valueFactory.Create);
+
+			lru.TryRemove(2).Should().BeFalse();
+		}
+
 		private class DisposableItem : IDisposable
 		{
 			public bool IsDisposed { get; private set; }

Lightweight.Caching.UnitTests/Lru/ConcurrentTLruTests.cs

@@ -30,5 +30,14 @@ public async Task WhenItemIsExpiredItIsRemoved()
 
             lru.TryGet(1, out var value).Should().BeFalse();
         }
+
+        [Fact]
+        public void WhenItemIsAddedThenRetrievedHitRatioIsHalf()
+        {
+            lru.GetOrAdd(1, valueFactory.Create);
+            bool result = lru.TryGet(1, out var value);
+
+            lru.HitRatio.Should().Be(0.5);
+        }
     }
 }

Lightweight.Caching.UnitTests/Lru/FastConcurrentLruTests.cs

@@ -0,0 +1,23 @@
+using FluentAssertions;
+using Lightweight.Caching.Lru;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Xunit;
+
+namespace Lightweight.Caching.UnitTests.Lru
+{
+    public class FastConcurrentLruTests
+    {
+        [Fact]
+        public void ConstructAddAndRetrieveWithCustomComparerReturnsValue()
+        {
+            var lru = new FastConcurrentLru<string, int>(9, 9, StringComparer.OrdinalIgnoreCase);
+
+            lru.GetOrAdd("foo", k => 1);
+
+            lru.TryGet("FOO", out var value).Should().BeTrue();
+            value.Should().Be(1);
+        }
+    }
+}

Lightweight.Caching.UnitTests/Lru/FastConcurrentTLruTests.cs

@@ -0,0 +1,23 @@
+using FluentAssertions;
+using Lightweight.Caching.Lru;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Xunit;
+
+namespace Lightweight.Caching.UnitTests.Lru
+{
+    public class FastConcurrentTLruTests
+    {
+        [Fact]
+        public void ConstructAddAndRetrieveWithCustomComparerReturnsValue()
+        {
+            var lru = new FastConcurrentTLru<string, int>(9, 9, StringComparer.OrdinalIgnoreCase, TimeSpan.FromSeconds(10));
+
+            lru.GetOrAdd("foo", k => 1);
+
+            lru.TryGet("FOO", out var value).Should().BeTrue();
+            value.Should().Be(1);
+        }
+    }
+}

Lightweight.Caching/ClassicLru.cs

@@ -93,6 +93,71 @@ public V GetOrAdd(K key, Func<K, V> valueFactory)
 			return this.GetOrAdd(key, valueFactory);
 		}
 
+		public async Task<V> GetOrAddAsync(K key, Func<K, Task<V>> valueFactory)
+		{
+			if (this.TryGet(key, out var value))
+			{
+				return value;
+			}
+
+			var node = new LinkedListNode<LruItem>(new LruItem(key, await valueFactory(key)));
+
+			if (this.dictionary.TryAdd(key, node))
+			{
+				LinkedListNode<LruItem> first = null;
+
+				lock (this.linkedList)
+				{
+					if (linkedList.Count >= capacity)
+					{
+						first = linkedList.First;
+						linkedList.RemoveFirst();
+					}
+
+					linkedList.AddLast(node);
+				}
+
+				// Remove from the dictionary outside the lock. This means that the dictionary at this moment
+				// contains an item that is not in the linked list. If another thread fetches this item, 
+				// LockAndMoveToEnd will ignore it, since it is detached. This means we potentially 'lose' an 
+				// item just as it was about to move to the back of the LRU list and be preserved. The next request
+				// for the same key will be a miss. Dictionary and list are eventually consistent.
+				// However, all operations inside the lock are extremely fast, so contention is minimized.
+				if (first != null)
+				{
+					dictionary.TryRemove(first.Value.Key, out var removed);
+				}
+
+				return node.Value.Value;
+			}
+
+			return await this.GetOrAddAsync(key, valueFactory);
+		}
+
+		public bool TryRemove(K key)
+        {
+			if (dictionary.TryRemove(key, out var node))
+			{
+				// If the node has already been removed from the list, ignore.
+				// E.g. thread A reads x from the dictionary. Thread B adds a new item, removes x from 
+				// the List & Dictionary. Now thread A will try to move x to the end of the list.
+				if (node.List != null)
+				{
+					lock (this.linkedList)
+					{
+						if (node.List != null)
+						{
+							linkedList.Remove(node);
+						}
+					}
+				}
+
+				return true;
+			}
+
+			return false;
+        }
+
 		// Thead A reads x from the dictionary. Thread B adds a new item. Thread A moves x to the end. Thread B now removes the new first Node (removal is atomic on both data structures).
 		private void LockAndMoveToEnd(LinkedListNode<LruItem> node)
 		{

Lightweight.Caching/ICache.cs

@@ -11,5 +11,9 @@ public interface ICache<K, V>
         bool TryGet(K key, out V value);
 
         V GetOrAdd(K key, Func<K, V> valueFactory);
+
+        Task<V> GetOrAddAsync(K key, Func<K, Task<V>> valueFactory);
+
+        bool TryRemove(K key);
     }
 }

Lightweight.Caching/Lru/TemplateConcurrentLru.cs

@@ -8,6 +8,22 @@
 
 namespace Lightweight.Caching.Lru
 {
+	/// <summary>
+	/// LRU implementation where LRU list is composed of 3 segments: hot, warm and cold. Cost of maintaining
+	/// segments is amortized across requests. Items are only cycled when capacity is exceeded. Pure read does
+	/// not cycle items if all segments are within capacity constraints.
+	/// There are no global locks. On cache miss, a new item is added. Tail items in each segment are dequeued,
+	/// examined, and are either enqueued or discarded.
+	/// </summary>
+	/// <remarks>
+	/// Each segment has a capacity. When segment capacity is exceeded, items are moved as follows:
+	/// 1. New items are added to hot, WasAccessed = false
+	/// 2. When items are accessed, update WasAccessed = true
+	/// 3. When items are moved WasAccessed is set to false.
+	/// 4. When hot is full, hot tail is moved to either Warm or Cold depending on WasAccessed. 
+	/// 5. When warm is full, warm tail is moved to warm head or cold depending on WasAccessed.
+	/// 6. When cold is full, cold tail is moved to warm head or removed from dictionary on depending on WasAccessed.
+	/// </remarks>
 	public class TemplateConcurrentLru<K, V, I, P, H> : ICache<K, V>
 		where I : LruItem<K, V>
 		where P : struct, IPolicy<K, V, I>
@@ -56,7 +72,7 @@ public TemplateConcurrentLru(
 			this.hitCounter = hitCounter;
 		}
 
-		public int Count => this.hotCount + this.warmCount + this.coldCount;
+		public int Count => this.dictionary.Count;
 
 		public int HotCount => this.hotCount;
 
@@ -103,7 +119,7 @@ public V GetOrAdd(K key, Func<K, V> valueFactory)
 			{
 				this.hotQueue.Enqueue(newItem);
 				Interlocked.Increment(ref hotCount);
-				BumpItems();
+				Cycle();
 				return newItem.Value;
 			}
 
@@ -125,30 +141,51 @@ public async Task<V> GetOrAddAsync(K key, Func<K, Task<V>> valueFactory)
 			{
 				this.hotQueue.Enqueue(newItem);
 				Interlocked.Increment(ref hotCount);
-				BumpItems();
+				Cycle();
 				return newItem.Value;
 			}
 
 			return await this.GetOrAddAsync(key, valueFactory).ConfigureAwait(false);
 		}
 
-		private void BumpItems()
+		public bool TryRemove(K key)
+		{
+			if (this.dictionary.TryRemove(key, out var removedItem))
+            {
+				// Mark as not accessed, it will later be cycled out of the queues because it can never be fetched 
+				// from the dictionary. Note: Hot/Warm/Cold count will reflect the removed item until it is cycled 
+				// from the queue.
+				removedItem.WasAccessed = false;
+
+				if (removedItem.Value is IDisposable d)
+				{
+					d.Dispose();
+				}
+
+				return true;
+            }
+
+			return false;
+		}
+
+		private void Cycle()
 		{
 			// There will be races when queue count == queue capacity. Two threads may each dequeue items.
-			// This will prematurely free slots for the next caller. Each thread will still only bump at most 5 items.
+			// This will prematurely free slots for the next caller. Each thread will still only cycle at most 5 items.
 			// Since TryDequeue is thread safe, only 1 thread can dequeue each item. Thus counts and queue state will always
 			// converge on correct over time.
-			BumpHot();
+			CycleHot();
+
 			// Multi-threaded stress tests show that due to races, the warm and cold count can increase beyond capacity when
-			// hit rate is very high. Double bump results in stable count under all conditions. When contention is low, 
-			// secondary bumps have no effect.
-			BumpWarm();
-			BumpWarm();
-			BumpCold();
-			BumpCold();
+			// hit rate is very high. Double cycle results in stable count under all conditions. When contention is low, 
+			// secondary cycles have no effect.
+			CycleWarm();
+			CycleWarm();
+			CycleCold();
+			CycleCold();
 		}
 
-		private void BumpHot()
+		private void CycleHot()
 		{
 			if (this.hotCount > this.hotCapacity)
 			{
@@ -166,7 +203,7 @@ private void BumpHot()
 			}
 		}
 
-		private void BumpWarm()
+		private void CycleWarm()
 		{
 			if (this.warmCount > this.warmCapacity)
 			{
@@ -195,7 +232,7 @@ private void BumpWarm()
 			}
 		}
 
-		private void BumpCold()
+		private void CycleCold()
 		{
 			if (this.coldCount > this.coldCapacity)
 			{