Add OrderedSet.dedup to deduplicate iterators

Since this is a generator, it yields items
without waiting for the whole input.

Also add OrderedSet.async_dedup,
which does the same thing to an AsyncIterator

Author: Techcable

pyproject.toml

@@ -24,7 +24,12 @@ pydantic = [
 
 [dependency-groups]
 mypy = ["mypy~=1.0", {include-group = "typing"}]
-test = ["pytest~=8.3", "pytest-sugar~=1.0", "techcable.orderedset[pydantic]"]
+test = [
+    "pytest~=8.3",
+    "pytest-asyncio>=0.26",
+    "pytest-sugar~=1.0",
+    "techcable.orderedset[pydantic]"
+]
 typing = ["typing-extensions~=4.12", "techcable.orderedset[pydantic]"]
 dev = [{include-group = "mypy"}, {include-group = "test"}]
 

src/techcable/orderedset/_orderedset.py

@@ -1,7 +1,17 @@
 from __future__ import annotations
 
 import operator
-from collections.abc import Callable, Iterable, Iterator, MutableSet, Sequence, Set
+from collections.abc import (
+    AsyncGenerator,
+    AsyncIterable,
+    Callable,
+    Generator,
+    Iterable,
+    Iterator,
+    MutableSet,
+    Sequence,
+    Set,
+)
 from typing import TYPE_CHECKING, Any, Optional, TypeVar, overload
 
 if TYPE_CHECKING:
@@ -249,5 +259,40 @@ def __get_pydantic_core_schema__(
             ),
         )
 
+    @classmethod
+    def dedup(self, source: Iterable[T], /) -> Generator[T]:
+        """A utility method to deduplicate the specified iterable,
+        while preserving the original order.
+
+        This is a generator, so does not need to wait for the entire input,
+        and will return items as soon as they are available.
+
+        Since: v0.1.4
+        """
+        oset: OrderedSet[T] = OrderedSet()
+        for item in source:
+            if oset.append(item):
+                # new value
+                yield item
+
+    @classmethod
+    async def dedup_async(self, source: AsyncIterable[T], /) -> AsyncGenerator[T]:
+        """A utility method to deduplicate the specified iterable,
+        while preserving the original order.
+
+        This is a generator, so does not need to wait for the entire input.
+        Because it is asynchronous, it does not block the thread while waiting.
+
+        This is an asynchronous version of `OrderedSet.dedup`.
+
+        Since: v0.1.4
+        """
+        # Defined in PEP 525
+        oset: OrderedSet[T] = OrderedSet()
+        async for item in source:
+            if oset.append(item):
+                # new value
+                yield item
+
 
 __all__ = ("OrderedSet",)

tests/test_simple.py

@@ -1,6 +1,11 @@
+from __future__ import annotations
+
 import random
+from collections.abc import AsyncGenerator
 from typing import Any, TypeVar
 
+import pytest
+
 from techcable.orderedset import OrderedSet
 
 T = TypeVar("T")
@@ -20,6 +25,7 @@ def _remove_duplicates(l: list[T]) -> list[T]:
     ["foo", "bar", "baz", "foo"],
     [1, 2, 7, 13, 9, 12, 2, 8, 7],
     [float("NaN"), 2.8, float("NaN"), 7.9],
+    [2.7, 3, 2.7, 9, 8.2, 3, 4.1],
 ]
 
 
@@ -43,3 +49,30 @@ def test_remove():
         assert orig_oset == OrderedSet(orig_data), "Copy didn't work"
         assert oset == (orig_oset - {target})
         assert oset == OrderedSet(data)
+
+
+def test_dedup():
+    for example in EXAMPLE_DATA:
+        assert list(OrderedSet.dedup(example)) == _remove_duplicates(example)
+
+
+@pytest.mark.asyncio
+async def test_async_dedup():
+    for example in EXAMPLE_DATA:
+        async_counter = 0
+
+        async def increment_counter() -> None:
+            nonlocal async_counter
+            async_counter += 1
+
+        async def source(items: list[T], /) -> AsyncGenerator[T]:
+            for item in items:
+                await increment_counter()
+                yield item
+
+        result = []
+        async for item in OrderedSet.dedup_async(source(example)):
+            result.append(item)
+
+        assert async_counter == len(example)
+        assert result == _remove_duplicates(example)