Merge pull request #1670 from Bhargavasomu/lazy_evaluation_prev_hashes

Lazily evaluate vm.previous_hashes

Author: carver

eth/_utils/generator.py

@@ -0,0 +1,25 @@
+import itertools
+from typing import (  # noqa: F401
+    Generic,
+    Iterable,
+    Iterator,
+    List,
+    TypeVar,
+)
+
+
+TItem = TypeVar('TItem')
+
+
+class CachedIterable(Generic[TItem], Iterable[TItem]):
+    def __init__(self, iterable: Iterable[TItem]) -> None:
+        self._cached_results = []  # type: List[TItem]
+        self._iterator = iter(iterable)
+
+    def __iter__(self) -> Iterator[TItem]:
+        return itertools.chain(self._cached_results, self._cache_and_yield())
+
+    def _cache_and_yield(self) -> Iterator[TItem]:
+        for item in self._iterator:
+            self._cached_results.append(item)
+            yield item

eth/rlp/headers.py

@@ -1,7 +1,7 @@
 import time
 from typing import (
+    Iterable,
     Optional,
-    Tuple,
     Union,
     overload,
 )
@@ -203,7 +203,7 @@ def from_parent(cls,
         return header
 
     def create_execution_context(
-            self, prev_hashes: Tuple[Hash32, ...]) -> ExecutionContext:
+            self, prev_hashes: Iterable[Hash32]) -> ExecutionContext:
 
         return ExecutionContext(
             coinbase=self.coinbase,

eth/vm/base.py

@@ -4,7 +4,7 @@
     abstractmethod,
 )
 import contextlib
-import functools
+import itertools
 import logging
 from typing import (
     Any,
@@ -28,7 +28,6 @@
 )
 
 from eth_utils import (
-    to_tuple,
     ValidationError,
 )
 
@@ -258,7 +257,6 @@ def get_nephew_reward(cls) -> int:
 
     @classmethod
     @abstractmethod
-    @to_tuple
     def get_prev_hashes(cls,
                         last_block_hash: Hash32,
                         chaindb: BaseChainDB) -> Optional[Iterable[Hash32]]:
@@ -691,8 +689,6 @@ def get_block_class(cls) -> Type[BaseBlock]:
             return cls.block_class
 
     @classmethod
-    @functools.lru_cache(maxsize=32)
-    @to_tuple
     def get_prev_hashes(cls,
                         last_block_hash: Hash32,
                         chaindb: BaseChainDB) -> Optional[Iterable[Hash32]]:
@@ -709,7 +705,7 @@ def get_prev_hashes(cls,
                 break
 
     @property
-    def previous_hashes(self) -> Optional[Tuple[Hash32, ...]]:
+    def previous_hashes(self) -> Optional[Iterable[Hash32]]:
         """
         Convenience API for accessing the previous 255 block hashes.
         """
@@ -919,7 +915,7 @@ def get_state_class(cls) -> Type[BaseState]:
     def state_in_temp_block(self) -> Iterator[BaseState]:
         header = self.block.header
         temp_block = self.generate_block_from_parent_header_and_coinbase(header, header.coinbase)
-        prev_hashes = (header.hash, ) + self.previous_hashes
+        prev_hashes = itertools.chain((header.hash, ), self.previous_hashes)
 
         state = self.get_state_class()(
             db=self.chaindb.db,

eth/vm/execution_context.py

@@ -1,10 +1,14 @@
-from typing import Tuple
+from typing import (
+    Iterable,
+)
 
 from eth_typing import (
     Address,
     Hash32,
 )
 
+from eth._utils.generator import CachedIterable
+
 
 class ExecutionContext:
     _coinbase = None
@@ -22,13 +26,13 @@ def __init__(
             block_number: int,
             difficulty: int,
             gas_limit: int,
-            prev_hashes: Tuple[Hash32, ...]) -> None:
+            prev_hashes: Iterable[Hash32]) -> None:
         self._coinbase = coinbase
         self._timestamp = timestamp
         self._block_number = block_number
         self._difficulty = difficulty
         self._gas_limit = gas_limit
-        self._prev_hashes = prev_hashes
+        self._prev_hashes = CachedIterable(prev_hashes)
 
     @property
     def coinbase(self) -> Address:
@@ -51,5 +55,5 @@ def gas_limit(self) -> int:
         return self._gas_limit
 
     @property
-    def prev_hashes(self) -> Tuple[Hash32, ...]:
+    def prev_hashes(self) -> Iterable[Hash32]:
         return self._prev_hashes

eth/vm/state.py

@@ -19,6 +19,7 @@
     Address,
     Hash32,
 )
+from eth_utils.toolz import nth
 
 from eth.constants import (
     BLANK_ROOT_HASH,
@@ -201,12 +202,16 @@ def get_ancestor_hash(self, block_number: int) -> Hash32:
         is_ancestor_depth_out_of_range = (
             ancestor_depth >= MAX_PREV_HEADER_DEPTH or
             ancestor_depth < 0 or
-            ancestor_depth >= len(self.execution_context.prev_hashes)
+            block_number < 0
         )
         if is_ancestor_depth_out_of_range:
             return Hash32(b'')
-        ancestor_hash = self.execution_context.prev_hashes[ancestor_depth]
-        return ancestor_hash
+
+        try:
+            return nth(ancestor_depth, self.execution_context.prev_hashes)
+        except StopIteration:
+            # Ancestor with specified depth not present
+            return Hash32(b'')
 
     #
     # Computation

tests/core/generator-utils/test_cached_iterable.py

@@ -0,0 +1,32 @@
+from eth._utils.generator import CachedIterable
+
+from eth_utils.toolz import (
+    first,
+    nth,
+)
+import itertools
+
+
+def test_cached_generator():
+    use_once = itertools.count()
+    repeated_use = CachedIterable(use_once)
+
+    for find_val in [1, 0, 10, 5]:
+        assert find_val == nth(find_val, repeated_use)
+
+
+def test_laziness():
+    def crash_after_first_val():
+        yield 1
+        raise Exception("oops, iterated past first value")
+
+    repeated_use = CachedIterable(crash_after_first_val())
+    assert first(repeated_use) == 1
+    assert first(repeated_use) == 1
+
+
+def test_cached_generator_iterable():
+    input_vals = [2]
+    repeated_use = CachedIterable(input_vals)
+    assert list(repeated_use) == input_vals
+    assert list(repeated_use) == input_vals