Reduce memory usage of tests

lithp · lithp · commit 490624ec85ac · 2018-12-14T12:21:26.000-08:00
- eth/vm/memory.py:Memory.read() used to return a bytes() initialized
  with the requested memory, this created an often unnecessary copy of
  the data. It now returns a memoryview(), which is a python wrapper
  around a pointer to the raw data.
diff --git a/eth/precompiles/ecadd.py b/eth/precompiles/ecadd.py
@@ -32,7 +32,7 @@ def ecadd(computation: BaseComputation) -> BaseComputation:
     computation.consume_gas(constants.GAS_ECADD, reason='ECADD Precompile')
 
     try:
-        result = _ecadd(computation.msg.data)
+        result = _ecadd(bytes(computation.msg.data))
     except ValidationError:
         raise VMError("Invalid ECADD parameters")
 
diff --git a/eth/precompiles/ecmul.py b/eth/precompiles/ecmul.py
@@ -32,7 +32,7 @@ def ecmul(computation: BaseComputation) -> BaseComputation:
     computation.consume_gas(constants.GAS_ECMUL, reason='ECMUL Precompile')
 
     try:
-        result = _ecmull(computation.msg.data)
+        result = _ecmull(bytes(computation.msg.data))
     except ValidationError:
         raise VMError("Invalid ECMUL parameters")
 
diff --git a/eth/precompiles/ecpairing.py b/eth/precompiles/ecpairing.py
@@ -1,4 +1,4 @@
-from typing import Tuple
+from typing import Tuple, Union
 
 from cytoolz import (
     curry,
@@ -60,7 +60,7 @@ def ecpairing(computation: BaseComputation) -> BaseComputation:
     return computation
 
 
-def _ecpairing(data: bytes) -> bool:
+def _ecpairing(data: Union[bytes, memoryview]) -> bool:
     exponent = bn128.FQ12.one()
 
     processing_pipeline = (
diff --git a/eth/precompiles/ecrecover.py b/eth/precompiles/ecrecover.py
@@ -27,16 +27,17 @@
 
 def ecrecover(computation: BaseComputation) -> BaseComputation:
     computation.consume_gas(constants.GAS_ECRECOVER, reason="ECRecover Precompile")
-    raw_message_hash = computation.msg.data[:32]
+    data = bytes(computation.msg.data)
+    raw_message_hash = data[:32]
     message_hash = pad32r(raw_message_hash)
 
-    v_bytes = pad32r(computation.msg.data[32:64])
+    v_bytes = pad32r(data[32:64])
     v = big_endian_to_int(v_bytes)
 
-    r_bytes = pad32r(computation.msg.data[64:96])
+    r_bytes = pad32r(data[64:96])
     r = big_endian_to_int(r_bytes)
 
-    s_bytes = pad32r(computation.msg.data[96:128])
+    s_bytes = pad32r(data[96:128])
     s = big_endian_to_int(s_bytes)
 
     try:
diff --git a/eth/precompiles/identity.py b/eth/precompiles/identity.py
@@ -14,5 +14,5 @@ def identity(computation: BaseComputation) -> BaseComputation:
 
     computation.consume_gas(gas_fee, reason="Identity Precompile")
 
-    computation.output = computation.msg.data
+    computation.output = bytes(computation.msg.data)
     return computation
diff --git a/eth/precompiles/modexp.py b/eth/precompiles/modexp.py
@@ -1,5 +1,5 @@
 from typing import (
-    Tuple,
+    Tuple
 )
 
 from eth_utils import (
@@ -125,12 +125,14 @@ def modexp(computation: BaseComputation) -> BaseComputation:
     """
     https://github.com/ethereum/EIPs/pull/198
     """
-    gas_fee = _compute_modexp_gas_fee(computation.msg.data)
+    data = bytes(computation.msg.data)
+
+    gas_fee = _compute_modexp_gas_fee(data)
     computation.consume_gas(gas_fee, reason='MODEXP Precompile')
 
-    result = _modexp(computation.msg.data)
+    result = _modexp(data)
 
-    _, _, modulus_length = _extract_lengths(computation.msg.data)
+    _, _, modulus_length = _extract_lengths(data)
 
     # Modulo 0 is undefined, return zero
     # https://math.stackexchange.com/questions/516251/why-is-n-mod-0-undefined
diff --git a/eth/vm/computation.py b/eth/vm/computation.py
@@ -243,7 +243,7 @@ def memory_write(self, start_position: int, size: int, value: bytes) -> None:
         """
         return self._memory.write(start_position, size, value)
 
-    def memory_read(self, start_position: int, size: int) -> bytes:
+    def memory_read(self, start_position: int, size: int) -> memoryview:
         """
         Read and return ``size`` bytes from memory starting at ``start_position``.
         """
@@ -360,7 +360,7 @@ def prepare_child_message(self,
                               gas: int,
                               to: Address,
                               value: int,
-                              data: bytes,
+                              data: Union[bytes, memoryview],
                               code: bytes,
                               **kwargs: Any) -> Message:
         """
@@ -442,7 +442,12 @@ def get_accounts_for_deletion(self) -> Tuple[Tuple[bytes, bytes], ...]:
     #
     # EVM logging
     #
-    def add_log_entry(self, account: Address, topics: List[int], data: bytes) -> None:
+    def add_log_entry(
+            self,
+            account: Address,
+            topics: List[int],
+            data: Union[bytes, memoryview]) -> None:
+        data = bytes(data)
         validate_canonical_address(account, title="Log entry address")
         for topic in topics:
             validate_uint256(topic, title="Log entry topic")
diff --git a/eth/vm/logic/context.py b/eth/vm/logic/context.py
@@ -42,7 +42,7 @@ def calldataload(computation: BaseComputation) -> None:
     """
     start_position = computation.stack_pop(type_hint=constants.UINT256)
 
-    value = computation.msg.data[start_position:start_position + 32]
+    value = bytes(computation.msg.data[start_position:start_position + 32])
     padded_value = value.ljust(32, b'\x00')
     normalized_value = padded_value.lstrip(b'\x00')
 
@@ -68,7 +68,9 @@ def calldatacopy(computation: BaseComputation) -> None:
 
     computation.consume_gas(copy_gas_cost, reason="CALLDATACOPY fee")
 
-    value = computation.msg.data[calldata_start_position: calldata_start_position + size]
+    value = bytes(
+        computation.msg.data[calldata_start_position: calldata_start_position + size]
+    )
     padded_value = value.ljust(size, b'\x00')
 
     computation.memory_write(mem_start_position, size, padded_value)
diff --git a/eth/vm/logic/memory.py b/eth/vm/logic/memory.py
@@ -33,7 +33,7 @@ def mload(computation: BaseComputation) -> None:
     computation.extend_memory(start_position, 32)
 
     value = computation.memory_read(start_position, 32)
-    computation.stack_push(value)
+    computation.stack_push(bytes(value))
 
 
 def msize(computation: BaseComputation) -> None:
diff --git a/eth/vm/logic/sha3.py b/eth/vm/logic/sha3.py
@@ -18,6 +18,6 @@ def sha3(computation: BaseComputation) -> None:
     gas_cost = constants.GAS_SHA3WORD * word_count
     computation.consume_gas(gas_cost, reason="SHA3: word gas cost")
 
-    result = keccak(sha3_bytes)
+    result = keccak(bytes(sha3_bytes))
 
     computation.stack_push(result)
diff --git a/eth/vm/logic/system.py b/eth/vm/logic/system.py
@@ -163,7 +163,9 @@ def __call__(self, computation: BaseComputation) -> None:
             computation.stack_push(0)
             return
 
-        call_data = computation.memory_read(stack_data.memory_start, stack_data.memory_length)
+        call_data = bytes(
+            computation.memory_read(stack_data.memory_start, stack_data.memory_length)
+        )
 
         create_msg_gas = self.max_child_gas_modifier(
             computation.get_gas_remaining()
diff --git a/eth/vm/memory.py b/eth/vm/memory.py
@@ -1,4 +1,5 @@
 import itertools
+from typing import Union
 import logging
 
 from eth.validation import (
@@ -32,7 +33,16 @@ def extend(self, start_position: int, size: int) -> None:
             return
 
         size_to_extend = new_size - len(self)
-        self._bytes.extend(itertools.repeat(0, size_to_extend))
+        try:
+            self._bytes.extend(itertools.repeat(0, size_to_extend))
+        except BufferError:
+            # we can't extend the buffer (which might involve relocating it) if a
+            # memoryview (which stores a pointer into the buffer) has been created by
+            # read() and not released. Callers of read() will never try to write to the
+            # buffer so we're not missing anything by making a new buffer and forgetting
+            # about the old one. We're keeping too much memory around but this is still a
+            # net savings over having read() return a new bytes() object every time.
+            self._bytes = self._bytes + bytearray(size_to_extend)
 
     def __len__(self) -> int:
         return len(self._bytes)
@@ -51,8 +61,8 @@ def write(self, start_position: int, size: int, value: bytes) -> None:
             for idx, v in enumerate(value):
                 self._bytes[start_position + idx] = v
 
-    def read(self, start_position: int, size: int) -> bytes:
+    def read(self, start_position: int, size: int) -> Union[bytes, memoryview]:
         """
         Read a value from memory.
         """
-        return bytes(self._bytes[start_position:start_position + size])
+        return memoryview(self._bytes)[start_position:start_position + size]
diff --git a/eth/vm/message.py b/eth/vm/message.py
@@ -1,4 +1,5 @@
 import logging
+from typing import Union
 
 from eth_typing import Address
 
@@ -31,7 +32,7 @@ def __init__(self,
                  to: Address,
                  sender: Address,
                  value: int,
-                 data: bytes,
+                 data: Union[bytes, memoryview],
                  code: bytes,
                  depth: int=0,
                  create_address: Address=None,
@@ -51,7 +52,7 @@ def __init__(self,
         validate_uint256(value, title="Message.value")
         self.value = value
 
-        validate_is_bytes(data, title="Message.data")
+#        validate_is_bytes(data, title="Message.data")
         self.data = data
 
         validate_is_integer(depth, title="Message.depth")
