Mine all transactions in one batch

Provides a meaningful speedup during tests, and especially in the
benchmarks. Requires a rework of the VM.mine_block() API to stop relying
on the side-effect of setting vm._block.

Author: carver

eth/abc.py

@@ -480,6 +480,16 @@ class BlockAndMetaWitness(NamedTuple):
     meta_witness: MetaWitnessAPI
 
 
+class BlockPersistResult(NamedTuple):
+    """
+    After persisting a block into the active chain, this information
+    becomes available.
+    """
+    imported_block: BlockAPI
+    new_canonical_blocks: Tuple[BlockAPI, ...]
+    old_canonical_blocks: Tuple[BlockAPI, ...]
+
+
 class BlockImportResult(NamedTuple):
     """
     After importing and persisting a block into the active chain, this information
@@ -2785,9 +2795,9 @@ def import_block(self, block: BlockAPI) -> BlockAndMetaWitness:
         ...
 
     @abstractmethod
-    def mine_block(self, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
+    def mine_block(self, block: BlockAPI, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
         """
-        Mine the current block. Proxies to self.pack_block method.
+        Mine the given block. Proxies to self.pack_block method.
         """
         ...
 
@@ -3586,6 +3596,24 @@ def __init__(self, base_db: AtomicDatabaseAPI, header: BlockHeaderAPI = None) ->
         """
         ...
 
+    @abstractmethod
+    def mine_all(
+            self,
+            transactions: Sequence[SignedTransactionAPI],
+            *args: Any,
+            parent_header: BlockHeaderAPI = None,
+            **kwargs: Any,
+    ) -> Tuple[BlockImportResult, Tuple[ReceiptAPI, ...], Tuple[ComputationAPI, ...]]:
+        """
+        Build a block with the given transactions, and mine it.
+
+        Optionally, supply the parent block header to mine on top of.
+
+        This is much faster than individually running :meth:`apply_transaction`
+        and then :meth:`mine_block`.
+        """
+        ...
+
     @abstractmethod
     def apply_transaction(self,
                           transaction: SignedTransactionAPI

eth/chains/base.py

@@ -44,6 +44,7 @@
     AtomicDatabaseAPI,
     BlockHeaderAPI,
     BlockImportResult,
+    BlockPersistResult,
     ChainAPI,
     ChainDatabaseAPI,
     ConsensusContextAPI,
@@ -481,17 +482,27 @@ def import_block(self,
             except ValidationError:
                 self.logger.warning("Proposed %s doesn't follow EVM rules, rejecting...", block)
                 raise
-            self.validate_block(imported_block)
+
+        persist_result = self.persist_block(imported_block, perform_validation)
+        return BlockImportResult(*persist_result, block_result.meta_witness)
+
+    def persist_block(
+            self,
+            block: BlockAPI,
+            perform_validation: bool = True) -> BlockPersistResult:
+
+        if perform_validation:
+            self.validate_block(block)
 
         (
             new_canonical_hashes,
             old_canonical_hashes,
-        ) = self.chaindb.persist_block(imported_block)
+        ) = self.chaindb.persist_block(block)
 
         self.logger.debug(
-            'IMPORTED_BLOCK: number %s | hash %s',
-            imported_block.number,
-            encode_hex(imported_block.hash),
+            'Persisted block: number %s | hash %s',
+            block.number,
+            encode_hex(block.hash),
         )
 
         new_canonical_blocks = tuple(
@@ -505,11 +516,10 @@ def import_block(self,
             in old_canonical_hashes
         )
 
-        return BlockImportResult(
-            imported_block=imported_block,
+        return BlockPersistResult(
+            imported_block=block,
             new_canonical_blocks=new_canonical_blocks,
             old_canonical_blocks=old_canonical_blocks,
-            meta_witness=block_result.meta_witness,
         )
 
     #
@@ -667,11 +677,43 @@ def import_block(self,
         self.header = self.ensure_header()
         return result
 
+    def mine_all(
+            self,
+            transactions: Sequence[SignedTransactionAPI],
+            *args: Any,
+            parent_header: BlockHeaderAPI = None,
+            **kwargs: Any,
+    ) -> Tuple[BlockImportResult, Tuple[ReceiptAPI, ...], Tuple[ComputationAPI, ...]]:
+
+        if parent_header is None:
+            base_header = self.header
+        else:
+            base_header = self.create_header_from_parent(parent_header)
+
+        vm = self.get_vm(base_header)
+
+        new_header, receipts, computations = vm.apply_all_transactions(transactions, base_header)
+        filled_block = vm.set_block_transactions(vm.get_block(), new_header, transactions, receipts)
+
+        block_result = vm.mine_block(filled_block, *args, **kwargs)
+        imported_block = block_result.block
+
+        block_persist_result = self.persist_block(imported_block)
+        block_import_result = BlockImportResult(*block_persist_result, block_result.meta_witness)
+
+        self.header = self.create_header_from_parent(imported_block.header)
+        return (block_import_result, receipts, computations)
+
     def mine_block(self, *args: Any, **kwargs: Any) -> BlockAPI:
+        """
+        Mine whatever transactions have been incrementally applied so far.
+        """
         return self.mine_block_extended(*args, **kwargs).block
 
     def mine_block_extended(self, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
-        mine_result = self.get_vm(self.header).mine_block(*args, **kwargs)
+        vm = self.get_vm(self.header)
+        current_block = vm.get_block()
+        mine_result = vm.mine_block(current_block, *args, **kwargs)
         mined_block = mine_result.block
 
         self.validate_block(mined_block)

eth/tools/builder/chain/builders.py

@@ -366,10 +366,8 @@ def mine_block(chain: MiningChainAPI, **kwargs: Any) -> MiningChainAPI:
         raise ValidationError('`mine_block` may only be used on MiningChain instances')
 
     transactions = kwargs.pop('transactions', ())
-    for tx in transactions:
-        chain.apply_transaction(tx)
 
-    chain.mine_block(**kwargs)
+    chain.mine_all(transactions, **kwargs)
     return chain
 
 

eth/tools/factories/transaction.py

@@ -15,13 +15,16 @@ def new_transaction(
         gas_price=10,
         gas=100000,
         data=b'',
+        nonce=None,
         chain_id=None):
     """
     Create and return a transaction sending amount from <from_> to <to>.
 
     The transaction will be signed with the given private key.
     """
-    nonce = vm.state.get_nonce(from_)
+    if nonce is None:
+        nonce = vm.state.get_nonce(from_)
+
     tx = vm.create_unsigned_transaction(
         nonce=nonce,
         gas_price=gas_price,

eth/vm/base.py

@@ -305,17 +305,17 @@ def import_block(self, block: BlockAPI) -> BlockAndMetaWitness:
         # run all of the transactions.
         new_header, receipts, _ = self.apply_all_transactions(block.transactions, header)
 
-        self._block = self.set_block_transactions(
+        block_with_transactions = self.set_block_transactions(
             self.get_block(),
             new_header,
             block.transactions,
             receipts,
         )
 
-        return self.mine_block()
+        return self.mine_block(block_with_transactions)
 
-    def mine_block(self, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
-        packed_block = self.pack_block(self.get_block(), *args, **kwargs)
+    def mine_block(self, block: BlockAPI, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
+        packed_block = self.pack_block(block, *args, **kwargs)
 
         block_result = self.finalize_block(packed_block)
 

tests/core/chain-object/test_chain.py

@@ -4,6 +4,7 @@
 from eth_utils import decode_hex
 
 from eth import constants
+from eth.abc import MiningChainAPI
 from eth.chains.mainnet import MAINNET_GENESIS_HEADER
 from eth.chains.ropsten import ROPSTEN_GENESIS_HEADER
 from eth.exceptions import (
@@ -38,6 +39,15 @@ def tx(chain, funded_address, funded_address_private_key):
     return new_transaction(vm, from_, recipient, amount, funded_address_private_key)
 
 
+@pytest.fixture()
+def tx2(chain, funded_address, funded_address_private_key):
+    recipient = b'\x88' * 20
+    amount = 100
+    vm = chain.get_vm()
+    from_ = funded_address
+    return new_transaction(vm, from_, recipient, amount, funded_address_private_key, nonce=1)
+
+
 @pytest.mark.xfail(reason="modification to initial allocation made the block fixture invalid")
 def test_import_block_validation(valid_chain, funded_address, funded_address_initial_balance):
     block = rlp.decode(valid_block_rlp, sedes=FrontierBlock)
@@ -63,6 +73,7 @@ def test_import_block(chain, tx):
     else:
         # working on a non-mining chain, so we have to build the block to apply manually
         new_block, receipts, computations = chain.build_block_with_transactions([tx])
+        assert len(computations) == 1
         computations[0].raise_if_error()
 
     block_import_result = chain.import_block(new_block)
@@ -74,6 +85,48 @@ def test_import_block(chain, tx):
     assert chain.get_canonical_transaction(tx.hash) == tx
 
 
+def test_mine_all(chain, tx, tx2, funded_address):
+    if hasattr(chain, 'mine_all'):
+        start_balance = chain.get_vm().state.get_balance(funded_address)
+
+        mine_result = chain.mine_all([tx, tx2])
+        block = mine_result[0].imported_block
+
+        assert block.transactions == (tx, tx2)
+        assert chain.get_block_by_hash(block.hash) == block
+        assert chain.get_canonical_block_by_number(block.number) == block
+        assert chain.get_canonical_transaction(tx.hash) == tx
+
+        end_balance = chain.get_vm().state.get_balance(funded_address)
+        expected_spend = 2 * (100 + 21000 * 10)  # sent + gas * gasPrice
+
+        assert start_balance - end_balance == expected_spend
+    elif isinstance(chain, MiningChainAPI):
+        raise AssertionError()  # Mining chains should have the 'mine_all' method
+
+
+def test_mine_all_uncle(chain, tx, tx2, funded_address):
+    if hasattr(chain, 'mine_all'):
+        starting_tip = chain.get_canonical_head()
+        canonical = chain.mine_all([tx])
+        uncled = chain.mine_all([], parent_header=starting_tip)
+        uncled_header = uncled[0].imported_block.header
+
+        new_tip = chain.mine_all(
+            [tx2],
+            parent_header=canonical[0].imported_block.header,
+            uncles=[uncled_header],
+        )
+
+        block = new_tip[0].imported_block
+
+        assert block.transactions == (tx2,)
+        assert chain.get_block_by_hash(block.hash) == block
+        assert block.uncles == (uncled_header,)
+    elif isinstance(chain, MiningChainAPI):
+        raise AssertionError()  # Mining chains should have the 'mine_all' method
+
+
 def test_empty_transaction_lookups(chain):
     with pytest.raises(TransactionNotFound):
         chain.get_canonical_transaction(b'\0' * 32)

tests/core/vm/test_rewards.py

@@ -229,11 +229,15 @@ def test_uncle_block_inclusion_validity(vm_fn):
     uncle = fork_chain.get_block_header_by_hash(fork_chain.header.parent_hash)
     assert uncle.block_number == 2
 
+    chain = build(
+        chain,
+        # Mines blocks from 3 to 8 (both numbers inclusive)
+        mine_blocks(6),
+    )
+
     with pytest.raises(ValidationError):
         chain = build(
             chain,
-            # Mines blocks from 3 to 8 (both numbers inclusive)
-            mine_blocks(6),
             # Mine block 9 with uncle
             mine_block(uncles=[uncle]),
         )

tests/core/vm/test_vm_state.py

@@ -21,7 +21,7 @@ def state(chain_without_block_validation):
 def test_block_properties(chain_without_block_validation):
     chain = chain_without_block_validation
     vm = chain.get_vm()
-    mined_block = vm.mine_block().block
+    mined_block = vm.mine_block(vm.get_block()).block
     block_import_result = chain.import_block(mined_block)
     block = block_import_result.imported_block