Failing blockchain tests now emit a state diff

- Instead of failing with "these block hashes do not match", blockchain
  tests now fail with "this part of the state tree does not match".

- The tests were inefficient, they first checked that the state hash
  matched and then checked that the states matched. It's unlikely the
  states will ever not match!

- Some validations needed to be deferred until after state-tree
  comparison, or else they would fail the test early with an unhelpful
  message.

Author: lithp

eth/tools/fixtures/helpers.py

@@ -193,8 +193,10 @@ def new_chain_from_fixture(fixture: Dict[str, Any],
     )
 
 
-def apply_fixture_block_to_chain(block_fixture: Dict[str, Any],
-                                 chain: BaseChain) -> Tuple[BaseBlock, BaseBlock, BaseBlock]:
+def apply_fixture_block_to_chain(
+        block_fixture: Dict[str, Any],
+        chain: BaseChain,
+        perform_validation: bool=True) -> Tuple[BaseBlock, BaseBlock, BaseBlock]:
     '''
     :return: (premined_block, mined_block, rlp_encoded_mined_block)
     '''
@@ -209,7 +211,7 @@ def apply_fixture_block_to_chain(block_fixture: Dict[str, Any],
 
     block = rlp.decode(block_fixture['rlp'], sedes=block_class)
 
-    mined_block, _, _ = chain.import_block(block)
+    mined_block, _, _ = chain.import_block(block, perform_validation=perform_validation)
 
     rlp_encoded_mined_block = rlp.encode(mined_block, sedes=block_class)
 

tests/json-fixtures/test_blockchain.py

@@ -111,8 +111,10 @@ def test_blockchain_fixtures(fixture_data, fixture):
     # 2 - loop over blocks:
     #     - apply transactions
     #     - mine block
-    # 4 - profit!!
+    # 3 - diff resulting state with expected state
+    # 4 - check that all previous blocks were valid
 
+    mined_blocks = list()
     for block_fixture in fixture['blocks']:
         should_be_good_block = 'blockHeader' in block_fixture
 
@@ -121,8 +123,12 @@ def test_blockchain_fixtures(fixture_data, fixture):
             continue
 
         if should_be_good_block:
-            (block, mined_block, block_rlp) = apply_fixture_block_to_chain(block_fixture, chain)
-            assert_mined_block_unchanged(block, mined_block)
+            (block, mined_block, block_rlp) = apply_fixture_block_to_chain(
+                block_fixture,
+                chain,
+                perform_validation=False  # we manually validate below
+            )
+            mined_blocks.append((block, mined_block))
         else:
             try:
                 apply_fixture_block_to_chain(block_fixture, chain)
@@ -133,6 +139,10 @@ def test_blockchain_fixtures(fixture_data, fixture):
                 raise AssertionError("Block should have caused a validation error")
 
     latest_block_hash = chain.get_canonical_block_by_number(chain.get_block().number - 1).hash
-    assert latest_block_hash == fixture['lastblockhash']
+    if latest_block_hash != fixture['lastblockhash']:
+        verify_account_db(fixture['postState'], chain.get_vm().state.account_db)
+        assert False, 'the state must be different if the hashes are'
 
-    verify_account_db(fixture['postState'], chain.get_vm().state.account_db)
+    for block, mined_block in mined_blocks:
+        assert_mined_block_unchanged(block, mined_block)
+        chain.validate_block(block)