Ignoring unknown keys in RPC responses by default (#1312)

* Allow ignoring unknown keys in RPC responses by set env

* remove unknown=excluded from node_client

* set execution_resources as optional for L1_HANDLER_TXN_TRACE schema

Author: tkumor3

starknet_py/net/client_models.py

@@ -1,6 +1,10 @@
 """
 Dataclasses representing responses from Starknet.
 They need to stay backwards compatible for old transactions/blocks to be fetchable.
+
+If you encounter a ValidationError in the context of an RPC response, it is possible to disable validation.
+This can be achieved by setting the environment variable, STARKNET_PY_MARSHMALLOW_UKNOWN_EXCLUDE,
+to true. Consequently, any unknown fields in response will be excluded.
 """
 
 import json
@@ -985,6 +989,7 @@ class L1HandlerTransactionTrace:
     Dataclass representing a transaction trace of an L1_HANDLER transaction.
     """
 
+    execution_resources: Optional[ExecutionResources]
     function_invocation: FunctionInvocation
     state_diff: Optional[StateDiff] = None
 

starknet_py/net/full_node_client.py

@@ -126,6 +126,7 @@ async def get_block_with_txs(
     ) -> Union[StarknetBlock, PendingStarknetBlock]:
         return await self.get_block(block_hash=block_hash, block_number=block_number)
 
+    # TODO (#1323): remove unknown=EXCLUDE after devnet response fix
     async def get_block_with_tx_hashes(
         self,
         block_hash: Optional[Union[Hash, Tag]] = None,
@@ -147,7 +148,7 @@ async def get_block_with_tx_hashes(
             )
         return cast(
             StarknetBlockWithTxHashes,
-            StarknetBlockWithTxHashesSchema().load(res, unknown=EXCLUDE),
+            StarknetBlockWithTxHashesSchema().load(res),
         )
 
     async def get_block_with_receipts(
@@ -167,11 +168,11 @@ async def get_block_with_receipts(
         if block_identifier == {"block_id": "pending"}:
             return cast(
                 PendingStarknetBlockWithReceipts,
-                PendingStarknetBlockWithReceiptsSchema().load(res, unknown=EXCLUDE),
+                PendingStarknetBlockWithReceiptsSchema().load(res),
             )
         return cast(
             StarknetBlockWithReceipts,
-            StarknetBlockWithReceiptsSchema().load(res, unknown=EXCLUDE),
+            StarknetBlockWithReceiptsSchema().load(res),
         )
 
     # TODO (#809): add tests with multiple emitted keys
@@ -281,6 +282,7 @@ async def _get_events_chunk(
             return res["events"], res["continuation_token"]
         return res["events"], None
 
+    # TODO (#1323): remove unknown=EXCLUDE after devnet fix response
     async def get_state_update(
         self,
         block_hash: Optional[Union[Hash, Tag]] = None,
@@ -300,9 +302,7 @@ async def get_state_update(
                 PendingBlockStateUpdate,
                 PendingBlockStateUpdateSchema().load(res, unknown=EXCLUDE),
             )
-        return cast(
-            BlockStateUpdate, BlockStateUpdateSchema().load(res, unknown=EXCLUDE)
-        )
+        return cast(BlockStateUpdate, BlockStateUpdateSchema().load(res))
 
     async def get_storage_at(
         self,
@@ -337,7 +337,7 @@ async def get_transaction(
             )
         except ClientError as ex:
             raise TransactionNotReceivedError() from ex
-        return cast(Transaction, TypesOfTransactionsSchema().load(res, unknown=EXCLUDE))
+        return cast(Transaction, TypesOfTransactionsSchema().load(res))
 
     async def get_l1_message_hash(self, tx_hash: Hash) -> Hash:
         """
@@ -358,9 +358,7 @@ async def get_transaction_receipt(self, tx_hash: Hash) -> TransactionReceipt:
             method_name="getTransactionReceipt",
             params={"transaction_hash": _to_rpc_felt(tx_hash)},
         )
-        return cast(
-            TransactionReceipt, TransactionReceiptSchema().load(res, unknown=EXCLUDE)
-        )
+        return cast(TransactionReceipt, TransactionReceiptSchema().load(res))
 
     async def estimate_fee(
         self,
@@ -392,9 +390,7 @@ async def estimate_fee(
 
         return cast(
             EstimatedFee,
-            EstimatedFeeSchema().load(
-                res, unknown=EXCLUDE, many=(not single_transaction)
-            ),
+            EstimatedFeeSchema().load(res, many=not single_transaction),
         )
 
     async def estimate_message_fee(
@@ -440,7 +436,7 @@ async def estimate_message_fee(
                     **block_identifier,
                 },
             )
-            return cast(EstimatedFee, EstimatedFeeSchema().load(res, unknown=EXCLUDE))
+            return cast(EstimatedFee, EstimatedFeeSchema().load(res))
         except ClientError as err:
             if err.code == RPC_CONTRACT_ERROR:
                 raise ClientError(
@@ -499,9 +495,7 @@ async def send_transaction(self, transaction: Invoke) -> SentTransactionResponse
             params={"invoke_transaction": params},
         )
 
-        return cast(
-            SentTransactionResponse, SentTransactionSchema().load(res, unknown=EXCLUDE)
-        )
+        return cast(SentTransactionResponse, SentTransactionSchema().load(res))
 
     async def deploy_account(
         self, transaction: DeployAccount
@@ -515,7 +509,7 @@ async def deploy_account(
 
         return cast(
             DeployAccountTransactionResponse,
-            DeployAccountTransactionResponseSchema().load(res, unknown=EXCLUDE),
+            DeployAccountTransactionResponseSchema().load(res),
         )
 
     async def declare(self, transaction: Declare) -> DeclareTransactionResponse:
@@ -528,7 +522,7 @@ async def declare(self, transaction: Declare) -> DeclareTransactionResponse:
 
         return cast(
             DeclareTransactionResponse,
-            DeclareTransactionResponseSchema().load(res, unknown=EXCLUDE),
+            DeclareTransactionResponseSchema().load(res),
         )
 
     async def get_class_hash_at(
@@ -571,9 +565,9 @@ async def get_class_by_hash(
         if "sierra_program" in res:
             return cast(
                 SierraContractClass,
-                SierraContractClassSchema().load(res, unknown=EXCLUDE),
+                SierraContractClassSchema().load(res),
             )
-        return cast(ContractClass, ContractClassSchema().load(res, unknown=EXCLUDE))
+        return cast(ContractClass, ContractClassSchema().load(res))
 
     # Only RPC methods
 
@@ -602,7 +596,7 @@ async def get_transaction_by_block_id(
                 "index": index,
             },
         )
-        return cast(Transaction, TypesOfTransactionsSchema().load(res, unknown=EXCLUDE))
+        return cast(Transaction, TypesOfTransactionsSchema().load(res))
 
     async def get_block_transaction_count(
         self,
@@ -656,9 +650,9 @@ async def get_class_at(
         if "sierra_program" in res:
             return cast(
                 SierraContractClass,
-                SierraContractClassSchema().load(res, unknown=EXCLUDE),
+                SierraContractClassSchema().load(res),
             )
-        return cast(ContractClass, ContractClassSchema().load(res, unknown=EXCLUDE))
+        return cast(ContractClass, ContractClassSchema().load(res))
 
     async def get_contract_nonce(
         self,
@@ -698,7 +692,7 @@ async def get_transaction_status(self, tx_hash: Hash) -> TransactionStatusRespon
         )
         return cast(
             TransactionStatusResponse,
-            TransactionStatusResponseSchema().load(res, unknown=EXCLUDE),
+            TransactionStatusResponseSchema().load(res),
         )
 
     # ------------------------------- Trace API -------------------------------
@@ -719,9 +713,7 @@ async def trace_transaction(
                 "transaction_hash": _to_rpc_felt(tx_hash),
             },
         )
-        return cast(
-            TransactionTrace, TransactionTraceSchema().load(res, unknown=EXCLUDE)
-        )
+        return cast(TransactionTrace, TransactionTraceSchema().load(res))
 
     async def simulate_transactions(
         self,
@@ -772,7 +764,7 @@ async def simulate_transactions(
         )
         return cast(
             List[SimulatedTransaction],
-            SimulatedTransactionSchema().load(res, unknown=EXCLUDE, many=True),
+            SimulatedTransactionSchema().load(res, many=True),
         )
 
     async def trace_block_transactions(
@@ -799,7 +791,7 @@ async def trace_block_transactions(
         )
         return cast(
             List[BlockTransactionTrace],
-            BlockTransactionTraceSchema().load(res, unknown=EXCLUDE, many=True),
+            BlockTransactionTraceSchema().load(res, many=True),
         )
 
 

starknet_py/net/schemas/rpc.py

@@ -1,6 +1,6 @@
 # pylint: disable=too-many-lines
 
-from marshmallow import EXCLUDE, Schema, fields, post_load
+from marshmallow import EXCLUDE, fields, post_load
 from marshmallow_oneofschema import OneOfSchema
 
 from starknet_py.abi.v0.schemas import ContractAbiEntrySchema
@@ -87,6 +87,7 @@
     Uint128,
 )
 from starknet_py.net.schemas.utils import _extract_tx_version
+from starknet_py.utils.schema import Schema
 
 # pylint: disable=unused-argument, no-self-use
 
@@ -506,7 +507,7 @@ class BlockHeaderSchema(Schema):
 
 class PendingStarknetBlockSchema(PendingBlockHeaderSchema):
     transactions = fields.List(
-        fields.Nested(TypesOfTransactionsSchema(unknown=EXCLUDE)),
+        fields.Nested(TypesOfTransactionsSchema()),
         data_key="transactions",
         required=True,
     )
@@ -519,7 +520,7 @@ def make_dataclass(self, data, **kwargs) -> PendingStarknetBlock:
 class StarknetBlockSchema(BlockHeaderSchema):
     status = BlockStatusField(data_key="status", required=True)
     transactions = fields.List(
-        fields.Nested(TypesOfTransactionsSchema(unknown=EXCLUDE)),
+        fields.Nested(TypesOfTransactionsSchema()),
         data_key="transactions",
         required=True,
     )
@@ -964,6 +965,11 @@ def make_dataclass(self, data, **kwargs) -> DeployAccountTransactionTrace:
 
 
 class L1HandlerTransactionTraceSchema(Schema):
+    # TODO (#1323): Explain with starknet, because spec doesn't contain execution_resources
+    execution_resources = fields.Nested(
+        ExecutionResourcesSchema(), data_key="execution_resources", load_default=None
+    )
+
     function_invocation = fields.Nested(
         FunctionInvocationSchema(), data_key="function_invocation", required=True
     )
@@ -977,16 +983,15 @@ def make_dataclass(self, data, **kwargs) -> L1HandlerTransactionTrace:
 
 
 class TransactionTraceSchema(OneOfSchema):
+    type_field = "type"
+
     type_schemas = {
         "INVOKE": InvokeTransactionTraceSchema(),
         "DECLARE": DeclareTransactionTraceSchema(),
         "DEPLOY_ACCOUNT": DeployAccountTransactionTraceSchema(),
         "L1_HANDLER": L1HandlerTransactionTraceSchema(),
     }
 
-    def get_data_type(self, data):
-        return data["type"]
-
 
 class SimulatedTransactionSchema(Schema):
     # `unknown=EXCLUDE` in order to skip `type=...` field we don't want

starknet_py/utils/schema.py

@@ -0,0 +1,13 @@
+import os
+
+from marshmallow import EXCLUDE, RAISE
+from marshmallow import Schema as MarshmallowSchema
+
+MARSHMALLOW_UKNOWN_EXCLUDE = os.environ.get("STARKNET_PY_MARSHMALLOW_UKNOWN_EXCLUDE")
+
+
+class Schema(MarshmallowSchema):
+    class Meta:
+        unknown = (
+            EXCLUDE if (MARSHMALLOW_UKNOWN_EXCLUDE or "").lower() == "true" else RAISE
+        )