Added support for V14 metadata decoding.

Author: thewhaleking

async_substrate_interface/async_substrate.py

@@ -65,6 +65,7 @@
 from async_substrate_interface.utils.decoding import (
     _determine_if_old_runtime_call,
     _bt_decode_to_dict_or_list,
+    legacy_scale_decode,
 )
 from async_substrate_interface.utils.storage import StorageKey
 from async_substrate_interface.type_registry import _TYPE_REGISTRY
@@ -816,7 +817,7 @@ async def initialize(self):
                 )
 
                 if ss58_prefix_constant:
-                    self.ss58_format = ss58_prefix_constant
+                    self.ss58_format = ss58_prefix_constant.value
         self.initialized = True
         self._initializing = False
 
@@ -908,7 +909,7 @@ async def _get_current_block_hash(
 
     async def _load_registry_at_block(
         self, block_hash: Optional[str]
-    ) -> tuple[MetadataV15, PortableRegistry]:
+    ) -> tuple[Optional[MetadataV15], Optional[PortableRegistry]]:
         # Should be called for any block that fails decoding.
         # Possibly the metadata was different.
         try:
@@ -922,7 +923,11 @@ async def _load_registry_at_block(
                 "Client error: Execution failed: Other: Exported method Metadata_metadata_at_version is not found"
                 in e.args
             ):
-                raise MetadataAtVersionNotFound
+                logger.warning(
+                    "Exported method Metadata_metadata_at_version is not found. This indicates the block is quite old, "
+                    "decoding for this block will use legacy Python decoding."
+                )
+                return None, None
             else:
                 raise e
         metadata_option_hex_str = metadata_rpc_result["result"]
@@ -964,6 +969,7 @@ async def decode_scale(
         return_scale_obj: bool = False,
         block_hash: Optional[str] = None,
         runtime: Optional[Runtime] = None,
+        force_legacy: bool = False,
     ) -> Union[ScaleObj, Any]:
         """
         Helper function to decode arbitrary SCALE-bytes (e.g. 0x02000000) according to given RUST type_string
@@ -979,6 +985,7 @@ async def decode_scale(
             block_hash: Hash of the block where the desired runtime is located. Ignored if supplying `runtime`
             runtime: Optional Runtime object whose registry to use for decoding. If not specified, runtime will be
                 loaded based on the block hash specified (or latest block if no block_hash is specified)
+            force_legacy: Whether to explicitly use legacy Python-only decoding (non bt-decode).
 
         Returns:
             Decoded object
@@ -991,10 +998,10 @@ async def decode_scale(
         else:
             if not runtime:
                 runtime = await self.init_runtime(block_hash=block_hash)
-                runtime_registry = runtime.registry
+            if runtime.metadata_v15 is not None or force_legacy is True:
+                obj = decode_by_type_string(type_string, runtime.registry, scale_bytes)
             else:
-                runtime_registry = runtime.registry
-            obj = decode_by_type_string(type_string, runtime_registry, scale_bytes)
+                obj = legacy_scale_decode(type_string, scale_bytes, runtime)
         if return_scale_obj:
             return ScaleObj(obj)
         else:
@@ -1933,7 +1940,12 @@ def convert_event_data(data):
         )
         if storage_obj:
             for item in list(storage_obj):
-                events.append(convert_event_data(item))
+                try:
+                    events.append(convert_event_data(item))
+                except (
+                    AttributeError
+                ):  # indicates this was legacy decoded with scalecodec
+                    events.append(item)
         return events
 
     async def get_metadata(self, block_hash=None) -> MetadataV15:
@@ -2909,7 +2921,7 @@ async def _do_runtime_call_old(
         result_vec_u8_bytes = hex_to_bytes(result_data["result"])
         result_bytes = await self.decode_scale(
             "Vec<u8>", result_vec_u8_bytes, runtime=runtime
-        )
+        )  # TODO may need to force_legacy after testing.
 
         # Decode result
         # Get correct type
@@ -2945,22 +2957,32 @@ async def runtime_call(
             params = {}
 
         try:
-            metadata_v15_value = runtime.metadata_v15.value()
+            if runtime.metadata_v15 is None:
+                _ = self.runtime_config.type_registry["runtime_api"][api]["methods"][
+                    method
+                ]
+                runtime_api_types = self.runtime_config.type_registry["runtime_api"][
+                    api
+                ].get("types", {})
+                runtime.runtime_config.update_type_registry_types(runtime_api_types)
+                return await self._do_runtime_call_old(
+                    api, method, params, block_hash, runtime=runtime
+                )
 
-            apis = {entry["name"]: entry for entry in metadata_v15_value["apis"]}
-            api_entry = apis[api]
-            methods = {entry["name"]: entry for entry in api_entry["methods"]}
-            runtime_call_def = methods[method]
+            else:
+                metadata_v15_value = runtime.metadata_v15.value()
+
+                apis = {entry["name"]: entry for entry in metadata_v15_value["apis"]}
+                api_entry = apis[api]
+                methods = {entry["name"]: entry for entry in api_entry["methods"]}
+                runtime_call_def = methods[method]
+                if _determine_if_old_runtime_call(runtime_call_def, metadata_v15_value):
+                    return await self._do_runtime_call_old(
+                        api, method, params, block_hash, runtime=runtime
+                    )
         except KeyError:
             raise ValueError(f"Runtime API Call '{api}.{method}' not found in registry")
 
-        if _determine_if_old_runtime_call(runtime_call_def, metadata_v15_value):
-            result = await self._do_runtime_call_old(
-                api, method, params, block_hash, runtime=runtime
-            )
-
-            return result
-
         if isinstance(params, list) and len(params) != len(runtime_call_def["inputs"]):
             raise ValueError(
                 f"Number of parameter provided ({len(params)}) does not "

async_substrate_interface/sync_substrate.py

@@ -560,7 +560,7 @@ def initialize(self):
                     "System", "SS58Prefix", block_hash=self.last_block_hash
                 )
                 if ss58_prefix_constant:
-                    self.ss58_format = ss58_prefix_constant
+                    self.ss58_format = ss58_prefix_constant.value
         self.initialized = True
 
     def __exit__(self, exc_type, exc_val, exc_tb):

async_substrate_interface/utils/decoding.py

@@ -1,6 +1,7 @@
 from typing import Union, TYPE_CHECKING
 
 from bt_decode import AxonInfo, PrometheusInfo, decode_list
+from scalecodec import ScaleBytes
 
 from async_substrate_interface.utils import hex_to_bytes
 from async_substrate_interface.types import ScaleObj
@@ -55,10 +56,16 @@ def _bt_decode_to_dict_or_list(obj) -> Union[dict, list[dict]]:
 def _decode_scale_list_with_runtime(
     type_strings: list[str],
     scale_bytes_list: list[bytes],
-    runtime_registry,
+    runtime: "Runtime",
     return_scale_obj: bool = False,
 ):
-    obj = decode_list(type_strings, runtime_registry, scale_bytes_list)
+    if runtime.metadata_v15 is not None:
+        obj = decode_list(type_strings, runtime.registry, scale_bytes_list)
+    else:
+        obj = [
+            legacy_scale_decode(x, y, runtime)
+            for (x, y) in zip(type_strings, scale_bytes_list)
+        ]
     if return_scale_obj:
         return [ScaleObj(x) for x in obj]
     else:
@@ -109,7 +116,7 @@ def concat_hash_len(key_hasher: str) -> int:
     all_decoded = _decode_scale_list_with_runtime(
         pre_decoded_key_types + pre_decoded_value_types,
         pre_decoded_keys + pre_decoded_values,
-        runtime.registry,
+        runtime,
     )
     middl_index = len(all_decoded) // 2
     decoded_keys = all_decoded[:middl_index]
@@ -132,3 +139,18 @@ def concat_hash_len(key_hasher: str) -> int:
         item_value = dv
         result.append([item_key, item_value])
     return result
+
+
+def legacy_scale_decode(
+    type_string: str, scale_bytes: Union[str, ScaleBytes], runtime: "Runtime"
+):
+    if isinstance(scale_bytes, (str, bytes)):
+        scale_bytes = ScaleBytes(scale_bytes)
+
+    obj = runtime.runtime_config.create_scale_object(
+        type_string=type_string, data=scale_bytes, metadata=runtime.metadata
+    )
+
+    obj.decode(check_remaining=runtime.config.get("strict_scale_decode"))
+
+    return obj.value

tests/unit_tests/asyncio_/test_substrate_interface.py

@@ -8,6 +8,9 @@
 from async_substrate_interface.types import ScaleObj
 
 
+ARCHIVE_ENTRYPOINT = "wss://archive.chain.opentensor.ai:443"
+
+
 @pytest.mark.asyncio
 async def test_invalid_url_raises_exception():
     """Test that invalid URI raises an InvalidURI exception."""
@@ -113,3 +116,22 @@ async def test_websocket_shutdown_timer():
         await substrate.get_chain_head()
         await asyncio.sleep(6)  # same sleep time as before
         assert substrate.ws._initialized is True  # connection should still be open
+
+
+@pytest.mark.asyncio
+async def test_legacy_decoding():
+    pre_metadata_v15_block = 3_014_300  # several blocks before metadata v15 was added
+
+    async with AsyncSubstrateInterface(ARCHIVE_ENTRYPOINT) as substrate:
+        block_hash = await substrate.get_block_hash(pre_metadata_v15_block)
+        events = await substrate.get_events(block_hash)
+        assert isinstance(events, list)
+
+        query_map_result = await substrate.query_map(
+            module="SubtensorModule",
+            storage_function="NetworksAdded",
+            block_hash=block_hash,
+        )
+        async for key, value in query_map_result:
+            assert isinstance(key, int)
+            assert isinstance(value, ScaleObj)