Apply legacy v14 support to sync substrate

Author: thewhaleking

async_substrate_interface/async_substrate.py

@@ -40,7 +40,6 @@
     ExtrinsicNotFound,
     BlockNotFound,
     MaxRetriesExceeded,
-    MetadataAtVersionNotFound,
     StateDiscardedError,
 )
 from async_substrate_interface.protocols import Keypair
@@ -923,10 +922,6 @@ async def _load_registry_at_block(
                 "Client error: Execution failed: Other: Exported method Metadata_metadata_at_version is not found"
                 in e.args
             ):
-                logger.warning(
-                    "Exported method Metadata_metadata_at_version is not found. This indicates the block is quite old, "
-                    "decoding for this block will use legacy Python decoding."
-                )
                 return None, None
             else:
                 raise e
@@ -1091,9 +1086,15 @@ async def _get_runtime_for_version(
             raise SubstrateRequestException(
                 f"No metadata for block '{runtime_block_hash}'"
             )
-        logger.debug(
-            f"Retrieved metadata and metadata v15 for {runtime_version} from Substrate node"
-        )
+        if metadata_v15 is not None:
+            logger.debug(
+                f"Retrieved metadata and metadata v15 for {runtime_version} from Substrate node"
+            )
+        else:
+            logger.debug(
+                f"Exported method Metadata_metadata_at_version is not found for {runtime_version}. This indicates the "
+                f"block is quite old, decoding for this block will use legacy Python decoding."
+            )
         implements_scale_info = metadata.portable_registry is not None
         runtime = Runtime(
             chain=self.chain,
@@ -2921,7 +2922,7 @@ async def _do_runtime_call_old(
         result_vec_u8_bytes = hex_to_bytes(result_data["result"])
         result_bytes = await self.decode_scale(
             "Vec<u8>", result_vec_u8_bytes, runtime=runtime
-        )  # TODO may need to force_legacy after testing.
+        )
 
         # Decode result
         # Get correct type

async_substrate_interface/sync_substrate.py

@@ -23,7 +23,6 @@
     SubstrateRequestException,
     BlockNotFound,
     MaxRetriesExceeded,
-    MetadataAtVersionNotFound,
     StateDiscardedError,
 )
 from async_substrate_interface.protocols import Keypair
@@ -45,6 +44,7 @@
     _determine_if_old_runtime_call,
     _bt_decode_to_dict_or_list,
     decode_query_map,
+    legacy_scale_decode,
 )
 from async_substrate_interface.utils.storage import StorageKey
 from async_substrate_interface.type_registry import _TYPE_REGISTRY
@@ -576,17 +576,6 @@ def metadata(self):
         else:
             return self.runtime.metadata
 
-    @property
-    def implements_scaleinfo(self) -> Optional[bool]:
-        """
-        Returns True if current runtime implements a `PortableRegistry` (`MetadataV14` and higher). Returns `None` if
-        no currently loaded runtime.
-        """
-        if self.runtime and self.runtime.metadata:
-            return self.runtime.metadata.portable_registry is not None
-        else:
-            return None
-
     @property
     def properties(self):
         if self._properties is None:
@@ -669,7 +658,7 @@ def _load_registry_at_block(self, block_hash: Optional[str]) -> MetadataV15:
                 "Client error: Execution failed: Other: Exported method Metadata_metadata_at_version is not found"
                 in e.args
             ):
-                raise MetadataAtVersionNotFound
+                return None, None
             else:
                 raise e
         metadata_option_hex_str = metadata_rpc_result["result"]
@@ -701,7 +690,12 @@ def decode_scale(
             # Decode AccountId bytes to SS58 address
             return ss58_encode(scale_bytes, SS58_FORMAT)
         else:
-            obj = decode_by_type_string(type_string, self.runtime.registry, scale_bytes)
+            if self.runtime.metadata_v15 is not None:
+                obj = decode_by_type_string(
+                    type_string, self.runtime.registry, scale_bytes
+                )
+            else:
+                obj = legacy_scale_decode(type_string, scale_bytes, self.runtime)
         if return_scale_obj:
             return ScaleObj(obj)
         else:
@@ -714,7 +708,7 @@ def load_runtime(self, runtime):
         self.runtime.reload_type_registry(use_remote_preset=False, auto_discover=True)
 
         self.runtime_config.set_active_spec_version_id(runtime.runtime_version)
-        if self.implements_scaleinfo:
+        if self.runtime.implements_scaleinfo:
             logger.debug("Add PortableRegistry from metadata to type registry")
             self.runtime_config.add_portable_registry(runtime.metadata)
         # Set runtime compatibility flags
@@ -754,7 +748,8 @@ def init_runtime(
             if runtime := self.runtime_cache.retrieve(block=block_id):
                 self.runtime = runtime
                 self.runtime.load_runtime()
-                self.runtime.load_registry_type_map()
+                if self.runtime.registry:
+                    self.runtime.load_registry_type_map()
                 return self.runtime
             block_hash = self.get_block_hash(block_id)
 
@@ -765,7 +760,8 @@ def init_runtime(
             if runtime := self.runtime_cache.retrieve(block_hash=block_hash):
                 self.runtime = runtime
                 self.runtime.load_runtime()
-                self.runtime.load_registry_type_map()
+                if self.runtime.registry:
+                    self.runtime.load_registry_type_map()
                 return self.runtime
 
         runtime_version = self.get_block_runtime_version_for(block_hash)
@@ -780,12 +776,14 @@ def init_runtime(
         if runtime := self.runtime_cache.retrieve(runtime_version=runtime_version):
             self.runtime = runtime
             self.runtime.load_runtime()
-            self.runtime.load_registry_type_map()
+            if self.runtime.registry:
+                self.runtime.load_registry_type_map()
             return runtime
         else:
             self.runtime = self.get_runtime_for_version(runtime_version, block_hash)
             self.runtime.load_runtime()
-            self.runtime.load_registry_type_map()
+            if self.runtime.registry:
+                self.runtime.load_registry_type_map()
             return self.runtime
 
     def get_runtime_for_version(
@@ -819,9 +817,15 @@ def get_runtime_for_version(
         metadata_v15, registry = self._load_registry_at_block(
             block_hash=runtime_block_hash
         )
-        logger.debug(
-            f"Retrieved metadata v15 for {runtime_version} from Substrate node"
-        )
+        if metadata_v15 is not None:
+            logger.debug(
+                f"Retrieved metadata and metadata v15 for {runtime_version} from Substrate node"
+            )
+        else:
+            logger.debug(
+                f"Exported method Metadata_metadata_at_version is not found for {runtime_version}. This indicates the "
+                f"block is quite old, decoding for this block will use legacy Python decoding."
+            )
 
         runtime = Runtime(
             chain=self.chain,
@@ -1204,7 +1208,7 @@ def decode_block(block_data, block_data_hash=None) -> dict[str, Any]:
                             block_data["header"]["digest"]["logs"][idx] = log_digest
 
                             if include_author and "PreRuntime" in log_digest.value:
-                                if self.implements_scaleinfo:
+                                if self.runtime.implements_scaleinfo:
                                     engine = bytes(log_digest[1][0])
                                     # Retrieve validator set
                                     parent_hash = block_data["header"]["parentHash"]
@@ -1609,7 +1613,12 @@ def convert_event_data(data):
         )
         if storage_obj:
             for item in list(storage_obj):
-                events.append(convert_event_data(item))
+                try:
+                    events.append(convert_event_data(item))
+                except (
+                    AttributeError
+                ):  # indicates this was legacy decoded with scalecodec
+                    events.append(item)
         return events
 
     def get_metadata(self, block_hash=None) -> MetadataV15:
@@ -2530,20 +2539,28 @@ def runtime_call(
             params = {}
 
         try:
-            metadata_v15_value = runtime.metadata_v15.value()
+            if runtime.metadata_v15 is None:
+                _ = self.runtime_config.type_registry["runtime_api"][api]["methods"][
+                    method
+                ]
+                runtime_api_types = self.runtime_config.type_registry["runtime_api"][
+                    api
+                ].get("types", {})
+                runtime.runtime_config.update_type_registry_types(runtime_api_types)
+                return self._do_runtime_call_old(api, method, params, block_hash)
+            else:
+                metadata_v15_value = runtime.metadata_v15.value()
+
+                apis = {entry["name"]: entry for entry in metadata_v15_value["apis"]}
+                api_entry = apis[api]
+                methods = {entry["name"]: entry for entry in api_entry["methods"]}
+                runtime_call_def = methods[method]
+                if _determine_if_old_runtime_call(runtime_call_def, metadata_v15_value):
+                    return self._do_runtime_call_old(api, method, params, block_hash)
 
-            apis = {entry["name"]: entry for entry in metadata_v15_value["apis"]}
-            api_entry = apis[api]
-            methods = {entry["name"]: entry for entry in api_entry["methods"]}
-            runtime_call_def = methods[method]
         except KeyError:
             raise ValueError(f"Runtime API Call '{api}.{method}' not found in registry")
 
-        if _determine_if_old_runtime_call(runtime_call_def, metadata_v15_value):
-            result = self._do_runtime_call_old(api, method, params, block_hash)
-
-            return result
-
         if isinstance(params, list) and len(params) != len(runtime_call_def["inputs"]):
             raise ValueError(
                 f"Number of parameter provided ({len(params)}) does not "
@@ -2745,7 +2762,7 @@ def get_type_registry(self, block_hash: str = None, max_recursion: int = 4) -> d
         """
         self.init_runtime(block_hash=block_hash)
 
-        if not self.implements_scaleinfo:
+        if not self.runtime.implements_scaleinfo:
             raise NotImplementedError("MetadataV14 or higher runtimes is required")
 
         type_registry = {}

tests/helpers/settings.py

@@ -32,3 +32,5 @@
 AURA_NODE_URL = (
     environ.get("SUBSTRATE_AURA_NODE_URL") or "wss://acala-rpc-1.aca-api.network"
 )
+
+ARCHIVE_ENTRYPOINT = "wss://archive.chain.opentensor.ai:443"

tests/unit_tests/asyncio_/test_substrate_interface.py

@@ -6,9 +6,7 @@
 
 from async_substrate_interface.async_substrate import AsyncSubstrateInterface
 from async_substrate_interface.types import ScaleObj
-
-
-ARCHIVE_ENTRYPOINT = "wss://archive.chain.opentensor.ai:443"
+from tests.helpers.settings import ARCHIVE_ENTRYPOINT
 
 
 @pytest.mark.asyncio
@@ -137,9 +135,9 @@ async def test_legacy_decoding():
             assert isinstance(key, int)
             assert isinstance(value, ScaleObj)
 
-        unix = await substrate.query(
+        timestamp = await substrate.query(
             "Timestamp",
             "Now",
             block_hash=block_hash,
         )
-        assert unix.value == 1716358476004
+        assert timestamp.value == 1716358476004

tests/unit_tests/sync/test_substrate_interface.py

@@ -3,6 +3,8 @@
 from async_substrate_interface.sync_substrate import SubstrateInterface
 from async_substrate_interface.types import ScaleObj
 
+from tests.helpers.settings import ARCHIVE_ENTRYPOINT
+
 
 def test_runtime_call(monkeypatch):
     substrate = SubstrateInterface("ws://localhost", _mock=True)
@@ -72,3 +74,30 @@ def test_runtime_call(monkeypatch):
     substrate.rpc_request.assert_any_call(
         "state_call", ["SubstrateApi_SubstrateMethod", "", None]
     )
+    substrate.close()
+
+
+def test_legacy_decoding():
+    # roughly 4000 blocks before metadata v15 was added
+    pre_metadata_v15_block = 3_010_611
+
+    with SubstrateInterface(ARCHIVE_ENTRYPOINT) as substrate:
+        block_hash = substrate.get_block_hash(pre_metadata_v15_block)
+        events = substrate.get_events(block_hash)
+        assert isinstance(events, list)
+
+        query_map_result = substrate.query_map(
+            module="SubtensorModule",
+            storage_function="NetworksAdded",
+            block_hash=block_hash,
+        )
+        for key, value in query_map_result:
+            assert isinstance(key, int)
+            assert isinstance(value, ScaleObj)
+
+        timestamp = substrate.query(
+            "Timestamp",
+            "Now",
+            block_hash=block_hash,
+        )
+        assert timestamp.value == 1716358476004