From d4ace76273b4965f50ebe68fc984f1272238f123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Thu, 3 Jul 2025 16:33:32 +0100 Subject: [PATCH 01/28] rename nyxd-scraper to sqlite wip: made storage mostly generic minus modules changed error types to make modules dyn compatible implemented traits for sqlite instance using sqlite instance for rewarder and chain watcher psql scaffolding initial postgres support - missing some proto -> json parsing use postgres in chain scraper added message registry to block processor message content parsing in psql involved addresses adding null value for logs Revert "use postgres in chain scraper" This reverts commit 83c84bfd2dcb7e357252ed60f69a73eb6fe4aa1e. using SignerInfo proto definitions for db serialisation added ibc messages to MessageRegistry --- Cargo.toml | 8 +- ...03b5ee5b27879b0026bb0480b3f2722318a75.json | 15 + ...16422b1f723d0a316314b50c43c8b29f8891d.json | 14 + ...f3cb2eed5b5d35391fd30a4d5f44f2e2178b7.json | 26 + ...a93e944b0d44ed1f7c1036f306e34372da11c.json | 20 + ...c57402119ec7c3aae731b0da831327301466f.json | 14 + ...a70635028f392fe794d6131827b083e1755e1.json | 14 + ...3f419a849d4ec45af40b052a4cbf09b44f3ec.json | 20 + ...5b67fd9281ce1de0653efa53b9d9b93cf335d.json | 14 + ...7a22f0444fbc679db1c06b651fb8b5538b278.json | 18 + ...ed56b6e270ce186f0e49528865d1924343b78.json | 19 + ...19cc462d04222fb20ad76de2a40f3f4f8fe15.json | 22 + ...6b2e2d6a9ad4b225c4c883aafc4e9f0428008.json | 22 + ...e77b7bc0a1af315ffd42c3e68156d6e4ace70.json | 24 + ...ab7bd82ecd68041aa932a56c8ce09623251e4.json | 28 + ...9ce71582635df47f52dcf3fd1df4e7be6b96d.json | 20 + ...890bbf6150ab394c72783114340d4def5f9ef.json | 19 + ...3cf00236490b86779559d84740ec18bcfa3a9.json | 14 + ...13d0598fd856aa019a0cbbae12d7cafb4672f.json | 14 + common/nyxd-scraper-psql/Cargo.toml | 34 ++ common/nyxd-scraper-psql/Makefile | 102 ++++ common/nyxd-scraper-psql/README.md | 79 +++ common/nyxd-scraper-psql/build.rs | 8 + common/nyxd-scraper-psql/docker-compose.yml | 21 + .../sql_migrations/01_metadata.sql | 0 .../sql_migrations/02_cosmos.sql | 127 +++++ common/nyxd-scraper-psql/src/error.rs | 43 ++ common/nyxd-scraper-psql/src/lib.rs | 21 + .../src/storage/block_storage.rs | 246 ++++++++ .../nyxd-scraper-psql/src/storage/helpers.rs | 25 + .../nyxd-scraper-psql/src/storage/manager.rs | 538 ++++++++++++++++++ common/nyxd-scraper-psql/src/storage/mod.rs | 8 + .../src/storage/models.rs | 0 .../src/storage/transaction.rs | 291 ++++++++++ .../Cargo.toml | 21 +- .../src/block_processor/helpers.rs | 0 .../src/block_processor/mod.rs | 25 +- .../src/block_processor/pruning.rs | 0 .../src/block_processor/types.rs | 0 .../src/block_requester/mod.rs | 0 .../src/constants.rs | 0 .../src/cosmos_module/message_registry.rs | 146 +++++ .../src/cosmos_module/mod.rs | 11 + .../src/cosmos_module/modules/auth.rs | 14 + .../src/cosmos_module/modules/authz.rs | 16 + .../src/cosmos_module/modules/bank.rs | 19 + .../src/cosmos_module/modules/capability.rs | 11 + .../src/cosmos_module/modules/consensus.rs | 11 + .../src/cosmos_module/modules/crisis.rs | 15 + .../src/cosmos_module/modules/distribution.rs | 22 + .../src/cosmos_module/modules/evidence.rs | 14 + .../src/cosmos_module/modules/feegrant.rs | 18 + .../src/cosmos_module/modules/gov_v1.rs | 21 + .../src/cosmos_module/modules/gov_v1beta1.rs | 19 + .../src/cosmos_module/modules/group.rs | 27 + .../src/cosmos_module/modules/ibc_core.rs | 70 +++ .../src/cosmos_module/modules/ibc_fee.rs | 18 + .../ibc_interchain_accounts_controller.rs | 17 + .../cosmos_module/modules/ibc_transfer_v1.rs | 14 + .../cosmos_module/modules/ibc_transfer_v2.rs | 10 + .../src/cosmos_module/modules/mint.rs | 14 + .../src/cosmos_module/modules/mod.rs | 28 + .../src/cosmos_module/modules/nft.rs | 11 + .../src/cosmos_module/modules/params.rs | 11 + .../src/cosmos_module/modules/slashing.rs | 11 + .../src/cosmos_module/modules/staking.rs | 23 + .../src/cosmos_module/modules/upgrade.rs | 15 + .../src/cosmos_module/modules/vesting.rs | 18 + .../src/cosmos_module/modules/wasm.rs | 104 ++++ .../src/error.rs | 74 ++- common/nyxd-scraper-shared/src/helpers.rs | 66 +++ .../src/lib.rs | 13 +- .../src/modules/block_module.rs | 4 +- .../src/modules/mod.rs | 0 .../src/modules/msg_module.rs | 4 +- .../src/modules/tx_module.rs | 4 +- .../src/rpc_client.rs | 0 .../src/scraper/mod.rs | 45 +- .../src/scraper/subscriber.rs | 0 .../src/storage/helpers.rs | 18 + common/nyxd-scraper-shared/src/storage/mod.rs | 124 ++++ common/nyxd-scraper-sqlite/Cargo.toml | 28 + .../README.md | 0 .../build.rs | 0 .../sql_migrations/01_metadata.sql | 10 + .../sql_migrations/02_cosmos.sql | 0 common/nyxd-scraper-sqlite/src/error.rs | 36 ++ common/nyxd-scraper-sqlite/src/lib.rs | 21 + .../src/storage/block_storage.rs | 251 ++++++++ .../src/storage/manager.rs | 2 +- .../src/storage/mod.rs} | 5 + .../nyxd-scraper-sqlite/src/storage/models.rs | 30 + .../src/storage/transaction.rs | 236 ++++++++ common/nyxd-scraper/src/helpers.rs | 46 -- common/nyxd-scraper/src/storage/mod.rs | 400 ------------- nym-validator-rewarder/Cargo.toml | 2 +- .../src/cli/process_block.rs | 4 +- .../src/cli/process_until.rs | 4 +- nym-validator-rewarder/src/config/mod.rs | 8 +- .../src/config/persistence/paths.rs | 7 +- nym-validator-rewarder/src/error.rs | 9 +- .../src/rewarder/block_signing/mod.rs | 4 +- .../src/rewarder/block_signing/types.rs | 2 +- .../src/rewarder/helpers.rs | 2 +- nym-validator-rewarder/src/rewarder/mod.rs | 4 +- nym-wallet/Cargo.lock | 42 +- nyx-chain-watcher/Cargo.toml | 2 +- nyx-chain-watcher/src/chain_scraper/mod.rs | 16 +- nyx-chain-watcher/src/http/state.rs | 2 +- 109 files changed, 3631 insertions(+), 556 deletions(-) create mode 100644 common/nyxd-scraper-psql/.sqlx/query-0d3709efacf763b06bf14803bb803b5ee5b27879b0026bb0480b3f2722318a75.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-1c2fb0e9ffceca21ef8dbea19b116422b1f723d0a316314b50c43c8b29f8891d.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-1e344c1dff8b98eb0eb2e530e28f3cb2eed5b5d35391fd30a4d5f44f2e2178b7.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-2561fb016951ea4cd29e43fb9a4a93e944b0d44ed1f7c1036f306e34372da11c.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-36ba5941aca6e7b604a10b8b0aba70635028f392fe794d6131827b083e1755e1.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-3bdf81a9db6075f6f77224c30553f419a849d4ec45af40b052a4cbf09b44f3ec.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-52c27143720ddfdfd0f5644b60f5b67fd9281ce1de0653efa53b9d9b93cf335d.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-62e14613f5ffe692346a79086857a22f0444fbc679db1c06b651fb8b5538b278.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-64a484fd46d8ec46797f944a4cced56b6e270ce186f0e49528865d1924343b78.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-7e82426f5dbcadf1631ba1a806e19cc462d04222fb20ad76de2a40f3f4f8fe15.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-9455331f9be5a3be28e2bd399a36b2e2d6a9ad4b225c4c883aafc4e9f0428008.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-bc7795e58ce71893c3f32a19db8e77b7bc0a1af315ffd42c3e68156d6e4ace70.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-be43d4873911deca784b7be0531ab7bd82ecd68041aa932a56c8ce09623251e4.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-cdba9b267f143c8a8c6c3d6ed713cf00236490b86779559d84740ec18bcfa3a9.json create mode 100644 common/nyxd-scraper-psql/.sqlx/query-d89558c37c51e8e6b1e6a9d5a2b13d0598fd856aa019a0cbbae12d7cafb4672f.json create mode 100644 common/nyxd-scraper-psql/Cargo.toml create mode 100644 common/nyxd-scraper-psql/Makefile create mode 100644 common/nyxd-scraper-psql/README.md create mode 100644 common/nyxd-scraper-psql/build.rs create mode 100644 common/nyxd-scraper-psql/docker-compose.yml rename common/{nyxd-scraper => nyxd-scraper-psql}/sql_migrations/01_metadata.sql (100%) create mode 100644 common/nyxd-scraper-psql/sql_migrations/02_cosmos.sql create mode 100644 common/nyxd-scraper-psql/src/error.rs create mode 100644 common/nyxd-scraper-psql/src/lib.rs create mode 100644 common/nyxd-scraper-psql/src/storage/block_storage.rs create mode 100644 common/nyxd-scraper-psql/src/storage/helpers.rs create mode 100644 common/nyxd-scraper-psql/src/storage/manager.rs create mode 100644 common/nyxd-scraper-psql/src/storage/mod.rs rename common/{nyxd-scraper => nyxd-scraper-psql}/src/storage/models.rs (100%) create mode 100644 common/nyxd-scraper-psql/src/storage/transaction.rs rename common/{nyxd-scraper => nyxd-scraper-shared}/Cargo.toml (61%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/block_processor/helpers.rs (100%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/block_processor/mod.rs (97%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/block_processor/pruning.rs (100%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/block_processor/types.rs (100%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/block_requester/mod.rs (100%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/constants.rs (100%) create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/message_registry.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/mod.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/auth.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/authz.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/bank.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/capability.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/consensus.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/crisis.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/distribution.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/evidence.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/feegrant.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1beta1.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/group.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_core.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_fee.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_interchain_accounts_controller.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_transfer_v1.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_transfer_v2.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/mint.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/mod.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/nft.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/params.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/slashing.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/staking.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/upgrade.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/vesting.rs create mode 100644 common/nyxd-scraper-shared/src/cosmos_module/modules/wasm.rs rename common/{nyxd-scraper => nyxd-scraper-shared}/src/error.rs (72%) create mode 100644 common/nyxd-scraper-shared/src/helpers.rs rename common/{nyxd-scraper => nyxd-scraper-shared}/src/lib.rs (69%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/modules/block_module.rs (79%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/modules/mod.rs (100%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/modules/msg_module.rs (83%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/modules/tx_module.rs (79%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/rpc_client.rs (100%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/scraper/mod.rs (92%) rename common/{nyxd-scraper => nyxd-scraper-shared}/src/scraper/subscriber.rs (100%) create mode 100644 common/nyxd-scraper-shared/src/storage/helpers.rs create mode 100644 common/nyxd-scraper-shared/src/storage/mod.rs create mode 100644 common/nyxd-scraper-sqlite/Cargo.toml rename common/{nyxd-scraper => nyxd-scraper-sqlite}/README.md (100%) rename common/{nyxd-scraper => nyxd-scraper-sqlite}/build.rs (100%) create mode 100644 common/nyxd-scraper-sqlite/sql_migrations/01_metadata.sql rename common/{nyxd-scraper => nyxd-scraper-sqlite}/sql_migrations/02_cosmos.sql (100%) create mode 100644 common/nyxd-scraper-sqlite/src/error.rs create mode 100644 common/nyxd-scraper-sqlite/src/lib.rs create mode 100644 common/nyxd-scraper-sqlite/src/storage/block_storage.rs rename common/{nyxd-scraper => nyxd-scraper-sqlite}/src/storage/manager.rs (99%) rename common/{nyxd-scraper/src/storage/helpers.rs => nyxd-scraper-sqlite/src/storage/mod.rs} (57%) create mode 100644 common/nyxd-scraper-sqlite/src/storage/models.rs create mode 100644 common/nyxd-scraper-sqlite/src/storage/transaction.rs delete mode 100644 common/nyxd-scraper/src/helpers.rs delete mode 100644 common/nyxd-scraper/src/storage/mod.rs diff --git a/Cargo.toml b/Cargo.toml index eaad10bbc4f..838d88185ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,7 +87,9 @@ members = [ "common/nymsphinx/params", "common/nymsphinx/routing", "common/nymsphinx/types", - "common/nyxd-scraper", + "common/nyxd-scraper-sqlite", + "common/nyxd-scraper-psql", + "common/nyxd-scraper-shared", "common/pemstore", "common/registration", "common/serde-helpers", @@ -398,7 +400,9 @@ cw-multi-test = "=2.3.2" bip32 = { version = "0.5.3", default-features = false } -cosmrs = { version = "0.21.1" } +cosmrs = { version = "0.22.0" } +cosmos-sdk-proto = { version = "0.27.0" } +ibc-proto = { version = "0.52.0" } tendermint = "0.40.4" tendermint-rpc = "0.40.4" prost = { version = "0.13", default-features = false } diff --git a/common/nyxd-scraper-psql/.sqlx/query-0d3709efacf763b06bf14803bb803b5ee5b27879b0026bb0480b3f2722318a75.json b/common/nyxd-scraper-psql/.sqlx/query-0d3709efacf763b06bf14803bb803b5ee5b27879b0026bb0480b3f2722318a75.json new file mode 100644 index 00000000000..36ba8bb96b3 --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-0d3709efacf763b06bf14803bb803b5ee5b27879b0026bb0480b3f2722318a75.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO validator (consensus_address, consensus_pubkey)\n VALUES ($1, $2)\n ON CONFLICT DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0d3709efacf763b06bf14803bb803b5ee5b27879b0026bb0480b3f2722318a75" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-1c2fb0e9ffceca21ef8dbea19b116422b1f723d0a316314b50c43c8b29f8891d.json b/common/nyxd-scraper-psql/.sqlx/query-1c2fb0e9ffceca21ef8dbea19b116422b1f723d0a316314b50c43c8b29f8891d.json new file mode 100644 index 00000000000..2e10a89220b --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-1c2fb0e9ffceca21ef8dbea19b116422b1f723d0a316314b50c43c8b29f8891d.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM pre_commit WHERE height < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1c2fb0e9ffceca21ef8dbea19b116422b1f723d0a316314b50c43c8b29f8891d" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-1e344c1dff8b98eb0eb2e530e28f3cb2eed5b5d35391fd30a4d5f44f2e2178b7.json b/common/nyxd-scraper-psql/.sqlx/query-1e344c1dff8b98eb0eb2e530e28f3cb2eed5b5d35391fd30a4d5f44f2e2178b7.json new file mode 100644 index 00000000000..8c9fee0b4a8 --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-1e344c1dff8b98eb0eb2e530e28f3cb2eed5b5d35391fd30a4d5f44f2e2178b7.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO transaction\n (hash, height, index, success, messages, memo, signatures, signer_infos, fee, gas_wanted, gas_used, raw_log, logs)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)\n ON CONFLICT (hash) DO UPDATE\n SET height = excluded.height,\n index = excluded.index,\n success = excluded.success,\n messages = excluded.messages,\n memo = excluded.memo,\n signatures = excluded.signatures,\n signer_infos = excluded.signer_infos,\n fee = excluded.fee,\n gas_wanted = excluded.gas_wanted,\n gas_used = excluded.gas_used,\n raw_log = excluded.raw_log,\n logs = excluded.logs\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int4", + "Bool", + "Json", + "Text", + "TextArray", + "Jsonb", + "Jsonb", + "Int8", + "Int8", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "1e344c1dff8b98eb0eb2e530e28f3cb2eed5b5d35391fd30a4d5f44f2e2178b7" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-2561fb016951ea4cd29e43fb9a4a93e944b0d44ed1f7c1036f306e34372da11c.json b/common/nyxd-scraper-psql/.sqlx/query-2561fb016951ea4cd29e43fb9a4a93e944b0d44ed1f7c1036f306e34372da11c.json new file mode 100644 index 00000000000..0d1b70f8cce --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-2561fb016951ea4cd29e43fb9a4a93e944b0d44ed1f7c1036f306e34372da11c.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT height\n FROM block\n ORDER BY height ASC\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "height", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "2561fb016951ea4cd29e43fb9a4a93e944b0d44ed1f7c1036f306e34372da11c" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f.json b/common/nyxd-scraper-psql/.sqlx/query-2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f.json new file mode 100644 index 00000000000..7efbd0abe8c --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE metadata SET last_processed_height = GREATEST(last_processed_height, $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-36ba5941aca6e7b604a10b8b0aba70635028f392fe794d6131827b083e1755e1.json b/common/nyxd-scraper-psql/.sqlx/query-36ba5941aca6e7b604a10b8b0aba70635028f392fe794d6131827b083e1755e1.json new file mode 100644 index 00000000000..dede45475e4 --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-36ba5941aca6e7b604a10b8b0aba70635028f392fe794d6131827b083e1755e1.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE pruning SET last_pruned_height = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "36ba5941aca6e7b604a10b8b0aba70635028f392fe794d6131827b083e1755e1" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-3bdf81a9db6075f6f77224c30553f419a849d4ec45af40b052a4cbf09b44f3ec.json b/common/nyxd-scraper-psql/.sqlx/query-3bdf81a9db6075f6f77224c30553f419a849d4ec45af40b052a4cbf09b44f3ec.json new file mode 100644 index 00000000000..e638bce9220 --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-3bdf81a9db6075f6f77224c30553f419a849d4ec45af40b052a4cbf09b44f3ec.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT last_pruned_height FROM pruning\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_pruned_height", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "3bdf81a9db6075f6f77224c30553f419a849d4ec45af40b052a4cbf09b44f3ec" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-52c27143720ddfdfd0f5644b60f5b67fd9281ce1de0653efa53b9d9b93cf335d.json b/common/nyxd-scraper-psql/.sqlx/query-52c27143720ddfdfd0f5644b60f5b67fd9281ce1de0653efa53b9d9b93cf335d.json new file mode 100644 index 00000000000..58af4f89c42 --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-52c27143720ddfdfd0f5644b60f5b67fd9281ce1de0653efa53b9d9b93cf335d.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM message WHERE height < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "52c27143720ddfdfd0f5644b60f5b67fd9281ce1de0653efa53b9d9b93cf335d" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-62e14613f5ffe692346a79086857a22f0444fbc679db1c06b651fb8b5538b278.json b/common/nyxd-scraper-psql/.sqlx/query-62e14613f5ffe692346a79086857a22f0444fbc679db1c06b651fb8b5538b278.json new file mode 100644 index 00000000000..a7c102469df --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-62e14613f5ffe692346a79086857a22f0444fbc679db1c06b651fb8b5538b278.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO pre_commit (validator_address, height, timestamp, voting_power, proposer_priority)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (validator_address, timestamp) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Timestamp", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "62e14613f5ffe692346a79086857a22f0444fbc679db1c06b651fb8b5538b278" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-64a484fd46d8ec46797f944a4cced56b6e270ce186f0e49528865d1924343b78.json b/common/nyxd-scraper-psql/.sqlx/query-64a484fd46d8ec46797f944a4cced56b6e270ce186f0e49528865d1924343b78.json new file mode 100644 index 00000000000..08983f2af9f --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-64a484fd46d8ec46797f944a4cced56b6e270ce186f0e49528865d1924343b78.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO block (height, hash, num_txs, total_gas, proposer_address, timestamp)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Int4", + "Int8", + "Text", + "Timestamp" + ] + }, + "nullable": [] + }, + "hash": "64a484fd46d8ec46797f944a4cced56b6e270ce186f0e49528865d1924343b78" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-7e82426f5dbcadf1631ba1a806e19cc462d04222fb20ad76de2a40f3f4f8fe15.json b/common/nyxd-scraper-psql/.sqlx/query-7e82426f5dbcadf1631ba1a806e19cc462d04222fb20ad76de2a40f3f4f8fe15.json new file mode 100644 index 00000000000..3a60c573ed8 --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-7e82426f5dbcadf1631ba1a806e19cc462d04222fb20ad76de2a40f3f4f8fe15.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT height\n FROM block\n WHERE timestamp < $1\n ORDER BY timestamp DESC\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "height", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Timestamp" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7e82426f5dbcadf1631ba1a806e19cc462d04222fb20ad76de2a40f3f4f8fe15" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-9455331f9be5a3be28e2bd399a36b2e2d6a9ad4b225c4c883aafc4e9f0428008.json b/common/nyxd-scraper-psql/.sqlx/query-9455331f9be5a3be28e2bd399a36b2e2d6a9ad4b225c4c883aafc4e9f0428008.json new file mode 100644 index 00000000000..309aa81d9c7 --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-9455331f9be5a3be28e2bd399a36b2e2d6a9ad4b225c4c883aafc4e9f0428008.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT height\n FROM block\n WHERE timestamp > $1\n ORDER BY timestamp\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "height", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Timestamp" + ] + }, + "nullable": [ + false + ] + }, + "hash": "9455331f9be5a3be28e2bd399a36b2e2d6a9ad4b225c4c883aafc4e9f0428008" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-bc7795e58ce71893c3f32a19db8e77b7bc0a1af315ffd42c3e68156d6e4ace70.json b/common/nyxd-scraper-psql/.sqlx/query-bc7795e58ce71893c3f32a19db8e77b7bc0a1af315ffd42c3e68156d6e4ace70.json new file mode 100644 index 00000000000..caca484b94d --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-bc7795e58ce71893c3f32a19db8e77b7bc0a1af315ffd42c3e68156d6e4ace70.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT COUNT(*) as count FROM pre_commit\n WHERE\n validator_address = $1\n AND height >= $2\n AND height <= $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bc7795e58ce71893c3f32a19db8e77b7bc0a1af315ffd42c3e68156d6e4ace70" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-be43d4873911deca784b7be0531ab7bd82ecd68041aa932a56c8ce09623251e4.json b/common/nyxd-scraper-psql/.sqlx/query-be43d4873911deca784b7be0531ab7bd82ecd68041aa932a56c8ce09623251e4.json new file mode 100644 index 00000000000..f1df706371b --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-be43d4873911deca784b7be0531ab7bd82ecd68041aa932a56c8ce09623251e4.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT * FROM validator\n WHERE EXISTS (\n SELECT 1 FROM pre_commit\n WHERE height = $1\n AND pre_commit.validator_address = validator.consensus_address\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "consensus_address", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "consensus_pubkey", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "be43d4873911deca784b7be0531ab7bd82ecd68041aa932a56c8ce09623251e4" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d.json b/common/nyxd-scraper-psql/.sqlx/query-c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d.json new file mode 100644 index 00000000000..15c137540ce --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT last_processed_height FROM metadata\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_processed_height", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef.json b/common/nyxd-scraper-psql/.sqlx/query-cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef.json new file mode 100644 index 00000000000..b5fd43d8d72 --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO message(transaction_hash, index, type, value, involved_accounts_addresses, height)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (transaction_hash, index) DO UPDATE\n SET height = excluded.height,\n type = excluded.type,\n value = excluded.value,\n involved_accounts_addresses = excluded.involved_accounts_addresses\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Text", + "Json", + "TextArray", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-cdba9b267f143c8a8c6c3d6ed713cf00236490b86779559d84740ec18bcfa3a9.json b/common/nyxd-scraper-psql/.sqlx/query-cdba9b267f143c8a8c6c3d6ed713cf00236490b86779559d84740ec18bcfa3a9.json new file mode 100644 index 00000000000..2ae11a8fbb4 --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-cdba9b267f143c8a8c6c3d6ed713cf00236490b86779559d84740ec18bcfa3a9.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM block WHERE height < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cdba9b267f143c8a8c6c3d6ed713cf00236490b86779559d84740ec18bcfa3a9" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-d89558c37c51e8e6b1e6a9d5a2b13d0598fd856aa019a0cbbae12d7cafb4672f.json b/common/nyxd-scraper-psql/.sqlx/query-d89558c37c51e8e6b1e6a9d5a2b13d0598fd856aa019a0cbbae12d7cafb4672f.json new file mode 100644 index 00000000000..1970629169b --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-d89558c37c51e8e6b1e6a9d5a2b13d0598fd856aa019a0cbbae12d7cafb4672f.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM transaction WHERE height < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "d89558c37c51e8e6b1e6a9d5a2b13d0598fd856aa019a0cbbae12d7cafb4672f" +} diff --git a/common/nyxd-scraper-psql/Cargo.toml b/common/nyxd-scraper-psql/Cargo.toml new file mode 100644 index 00000000000..0c2253cf9ba --- /dev/null +++ b/common/nyxd-scraper-psql/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "nyxd-scraper-psql" +version = "0.1.0" +authors.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +readme.workspace = true + +[dependencies] +async-trait = { workspace = true } +base64 = { workspace = true } +itertools = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres", "macros", "migrate", "time"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tracing.workspace = true + +nyxd-scraper-shared = { path = "../nyxd-scraper-shared" } + +# temp due to cosmrs redefinitions for serde +cosmrs = { workspace = true } + +[build-dependencies] +sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres", "macros", "migrate"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } + +[lints] +workspace = true diff --git a/common/nyxd-scraper-psql/Makefile b/common/nyxd-scraper-psql/Makefile new file mode 100644 index 00000000000..67eda86cd1f --- /dev/null +++ b/common/nyxd-scraper-psql/Makefile @@ -0,0 +1,102 @@ +# Makefile for nyxd-scraper-psql database management + +# --- Configuration --- +TEST_DATABASE_URL := postgres://testuser:testpass@localhost:5433/nyxd_scraper_test + +# Docker compose service names +DB_SERVICE_NAME := postgres-test +DB_CONTAINER_NAME := nyxd_scraper_psql_test + +# Default target +.PHONY: default +default: help + +# --- Main Targets --- +.PHONY: prepare-pg +prepare-pg: test-db-up test-db-wait test-db-migrate test-db-prepare test-db-down ## Setup PostgreSQL and prepare SQLx offline cache + +.PHONY: test-db +test-db: test-db-up test-db-wait test-db-migrate test-db-run test-db-down ## Run tests with PostgreSQL database + +.PHONY: dev-db +dev-db: test-db-up test-db-wait test-db-migrate ## Start PostgreSQL for development (keeps running) + @echo "PostgreSQL is running on port 5433" + @echo "Connection string: $(TEST_DATABASE_URL)" + +# --- Docker Compose Targets --- +.PHONY: test-db-up +test-db-up: ## Start the PostgreSQL test database in the background + @echo "Starting PostgreSQL test database..." + docker compose up -d $(DB_SERVICE_NAME) + +.PHONY: test-db-wait +test-db-wait: ## Wait for the PostgreSQL database to be healthy + @echo "Waiting for PostgreSQL database..." + @while ! docker inspect --format='{{.State.Health.Status}}' $(DB_CONTAINER_NAME) 2>/dev/null | grep -q 'healthy'; do \ + echo -n "."; \ + sleep 1; \ + done; \ + echo " Database is healthy!" + +.PHONY: test-db-down +test-db-down: ## Stop and remove the test database + @echo "Stopping PostgreSQL test database..." + docker compose down + +# --- SQLx Targets --- +.PHONY: test-db-migrate +test-db-migrate: ## Run database migrations against PostgreSQL + @echo "Running PostgreSQL migrations..." + DATABASE_URL="$(TEST_DATABASE_URL)" sqlx migrate run --source sql_migrations + +.PHONY: test-db-prepare +test-db-prepare: ## Run sqlx prepare for compile-time query verification + @echo "Running sqlx prepare for PostgreSQL..." + DATABASE_URL="$(TEST_DATABASE_URL)" cargo sqlx prepare + +# --- Build and Test Targets --- +.PHONY: test-db-run +test-db-run: ## Run tests with PostgreSQL feature + @echo "Running tests with PostgreSQL..." + DATABASE_URL="$(TEST_DATABASE_URL)" cargo test --features pg --no-default-features + +.PHONY: build-pg +build-pg: ## Build with PostgreSQL feature + @echo "Building with PostgreSQL feature..." + cargo build + +.PHONY: check-pg +check-pg: ## Check code with PostgreSQL feature + @echo "Checking code with PostgreSQL feature..." + cargo check + +.PHONY: clippy +clippy: clippy-pg + +.PHONY: clippy-pg +clippy-pg: ## Run clippy with PostgreSQL feature + @echo "Running clippy with PostgreSQL feature..." + cargo clippy -- -D warnings + +# --- Cleanup Targets --- +.PHONY: clean +clean: ## Clean build artifacts and SQLx cache + cargo clean + rm -rf .sqlx + +.PHONY: clean-db +clean-db: test-db-down ## Stop database and clean volumes + docker volume rm -f nym-node-status-api_postgres_test_data 2>/dev/null || true + +# --- Utility Targets --- +.PHONY: sqlx-cli +sqlx-cli: ## Install sqlx-cli if not already installed + @command -v sqlx >/dev/null 2>&1 || cargo install sqlx-cli --features postgres + +.PHONY: psql +psql: ## Connect to the running PostgreSQL database with psql + @docker exec -it $(DB_CONTAINER_NAME) psql -U testuser -d nyxd_scraper_test + +.PHONY: help +help: ## Show help for Makefile targets + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' \ No newline at end of file diff --git a/common/nyxd-scraper-psql/README.md b/common/nyxd-scraper-psql/README.md new file mode 100644 index 00000000000..bfa3e04fe21 --- /dev/null +++ b/common/nyxd-scraper-psql/README.md @@ -0,0 +1,79 @@ +## Quick Start with PostgreSQL + +### 1. Install Prerequisites + +```bash +# Install sqlx-cli if not already installed +make sqlx-cli +``` + +### 2. Prepare PostgreSQL for Development + +```bash +# This will: +# - Start PostgreSQL in Docker +# - Run migrations +# - Generate SQLx offline query cache +# - Stop the database +make prepare-pg +``` + +### 3. Build with PostgreSQL + +```bash +# Build with PostgreSQL feature +make build-pg + +# Or manually: +cargo build +``` + +### 4. Run with PostgreSQL + +```bash +# Start PostgreSQL for development (keeps running) +make dev-db + +# In another terminal, run the application +DATABASE_URL=postgres://testuser:testpass@localhost:5433/nym_node_status_api_test \ +cargo run +``` + +## Makefile Targets + +```bash +make help # Show all available targets +make prepare-pg # Setup PostgreSQL and prepare SQLx cache +make dev-db # Start PostgreSQL for development +make test-db # Run tests with PostgreSQL +make build-pg # Build with PostgreSQL +make psql # Connect to running PostgreSQL +make clean # Clean build artifacts +make clean-db # Stop database and clean volumes +``` + +## Environment Variables + +See `.env.example` for all configuration options. Key variable: + +```bash +# For PostgreSQL: +DATABASE_URL=postgres://testuser:testpass@localhost:5433/nym_node_status_api_test +``` + +## Troubleshooting + +### SQLx Offline Mode + +If you see "no cached data for this query" errors: + +1. Ensure PostgreSQL is running: `make dev-db` +2. Run: `make test-db-prepare` + +### Connection Refused + +If you see "Connection refused" errors: + +1. Check Docker is running: `docker ps` +2. Check PostgreSQL container: `docker ps | grep nym_node_status_api_postgres_test` +3. Restart database: `make test-db-down && make dev-db` \ No newline at end of file diff --git a/common/nyxd-scraper-psql/build.rs b/common/nyxd-scraper-psql/build.rs new file mode 100644 index 00000000000..2903970a552 --- /dev/null +++ b/common/nyxd-scraper-psql/build.rs @@ -0,0 +1,8 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +fn main() { + if let Ok(database_url) = std::env::var("DATABASE_URL") { + println!("cargo::rustc-env=DATABASE_URL={database_url}"); + } +} diff --git a/common/nyxd-scraper-psql/docker-compose.yml b/common/nyxd-scraper-psql/docker-compose.yml new file mode 100644 index 00000000000..3965792b45e --- /dev/null +++ b/common/nyxd-scraper-psql/docker-compose.yml @@ -0,0 +1,21 @@ +services: + postgres-test: + image: postgres:16-alpine + container_name: nyxd_scraper_psql_test + environment: + POSTGRES_DB: nyxd_scraper_test + POSTGRES_USER: testuser + POSTGRES_PASSWORD: testpass + ports: + - '5433:5432' # Map to 5433 to avoid conflicts with default PostgreSQL + healthcheck: + test: [ 'CMD-SHELL', 'pg_isready -U testuser -d nyxd_scraper_test' ] + interval: 5s + timeout: 5s + retries: 5 + # Optional: Add volume for persistent data during development + # volumes: + # - postgres_test_data:/var/lib/postgresql/data + +# volumes: +# postgres_test_data: \ No newline at end of file diff --git a/common/nyxd-scraper/sql_migrations/01_metadata.sql b/common/nyxd-scraper-psql/sql_migrations/01_metadata.sql similarity index 100% rename from common/nyxd-scraper/sql_migrations/01_metadata.sql rename to common/nyxd-scraper-psql/sql_migrations/01_metadata.sql diff --git a/common/nyxd-scraper-psql/sql_migrations/02_cosmos.sql b/common/nyxd-scraper-psql/sql_migrations/02_cosmos.sql new file mode 100644 index 00000000000..f48da6e9030 --- /dev/null +++ b/common/nyxd-scraper-psql/sql_migrations/02_cosmos.sql @@ -0,0 +1,127 @@ +CREATE TABLE validator +( + consensus_address TEXT NOT NULL PRIMARY KEY, /* Validator consensus address */ + consensus_pubkey TEXT NOT NULL UNIQUE /* Validator consensus public key */ +); + +CREATE TABLE pre_commit +( + validator_address TEXT NOT NULL REFERENCES validator (consensus_address), + height BIGINT NOT NULL, + timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL, + voting_power BIGINT NOT NULL, + proposer_priority BIGINT NOT NULL, + UNIQUE (validator_address, timestamp) +); +CREATE INDEX pre_commit_validator_address_index ON pre_commit (validator_address); +CREATE INDEX pre_commit_height_index ON pre_commit (height); + +CREATE TABLE block +( + height BIGINT UNIQUE PRIMARY KEY, + hash TEXT NOT NULL UNIQUE, + num_txs INTEGER DEFAULT 0, + total_gas BIGINT DEFAULT 0, + proposer_address TEXT REFERENCES validator (consensus_address), + timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL +); +CREATE INDEX block_height_index ON block (height); +CREATE INDEX block_hash_index ON block (hash); +CREATE INDEX block_proposer_address_index ON block (proposer_address); +ALTER TABLE block + SET ( + autovacuum_vacuum_scale_factor = 0, + autovacuum_analyze_scale_factor = 0, + autovacuum_vacuum_threshold = 10000, + autovacuum_analyze_threshold = 10000 + ); + +CREATE TABLE transaction +( + hash TEXT NOT NULL, + height BIGINT NOT NULL REFERENCES block (height), + "index" INTEGER NOT NULL, -- <<<=== not present in original bdjuno table, but it's quite useful + success BOOLEAN NOT NULL, + + /* Body */ + messages JSON NOT NULL DEFAULT '[]'::JSON, + memo TEXT, + signatures TEXT[] NOT NULL, + + /* AuthInfo */ + signer_infos JSONB NOT NULL DEFAULT '[]'::JSONB, + fee JSONB NOT NULL DEFAULT '{}'::JSONB, + + /* Tx response */ + gas_wanted BIGINT DEFAULT 0, + gas_used BIGINT DEFAULT 0, + raw_log TEXT, + logs JSONB, + + CONSTRAINT unique_tx UNIQUE (hash) +); +CREATE INDEX transaction_hash_index ON transaction (hash); +CREATE INDEX transaction_height_index ON transaction (height); + +CREATE TABLE message_type +( + type TEXT NOT NULL UNIQUE, + module TEXT NOT NULL, + label TEXT NOT NULL, + height BIGINT NOT NULL +); +CREATE INDEX message_type_module_index ON message_type (module); +CREATE INDEX message_type_type_index ON message_type (type); + +CREATE TABLE message +( + transaction_hash TEXT NOT NULL, + index BIGINT NOT NULL, + type TEXT NOT NULL REFERENCES message_type (type), + value JSON NOT NULL, + involved_accounts_addresses TEXT[] NOT NULL, + + height BIGINT NOT NULL, + FOREIGN KEY (transaction_hash) REFERENCES transaction (hash), + CONSTRAINT unique_message_per_tx UNIQUE (transaction_hash, index) +); +CREATE INDEX message_transaction_hash_index ON message (transaction_hash); +CREATE INDEX message_type_index ON message (type); +CREATE INDEX message_involved_accounts_index ON message USING GIN (involved_accounts_addresses); + +/** + * This function is used to find all the utils that involve any of the given addresses and have + * type that is one of the specified types. + */ +CREATE FUNCTION messages_by_address( + addresses TEXT[], + types TEXT[], + "limit" BIGINT = 100, + "offset" BIGINT = 0) + RETURNS SETOF message AS +$$ +SELECT * +FROM message +WHERE (cardinality(types) = 0 OR type = ANY (types)) + AND addresses && involved_accounts_addresses +ORDER BY height DESC +LIMIT "limit" OFFSET "offset" +$$ LANGUAGE sql STABLE; + +CREATE FUNCTION messages_by_type( + types text[], + "limit" bigint DEFAULT 100, + "offset" bigint DEFAULT 0) + RETURNS SETOF message AS +$$ +SELECT * +FROM message +WHERE (cardinality(types) = 0 OR type = ANY (types)) +ORDER BY height DESC +LIMIT "limit" OFFSET "offset" +$$ LANGUAGE sql STABLE; + +CREATE TABLE pruning +( + last_pruned_height BIGINT NOT NULL +); \ No newline at end of file diff --git a/common/nyxd-scraper-psql/src/error.rs b/common/nyxd-scraper-psql/src/error.rs new file mode 100644 index 00000000000..9e94af39edd --- /dev/null +++ b/common/nyxd-scraper-psql/src/error.rs @@ -0,0 +1,43 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use nyxd_scraper_shared::helpers::MalformedDataError; +use nyxd_scraper_shared::storage::NyxdScraperStorageError; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum PostgresScraperError { + #[error("experienced internal database error: {0}")] + InternalDatabaseError(#[from] sqlx::error::Error), + + #[error("failed to perform startup SQL migration: {0}")] + StartupMigrationFailure(#[from] sqlx::migrate::MigrateError), + + #[error("failed to begin storage tx: {source}")] + StorageTxBeginFailure { + #[source] + source: sqlx::error::Error, + }, + + #[error("failed to commit storage tx: {source}")] + StorageTxCommitFailure { + #[source] + source: sqlx::error::Error, + }, + + #[error(transparent)] + MalformedData(#[from] MalformedDataError), + + // TOOD: add struct name + #[error("json serialisation failure: {source}")] + SerialisationFailure { + #[from] + source: serde_json::Error, + }, +} + +impl From for NyxdScraperStorageError { + fn from(err: PostgresScraperError) -> Self { + NyxdScraperStorageError::new(err) + } +} diff --git a/common/nyxd-scraper-psql/src/lib.rs b/common/nyxd-scraper-psql/src/lib.rs new file mode 100644 index 00000000000..85fcbb46efa --- /dev/null +++ b/common/nyxd-scraper-psql/src/lib.rs @@ -0,0 +1,21 @@ +// Copyright 2023 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::storage::block_storage::PostgresScraperStorage; +use nyxd_scraper_shared::NyxdScraper; + +pub use nyxd_scraper_shared::constants; +pub use nyxd_scraper_shared::error::ScraperError; +pub use nyxd_scraper_shared::{ + BlockModule, MsgModule, NyxdScraperTransaction, ParsedTransactionResponse, PruningOptions, + PruningStrategy, StartingBlockOpts, TxModule, +}; +pub use storage::models; + +pub mod error; +pub mod storage; + +pub type PostgresNyxdScraper = NyxdScraper; + +// TODO: for now just use exactly the same config +pub use nyxd_scraper_shared::Config; diff --git a/common/nyxd-scraper-psql/src/storage/block_storage.rs b/common/nyxd-scraper-psql/src/storage/block_storage.rs new file mode 100644 index 00000000000..b16f6c58916 --- /dev/null +++ b/common/nyxd-scraper-psql/src/storage/block_storage.rs @@ -0,0 +1,246 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::error::PostgresScraperError; +use crate::models::{CommitSignature, Validator}; +use crate::storage::manager::{ + prune_blocks, prune_messages, prune_pre_commits, prune_transactions, update_last_pruned, + StorageManager, +}; +use crate::storage::transaction::PostgresStorageTransaction; +use async_trait::async_trait; +use nyxd_scraper_shared::storage::helpers::log_db_operation_time; +use nyxd_scraper_shared::storage::{NyxdScraperStorage, NyxdScraperStorageError}; +use nyxd_scraper_shared::{default_message_registry, MessageRegistry}; +use sqlx::types::time::{OffsetDateTime, PrimitiveDateTime}; +use tokio::time::Instant; +use tracing::{debug, error, info, instrument}; + +#[derive(Clone)] +pub struct PostgresScraperStorage { + pub(crate) manager: StorageManager, + + // kinda like very limited cosmos sdk codec + pub(crate) message_registry: MessageRegistry, +} + +impl PostgresScraperStorage { + #[instrument] + pub async fn init(connection_string: &str) -> Result { + debug!("initialising scraper database with '{connection_string}'",); + + let connection_pool = match sqlx::PgPool::connect(connection_string).await { + Ok(db) => db, + Err(err) => { + error!("Failed to connect to SQLx database: {err}"); + return Err(err.into()); + } + }; + + if let Err(err) = sqlx::migrate!("./sql_migrations") + .run(&connection_pool) + .await + { + error!("Failed to initialize SQLx database: {err}"); + return Err(err.into()); + } + + info!("Database migration finished!"); + + let manager = StorageManager { connection_pool }; + manager.set_initial_metadata().await?; + + let storage = PostgresScraperStorage { + manager, + message_registry: default_message_registry(), + }; + + Ok(storage) + } + + #[instrument(skip(self))] + pub async fn prune_storage( + &self, + oldest_to_keep: u32, + current_height: u32, + ) -> Result<(), PostgresScraperError> { + let start = Instant::now(); + + let mut tx = self.begin_processing_tx().await?; + + prune_messages(oldest_to_keep.into(), &mut **tx).await?; + prune_transactions(oldest_to_keep.into(), &mut **tx).await?; + prune_pre_commits(oldest_to_keep.into(), &mut **tx).await?; + prune_blocks(oldest_to_keep.into(), &mut **tx).await?; + update_last_pruned(current_height.into(), &mut **tx).await?; + + let commit_start = Instant::now(); + tx.inner + .commit() + .await + .map_err(|source| PostgresScraperError::StorageTxCommitFailure { source })?; + log_db_operation_time("committing pruning tx", commit_start); + + log_db_operation_time("pruning storage", start); + Ok(()) + } + + #[instrument(skip_all)] + pub async fn begin_processing_tx( + &self, + ) -> Result { + debug!("starting storage tx"); + self.manager + .connection_pool + .begin() + .await + .map(|inner| PostgresStorageTransaction { + inner, + registry: self.message_registry.clone(), + }) + .map_err(|source| PostgresScraperError::StorageTxBeginFailure { source }) + } + + pub async fn lowest_block_height(&self) -> Result, PostgresScraperError> { + Ok(self.manager.get_lowest_block().await?) + } + + pub async fn get_first_block_height_after( + &self, + time: OffsetDateTime, + ) -> Result, PostgresScraperError> { + let time = PrimitiveDateTime::new(time.date(), time.time()); + + Ok(self.manager.get_first_block_height_after(time).await?) + } + + pub async fn get_last_block_height_before( + &self, + time: OffsetDateTime, + ) -> Result, PostgresScraperError> { + let time = PrimitiveDateTime::new(time.date(), time.time()); + + Ok(self.manager.get_last_block_height_before(time).await?) + } + + pub async fn get_blocks_between( + &self, + start_time: OffsetDateTime, + end_time: OffsetDateTime, + ) -> Result { + let Some(block_start) = self.get_first_block_height_after(start_time).await? else { + return Ok(0); + }; + let Some(block_end) = self.get_last_block_height_before(end_time).await? else { + return Ok(0); + }; + + Ok(block_end - block_start) + } + + pub async fn get_signed_between( + &self, + consensus_address: &str, + start_height: i64, + end_height: i64, + ) -> Result { + Ok(self + .manager + .get_signed_between(consensus_address, start_height, end_height) + .await?) + } + + pub async fn get_signed_between_times( + &self, + consensus_address: &str, + start_time: OffsetDateTime, + end_time: OffsetDateTime, + ) -> Result { + let Some(block_start) = self.get_first_block_height_after(start_time).await? else { + return Ok(0); + }; + let Some(block_end) = self.get_last_block_height_before(end_time).await? else { + return Ok(0); + }; + + self.get_signed_between(consensus_address, block_start, block_end) + .await + } + + pub async fn get_precommit( + &self, + consensus_address: &str, + height: i64, + ) -> Result, PostgresScraperError> { + Ok(self + .manager + .get_precommit(consensus_address, height) + .await?) + } + + pub async fn get_block_signers( + &self, + height: i64, + ) -> Result, PostgresScraperError> { + Ok(self.manager.get_block_validators(height).await?) + } + + pub async fn get_all_known_validators(&self) -> Result, PostgresScraperError> { + Ok(self.manager.get_validators().await?) + } + + pub async fn get_last_processed_height(&self) -> Result { + Ok(self.manager.get_last_processed_height().await?) + } + + pub async fn get_pruned_height(&self) -> Result { + Ok(self.manager.get_pruned_height().await?) + } +} + +#[async_trait] +impl NyxdScraperStorage for PostgresScraperStorage { + type StorageTransaction = PostgresStorageTransaction; + + async fn initialise(storage: &str) -> Result { + PostgresScraperStorage::init(storage) + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn begin_processing_tx( + &self, + ) -> Result { + self.begin_processing_tx() + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn get_last_processed_height(&self) -> Result { + self.get_last_processed_height() + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn get_pruned_height(&self) -> Result { + self.get_pruned_height() + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn lowest_block_height(&self) -> Result, NyxdScraperStorageError> { + self.lowest_block_height() + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn prune_storage( + &self, + oldest_to_keep: u32, + current_height: u32, + ) -> Result<(), NyxdScraperStorageError> { + self.prune_storage(oldest_to_keep, current_height) + .await + .map_err(NyxdScraperStorageError::from) + } +} diff --git a/common/nyxd-scraper-psql/src/storage/helpers.rs b/common/nyxd-scraper-psql/src/storage/helpers.rs new file mode 100644 index 00000000000..8d56ee72f78 --- /dev/null +++ b/common/nyxd-scraper-psql/src/storage/helpers.rs @@ -0,0 +1,25 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use cosmrs::AccountId; +use itertools::Itertools; +use nyxd_scraper_shared::ParsedTransactionResponse; +use std::str::FromStr; + +// replicate behaviour of `CosmosMessageAddressesParser` from juno +pub(crate) fn parse_addresses_from_events(tx: &ParsedTransactionResponse) -> Vec { + let mut addresses: Vec = Vec::new(); + for event in &tx.tx_result.events { + for attribute in &event.attributes { + let Ok(value) = attribute.value_str() else { + continue; + }; + + // Try parsing the address as an account address + if let Ok(address) = AccountId::from_str(value) { + addresses.push(address.to_string()); + } + } + } + addresses.into_iter().unique().collect() +} diff --git a/common/nyxd-scraper-psql/src/storage/manager.rs b/common/nyxd-scraper-psql/src/storage/manager.rs new file mode 100644 index 00000000000..165dfd42f45 --- /dev/null +++ b/common/nyxd-scraper-psql/src/storage/manager.rs @@ -0,0 +1,538 @@ +// Copyright 2023 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::storage::models::{CommitSignature, Validator}; +use nyxd_scraper_shared::storage::helpers::log_db_operation_time; +use sqlx::types::time::PrimitiveDateTime; +use sqlx::types::{Json, JsonValue}; +use sqlx::{Executor, Postgres}; +use tokio::time::Instant; +use tracing::{instrument, trace}; + +#[derive(Clone)] +pub(crate) struct StorageManager { + pub(crate) connection_pool: sqlx::Pool, +} + +impl StorageManager { + pub(crate) async fn set_initial_metadata(&self) -> Result<(), sqlx::Error> { + if sqlx::query("SELECT * from metadata") + .fetch_optional(&self.connection_pool) + .await? + .is_none() + { + sqlx::query("INSERT INTO metadata (id, last_processed_height) VALUES (0, 0)") + .execute(&self.connection_pool) + .await?; + } + Ok(()) + } + + pub(crate) async fn get_lowest_block(&self) -> Result, sqlx::Error> { + trace!("get_lowest_block"); + let start = Instant::now(); + + let maybe_record = sqlx::query!( + r#" + SELECT height + FROM block + ORDER BY height ASC + LIMIT 1 + "#, + ) + .fetch_optional(&self.connection_pool) + .await?; + log_db_operation_time("get_lowest_block", start); + + Ok(maybe_record.map(|x| x.height)) + } + + pub(crate) async fn get_first_block_height_after( + &self, + time: PrimitiveDateTime, + ) -> Result, sqlx::Error> { + trace!("get_first_block_height_after"); + let start = Instant::now(); + + let maybe_record = sqlx::query!( + r#" + SELECT height + FROM block + WHERE timestamp > $1 + ORDER BY timestamp + LIMIT 1 + "#, + time + ) + .fetch_optional(&self.connection_pool) + .await?; + log_db_operation_time("get_first_block_height_after", start); + + Ok(maybe_record.map(|x| x.height)) + } + + pub(crate) async fn get_last_block_height_before( + &self, + time: PrimitiveDateTime, + ) -> Result, sqlx::Error> { + trace!("get_last_block_height_before"); + let start = Instant::now(); + + let maybe_record = sqlx::query!( + r#" + SELECT height + FROM block + WHERE timestamp < $1 + ORDER BY timestamp DESC + LIMIT 1 + "#, + time + ) + .fetch_optional(&self.connection_pool) + .await?; + log_db_operation_time("get_last_block_height_before", start); + + Ok(maybe_record.map(|x| x.height)) + } + + pub(crate) async fn get_signed_between( + &self, + consensus_address: &str, + start_height: i64, + end_height: i64, + ) -> Result { + trace!("get_signed_between"); + let start = Instant::now(); + + let count = sqlx::query!( + r#" + SELECT COUNT(*) as count FROM pre_commit + WHERE + validator_address = $1 + AND height >= $2 + AND height <= $3 + "#, + consensus_address, + start_height, + end_height + ) + .fetch_one(&self.connection_pool) + .await? + .count; + log_db_operation_time("get_signed_between", start); + + Ok(count.unwrap_or(0)) + } + + pub(crate) async fn get_precommit( + &self, + consensus_address: &str, + height: i64, + ) -> Result, sqlx::Error> { + trace!("get_precommit"); + let start = Instant::now(); + + let res = sqlx::query_as( + r#" + SELECT * FROM pre_commit + WHERE validator_address = $1 + AND height = $2 + "#, + ) + .bind(consensus_address) + .bind(height) + .fetch_optional(&self.connection_pool) + .await?; + log_db_operation_time("get_precommit", start); + + Ok(res) + } + + pub(crate) async fn get_block_validators( + &self, + height: i64, + ) -> Result, sqlx::Error> { + trace!("get_block_validators"); + let start = Instant::now(); + + let res = sqlx::query_as!( + Validator, + r#" + SELECT * FROM validator + WHERE EXISTS ( + SELECT 1 FROM pre_commit + WHERE height = $1 + AND pre_commit.validator_address = validator.consensus_address + ) + "#, + height + ) + .fetch_all(&self.connection_pool) + .await?; + log_db_operation_time("get_block_validators", start); + + Ok(res) + } + + pub(crate) async fn get_validators(&self) -> Result, sqlx::Error> { + trace!("get_validators"); + let start = Instant::now(); + + let res = sqlx::query_as("SELECT * FROM validator") + .fetch_all(&self.connection_pool) + .await?; + log_db_operation_time("get_validators", start); + + Ok(res) + } + + pub(crate) async fn get_last_processed_height(&self) -> Result { + trace!("get_last_processed_height"); + let start = Instant::now(); + + let maybe_record = sqlx::query!( + r#" + SELECT last_processed_height FROM metadata + "# + ) + .fetch_optional(&self.connection_pool) + .await?; + log_db_operation_time("get_last_processed_height", start); + + if let Some(row) = maybe_record { + Ok(row.last_processed_height as i64) + } else { + Ok(-1) + } + } + + pub(crate) async fn get_pruned_height(&self) -> Result { + trace!("get_pruned_height"); + let start = Instant::now(); + + let maybe_record = sqlx::query!( + r#" + SELECT last_pruned_height FROM pruning + "# + ) + .fetch_optional(&self.connection_pool) + .await?; + + log_db_operation_time("get_pruned_height", start); + + if let Some(row) = maybe_record { + Ok(row.last_pruned_height) + } else { + Ok(-1) + } + } +} + +// make those generic over executor so that they could be performed over connection pool and a tx + +#[instrument(skip(executor))] +pub(crate) async fn insert_validator<'a, E>( + consensus_address: String, + consensus_pubkey: String, + executor: E, +) -> Result<(), sqlx::Error> +where + E: Executor<'a, Database = Postgres>, +{ + trace!("insert_validator"); + let start = Instant::now(); + + sqlx::query!( + r#" + INSERT INTO validator (consensus_address, consensus_pubkey) + VALUES ($1, $2) + ON CONFLICT DO NOTHING + "#, + consensus_address, + consensus_pubkey + ) + .execute(executor) + .await?; + log_db_operation_time("insert_validator", start); + + Ok(()) +} + +#[instrument(skip(executor))] +pub(crate) async fn insert_block<'a, E>( + height: i64, + hash: String, + num_txs: i32, + total_gas: i64, + proposer_address: String, + timestamp: PrimitiveDateTime, + executor: E, +) -> Result<(), sqlx::Error> +where + E: Executor<'a, Database = Postgres>, +{ + trace!("insert_block"); + let start = Instant::now(); + + sqlx::query!( + r#" + INSERT INTO block (height, hash, num_txs, total_gas, proposer_address, timestamp) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT DO NOTHING + "#, + height, + hash, + num_txs, + total_gas, + proposer_address, + timestamp + ) + .execute(executor) + .await?; + log_db_operation_time("insert_block", start); + + Ok(()) +} + +#[instrument(skip(executor))] +pub(crate) async fn insert_precommit<'a, E>( + validator_address: String, + height: i64, + timestamp: PrimitiveDateTime, + voting_power: i64, + proposer_priority: i64, + executor: E, +) -> Result<(), sqlx::Error> +where + E: Executor<'a, Database = Postgres>, +{ + trace!("insert_precommit"); + let start = Instant::now(); + + sqlx::query!( + r#" + INSERT INTO pre_commit (validator_address, height, timestamp, voting_power, proposer_priority) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (validator_address, timestamp) DO NOTHING + "#, + validator_address, + height, + timestamp, + voting_power, + proposer_priority + ) + .execute(executor) + .await?; + log_db_operation_time("insert_precommit", start); + + Ok(()) +} + +#[instrument(skip(executor))] +#[allow(clippy::too_many_arguments)] +pub(crate) async fn insert_transaction<'a, E>( + hash: String, + height: i64, + index: i32, + success: bool, + messages: JsonValue, + memo: String, + signatures: Vec, + signer_infos: JsonValue, + fee: JsonValue, + gas_wanted: i64, + gas_used: i64, + raw_log: String, + logs: JsonValue, + executor: E, +) -> Result<(), sqlx::Error> +where + E: Executor<'a, Database = Postgres>, +{ + trace!("insert_transaction"); + let start = Instant::now(); + + sqlx::query!( + r#" + INSERT INTO transaction + (hash, height, index, success, messages, memo, signatures, signer_infos, fee, gas_wanted, gas_used, raw_log, logs) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + ON CONFLICT (hash) DO UPDATE + SET height = excluded.height, + index = excluded.index, + success = excluded.success, + messages = excluded.messages, + memo = excluded.memo, + signatures = excluded.signatures, + signer_infos = excluded.signer_infos, + fee = excluded.fee, + gas_wanted = excluded.gas_wanted, + gas_used = excluded.gas_used, + raw_log = excluded.raw_log, + logs = excluded.logs + "#, + hash, + height, + index, + success, + messages, + memo, + &signatures, + signer_infos, + fee, + gas_wanted, + gas_used, + raw_log, + logs, + ) + .execute(executor) + .await?; + log_db_operation_time("insert_transaction", start); + + Ok(()) +} + +#[instrument(skip(executor))] +pub(crate) async fn insert_message<'a, E>( + transaction_hash: String, + index: i64, + typ: String, + value: JsonValue, + involved_account_addresses: Vec, + height: i64, + executor: E, +) -> Result<(), sqlx::Error> +where + E: Executor<'a, Database = Postgres>, +{ + trace!("insert_message"); + let start = Instant::now(); + + sqlx::query!( + r#" + INSERT INTO message(transaction_hash, index, type, value, involved_accounts_addresses, height) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (transaction_hash, index) DO UPDATE + SET height = excluded.height, + type = excluded.type, + value = excluded.value, + involved_accounts_addresses = excluded.involved_accounts_addresses + "#, + transaction_hash, + index, + typ, + value, + &involved_account_addresses, + height + ) + .execute(executor) + .await?; + log_db_operation_time("insert_message", start); + + Ok(()) +} + +#[instrument(skip(executor))] +pub(crate) async fn update_last_processed<'a, E>( + height: i32, + executor: E, +) -> Result<(), sqlx::Error> +where + E: Executor<'a, Database = Postgres>, +{ + trace!("update_last_processed"); + let start = Instant::now(); + + sqlx::query!( + "UPDATE metadata SET last_processed_height = GREATEST(last_processed_height, $1)", + height + ) + .execute(executor) + .await?; + log_db_operation_time("update_last_processed", start); + + Ok(()) +} + +#[instrument(skip(executor))] +pub(crate) async fn update_last_pruned<'a, E>(height: i64, executor: E) -> Result<(), sqlx::Error> +where + E: Executor<'a, Database = Postgres>, +{ + trace!("update_last_pruned"); + let start = Instant::now(); + + sqlx::query!("UPDATE pruning SET last_pruned_height = $1", height) + .execute(executor) + .await?; + log_db_operation_time("update_last_pruned", start); + + Ok(()) +} + +pub(crate) async fn prune_blocks<'a, E>(oldest_to_keep: i64, executor: E) -> Result<(), sqlx::Error> +where + E: Executor<'a, Database = Postgres>, +{ + trace!("prune_blocks"); + let start = Instant::now(); + + sqlx::query!("DELETE FROM block WHERE height < $1", oldest_to_keep) + .execute(executor) + .await?; + log_db_operation_time("prune_blocks", start); + + Ok(()) +} + +pub(crate) async fn prune_pre_commits<'a, E>( + oldest_to_keep: i64, + executor: E, +) -> Result<(), sqlx::Error> +where + E: Executor<'a, Database = Postgres>, +{ + trace!("prune_pre_commits"); + let start = Instant::now(); + + sqlx::query!("DELETE FROM pre_commit WHERE height < $1", oldest_to_keep) + .execute(executor) + .await?; + log_db_operation_time("prune_pre_commits", start); + + Ok(()) +} + +pub(crate) async fn prune_transactions<'a, E>( + oldest_to_keep: i64, + executor: E, +) -> Result<(), sqlx::Error> +where + E: Executor<'a, Database = Postgres>, +{ + trace!("prune_transactions"); + let start = Instant::now(); + sqlx::query!("DELETE FROM transaction WHERE height < $1", oldest_to_keep) + .execute(executor) + .await?; + log_db_operation_time("prune_transactions", start); + + Ok(()) +} + +pub(crate) async fn prune_messages<'a, E>( + oldest_to_keep: i64, + executor: E, +) -> Result<(), sqlx::Error> +where + E: Executor<'a, Database = Postgres>, +{ + trace!("prune_messages"); + let start = Instant::now(); + sqlx::query!("DELETE FROM message WHERE height < $1", oldest_to_keep) + .execute(executor) + .await?; + log_db_operation_time("prune_messages", start); + + Ok(()) +} diff --git a/common/nyxd-scraper-psql/src/storage/mod.rs b/common/nyxd-scraper-psql/src/storage/mod.rs new file mode 100644 index 00000000000..091f5c0f2e4 --- /dev/null +++ b/common/nyxd-scraper-psql/src/storage/mod.rs @@ -0,0 +1,8 @@ +// Copyright 2023 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +pub mod block_storage; +mod helpers; +mod manager; +pub mod models; +pub mod transaction; diff --git a/common/nyxd-scraper/src/storage/models.rs b/common/nyxd-scraper-psql/src/storage/models.rs similarity index 100% rename from common/nyxd-scraper/src/storage/models.rs rename to common/nyxd-scraper-psql/src/storage/models.rs diff --git a/common/nyxd-scraper-psql/src/storage/transaction.rs b/common/nyxd-scraper-psql/src/storage/transaction.rs new file mode 100644 index 00000000000..2baa2c81308 --- /dev/null +++ b/common/nyxd-scraper-psql/src/storage/transaction.rs @@ -0,0 +1,291 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::error::PostgresScraperError; +use crate::storage::helpers::{parse_addresses_from_events, PlaceholderStruct}; +use crate::storage::manager::{ + insert_block, insert_message, insert_precommit, insert_transaction, insert_validator, +}; +use async_trait::async_trait; +use base64::engine::general_purpose; +use base64::Engine as _; +use cosmrs::proto; +use nyxd_scraper_shared::helpers::{ + validator_consensus_address, validator_info, validator_pubkey_to_bech32, +}; +use nyxd_scraper_shared::storage::validators::Response; +use nyxd_scraper_shared::storage::{ + validators, Block, Commit, CommitSig, NyxdScraperStorageError, NyxdScraperTransaction, +}; +use nyxd_scraper_shared::{Any, MessageRegistry, ParsedTransactionResponse}; +use serde_json::json; +use sqlx::types::time::{OffsetDateTime, PrimitiveDateTime}; +use sqlx::{Postgres, Transaction}; +use std::ops::{Deref, DerefMut}; +use tracing::{debug, trace, warn}; + +pub struct PostgresStorageTransaction { + pub(super) inner: Transaction<'static, Postgres>, + + pub(super) registry: MessageRegistry, +} + +impl Deref for PostgresStorageTransaction { + type Target = Transaction<'static, Postgres>; + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for PostgresStorageTransaction { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl PostgresStorageTransaction { + fn decode_or_skip(&self, msg: &Any) -> Option { + match self.registry.try_decode(msg) { + Ok(decoded) => Some(decoded), + Err(err) => { + warn!("{err}"); + None + } + } + } + + async fn persist_validators( + &mut self, + validators: &validators::Response, + ) -> Result<(), PostgresScraperError> { + debug!("persisting {} validators", validators.total); + for validator in &validators.validators { + let consensus_address = validator_consensus_address(validator.address)?; + let consensus_pubkey = validator_pubkey_to_bech32(validator.pub_key)?; + + insert_validator( + consensus_address.to_string(), + consensus_pubkey.to_string(), + self.inner.as_mut(), + ) + .await?; + } + + Ok(()) + } + + async fn persist_block_data( + &mut self, + block: &Block, + total_gas: i64, + ) -> Result<(), PostgresScraperError> { + let proposer_address = + validator_consensus_address(block.header.proposer_address)?.to_string(); + + let offset_datetime: OffsetDateTime = block.header.time.into(); + let time = PrimitiveDateTime::new(offset_datetime.date(), offset_datetime.time()); + + insert_block( + block.header.height.into(), + block.header.hash().to_string(), + block.data.len() as i32, + total_gas, + proposer_address, + time, + self.inner.as_mut(), + ) + .await?; + Ok(()) + } + + async fn persist_commits( + &mut self, + commits: &Commit, + validators: &validators::Response, + ) -> Result<(), PostgresScraperError> { + debug!("persisting up to {} commits", commits.signatures.len()); + let height: i64 = commits.height.into(); + + for commit_sig in &commits.signatures { + let (validator_id, timestamp, signature) = match commit_sig { + CommitSig::BlockIdFlagAbsent => { + trace!("absent signature"); + continue; + } + CommitSig::BlockIdFlagCommit { + validator_address, + timestamp, + signature, + } => (validator_address, timestamp, signature), + CommitSig::BlockIdFlagNil { + validator_address, + timestamp, + signature, + } => (validator_address, timestamp, signature), + }; + + let validator = validator_info(*validator_id, validators)?; + let validator_address = validator_consensus_address(*validator_id)?; + + if signature.is_none() { + warn!("empty signature for {validator_address} at height {height}"); + continue; + } + + let offset_datetime: OffsetDateTime = (*timestamp).into(); + let time = PrimitiveDateTime::new(offset_datetime.date(), offset_datetime.time()); + + insert_precommit( + validator_address.to_string(), + height, + time, + validator.power.into(), + validator.proposer_priority.value(), + self.inner.as_mut(), + ) + .await?; + } + + Ok(()) + } + + async fn persist_txs( + &mut self, + txs: &[ParsedTransactionResponse], + ) -> Result<(), PostgresScraperError> { + debug!("persisting {} txs", txs.len()); + + for chain_tx in txs { + // bdjuno style, base64 encode them + let signatures = chain_tx + .tx + .signatures + .iter() + .map(|sig| general_purpose::STANDARD.encode(sig)) + .collect(); + + let messages = chain_tx + .tx + .body + .messages + .iter() + .filter_map(|msg| self.decode_or_skip(msg)) + .collect::>(); + + let signer_infos = chain_tx + .tx + .auth_info + .signer_infos + .iter() + .map(|info| proto::cosmos::tx::v1beta1::SignerInfo::from(info.clone())) + .collect::>(); + + insert_transaction( + chain_tx.hash.to_string(), + chain_tx.height.into(), + chain_tx.index as i32, + chain_tx.tx_result.code.is_ok(), + serde_json::Value::Array(messages), + chain_tx.tx.body.memo.clone(), + signatures, + serde_json::to_value(signer_infos)?, + serde_json::to_value(&chain_tx.tx.auth_info.fee)?, + chain_tx.tx_result.gas_wanted, + chain_tx.tx_result.gas_used, + chain_tx.tx_result.log.clone(), + json!("null"), + self.inner.as_mut(), + ) + .await?; + } + + Ok(()) + } + + async fn persist_messages( + &mut self, + txs: &[ParsedTransactionResponse], + ) -> Result<(), PostgresScraperError> { + debug!("persisting messages"); + + for chain_tx in txs { + let involved_addresses = parse_addresses_from_events(chain_tx); + for (index, msg) in chain_tx.tx.body.messages.iter().enumerate() { + insert_message( + chain_tx.hash.to_string(), + index as i64, + msg.type_url.clone(), + serde_json::to_value(self.decode_or_skip(msg))?, + involved_addresses.clone(), + chain_tx.height.into(), + self.inner.as_mut(), + ) + .await? + } + } + + Ok(()) + } +} + +#[async_trait] +impl NyxdScraperTransaction for PostgresStorageTransaction { + async fn commit(self) -> Result<(), NyxdScraperStorageError> { + self.inner + .commit() + .await + .map_err(PostgresScraperError::from) + .map_err(NyxdScraperStorageError::from) + } + + async fn persist_validators( + &mut self, + validators: &Response, + ) -> Result<(), NyxdScraperStorageError> { + self.persist_validators(validators) + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn persist_block_data( + &mut self, + block: &Block, + total_gas: i64, + ) -> Result<(), NyxdScraperStorageError> { + self.persist_block_data(block, total_gas) + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn persist_commits( + &mut self, + commits: &Commit, + validators: &Response, + ) -> Result<(), NyxdScraperStorageError> { + self.persist_commits(commits, validators) + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn persist_txs( + &mut self, + txs: &[ParsedTransactionResponse], + ) -> Result<(), NyxdScraperStorageError> { + self.persist_txs(txs) + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn persist_messages( + &mut self, + txs: &[ParsedTransactionResponse], + ) -> Result<(), NyxdScraperStorageError> { + self.persist_messages(txs) + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn update_last_processed(&mut self, height: i64) -> Result<(), NyxdScraperStorageError> { + self.update_last_processed(height).await + } +} diff --git a/common/nyxd-scraper/Cargo.toml b/common/nyxd-scraper-shared/Cargo.toml similarity index 61% rename from common/nyxd-scraper/Cargo.toml rename to common/nyxd-scraper-shared/Cargo.toml index 025e906d56e..3f7a343ec23 100644 --- a/common/nyxd-scraper/Cargo.toml +++ b/common/nyxd-scraper-shared/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "nyxd-scraper" +name = "nyxd-scraper-shared" version = "0.1.0" authors.workspace = true repository.workspace = true @@ -8,19 +8,22 @@ documentation.workspace = true edition.workspace = true license.workspace = true rust-version.workspace = true - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +readme.workspace = true [dependencies] async-trait.workspace = true +base64.workspace = true const_format = { workspace = true } cosmrs.workspace = true +cosmos-sdk-proto = { workspace = true, features = ["serde", "cosmwasm"] } # we need to explicitly include serde feature eyre = { workspace = true } futures.workspace = true humantime = { workspace = true } +ibc-proto = { workspace = true, features = ["serde"] } +prost = { workspace = true } sha2 = { workspace = true } serde = { workspace = true, features = ["derive"] } -sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate", "time"] } +serde_json = { workspace = true } tendermint.workspace = true tendermint-rpc = { workspace = true, features = ["websocket-client", "http-client"] } thiserror.workspace = true @@ -32,11 +35,5 @@ tracing.workspace = true url.workspace = true -# TEMP -#nym-bin-common = { path = "../bin-common", features = ["basic_tracing"]} - - -[build-dependencies] -anyhow = { workspace = true } -sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] } -tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +[lints] +workspace = true diff --git a/common/nyxd-scraper/src/block_processor/helpers.rs b/common/nyxd-scraper-shared/src/block_processor/helpers.rs similarity index 100% rename from common/nyxd-scraper/src/block_processor/helpers.rs rename to common/nyxd-scraper-shared/src/block_processor/helpers.rs diff --git a/common/nyxd-scraper/src/block_processor/mod.rs b/common/nyxd-scraper-shared/src/block_processor/mod.rs similarity index 97% rename from common/nyxd-scraper/src/block_processor/mod.rs rename to common/nyxd-scraper-shared/src/block_processor/mod.rs index a99bc1c21f9..17979a7ed1f 100644 --- a/common/nyxd-scraper/src/block_processor/mod.rs +++ b/common/nyxd-scraper-shared/src/block_processor/mod.rs @@ -8,7 +8,7 @@ use crate::block_requester::BlockRequest; use crate::error::ScraperError; use crate::modules::{BlockModule, MsgModule, TxModule}; use crate::rpc_client::RpcClient; -use crate::storage::{ScraperStorage, persist_block}; +use crate::storage::{persist_block, NyxdScraperStorage, NyxdScraperTransaction}; use futures::StreamExt; use std::cmp::max; use std::collections::{BTreeMap, HashSet, VecDeque}; @@ -77,7 +77,7 @@ impl BlockProcessorConfig { } } -pub struct BlockProcessor { +pub struct BlockProcessor { config: BlockProcessorConfig, cancel: CancellationToken, synced: Arc, @@ -90,7 +90,7 @@ pub struct BlockProcessor { rpc_client: RpcClient, incoming: UnboundedReceiverStream, block_requester: Sender, - storage: ScraperStorage, + storage: S, // future work: rather than sending each msg to every msg module, // let them subscribe based on `type_url` inside the message itself @@ -101,14 +101,17 @@ pub struct BlockProcessor { } #[allow(clippy::too_many_arguments)] -impl BlockProcessor { +impl BlockProcessor +where + S: NyxdScraperStorage, +{ pub async fn new( config: BlockProcessorConfig, cancel: CancellationToken, synced: Arc, incoming: UnboundedReceiver, block_requester: Sender, - storage: ScraperStorage, + storage: S, rpc_client: RpcClient, ) -> Result { let last_processed = storage.get_last_processed_height().await?; @@ -164,7 +167,11 @@ impl BlockProcessor { // process the entire block as a transaction so that if anything fails, // we won't end up with a corrupted storage. - let mut tx = self.storage.begin_processing_tx().await?; + let mut tx = self + .storage + .begin_processing_tx() + .await + .map_err(ScraperError::tx_begin_failure)?; persist_block(&full_info, &mut tx, self.config.store_precommits).await?; @@ -192,10 +199,8 @@ impl BlockProcessor { } let commit_start = Instant::now(); - tx.commit() - .await - .map_err(|source| ScraperError::StorageTxCommitFailure { source })?; - crate::storage::log_db_operation_time("committing processing tx", commit_start); + tx.commit().await.map_err(ScraperError::tx_commit_failure)?; + crate::storage::helpers::log_db_operation_time("committing processing tx", commit_start); self.last_processed_height = full_info.block.header.height.value() as u32; self.last_processed_at = Instant::now(); diff --git a/common/nyxd-scraper/src/block_processor/pruning.rs b/common/nyxd-scraper-shared/src/block_processor/pruning.rs similarity index 100% rename from common/nyxd-scraper/src/block_processor/pruning.rs rename to common/nyxd-scraper-shared/src/block_processor/pruning.rs diff --git a/common/nyxd-scraper/src/block_processor/types.rs b/common/nyxd-scraper-shared/src/block_processor/types.rs similarity index 100% rename from common/nyxd-scraper/src/block_processor/types.rs rename to common/nyxd-scraper-shared/src/block_processor/types.rs diff --git a/common/nyxd-scraper/src/block_requester/mod.rs b/common/nyxd-scraper-shared/src/block_requester/mod.rs similarity index 100% rename from common/nyxd-scraper/src/block_requester/mod.rs rename to common/nyxd-scraper-shared/src/block_requester/mod.rs diff --git a/common/nyxd-scraper/src/constants.rs b/common/nyxd-scraper-shared/src/constants.rs similarity index 100% rename from common/nyxd-scraper/src/constants.rs rename to common/nyxd-scraper-shared/src/constants.rs diff --git a/common/nyxd-scraper-shared/src/cosmos_module/message_registry.rs b/common/nyxd-scraper-shared/src/cosmos_module/message_registry.rs new file mode 100644 index 00000000000..bacc9b499ba --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/message_registry.rs @@ -0,0 +1,146 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::modules::auth::Auth; +use crate::cosmos_module::modules::authz::Authz; +use crate::cosmos_module::modules::bank::Bank; +use crate::cosmos_module::modules::capability::Capability; +use crate::cosmos_module::modules::consensus::Consensus; +use crate::cosmos_module::modules::crisis::Crisis; +use crate::cosmos_module::modules::distribution::Distribution; +use crate::cosmos_module::modules::evidence::Evidence; +use crate::cosmos_module::modules::feegrant::Feegrant; +use crate::cosmos_module::modules::gov_v1::GovV1; +use crate::cosmos_module::modules::gov_v1beta1::GovV1Beta1; +use crate::cosmos_module::modules::group::Group; +use crate::cosmos_module::modules::ibc_core::IbcCore; +use crate::cosmos_module::modules::ibc_fee::IbcFee; +use crate::cosmos_module::modules::ibc_interchain_accounts_controller::IbcInterchainAccountsController; +use crate::cosmos_module::modules::ibc_transfer_v1::IbcTransferV1; +use crate::cosmos_module::modules::ibc_transfer_v2::IbcTransferV2; +use crate::cosmos_module::modules::mint::Mint; +use crate::cosmos_module::modules::nft::Nft; +use crate::cosmos_module::modules::params::Params; +use crate::cosmos_module::modules::slashing::Slashing; +use crate::cosmos_module::modules::staking::Staking; +use crate::cosmos_module::modules::upgrade::Upgrade; +use crate::cosmos_module::modules::vesting::Vesting; +use crate::cosmos_module::modules::wasm::Wasm; +use crate::cosmos_module::CosmosModule; +use crate::error::ScraperError; +use cosmrs::proto::prost::Name; +use cosmrs::proto::traits::Message; +use cosmrs::Any; +use serde::Serialize; +use std::collections::HashMap; + +pub(crate) fn default_proto_to_json( + msg: &Any, +) -> Result { + let proto = ::decode(msg.value.as_slice()).map_err(|error| { + ScraperError::InvalidProtoRepresentation { + type_url: msg.type_url.clone(), + error, + } + })?; + let mut base_serde = + serde_json::to_value(&proto).map_err(|error| ScraperError::JsonSerialisationFailure { + type_url: msg.type_url.clone(), + error, + })?; + + // in bdjuno's output we also had @type field with the type_url + let obj = base_serde.as_object_mut().ok_or_else(|| { + ScraperError::JsonSerialisationFailureNotObject { + type_url: msg.type_url.clone(), + } + })?; + obj.insert( + "@type".to_string(), + serde_json::Value::String(msg.type_url.clone()), + ); + + Ok(base_serde) +} + +type ConvertFn = fn(&Any) -> Result; + +#[derive(Default, Clone)] +pub struct MessageRegistry { + // type url to function converting bytes to proto and finally to json + registered_types: HashMap, +} + +impl MessageRegistry { + pub fn new() -> Self { + MessageRegistry { + registered_types: Default::default(), + } + } + + pub fn register(&mut self) + where + T: Message + Default + Name + Serialize + 'static, + { + self.register_with_custom_fn::(default_proto_to_json::) + } + + #[allow(clippy::panic)] + pub fn register_with_custom_fn(&mut self, convert_fn: ConvertFn) + where + T: Message + Default + Name + Serialize + 'static, + { + if self + .registered_types + .insert(::type_url(), convert_fn) + .is_some() + { + // don't allow duplicate registration because it most likely implies bug in the code + panic!("duplicate registration of type {}", ::type_url()); + } + } + + pub fn try_decode(&self, raw: &Any) -> Result { + self.registered_types.get(&raw.type_url).ok_or( + ScraperError::MissingTypeUrlRegistration { + type_url: raw.type_url.clone(), + }, + )?(raw) + } +} + +pub fn default_message_registry() -> MessageRegistry { + let mut registry = MessageRegistry::new(); + let modules: Vec> = vec![ + Box::new(Auth), + Box::new(Authz), + Box::new(Bank), + Box::new(Capability), + Box::new(Consensus), + Box::new(Wasm), + Box::new(Crisis), + Box::new(Distribution), + Box::new(Evidence), + Box::new(Feegrant), + Box::new(GovV1), + Box::new(GovV1Beta1), + Box::new(Group), + Box::new(IbcCore), + Box::new(IbcFee), + Box::new(IbcTransferV1), + Box::new(IbcTransferV2), + Box::new(IbcInterchainAccountsController), + Box::new(Mint), + Box::new(Nft), + Box::new(Params), + Box::new(Slashing), + Box::new(Staking), + Box::new(Upgrade), + Box::new(Vesting), + ]; + + for module in modules { + module.register_messages(&mut registry) + } + registry +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/mod.rs b/common/nyxd-scraper-shared/src/cosmos_module/mod.rs new file mode 100644 index 00000000000..d60bb7caf1f --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/mod.rs @@ -0,0 +1,11 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; + +pub mod message_registry; +mod modules; + +pub trait CosmosModule { + fn register_messages(&self, registry: &mut MessageRegistry); +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/auth.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/auth.rs new file mode 100644 index 00000000000..f43c6b507bf --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/auth.rs @@ -0,0 +1,14 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::auth::v1beta1::MsgUpdateParams; + +pub(crate) struct Auth; + +impl CosmosModule for Auth { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::() + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/authz.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/authz.rs new file mode 100644 index 00000000000..62c3675c652 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/authz.rs @@ -0,0 +1,16 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::authz::v1beta1::{MsgExec, MsgGrant, MsgRevoke}; + +pub(crate) struct Authz; + +impl CosmosModule for Authz { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/bank.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/bank.rs new file mode 100644 index 00000000000..5f2db1cb782 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/bank.rs @@ -0,0 +1,19 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::bank::v1beta1::{ + MsgMultiSend, MsgSend, MsgSetSendEnabled, MsgUpdateParams, +}; + +pub(crate) struct Bank; + +impl CosmosModule for Bank { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/capability.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/capability.rs new file mode 100644 index 00000000000..2d0837f4332 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/capability.rs @@ -0,0 +1,11 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; + +pub(crate) struct Capability; + +impl CosmosModule for Capability { + fn register_messages(&self, _registry: &mut MessageRegistry) {} +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/consensus.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/consensus.rs new file mode 100644 index 00000000000..8e40ba95b75 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/consensus.rs @@ -0,0 +1,11 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; + +pub(crate) struct Consensus; + +impl CosmosModule for Consensus { + fn register_messages(&self, _registry: &mut MessageRegistry) {} +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/crisis.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/crisis.rs new file mode 100644 index 00000000000..26653044171 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/crisis.rs @@ -0,0 +1,15 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::crisis::v1beta1::{MsgUpdateParams, MsgVerifyInvariant}; + +pub(crate) struct Crisis; + +impl CosmosModule for Crisis { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/distribution.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/distribution.rs new file mode 100644 index 00000000000..7e3f5e273cd --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/distribution.rs @@ -0,0 +1,22 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::distribution::v1beta1::{ + MsgCommunityPoolSpend, MsgFundCommunityPool, MsgSetWithdrawAddress, MsgUpdateParams, + MsgWithdrawDelegatorReward, MsgWithdrawValidatorCommission, +}; + +pub(crate) struct Distribution; + +impl CosmosModule for Distribution { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/evidence.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/evidence.rs new file mode 100644 index 00000000000..03532eba91f --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/evidence.rs @@ -0,0 +1,14 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::evidence::v1beta1::MsgSubmitEvidence; + +pub(crate) struct Evidence; + +impl CosmosModule for Evidence { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::() + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/feegrant.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/feegrant.rs new file mode 100644 index 00000000000..6948e63198c --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/feegrant.rs @@ -0,0 +1,18 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::feegrant::v1beta1::{ + MsgGrantAllowance, MsgPruneAllowances, MsgRevokeAllowance, +}; + +pub(crate) struct Feegrant; + +impl CosmosModule for Feegrant { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1.rs new file mode 100644 index 00000000000..be69613a12f --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1.rs @@ -0,0 +1,21 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::gov::v1::{ + MsgDeposit, MsgExecLegacyContent, MsgSubmitProposal, MsgUpdateParams, MsgVote, MsgVoteWeighted, +}; + +pub(crate) struct GovV1; + +impl CosmosModule for GovV1 { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1beta1.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1beta1.rs new file mode 100644 index 00000000000..cf64d12e0ec --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1beta1.rs @@ -0,0 +1,19 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::gov::v1beta1::{ + MsgDeposit, MsgSubmitProposal, MsgVote, MsgVoteWeighted, +}; + +pub(crate) struct GovV1Beta1; + +impl CosmosModule for GovV1Beta1 { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/group.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/group.rs new file mode 100644 index 00000000000..bddbf750910 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/group.rs @@ -0,0 +1,27 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use tracing::warn; + +pub(crate) struct Group; + +impl CosmosModule for Group { + fn register_messages(&self, _registry: &mut MessageRegistry) { + warn!("mising cosmos-sdk-proto definition for 'group::MsgCreateGroup'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgUpdateGroupMembers'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgUpdateGroupAdmin'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgUpdateGroupMetadata'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgCreateGroupWithPolicy'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgCreateGroupPolicy'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgUpdateGroupPolicyAdmin'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgUpdateGroupPolicyDecisionPolicy'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgUpdateGroupPolicyMetadata'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgSubmitProposal'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgWithdrawProposal'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgVote'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgExec'"); + warn!("mising cosmos-sdk-proto definition for 'group::MsgLeaveGroup'"); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_core.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_core.rs new file mode 100644 index 00000000000..304602fa545 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_core.rs @@ -0,0 +1,70 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::{CosmosModule, MessageRegistry}; +use ibc_proto::ibc::core::channel::{ + self, + v1::{ + MsgAcknowledgement, MsgChannelCloseConfirm, MsgChannelCloseInit, MsgChannelOpenAck, + MsgChannelOpenConfirm, MsgChannelOpenInit, MsgChannelOpenTry, MsgChannelUpgradeAck, + MsgChannelUpgradeCancel, MsgChannelUpgradeConfirm, MsgChannelUpgradeInit, + MsgChannelUpgradeOpen, MsgChannelUpgradeTimeout, MsgChannelUpgradeTry, + MsgPruneAcknowledgements, MsgRecvPacket, MsgTimeout, MsgTimeoutOnClose, + }, +}; +use ibc_proto::ibc::core::client::{ + self, + v1::{ + MsgCreateClient, MsgIbcSoftwareUpgrade, MsgRecoverClient, MsgSubmitMisbehaviour, + MsgUpdateClient, MsgUpgradeClient, + }, +}; +use ibc_proto::ibc::core::connection::{ + self, + v1::{ + MsgConnectionOpenAck, MsgConnectionOpenConfirm, MsgConnectionOpenInit, MsgConnectionOpenTry, + }, +}; + +pub(crate) struct IbcCore; + +impl CosmosModule for IbcCore { + fn register_messages(&self, registry: &mut MessageRegistry) { + // channel + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + + // client + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + + // connection + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_fee.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_fee.rs new file mode 100644 index 00000000000..b5e3d16b534 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_fee.rs @@ -0,0 +1,18 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::{CosmosModule, MessageRegistry}; +use ibc_proto::ibc::applications::fee::v1::{ + MsgPayPacketFee, MsgPayPacketFeeAsync, MsgRegisterPayee, RegisteredCounterpartyPayee, +}; + +pub(crate) struct IbcFee; + +impl CosmosModule for IbcFee { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_interchain_accounts_controller.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_interchain_accounts_controller.rs new file mode 100644 index 00000000000..fc6ef915eb3 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_interchain_accounts_controller.rs @@ -0,0 +1,17 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::{CosmosModule, MessageRegistry}; +use ibc_proto::ibc::applications::interchain_accounts::controller::v1::{ + MsgRegisterInterchainAccount, MsgSendTx, MsgUpdateParams, +}; + +pub(crate) struct IbcInterchainAccountsController; + +impl CosmosModule for IbcInterchainAccountsController { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_transfer_v1.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_transfer_v1.rs new file mode 100644 index 00000000000..0f2f92524a1 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_transfer_v1.rs @@ -0,0 +1,14 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::{CosmosModule, MessageRegistry}; +use ibc_proto::ibc::applications::transfer::v1::{MsgTransfer, MsgUpdateParams}; + +pub(crate) struct IbcTransferV1; + +impl CosmosModule for IbcTransferV1 { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_transfer_v2.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_transfer_v2.rs new file mode 100644 index 00000000000..d0e707e35fa --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/ibc_transfer_v2.rs @@ -0,0 +1,10 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::{CosmosModule, MessageRegistry}; + +pub(crate) struct IbcTransferV2; + +impl CosmosModule for IbcTransferV2 { + fn register_messages(&self, _registry: &mut MessageRegistry) {} +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/mint.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/mint.rs new file mode 100644 index 00000000000..9cd59c50384 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/mint.rs @@ -0,0 +1,14 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::mint::v1beta1::MsgUpdateParams; + +pub(crate) struct Mint; + +impl CosmosModule for Mint { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::() + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/mod.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/mod.rs new file mode 100644 index 00000000000..5c13923dcba --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/mod.rs @@ -0,0 +1,28 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +pub(crate) mod auth; +pub(crate) mod authz; +pub(crate) mod bank; +pub(crate) mod capability; +pub(crate) mod consensus; +pub(crate) mod crisis; +pub(crate) mod distribution; +pub(crate) mod evidence; +pub(crate) mod feegrant; +pub(crate) mod gov_v1; +pub(crate) mod gov_v1beta1; +pub(crate) mod group; +pub(crate) mod ibc_core; +pub(crate) mod ibc_fee; +pub(crate) mod ibc_interchain_accounts_controller; +pub(crate) mod ibc_transfer_v1; +pub(crate) mod ibc_transfer_v2; +pub(crate) mod mint; +pub(crate) mod nft; +pub(crate) mod params; +pub(crate) mod slashing; +pub(crate) mod staking; +pub(crate) mod upgrade; +pub(crate) mod vesting; +pub(crate) mod wasm; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/nft.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/nft.rs new file mode 100644 index 00000000000..cebfd52fba8 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/nft.rs @@ -0,0 +1,11 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; + +pub(crate) struct Nft; + +impl CosmosModule for Nft { + fn register_messages(&self, _registry: &mut MessageRegistry) {} +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/params.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/params.rs new file mode 100644 index 00000000000..909cc12ede6 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/params.rs @@ -0,0 +1,11 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; + +pub(crate) struct Params; + +impl CosmosModule for Params { + fn register_messages(&self, _registry: &mut MessageRegistry) {} +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/slashing.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/slashing.rs new file mode 100644 index 00000000000..5ae5328c372 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/slashing.rs @@ -0,0 +1,11 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; + +pub(crate) struct Slashing; + +impl CosmosModule for Slashing { + fn register_messages(&self, _registry: &mut MessageRegistry) {} +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/staking.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/staking.rs new file mode 100644 index 00000000000..44c836eb298 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/staking.rs @@ -0,0 +1,23 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::staking::v1beta1::{ + MsgBeginRedelegate, MsgCancelUnbondingDelegation, MsgCreateValidator, MsgDelegate, + MsgEditValidator, MsgUndelegate, MsgUpdateParams, +}; + +pub(crate) struct Staking; + +impl CosmosModule for Staking { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/upgrade.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/upgrade.rs new file mode 100644 index 00000000000..5a100f816c1 --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/upgrade.rs @@ -0,0 +1,15 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::upgrade::v1beta1::{MsgCancelUpgrade, MsgSoftwareUpgrade}; + +pub(crate) struct Upgrade; + +impl CosmosModule for Upgrade { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/vesting.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/vesting.rs new file mode 100644 index 00000000000..f690cab22bc --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/vesting.rs @@ -0,0 +1,18 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::MessageRegistry; +use crate::cosmos_module::CosmosModule; +use cosmos_sdk_proto::cosmos::vesting::v1beta1::{ + MsgCreatePeriodicVestingAccount, MsgCreatePermanentLockedAccount, MsgCreateVestingAccount, +}; + +pub(crate) struct Vesting; + +impl CosmosModule for Vesting { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/wasm.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/wasm.rs new file mode 100644 index 00000000000..38d26b8bacb --- /dev/null +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/wasm.rs @@ -0,0 +1,104 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::cosmos_module::message_registry::{default_proto_to_json, MessageRegistry}; +use crate::cosmos_module::CosmosModule; +use crate::error::ScraperError; +use base64::engine::general_purpose::STANDARD; +use base64::Engine; +use cosmos_sdk_proto::cosmwasm::wasm::v1::{ + MsgAddCodeUploadParamsAddresses, MsgClearAdmin, MsgExecuteContract, MsgIbcCloseChannel, + MsgIbcSend, MsgInstantiateContract, MsgInstantiateContract2, MsgMigrateContract, MsgPinCodes, + MsgRemoveCodeUploadParamsAddresses, MsgStoreAndInstantiateContract, MsgStoreAndMigrateContract, + MsgStoreCode, MsgSudoContract, MsgUnpinCodes, MsgUpdateAdmin, MsgUpdateContractLabel, + MsgUpdateInstantiateConfig, MsgUpdateParams, +}; +use cosmrs::Any; +use prost::Message; +use serde::Serialize; +use tracing::warn; + +pub(crate) struct Wasm; + +fn decode_wasm_message( + msg: &Any, +) -> Result { + let field = "msg"; + // 1. perform basic decoding + let mut base = default_proto_to_json::(msg)?; + let Some(encoded_field) = base.get_mut(field) else { + warn!( + "missing field 'msg' in wasm message of type {} - can't perform additional decoding", + msg.type_url + ); + return Ok(base); + }; + + // 2. decode 'msg' field + let as_str = + encoded_field + .as_str() + .ok_or(ScraperError::JsonWasmSerialisationFailureNotString { + field: field.to_string(), + type_url: msg.type_url.clone(), + })?; + + let decoded = STANDARD.decode(as_str).map_err(|error| { + ScraperError::JsonWasmSerialisationFailureInvalidBase64Encoding { + field: field.to_string(), + type_url: msg.type_url.clone(), + error, + } + })?; + + // 3. replace original 'msg' with the new json + let re_decoded: serde_json::Value = serde_json::from_slice(&decoded).map_err(|error| { + ScraperError::JsonSerialisationFailure { + type_url: format!("{}.{field}", msg.type_url), + error, + } + })?; + + *encoded_field = re_decoded; + Ok(base) +} + +impl CosmosModule for Wasm { + fn register_messages(&self, registry: &mut MessageRegistry) { + registry.register::(); + registry.register::(); + registry.register::(); + + registry.register_with_custom_fn::(|msg| { + decode_wasm_message::(msg) + }); + registry.register_with_custom_fn::(|msg| { + decode_wasm_message::(msg) + }); + registry.register_with_custom_fn::(|msg| { + decode_wasm_message::(msg) + }); + registry.register_with_custom_fn::(|msg| { + decode_wasm_message::(msg) + }); + registry.register_with_custom_fn::(|msg| { + decode_wasm_message::(msg) + }); + registry.register_with_custom_fn::(|msg| { + decode_wasm_message::(msg) + }); + registry.register_with_custom_fn::(|msg| { + decode_wasm_message::(msg) + }); + + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + registry.register::(); + } +} diff --git a/common/nyxd-scraper/src/error.rs b/common/nyxd-scraper-shared/src/error.rs similarity index 72% rename from common/nyxd-scraper/src/error.rs rename to common/nyxd-scraper-shared/src/error.rs index d9dd359f6ad..76723950bef 100644 --- a/common/nyxd-scraper/src/error.rs +++ b/common/nyxd-scraper-shared/src/error.rs @@ -4,17 +4,16 @@ use crate::block_processor::pruning::{ EVERYTHING_PRUNING_INTERVAL, EVERYTHING_PRUNING_KEEP_RECENT, }; +use crate::helpers::MalformedDataError; +use crate::storage::NyxdScraperStorageError; use tendermint::Hash; use thiserror::Error; use tokio::sync::mpsc::error::SendError; #[derive(Debug, Error)] pub enum ScraperError { - #[error("experienced internal database error: {0}")] - InternalDatabaseError(#[from] sqlx::Error), - - #[error("failed to perform startup SQL migration: {0}")] - StartupMigrationFailure(#[from] sqlx::migrate::MigrateError), + #[error("storage error: {0}")] + StorageError(#[from] NyxdScraperStorageError), #[error("the block scraper is already running")] ScraperAlreadyRunning, @@ -106,40 +105,26 @@ pub enum ScraperError { #[error("failed to begin storage tx: {source}")] StorageTxBeginFailure { #[source] - source: sqlx::Error, + source: NyxdScraperStorageError, }, #[error("failed to commit storage tx: {source}")] StorageTxCommitFailure { #[source] - source: sqlx::Error, + source: NyxdScraperStorageError, }, #[error("failed to send on a closed channel")] ClosedChannelError, - #[error("failed to parse validator's address: {source}")] - MalformedValidatorAddress { - #[source] - source: eyre::Report, - }, - - #[error("failed to parse validator's address: {source}")] - MalformedValidatorPubkey { - #[source] - source: eyre::Report, - }, + #[error(transparent)] + MalformedData(#[from] MalformedDataError), #[error( "could not find the block proposer ('{proposer}') for height {height} in the validator set" )] BlockProposerNotInValidatorSet { height: u32, proposer: String }, - #[error( - "could not find validator information for {address}; the validator has signed a commit" - )] - MissingValidatorInfoCommitted { address: String }, - #[error( "pruning.interval must not be set to 0. If you want to disable pruning, select pruning.strategy = \"nothing\"" )] @@ -156,6 +141,49 @@ pub enum ScraperError { EVERYTHING_PRUNING_KEEP_RECENT )] TooSmallKeepRecent { keep_recent: u32 }, + + #[error("'{type_url}' is not registered in the message registry")] + MissingTypeUrlRegistration { type_url: String }, + + #[error("failed to decode message of type '{type_url}': {error}")] + InvalidProtoRepresentation { + type_url: String, + #[source] + error: prost::DecodeError, + }, + + #[error("failed to encode message of type '{type_url}' to json: '{error}'")] + JsonSerialisationFailure { + type_url: String, + #[source] + error: serde_json::Error, + }, + + #[error("serialisation of message of type '{type_url}' didn't result in an object!")] + JsonSerialisationFailureNotObject { type_url: String }, + + #[error("field '{field}' in '{type_url}' is not a string")] + JsonWasmSerialisationFailureNotString { field: String, type_url: String }, + + #[error("field '{field}' in '{type_url}' has invalid base64 encoding: {error}")] + JsonWasmSerialisationFailureInvalidBase64Encoding { + field: String, + type_url: String, + #[source] + error: base64::DecodeError, + }, +} + +impl ScraperError { + pub fn tx_begin_failure(source: NyxdScraperStorageError) -> ScraperError +where { + ScraperError::StorageTxBeginFailure { source } + } + + pub fn tx_commit_failure(source: NyxdScraperStorageError) -> ScraperError +where { + ScraperError::StorageTxCommitFailure { source } + } } impl From> for ScraperError { diff --git a/common/nyxd-scraper-shared/src/helpers.rs b/common/nyxd-scraper-shared/src/helpers.rs new file mode 100644 index 00000000000..2a20a2751be --- /dev/null +++ b/common/nyxd-scraper-shared/src/helpers.rs @@ -0,0 +1,66 @@ +// Copyright 2023 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::block_processor::types::ParsedTransactionResponse; +use crate::constants::{BECH32_CONESNSUS_PUBKEY_PREFIX, BECH32_CONSENSUS_ADDRESS_PREFIX}; +use cosmrs::AccountId; +use sha2::{Digest, Sha256}; +use tendermint::{account, PublicKey}; +use tendermint::{validator, Hash}; +use tendermint_rpc::endpoint::validators; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum MalformedDataError { + #[error("failed to parse validator's address: {source}")] + MalformedValidatorAddress { + #[source] + source: eyre::Report, + }, + + #[error("failed to parse validator's address: {source}")] + MalformedValidatorPubkey { + #[source] + source: eyre::Report, + }, + + #[error( + "could not find validator information for {address}; the validator has signed a commit" + )] + MissingValidatorInfoCommitted { address: String }, +} + +pub fn tx_hash>(raw_tx: M) -> Hash { + Hash::Sha256(Sha256::digest(raw_tx).into()) +} + +pub fn validator_pubkey_to_bech32(pubkey: PublicKey) -> Result { + // TODO: this one seem to attach additional prefix to they pubkeys, is that what we want instead maybe? + // Ok(pubkey.to_bech32(BECH32_CONESNSUS_PUBKEY_PREFIX)) + AccountId::new(BECH32_CONESNSUS_PUBKEY_PREFIX, &pubkey.to_bytes()) + .map_err(|source| MalformedDataError::MalformedValidatorPubkey { source }) +} + +pub fn validator_consensus_address(id: account::Id) -> Result { + AccountId::new(BECH32_CONSENSUS_ADDRESS_PREFIX, id.as_ref()) + .map_err(|source| MalformedDataError::MalformedValidatorAddress { source }) +} + +pub fn tx_gas_sum(txs: &[ParsedTransactionResponse]) -> i64 { + txs.iter().map(|tx| tx.tx_result.gas_used).sum() +} + +pub fn validator_info( + id: account::Id, + validators: &validators::Response, +) -> Result<&validator::Info, MalformedDataError> { + match validators.validators.iter().find(|v| v.address == id) { + Some(info) => Ok(info), + None => { + let addr = validator_consensus_address(id)?; + Err(MalformedDataError::MissingValidatorInfoCommitted { + address: addr.to_string(), + }) + } + } +} diff --git a/common/nyxd-scraper/src/lib.rs b/common/nyxd-scraper-shared/src/lib.rs similarity index 69% rename from common/nyxd-scraper/src/lib.rs rename to common/nyxd-scraper-shared/src/lib.rs index 7d22921a3eb..7d3f198b7c7 100644 --- a/common/nyxd-scraper/src/lib.rs +++ b/common/nyxd-scraper-shared/src/lib.rs @@ -1,14 +1,12 @@ // Copyright 2023 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -#![warn(clippy::expect_used)] -#![warn(clippy::unwrap_used)] - pub(crate) mod block_processor; pub(crate) mod block_requester; pub mod constants; +mod cosmos_module; pub mod error; -pub(crate) mod helpers; +pub mod helpers; pub mod modules; pub(crate) mod rpc_client; pub(crate) mod scraper; @@ -16,6 +14,11 @@ pub mod storage; pub use block_processor::pruning::{PruningOptions, PruningStrategy}; pub use block_processor::types::ParsedTransactionResponse; +pub use cosmos_module::{ + message_registry::{default_message_registry, MessageRegistry}, + CosmosModule, +}; +pub use cosmrs::Any; pub use modules::{BlockModule, MsgModule, TxModule}; pub use scraper::{Config, NyxdScraper, StartingBlockOpts}; -pub use storage::models; +pub use storage::{NyxdScraperStorage, NyxdScraperTransaction}; diff --git a/common/nyxd-scraper/src/modules/block_module.rs b/common/nyxd-scraper-shared/src/modules/block_module.rs similarity index 79% rename from common/nyxd-scraper/src/modules/block_module.rs rename to common/nyxd-scraper-shared/src/modules/block_module.rs index 9ca1ba9b204..1ea3c2899d9 100644 --- a/common/nyxd-scraper/src/modules/block_module.rs +++ b/common/nyxd-scraper-shared/src/modules/block_module.rs @@ -3,7 +3,7 @@ use crate::block_processor::types::FullBlockInformation; use crate::error::ScraperError; -use crate::storage::StorageTransaction; +use crate::storage::NyxdScraperTransaction; use async_trait::async_trait; #[async_trait] @@ -11,6 +11,6 @@ pub trait BlockModule { async fn handle_block( &mut self, block: &FullBlockInformation, - storage_tx: &mut StorageTransaction, + storage_tx: &mut dyn NyxdScraperTransaction, ) -> Result<(), ScraperError>; } diff --git a/common/nyxd-scraper/src/modules/mod.rs b/common/nyxd-scraper-shared/src/modules/mod.rs similarity index 100% rename from common/nyxd-scraper/src/modules/mod.rs rename to common/nyxd-scraper-shared/src/modules/mod.rs diff --git a/common/nyxd-scraper/src/modules/msg_module.rs b/common/nyxd-scraper-shared/src/modules/msg_module.rs similarity index 83% rename from common/nyxd-scraper/src/modules/msg_module.rs rename to common/nyxd-scraper-shared/src/modules/msg_module.rs index 1d195bee14a..60f53b1553d 100644 --- a/common/nyxd-scraper/src/modules/msg_module.rs +++ b/common/nyxd-scraper-shared/src/modules/msg_module.rs @@ -3,7 +3,7 @@ use crate::block_processor::types::ParsedTransactionResponse; use crate::error::ScraperError; -use crate::storage::StorageTransaction; +use crate::storage::NyxdScraperTransaction; use async_trait::async_trait; use cosmrs::Any; @@ -16,6 +16,6 @@ pub trait MsgModule { index: usize, msg: &Any, tx: &ParsedTransactionResponse, - storage_tx: &mut StorageTransaction, + storage_tx: &mut dyn NyxdScraperTransaction, ) -> Result<(), ScraperError>; } diff --git a/common/nyxd-scraper/src/modules/tx_module.rs b/common/nyxd-scraper-shared/src/modules/tx_module.rs similarity index 79% rename from common/nyxd-scraper/src/modules/tx_module.rs rename to common/nyxd-scraper-shared/src/modules/tx_module.rs index 07d012ab5a8..8d2f5b22b1e 100644 --- a/common/nyxd-scraper/src/modules/tx_module.rs +++ b/common/nyxd-scraper-shared/src/modules/tx_module.rs @@ -3,7 +3,7 @@ use crate::block_processor::types::ParsedTransactionResponse; use crate::error::ScraperError; -use crate::storage::StorageTransaction; +use crate::storage::NyxdScraperTransaction; use async_trait::async_trait; #[async_trait] @@ -11,6 +11,6 @@ pub trait TxModule { async fn handle_tx( &mut self, tx: &ParsedTransactionResponse, - storage_tx: &mut StorageTransaction, + storage_tx: &mut dyn NyxdScraperTransaction, ) -> Result<(), ScraperError>; } diff --git a/common/nyxd-scraper/src/rpc_client.rs b/common/nyxd-scraper-shared/src/rpc_client.rs similarity index 100% rename from common/nyxd-scraper/src/rpc_client.rs rename to common/nyxd-scraper-shared/src/rpc_client.rs diff --git a/common/nyxd-scraper/src/scraper/mod.rs b/common/nyxd-scraper-shared/src/scraper/mod.rs similarity index 92% rename from common/nyxd-scraper/src/scraper/mod.rs rename to common/nyxd-scraper-shared/src/scraper/mod.rs index 5d067044da5..df9224ae220 100644 --- a/common/nyxd-scraper/src/scraper/mod.rs +++ b/common/nyxd-scraper-shared/src/scraper/mod.rs @@ -9,9 +9,9 @@ use crate::error::ScraperError; use crate::modules::{BlockModule, MsgModule, TxModule}; use crate::rpc_client::RpcClient; use crate::scraper::subscriber::ChainSubscriber; -use crate::storage::ScraperStorage; +use crate::storage::NyxdScraperStorage; use futures::future::join_all; -use std::path::PathBuf; +use std::marker::PhantomData; use std::sync::Arc; use tokio::sync::Notify; use tokio::sync::mpsc::{ @@ -40,7 +40,8 @@ pub struct Config { /// Url to the rpc endpoint of a validator, for example `https://rpc.nymtech.net/` pub rpc_url: Url, - pub database_path: PathBuf, + /// Points to either underlying file (sqlite) or connection string (postgres) + pub database_storage: String, pub pruning_options: PruningOptions, @@ -49,7 +50,8 @@ pub struct Config { pub start_block: StartingBlockOpts, } -pub struct NyxdScraperBuilder { +pub struct NyxdScraperBuilder { + _storage: PhantomData, config: Config, block_modules: Vec>, @@ -57,9 +59,13 @@ pub struct NyxdScraperBuilder { msg_modules: Vec>, } -impl NyxdScraperBuilder { - pub async fn build_and_start(self) -> Result { - let scraper = NyxdScraper::new(self.config).await?; +impl NyxdScraperBuilder +where + S: NyxdScraperStorage + Send + Sync + 'static, + S::StorageTransaction: Send + Sync + 'static, +{ + pub async fn build_and_start(self) -> Result, ScraperError> { + let scraper = NyxdScraper::::new(self.config).await?; let (processing_tx, processing_rx) = unbounded_channel(); let (req_tx, req_rx) = channel(5); @@ -110,6 +116,7 @@ impl NyxdScraperBuilder { pub fn new(config: Config) -> Self { NyxdScraperBuilder { + _storage: PhantomData, config, block_modules: vec![], tx_modules: vec![], @@ -133,24 +140,28 @@ impl NyxdScraperBuilder { } } -pub struct NyxdScraper { +pub struct NyxdScraper { config: Config, task_tracker: TaskTracker, cancel_token: CancellationToken, startup_sync: Arc, - storage: ScraperStorage, + storage: S, rpc_client: RpcClient, } -impl NyxdScraper { - pub fn builder(config: Config) -> NyxdScraperBuilder { +impl NyxdScraper +where + S: NyxdScraperStorage + Send + Sync + 'static, + S::StorageTransaction: Send + Sync + 'static, +{ + pub fn builder(config: Config) -> NyxdScraperBuilder { NyxdScraperBuilder::new(config) } pub async fn new(config: Config) -> Result { config.pruning_options.validate()?; - let storage = ScraperStorage::init(&config.database_path).await?; + let storage = S::initialise(&config.database_storage).await?; let rpc_client = RpcClient::new(&config.rpc_url)?; Ok(NyxdScraper { @@ -163,14 +174,14 @@ impl NyxdScraper { }) } - pub fn storage(&self) -> ScraperStorage { - self.storage.clone() + pub fn storage(&self) -> &S { + &self.storage } fn start_tasks( &self, mut block_requester: BlockRequester, - mut block_processor: BlockProcessor, + mut block_processor: BlockProcessor, mut chain_subscriber: ChainSubscriber, ) { self.task_tracker @@ -336,7 +347,7 @@ impl NyxdScraper { &self, req_tx: Sender, processing_rx: UnboundedReceiver, - ) -> Result { + ) -> Result, ScraperError> { let block_processor_config = BlockProcessorConfig::new( self.config.pruning_options, self.config.store_precommits, @@ -344,7 +355,7 @@ impl NyxdScraper { self.config.start_block.use_best_effort_start_height, ); - BlockProcessor::new( + BlockProcessor::::new( block_processor_config, self.cancel_token.clone(), self.startup_sync.clone(), diff --git a/common/nyxd-scraper/src/scraper/subscriber.rs b/common/nyxd-scraper-shared/src/scraper/subscriber.rs similarity index 100% rename from common/nyxd-scraper/src/scraper/subscriber.rs rename to common/nyxd-scraper-shared/src/scraper/subscriber.rs diff --git a/common/nyxd-scraper-shared/src/storage/helpers.rs b/common/nyxd-scraper-shared/src/storage/helpers.rs new file mode 100644 index 00000000000..488103fcc64 --- /dev/null +++ b/common/nyxd-scraper-shared/src/storage/helpers.rs @@ -0,0 +1,18 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use tokio::time::Instant; +use tracing::{debug, error, info, trace, warn}; + +pub fn log_db_operation_time(op_name: &str, start_time: Instant) { + let elapsed = start_time.elapsed(); + let formatted = humantime::format_duration(elapsed); + + match elapsed.as_millis() { + v if v > 10000 => error!("{op_name} took {formatted} to execute"), + v if v > 1000 => warn!("{op_name} took {formatted} to execute"), + v if v > 100 => info!("{op_name} took {formatted} to execute"), + v if v > 10 => debug!("{op_name} took {formatted} to execute"), + _ => trace!("{op_name} took {formatted} to execute"), + } +} diff --git a/common/nyxd-scraper-shared/src/storage/mod.rs b/common/nyxd-scraper-shared/src/storage/mod.rs new file mode 100644 index 00000000000..97fa3ce26c1 --- /dev/null +++ b/common/nyxd-scraper-shared/src/storage/mod.rs @@ -0,0 +1,124 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::error::ScraperError; +use async_trait::async_trait; +use thiserror::Error; +use tracing::warn; + +pub use crate::block_processor::types::FullBlockInformation; +pub use crate::ParsedTransactionResponse; +pub use tendermint::block::{Commit, CommitSig}; +pub use tendermint::Block; +pub use tendermint_rpc::endpoint::validators; + +pub mod helpers; + +// a workaround for needing associated type (which is a no-no in dynamic dispatch) +#[derive(Error, Debug)] +#[error(transparent)] +pub struct NyxdScraperStorageError(Box); + +impl NyxdScraperStorageError { + pub fn new(error: E) -> Self + where + E: std::error::Error + Send + Sync + 'static, + { + NyxdScraperStorageError(Box::new(error)) + } +} + +#[async_trait] +pub trait NyxdScraperStorage: Clone + Sized { + type StorageTransaction: NyxdScraperTransaction; + + /// Either connection string (postgres) or storage path (sqlite) + async fn initialise(storage: &str) -> Result; + + async fn begin_processing_tx( + &self, + ) -> Result; + + async fn get_last_processed_height(&self) -> Result; + + async fn get_pruned_height(&self) -> Result; + + async fn lowest_block_height(&self) -> Result, NyxdScraperStorageError>; + + async fn prune_storage( + &self, + oldest_to_keep: u32, + current_height: u32, + ) -> Result<(), NyxdScraperStorageError>; +} + +#[async_trait] +pub trait NyxdScraperTransaction { + async fn commit(mut self) -> Result<(), NyxdScraperStorageError>; + + async fn persist_validators( + &mut self, + validators: &validators::Response, + ) -> Result<(), NyxdScraperStorageError>; + + async fn persist_block_data( + &mut self, + block: &Block, + total_gas: i64, + ) -> Result<(), NyxdScraperStorageError>; + + async fn persist_commits( + &mut self, + commits: &Commit, + validators: &validators::Response, + ) -> Result<(), NyxdScraperStorageError>; + + async fn persist_txs( + &mut self, + txs: &[ParsedTransactionResponse], + ) -> Result<(), NyxdScraperStorageError>; + + async fn persist_messages( + &mut self, + txs: &[ParsedTransactionResponse], + ) -> Result<(), NyxdScraperStorageError>; + + async fn update_last_processed(&mut self, height: i64) -> Result<(), NyxdScraperStorageError>; +} + +pub async fn persist_block( + block: &FullBlockInformation, + tx: &mut Tx, + store_precommits: bool, +) -> Result<(), ScraperError> +where + Tx: NyxdScraperTransaction, +{ + let total_gas = crate::helpers::tx_gas_sum(&block.transactions); + + // SANITY CHECK: make sure the block proposer is present in the validator set + block.ensure_proposer()?; + + tx.persist_validators(&block.validators).await?; + + tx.persist_block_data(&block.block, total_gas).await?; + + if store_precommits { + if let Some(commit) = &block.block.last_commit { + tx.persist_commits(commit, &block.validators).await?; + } else { + warn!("no commits for block {}", block.block.header.height) + } + } + + // persist txs + tx.persist_txs(&block.transactions).await?; + + // persist messages (inside the transactions) + tx.persist_messages(&block.transactions).await?; + + tx.update_last_processed(block.block.header.height.into()) + .await?; + + Ok(()) +} diff --git a/common/nyxd-scraper-sqlite/Cargo.toml b/common/nyxd-scraper-sqlite/Cargo.toml new file mode 100644 index 00000000000..53e492b5e0c --- /dev/null +++ b/common/nyxd-scraper-sqlite/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "nyxd-scraper-sqlite" +version = "0.1.0" +authors.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +edition.workspace = true +license.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-trait = { workspace = true } +sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate", "time"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tracing.workspace = true + +nyxd-scraper-shared = { path = "../nyxd-scraper-shared" } + + +[build-dependencies] +sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } + +[lints] +workspace = true \ No newline at end of file diff --git a/common/nyxd-scraper/README.md b/common/nyxd-scraper-sqlite/README.md similarity index 100% rename from common/nyxd-scraper/README.md rename to common/nyxd-scraper-sqlite/README.md diff --git a/common/nyxd-scraper/build.rs b/common/nyxd-scraper-sqlite/build.rs similarity index 100% rename from common/nyxd-scraper/build.rs rename to common/nyxd-scraper-sqlite/build.rs diff --git a/common/nyxd-scraper-sqlite/sql_migrations/01_metadata.sql b/common/nyxd-scraper-sqlite/sql_migrations/01_metadata.sql new file mode 100644 index 00000000000..bce13940e81 --- /dev/null +++ b/common/nyxd-scraper-sqlite/sql_migrations/01_metadata.sql @@ -0,0 +1,10 @@ +/* + * Copyright 2023 - Nym Technologies SA + * SPDX-License-Identifier: Apache-2.0 + */ + +CREATE TABLE METADATA +( + id INTEGER PRIMARY KEY CHECK (id = 0), + last_processed_height INTEGER NOT NULL +); \ No newline at end of file diff --git a/common/nyxd-scraper/sql_migrations/02_cosmos.sql b/common/nyxd-scraper-sqlite/sql_migrations/02_cosmos.sql similarity index 100% rename from common/nyxd-scraper/sql_migrations/02_cosmos.sql rename to common/nyxd-scraper-sqlite/sql_migrations/02_cosmos.sql diff --git a/common/nyxd-scraper-sqlite/src/error.rs b/common/nyxd-scraper-sqlite/src/error.rs new file mode 100644 index 00000000000..c91a2c2b8d8 --- /dev/null +++ b/common/nyxd-scraper-sqlite/src/error.rs @@ -0,0 +1,36 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use nyxd_scraper_shared::helpers::MalformedDataError; +use nyxd_scraper_shared::storage::NyxdScraperStorageError; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum SqliteScraperError { + #[error("experienced internal database error: {0}")] + InternalDatabaseError(#[from] sqlx::error::Error), + + #[error("failed to perform startup SQL migration: {0}")] + StartupMigrationFailure(#[from] sqlx::migrate::MigrateError), + + #[error("failed to begin storage tx: {source}")] + StorageTxBeginFailure { + #[source] + source: sqlx::error::Error, + }, + + #[error("failed to commit storage tx: {source}")] + StorageTxCommitFailure { + #[source] + source: sqlx::error::Error, + }, + + #[error(transparent)] + MalformedData(#[from] MalformedDataError), +} + +impl From for NyxdScraperStorageError { + fn from(err: SqliteScraperError) -> Self { + NyxdScraperStorageError::new(err) + } +} diff --git a/common/nyxd-scraper-sqlite/src/lib.rs b/common/nyxd-scraper-sqlite/src/lib.rs new file mode 100644 index 00000000000..56c202b63ff --- /dev/null +++ b/common/nyxd-scraper-sqlite/src/lib.rs @@ -0,0 +1,21 @@ +// Copyright 2023 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::storage::block_storage::SqliteScraperStorage; +use nyxd_scraper_shared::NyxdScraper; + +pub use nyxd_scraper_shared::constants; +pub use nyxd_scraper_shared::error::ScraperError; +pub use nyxd_scraper_shared::{ + BlockModule, MsgModule, NyxdScraperTransaction, ParsedTransactionResponse, PruningOptions, + PruningStrategy, StartingBlockOpts, TxModule, +}; +pub use storage::models; + +pub mod error; +pub mod storage; + +pub type SqliteNyxdScraper = NyxdScraper; + +// TODO: for now just use exactly the same config +pub use nyxd_scraper_shared::Config; diff --git a/common/nyxd-scraper-sqlite/src/storage/block_storage.rs b/common/nyxd-scraper-sqlite/src/storage/block_storage.rs new file mode 100644 index 00000000000..0df04ddbabb --- /dev/null +++ b/common/nyxd-scraper-sqlite/src/storage/block_storage.rs @@ -0,0 +1,251 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::error::SqliteScraperError; +use crate::models::{CommitSignature, Validator}; +use crate::storage::manager::{ + prune_blocks, prune_messages, prune_pre_commits, prune_transactions, update_last_pruned, + StorageManager, +}; +use crate::storage::transaction::SqliteStorageTransaction; +use async_trait::async_trait; +use nyxd_scraper_shared::storage::helpers::log_db_operation_time; +use nyxd_scraper_shared::storage::{NyxdScraperStorage, NyxdScraperStorageError}; +use sqlx::sqlite::{SqliteAutoVacuum, SqliteSynchronous}; +use sqlx::types::time::OffsetDateTime; +use sqlx::ConnectOptions; +use std::fmt::Debug; +use std::path::Path; +use tokio::time::Instant; +use tracing::{debug, error, info, instrument}; + +#[derive(Clone)] +pub struct SqliteScraperStorage { + pub(crate) manager: StorageManager, +} + +impl SqliteScraperStorage { + #[instrument] + pub async fn init + Debug>( + database_path: P, + ) -> Result { + let database_path = database_path.as_ref(); + debug!( + "initialising scraper database path to '{}'", + database_path.display() + ); + + let opts = sqlx::sqlite::SqliteConnectOptions::new() + .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) + .synchronous(SqliteSynchronous::Normal) + .auto_vacuum(SqliteAutoVacuum::Incremental) + .filename(database_path) + .create_if_missing(true) + .disable_statement_logging(); + + // TODO: do we want auto_vacuum ? + + let connection_pool = match sqlx::SqlitePool::connect_with(opts).await { + Ok(db) => db, + Err(err) => { + error!("Failed to connect to SQLx database: {err}"); + return Err(err.into()); + } + }; + + if let Err(err) = sqlx::migrate!("./sql_migrations") + .run(&connection_pool) + .await + { + error!("Failed to initialize SQLx database: {err}"); + return Err(err.into()); + } + + info!("Database migration finished!"); + + let manager = StorageManager { connection_pool }; + manager.set_initial_metadata().await?; + + let storage = SqliteScraperStorage { manager }; + + Ok(storage) + } + + #[instrument(skip(self))] + pub async fn prune_storage( + &self, + oldest_to_keep: u32, + current_height: u32, + ) -> Result<(), SqliteScraperError> { + let start = Instant::now(); + + let mut tx = self.begin_processing_tx().await?; + + prune_messages(oldest_to_keep.into(), &mut **tx).await?; + prune_transactions(oldest_to_keep.into(), &mut **tx).await?; + prune_pre_commits(oldest_to_keep.into(), &mut **tx).await?; + prune_blocks(oldest_to_keep.into(), &mut **tx).await?; + update_last_pruned(current_height.into(), &mut **tx).await?; + + let commit_start = Instant::now(); + tx.0.commit() + .await + .map_err(|source| SqliteScraperError::StorageTxCommitFailure { source })?; + log_db_operation_time("committing pruning tx", commit_start); + + log_db_operation_time("pruning storage", start); + Ok(()) + } + + #[instrument(skip_all)] + pub async fn begin_processing_tx( + &self, + ) -> Result { + debug!("starting storage tx"); + self.manager + .connection_pool + .begin() + .await + .map(SqliteStorageTransaction) + .map_err(|source| SqliteScraperError::StorageTxBeginFailure { source }) + } + + pub async fn lowest_block_height(&self) -> Result, SqliteScraperError> { + Ok(self.manager.get_lowest_block().await?) + } + + pub async fn get_first_block_height_after( + &self, + time: OffsetDateTime, + ) -> Result, SqliteScraperError> { + Ok(self.manager.get_first_block_height_after(time).await?) + } + + pub async fn get_last_block_height_before( + &self, + time: OffsetDateTime, + ) -> Result, SqliteScraperError> { + Ok(self.manager.get_last_block_height_before(time).await?) + } + + pub async fn get_blocks_between( + &self, + start_time: OffsetDateTime, + end_time: OffsetDateTime, + ) -> Result { + let Some(block_start) = self.get_first_block_height_after(start_time).await? else { + return Ok(0); + }; + let Some(block_end) = self.get_last_block_height_before(end_time).await? else { + return Ok(0); + }; + + Ok(block_end - block_start) + } + + pub async fn get_signed_between( + &self, + consensus_address: &str, + start_height: i64, + end_height: i64, + ) -> Result { + Ok(self + .manager + .get_signed_between(consensus_address, start_height, end_height) + .await?) + } + + pub async fn get_signed_between_times( + &self, + consensus_address: &str, + start_time: OffsetDateTime, + end_time: OffsetDateTime, + ) -> Result { + let Some(block_start) = self.get_first_block_height_after(start_time).await? else { + return Ok(0); + }; + let Some(block_end) = self.get_last_block_height_before(end_time).await? else { + return Ok(0); + }; + + self.get_signed_between(consensus_address, block_start, block_end) + .await + } + + pub async fn get_precommit( + &self, + consensus_address: &str, + height: i64, + ) -> Result, SqliteScraperError> { + Ok(self + .manager + .get_precommit(consensus_address, height) + .await?) + } + + pub async fn get_block_signers( + &self, + height: i64, + ) -> Result, SqliteScraperError> { + Ok(self.manager.get_block_validators(height).await?) + } + + pub async fn get_all_known_validators(&self) -> Result, SqliteScraperError> { + Ok(self.manager.get_validators().await?) + } + + pub async fn get_last_processed_height(&self) -> Result { + Ok(self.manager.get_last_processed_height().await?) + } + + pub async fn get_pruned_height(&self) -> Result { + Ok(self.manager.get_pruned_height().await?) + } +} + +#[async_trait] +impl NyxdScraperStorage for SqliteScraperStorage { + type StorageTransaction = SqliteStorageTransaction; + + async fn initialise(storage: &str) -> Result { + SqliteScraperStorage::init(storage) + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn begin_processing_tx( + &self, + ) -> Result { + self.begin_processing_tx() + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn get_last_processed_height(&self) -> Result { + self.get_last_processed_height() + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn get_pruned_height(&self) -> Result { + self.get_pruned_height() + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn lowest_block_height(&self) -> Result, NyxdScraperStorageError> { + self.lowest_block_height() + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn prune_storage( + &self, + oldest_to_keep: u32, + current_height: u32, + ) -> Result<(), NyxdScraperStorageError> { + self.prune_storage(oldest_to_keep, current_height) + .await + .map_err(NyxdScraperStorageError::from) + } +} diff --git a/common/nyxd-scraper/src/storage/manager.rs b/common/nyxd-scraper-sqlite/src/storage/manager.rs similarity index 99% rename from common/nyxd-scraper/src/storage/manager.rs rename to common/nyxd-scraper-sqlite/src/storage/manager.rs index fcc3485b952..34d894407db 100644 --- a/common/nyxd-scraper/src/storage/manager.rs +++ b/common/nyxd-scraper-sqlite/src/storage/manager.rs @@ -1,8 +1,8 @@ // Copyright 2023 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::storage::log_db_operation_time; use crate::storage::models::{CommitSignature, Validator}; +use nyxd_scraper_shared::storage::helpers::log_db_operation_time; use sqlx::types::time::OffsetDateTime; use sqlx::{Executor, Sqlite}; use tokio::time::Instant; diff --git a/common/nyxd-scraper/src/storage/helpers.rs b/common/nyxd-scraper-sqlite/src/storage/mod.rs similarity index 57% rename from common/nyxd-scraper/src/storage/helpers.rs rename to common/nyxd-scraper-sqlite/src/storage/mod.rs index 38dc36ebc38..e16997df982 100644 --- a/common/nyxd-scraper/src/storage/helpers.rs +++ b/common/nyxd-scraper-sqlite/src/storage/mod.rs @@ -1,2 +1,7 @@ // Copyright 2023 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 + +pub mod block_storage; +mod manager; +pub mod models; +pub mod transaction; diff --git a/common/nyxd-scraper-sqlite/src/storage/models.rs b/common/nyxd-scraper-sqlite/src/storage/models.rs new file mode 100644 index 00000000000..74a3dd9b1cf --- /dev/null +++ b/common/nyxd-scraper-sqlite/src/storage/models.rs @@ -0,0 +1,30 @@ +// Copyright 2023 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use sqlx::types::time::OffsetDateTime; +use sqlx::FromRow; + +#[derive(Debug, Clone, Eq, PartialEq, Hash, FromRow)] +pub struct Validator { + pub consensus_address: String, + pub consensus_pubkey: String, +} + +#[derive(Debug, Clone, FromRow)] +pub struct Block { + pub height: i64, + pub hash: String, + pub num_txs: u32, + pub total_gas: i64, + pub proposer_address: String, + pub timestamp: OffsetDateTime, +} + +#[derive(Debug, Clone, FromRow)] +pub struct CommitSignature { + pub height: i64, + pub validator_address: String, + pub voting_power: i64, + pub proposer_priority: i64, + pub timestamp: OffsetDateTime, +} diff --git a/common/nyxd-scraper-sqlite/src/storage/transaction.rs b/common/nyxd-scraper-sqlite/src/storage/transaction.rs new file mode 100644 index 00000000000..713cbb32615 --- /dev/null +++ b/common/nyxd-scraper-sqlite/src/storage/transaction.rs @@ -0,0 +1,236 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use crate::error::SqliteScraperError; +use crate::storage::manager::{ + insert_block, insert_message, insert_precommit, insert_transaction, insert_validator, +}; +use async_trait::async_trait; +use nyxd_scraper_shared::helpers::{ + validator_consensus_address, validator_info, validator_pubkey_to_bech32, +}; +use nyxd_scraper_shared::storage::validators::Response; +use nyxd_scraper_shared::storage::{ + validators, Block, Commit, CommitSig, NyxdScraperStorageError, NyxdScraperTransaction, +}; +use nyxd_scraper_shared::ParsedTransactionResponse; +use sqlx::{Sqlite, Transaction}; +use std::ops::{Deref, DerefMut}; +use tracing::{debug, trace, warn}; + +pub struct SqliteStorageTransaction(pub(crate) Transaction<'static, Sqlite>); + +impl Deref for SqliteStorageTransaction { + type Target = Transaction<'static, Sqlite>; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for SqliteStorageTransaction { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl SqliteStorageTransaction { + async fn persist_validators( + &mut self, + validators: &validators::Response, + ) -> Result<(), SqliteScraperError> { + debug!("persisting {} validators", validators.total); + for validator in &validators.validators { + let consensus_address = validator_consensus_address(validator.address)?; + let consensus_pubkey = validator_pubkey_to_bech32(validator.pub_key)?; + + insert_validator( + consensus_address.to_string(), + consensus_pubkey.to_string(), + self.0.as_mut(), + ) + .await?; + } + + Ok(()) + } + + async fn persist_block_data( + &mut self, + block: &Block, + total_gas: i64, + ) -> Result<(), SqliteScraperError> { + let proposer_address = + validator_consensus_address(block.header.proposer_address)?.to_string(); + + insert_block( + block.header.height.into(), + block.header.hash().to_string(), + block.data.len() as u32, + total_gas, + proposer_address, + block.header.time.into(), + self.0.as_mut(), + ) + .await?; + Ok(()) + } + + async fn persist_commits( + &mut self, + commits: &Commit, + validators: &validators::Response, + ) -> Result<(), SqliteScraperError> { + debug!("persisting up to {} commits", commits.signatures.len()); + let height: i64 = commits.height.into(); + + for commit_sig in &commits.signatures { + let (validator_id, timestamp, signature) = match commit_sig { + CommitSig::BlockIdFlagAbsent => { + trace!("absent signature"); + continue; + } + CommitSig::BlockIdFlagCommit { + validator_address, + timestamp, + signature, + } => (validator_address, timestamp, signature), + CommitSig::BlockIdFlagNil { + validator_address, + timestamp, + signature, + } => (validator_address, timestamp, signature), + }; + + let validator = validator_info(*validator_id, validators)?; + let validator_address = validator_consensus_address(*validator_id)?; + + if signature.is_none() { + warn!("empty signature for {validator_address} at height {height}"); + continue; + } + + insert_precommit( + validator_address.to_string(), + height, + (*timestamp).into(), + validator.power.into(), + validator.proposer_priority.value(), + self.0.as_mut(), + ) + .await?; + } + + Ok(()) + } + + async fn persist_txs( + &mut self, + txs: &[ParsedTransactionResponse], + ) -> Result<(), SqliteScraperError> { + debug!("persisting {} txs", txs.len()); + + for chain_tx in txs { + insert_transaction( + chain_tx.hash.to_string(), + chain_tx.height.into(), + chain_tx.index as i64, + chain_tx.tx_result.code.is_ok(), + chain_tx.tx.body.messages.len() as i64, + chain_tx.tx.body.memo.clone(), + chain_tx.tx_result.gas_wanted, + chain_tx.tx_result.gas_used, + chain_tx.tx_result.log.clone(), + self.0.as_mut(), + ) + .await?; + } + + Ok(()) + } + + async fn persist_messages( + &mut self, + txs: &[ParsedTransactionResponse], + ) -> Result<(), SqliteScraperError> { + debug!("persisting messages"); + + for chain_tx in txs { + for (index, msg) in chain_tx.tx.body.messages.iter().enumerate() { + insert_message( + chain_tx.hash.to_string(), + index as i64, + msg.type_url.clone(), + chain_tx.height.into(), + self.0.as_mut(), + ) + .await? + } + } + + Ok(()) + } +} + +#[async_trait] +impl NyxdScraperTransaction for SqliteStorageTransaction { + async fn commit(self) -> Result<(), NyxdScraperStorageError> { + self.0 + .commit() + .await + .map_err(SqliteScraperError::from) + .map_err(NyxdScraperStorageError::from) + } + + async fn persist_validators( + &mut self, + validators: &Response, + ) -> Result<(), NyxdScraperStorageError> { + self.persist_validators(validators) + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn persist_block_data( + &mut self, + block: &Block, + total_gas: i64, + ) -> Result<(), NyxdScraperStorageError> { + self.persist_block_data(block, total_gas) + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn persist_commits( + &mut self, + commits: &Commit, + validators: &Response, + ) -> Result<(), NyxdScraperStorageError> { + self.persist_commits(commits, validators) + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn persist_txs( + &mut self, + txs: &[ParsedTransactionResponse], + ) -> Result<(), NyxdScraperStorageError> { + self.persist_txs(txs) + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn persist_messages( + &mut self, + txs: &[ParsedTransactionResponse], + ) -> Result<(), NyxdScraperStorageError> { + self.persist_messages(txs) + .await + .map_err(NyxdScraperStorageError::from) + } + + async fn update_last_processed(&mut self, height: i64) -> Result<(), NyxdScraperStorageError> { + self.update_last_processed(height) + .await + .map_err(NyxdScraperStorageError::from) + } +} diff --git a/common/nyxd-scraper/src/helpers.rs b/common/nyxd-scraper/src/helpers.rs deleted file mode 100644 index 44bef81b89c..00000000000 --- a/common/nyxd-scraper/src/helpers.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2023 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -use crate::block_processor::types::ParsedTransactionResponse; -use crate::constants::{BECH32_CONESNSUS_PUBKEY_PREFIX, BECH32_CONSENSUS_ADDRESS_PREFIX}; -use crate::error::ScraperError; -use cosmrs::AccountId; -use sha2::{Digest, Sha256}; -use tendermint::{Hash, validator}; -use tendermint::{PublicKey, account}; -use tendermint_rpc::endpoint::validators; - -pub(crate) fn tx_hash>(raw_tx: M) -> Hash { - Hash::Sha256(Sha256::digest(raw_tx).into()) -} - -pub(crate) fn validator_pubkey_to_bech32(pubkey: PublicKey) -> Result { - // TODO: this one seem to attach additional prefix to they pubkeys, is that what we want instead maybe? - // Ok(pubkey.to_bech32(BECH32_CONESNSUS_PUBKEY_PREFIX)) - AccountId::new(BECH32_CONESNSUS_PUBKEY_PREFIX, &pubkey.to_bytes()) - .map_err(|source| ScraperError::MalformedValidatorPubkey { source }) -} - -pub(crate) fn validator_consensus_address(id: account::Id) -> Result { - AccountId::new(BECH32_CONSENSUS_ADDRESS_PREFIX, id.as_ref()) - .map_err(|source| ScraperError::MalformedValidatorAddress { source }) -} - -pub(crate) fn tx_gas_sum(txs: &[ParsedTransactionResponse]) -> i64 { - txs.iter().map(|tx| tx.tx_result.gas_used).sum() -} - -pub(crate) fn validator_info( - id: account::Id, - validators: &validators::Response, -) -> Result<&validator::Info, ScraperError> { - match validators.validators.iter().find(|v| v.address == id) { - Some(info) => Ok(info), - None => { - let addr = validator_consensus_address(id)?; - Err(ScraperError::MissingValidatorInfoCommitted { - address: addr.to_string(), - }) - } - } -} diff --git a/common/nyxd-scraper/src/storage/mod.rs b/common/nyxd-scraper/src/storage/mod.rs deleted file mode 100644 index 2f0a0a8660c..00000000000 --- a/common/nyxd-scraper/src/storage/mod.rs +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright 2023 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 - -use crate::{ - block_processor::types::{FullBlockInformation, ParsedTransactionResponse}, - error::ScraperError, - storage::{ - manager::{ - StorageManager, insert_block, insert_message, insert_precommit, insert_transaction, - insert_validator, prune_blocks, prune_messages, prune_pre_commits, prune_transactions, - update_last_processed, update_last_pruned, - }, - models::{CommitSignature, Validator}, - }, -}; -use sqlx::{ - ConnectOptions, Sqlite, Transaction, - sqlite::{SqliteAutoVacuum, SqliteSynchronous}, - types::time::OffsetDateTime, -}; -use std::{fmt::Debug, path::Path}; -use tendermint::{ - Block, - block::{Commit, CommitSig}, -}; -use tendermint_rpc::endpoint::validators; -use tokio::time::Instant; -use tracing::{debug, error, info, instrument, trace, warn}; - -mod helpers; -mod manager; -pub mod models; - -pub type StorageTransaction = Transaction<'static, Sqlite>; - -#[derive(Clone)] -pub struct ScraperStorage { - pub(crate) manager: StorageManager, -} - -pub(crate) fn log_db_operation_time(op_name: &str, start_time: Instant) { - let elapsed = start_time.elapsed(); - let formatted = humantime::format_duration(elapsed); - - match elapsed.as_millis() { - v if v > 10000 => error!("{op_name} took {formatted} to execute"), - v if v > 1000 => warn!("{op_name} took {formatted} to execute"), - v if v > 100 => info!("{op_name} took {formatted} to execute"), - v if v > 10 => debug!("{op_name} took {formatted} to execute"), - _ => trace!("{op_name} took {formatted} to execute"), - } -} - -impl ScraperStorage { - #[instrument] - pub async fn init + Debug>(database_path: P) -> Result { - let database_path = database_path.as_ref(); - debug!( - "initialising scraper database path to '{}'", - database_path.display() - ); - - let opts = sqlx::sqlite::SqliteConnectOptions::new() - .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal) - .synchronous(SqliteSynchronous::Normal) - .auto_vacuum(SqliteAutoVacuum::Incremental) - .filename(database_path) - .create_if_missing(true) - .disable_statement_logging(); - - // TODO: do we want auto_vacuum ? - - let connection_pool = match sqlx::SqlitePool::connect_with(opts).await { - Ok(db) => db, - Err(err) => { - error!("Failed to connect to SQLx database: {err}"); - return Err(err.into()); - } - }; - - if let Err(err) = sqlx::migrate!("./sql_migrations") - .run(&connection_pool) - .await - { - error!("Failed to initialize SQLx database: {err}"); - return Err(err.into()); - } - - info!("Database migration finished!"); - - let manager = StorageManager { connection_pool }; - manager.set_initial_metadata().await?; - - let storage = ScraperStorage { manager }; - - Ok(storage) - } - - #[instrument(skip(self))] - pub async fn prune_storage( - &self, - oldest_to_keep: u32, - current_height: u32, - ) -> Result<(), ScraperError> { - let start = Instant::now(); - - let mut tx = self.begin_processing_tx().await?; - - prune_messages(oldest_to_keep.into(), &mut *tx).await?; - prune_transactions(oldest_to_keep.into(), &mut *tx).await?; - prune_pre_commits(oldest_to_keep.into(), &mut *tx).await?; - prune_blocks(oldest_to_keep.into(), &mut *tx).await?; - update_last_pruned(current_height.into(), &mut *tx).await?; - - let commit_start = Instant::now(); - tx.commit() - .await - .map_err(|source| ScraperError::StorageTxCommitFailure { source })?; - log_db_operation_time("committing pruning tx", commit_start); - - log_db_operation_time("pruning storage", start); - Ok(()) - } - - #[instrument(skip_all)] - pub async fn begin_processing_tx(&self) -> Result { - debug!("starting storage tx"); - self.manager - .connection_pool - .begin() - .await - .map_err(|source| ScraperError::StorageTxBeginFailure { source }) - } - - pub async fn lowest_block_height(&self) -> Result, ScraperError> { - Ok(self.manager.get_lowest_block().await?) - } - - pub async fn get_first_block_height_after( - &self, - time: OffsetDateTime, - ) -> Result, ScraperError> { - Ok(self.manager.get_first_block_height_after(time).await?) - } - - pub async fn get_last_block_height_before( - &self, - time: OffsetDateTime, - ) -> Result, ScraperError> { - Ok(self.manager.get_last_block_height_before(time).await?) - } - - pub async fn get_blocks_between( - &self, - start_time: OffsetDateTime, - end_time: OffsetDateTime, - ) -> Result { - let Some(block_start) = self.get_first_block_height_after(start_time).await? else { - return Ok(0); - }; - let Some(block_end) = self.get_last_block_height_before(end_time).await? else { - return Ok(0); - }; - - Ok(block_end - block_start) - } - - pub async fn get_signed_between( - &self, - consensus_address: &str, - start_height: i64, - end_height: i64, - ) -> Result { - Ok(self - .manager - .get_signed_between(consensus_address, start_height, end_height) - .await?) - } - - pub async fn get_signed_between_times( - &self, - consensus_address: &str, - start_time: OffsetDateTime, - end_time: OffsetDateTime, - ) -> Result { - let Some(block_start) = self.get_first_block_height_after(start_time).await? else { - return Ok(0); - }; - let Some(block_end) = self.get_last_block_height_before(end_time).await? else { - return Ok(0); - }; - - self.get_signed_between(consensus_address, block_start, block_end) - .await - } - - pub async fn get_precommit( - &self, - consensus_address: &str, - height: i64, - ) -> Result, ScraperError> { - Ok(self - .manager - .get_precommit(consensus_address, height) - .await?) - } - - pub async fn get_block_signers(&self, height: i64) -> Result, ScraperError> { - Ok(self.manager.get_block_validators(height).await?) - } - - pub async fn get_all_known_validators(&self) -> Result, ScraperError> { - Ok(self.manager.get_validators().await?) - } - - pub async fn get_last_processed_height(&self) -> Result { - Ok(self.manager.get_last_processed_height().await?) - } - - pub async fn get_pruned_height(&self) -> Result { - Ok(self.manager.get_pruned_height().await?) - } -} - -pub async fn persist_block( - block: &FullBlockInformation, - tx: &mut StorageTransaction, - store_precommits: bool, -) -> Result<(), ScraperError> { - let total_gas = crate::helpers::tx_gas_sum(&block.transactions); - - // SANITY CHECK: make sure the block proposer is present in the validator set - block.ensure_proposer()?; - - // persist validators - persist_validators(&block.validators, tx).await?; - - // persist block data - persist_block_data(&block.block, total_gas, tx).await?; - - if store_precommits { - if let Some(commit) = &block.block.last_commit { - persist_commits(commit, &block.validators, tx).await?; - } else { - warn!("no commits for block {}", block.block.header.height) - } - } - - // persist txs - persist_txs(&block.transactions, tx).await?; - - // persist messages (inside the transactions) - persist_messages(&block.transactions, tx).await?; - - update_last_processed(block.block.header.height.into(), tx.as_mut()).await?; - - Ok(()) -} - -async fn persist_validators( - validators: &validators::Response, - tx: &mut StorageTransaction, -) -> Result<(), ScraperError> { - debug!("persisting {} validators", validators.total); - for validator in &validators.validators { - let consensus_address = crate::helpers::validator_consensus_address(validator.address)?; - let consensus_pubkey = crate::helpers::validator_pubkey_to_bech32(validator.pub_key)?; - - insert_validator( - consensus_address.to_string(), - consensus_pubkey.to_string(), - tx.as_mut(), - ) - .await?; - } - - Ok(()) -} - -async fn persist_block_data( - block: &Block, - total_gas: i64, - tx: &mut StorageTransaction, -) -> Result<(), ScraperError> { - let proposer_address = - crate::helpers::validator_consensus_address(block.header.proposer_address)?.to_string(); - - insert_block( - block.header.height.into(), - block.header.hash().to_string(), - block.data.len() as u32, - total_gas, - proposer_address, - block.header.time.into(), - tx.as_mut(), - ) - .await?; - Ok(()) -} - -async fn persist_commits( - commits: &Commit, - validators: &validators::Response, - tx: &mut StorageTransaction, -) -> Result<(), ScraperError> { - debug!("persisting up to {} commits", commits.signatures.len()); - let height: i64 = commits.height.into(); - - for commit_sig in &commits.signatures { - let (validator_id, timestamp, signature) = match commit_sig { - CommitSig::BlockIdFlagAbsent => { - trace!("absent signature"); - continue; - } - CommitSig::BlockIdFlagCommit { - validator_address, - timestamp, - signature, - } => (validator_address, timestamp, signature), - CommitSig::BlockIdFlagNil { - validator_address, - timestamp, - signature, - } => (validator_address, timestamp, signature), - }; - - let validator = match crate::helpers::validator_info(*validator_id, validators) { - Ok(validator_info) => validator_info, - Err(err) => { - error!("{err}"); - continue; - } - }; - let validator_address = crate::helpers::validator_consensus_address(*validator_id)?; - - if signature.is_none() { - warn!("empty signature for {validator_address} at height {height}"); - continue; - } - - insert_precommit( - validator_address.to_string(), - height, - (*timestamp).into(), - validator.power.into(), - validator.proposer_priority.value(), - tx.as_mut(), - ) - .await?; - } - - Ok(()) -} - -async fn persist_txs( - txs: &[ParsedTransactionResponse], - tx: &mut StorageTransaction, -) -> Result<(), ScraperError> { - debug!("persisting {} txs", txs.len()); - - for chain_tx in txs { - insert_transaction( - chain_tx.hash.to_string(), - chain_tx.height.into(), - chain_tx.index as i64, - chain_tx.tx_result.code.is_ok(), - chain_tx.tx.body.messages.len() as i64, - chain_tx.tx.body.memo.clone(), - chain_tx.tx_result.gas_wanted, - chain_tx.tx_result.gas_used, - chain_tx.tx_result.log.clone(), - tx.as_mut(), - ) - .await?; - } - - Ok(()) -} - -async fn persist_messages( - txs: &[ParsedTransactionResponse], - tx: &mut StorageTransaction, -) -> Result<(), ScraperError> { - debug!("persisting messages"); - - for chain_tx in txs { - for (index, msg) in chain_tx.tx.body.messages.iter().enumerate() { - insert_message( - chain_tx.hash.to_string(), - index as i64, - msg.type_url.clone(), - chain_tx.height.into(), - tx.as_mut(), - ) - .await? - } - } - - Ok(()) -} diff --git a/nym-validator-rewarder/Cargo.toml b/nym-validator-rewarder/Cargo.toml index be1ae056dc2..0655bd21eff 100644 --- a/nym-validator-rewarder/Cargo.toml +++ b/nym-validator-rewarder/Cargo.toml @@ -44,7 +44,7 @@ nym-task = { path = "../common/task" } nym-validator-client = { path = "../common/client-libs/validator-client" } nym-http-api-client = { path = "../common/http-api-client" } nym-coconut-dkg-common = { path = "../common/cosmwasm-smart-contracts/coconut-dkg" } -nyxd-scraper = { path = "../common/nyxd-scraper" } +nyxd-scraper-sqlite = { path = "../common/nyxd-scraper-sqlite" } nym-ticketbooks-merkle = { path = "../common/ticketbooks-merkle" } nym-serde-helpers = { path = "../common/serde-helpers", features = ["base64"] } nym-pemstore = { path = "../common/pemstore" } diff --git a/nym-validator-rewarder/src/cli/process_block.rs b/nym-validator-rewarder/src/cli/process_block.rs index 0174f1165c6..aeb249076de 100644 --- a/nym-validator-rewarder/src/cli/process_block.rs +++ b/nym-validator-rewarder/src/cli/process_block.rs @@ -3,7 +3,7 @@ use crate::cli::{ConfigOverridableArgs, try_load_current_config}; use crate::error::NymRewarderError; -use nyxd_scraper::NyxdScraper; +use nyxd_scraper_sqlite::SqliteNyxdScraper; use std::path::PathBuf; #[derive(Debug, clap::Args)] @@ -24,7 +24,7 @@ pub(crate) async fn execute(args: Args) -> Result<(), NymRewarderError> { let config = try_load_current_config(&args.custom_config_path)?.with_override(args.config_override); - NyxdScraper::new(config.scraper_config()) + SqliteNyxdScraper::new(config.scraper_config()) .await? .unsafe_process_single_block(args.height) .await?; diff --git a/nym-validator-rewarder/src/cli/process_until.rs b/nym-validator-rewarder/src/cli/process_until.rs index 159f954e574..06c3c68f870 100644 --- a/nym-validator-rewarder/src/cli/process_until.rs +++ b/nym-validator-rewarder/src/cli/process_until.rs @@ -3,7 +3,7 @@ use crate::cli::{ConfigOverridableArgs, try_load_current_config}; use crate::error::NymRewarderError; -use nyxd_scraper::NyxdScraper; +use nyxd_scraper_sqlite::SqliteNyxdScraper; use std::path::PathBuf; #[derive(Debug, clap::Args)] @@ -37,7 +37,7 @@ pub(crate) async fn execute(args: Args) -> Result<(), NymRewarderError> { let config = try_load_current_config(&args.custom_config_path)?.with_override(args.config_override); - NyxdScraper::new(config.scraper_config()) + SqliteNyxdScraper::new(config.scraper_config()) .await? .unsafe_process_block_range(args.start_height, args.stop_height) .await?; diff --git a/nym-validator-rewarder/src/config/mod.rs b/nym-validator-rewarder/src/config/mod.rs index 618d4ef27a5..b3824f21ff1 100644 --- a/nym-validator-rewarder/src/config/mod.rs +++ b/nym-validator-rewarder/src/config/mod.rs @@ -12,7 +12,7 @@ use nym_config::{ must_get_home, read_config_from_toml_file, save_formatted_config_to_file, }; use nym_validator_client::nyxd::{AccountId, Coin}; -use nyxd_scraper::{PruningOptions, StartingBlockOpts}; +use nyxd_scraper_sqlite::{PruningOptions, StartingBlockOpts}; use serde::{Deserialize, Serialize}; use serde_with::{DisplayFromStr, serde_as}; use std::io; @@ -119,11 +119,11 @@ impl Config { } } - pub fn scraper_config(&self) -> nyxd_scraper::Config { - nyxd_scraper::Config { + pub fn scraper_config(&self) -> nyxd_scraper_sqlite::Config { + nyxd_scraper_sqlite::Config { websocket_url: self.nyxd_scraper.websocket_url.clone(), rpc_url: self.base.upstream_nyxd.clone(), - database_path: self.storage_paths.nyxd_scraper.clone(), + database_storage: self.storage_paths.nyxd_scraper.clone(), pruning_options: self.nyxd_scraper.pruning, store_precommits: self.nyxd_scraper.store_precommits, start_block: StartingBlockOpts { diff --git a/nym-validator-rewarder/src/config/persistence/paths.rs b/nym-validator-rewarder/src/config/persistence/paths.rs index 6d9baa66f97..47dd3bf8111 100644 --- a/nym-validator-rewarder/src/config/persistence/paths.rs +++ b/nym-validator-rewarder/src/config/persistence/paths.rs @@ -15,7 +15,7 @@ pub const DEFAULT_ED25519_PUBLIC_IDENTITY_KEY_FILENAME: &str = "ed25519_identity #[derive(Debug, Clone, Deserialize, Serialize)] pub struct ValidatorRewarderPaths { - pub nyxd_scraper: PathBuf, + pub nyxd_scraper: String, pub reward_history: PathBuf, @@ -46,7 +46,10 @@ impl ValidatorRewarderPaths { impl Default for ValidatorRewarderPaths { fn default() -> Self { ValidatorRewarderPaths { - nyxd_scraper: default_data_directory().join(DEFAULT_SCRAPER_DB_FILENAME), + // validator rewarder uses sqlite + nyxd_scraper: (default_data_directory().join(DEFAULT_SCRAPER_DB_FILENAME)) + .to_string_lossy() + .to_string(), reward_history: default_data_directory().join(DEFAULT_REWARD_HISTORY_DB_FILENAME), private_ed25519_identity_key_file: default_data_directory() .join(DEFAULT_ED25519_PRIVATE_IDENTITY_KEY_FILENAME), diff --git a/nym-validator-rewarder/src/error.rs b/nym-validator-rewarder/src/error.rs index 8adbe9883ff..d20ee0e81c9 100644 --- a/nym-validator-rewarder/src/error.rs +++ b/nym-validator-rewarder/src/error.rs @@ -9,6 +9,7 @@ use nym_validator_client::nym_api::error::NymAPIError; use nym_validator_client::nyxd::error::NyxdError; use nym_validator_client::nyxd::tx::ErrorReport; use nym_validator_client::nyxd::{AccountId, Coin}; +use nyxd_scraper_sqlite::error::SqliteScraperError; use std::io; use std::path::PathBuf; use thiserror::Error; @@ -81,7 +82,13 @@ pub enum NymRewarderError { #[error("chain scraping failure: {source}")] ScraperFailure { #[from] - source: nyxd_scraper::error::ScraperError, + source: nyxd_scraper_sqlite::ScraperError, + }, + + #[error("chain scraper storage failure: {source}")] + ScraperStorageFailure { + #[from] + source: SqliteScraperError, }, // this should never happen but unwrapping everywhere was more cumbersome than just propagating the error diff --git a/nym-validator-rewarder/src/rewarder/block_signing/mod.rs b/nym-validator-rewarder/src/rewarder/block_signing/mod.rs index 57406c34ccb..d86a3365b85 100644 --- a/nym-validator-rewarder/src/rewarder/block_signing/mod.rs +++ b/nym-validator-rewarder/src/rewarder/block_signing/mod.rs @@ -7,7 +7,7 @@ use crate::rewarder::epoch::Epoch; use crate::rewarder::nyxd_client::NyxdClient; use nym_validator_client::nyxd::module_traits::staking; use nym_validator_client::nyxd::{AccountId, PageRequest}; -use nyxd_scraper::NyxdScraper; +use nyxd_scraper_sqlite::SqliteNyxdScraper; use std::cmp::min; use std::collections::HashMap; use std::ops::Range; @@ -17,7 +17,7 @@ pub(crate) mod types; pub struct EpochSigning { pub(crate) nyxd_client: NyxdClient, - pub(crate) nyxd_scraper: NyxdScraper, + pub(crate) nyxd_scraper: SqliteNyxdScraper, pub(crate) whitelist: Vec, } diff --git a/nym-validator-rewarder/src/rewarder/block_signing/types.rs b/nym-validator-rewarder/src/rewarder/block_signing/types.rs index 497d6546da1..559e6991b6e 100644 --- a/nym-validator-rewarder/src/rewarder/block_signing/types.rs +++ b/nym-validator-rewarder/src/rewarder/block_signing/types.rs @@ -7,7 +7,7 @@ use crate::{ }; use cosmwasm_std::{Decimal, Uint128}; use nym_validator_client::nyxd::{AccountId, Coin, module_traits::staking}; -use nyxd_scraper::models; +use nyxd_scraper_sqlite::models; use std::collections::HashMap; use tracing::info; diff --git a/nym-validator-rewarder/src/rewarder/helpers.rs b/nym-validator-rewarder/src/rewarder/helpers.rs index 578d49c498c..572e4ecbc79 100644 --- a/nym-validator-rewarder/src/rewarder/helpers.rs +++ b/nym-validator-rewarder/src/rewarder/helpers.rs @@ -3,7 +3,7 @@ use crate::error::NymRewarderError; use nym_validator_client::nyxd::{AccountId, PublicKey}; -use nyxd_scraper::constants::{BECH32_CONSENSUS_ADDRESS_PREFIX, BECH32_PREFIX}; +use nyxd_scraper_sqlite::constants::{BECH32_CONSENSUS_ADDRESS_PREFIX, BECH32_PREFIX}; use sha2::{Digest, Sha256}; pub(crate) fn consensus_pubkey_to_address( diff --git a/nym-validator-rewarder/src/rewarder/mod.rs b/nym-validator-rewarder/src/rewarder/mod.rs index e46dcc3e791..a5d43bca058 100644 --- a/nym-validator-rewarder/src/rewarder/mod.rs +++ b/nym-validator-rewarder/src/rewarder/mod.rs @@ -16,7 +16,7 @@ use nym_crypto::asymmetric::ed25519; use nym_ecash_time::{EcashTime, ecash_today, ecash_today_date}; use nym_task::ShutdownManager; use nym_validator_client::nyxd::{AccountId, Coin, Hash}; -use nyxd_scraper::NyxdScraper; +use nyxd_scraper_sqlite::SqliteNyxdScraper; use std::sync::Arc; use time::Date; use tracing::{error, info, instrument, warn}; @@ -187,7 +187,7 @@ impl Rewarder { info!("the block signing rewarding is running in monitor only mode"); } - let nyxd_scraper = NyxdScraper::new(config.scraper_config()).await?; + let nyxd_scraper = SqliteNyxdScraper::new(config.scraper_config()).await?; Some(EpochSigning { nyxd_scraper, diff --git a/nym-wallet/Cargo.lock b/nym-wallet/Cargo.lock index 19db598a322..e539f36c598 100644 --- a/nym-wallet/Cargo.lock +++ b/nym-wallet/Cargo.lock @@ -11,7 +11,7 @@ dependencies = [ "bip39", "cfg-if", "colored 2.2.0", - "cosmrs", + "cosmrs 0.21.1", "cosmwasm-std", "dirs 4.0.0", "dotenvy", @@ -1249,6 +1249,16 @@ dependencies = [ "tendermint-proto", ] +[[package]] +name = "cosmos-sdk-proto" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ac39be7373404accccaede7cc1ec942ccef14f0ca18d209967a756bf1dbb1f" +dependencies = [ + "prost", + "tendermint-proto", +] + [[package]] name = "cosmrs" version = "0.21.1" @@ -1256,7 +1266,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1394c263335da09e8ba8c4b2c675d804e3e0deb44cce0866a5f838d3ddd43d02" dependencies = [ "bip32", - "cosmos-sdk-proto", + "cosmos-sdk-proto 0.26.1", + "ecdsa", + "eyre", + "k256", + "rand_core 0.6.4", + "serde", + "serde_json", + "signature", + "subtle-encoding", + "tendermint", + "thiserror 1.0.69", +] + +[[package]] +name = "cosmrs" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34e74fa7a22930fe0579bef560f2d64b78415d4c47b9dd976c0635136809471d" +dependencies = [ + "bip32", + "cosmos-sdk-proto 0.27.0", "ecdsa", "eyre", "k256", @@ -4156,7 +4186,7 @@ name = "nym-api-requests" version = "0.1.0" dependencies = [ "bs58", - "cosmrs", + "cosmrs 0.22.0", "cosmwasm-std", "ecdsa", "hex", @@ -4568,7 +4598,7 @@ name = "nym-types" version = "1.0.0" dependencies = [ "base64 0.22.1", - "cosmrs", + "cosmrs 0.22.0", "cosmwasm-std", "eyre", "hmac", @@ -4618,7 +4648,7 @@ dependencies = [ "bip32", "bip39", "colored 2.2.0", - "cosmrs", + "cosmrs 0.22.0", "cosmwasm-std", "cw-controllers", "cw-utils", @@ -4688,7 +4718,7 @@ dependencies = [ name = "nym-wallet-types" version = "1.0.0" dependencies = [ - "cosmrs", + "cosmrs 0.21.1", "cosmwasm-std", "hex-literal", "nym-config", diff --git a/nyx-chain-watcher/Cargo.toml b/nyx-chain-watcher/Cargo.toml index bbc2ed19396..3a2f09149b5 100644 --- a/nyx-chain-watcher/Cargo.toml +++ b/nyx-chain-watcher/Cargo.toml @@ -24,7 +24,7 @@ nym-bin-common = { path = "../common/bin-common", features = ["output_format"] } nym-network-defaults = { path = "../common/network-defaults" } nym-task = { path = "../common/task" } nym-validator-client = { path = "../common/client-libs/validator-client" } -nyxd-scraper = { path = "../common/nyxd-scraper" } +nyxd-scraper-sqlite = { path = "../common/nyxd-scraper-sqlite" } reqwest = { workspace = true, features = ["rustls-tls"] } schemars = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/nyx-chain-watcher/src/chain_scraper/mod.rs b/nyx-chain-watcher/src/chain_scraper/mod.rs index 6f2e493f315..f8738f195c4 100644 --- a/nyx-chain-watcher/src/chain_scraper/mod.rs +++ b/nyx-chain-watcher/src/chain_scraper/mod.rs @@ -6,9 +6,9 @@ use crate::env::vars::{ use crate::http::state::BankScraperModuleState; use async_trait::async_trait; use nym_validator_client::nyxd::{Any, Coin, CosmosCoin, Hash, Msg, MsgSend, Name}; -use nyxd_scraper::{ - MsgModule, NyxdScraper, ParsedTransactionResponse, PruningOptions, error::ScraperError, - storage::StorageTransaction, +use nyxd_scraper_sqlite::{ + MsgModule, NyxdScraperTransaction, ParsedTransactionResponse, PruningOptions, ScraperError, + SqliteNyxdScraper, }; use sqlx::SqlitePool; use std::fs; @@ -18,7 +18,7 @@ pub(crate) async fn run_chain_scraper( config: &crate::config::Config, db_pool: SqlitePool, shared_state: BankScraperModuleState, -) -> anyhow::Result { +) -> anyhow::Result { let websocket_url = std::env::var("NYXD_WS").expect("NYXD_WS not defined"); let rpc_url = std::env::var("NYXD").expect("NYXD not defined"); @@ -50,13 +50,13 @@ pub(crate) async fn run_chain_scraper( fs::remove_file(config.chain_scraper_database_path())?; } - let scraper = NyxdScraper::builder(nyxd_scraper::Config { + let scraper = SqliteNyxdScraper::builder(nyxd_scraper_sqlite::Config { websocket_url, rpc_url, - database_path: config.chain_scraper_database_path().into(), + database_storage: config.chain_scraper_database_path().into(), pruning_options: PruningOptions::nothing(), store_precommits: false, - start_block: nyxd_scraper::StartingBlockOpts { + start_block: nyxd_scraper_sqlite::StartingBlockOpts { start_block_height, use_best_effort_start_height, }, @@ -157,7 +157,7 @@ impl MsgModule for BankScraperModule { index: usize, msg: &Any, tx: &ParsedTransactionResponse, - _storage_tx: &mut StorageTransaction, + _storage_tx: &mut dyn NyxdScraperTransaction, ) -> Result<(), ScraperError> { let memo = tx.tx.body.memo.clone(); diff --git a/nyx-chain-watcher/src/http/state.rs b/nyx-chain-watcher/src/http/state.rs index 8cb9d93e9f5..8ab26dff9b1 100644 --- a/nyx-chain-watcher/src/http/state.rs +++ b/nyx-chain-watcher/src/http/state.rs @@ -7,7 +7,7 @@ use axum::extract::FromRef; use nym_bin_common::bin_info; use nym_bin_common::build_information::BinaryBuildInformation; use nym_validator_client::nyxd::{Coin, MsgSend}; -use nyxd_scraper::ParsedTransactionResponse; +use nyxd_scraper_sqlite::ParsedTransactionResponse; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::ops::Deref; From 188fb5b9702e7ef0a43a988d8a3ee46bd2162f59 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 22 Jul 2025 00:47:14 +0100 Subject: [PATCH 02/28] add the data observatory --- Cargo.lock | 171 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 155 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62d01b3e6da..280cf9f29b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1328,7 +1328,7 @@ dependencies = [ "futures-core", "prost", "prost-types", - "tonic", + "tonic 0.12.3", "tracing-core", ] @@ -1352,7 +1352,7 @@ dependencies = [ "thread_local", "tokio", "tokio-stream", - "tonic", + "tonic 0.12.3", "tracing", "tracing-core", "tracing-subscriber", @@ -1442,6 +1442,19 @@ dependencies = [ "tendermint-proto", ] +[[package]] +name = "cosmos-sdk-proto" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ac39be7373404accccaede7cc1ec942ccef14f0ca18d209967a756bf1dbb1f" +dependencies = [ + "informalsystems-pbjson", + "prost", + "serde", + "tendermint-proto", + "tonic 0.13.1", +] + [[package]] name = "cosmrs" version = "0.21.1" @@ -1449,7 +1462,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1394c263335da09e8ba8c4b2c675d804e3e0deb44cce0866a5f838d3ddd43d02" dependencies = [ "bip32", - "cosmos-sdk-proto", + "cosmos-sdk-proto 0.26.1", + "ecdsa", + "eyre", + "k256", + "rand_core 0.6.4", + "serde", + "serde_json", + "signature", + "subtle-encoding", + "tendermint", + "thiserror 1.0.69", +] + +[[package]] +name = "cosmrs" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34e74fa7a22930fe0579bef560f2d64b78415d4c47b9dd976c0635136809471d" +dependencies = [ + "bip32", + "cosmos-sdk-proto 0.27.0", "ecdsa", "eyre", "k256", @@ -3604,6 +3637,39 @@ dependencies = [ "cc", ] +[[package]] +name = "ibc-proto" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a650b51e384e54264b53974feb38e95e37aac70f7f2f9c07eb8022fe15eb8e20" +dependencies = [ + "base64 0.22.1", + "bytes", + "cosmos-sdk-proto 0.27.0", + "flex-error", + "ics23", + "informalsystems-pbjson", + "prost", + "serde", + "subtle-encoding", + "tendermint-proto", + "tonic 0.13.1", +] + +[[package]] +name = "ics23" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b17f1a5bd7d12ad30a21445cfa5f52fd7651cb3243ba866f9916b1ec112f12" +dependencies = [ + "anyhow", + "bytes", + "hex", + "informalsystems-pbjson", + "prost", + "serde", +] + [[package]] name = "icu_collections" version = "2.0.0" @@ -3833,6 +3899,16 @@ dependencies = [ "web-time", ] +[[package]] +name = "informalsystems-pbjson" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa4a0980c8379295100d70854354e78df2ee1c6ca0f96ffe89afeb3140e3a3d" +dependencies = [ + "base64 0.21.7", + "serde", +] + [[package]] name = "inotify" version = "0.9.6" @@ -4920,7 +4996,7 @@ name = "nym-api-requests" version = "0.1.0" dependencies = [ "bs58", - "cosmrs", + "cosmrs 0.22.0", "cosmwasm-std", "ecdsa", "hex", @@ -5091,7 +5167,7 @@ dependencies = [ "clap", "colored", "comfy-table", - "cosmrs", + "cosmrs 0.22.0", "cosmwasm-std", "csv", "cw-utils", @@ -5257,7 +5333,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "cosmrs", + "cosmrs 0.22.0", "nym-crypto", "nym-gateway-requests", "serde", @@ -5632,7 +5708,7 @@ version = "0.1.0" dependencies = [ "bincode", "bls12_381", - "cosmrs", + "cosmrs 0.22.0", "log", "nym-api-requests", "nym-credentials-interface", @@ -7358,7 +7434,7 @@ name = "nym-types" version = "1.0.0" dependencies = [ "base64 0.22.1", - "cosmrs", + "cosmrs 0.22.0", "cosmwasm-std", "eyre", "hmac", @@ -7412,7 +7488,7 @@ dependencies = [ "bip32", "bip39", "colored", - "cosmrs", + "cosmrs 0.22.0", "cosmwasm-std", "cw-controllers", "cw-utils", @@ -7480,7 +7556,7 @@ dependencies = [ "nym-task", "nym-ticketbooks-merkle", "nym-validator-client", - "nyxd-scraper", + "nyxd-scraper-sqlite", "rand 0.8.5", "rand_chacha 0.3.1", "serde", @@ -7560,7 +7636,7 @@ dependencies = [ name = "nym-wallet-types" version = "1.0.0" dependencies = [ - "cosmrs", + "cosmrs 0.21.1", "cosmwasm-std", "hex-literal", "nym-config", @@ -7722,7 +7798,7 @@ dependencies = [ "nym-network-defaults", "nym-task", "nym-validator-client", - "nyxd-scraper", + "nyxd-scraper-sqlite", "reqwest 0.12.22", "schemars 0.8.22", "serde", @@ -7740,19 +7816,39 @@ dependencies = [ ] [[package]] -name = "nyxd-scraper" +name = "nyxd-scraper-psql" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "cosmrs 0.22.0", + "itertools 0.14.0", + "nyxd-scraper-shared", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.12", + "tokio", + "tracing", +] + +[[package]] +name = "nyxd-scraper-shared" version = "0.1.0" dependencies = [ - "anyhow", "async-trait", + "base64 0.22.1", "const_format", - "cosmrs", + "cosmos-sdk-proto 0.27.0", + "cosmrs 0.22.0", "eyre", "futures", "humantime", + "ibc-proto", + "prost", "serde", + "serde_json", "sha2 0.10.9", - "sqlx", "tendermint", "tendermint-rpc", "thiserror 2.0.12", @@ -7764,6 +7860,18 @@ dependencies = [ "url", ] +[[package]] +name = "nyxd-scraper-sqlite" +version = "0.1.0" +dependencies = [ + "async-trait", + "nyxd-scraper-shared", + "sqlx", + "thiserror 2.0.12", + "tokio", + "tracing", +] + [[package]] name = "objc2-core-foundation" version = "0.3.1" @@ -11011,6 +11119,34 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "h2 0.4.11", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.6.0", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower 0.5.2", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -11039,9 +11175,12 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", + "indexmap 2.10.0", "pin-project-lite", + "slab", "sync_wrapper 1.0.2", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", From b7c99f802d78a97ab5c95d7ddcee162aaf48ec1d Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 22 Jul 2025 00:47:14 +0100 Subject: [PATCH 03/28] add the data observatory --- Cargo.lock | 647 +++++++++--------- Cargo.toml | 1 + .../{01_metadata.sql => 0001_metadata.sql} | 2 +- .../{02_cosmos.sql => 0002_cosmos.sql} | 28 +- .../nyxd-scraper-psql/src/storage/manager.rs | 81 ++- .../nyxd-scraper-psql/src/storage/models.rs | 10 +- .../src/storage/transaction.rs | 59 +- .../src/storage/transaction.rs | 7 + nym-data-observatory/.gitignore | 2 + ...378d582b7da78c45bc0de934f92c1abe14bda.json | 19 + ...d60df8ad35a747808d5d1d3d525a76bbf0618.json | 52 ++ ...677d46a284e0446b96a2fc5bd77630c62d4b8.json | 50 ++ ...731dfc6affe6c672df121140a5c7141f71c63.json | 16 + nym-data-observatory/Cargo.toml | 48 ++ nym-data-observatory/Dockerfile | 32 + nym-data-observatory/Makefile | 104 +++ nym-data-observatory/README.md | 96 +++ nym-data-observatory/build.rs | 8 + nym-data-observatory/docker-compose.yml | 21 + .../migrations/0101_price_data.sql | 8 + .../migrations/0102_payment_transactions.sql | 10 + .../0103_create_transactions_table.sql | 12 + .../0104_add_listener_failure_table.sql | 11 + nym-data-observatory/src/chain_scraper/mod.rs | 66 ++ .../src/chain_scraper/webhook.rs | 147 ++++ .../src/cli/commands/build_info.rs | 17 + nym-data-observatory/src/cli/commands/init.rs | 46 ++ nym-data-observatory/src/cli/commands/mod.rs | 3 + .../src/cli/commands/run/args.rs | 33 + .../src/cli/commands/run/config.rs | 68 ++ .../src/cli/commands/run/mod.rs | 211 ++++++ nym-data-observatory/src/cli/mod.rs | 67 ++ .../src/config/data_observatory.rs | 20 + nym-data-observatory/src/config/mod.rs | 219 ++++++ nym-data-observatory/src/config/template.rs | 29 + nym-data-observatory/src/db/mod.rs | 38 + nym-data-observatory/src/db/models.rs | 58 ++ nym-data-observatory/src/db/queries/mod.rs | 5 + nym-data-observatory/src/db/queries/price.rs | 119 ++++ nym-data-observatory/src/env.rs | 27 + nym-data-observatory/src/error.rs | 43 ++ nym-data-observatory/src/http/api/mod.rs | 79 +++ nym-data-observatory/src/http/api/price.rs | 44 ++ nym-data-observatory/src/http/api/status.rs | 79 +++ nym-data-observatory/src/http/api_docs.rs | 14 + nym-data-observatory/src/http/error.rs | 21 + nym-data-observatory/src/http/mod.rs | 6 + nym-data-observatory/src/http/models.rs | 68 ++ nym-data-observatory/src/http/server.rs | 59 ++ nym-data-observatory/src/http/state.rs | 124 ++++ nym-data-observatory/src/logging.rs | 43 ++ nym-data-observatory/src/main.rs | 34 + nym-data-observatory/src/models.rs | 22 + nym-data-observatory/src/price_scraper/mod.rs | 76 ++ nyx-chain-watcher/src/cli/commands/run/mod.rs | 2 +- .../src/{payment_listener => listener}/mod.rs | 2 +- .../{payment_listener => listener}/watcher.rs | 0 nyx-chain-watcher/src/main.rs | 2 +- 58 files changed, 2863 insertions(+), 352 deletions(-) rename common/nyxd-scraper-psql/sql_migrations/{01_metadata.sql => 0001_metadata.sql} (82%) rename common/nyxd-scraper-psql/sql_migrations/{02_cosmos.sql => 0002_cosmos.sql} (88%) create mode 100644 nym-data-observatory/.gitignore create mode 100644 nym-data-observatory/.sqlx/query-140df23f816ff5d7501128682ce378d582b7da78c45bc0de934f92c1abe14bda.json create mode 100644 nym-data-observatory/.sqlx/query-a57b74a049b33aee36b72741056d60df8ad35a747808d5d1d3d525a76bbf0618.json create mode 100644 nym-data-observatory/.sqlx/query-f81a3275a1c7cbeefb3fdf7904c677d46a284e0446b96a2fc5bd77630c62d4b8.json create mode 100644 nym-data-observatory/.sqlx/query-fbf7dc2d779476fffcefafaa0a1731dfc6affe6c672df121140a5c7141f71c63.json create mode 100644 nym-data-observatory/Cargo.toml create mode 100644 nym-data-observatory/Dockerfile create mode 100644 nym-data-observatory/Makefile create mode 100644 nym-data-observatory/README.md create mode 100644 nym-data-observatory/build.rs create mode 100644 nym-data-observatory/docker-compose.yml create mode 100644 nym-data-observatory/migrations/0101_price_data.sql create mode 100644 nym-data-observatory/migrations/0102_payment_transactions.sql create mode 100644 nym-data-observatory/migrations/0103_create_transactions_table.sql create mode 100644 nym-data-observatory/migrations/0104_add_listener_failure_table.sql create mode 100644 nym-data-observatory/src/chain_scraper/mod.rs create mode 100644 nym-data-observatory/src/chain_scraper/webhook.rs create mode 100644 nym-data-observatory/src/cli/commands/build_info.rs create mode 100644 nym-data-observatory/src/cli/commands/init.rs create mode 100644 nym-data-observatory/src/cli/commands/mod.rs create mode 100644 nym-data-observatory/src/cli/commands/run/args.rs create mode 100644 nym-data-observatory/src/cli/commands/run/config.rs create mode 100644 nym-data-observatory/src/cli/commands/run/mod.rs create mode 100644 nym-data-observatory/src/cli/mod.rs create mode 100644 nym-data-observatory/src/config/data_observatory.rs create mode 100644 nym-data-observatory/src/config/mod.rs create mode 100644 nym-data-observatory/src/config/template.rs create mode 100644 nym-data-observatory/src/db/mod.rs create mode 100644 nym-data-observatory/src/db/models.rs create mode 100644 nym-data-observatory/src/db/queries/mod.rs create mode 100644 nym-data-observatory/src/db/queries/price.rs create mode 100644 nym-data-observatory/src/env.rs create mode 100644 nym-data-observatory/src/error.rs create mode 100644 nym-data-observatory/src/http/api/mod.rs create mode 100644 nym-data-observatory/src/http/api/price.rs create mode 100644 nym-data-observatory/src/http/api/status.rs create mode 100644 nym-data-observatory/src/http/api_docs.rs create mode 100644 nym-data-observatory/src/http/error.rs create mode 100644 nym-data-observatory/src/http/mod.rs create mode 100644 nym-data-observatory/src/http/models.rs create mode 100644 nym-data-observatory/src/http/server.rs create mode 100644 nym-data-observatory/src/http/state.rs create mode 100644 nym-data-observatory/src/logging.rs create mode 100644 nym-data-observatory/src/main.rs create mode 100644 nym-data-observatory/src/models.rs create mode 100644 nym-data-observatory/src/price_scraper/mod.rs rename nyx-chain-watcher/src/{payment_listener => listener}/mod.rs (98%) rename nyx-chain-watcher/src/{payment_listener => listener}/watcher.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 280cf9f29b8..3985bd4ea6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,7 +11,7 @@ dependencies = [ "macroific", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -24,15 +24,6 @@ dependencies = [ "psl-types", ] -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - [[package]] name = "adler2" version = "2.0.1" @@ -133,9 +124,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "ammonia" -version = "4.1.2" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e913097e1a2124b46746c980134e8c954bc17a6a59bb3fde96f088d126dde6" +checksum = "d6b346764dd0814805de8abf899fe03065bcee69bb1a4771c785817e39f3978f" dependencies = [ "cssparser", "html5ever", @@ -217,9 +208,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "arbitrary" @@ -408,7 +399,7 @@ dependencies = [ "rustc-hash", "serde", "serde_derive", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -490,7 +481,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -501,7 +492,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -655,7 +646,7 @@ checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -688,21 +679,6 @@ dependencies = [ "url", ] -[[package]] -name = "backtrace" -version = "0.3.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", -] - [[package]] name = "base16ct" version = "0.2.0" @@ -1065,7 +1041,7 @@ dependencies = [ "semver 1.0.26", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -1159,7 +1135,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -1250,7 +1226,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -1534,7 +1510,7 @@ checksum = "a782b93fae93e57ca8ad3e9e994e784583f5933aeaaa5c80a545c4b437be2047" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -1558,7 +1534,7 @@ checksum = "e01c9214319017f6ebd8e299036e1f717fa9bb6724e758f7d6fb2477599d1a29" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -1802,7 +1778,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" dependencies = [ "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -1906,7 +1882,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -1954,7 +1930,7 @@ dependencies = [ "schemars 0.8.22", "serde", "sha2 0.10.9", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -2058,7 +2034,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -2069,7 +2045,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -2122,7 +2098,7 @@ dependencies = [ "macroific", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -2165,7 +2141,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -2186,7 +2162,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -2196,7 +2172,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -2225,7 +2201,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", "unicode-xid", ] @@ -2237,7 +2213,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", "unicode-xid", ] @@ -2306,7 +2282,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -2359,7 +2335,7 @@ version = "0.1.0" dependencies = [ "cosmwasm-std", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -2521,7 +2497,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -2618,7 +2594,7 @@ dependencies = [ "console_error_panic_hook", "js-sys", "serde-wasm-bindgen 0.6.5", - "thiserror 2.0.12", + "thiserror 2.0.17", "wasm-bindgen", "wasm-bindgen-futures", "wasm-storage", @@ -2651,7 +2627,7 @@ dependencies = [ "macroific", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -2881,7 +2857,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -2992,12 +2968,6 @@ dependencies = [ "polyval", ] -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - [[package]] name = "glob" version = "0.3.2" @@ -3287,7 +3257,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustls 0.23.29", - "thiserror 2.0.12", + "thiserror 2.0.17", "tinyvec", "tokio", "tokio-rustls 0.26.2", @@ -3313,7 +3283,7 @@ dependencies = [ "resolv-conf", "rustls 0.23.29", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tokio-rustls 0.26.2", "tracing", @@ -3591,9 +3561,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f66d5bd4c6f02bf0542fad85d626775bab9258cf795a4256dcaf3161114d1df" +checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" dependencies = [ "base64 0.22.1", "bytes", @@ -3607,7 +3577,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.0", "tokio", "tower-service", "tracing", @@ -3791,7 +3761,7 @@ checksum = "0ab604ee7085efba6efc65e4ebca0e9533e3aff6cb501d7d77b211e3a781c6d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -3844,7 +3814,7 @@ dependencies = [ "js-sys", "sealed", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "wasm-bindgen", "wasm-bindgen-futures", @@ -3860,7 +3830,7 @@ dependencies = [ "macroific", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -3979,17 +3949,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "io-uring" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" -dependencies = [ - "bitflags 2.9.1", - "cfg-if", - "libc", -] - [[package]] name = "ip_network" version = "0.4.1" @@ -4120,7 +4079,7 @@ checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -4145,9 +4104,9 @@ dependencies = [ [[package]] name = "jwt-simple" -version = "0.12.12" +version = "0.12.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "731011e9647a71ff4f8474176ff6ce6e0d2de87a0173f15613af3a84c3e3401a" +checksum = "6ad8761f175784dfbb83709f322fc4daf6b27afd5bf375492f2876f9e925ef5a" dependencies = [ "anyhow", "binstring", @@ -4165,7 +4124,7 @@ dependencies = [ "serde", "serde_json", "superboring", - "thiserror 2.0.12", + "thiserror 2.0.17", "zeroize", ] @@ -4392,7 +4351,7 @@ dependencies = [ "proc-macro2", "quote", "sealed", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -4404,7 +4363,7 @@ dependencies = [ "proc-macro2", "quote", "sealed", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -4417,7 +4376,7 @@ dependencies = [ "macroific_core", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -4445,7 +4404,7 @@ checksum = "ac84fd3f360fcc43dc5f5d186f02a94192761a080e8bc58621ad4d12296a58cf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -4568,7 +4527,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde-wasm-bindgen 0.6.5", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tsify", "url", @@ -4976,7 +4935,7 @@ dependencies = [ "tempfile", "tendermint", "test-with", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-stream", @@ -5022,7 +4981,7 @@ dependencies = [ "sha2 0.10.9", "tendermint", "tendermint-rpc", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tracing", "ts-rs", @@ -5055,7 +5014,7 @@ dependencies = [ "nym-validator-client", "nym-wireguard-types", "semver 1.0.26", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tokio-util", "tracing", @@ -5080,7 +5039,7 @@ dependencies = [ "serde", "sha2 0.10.9", "strum_macros", - "thiserror 2.0.12", + "thiserror 2.0.17", "tracing", "x25519-dalek", ] @@ -5099,7 +5058,7 @@ dependencies = [ "nym-task", "nym-validator-client", "rand 0.8.5", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -5206,7 +5165,7 @@ dependencies = [ "serde_json", "tap", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "toml 0.8.23", @@ -5243,7 +5202,7 @@ dependencies = [ "serde", "serde_json", "tap", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-tungstenite", @@ -5296,7 +5255,7 @@ dependencies = [ "sha2 0.10.9", "si-scale", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-stream", @@ -5323,7 +5282,7 @@ dependencies = [ "nym-sphinx-params", "nym-statistics-common", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "url", ] @@ -5338,7 +5297,7 @@ dependencies = [ "nym-gateway-requests", "serde", "sqlx", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tracing", @@ -5358,7 +5317,7 @@ dependencies = [ "nym-task", "sqlx", "sqlx-pool-guard", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tracing", @@ -5381,7 +5340,7 @@ dependencies = [ "serde", "serde-wasm-bindgen 0.6.5", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio_with_wasm", "tsify", "wasm-bindgen", @@ -5442,7 +5401,7 @@ dependencies = [ "serde", "sha2 0.10.9", "subtle 2.6.1", - "thiserror 2.0.12", + "thiserror 2.0.17", "zeroize", ] @@ -5455,7 +5414,7 @@ dependencies = [ "log", "nym-network-defaults", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "toml 0.8.23", "url", ] @@ -5472,7 +5431,7 @@ dependencies = [ "nym-ip-packet-requests", "nym-sdk", "pnet_packet", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tokio-util", "tracing", @@ -5490,7 +5449,7 @@ dependencies = [ "schemars 0.8.22", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", "utoipa", "vergen 8.3.1", ] @@ -5559,7 +5518,7 @@ dependencies = [ "strum", "strum_macros", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-util", @@ -5600,7 +5559,7 @@ dependencies = [ "strum", "strum_macros", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-util", @@ -5651,7 +5610,7 @@ dependencies = [ "serde", "sqlx", "sqlx-pool-guard", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "zeroize", @@ -5670,7 +5629,7 @@ dependencies = [ "nym-credentials-interface", "nym-ecash-time", "nym-validator-client", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", ] @@ -5696,7 +5655,7 @@ dependencies = [ "nym-upgrade-mode-check", "nym-validator-client", "si-scale", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tracing", @@ -5721,7 +5680,7 @@ dependencies = [ "nym-validator-client", "rand 0.8.5", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "zeroize", ] @@ -5739,7 +5698,7 @@ dependencies = [ "serde", "strum", "strum_macros", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "utoipa", ] @@ -5773,11 +5732,43 @@ dependencies = [ "serde_json", "sha2 0.10.9", "subtle-encoding", - "thiserror 2.0.12", + "thiserror 2.0.17", "x25519-dalek", "zeroize", ] +[[package]] +name = "nym-data-observatory" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "chrono", + "clap", + "nym-bin-common", + "nym-config", + "nym-network-defaults", + "nym-task", + "nym-validator-client", + "nyxd-scraper-psql", + "nyxd-scraper-shared", + "reqwest 0.12.22", + "schemars 0.8.22", + "serde", + "sqlx", + "thiserror 2.0.17", + "time", + "tokio", + "tokio-util", + "tower-http", + "tracing", + "tracing-subscriber", + "utoipa", + "utoipa-swagger-ui", + "utoipauto", +] + [[package]] name = "nym-dkg" version = "0.1.0" @@ -5796,7 +5787,7 @@ dependencies = [ "serde", "serde_derive", "sha2 0.10.9", - "thiserror 2.0.12", + "thiserror 2.0.17", "zeroize", ] @@ -5811,7 +5802,7 @@ dependencies = [ "cw-utils", "cw2", "nym-multisig-contract-common", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -5824,7 +5815,7 @@ dependencies = [ "nym-network-defaults", "nym-validator-client", "semver 1.0.26", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tracing", "url", @@ -5838,7 +5829,7 @@ dependencies = [ "nym-crypto", "semver 1.0.26", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tracing", "url", @@ -5860,7 +5851,7 @@ dependencies = [ "reqwest 0.12.22", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", "tracing", "utoipa", ] @@ -5923,7 +5914,7 @@ dependencies = [ "nym-wireguard-types", "rand 0.8.5", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-stream", @@ -5956,7 +5947,7 @@ dependencies = [ "rand 0.8.5", "serde", "si-scale", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-stream", @@ -6006,7 +5997,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tokio-util", "tracing", @@ -6039,7 +6030,7 @@ dependencies = [ "serde_json", "strum", "subtle 2.6.1", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tracing", @@ -6058,7 +6049,7 @@ dependencies = [ "nym-statistics-common", "sqlx", "strum", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tracing", @@ -6077,7 +6068,7 @@ dependencies = [ "nym-gateway-requests", "nym-sphinx", "sqlx", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tracing", @@ -6094,7 +6085,7 @@ dependencies = [ "nym-ffi-shared", "nym-sdk", "nym-sphinx-anonymous-replies", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "uniffi", "uniffi_build", @@ -6135,7 +6126,7 @@ dependencies = [ "serde_json", "serde_plain", "serde_yaml", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tracing", "tracing-subscriber", @@ -6152,7 +6143,7 @@ dependencies = [ "proc-macro2", "quote", "reqwest 0.12.22", - "syn 2.0.106", + "syn 2.0.104", "uuid", ] @@ -6184,7 +6175,7 @@ version = "0.1.0" dependencies = [ "nym-credential-storage", "nym-credentials", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tracing", "zeroize", @@ -6210,7 +6201,7 @@ version = "0.1.0" dependencies = [ "log", "rand 0.8.5", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -6222,7 +6213,7 @@ dependencies = [ "futures", "nym-ip-packet-requests", "nym-sdk", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tokio-util", "tracing", @@ -6240,7 +6231,7 @@ dependencies = [ "nym-sphinx", "rand 0.8.5", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-util", @@ -6281,7 +6272,7 @@ dependencies = [ "reqwest 0.12.22", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-tun", @@ -6297,7 +6288,7 @@ dependencies = [ "k256", "ledger-transport", "ledger-transport-hid", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -6344,7 +6335,7 @@ dependencies = [ "semver 1.0.26", "serde", "serde_repr", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "ts-rs", "utoipa", @@ -6370,7 +6361,7 @@ dependencies = [ "nym-task", "rand 0.8.5", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-util", @@ -6389,7 +6380,7 @@ dependencies = [ "cw4", "schemars 0.8.22", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -6483,7 +6474,7 @@ dependencies = [ "sqlx", "tap", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-tungstenite", @@ -6561,7 +6552,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "sysinfo", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-stream", @@ -6611,7 +6602,7 @@ dependencies = [ "serde_json", "strum", "strum_macros", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "url", @@ -6680,7 +6671,7 @@ dependencies = [ "sqlx", "strum", "strum_macros", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-stream", @@ -6726,7 +6717,7 @@ dependencies = [ "rand_chacha 0.3.1", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "wasm-utils", ] @@ -6741,7 +6732,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde-wasm-bindgen 0.6.5", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tsify", "wasm-bindgen", @@ -6768,7 +6759,7 @@ dependencies = [ "snow", "strum", "strum_macros", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tokio-util", "tracing", @@ -6815,7 +6806,7 @@ name = "nym-ordered-buffer" version = "0.1.0" dependencies = [ "log", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -6832,7 +6823,7 @@ dependencies = [ "rand 0.8.5", "rayon", "sphinx-packet", - "thiserror 2.0.12", + "thiserror 2.0.17", "x25519-dalek", "zeroize", ] @@ -6856,7 +6847,7 @@ dependencies = [ "nym-contracts-common", "schemars 0.8.22", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -6868,7 +6859,7 @@ dependencies = [ "cw-controllers", "schemars 0.8.22", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", ] @@ -6885,7 +6876,7 @@ dependencies = [ "nym-registration-common", "nym-sdk", "nym-validator-client", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tokio-util", "tracing", @@ -6950,7 +6941,7 @@ dependencies = [ "serde", "tap", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-stream", @@ -6980,7 +6971,7 @@ version = "0.1.0" dependencies = [ "bincode", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -6996,7 +6987,7 @@ dependencies = [ "nym-sphinx-anonymous-replies", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", ] @@ -7046,7 +7037,7 @@ dependencies = [ "serde", "serde_json", "tap", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "url", @@ -7080,7 +7071,7 @@ dependencies = [ "schemars 0.8.22", "serde", "tap", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "url", ] @@ -7112,7 +7103,7 @@ dependencies = [ "serde", "serde_json", "tap", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -7136,7 +7127,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rand_distr", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tracing", ] @@ -7155,7 +7146,7 @@ dependencies = [ "nym-topology", "rand 0.8.5", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "zeroize", ] @@ -7169,7 +7160,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -7185,7 +7176,7 @@ dependencies = [ "nym-topology", "rand 0.8.5", "rand_chacha 0.3.1", - "thiserror 2.0.12", + "thiserror 2.0.17", "tracing", "wasm-bindgen", ] @@ -7203,7 +7194,7 @@ dependencies = [ "nym-sphinx-types", "rand 0.8.5", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "utoipa", "wasmtimer", ] @@ -7222,7 +7213,7 @@ dependencies = [ "nym-sphinx-types", "nym-topology", "rand 0.8.5", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -7233,7 +7224,7 @@ dependencies = [ "nym-sphinx-anonymous-replies", "nym-sphinx-params", "nym-sphinx-types", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -7246,7 +7237,7 @@ dependencies = [ "nym-sphinx-forwarding", "nym-sphinx-params", "nym-sphinx-types", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tokio-util", "tracing", @@ -7259,7 +7250,7 @@ dependencies = [ "nym-crypto", "nym-sphinx-types", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -7268,7 +7259,7 @@ version = "0.1.0" dependencies = [ "nym-sphinx-addressing", "nym-sphinx-types", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -7277,7 +7268,7 @@ version = "0.2.0" dependencies = [ "nym-outfox", "sphinx-packet", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -7328,7 +7319,7 @@ dependencies = [ "strum", "strum_macros", "sysinfo", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "utoipa", @@ -7346,7 +7337,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", "zeroize", ] @@ -7359,7 +7350,7 @@ dependencies = [ "futures", "log", "nym-test-utils", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tokio-util", "tracing", @@ -7409,7 +7400,7 @@ dependencies = [ "reqwest 0.12.22", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tracing", "tsify", @@ -7424,7 +7415,7 @@ dependencies = [ "etherparse", "log", "nym-wireguard-types", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tokio-tun", ] @@ -7453,7 +7444,7 @@ dependencies = [ "strum", "strum_macros", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.17", "ts-rs", "url", "utoipa", @@ -7472,7 +7463,7 @@ dependencies = [ "reqwest 0.12.22", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tracing", "utoipa", @@ -7519,7 +7510,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tendermint-rpc", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tracing", @@ -7564,7 +7555,7 @@ dependencies = [ "serde_with", "sha2 0.10.9", "sqlx", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tracing", @@ -7585,7 +7576,7 @@ dependencies = [ "nym-task", "nym-validator-client", "rand 0.8.5", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-util", @@ -7603,7 +7594,7 @@ dependencies = [ "nym-contracts-common", "nym-mixnet-contract-common", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "ts-rs", ] @@ -7624,7 +7615,7 @@ dependencies = [ "serde", "serde-wasm-bindgen 0.6.5", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tsify", "wasm-bindgen", @@ -7669,7 +7660,7 @@ dependencies = [ "nym-node-metrics", "nym-task", "nym-wireguard-types", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tokio-stream", "tracing", @@ -7714,7 +7705,7 @@ dependencies = [ "nym-credentials-interface", "schemars 0.8.22", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "utoipa", ] @@ -7750,7 +7741,7 @@ dependencies = [ "nym-crypto", "rand 0.8.5", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "x25519-dalek", ] @@ -7777,7 +7768,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tar", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tracing", @@ -7803,7 +7794,7 @@ dependencies = [ "schemars 0.8.22", "serde", "sqlx", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-util", @@ -7827,7 +7818,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tracing", ] @@ -7851,7 +7842,7 @@ dependencies = [ "sha2 0.10.9", "tendermint", "tendermint-rpc", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-stream", @@ -7867,39 +7858,30 @@ dependencies = [ "async-trait", "nyxd-scraper-shared", "sqlx", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tracing", ] [[package]] name = "objc2-core-foundation" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.9.1", ] [[package]] name = "objc2-io-kit" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" dependencies = [ "libc", "objc2-core-foundation", ] -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.21.3" @@ -8202,7 +8184,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" dependencies = [ "memchr", - "thiserror 2.0.12", + "thiserror 2.0.17", "ucd-trie", ] @@ -8226,7 +8208,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -8289,7 +8271,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -8318,7 +8300,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -8412,7 +8394,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -8570,11 +8552,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit", + "toml_edit 0.23.5", ] [[package]] @@ -8596,7 +8578,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -8629,7 +8611,7 @@ dependencies = [ "memchr", "parking_lot", "protobuf", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -8652,7 +8634,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -8729,7 +8711,7 @@ dependencies = [ "rustc-hash", "rustls 0.23.29", "socket2 0.5.10", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tracing", "web-time", @@ -8750,7 +8732,7 @@ dependencies = [ "rustls 0.23.29", "rustls-pki-types", "slab", - "thiserror 2.0.12", + "thiserror 2.0.17", "tinyvec", "tracing", "web-time", @@ -8897,7 +8879,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -8917,7 +8899,7 @@ checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -9038,7 +9020,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21918d6644020c6f6ef1993242989bf6d4952d2e025617744f184c02df51c356" dependencies = [ - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -9152,7 +9134,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.106", + "syn 2.0.104", "walkdir", ] @@ -9182,12 +9164,6 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "rustc-demangle" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -9441,7 +9417,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals 0.29.1", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -9473,7 +9449,7 @@ checksum = "1783eabc414609e28a5ba76aee5ddd52199f7107a0b24c2e9746a1ecc34a683d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -9494,7 +9470,7 @@ checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -9579,10 +9555,11 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] @@ -9626,15 +9603,24 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -9645,7 +9631,7 @@ checksum = "e578a843d40b4189a4d66bba51d7684f57da5bd7c304c64e14bd63efbef49509" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -9656,19 +9642,20 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] name = "serde_json" -version = "1.0.141" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] @@ -9684,7 +9671,7 @@ dependencies = [ "serde_json", "serde_json_path_core", "serde_json_path_macros", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -9696,7 +9683,7 @@ dependencies = [ "inventory", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -9718,7 +9705,7 @@ checksum = "aafbefbe175fa9bf03ca83ef89beecff7d2a95aaacd5732325b90ac8c3bd7b90" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -9748,7 +9735,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -9801,7 +9788,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -9953,9 +9940,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.11" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" [[package]] name = "sluice" @@ -10128,7 +10115,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "tokio-stream", @@ -10147,7 +10134,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -10170,7 +10157,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.106", + "syn 2.0.104", "tokio", "url", ] @@ -10213,7 +10200,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tracing", "whoami", @@ -10265,7 +10252,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tracing", "whoami", @@ -10291,7 +10278,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tracing", "url", @@ -10379,7 +10366,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -10435,9 +10422,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.106" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", @@ -10467,14 +10454,14 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] name = "sysinfo" -version = "0.37.0" +version = "0.37.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07cec4dc2d2e357ca1e610cfb07de2fa7a10fc3e9fe89f72545f3d244ea87753" +checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" dependencies = [ "libc", "memchr", @@ -10664,7 +10651,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -10700,7 +10687,7 @@ dependencies = [ "serde_json", "sqlx", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tokio", "toml 0.8.23", @@ -10729,11 +10716,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl 2.0.17", ] [[package]] @@ -10744,18 +10731,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -10860,34 +10847,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.47.1" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio 1.0.4", "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", "socket2 0.6.0", "tokio-macros", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -11036,7 +11020,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37e04c1865c281139e5ccf633cb9f76ffdaabeebfe53b703984cf82878e2aabb" dependencies = [ "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -11056,8 +11040,8 @@ checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", - "toml_datetime", - "toml_edit", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", ] [[package]] @@ -11069,6 +11053,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -11078,11 +11071,32 @@ dependencies = [ "indexmap 2.10.0", "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.11", "toml_write", "winnow", ] +[[package]] +name = "toml_edit" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2ad0b7ae9cfeef5605163839cb9221f453399f15cfb5c10be9885fcf56611f9" +dependencies = [ + "indexmap 2.10.0", + "toml_datetime 0.7.3", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" +dependencies = [ + "winnow", +] + [[package]] name = "toml_write" version = "0.1.2" @@ -11247,7 +11261,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -11354,7 +11368,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -11411,7 +11425,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e640d9b0964e9d39df633548591090ab92f7a4567bc31d3891af23471a3365c6" dependencies = [ "lazy_static", - "thiserror 2.0.12", + "thiserror 2.0.17", "ts-rs-macros", ] @@ -11438,7 +11452,7 @@ checksum = "0e9d8656589772eeec2cf7a8264d9cda40fb28b9bc53118ceb9e8c07f8f38730" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", "termcolor", ] @@ -11465,7 +11479,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals 0.28.0", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -11527,7 +11541,7 @@ checksum = "016c26257f448222014296978b2c8456e2cad4de308c35bdb1e383acd569ef5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -11675,7 +11689,7 @@ dependencies = [ "indexmap 2.10.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -11690,7 +11704,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.106", + "syn 2.0.104", "toml 0.5.11", "uniffi_meta", ] @@ -11817,7 +11831,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.106", + "syn 2.0.104", "uuid", ] @@ -11856,7 +11870,7 @@ checksum = "268d76aaebb80eba79240b805972e52d7d410d4bcc52321b951318b0f440cd60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -11867,7 +11881,7 @@ checksum = "382673bda1d05c85b4550d32fd4192ccd4cffe9a908543a0795d1e7682b36246" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", "utoipauto-core", ] @@ -12086,7 +12100,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", "wasm-bindgen-shared", ] @@ -12121,7 +12135,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -12156,7 +12170,7 @@ checksum = "17d5042cc5fa009658f9a7333ef24291b1291a25b6382dd68862a7f3b969f69b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -12182,7 +12196,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde-wasm-bindgen 0.6.5", - "thiserror 2.0.12", + "thiserror 2.0.17", "time", "tsify", "url", @@ -12204,7 +12218,7 @@ dependencies = [ "nym-store-cipher", "serde", "serde-wasm-bindgen 0.6.5", - "thiserror 2.0.12", + "thiserror 2.0.17", "wasm-bindgen", "wasm-utils", ] @@ -12383,7 +12397,7 @@ dependencies = [ "windows-collections", "windows-core", "windows-future", - "windows-link", + "windows-link 0.1.3", "windows-numerics", ] @@ -12404,7 +12418,7 @@ checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", - "windows-link", + "windows-link 0.1.3", "windows-result", "windows-strings", ] @@ -12416,7 +12430,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core", - "windows-link", + "windows-link 0.1.3", "windows-threading", ] @@ -12428,7 +12442,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -12439,7 +12453,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -12448,6 +12462,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-numerics" version = "0.2.0" @@ -12455,7 +12475,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ "windows-core", - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -12464,7 +12484,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -12473,7 +12493,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -12521,6 +12541,15 @@ dependencies = [ "windows-targets 0.53.2", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -12589,7 +12618,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -12863,7 +12892,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", "synstructure", ] @@ -12884,7 +12913,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -12904,15 +12933,15 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" dependencies = [ "zeroize_derive", ] @@ -12925,7 +12954,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -12958,7 +12987,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.104", ] [[package]] @@ -12974,7 +13003,7 @@ dependencies = [ "flate2", "indexmap 2.10.0", "memchr", - "thiserror 2.0.12", + "thiserror 2.0.17", "zopfli", ] @@ -12995,7 +13024,7 @@ dependencies = [ "rand 0.8.5", "reqwest 0.12.22", "serde", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tsify", "uuid", @@ -13055,7 +13084,7 @@ dependencies = [ "reqwest 0.12.22", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.17", "tokio", "tracing", "url", diff --git a/Cargo.toml b/Cargo.toml index 838d88185ad..28f7f6c3194 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,6 +126,7 @@ members = [ "nym-credential-proxy/nym-credential-proxy", "nym-credential-proxy/nym-credential-proxy-requests", "nym-credential-proxy/vpn-api-lib-wasm", + "nym-data-observatory", "nym-ip-packet-client", "nym-network-monitor", "nym-node", diff --git a/common/nyxd-scraper-psql/sql_migrations/01_metadata.sql b/common/nyxd-scraper-psql/sql_migrations/0001_metadata.sql similarity index 82% rename from common/nyxd-scraper-psql/sql_migrations/01_metadata.sql rename to common/nyxd-scraper-psql/sql_migrations/0001_metadata.sql index bce13940e81..43070210c4a 100644 --- a/common/nyxd-scraper-psql/sql_migrations/01_metadata.sql +++ b/common/nyxd-scraper-psql/sql_migrations/0001_metadata.sql @@ -6,5 +6,5 @@ CREATE TABLE METADATA ( id INTEGER PRIMARY KEY CHECK (id = 0), - last_processed_height INTEGER NOT NULL + last_processed_height BIGINT NOT NULL ); \ No newline at end of file diff --git a/common/nyxd-scraper-psql/sql_migrations/02_cosmos.sql b/common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql similarity index 88% rename from common/nyxd-scraper-psql/sql_migrations/02_cosmos.sql rename to common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql index f48da6e9030..d33b75fae9d 100644 --- a/common/nyxd-scraper-psql/sql_migrations/02_cosmos.sql +++ b/common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql @@ -44,7 +44,7 @@ CREATE TABLE transaction success BOOLEAN NOT NULL, /* Body */ - messages JSON NOT NULL DEFAULT '[]'::JSON, + messages JSONB NOT NULL DEFAULT '[]'::JSONB, memo TEXT, signatures TEXT[] NOT NULL, @@ -63,31 +63,34 @@ CREATE TABLE transaction CREATE INDEX transaction_hash_index ON transaction (hash); CREATE INDEX transaction_height_index ON transaction (height); -CREATE TABLE message_type +CREATE TYPE COIN AS ( - type TEXT NOT NULL UNIQUE, - module TEXT NOT NULL, - label TEXT NOT NULL, - height BIGINT NOT NULL + denom TEXT, + amount TEXT ); -CREATE INDEX message_type_module_index ON message_type (module); -CREATE INDEX message_type_type_index ON message_type (type); CREATE TABLE message ( transaction_hash TEXT NOT NULL, index BIGINT NOT NULL, - type TEXT NOT NULL REFERENCES message_type (type), - value JSON NOT NULL, + type TEXT NOT NULL, + value JSONB NOT NULL, involved_accounts_addresses TEXT[] NOT NULL, - height BIGINT NOT NULL, + + funds COIN[] DEFAULT '{}', + + wasm_sender TEXT, + wasm_contract_address TEXT, + wasm_message_type TEXT, + FOREIGN KEY (transaction_hash) REFERENCES transaction (hash), CONSTRAINT unique_message_per_tx UNIQUE (transaction_hash, index) ); CREATE INDEX message_transaction_hash_index ON message (transaction_hash); CREATE INDEX message_type_index ON message (type); CREATE INDEX message_involved_accounts_index ON message USING GIN (involved_accounts_addresses); +CREATE INDEX message_wasm_contract_message_type_index ON message (wasm_message_type); /** * This function is used to find all the utils that involve any of the given addresses and have @@ -124,4 +127,5 @@ $$ LANGUAGE sql STABLE; CREATE TABLE pruning ( last_pruned_height BIGINT NOT NULL -); \ No newline at end of file +); + diff --git a/common/nyxd-scraper-psql/src/storage/manager.rs b/common/nyxd-scraper-psql/src/storage/manager.rs index 165dfd42f45..ef38189241b 100644 --- a/common/nyxd-scraper-psql/src/storage/manager.rs +++ b/common/nyxd-scraper-psql/src/storage/manager.rs @@ -1,10 +1,11 @@ // Copyright 2023 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 +use crate::models::Coin; use crate::storage::models::{CommitSignature, Validator}; use nyxd_scraper_shared::storage::helpers::log_db_operation_time; use sqlx::types::time::PrimitiveDateTime; -use sqlx::types::{Json, JsonValue}; +use sqlx::types::JsonValue; use sqlx::{Executor, Postgres}; use tokio::time::Instant; use tracing::{instrument, trace}; @@ -200,7 +201,8 @@ impl StorageManager { log_db_operation_time("get_last_processed_height", start); if let Some(row) = maybe_record { - Ok(row.last_processed_height as i64) + #[allow(clippy::useless_conversion)] + Ok(row.last_processed_height.into()) } else { Ok(-1) } @@ -392,6 +394,7 @@ where Ok(()) } +#[allow(clippy::too_many_arguments)] #[instrument(skip(executor))] pub(crate) async fn insert_message<'a, E>( transaction_hash: String, @@ -400,6 +403,10 @@ pub(crate) async fn insert_message<'a, E>( value: JsonValue, involved_account_addresses: Vec, height: i64, + wasm_sender: Option, + wasm_contract_address: Option, + wasm_message_type: Option, + funds: Option>, executor: E, ) -> Result<(), sqlx::Error> where @@ -408,25 +415,55 @@ where trace!("insert_message"); let start = Instant::now(); - sqlx::query!( - r#" - INSERT INTO message(transaction_hash, index, type, value, involved_accounts_addresses, height) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (transaction_hash, index) DO UPDATE - SET height = excluded.height, - type = excluded.type, - value = excluded.value, - involved_accounts_addresses = excluded.involved_accounts_addresses - "#, - transaction_hash, - index, - typ, - value, - &involved_account_addresses, - height - ) - .execute(executor) - .await?; + // sqlx doesn't understand option types + if let Some(coins) = funds { + sqlx::query!( + r#" + INSERT INTO message(transaction_hash, index, type, value, involved_accounts_addresses, height, wasm_sender, wasm_contract_address, wasm_message_type, funds) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (transaction_hash, index) DO UPDATE + SET height = excluded.height, + type = excluded.type, + value = excluded.value, + involved_accounts_addresses = excluded.involved_accounts_addresses + "#, + transaction_hash, + index, + typ, + value, + &involved_account_addresses, + height, + wasm_sender, + wasm_contract_address, + wasm_message_type, + &coins as _, + ) + .execute(executor) + .await?; + } else { + sqlx::query!( + r#" + INSERT INTO message(transaction_hash, index, type, value, involved_accounts_addresses, height, wasm_sender, wasm_contract_address, wasm_message_type) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (transaction_hash, index) DO UPDATE + SET height = excluded.height, + type = excluded.type, + value = excluded.value, + involved_accounts_addresses = excluded.involved_accounts_addresses + "#, + transaction_hash, + index, + typ, + value, + &involved_account_addresses, + height, + wasm_sender, + wasm_contract_address, + wasm_message_type, + ) + .execute(executor) + .await?; + } log_db_operation_time("insert_message", start); Ok(()) @@ -434,7 +471,7 @@ where #[instrument(skip(executor))] pub(crate) async fn update_last_processed<'a, E>( - height: i32, + height: i64, executor: E, ) -> Result<(), sqlx::Error> where diff --git a/common/nyxd-scraper-psql/src/storage/models.rs b/common/nyxd-scraper-psql/src/storage/models.rs index 6c4c16c633f..878eff203b4 100644 --- a/common/nyxd-scraper-psql/src/storage/models.rs +++ b/common/nyxd-scraper-psql/src/storage/models.rs @@ -1,8 +1,9 @@ // Copyright 2023 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use sqlx::FromRow; +use serde::{Deserialize, Serialize}; use sqlx::types::time::OffsetDateTime; +use sqlx::FromRow; #[derive(Debug, Clone, Eq, PartialEq, Hash, FromRow)] pub struct Validator { @@ -28,3 +29,10 @@ pub struct CommitSignature { pub proposer_priority: i64, pub timestamp: OffsetDateTime, } + +#[derive(Debug, Serialize, Deserialize, sqlx::Type)] +#[sqlx(type_name = "coin")] +pub struct Coin { + pub amount: String, + pub denom: String, +} diff --git a/common/nyxd-scraper-psql/src/storage/transaction.rs b/common/nyxd-scraper-psql/src/storage/transaction.rs index 2baa2c81308..af118f300a6 100644 --- a/common/nyxd-scraper-psql/src/storage/transaction.rs +++ b/common/nyxd-scraper-psql/src/storage/transaction.rs @@ -2,14 +2,18 @@ // SPDX-License-Identifier: Apache-2.0 use crate::error::PostgresScraperError; -use crate::storage::helpers::{parse_addresses_from_events, PlaceholderStruct}; +use crate::models::Coin; +use crate::storage::helpers::parse_addresses_from_events; use crate::storage::manager::{ insert_block, insert_message, insert_precommit, insert_transaction, insert_validator, + update_last_processed, }; use async_trait::async_trait; use base64::engine::general_purpose; use base64::Engine as _; use cosmrs::proto; +use cosmrs::proto::cosmwasm::wasm::v1::MsgExecuteContract; +use cosmrs::proto::prost::Message; use nyxd_scraper_shared::helpers::{ validator_consensus_address, validator_info, validator_pubkey_to_bech32, }; @@ -18,7 +22,7 @@ use nyxd_scraper_shared::storage::{ validators, Block, Commit, CommitSig, NyxdScraperStorageError, NyxdScraperTransaction, }; use nyxd_scraper_shared::{Any, MessageRegistry, ParsedTransactionResponse}; -use serde_json::json; +use serde_json::{json, Value}; use sqlx::types::time::{OffsetDateTime, PrimitiveDateTime}; use sqlx::{Postgres, Transaction}; use std::ops::{Deref, DerefMut}; @@ -211,13 +215,44 @@ impl PostgresStorageTransaction { for chain_tx in txs { let involved_addresses = parse_addresses_from_events(chain_tx); for (index, msg) in chain_tx.tx.body.messages.iter().enumerate() { + let mut wasm_sender: Option = None; + let mut wasm_contract_address: Option = None; + let mut wasm_message_type: Option = None; + let mut funds: Option> = None; + + let value = serde_json::to_value(self.decode_or_skip(msg))?; + + if msg.type_url == "/cosmwasm.wasm.v1.MsgExecuteContract" { + if let Ok(wasm_execute) = MsgExecuteContract::decode(msg.value.as_ref()) { + wasm_sender = Some(wasm_execute.sender); + wasm_contract_address = Some(wasm_execute.contract); + if let Some(raw_msg) = value.get("msg") { + wasm_message_type = get_first_field_name(raw_msg); + } + funds = Some( + wasm_execute + .funds + .iter() + .map(|c| Coin { + amount: c.amount.to_string(), + denom: c.denom.clone(), + }) + .collect(), + ); + } + } + insert_message( chain_tx.hash.to_string(), index as i64, msg.type_url.clone(), - serde_json::to_value(self.decode_or_skip(msg))?, + value, involved_addresses.clone(), chain_tx.height.into(), + wasm_sender, + wasm_contract_address, + wasm_message_type, + funds, self.inner.as_mut(), ) .await? @@ -226,6 +261,20 @@ impl PostgresStorageTransaction { Ok(()) } + + async fn update_last_processed(&mut self, height: i64) -> Result<(), PostgresScraperError> { + debug!("update_last_processed"); + update_last_processed(height, self.inner.as_mut()).await?; + Ok(()) + } +} + +fn get_first_field_name(value: &Value) -> Option { + debug!("value:\n{value}"); + match value.as_object() { + Some(map) => map.keys().next().cloned(), + None => None, + } } #[async_trait] @@ -286,6 +335,8 @@ impl NyxdScraperTransaction for PostgresStorageTransaction { } async fn update_last_processed(&mut self, height: i64) -> Result<(), NyxdScraperStorageError> { - self.update_last_processed(height).await + self.update_last_processed(height) + .await + .map_err(NyxdScraperStorageError::from) } } diff --git a/common/nyxd-scraper-sqlite/src/storage/transaction.rs b/common/nyxd-scraper-sqlite/src/storage/transaction.rs index 713cbb32615..76f5ecd67e9 100644 --- a/common/nyxd-scraper-sqlite/src/storage/transaction.rs +++ b/common/nyxd-scraper-sqlite/src/storage/transaction.rs @@ -4,6 +4,7 @@ use crate::error::SqliteScraperError; use crate::storage::manager::{ insert_block, insert_message, insert_precommit, insert_transaction, insert_validator, + update_last_processed, }; use async_trait::async_trait; use nyxd_scraper_shared::helpers::{ @@ -169,6 +170,12 @@ impl SqliteStorageTransaction { Ok(()) } + + async fn update_last_processed(&mut self, height: i64) -> Result<(), SqliteScraperError> { + debug!("update_last_processed"); + update_last_processed(height, self.0.as_mut()).await?; + Ok(()) + } } #[async_trait] diff --git a/nym-data-observatory/.gitignore b/nym-data-observatory/.gitignore new file mode 100644 index 00000000000..36150a96516 --- /dev/null +++ b/nym-data-observatory/.gitignore @@ -0,0 +1,2 @@ +0001_metadata.sql +0002_cosmos.sql \ No newline at end of file diff --git a/nym-data-observatory/.sqlx/query-140df23f816ff5d7501128682ce378d582b7da78c45bc0de934f92c1abe14bda.json b/nym-data-observatory/.sqlx/query-140df23f816ff5d7501128682ce378d582b7da78c45bc0de934f92c1abe14bda.json new file mode 100644 index 00000000000..b190fd0fb17 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-140df23f816ff5d7501128682ce378d582b7da78c45bc0de934f92c1abe14bda.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO price_history\n (timestamp, chf, usd, eur, gbp, btc)\n VALUES\n ($1, $2, $3, $4, $5, $6)\n ON CONFLICT(timestamp) DO UPDATE SET\n chf=excluded.chf,\n usd=excluded.usd,\n eur=excluded.eur,\n gbp=excluded.gbp,\n btc=excluded.btc;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Float8", + "Float8", + "Float8", + "Float8", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "140df23f816ff5d7501128682ce378d582b7da78c45bc0de934f92c1abe14bda" +} diff --git a/nym-data-observatory/.sqlx/query-a57b74a049b33aee36b72741056d60df8ad35a747808d5d1d3d525a76bbf0618.json b/nym-data-observatory/.sqlx/query-a57b74a049b33aee36b72741056d60df8ad35a747808d5d1d3d525a76bbf0618.json new file mode 100644 index 00000000000..0a8728ccb80 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-a57b74a049b33aee36b72741056d60df8ad35a747808d5d1d3d525a76bbf0618.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT timestamp, chf, usd, eur, gbp, btc FROM price_history WHERE timestamp >= $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "chf", + "type_info": "Float8" + }, + { + "ordinal": 2, + "name": "usd", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "eur", + "type_info": "Float8" + }, + { + "ordinal": 4, + "name": "gbp", + "type_info": "Float8" + }, + { + "ordinal": 5, + "name": "btc", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false + ] + }, + "hash": "a57b74a049b33aee36b72741056d60df8ad35a747808d5d1d3d525a76bbf0618" +} diff --git a/nym-data-observatory/.sqlx/query-f81a3275a1c7cbeefb3fdf7904c677d46a284e0446b96a2fc5bd77630c62d4b8.json b/nym-data-observatory/.sqlx/query-f81a3275a1c7cbeefb3fdf7904c677d46a284e0446b96a2fc5bd77630c62d4b8.json new file mode 100644 index 00000000000..e58a1a74516 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-f81a3275a1c7cbeefb3fdf7904c677d46a284e0446b96a2fc5bd77630c62d4b8.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT timestamp, chf, usd, eur, gbp, btc FROM price_history ORDER BY timestamp DESC LIMIT 1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "chf", + "type_info": "Float8" + }, + { + "ordinal": 2, + "name": "usd", + "type_info": "Float8" + }, + { + "ordinal": 3, + "name": "eur", + "type_info": "Float8" + }, + { + "ordinal": 4, + "name": "gbp", + "type_info": "Float8" + }, + { + "ordinal": 5, + "name": "btc", + "type_info": "Float8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false, + false, + false + ] + }, + "hash": "f81a3275a1c7cbeefb3fdf7904c677d46a284e0446b96a2fc5bd77630c62d4b8" +} diff --git a/nym-data-observatory/.sqlx/query-fbf7dc2d779476fffcefafaa0a1731dfc6affe6c672df121140a5c7141f71c63.json b/nym-data-observatory/.sqlx/query-fbf7dc2d779476fffcefafaa0a1731dfc6affe6c672df121140a5c7141f71c63.json new file mode 100644 index 00000000000..d70649da8b8 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-fbf7dc2d779476fffcefafaa0a1731dfc6affe6c672df121140a5c7141f71c63.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO watcher_execution(start_ts, end_ts, error_message)\n VALUES ($1, $2, $3)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Timestamptz", + "Timestamptz", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fbf7dc2d779476fffcefafaa0a1731dfc6affe6c672df121140a5c7141f71c63" +} diff --git a/nym-data-observatory/Cargo.toml b/nym-data-observatory/Cargo.toml new file mode 100644 index 00000000000..e215a48daa1 --- /dev/null +++ b/nym-data-observatory/Cargo.toml @@ -0,0 +1,48 @@ +# Copyright 2024 - Nym Technologies SA +# SPDX-License-Identifier: GPL-3.0-only + +[package] +name = "nym-data-observatory" +version = "0.1.0" +authors.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +readme.workspace = true + +[dependencies] +anyhow = { workspace = true } +async-trait.workspace = true +axum = { workspace = true, features = ["tokio"] } +chrono = { workspace = true } +clap = { workspace = true, features = ["cargo", "derive", "env"] } +nym-config = { path = "../common/config" } +nym-bin-common = { path = "../common/bin-common", features = ["output_format"] } +nym-network-defaults = { path = "../common/network-defaults" } +nym-task = { path = "../common/task" } +nym-validator-client = { path = "../common/client-libs/validator-client" } +nyxd-scraper-psql = { path = "../common/nyxd-scraper-psql" } +nyxd-scraper-shared = { path = "../common/nyxd-scraper-shared" } +reqwest = { workspace = true, features = ["rustls-tls"] } +schemars = { workspace = true } +serde = { workspace = true, features = ["derive"] } +sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres", "time"] } +thiserror = { workspace = true } +time = { workspace = true } +tokio = { workspace = true, features = ["process", "rt-multi-thread"] } +tokio-util = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } +tower-http = { workspace = true, features = ["cors", "trace"] } +utoipa = { workspace = true, features = ["axum_extras", "time"] } +utoipa-swagger-ui = { workspace = true, features = ["axum"] } +utoipauto = { workspace = true } + + +[build-dependencies] +anyhow = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] } diff --git a/nym-data-observatory/Dockerfile b/nym-data-observatory/Dockerfile new file mode 100644 index 00000000000..780564bec18 --- /dev/null +++ b/nym-data-observatory/Dockerfile @@ -0,0 +1,32 @@ +# this will only work with VPN, otherwise remove the harbor part +FROM harbor.nymte.ch/dockerhub/rust:latest AS builder + +COPY ./ /usr/src/nym +WORKDIR /usr/src/nym/nym-data-observatory + +RUN cargo build --release + +#------------------------------------------------------------------- +# The following environment variables are required at runtime: +# +# NYM_DATA_OBSERVATORY_DB_URL=postgres://nym_data_observatory:data-data-data@localhost/nym_data_observatory_data +# +# And optionally: +# +# NYM_DATA_OBSERVATORY_WEBHOOK_URL="https://webhook.site" +# NYM_DATA_OBSERVATORY_WEBHOOK_AUTH=1234 +# NYX_CHAIN_WATCHER_CONFIG_ENV_FILE_ARG = /mnt/sandbox.env for sandbox environment +# +# see https://github.com/nymtech/nym/blob/develop/nym-data-observatory/src/cli/commands/run/args.rs for details +# and https://github.com/nymtech/nym/blob/develop/nym-data-observatory/src/env.rs for env vars +#------------------------------------------------------------------- + +FROM harbor.nymte.ch/dockerhub/ubuntu:24.04 + +RUN apt update && apt install -yy curl ca-certificates + +WORKDIR /nym + +COPY --from=builder /usr/src/nym/target/release/nym-data-observatory ./ +ENTRYPOINT [ "/nym/nym-data-observatory", "run" ] + diff --git a/nym-data-observatory/Makefile b/nym-data-observatory/Makefile new file mode 100644 index 00000000000..30d5936a107 --- /dev/null +++ b/nym-data-observatory/Makefile @@ -0,0 +1,104 @@ +# Makefile for nyx_chain_scraper database management + +# --- Configuration --- +TEST_DATABASE_URL := postgres://testuser:testpass@localhost:5433/nym_data_observatory_test + +# Docker compose service names +DB_SERVICE_NAME := postgres-test +DB_CONTAINER_NAME := nym_data_observatory_test + +# Default target +.PHONY: default +default: help + +# --- Main Targets --- +.PHONY: prepare-pg +prepare-pg: test-db-up test-db-wait test-db-migrate test-db-prepare test-db-down ## Setup PostgreSQL and prepare SQLx offline cache + +.PHONY: test-db +test-db: test-db-up test-db-wait test-db-migrate test-db-run test-db-down ## Run tests with PostgreSQL database + +.PHONY: dev-db +dev-db: test-db-up test-db-wait test-db-migrate ## Start PostgreSQL for development (keeps running) + @echo "PostgreSQL is running on port 5433" + @echo "Connection string: $(TEST_DATABASE_URL)" + +# --- Docker Compose Targets --- +.PHONY: test-db-up +test-db-up: ## Start the PostgreSQL test database in the background + @echo "Starting PostgreSQL test database..." + docker compose up -d $(DB_SERVICE_NAME) + +.PHONY: test-db-wait +test-db-wait: ## Wait for the PostgreSQL database to be healthy + @echo "Waiting for PostgreSQL database..." + @while ! docker inspect --format='{{.State.Health.Status}}' $(DB_CONTAINER_NAME) 2>/dev/null | grep -q 'healthy'; do \ + echo -n "."; \ + sleep 1; \ + done; \ + echo " Database is healthy!" + +.PHONY: test-db-down +test-db-down: ## Stop and remove the test database + @echo "Stopping PostgreSQL test database..." + docker compose down + +# --- SQLx Targets --- +.PHONY: test-db-migrate +test-db-migrate: ## Run database migrations against PostgreSQL + @echo "Copying common PostgreSQL migrations..." + cp ../common/nyxd-scraper-psql/sql_migrations/* migrations + @echo "Running watcher PostgreSQL migrations..." + RUST_LOG=debug DATABASE_URL="$(TEST_DATABASE_URL)" sqlx migrate run --source migrations + +.PHONY: test-db-prepare +test-db-prepare: ## Run sqlx prepare for compile-time query verification + @echo "Running sqlx prepare for PostgreSQL..." + DATABASE_URL="$(TEST_DATABASE_URL)" cargo sqlx prepare -- + +# --- Build and Test Targets --- +.PHONY: test-db-run +test-db-run: ## Run tests with PostgreSQL feature + @echo "Running tests with PostgreSQL..." + DATABASE_URL="$(TEST_DATABASE_URL)" cargo test --no-default-features + +.PHONY: build-pg +build-pg: ## Build with PostgreSQL feature + @echo "Building with PostgreSQL feature..." + cargo build --no-default-features + +.PHONY: check-pg +check-pg: ## Check code with PostgreSQL feature + @echo "Checking code with PostgreSQL feature..." + cargo check --no-default-features + +.PHONY: clippy +clippy: clippy-pg + +.PHONY: clippy-pg +clippy-pg: ## Run clippy with PostgreSQL feature + @echo "Running clippy with PostgreSQL feature..." + DATABASE_URL="$(TEST_DATABASE_URL)" cargo clippy --no-default-features -- -D warnings + +# --- Cleanup Targets --- +.PHONY: clean +clean: ## Clean build artifacts and SQLx cache + cargo clean + rm -rf .sqlx + +.PHONY: clean-db +clean-db: test-db-down ## Stop database and clean volumes + docker volume rm -f nym_data_observatory_test_data 2>/dev/null || true + +# --- Utility Targets --- +.PHONY: sqlx-cli +sqlx-cli: ## Install sqlx-cli if not already installed + @command -v sqlx >/dev/null 2>&1 || cargo install sqlx-cli --features postgres + +.PHONY: psql +psql: ## Connect to the running PostgreSQL database with psql + @docker exec -it $(DB_CONTAINER_NAME) psql -U testuser -d nym_data_observatory_test + +.PHONY: help +help: ## Show help for Makefile targets + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' \ No newline at end of file diff --git a/nym-data-observatory/README.md b/nym-data-observatory/README.md new file mode 100644 index 00000000000..bc9b4484c72 --- /dev/null +++ b/nym-data-observatory/README.md @@ -0,0 +1,96 @@ +# Nym Data Observatory + +Collects data about the Nym network including: + +- **Chain scraper** - that parses blocks, transactions and messages on the Nyx chain +- **Price scraper** - to get the NYM/USD token price from CoinGecko +- **Webhooks** - trigger on messages or all messages to call with details + +## Running locally + +### 1. Install Prerequisites + +```bash +# Install sqlx-cli if not already installed +make sqlx-cli +``` + +### 2. Prepare PostgreSQL for Development + +```bash +# This will: +# - Start PostgreSQL in Docker +# - Run migrations +# - Generate SQLx offline query cache +# - Stop the database +make prepare-pg +``` + +### 3. Build + +```bash +make build-pg +``` + +### 4. Run with PostgreSQL + +```bash +# Start PostgreSQL for development (keeps running) +make test-db-up + +# In another terminal, run the application +NYM_DATA_OBSERVATORY_DB_URL=postgres://testuser:testpass@localhost:5433/nym_data_observatory_test \ +NYM_DATA_OBSERVATORY_WEBHOOK_URL="https://webhook.site" \ +NYM_DATA_OBSERVATORY_WEBHOOK_AUTH=1234 \ +cargo run -- run +``` + +To start from a block add the env var: `NYXD_SCRAPER_START_HEIGHT=19266184`. + +## Deploying + +Connect with `psql` to your local database: + +```sql +CREATE USER nym_data_observatory WITH PASSWORD 'data-data-data'; + +CREATE DATABASE nym_data_observatory_data; +GRANT ALL ON DATABASE nym_data_observatory_data TO nym_data_observatory; +``` + +Then run: + +``` +cargo run -- init --db_url postgres://nym_data_observatory:data-data-data@localhost/nym_data_observatory_data +``` + +and then: + +``` +NYM_DATA_OBSERVATORY_DB_URL=postgres://nym_data_observatory:data-data-data@localhost/nym_data_observatory_data \ +NYM_DATA_OBSERVATORY_WEBHOOK_URL="https://webhook.site" \ +NYM_DATA_OBSERVATORY_WEBHOOK_AUTH=1234 \ +cargo run -- run +``` + +## Troubleshooting + +### SQLx Offline Mode + +If you see "no cached data for this query" errors: + +1. Ensure PostgreSQL is running: `make dev-db` +2. Run: `make test-db-prepare` + +Also see [README_SQLX.md](../nyx-chain-watcher/README_SQLX.md). + +### Connection Refused + +If you see "Connection refused" errors: + +1. Check Docker is running: `docker ps` +2. Check PostgreSQL container: `docker ps | grep nym_data_observatory +3. Restart database: `make test-db-down && make dev-db` + + + diff --git a/nym-data-observatory/build.rs b/nym-data-observatory/build.rs new file mode 100644 index 00000000000..c1dcd9dcb30 --- /dev/null +++ b/nym-data-observatory/build.rs @@ -0,0 +1,8 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +fn main() { + if let Ok(database_url) = std::env::var("DATABASE_URL") { + println!("cargo:rustc-env=DATABASE_URL={database_url}"); + } +} diff --git a/nym-data-observatory/docker-compose.yml b/nym-data-observatory/docker-compose.yml new file mode 100644 index 00000000000..64152aa1bd0 --- /dev/null +++ b/nym-data-observatory/docker-compose.yml @@ -0,0 +1,21 @@ +services: + postgres-test: + image: postgres:16-alpine + container_name: nym_data_observatory_test + environment: + POSTGRES_DB: nym_data_observatory_test + POSTGRES_USER: testuser + POSTGRES_PASSWORD: testpass + ports: + - '5433:5432' # Map to 5433 to avoid conflicts with default PostgreSQL + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U testuser -d nyx_chain_scraper_test'] + interval: 5s + timeout: 5s + retries: 5 + # Optional: Add volume for persistent data during development +# volumes: +# - nym_data_observatory_test_data:/var/lib/postgresql/data + +#volumes: +# nym_data_observatory_test_data: \ No newline at end of file diff --git a/nym-data-observatory/migrations/0101_price_data.sql b/nym-data-observatory/migrations/0101_price_data.sql new file mode 100644 index 00000000000..80f30d736c8 --- /dev/null +++ b/nym-data-observatory/migrations/0101_price_data.sql @@ -0,0 +1,8 @@ +CREATE TABLE price_history ( + timestamp bigint PRIMARY KEY, + chf double precision NOT NULL, + usd double precision NOT NULL, + eur double precision NOT NULL, + btc double precision NOT NULL, + gbp double precision NOT NULL +); \ No newline at end of file diff --git a/nym-data-observatory/migrations/0102_payment_transactions.sql b/nym-data-observatory/migrations/0102_payment_transactions.sql new file mode 100644 index 00000000000..05c0ce55490 --- /dev/null +++ b/nym-data-observatory/migrations/0102_payment_transactions.sql @@ -0,0 +1,10 @@ +CREATE TABLE payments ( + id INTEGER PRIMARY KEY, + transaction_hash TEXT NOT NULL UNIQUE, + sender_address TEXT NOT NULL, + receiver_address TEXT NOT NULL, + amount double precision NOT NULL, + timestamp bigint NOT NULL, + height bigint NOT NULL, + memo TEXT +); diff --git a/nym-data-observatory/migrations/0103_create_transactions_table.sql b/nym-data-observatory/migrations/0103_create_transactions_table.sql new file mode 100644 index 00000000000..61f30fc0408 --- /dev/null +++ b/nym-data-observatory/migrations/0103_create_transactions_table.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS transactions ( + id INTEGER PRIMARY KEY, + tx_hash TEXT NOT NULL, + height BIGINT NOT NULL, + message_index BIGINT NOT NULL, + sender TEXT NOT NULL, + recipient TEXT NOT NULL, + amount TEXT NOT NULL, + memo TEXT, + created_at DATE DEFAULT CURRENT_TIMESTAMP, + UNIQUE(tx_hash, message_index) +); \ No newline at end of file diff --git a/nym-data-observatory/migrations/0104_add_listener_failure_table.sql b/nym-data-observatory/migrations/0104_add_listener_failure_table.sql new file mode 100644 index 00000000000..3c80ebf473e --- /dev/null +++ b/nym-data-observatory/migrations/0104_add_listener_failure_table.sql @@ -0,0 +1,11 @@ +/* + * Copyright 2025 - Nym Technologies SA + * SPDX-License-Identifier: GPL-3.0-only + */ + +CREATE TABLE watcher_execution +( + start_ts TIMESTAMPTZ NOT NULL, + end_ts TIMESTAMPTZ NOT NULL, + error_message TEXT +); \ No newline at end of file diff --git a/nym-data-observatory/src/chain_scraper/mod.rs b/nym-data-observatory/src/chain_scraper/mod.rs new file mode 100644 index 00000000000..3b6b908f8d2 --- /dev/null +++ b/nym-data-observatory/src/chain_scraper/mod.rs @@ -0,0 +1,66 @@ +use crate::db::DbPool; +use crate::env::vars::{ + NYXD_SCRAPER_START_HEIGHT, NYXD_SCRAPER_UNSAFE_NUKE_DB, + NYXD_SCRAPER_USE_BEST_EFFORT_START_HEIGHT, +}; +use nyxd_scraper_psql::{PostgresNyxdScraper, PruningOptions}; +use std::fs; +use tracing::{info, warn}; + +pub(crate) mod webhook; + +pub(crate) async fn run_chain_scraper( + config: &crate::config::Config, + _connection_pool: DbPool, +) -> anyhow::Result { + let websocket_url = std::env::var("NYXD_WS").expect("NYXD_WS not defined"); + + let rpc_url = std::env::var("NYXD").expect("NYXD not defined"); + let websocket_url = reqwest::Url::parse(&websocket_url)?; + let rpc_url = reqwest::Url::parse(&rpc_url)?; + + // why are those not part of CLI? : ( + let start_block_height = match std::env::var(NYXD_SCRAPER_START_HEIGHT).ok() { + None => None, + // blow up if passed malformed env value + Some(raw) => Some(raw.parse()?), + }; + + let use_best_effort_start_height = + match std::env::var(NYXD_SCRAPER_USE_BEST_EFFORT_START_HEIGHT).ok() { + None => false, + // blow up if passed malformed env value + Some(raw) => raw.parse()?, + }; + + let nuke_db: bool = match std::env::var(NYXD_SCRAPER_UNSAFE_NUKE_DB).ok() { + None => false, + // blow up if passed malformed env value + Some(raw) => raw.parse()?, + }; + + if nuke_db { + warn!("☢️☢️☢️ NUKING THE SCRAPER DATABASE"); + fs::remove_file(config.chain_scraper_connection_string())?; + } + + let scraper = PostgresNyxdScraper::builder(nyxd_scraper_psql::Config { + websocket_url, + rpc_url, + database_storage: config.chain_scraper_connection_string.clone(), + pruning_options: PruningOptions::nothing(), + store_precommits: false, + start_block: nyxd_scraper_psql::StartingBlockOpts { + start_block_height, + use_best_effort_start_height, + }, + }) + .with_msg_module(webhook::WebhookModule::new(config.clone())?); + + let instance = scraper.build_and_start().await?; + + info!("🚧 blocking until the chain has caught up..."); + instance.wait_for_startup_sync().await; + + Ok(instance) +} diff --git a/nym-data-observatory/src/chain_scraper/webhook.rs b/nym-data-observatory/src/chain_scraper/webhook.rs new file mode 100644 index 00000000000..221d99e0080 --- /dev/null +++ b/nym-data-observatory/src/chain_scraper/webhook.rs @@ -0,0 +1,147 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::config::data_observatory::{HttpAuthenticationOptions, WebhookConfig}; +use crate::models::WebhookPayload; +use anyhow::Context; +use async_trait::async_trait; +use nym_validator_client::nyxd::{Any, Msg, MsgSend, Name}; +use nyxd_scraper_psql::{ + MsgModule, NyxdScraperTransaction, ParsedTransactionResponse, ScraperError, +}; +use nyxd_scraper_shared::{default_message_registry, MessageRegistry}; +use reqwest::{Client, Url}; +use tracing::{error, info, warn}; +use utoipa::gen::serde_json; + +pub struct WebhookModule { + webhooks: Vec, + registry: MessageRegistry, +} + +impl WebhookModule { + pub fn new(config: crate::config::Config) -> anyhow::Result { + let webhooks = config + .data_observatory_config + .webhooks + .iter() + .map(|watcher_cfg| Webhook::new(watcher_cfg.clone())) + .collect::>>()?; + Ok(Self { + webhooks, + registry: default_message_registry(), + }) + } + + fn decode_or_skip(&self, msg: &Any) -> Option { + match self.registry.try_decode(msg) { + Ok(decoded) => Some(decoded), + Err(err) => { + warn!("webhook processing failed {err}"); + None + } + } + } +} + +#[async_trait] +impl MsgModule for WebhookModule { + fn type_url(&self) -> String { + ::Proto::type_url() + } + + async fn handle_msg( + &mut self, + index: usize, + msg: &Any, + tx: &ParsedTransactionResponse, + _storage_tx: &mut dyn NyxdScraperTransaction, + ) -> Result<(), ScraperError> { + let message = serde_json::to_value(self.decode_or_skip(msg)).ok(); + + let payload = WebhookPayload { + height: tx.height.value(), + message_index: index as u64, + transaction_hash: tx.hash.to_string(), + message, + }; + + for webhook in self.webhooks.clone() { + let payload = payload.clone(); + tokio::spawn(async move { + webhook + .invoke_webhook(&payload) + .await + .expect("webhook failed to process"); + }); + } + + Ok(()) + } +} + +#[derive(Clone)] +pub(crate) struct Webhook { + webhook_url: Url, + config: WebhookConfig, +} + +impl Webhook { + pub(crate) fn new(config: WebhookConfig) -> anyhow::Result { + Ok(Webhook { + webhook_url: config + .webhook_url + .as_str() + .parse() + .context("invalid config: provided webhook URL is malformed")?, + config, + }) + } + + pub(crate) fn id(&self) -> &str { + &self.config.id + } + + pub(crate) async fn invoke_webhook(&self, payload: &WebhookPayload) -> anyhow::Result<()> { + let client = Client::builder() + .user_agent(format!( + "nym-data-observatory/{}/webhook-{}", + env!("CARGO_PKG_VERSION"), + self.id() + )) + .build() + .context("failed to build reqwest client")?; + + let mut request_builder = client.post(self.webhook_url.clone()).json(payload); + + if let Some(auth) = &self.config.authentication { + match auth { + HttpAuthenticationOptions::AuthorizationBearerToken { token } => { + request_builder = request_builder.bearer_auth(token); + } + } + } + + match request_builder.send().await { + Ok(res) => info!( + "[webhook = {}] ✅ Webhook {} {} - tx {}, index {}", + self.config.id, + res.status(), + res.url(), + payload.transaction_hash, + payload.message_index, + ), + Err(err) => { + error!( + "[webhook = {}] ❌ Webhook {:?} {:?} error = {err}", + self.config.id, + err.status(), + err.url(), + ); + return Err(err.into()); + } + } + + Ok(()) + } +} diff --git a/nym-data-observatory/src/cli/commands/build_info.rs b/nym-data-observatory/src/cli/commands/build_info.rs new file mode 100644 index 00000000000..e3957b66fa2 --- /dev/null +++ b/nym-data-observatory/src/cli/commands/build_info.rs @@ -0,0 +1,17 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::error::NymDataObservatoryError; +use nym_bin_common::bin_info_owned; +use nym_bin_common::output_format::OutputFormat; + +#[derive(clap::Args, Debug)] +pub(crate) struct Args { + #[clap(short, long, default_value_t = OutputFormat::default())] + output: OutputFormat, +} + +pub(crate) fn execute(args: Args) -> Result<(), NymDataObservatoryError> { + println!("{}", args.output.format(&bin_info_owned!())); + Ok(()) +} diff --git a/nym-data-observatory/src/cli/commands/init.rs b/nym-data-observatory/src/cli/commands/init.rs new file mode 100644 index 00000000000..1c1fbde3e20 --- /dev/null +++ b/nym-data-observatory/src/cli/commands/init.rs @@ -0,0 +1,46 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::cli::DEFAULT_NYM_DATA_OBSERVATORY_ID; +use crate::config::data_observatory::HttpAuthenticationOptions::AuthorizationBearerToken; +use crate::config::data_observatory::WebhookConfig; +use crate::config::{default_config_filepath, Config, ConfigBuilder, DataObservatoryConfig}; +use crate::env::vars::*; +use crate::error::NymDataObservatoryError; +use nym_config::save_unformatted_config_to_file; + +#[derive(clap::Args, Debug)] +pub(crate) struct Args { + /// (Override) Postgres connection string for data storage + #[arg(long, env = NYM_DATA_OBSERVATORY_DB_URL, alias = "db_url")] + pub(crate) chain_history_db_connection_string: String, +} + +pub(crate) async fn execute(args: Args) -> Result<(), NymDataObservatoryError> { + let config_path = default_config_filepath(); + let data_dir = Config::default_data_directory(&config_path)?; + + let builder = ConfigBuilder::new( + config_path.clone(), + data_dir, + args.chain_history_db_connection_string, + ) + .with_data_observatory_config(DataObservatoryConfig { + webhooks: vec![WebhookConfig { + id: DEFAULT_NYM_DATA_OBSERVATORY_ID.to_string(), + webhook_url: "https://webhook.site".to_string(), + authentication: Some(AuthorizationBearerToken { + token: "1234".to_string(), + }), + description: None, + watch_for_chain_message_types: vec![ + "/cosmos.bank.v1beta1.MsgSend".to_string(), + "/ibc.applications.transfer.v1.MsgTransfer".to_string(), + ], + }], + }); + + let config = builder.build(); + + Ok(save_unformatted_config_to_file(&config, &config_path)?) +} diff --git a/nym-data-observatory/src/cli/commands/mod.rs b/nym-data-observatory/src/cli/commands/mod.rs new file mode 100644 index 00000000000..b1f63f4ae53 --- /dev/null +++ b/nym-data-observatory/src/cli/commands/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod build_info; +pub(crate) mod init; +pub(crate) mod run; diff --git a/nym-data-observatory/src/cli/commands/run/args.rs b/nym-data-observatory/src/cli/commands/run/args.rs new file mode 100644 index 00000000000..dc34af57151 --- /dev/null +++ b/nym-data-observatory/src/cli/commands/run/args.rs @@ -0,0 +1,33 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::env::vars::*; + +#[derive(clap::Args, Debug)] +pub(crate) struct Args { + /// (Override) Postgres connection string for chain scraper history + #[arg(long, env = NYM_DATA_OBSERVATORY_DB_URL, alias = "db_url")] + pub(crate) db_connection_string: Option, + + /// (Override) Watch for chain messages of these types + #[clap( + long, + value_delimiter = ',', + env = NYM_DATA_OBSERVATORY_WATCH_CHAIN_MESSAGE_TYPES + )] + pub watch_for_chain_message_types: Vec, + + /// (Override) The webhook to call when we find something + #[clap( + long, + env = NYM_DATA_OBSERVATORY_WEBHOOK_URL + )] + pub webhook_url: Option, + + /// (Override) Optionally, authenticate with the webhook + #[clap( + long, + env = NYM_DATA_OBSERVATORY_WEBHOOK_AUTH + )] + pub webhook_auth: Option, +} diff --git a/nym-data-observatory/src/cli/commands/run/config.rs b/nym-data-observatory/src/cli/commands/run/config.rs new file mode 100644 index 00000000000..0519b149ed6 --- /dev/null +++ b/nym-data-observatory/src/cli/commands/run/config.rs @@ -0,0 +1,68 @@ +use crate::cli::commands::run::args::Args; +use crate::cli::DEFAULT_NYM_DATA_OBSERVATORY_ID; +use crate::config::data_observatory::{HttpAuthenticationOptions, WebhookConfig}; +use crate::config::{default_config_filepath, Config, ConfigBuilder, DataObservatoryConfig}; +use crate::error::NymDataObservatoryError; +use tracing::{info, warn}; + +pub(crate) fn get_run_config(args: Args) -> Result { + info!("{args:#?}"); + + let Args { + mut watch_for_chain_message_types, + webhook_auth, + webhook_url, + .. + } = args; + + // if there are no args set, then try load the config + if args.db_connection_string.is_none() { + info!("Loading default config file..."); + return Config::read_from_toml_file_in_default_location(); + } + + // set default messages + if watch_for_chain_message_types.is_empty() { + watch_for_chain_message_types = vec!["/cosmos.bank.v1beta1.MsgSend".to_string()]; + } + + let config_path = default_config_filepath(); + let data_dir = Config::default_data_directory(&config_path)?; + + if args.db_connection_string.is_none() { + return Err(NymDataObservatoryError::DbConnectionStringMissing); + } + + let mut builder = ConfigBuilder::new( + config_path, + data_dir, + args.db_connection_string + .expect("db connection string is required"), + ); + + if let Some(webhook_url) = webhook_url { + let authentication = + webhook_auth.map(|token| HttpAuthenticationOptions::AuthorizationBearerToken { token }); + + let watcher_config = DataObservatoryConfig { + webhooks: vec![WebhookConfig { + id: DEFAULT_NYM_DATA_OBSERVATORY_ID.to_string(), + description: None, + watch_for_chain_message_types, + webhook_url, + authentication, + }], + }; + + info!("Overriding watcher config with env vars"); + + builder = builder.with_data_observatory_config(watcher_config); + } else { + warn!( + "You did not specify a webhook in {}. Only database items will be stored.", + crate::env::vars::NYM_DATA_OBSERVATORY_WEBHOOK_URL + ); + } + + Ok(builder.build()) +} diff --git a/nym-data-observatory/src/cli/commands/run/mod.rs b/nym-data-observatory/src/cli/commands/run/mod.rs new file mode 100644 index 00000000000..eb7ee796f09 --- /dev/null +++ b/nym-data-observatory/src/cli/commands/run/mod.rs @@ -0,0 +1,211 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::error::NymDataObservatoryError; +use anyhow::Context; +use std::time::Duration; +use time::OffsetDateTime; +use tokio::task::{JoinHandle, JoinSet}; +use tokio_util::sync::CancellationToken; +use tracing::{error, info}; + +mod args; +mod config; + +use crate::chain_scraper::run_chain_scraper; +use crate::db::DbPool; +use crate::http::state::PriceScraperState; +use crate::price_scraper::PriceScraper; +use crate::{db, http}; +pub(crate) use args::Args; +use nym_task::signal::wait_for_signal; + +async fn try_insert_watcher_execution_information( + db_pool: DbPool, + start: OffsetDateTime, + end: OffsetDateTime, + error_message: Option, +) { + let _ = sqlx::query!( + r#" + INSERT INTO watcher_execution(start_ts, end_ts, error_message) + VALUES ($1, $2, $3) + "#, + start.into(), + end.into(), + error_message + ) + .execute(&db_pool) + .await + .inspect_err(|err| error!("failed to insert run information: {err}")); +} + +async fn wait_for_shutdown( + db_pool: DbPool, + start: OffsetDateTime, + main_cancellation_token: CancellationToken, + scraper_cancellation_token: CancellationToken, + mut tasks: JoinSet>>, +) { + async fn finalize_shutdown( + db_pool: DbPool, + start: OffsetDateTime, + main_cancellation_token: CancellationToken, + scraper_cancellation_token: CancellationToken, + mut tasks: JoinSet>>, + error_message: Option, + ) { + // cancel all tasks + main_cancellation_token.cancel(); + scraper_cancellation_token.cancel(); + + // stupid nasty and hacky workaround to make sure all relevant tasks have finished before hard aborting them + // nasty stupid and hacky workaround + tokio::time::sleep(Duration::from_secs(1)).await; + tasks.abort_all(); + + // insert execution result into the db + try_insert_watcher_execution_information( + db_pool, + start, + OffsetDateTime::now_utc(), + error_message, + ) + .await + } + + tokio::select! { + // graceful shutdown + _ = wait_for_signal() => { + info!("received shutdown signal"); + finalize_shutdown(db_pool, start, main_cancellation_token, scraper_cancellation_token, tasks, None).await; + } + _ = scraper_cancellation_token.cancelled() => { + info!("the scraper has issued cancellation"); + finalize_shutdown(db_pool, start, main_cancellation_token, scraper_cancellation_token, tasks, Some("unexpected scraper task cancellation".into())).await; + } + _ = main_cancellation_token.cancelled() => { + info!("one of the tasks has cancelled the token"); + finalize_shutdown(db_pool, start, main_cancellation_token, scraper_cancellation_token, tasks, Some("unexpected main task cancellation".into())).await; + } + task_result = tasks.join_next() => { + // the first unwrap is fine => join set was not empty + let error_message = match task_result.unwrap() { + Err(_join_err) => Some("unexpected join error".to_string()), + Ok(Some(Ok(_))) => None, + Ok(Some(Err(err))) => Some(err.to_string()), + Ok(None) => { + Some("unexpected task cancellation".to_string()) + } + }; + + error!("unexpected task termination: {error_message:?}"); + finalize_shutdown(db_pool, start, main_cancellation_token, scraper_cancellation_token, tasks, error_message).await; + } + + } +} + +pub(crate) async fn execute(args: Args, http_port: u16) -> Result<(), NymDataObservatoryError> { + let start = OffsetDateTime::now_utc(); + + info!("passed arguments: {args:#?}"); + + let config = config::get_run_config(args)?; + + let db_connection_string = config.chain_scraper_connection_string(); + + info!("Config is {config:#?}"); + info!( + "Chain History Database path is {:?}", + std::path::Path::new(&config.chain_scraper_connection_string()).canonicalize() + ); + + let storage = db::Storage::init(db_connection_string).await?; + let watcher_pool = storage.pool_owned(); + + let mut tasks = JoinSet::new(); + let cancellation_token = CancellationToken::new(); + + let scraper_pool = storage.pool_owned(); + let shutdown_pool = storage.pool_owned(); + + // construct shared state + let price_scraper_shared_state = PriceScraperState::new(); + + // spawn all the tasks + + // 1. chain scraper (note: this doesn't really spawn the full scraper on this task, but we don't want to be blocking waiting for its startup) + let scraper_token_handle: JoinHandle> = tokio::spawn({ + let config = config.clone(); + async move { + // this only blocks until startup sync is done; it then runs on its own set of tasks + let scraper = run_chain_scraper(&config, scraper_pool).await?; + Ok(scraper.cancel_token()) + } + }); + + // 2. price scraper (note, this task never terminates on its own) + let price_scraper = PriceScraper::new(price_scraper_shared_state.clone(), watcher_pool); + { + let token = cancellation_token.clone(); + tasks.spawn(async move { + token + .run_until_cancelled(async move { + price_scraper.run().await; + Ok(()) + }) + .await + }); + } + + // 3. http api + let http_server = http::server::build_http_api( + storage.pool_owned(), + &config, + http_port, + price_scraper_shared_state, + ) + .await?; + { + let token = cancellation_token.clone(); + tasks.spawn(async move { + info!("Starting HTTP server on port {http_port}",); + async move { + Some( + http_server + .run(token.cancelled_owned()) + .await + .context("http server failure"), + ) + } + .await + }); + } + + // 1. wait for either shutdown or scraper having finished startup + tokio::select! { + _ = wait_for_signal() => { + info!("received shutdown signal while waiting for scraper to finish its startup"); + return Ok(()) + } + scraper_token = scraper_token_handle => { + let scraper_token = match scraper_token { + Ok(Ok(token)) => token, + Ok(Err(startup_err)) => { + error!("failed to startup the chain scraper: {startup_err}"); + return Err(startup_err.into()); + } + Err(runtime_err) => { + error!("failed to finish the scraper startup task: {runtime_err}"); + return Ok(()) + + } + }; + + wait_for_shutdown(shutdown_pool, start, cancellation_token, scraper_token, tasks).await + } + } + + Ok(()) +} diff --git a/nym-data-observatory/src/cli/mod.rs b/nym-data-observatory/src/cli/mod.rs new file mode 100644 index 00000000000..698bb9d293f --- /dev/null +++ b/nym-data-observatory/src/cli/mod.rs @@ -0,0 +1,67 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::cli::commands::{build_info, init, run}; +use crate::env::vars::*; +use crate::error::NymDataObservatoryError; +use clap::{Parser, Subcommand}; +use nym_bin_common::bin_info; +use std::sync::OnceLock; + +mod commands; + +pub const DEFAULT_NYM_DATA_OBSERVATORY_ID: &str = "default-nym-data-observatory"; + +// Helper for passing LONG_VERSION to clap +fn pretty_build_info_static() -> &'static str { + static PRETTY_BUILD_INFORMATION: OnceLock = OnceLock::new(); + PRETTY_BUILD_INFORMATION.get_or_init(|| bin_info!().pretty_print()) +} + +#[derive(Parser, Debug)] +#[clap(author = "Nymtech", version, long_version = pretty_build_info_static(), about)] +pub(crate) struct Cli { + /// Path pointing to an env file that configures the nym-data-observatory and overrides any preconfigured values. + #[clap( + short, + long, + env = NYM_DATA_OBSERVATORY_CONFIG_ENV_FILE_ARG + )] + pub(crate) config_env_file: Option, + + /// Flag used for disabling the printed banner in tty. + #[clap( + long, + env = NYM_DATA_OBSERVATORY_NO_BANNER_ARG + )] + pub(crate) no_banner: bool, + + /// Port to listen on + #[arg(long, default_value_t = 8000, env = "NYM_DATA_OBSERVATORY_HTTP_PORT")] + pub(crate) http_port: u16, + + #[clap(subcommand)] + command: Commands, +} + +impl Cli { + pub(crate) async fn execute(self) -> Result<(), NymDataObservatoryError> { + match self.command { + Commands::BuildInfo(args) => build_info::execute(args), + Commands::Run(args) => run::execute(*args, self.http_port).await, + Commands::Init(args) => init::execute(args).await, + } + } +} + +#[derive(Subcommand, Debug)] +pub(crate) enum Commands { + /// Show build information of this binary + BuildInfo(build_info::Args), + + /// Start this nym-chain-watcher + Run(Box), + + /// Initialise config + Init(init::Args), +} diff --git a/nym-data-observatory/src/config/data_observatory.rs b/nym-data-observatory/src/config/data_observatory.rs new file mode 100644 index 00000000000..36ef39e8811 --- /dev/null +++ b/nym-data-observatory/src/config/data_observatory.rs @@ -0,0 +1,20 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct DataObservatoryConfig { + pub webhooks: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebhookConfig { + pub id: String, + pub description: Option, + pub webhook_url: String, + pub watch_for_chain_message_types: Vec, + pub authentication: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum HttpAuthenticationOptions { + AuthorizationBearerToken { token: String }, +} diff --git a/nym-data-observatory/src/config/mod.rs b/nym-data-observatory/src/config/mod.rs new file mode 100644 index 00000000000..3e5b46dcbc7 --- /dev/null +++ b/nym-data-observatory/src/config/mod.rs @@ -0,0 +1,219 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::config::template::CONFIG_TEMPLATE; +use nym_bin_common::logging::LoggingSettings; +use nym_config::{ + must_get_home, read_config_from_toml_file, save_unformatted_config_to_file, NymConfigTemplate, + DEFAULT_CONFIG_DIR, DEFAULT_CONFIG_FILENAME, DEFAULT_DATA_DIR, NYM_DIR, +}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use tracing::{debug, error}; + +pub(crate) mod data_observatory; +mod template; + +pub use crate::config::data_observatory::DataObservatoryConfig; +use crate::error::NymDataObservatoryError; + +const DEFAULT_NYM_DATA_OBSERVATORY_DIR: &str = "nym-data-observatory"; + +/// Derive default path to nym-data-observatory's config directory. +/// It should get resolved to `$HOME/.nym/nym-data-observatory/config` +pub fn default_config_directory() -> PathBuf { + must_get_home() + .join(NYM_DIR) + .join(DEFAULT_NYM_DATA_OBSERVATORY_DIR) + .join(DEFAULT_CONFIG_DIR) +} + +/// Derive default path to nym-data-observatory's config file. +/// It should get resolved to `$HOME/.nym/nym-data-observatory/config/config.toml` +pub fn default_config_filepath() -> PathBuf { + default_config_directory().join(DEFAULT_CONFIG_FILENAME) +} + +pub struct ConfigBuilder { + pub config_path: PathBuf, + + pub data_dir: PathBuf, + + pub chain_scraper_connection_string: String, + pub data_observatory_config: Option, + + pub logging: Option, +} + +impl ConfigBuilder { + pub fn new( + config_path: PathBuf, + data_dir: PathBuf, + chain_scraper_connection_string: String, + ) -> Self { + ConfigBuilder { + config_path, + data_dir, + data_observatory_config: None, + logging: None, + chain_scraper_connection_string, + } + } + + #[allow(dead_code)] + pub fn with_data_observatory_config( + mut self, + data_observatory_config: impl Into, + ) -> Self { + self.data_observatory_config = Some(data_observatory_config.into()); + self + } + + #[allow(dead_code)] + pub fn with_logging(mut self, section: impl Into>) -> Self { + self.logging = section.into(); + self + } + + pub fn build(self) -> Config { + Config { + logging: self.logging.unwrap_or_default(), + save_path: Some(self.config_path), + data_observatory_config: self.data_observatory_config.unwrap_or_default(), + data_dir: self.data_dir, + chain_scraper_connection_string: self.chain_scraper_connection_string, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Config { + // additional metadata holding on-disk location of this config file + #[serde(skip)] + pub(crate) save_path: Option, + + #[serde(skip)] + pub(crate) data_dir: PathBuf, + + pub chain_scraper_connection_string: String, + + #[serde(default)] + pub data_observatory_config: DataObservatoryConfig, + + #[serde(default)] + pub logging: LoggingSettings, +} + +impl NymConfigTemplate for Config { + fn template(&self) -> &'static str { + CONFIG_TEMPLATE + } +} + +impl Config { + #[allow(unused)] + pub fn save(&self) -> Result<(), NymDataObservatoryError> { + let save_location = self.save_location(); + debug!( + "attempting to save config file to '{}'", + save_location.display() + ); + save_unformatted_config_to_file(self, &save_location).map_err(|source| { + NymDataObservatoryError::UnformattedConfigSaveFailure { + path: save_location, + source, + } + }) + } + + #[allow(unused)] + pub fn save_location(&self) -> PathBuf { + self.save_path + .clone() + .unwrap_or(self.default_save_location()) + } + + #[allow(unused)] + pub fn default_save_location(&self) -> PathBuf { + default_config_filepath() + } + + pub fn default_data_directory>( + config_path: P, + ) -> Result { + let config_path = config_path.as_ref(); + + // we got a proper path to the .toml file + let Some(config_dir) = config_path.parent() else { + error!( + "'{}' does not have a parent directory. Have you pointed to the fs root?", + config_path.display() + ); + return Err(NymDataObservatoryError::DataDirDerivationFailure); + }; + + let Some(config_dir_name) = config_dir.file_name() else { + error!( + "could not obtain parent directory name of '{}'. Have you used relative paths?", + config_path.display() + ); + return Err(NymDataObservatoryError::DataDirDerivationFailure); + }; + + if config_dir_name != DEFAULT_CONFIG_DIR { + error!( + "the parent directory of '{}' ({}) is not {DEFAULT_CONFIG_DIR}. currently this is not supported", + config_path.display(), config_dir_name.to_str().unwrap_or("UNKNOWN") + ); + return Err(NymDataObservatoryError::DataDirDerivationFailure); + } + + let Some(node_dir) = config_dir.parent() else { + error!( + "'{}' does not have a parent directory. Have you pointed to the fs root?", + config_dir.display() + ); + return Err(NymDataObservatoryError::DataDirDerivationFailure); + }; + + Ok(node_dir.join(DEFAULT_DATA_DIR)) + } + + pub fn chain_scraper_connection_string(&self) -> String { + self.chain_scraper_connection_string.clone() + } + + // simple wrapper that reads config file and assigns path location + fn read_from_path>( + path: P, + data_dir: P, + ) -> Result { + let path = path.as_ref(); + let data_dir = data_dir.as_ref(); + let mut loaded: Config = read_config_from_toml_file(path).map_err(|source| { + NymDataObservatoryError::ConfigLoadFailure { + path: path.to_path_buf(), + source, + } + })?; + loaded.data_dir = data_dir.to_path_buf(); + loaded.save_path = Some(path.to_path_buf()); + debug!("loaded config file from {}", path.display()); + Ok(loaded) + } + + #[allow(unused)] + pub fn read_from_toml_file>( + path: P, + data_dir: P, + ) -> Result { + Self::read_from_path(path, data_dir) + } + + pub fn read_from_toml_file_in_default_location() -> Result { + let config_path = default_config_filepath(); + let data_dir = Config::default_data_directory(&config_path)?; + Self::read_from_path(config_path, data_dir) + } +} diff --git a/nym-data-observatory/src/config/template.rs b/nym-data-observatory/src/config/template.rs new file mode 100644 index 00000000000..ddc3475aac4 --- /dev/null +++ b/nym-data-observatory/src/config/template.rs @@ -0,0 +1,29 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +// While using normal toml marshalling would have been way simpler with less overhead, +// I think it's useful to have comments attached to the saved config file to explain behaviour of +// particular fields. +// Note: any changes to the template must be reflected in the appropriate structs. +pub(crate) const CONFIG_TEMPLATE: &str = r#" +# This is a TOML config file. +# For more information, see https://github.com/toml-lang/toml + +[data_observatory_config] +{{#each data_observatory_config.webhooks }} +[[webhooks]] +id={{this.id}} +description='{{this.description}}' +webhook_url='{{this.webhook_url}}' +{{/each}} + + + + +##### logging configuration options ##### + +[logging] + +# TODO + +"#; diff --git a/nym-data-observatory/src/db/mod.rs b/nym-data-observatory/src/db/mod.rs new file mode 100644 index 00000000000..c00fa976136 --- /dev/null +++ b/nym-data-observatory/src/db/mod.rs @@ -0,0 +1,38 @@ +use anyhow::{anyhow, Result}; +use sqlx::{migrate::Migrator, postgres::PgConnectOptions, Postgres}; +use std::str::FromStr; + +pub(crate) mod models; +pub(crate) mod queries { + pub mod price; +} + +static _MIGRATOR: Migrator = sqlx::migrate!("./migrations"); + +pub(crate) type DbPool = sqlx::Pool; + +pub(crate) struct Storage { + pool: DbPool, +} + +impl Storage { + pub async fn init(connection_url: String) -> Result { + let connect_options = PgConnectOptions::from_str(&connection_url)?; + + let pool = DbPool::connect_with(connect_options) + .await + .map_err(|err| anyhow!("Failed to connect to {}: {}", &connection_url, err))?; + + // MIGRATOR + // .run(&pool) + // .await + // .map_err(|err| anyhow!("Failed to run migrations: {}", err))?; + + Ok(Storage { pool }) + } + + /// Cloning pool is cheap, it's the same underlying set of connections + pub fn pool_owned(&self) -> DbPool { + self.pool.clone() + } +} diff --git a/nym-data-observatory/src/db/models.rs b/nym-data-observatory/src/db/models.rs new file mode 100644 index 00000000000..c2ec1fcbcd0 --- /dev/null +++ b/nym-data-observatory/src/db/models.rs @@ -0,0 +1,58 @@ +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use time::OffsetDateTime; +use utoipa::ToSchema; + +#[derive(Clone, Serialize, Deserialize, Debug, ToSchema)] +pub(crate) struct CurrencyPrices { + pub(crate) chf: f64, + pub(crate) usd: f64, + pub(crate) eur: f64, + pub(crate) gbp: f64, + pub(crate) btc: f64, +} + +// Struct to hold Coingecko response +#[derive(Clone, Serialize, Deserialize, Debug, ToSchema)] +pub(crate) struct CoingeckoPriceResponse { + pub(crate) nym: CurrencyPrices, +} + +#[derive(Clone, Deserialize, Debug, ToSchema)] +pub(crate) struct PriceRecord { + pub(crate) timestamp: i64, + pub(crate) nym: CurrencyPrices, +} + +#[derive(Serialize, Deserialize, Debug, ToSchema)] +pub(crate) struct PriceHistory { + pub(crate) timestamp: i64, + pub(crate) chf: f64, + pub(crate) usd: f64, + pub(crate) eur: f64, + pub(crate) gbp: f64, + pub(crate) btc: f64, +} + +#[derive(Serialize, Deserialize, Debug, ToSchema)] +pub(crate) struct PaymentRecord { + pub(crate) transaction_hash: String, + pub(crate) sender_address: String, + pub(crate) receiver_address: String, + pub(crate) amount: f64, + pub(crate) timestamp: i64, + pub(crate) height: i64, +} + +#[derive(Serialize, Deserialize, Debug, FromRow)] +pub(crate) struct Transaction { + pub(crate) id: i64, + pub(crate) tx_hash: String, + pub(crate) height: i64, + pub(crate) message_index: i64, + pub(crate) sender: String, + pub(crate) recipient: String, + pub(crate) amount: String, + pub(crate) memo: Option, + pub(crate) created_at: Option, +} diff --git a/nym-data-observatory/src/db/queries/mod.rs b/nym-data-observatory/src/db/queries/mod.rs new file mode 100644 index 00000000000..e75aa1606a7 --- /dev/null +++ b/nym-data-observatory/src/db/queries/mod.rs @@ -0,0 +1,5 @@ +mod price; + +// re-exporting allows us to access all queries via `queries::bla`` +pub(crate) use payments::{get_last_checked_height, insert_payment}; +pub(crate) use price::{get_latest_price, insert_nym_prices}; diff --git a/nym-data-observatory/src/db/queries/price.rs b/nym-data-observatory/src/db/queries/price.rs new file mode 100644 index 00000000000..4a6ddbeea39 --- /dev/null +++ b/nym-data-observatory/src/db/queries/price.rs @@ -0,0 +1,119 @@ +use crate::db::models::{PriceHistory, PriceRecord}; +use crate::db::DbPool; +use chrono::Local; +use std::ops::Sub; + +pub(crate) async fn insert_nym_prices( + pool: &DbPool, + price_data: PriceRecord, +) -> anyhow::Result<()> { + let mut conn = pool.acquire().await?; + let timestamp = price_data.timestamp; + sqlx::query!( + "INSERT INTO price_history + (timestamp, chf, usd, eur, gbp, btc) + VALUES + ($1, $2, $3, $4, $5, $6) + ON CONFLICT(timestamp) DO UPDATE SET + chf=excluded.chf, + usd=excluded.usd, + eur=excluded.eur, + gbp=excluded.gbp, + btc=excluded.btc;", + timestamp, + price_data.nym.chf, + price_data.nym.usd, + price_data.nym.eur, + price_data.nym.gbp, + price_data.nym.btc, + ) + .execute(&mut *conn) + .await?; + + Ok(()) +} + +pub(crate) async fn get_latest_price(pool: &DbPool) -> anyhow::Result { + let result = sqlx::query!( + "SELECT timestamp, chf, usd, eur, gbp, btc FROM price_history ORDER BY timestamp DESC LIMIT 1;" + ) + .fetch_one(pool) + .await?; + + Ok(PriceHistory { + timestamp: result.timestamp, + chf: result.chf, + usd: result.usd, + eur: result.eur, + gbp: result.gbp, + btc: result.btc, + }) +} + +pub(crate) async fn get_average_price(pool: &DbPool) -> anyhow::Result { + // now less 1 day + let earliest_timestamp = Local::now().sub(chrono::Duration::days(1)).timestamp(); + + let result = sqlx::query!( + "SELECT timestamp, chf, usd, eur, gbp, btc FROM price_history WHERE timestamp >= $1;", + earliest_timestamp + ) + .fetch_all(pool) + .await?; + + let mut price = PriceHistory { + timestamp: Local::now().timestamp(), + chf: 0f64, + usd: 0f64, + eur: 0f64, + gbp: 0f64, + btc: 0f64, + }; + + let mut chf_count = 0; + let mut usd_count = 0; + let mut eur_count = 0; + let mut gbp_count = 0; + let mut btc_count = 0; + + for p in &result { + if p.chf != 0f64 { + price.chf += p.chf; + chf_count += 1; + } + if p.usd != 0f64 { + price.usd += p.usd; + usd_count += 1; + } + if p.eur != 0f64 { + price.eur += p.eur; + eur_count += 1; + } + if p.gbp != 0f64 { + price.gbp += p.gbp; + gbp_count += 1; + } + if p.btc != 0f64 { + price.btc += p.btc; + btc_count += 1; + } + } + + if chf_count > 0 { + price.chf /= chf_count as f64; + } + if usd_count > 0 { + price.usd /= usd_count as f64; + } + if eur_count > 0 { + price.eur /= eur_count as f64; + } + if gbp_count > 0 { + price.gbp /= gbp_count as f64; + } + if btc_count > 0 { + price.btc /= btc_count as f64; + } + + Ok(price) +} diff --git a/nym-data-observatory/src/env.rs b/nym-data-observatory/src/env.rs new file mode 100644 index 00000000000..45f054a8395 --- /dev/null +++ b/nym-data-observatory/src/env.rs @@ -0,0 +1,27 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +#[allow(unused)] +pub mod vars { + pub const NYM_DATA_OBSERVATORY_NO_BANNER_ARG: &str = "NYM_DATA_OBSERVATORY_NO_BANNER"; + pub const NYM_DATA_OBSERVATORY_CONFIG_ENV_FILE_ARG: &str = + "NYM_DATA_OBSERVATORY_CONFIG_ENV_FILE_ARG"; + + pub const NYM_DATA_OBSERVATORY_DB_URL: &str = "NYM_DATA_OBSERVATORY_DB_URL"; + + pub const NYXD_SCRAPER_START_HEIGHT: &str = "NYXD_SCRAPER_START_HEIGHT"; + pub const NYXD_SCRAPER_USE_BEST_EFFORT_START_HEIGHT: &str = + "NYXD_SCRAPER_USE_BEST_EFFORT_START_HEIGHT"; + + pub const NYXD_SCRAPER_UNSAFE_NUKE_DB: &str = "NYXD_SCRAPER_UNSAFE_NUKE_DB"; + + pub const NYM_DATA_OBSERVATORY_ID_ARG: &str = "NYM_DATA_OBSERVATORY_ID"; + pub const NYM_DATA_OBSERVATORY_OUTPUT_ARG: &str = "NYM_DATA_OBSERVATORY_OUTPUT"; + + pub const NYM_DATA_OBSERVATORY_CONFIG_PATH_ARG: &str = "NYM_DATA_OBSERVATORY_CONFIG"; + + pub const NYM_DATA_OBSERVATORY_WATCH_CHAIN_MESSAGE_TYPES: &str = + "NYM_DATA_OBSERVATORY_WATCH_CHAIN_MESSAGE_TYPES"; + pub const NYM_DATA_OBSERVATORY_WEBHOOK_URL: &str = "NYM_DATA_OBSERVATORY_WEBHOOK_URL"; + pub const NYM_DATA_OBSERVATORY_WEBHOOK_AUTH: &str = "NYM_DATA_OBSERVATORY_WEBHOOK_AUTH"; +} diff --git a/nym-data-observatory/src/error.rs b/nym-data-observatory/src/error.rs new file mode 100644 index 00000000000..86beec2f5f1 --- /dev/null +++ b/nym-data-observatory/src/error.rs @@ -0,0 +1,43 @@ +use std::io; +use std::path::PathBuf; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum NymDataObservatoryError { + // #[error("failed to save config file using path '{}'. detailed message: {source}", path.display())] + // ConfigSaveFailure { + // path: PathBuf, + // #[source] + // source: io::Error, + // }, + #[error("failed to save config file using path '{}'. detailed message: {source}", path.display())] + UnformattedConfigSaveFailure { + path: PathBuf, + #[source] + source: nym_config::error::NymConfigTomlError, + }, + + #[error("could not derive path to data directory of this nyx chain watcher")] + DataDirDerivationFailure, + + #[error("please provide a database connection string as an env var, cli argument or in a config file")] + DbConnectionStringMissing, + + // #[error("could not derive path to config directory of this nyx chain watcher")] + // ConfigDirDerivationFailure, + #[error("failed to load config file using path '{}'. detailed message: {source}", path.display())] + ConfigLoadFailure { + path: PathBuf, + #[source] + source: io::Error, + }, + + #[error(transparent)] + FileIoFailure(#[from] io::Error), + + #[error(transparent)] + AnyhowFailure(#[from] anyhow::Error), + + #[error(transparent)] + NymConfigTomlE(#[from] nym_config::error::NymConfigTomlError), +} diff --git a/nym-data-observatory/src/http/api/mod.rs b/nym-data-observatory/src/http/api/mod.rs new file mode 100644 index 00000000000..4d2b84ca294 --- /dev/null +++ b/nym-data-observatory/src/http/api/mod.rs @@ -0,0 +1,79 @@ +use anyhow::anyhow; +use axum::{response::Redirect, Router}; +use tokio::net::ToSocketAddrs; +use tower_http::{cors::CorsLayer, trace::TraceLayer}; +use utoipa::OpenApi; +use utoipa_swagger_ui::SwaggerUi; + +use crate::http::{api_docs, server::HttpServer, state::AppState}; + +pub(crate) mod price; +pub(crate) mod status; + +pub(crate) struct RouterBuilder { + unfinished_router: Router, +} + +impl RouterBuilder { + pub(crate) fn with_default_routes() -> Self { + let router = Router::new() + .merge( + SwaggerUi::new("/swagger") + .url("/api-docs/openapi.json", api_docs::ApiDoc::openapi()), + ) + .route( + "/", + axum::routing::get(|| async { Redirect::permanent("/swagger") }), + ) + .nest( + "/v1", + Router::new() + .nest("/status", status::routes()) + .nest("/price", price::routes()), + ); + + Self { + unfinished_router: router, + } + } + + pub(crate) fn with_state(self, state: AppState) -> RouterWithState { + RouterWithState { + router: self.finalize_routes().with_state(state), + } + } + + fn finalize_routes(self) -> Router { + // layers added later wrap earlier layers + self.unfinished_router + // CORS layer needs to wrap other API layers + .layer(setup_cors()) + // logger should be outermost layer + .layer(TraceLayer::new_for_http()) + } +} + +pub(crate) struct RouterWithState { + router: Router, +} + +impl RouterWithState { + pub(crate) async fn build_server( + self, + bind_address: A, + ) -> anyhow::Result { + tokio::net::TcpListener::bind(bind_address) + .await + .map(|listener| HttpServer::new(self.router, listener)) + .map_err(|err| anyhow!("Couldn't bind to address due to {}", err)) + } +} + +fn setup_cors() -> CorsLayer { + use axum::http::Method; + CorsLayer::new() + .allow_origin(tower_http::cors::Any) + .allow_methods([Method::POST, Method::GET, Method::PATCH, Method::OPTIONS]) + .allow_headers(tower_http::cors::Any) + .allow_credentials(false) +} diff --git a/nym-data-observatory/src/http/api/price.rs b/nym-data-observatory/src/http/api/price.rs new file mode 100644 index 00000000000..1971f35f7d6 --- /dev/null +++ b/nym-data-observatory/src/http/api/price.rs @@ -0,0 +1,44 @@ +use crate::db::models::PriceHistory; +use crate::db::queries::price::{get_average_price, get_latest_price}; +use crate::http::error::Error; +use crate::http::error::HttpResult; +use crate::http::state::AppState; +use axum::{extract::State, Json, Router}; + +pub(crate) fn routes() -> Router { + Router::new() + .route("/", axum::routing::get(price)) + .route("/average", axum::routing::get(average_price)) +} + +#[utoipa::path( + tag = "NYM Price", + get, + path = "/v1/price", + responses( + (status = 200, body = String) + ) +)] +/// Fetch the latest price cached by this API +async fn price(State(state): State) -> HttpResult> { + get_latest_price(state.db_pool()) + .await + .map(Json::from) + .map_err(|_| Error::internal()) +} + +#[utoipa::path( + tag = "NYM Price", + get, + path = "/v1/price/average", + responses( + (status = 200, body = String) + ) +)] +/// Fetch the average price cached by this API +async fn average_price(State(state): State) -> HttpResult> { + get_average_price(state.db_pool()) + .await + .map(Json::from) + .map_err(|_| Error::internal()) +} diff --git a/nym-data-observatory/src/http/api/status.rs b/nym-data-observatory/src/http/api/status.rs new file mode 100644 index 00000000000..35d7168cd94 --- /dev/null +++ b/nym-data-observatory/src/http/api/status.rs @@ -0,0 +1,79 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::http::models::status::{ + ApiStatus, HealthResponse, PriceScraperLastError, PriceScraperLastSuccess, + PriceScraperStatusResponse, +}; +use crate::http::state::{AppState, PriceScraperState, StatusState}; +use axum::extract::State; +use axum::routing::get; +use axum::{Json, Router}; +use nym_bin_common::build_information::BinaryBuildInformationOwned; + +pub(crate) fn routes() -> Router { + Router::new() + .route("/health", get(health)) + .route("/build-information", get(build_information)) + .route("/price-scraper", get(price_scraper_status)) +} + +#[utoipa::path( + tag = "Status", + get, + path = "/build-information", + context_path = "/v1/status", + responses( + (status = 200, body = BinaryBuildInformationOwned) + ) +)] +async fn build_information(State(state): State) -> Json { + Json(state.build_information.to_owned()) +} + +#[utoipa::path( + tag = "Status", + get, + path = "/health", + context_path = "/v1/status", + responses( + (status = 200, body = HealthResponse) + ) +)] +async fn health(State(state): State) -> Json { + let uptime = state.startup_time.elapsed(); + + let health = HealthResponse { + status: ApiStatus::Up, + uptime: uptime.as_secs(), + }; + Json(health) +} + +#[utoipa::path( + tag = "Status", + get, + path = "/price-scraper", + context_path = "/v1/status", + responses( + (status = 200, body = PriceScraperStatusResponse) + ) +)] +pub(crate) async fn price_scraper_status( + State(state): State, +) -> Json { + let guard = state.inner.read().await; + Json(PriceScraperStatusResponse { + last_success: guard + .last_success + .as_ref() + .map(|s| PriceScraperLastSuccess { + timestamp: s.timestamp, + response: s.response.clone(), + }), + last_failure: guard.last_failure.as_ref().map(|f| PriceScraperLastError { + timestamp: f.timestamp, + message: f.message.clone(), + }), + }) +} diff --git a/nym-data-observatory/src/http/api_docs.rs b/nym-data-observatory/src/http/api_docs.rs new file mode 100644 index 00000000000..c7dbe0118d0 --- /dev/null +++ b/nym-data-observatory/src/http/api_docs.rs @@ -0,0 +1,14 @@ +use utoipa::OpenApi; +use utoipauto::utoipauto; + +// manually import external structs which are behind feature flags because they +// can't be automatically discovered +// https://github.com/ProbablyClem/utoipauto/issues/13#issuecomment-1974911829 +#[utoipauto(paths = "./nym-data-observatory/src")] +#[derive(OpenApi)] +#[openapi( + info(title = "Nym Data Observatory API"), + tags(), + components(schemas()) +)] +pub(super) struct ApiDoc; diff --git a/nym-data-observatory/src/http/error.rs b/nym-data-observatory/src/http/error.rs new file mode 100644 index 00000000000..fa6b274b69b --- /dev/null +++ b/nym-data-observatory/src/http/error.rs @@ -0,0 +1,21 @@ +pub(crate) type HttpResult = Result; + +pub(crate) struct Error { + message: String, + status: axum::http::StatusCode, +} + +impl Error { + pub(crate) fn internal() -> Self { + Self { + message: String::from("Internal server error"), + status: axum::http::StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +impl axum::response::IntoResponse for Error { + fn into_response(self) -> axum::response::Response { + (self.status, self.message).into_response() + } +} diff --git a/nym-data-observatory/src/http/mod.rs b/nym-data-observatory/src/http/mod.rs new file mode 100644 index 00000000000..c57d17c47b3 --- /dev/null +++ b/nym-data-observatory/src/http/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod api; +pub(crate) mod api_docs; +pub(crate) mod error; +pub(crate) mod models; +pub(crate) mod server; +pub(crate) mod state; diff --git a/nym-data-observatory/src/http/models.rs b/nym-data-observatory/src/http/models.rs new file mode 100644 index 00000000000..c1798eb59d8 --- /dev/null +++ b/nym-data-observatory/src/http/models.rs @@ -0,0 +1,68 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +// if we ever create some sort of chain watcher client, those would need to be extracted + +pub mod status { + use crate::config::data_observatory::WebhookConfig; + use crate::db::models::CoingeckoPriceResponse; + use serde::{Deserialize, Serialize}; + use time::OffsetDateTime; + use utoipa::ToSchema; + + #[derive(Clone, Copy, Debug, Serialize, Deserialize, schemars::JsonSchema, ToSchema)] + #[serde(rename_all = "lowercase")] + pub enum ApiStatus { + Up, + } + + #[derive(Clone, Copy, Debug, Serialize, Deserialize, schemars::JsonSchema, ToSchema)] + pub struct HealthResponse { + pub status: ApiStatus, + pub uptime: u64, + } + + #[derive(Debug, Serialize, Deserialize, ToSchema)] + pub struct ActiveWebhooksResponse { + pub watchers: Vec, + } + + #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] + pub struct Webhook { + pub id: String, + pub description: String, + pub webhook_url: String, + pub watched_message_types: Vec, + } + + impl From<&WebhookConfig> for Webhook { + fn from(value: &WebhookConfig) -> Self { + Webhook { + id: value.id.clone(), + description: value.description.clone().unwrap_or_default(), + webhook_url: value.webhook_url.clone(), + watched_message_types: value.watch_for_chain_message_types.clone(), + } + } + } + + #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] + pub(crate) struct PriceScraperStatusResponse { + pub(crate) last_success: Option, + pub(crate) last_failure: Option, + } + + #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] + pub(crate) struct PriceScraperLastSuccess { + #[serde(with = "time::serde::rfc3339")] + pub(crate) timestamp: OffsetDateTime, + pub(crate) response: CoingeckoPriceResponse, + } + + #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] + pub(crate) struct PriceScraperLastError { + #[serde(with = "time::serde::rfc3339")] + pub(crate) timestamp: OffsetDateTime, + pub(crate) message: String, + } +} diff --git a/nym-data-observatory/src/http/server.rs b/nym-data-observatory/src/http/server.rs new file mode 100644 index 00000000000..fd01f01f871 --- /dev/null +++ b/nym-data-observatory/src/http/server.rs @@ -0,0 +1,59 @@ +use axum::Router; +use core::net::SocketAddr; +use tokio::net::TcpListener; +use tokio_util::sync::WaitForCancellationFutureOwned; + +use crate::config::Config; +use crate::http::state::PriceScraperState; +use crate::{ + db::DbPool, + http::{api::RouterBuilder, state::AppState}, +}; + +pub(crate) async fn build_http_api( + db_pool: DbPool, + config: &Config, + http_port: u16, + price_scraper_state: PriceScraperState, +) -> anyhow::Result { + let router_builder = RouterBuilder::with_default_routes(); + + let state = AppState::new( + db_pool, + config + .data_observatory_config + .webhooks + .iter() + .map(Into::into) + .collect(), + price_scraper_state, + ); + let router = router_builder.with_state(state); + + let bind_addr = format!("0.0.0.0:{http_port}"); + let server = router.build_server(bind_addr).await?; + Ok(server) +} + +pub(crate) struct HttpServer { + router: Router, + listener: TcpListener, +} + +impl HttpServer { + pub(crate) fn new(router: Router, listener: TcpListener) -> Self { + Self { router, listener } + } + + pub(crate) async fn run(self, receiver: WaitForCancellationFutureOwned) -> std::io::Result<()> { + // into_make_service_with_connect_info allows us to see client ip address + // in middleware, for logging, TLS, routing etc. + axum::serve( + self.listener, + self.router + .into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(receiver) + .await + } +} diff --git a/nym-data-observatory/src/http/state.rs b/nym-data-observatory/src/http/state.rs new file mode 100644 index 00000000000..da243731884 --- /dev/null +++ b/nym-data-observatory/src/http/state.rs @@ -0,0 +1,124 @@ +use crate::db::models::CoingeckoPriceResponse; +use crate::db::DbPool; +use crate::http::models::status::Webhook; +use axum::extract::FromRef; +use nym_bin_common::bin_info; +use nym_bin_common::build_information::BinaryBuildInformation; +use std::ops::Deref; +use std::sync::Arc; +use time::OffsetDateTime; +use tokio::sync::RwLock; +use tokio::time::Instant; + +#[derive(Debug, Clone)] +pub(crate) struct AppState { + db_pool: DbPool, + #[allow(dead_code)] + pub(crate) registered_webhooks: Arc>, + pub(crate) status_state: StatusState, + pub(crate) price_scraper_state: PriceScraperState, +} + +impl AppState { + pub(crate) fn new( + db_pool: DbPool, + registered_payment_watchers: Vec, + price_scraper_state: PriceScraperState, + ) -> Self { + Self { + db_pool, + registered_webhooks: Arc::new(registered_payment_watchers), + status_state: Default::default(), + price_scraper_state, + } + } + + pub(crate) fn db_pool(&self) -> &DbPool { + &self.db_pool + } +} + +#[derive(Clone, Debug)] +pub(crate) struct StatusState { + inner: Arc, +} + +impl Default for StatusState { + fn default() -> Self { + StatusState { + inner: Arc::new(StatusStateInner { + startup_time: Instant::now(), + build_information: bin_info!(), + }), + } + } +} + +impl Deref for StatusState { + type Target = StatusStateInner; + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +#[derive(Debug)] +pub(crate) struct StatusStateInner { + pub(crate) startup_time: Instant, + pub(crate) build_information: BinaryBuildInformation, +} + +#[derive(Debug, Clone)] +pub(crate) struct PriceScraperState { + pub(crate) inner: Arc>, +} + +impl PriceScraperState { + pub(crate) fn new() -> Self { + PriceScraperState { + inner: Arc::new(Default::default()), + } + } + + pub(crate) async fn new_failure>(&self, error: S) { + self.inner.write().await.last_failure = Some(PriceScraperLastError { + timestamp: OffsetDateTime::now_utc(), + message: error.into(), + }) + } + pub(crate) async fn new_success(&self, response: CoingeckoPriceResponse) { + self.inner.write().await.last_success = Some(PriceScraperLastSuccess { + timestamp: OffsetDateTime::now_utc(), + response, + }) + } +} + +#[derive(Debug, Default)] +pub(crate) struct PriceScraperStateInner { + pub(crate) last_success: Option, + pub(crate) last_failure: Option, +} + +#[derive(Debug)] +pub(crate) struct PriceScraperLastSuccess { + pub(crate) timestamp: OffsetDateTime, + pub(crate) response: CoingeckoPriceResponse, +} + +#[derive(Debug)] +pub(crate) struct PriceScraperLastError { + pub(crate) timestamp: OffsetDateTime, + pub(crate) message: String, +} + +impl FromRef for StatusState { + fn from_ref(input: &AppState) -> Self { + input.status_state.clone() + } +} + +impl FromRef for PriceScraperState { + fn from_ref(input: &AppState) -> Self { + input.price_scraper_state.clone() + } +} diff --git a/nym-data-observatory/src/logging.rs b/nym-data-observatory/src/logging.rs new file mode 100644 index 00000000000..1c8cbeebb16 --- /dev/null +++ b/nym-data-observatory/src/logging.rs @@ -0,0 +1,43 @@ +use tracing::level_filters::LevelFilter; +use tracing_subscriber::{filter::Directive, EnvFilter}; + +pub(crate) fn setup_tracing_logger() { + fn directive_checked(directive: String) -> Directive { + directive.parse().expect("Failed to parse log directive") + } + + let log_builder = tracing_subscriber::fmt() + // Use a more compact, abbreviated log format + .compact() + // Display source code file paths + .with_file(true) + // Display source code line numbers + .with_line_number(true) + // Don't display the event's target (module path) + .with_target(false); + + let mut filter = EnvFilter::builder() + // if RUST_LOG isn't set, set default level + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(); + // these crates are more granularly filtered + let filter_crates = [ + "nym_bin_common", + "nym_explorer_client", + "nym_network_defaults", + "nym_validator_client", + "reqwest", + "rustls", + "hyper", + "sqlx", + "h2", + "tendermint_rpc", + "tower_http", + "axum", + ]; + for crate_name in filter_crates { + filter = filter.add_directive(directive_checked(format!("{crate_name}=warn"))); + } + + log_builder.with_env_filter(filter).init(); +} diff --git a/nym-data-observatory/src/main.rs b/nym-data-observatory/src/main.rs new file mode 100644 index 00000000000..3a62b35d379 --- /dev/null +++ b/nym-data-observatory/src/main.rs @@ -0,0 +1,34 @@ +use clap::{crate_name, crate_version, Parser}; +use nym_bin_common::bin_info_owned; +use nym_bin_common::logging::maybe_print_banner; +use nym_network_defaults::setup_env; +use tracing::info; + +mod chain_scraper; +mod cli; +mod config; +mod db; +mod env; +mod error; +mod http; +mod logging; +pub mod models; +mod price_scraper; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let cli = cli::Cli::parse(); + setup_env(cli.config_env_file.as_ref()); + logging::setup_tracing_logger(); + + if !cli.no_banner { + maybe_print_banner(crate_name!(), crate_version!()); + } + + let bin_info = bin_info_owned!(); + info!("using the following version: {bin_info}"); + + cli.execute().await?; + + Ok(()) +} diff --git a/nym-data-observatory/src/models.rs b/nym-data-observatory/src/models.rs new file mode 100644 index 00000000000..44d983fac4d --- /dev/null +++ b/nym-data-observatory/src/models.rs @@ -0,0 +1,22 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use utoipa::gen::serde_json; +use utoipa::ToSchema; + +#[derive(Serialize, Deserialize, Clone, JsonSchema, ToSchema)] +pub struct WebhookPayload { + pub height: u64, + pub transaction_hash: String, + pub message_index: u64, + pub message: Option, +} + +pub mod openapi_schema { + use super::*; + + #[derive(ToSchema)] + pub struct Coin { + pub denom: String, + pub amount: String, + } +} diff --git a/nym-data-observatory/src/price_scraper/mod.rs b/nym-data-observatory/src/price_scraper/mod.rs new file mode 100644 index 00000000000..6442118579d --- /dev/null +++ b/nym-data-observatory/src/price_scraper/mod.rs @@ -0,0 +1,76 @@ +use crate::db::{ + models::{CoingeckoPriceResponse, PriceRecord}, + queries::price::insert_nym_prices, +}; +use core::str; +use tokio::time::Duration; + +use crate::db::DbPool; +use crate::http::state::PriceScraperState; + +const REFRESH_DELAY: Duration = Duration::from_secs(300); +const FAILURE_RETRY_DELAY: Duration = Duration::from_secs(60 * 2); +const COINGECKO_API_URL: &str = + "https://api.coingecko.com/api/v3/simple/price?ids=nym&vs_currencies=chf,usd,eur,gbp,btc"; + +pub(crate) struct PriceScraper { + shared_state: PriceScraperState, + db_pool: DbPool, +} + +impl PriceScraper { + pub(crate) fn new(shared_state: PriceScraperState, db_pool: DbPool) -> Self { + PriceScraper { + shared_state, + db_pool, + } + } + + async fn get_coingecko_prices(&self) -> anyhow::Result { + tracing::info!("💰 Fetching CoinGecko prices from {COINGECKO_API_URL}"); + + let response = reqwest::get(COINGECKO_API_URL) + .await? + .json::() + .await; + + tracing::info!("Got response {:?}", response); + match response { + Ok(resp) => { + let price_record = PriceRecord { + timestamp: time::OffsetDateTime::now_utc().unix_timestamp(), + nym: resp.nym.clone(), + }; + + insert_nym_prices(&self.db_pool, price_record).await?; + Ok(resp) + } + Err(err) => { + //tracing::info!("💰 CoinGecko price response: {:?}", response); + tracing::error!("Error sending request: {err}"); + Err(err.into()) + } + } + } + + pub(crate) async fn run(&self) { + loop { + tracing::info!("Running in a loop 🏃"); + match self.get_coingecko_prices().await { + Ok(coingecko_price_response) => { + self.shared_state + .new_success(coingecko_price_response) + .await; + tracing::info!("✅ Successfully fetched CoinGecko prices"); + tokio::time::sleep(REFRESH_DELAY).await; + } + Err(err) => { + tracing::error!("❌ Failed to get CoinGecko prices: {err}"); + tracing::info!("Retrying in {}s...", FAILURE_RETRY_DELAY.as_secs()); + self.shared_state.new_failure(err.to_string()).await; + tokio::time::sleep(FAILURE_RETRY_DELAY).await; + } + } + } + } +} diff --git a/nyx-chain-watcher/src/cli/commands/run/mod.rs b/nyx-chain-watcher/src/cli/commands/run/mod.rs index 150738dfb6e..704a292141c 100644 --- a/nyx-chain-watcher/src/cli/commands/run/mod.rs +++ b/nyx-chain-watcher/src/cli/commands/run/mod.rs @@ -15,7 +15,7 @@ mod config; use crate::chain_scraper::run_chain_scraper; use crate::db::DbPool; use crate::http::state::{BankScraperModuleState, PaymentListenerState, PriceScraperState}; -use crate::payment_listener::PaymentListener; +use crate::listener::PaymentListener; use crate::price_scraper::PriceScraper; use crate::{db, http}; pub(crate) use args::Args; diff --git a/nyx-chain-watcher/src/payment_listener/mod.rs b/nyx-chain-watcher/src/listener/mod.rs similarity index 98% rename from nyx-chain-watcher/src/payment_listener/mod.rs rename to nyx-chain-watcher/src/listener/mod.rs index f941e6c0126..f98db43521b 100644 --- a/nyx-chain-watcher/src/payment_listener/mod.rs +++ b/nyx-chain-watcher/src/listener/mod.rs @@ -7,8 +7,8 @@ use crate::db::queries; use crate::http::state::{ PaymentListenerFailureDetails, PaymentListenerState, ProcessedPayment, WatcherFailureDetails, }; +use crate::listener::watcher::PaymentWatcher; use crate::models::WebhookPayload; -use crate::payment_listener::watcher::PaymentWatcher; use anyhow::Context; use sqlx::SqlitePool; use tokio::time::{self, Duration}; diff --git a/nyx-chain-watcher/src/payment_listener/watcher.rs b/nyx-chain-watcher/src/listener/watcher.rs similarity index 100% rename from nyx-chain-watcher/src/payment_listener/watcher.rs rename to nyx-chain-watcher/src/listener/watcher.rs diff --git a/nyx-chain-watcher/src/main.rs b/nyx-chain-watcher/src/main.rs index 725eee7a06d..8281be987c4 100644 --- a/nyx-chain-watcher/src/main.rs +++ b/nyx-chain-watcher/src/main.rs @@ -12,9 +12,9 @@ mod env; mod error; pub(crate) mod helpers; mod http; +mod listener; mod logging; pub mod models; -mod payment_listener; mod price_scraper; #[tokio::main] From bfb762128d47e2640444077a17c6ee7508cc62ae Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 23 Jul 2025 17:01:54 +0100 Subject: [PATCH 04/28] move message parsing and change webhook --- Cargo.lock | 2 +- .../src/storage/block_storage.rs | 20 ++----- .../src/storage/transaction.rs | 27 +++------- .../src/block_processor/types.rs | 3 ++ common/nyxd-scraper-shared/src/rpc_client.rs | 53 ++++++++++++++----- nym-data-observatory/Cargo.toml | 3 +- .../src/chain_scraper/webhook.rs | 28 ++++------ 7 files changed, 65 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3985bd4ea6c..078a1a2f9f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5739,7 +5739,7 @@ dependencies = [ [[package]] name = "nym-data-observatory" -version = "0.1.0" +version = "1.0.0" dependencies = [ "anyhow", "async-trait", diff --git a/common/nyxd-scraper-psql/src/storage/block_storage.rs b/common/nyxd-scraper-psql/src/storage/block_storage.rs index b16f6c58916..e87e72b1eff 100644 --- a/common/nyxd-scraper-psql/src/storage/block_storage.rs +++ b/common/nyxd-scraper-psql/src/storage/block_storage.rs @@ -11,17 +11,13 @@ use crate::storage::transaction::PostgresStorageTransaction; use async_trait::async_trait; use nyxd_scraper_shared::storage::helpers::log_db_operation_time; use nyxd_scraper_shared::storage::{NyxdScraperStorage, NyxdScraperStorageError}; -use nyxd_scraper_shared::{default_message_registry, MessageRegistry}; use sqlx::types::time::{OffsetDateTime, PrimitiveDateTime}; use tokio::time::Instant; -use tracing::{debug, error, info, instrument}; +use tracing::{debug, error, info, instrument, warn}; #[derive(Clone)] pub struct PostgresScraperStorage { pub(crate) manager: StorageManager, - - // kinda like very limited cosmos sdk codec - pub(crate) message_registry: MessageRegistry, } impl PostgresScraperStorage { @@ -41,8 +37,8 @@ impl PostgresScraperStorage { .run(&connection_pool) .await { - error!("Failed to initialize SQLx database: {err}"); - return Err(err.into()); + warn!("Failed to initialize SQLx database: {err}"); + // return Err(err.into()); } info!("Database migration finished!"); @@ -50,10 +46,7 @@ impl PostgresScraperStorage { let manager = StorageManager { connection_pool }; manager.set_initial_metadata().await?; - let storage = PostgresScraperStorage { - manager, - message_registry: default_message_registry(), - }; + let storage = PostgresScraperStorage { manager }; Ok(storage) } @@ -94,10 +87,7 @@ impl PostgresScraperStorage { .connection_pool .begin() .await - .map(|inner| PostgresStorageTransaction { - inner, - registry: self.message_registry.clone(), - }) + .map(|inner| PostgresStorageTransaction { inner }) .map_err(|source| PostgresScraperError::StorageTxBeginFailure { source }) } diff --git a/common/nyxd-scraper-psql/src/storage/transaction.rs b/common/nyxd-scraper-psql/src/storage/transaction.rs index af118f300a6..a499a198859 100644 --- a/common/nyxd-scraper-psql/src/storage/transaction.rs +++ b/common/nyxd-scraper-psql/src/storage/transaction.rs @@ -21,7 +21,7 @@ use nyxd_scraper_shared::storage::validators::Response; use nyxd_scraper_shared::storage::{ validators, Block, Commit, CommitSig, NyxdScraperStorageError, NyxdScraperTransaction, }; -use nyxd_scraper_shared::{Any, MessageRegistry, ParsedTransactionResponse}; +use nyxd_scraper_shared::ParsedTransactionResponse; use serde_json::{json, Value}; use sqlx::types::time::{OffsetDateTime, PrimitiveDateTime}; use sqlx::{Postgres, Transaction}; @@ -30,8 +30,6 @@ use tracing::{debug, trace, warn}; pub struct PostgresStorageTransaction { pub(super) inner: Transaction<'static, Postgres>, - - pub(super) registry: MessageRegistry, } impl Deref for PostgresStorageTransaction { @@ -48,16 +46,6 @@ impl DerefMut for PostgresStorageTransaction { } impl PostgresStorageTransaction { - fn decode_or_skip(&self, msg: &Any) -> Option { - match self.registry.try_decode(msg) { - Ok(decoded) => Some(decoded), - Err(err) => { - warn!("{err}"); - None - } - } - } - async fn persist_validators( &mut self, validators: &validators::Response, @@ -169,11 +157,9 @@ impl PostgresStorageTransaction { .collect(); let messages = chain_tx - .tx - .body - .messages - .iter() - .filter_map(|msg| self.decode_or_skip(msg)) + .parsed_messages + .values() + .map(|v| v.clone()) .collect::>(); let signer_infos = chain_tx @@ -220,7 +206,8 @@ impl PostgresStorageTransaction { let mut wasm_message_type: Option = None; let mut funds: Option> = None; - let value = serde_json::to_value(self.decode_or_skip(msg))?; + let parsed_message = chain_tx.parsed_messages.get(&index); + let value = serde_json::to_value(parsed_message)?; if msg.type_url == "/cosmwasm.wasm.v1.MsgExecuteContract" { if let Ok(wasm_execute) = MsgExecuteContract::decode(msg.value.as_ref()) { @@ -270,7 +257,7 @@ impl PostgresStorageTransaction { } fn get_first_field_name(value: &Value) -> Option { - debug!("value:\n{value}"); + trace!("value:\n{value}"); match value.as_object() { Some(map) => map.keys().next().cloned(), None => None, diff --git a/common/nyxd-scraper-shared/src/block_processor/types.rs b/common/nyxd-scraper-shared/src/block_processor/types.rs index 1c456b93181..cb17bb698ba 100644 --- a/common/nyxd-scraper-shared/src/block_processor/types.rs +++ b/common/nyxd-scraper-shared/src/block_processor/types.rs @@ -3,6 +3,7 @@ use crate::error::ScraperError; use crate::helpers; +use std::collections::HashMap; use tendermint::{Block, Hash, abci, block, tx}; use tendermint_rpc::endpoint::{block as block_endpoint, block_results, validators}; use tendermint_rpc::event::{Event, EventData}; @@ -26,6 +27,8 @@ pub struct ParsedTransactionResponse { pub tx: cosmrs::tx::Tx, pub proof: Option, + + pub parsed_messages: HashMap, } #[derive(Debug)] diff --git a/common/nyxd-scraper-shared/src/rpc_client.rs b/common/nyxd-scraper-shared/src/rpc_client.rs index 5a3621abd4a..879b15bb6c2 100644 --- a/common/nyxd-scraper-shared/src/rpc_client.rs +++ b/common/nyxd-scraper-shared/src/rpc_client.rs @@ -6,15 +6,16 @@ use crate::block_processor::types::{ }; use crate::error::ScraperError; use crate::helpers::tx_hash; +use crate::{default_message_registry, Any, MessageRegistry}; use futures::StreamExt; use futures::future::join3; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use tendermint::Hash; use tendermint_rpc::endpoint::{block, block_results, tx, validators}; use tendermint_rpc::{Client, HttpClient, Paging}; use tokio::sync::Mutex; -use tracing::{debug, instrument}; +use tracing::{debug, instrument, warn}; use url::Url; #[derive(Clone)] @@ -22,6 +23,9 @@ pub struct RpcClient { // right now I don't care about anything nym specific, so a simple http client is sufficient, // once this is inadequate, we can switch to a NyxdClient inner: Arc, + + // kinda like very limited cosmos sdk codec + pub(crate) message_registry: MessageRegistry, } impl RpcClient { @@ -35,9 +39,20 @@ impl RpcClient { Ok(RpcClient { inner: Arc::new(http_client), + message_registry: default_message_registry(), }) } + fn decode_or_skip(&self, msg: &Any) -> Option { + match self.message_registry.try_decode(msg) { + Ok(decoded) => Some(decoded), + Err(err) => { + warn!("Failed to decode raw message: {err}"); + None + } + } + } + #[instrument(skip(self, block), fields(height = block.height))] pub async fn try_get_full_details( &self, @@ -56,19 +71,29 @@ impl RpcClient { let raw_transactions = raw_transactions?; let mut transactions = Vec::with_capacity(raw_transactions.len()); - for tx in raw_transactions { + for raw_tx in raw_transactions { + let mut parsed_messages = HashMap::new(); + let tx = cosmrs::Tx::from_bytes(&raw_tx.tx).map_err(|source| { + ScraperError::TxParseFailure { + hash: raw_tx.hash, + source, + } + })?; + + for (index, msg) in tx.body.messages.iter().enumerate() { + if let Some(value) = self.decode_or_skip(msg) { + parsed_messages.insert(index, value); + } + } + transactions.push(ParsedTransactionResponse { - hash: tx.hash, - height: tx.height, - index: tx.index, - tx_result: tx.tx_result, - tx: cosmrs::Tx::from_bytes(&tx.tx).map_err(|source| { - ScraperError::TxParseFailure { - hash: tx.hash, - source, - } - })?, - proof: tx.proof, + hash: raw_tx.hash, + height: raw_tx.height, + index: raw_tx.index, + tx_result: raw_tx.tx_result, + tx, + proof: raw_tx.proof, + parsed_messages, }) } diff --git a/nym-data-observatory/Cargo.toml b/nym-data-observatory/Cargo.toml index e215a48daa1..75d07705fe7 100644 --- a/nym-data-observatory/Cargo.toml +++ b/nym-data-observatory/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "nym-data-observatory" -version = "0.1.0" +version = "1.0.0" authors.workspace = true repository.workspace = true homepage.workspace = true @@ -11,7 +11,6 @@ documentation.workspace = true edition.workspace = true license.workspace = true rust-version.workspace = true -readme.workspace = true [dependencies] anyhow = { workspace = true } diff --git a/nym-data-observatory/src/chain_scraper/webhook.rs b/nym-data-observatory/src/chain_scraper/webhook.rs index 221d99e0080..50a06596e8b 100644 --- a/nym-data-observatory/src/chain_scraper/webhook.rs +++ b/nym-data-observatory/src/chain_scraper/webhook.rs @@ -9,14 +9,12 @@ use nym_validator_client::nyxd::{Any, Msg, MsgSend, Name}; use nyxd_scraper_psql::{ MsgModule, NyxdScraperTransaction, ParsedTransactionResponse, ScraperError, }; -use nyxd_scraper_shared::{default_message_registry, MessageRegistry}; use reqwest::{Client, Url}; -use tracing::{error, info, warn}; +use tracing::{error, info}; use utoipa::gen::serde_json; pub struct WebhookModule { webhooks: Vec, - registry: MessageRegistry, } impl WebhookModule { @@ -27,20 +25,7 @@ impl WebhookModule { .iter() .map(|watcher_cfg| Webhook::new(watcher_cfg.clone())) .collect::>>()?; - Ok(Self { - webhooks, - registry: default_message_registry(), - }) - } - - fn decode_or_skip(&self, msg: &Any) -> Option { - match self.registry.try_decode(msg) { - Ok(decoded) => Some(decoded), - Err(err) => { - warn!("webhook processing failed {err}"); - None - } - } + Ok(Self { webhooks }) } } @@ -53,11 +38,11 @@ impl MsgModule for WebhookModule { async fn handle_msg( &mut self, index: usize, - msg: &Any, + _msg: &Any, tx: &ParsedTransactionResponse, _storage_tx: &mut dyn NyxdScraperTransaction, ) -> Result<(), ScraperError> { - let message = serde_json::to_value(self.decode_or_skip(msg)).ok(); + let message = serde_json::to_value(tx.parsed_messages.get(&index)).ok(); let payload = WebhookPayload { height: tx.height.value(), @@ -66,6 +51,11 @@ impl MsgModule for WebhookModule { message, }; + println!( + "->>>>>>>>>>>>>>>>>>>>>>>>> {}", + serde_json::to_string(&payload).unwrap() + ); + for webhook in self.webhooks.clone() { let payload = payload.clone(); tokio::spawn(async move { From 3fc6f6f2c95b9a0526e5fb1dab0f0d922c63b0f5 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Fri, 25 Jul 2025 10:24:53 +0100 Subject: [PATCH 05/28] handle wasm messages in a module --- Cargo.lock | 2 + .../nyxd-scraper-psql/src/storage/manager.rs | 75 +++-------- .../nyxd-scraper-psql/src/storage/models.rs | 11 +- .../src/storage/transaction.rs | 42 +------ .../src/block_processor/types.rs | 2 + common/nyxd-scraper-shared/src/rpc_client.rs | 1 + nym-data-observatory/Cargo.toml | 2 + nym-data-observatory/migrations/0003_wasm.sql | 18 +++ ...ailure_table.sql => 0100_startup_info.sql} | 2 +- .../migrations/0102_payment_transactions.sql | 10 -- .../0103_create_transactions_table.sql | 12 -- nym-data-observatory/src/chain_scraper/mod.rs | 3 +- .../src/chain_scraper/webhook.rs | 7 +- .../src/cli/commands/run/mod.rs | 13 +- nym-data-observatory/src/db/mod.rs | 10 +- nym-data-observatory/src/db/queries/mod.rs | 2 + nym-data-observatory/src/db/queries/wasm.rs | 53 ++++++++ nym-data-observatory/src/main.rs | 1 + nym-data-observatory/src/modules/mod.rs | 4 + nym-data-observatory/src/modules/wasm.rs | 117 ++++++++++++++++++ 20 files changed, 245 insertions(+), 142 deletions(-) create mode 100644 nym-data-observatory/migrations/0003_wasm.sql rename nym-data-observatory/migrations/{0104_add_listener_failure_table.sql => 0100_startup_info.sql} (87%) delete mode 100644 nym-data-observatory/migrations/0102_payment_transactions.sql delete mode 100644 nym-data-observatory/migrations/0103_create_transactions_table.sql create mode 100644 nym-data-observatory/src/db/queries/wasm.rs create mode 100644 nym-data-observatory/src/modules/mod.rs create mode 100644 nym-data-observatory/src/modules/wasm.rs diff --git a/Cargo.lock b/Cargo.lock index 078a1a2f9f0..28c4e7d73df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5746,6 +5746,7 @@ dependencies = [ "axum", "chrono", "clap", + "cosmrs 0.22.0", "nym-bin-common", "nym-config", "nym-network-defaults", @@ -5756,6 +5757,7 @@ dependencies = [ "reqwest 0.12.22", "schemars 0.8.22", "serde", + "serde_json", "sqlx", "thiserror 2.0.17", "time", diff --git a/common/nyxd-scraper-psql/src/storage/manager.rs b/common/nyxd-scraper-psql/src/storage/manager.rs index ef38189241b..73e3b499fce 100644 --- a/common/nyxd-scraper-psql/src/storage/manager.rs +++ b/common/nyxd-scraper-psql/src/storage/manager.rs @@ -1,7 +1,6 @@ // Copyright 2023 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::models::Coin; use crate::storage::models::{CommitSignature, Validator}; use nyxd_scraper_shared::storage::helpers::log_db_operation_time; use sqlx::types::time::PrimitiveDateTime; @@ -403,10 +402,6 @@ pub(crate) async fn insert_message<'a, E>( value: JsonValue, involved_account_addresses: Vec, height: i64, - wasm_sender: Option, - wasm_contract_address: Option, - wasm_message_type: Option, - funds: Option>, executor: E, ) -> Result<(), sqlx::Error> where @@ -415,55 +410,25 @@ where trace!("insert_message"); let start = Instant::now(); - // sqlx doesn't understand option types - if let Some(coins) = funds { - sqlx::query!( - r#" - INSERT INTO message(transaction_hash, index, type, value, involved_accounts_addresses, height, wasm_sender, wasm_contract_address, wasm_message_type, funds) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (transaction_hash, index) DO UPDATE - SET height = excluded.height, - type = excluded.type, - value = excluded.value, - involved_accounts_addresses = excluded.involved_accounts_addresses - "#, - transaction_hash, - index, - typ, - value, - &involved_account_addresses, - height, - wasm_sender, - wasm_contract_address, - wasm_message_type, - &coins as _, - ) - .execute(executor) - .await?; - } else { - sqlx::query!( - r#" - INSERT INTO message(transaction_hash, index, type, value, involved_accounts_addresses, height, wasm_sender, wasm_contract_address, wasm_message_type) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ON CONFLICT (transaction_hash, index) DO UPDATE - SET height = excluded.height, - type = excluded.type, - value = excluded.value, - involved_accounts_addresses = excluded.involved_accounts_addresses - "#, - transaction_hash, - index, - typ, - value, - &involved_account_addresses, - height, - wasm_sender, - wasm_contract_address, - wasm_message_type, - ) - .execute(executor) - .await?; - } + sqlx::query!( + r#" + INSERT INTO message(transaction_hash, index, type, value, involved_accounts_addresses, height) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (transaction_hash, index) DO UPDATE + SET height = excluded.height, + type = excluded.type, + value = excluded.value, + involved_accounts_addresses = excluded.involved_accounts_addresses + "#, + transaction_hash, + index, + typ, + value, + &involved_account_addresses, + height, + ) + .execute(executor) + .await?; log_db_operation_time("insert_message", start); Ok(()) @@ -482,7 +447,7 @@ where sqlx::query!( "UPDATE metadata SET last_processed_height = GREATEST(last_processed_height, $1)", - height + height as i32 ) .execute(executor) .await?; diff --git a/common/nyxd-scraper-psql/src/storage/models.rs b/common/nyxd-scraper-psql/src/storage/models.rs index 878eff203b4..5894ed3146a 100644 --- a/common/nyxd-scraper-psql/src/storage/models.rs +++ b/common/nyxd-scraper-psql/src/storage/models.rs @@ -32,7 +32,16 @@ pub struct CommitSignature { #[derive(Debug, Serialize, Deserialize, sqlx::Type)] #[sqlx(type_name = "coin")] -pub struct Coin { +pub struct DbCoin { pub amount: String, pub denom: String, } + +impl From for DbCoin { + fn from(coin: cosmrs::proto::cosmos::base::v1beta1::Coin) -> Self { + Self { + amount: coin.amount, + denom: coin.denom, + } + } +} diff --git a/common/nyxd-scraper-psql/src/storage/transaction.rs b/common/nyxd-scraper-psql/src/storage/transaction.rs index a499a198859..7bc280e5669 100644 --- a/common/nyxd-scraper-psql/src/storage/transaction.rs +++ b/common/nyxd-scraper-psql/src/storage/transaction.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 use crate::error::PostgresScraperError; -use crate::models::Coin; use crate::storage::helpers::parse_addresses_from_events; use crate::storage::manager::{ insert_block, insert_message, insert_precommit, insert_transaction, insert_validator, @@ -12,8 +11,6 @@ use async_trait::async_trait; use base64::engine::general_purpose; use base64::Engine as _; use cosmrs::proto; -use cosmrs::proto::cosmwasm::wasm::v1::MsgExecuteContract; -use cosmrs::proto::prost::Message; use nyxd_scraper_shared::helpers::{ validator_consensus_address, validator_info, validator_pubkey_to_bech32, }; @@ -22,7 +19,7 @@ use nyxd_scraper_shared::storage::{ validators, Block, Commit, CommitSig, NyxdScraperStorageError, NyxdScraperTransaction, }; use nyxd_scraper_shared::ParsedTransactionResponse; -use serde_json::{json, Value}; +use serde_json::json; use sqlx::types::time::{OffsetDateTime, PrimitiveDateTime}; use sqlx::{Postgres, Transaction}; use std::ops::{Deref, DerefMut}; @@ -201,34 +198,9 @@ impl PostgresStorageTransaction { for chain_tx in txs { let involved_addresses = parse_addresses_from_events(chain_tx); for (index, msg) in chain_tx.tx.body.messages.iter().enumerate() { - let mut wasm_sender: Option = None; - let mut wasm_contract_address: Option = None; - let mut wasm_message_type: Option = None; - let mut funds: Option> = None; - let parsed_message = chain_tx.parsed_messages.get(&index); let value = serde_json::to_value(parsed_message)?; - if msg.type_url == "/cosmwasm.wasm.v1.MsgExecuteContract" { - if let Ok(wasm_execute) = MsgExecuteContract::decode(msg.value.as_ref()) { - wasm_sender = Some(wasm_execute.sender); - wasm_contract_address = Some(wasm_execute.contract); - if let Some(raw_msg) = value.get("msg") { - wasm_message_type = get_first_field_name(raw_msg); - } - funds = Some( - wasm_execute - .funds - .iter() - .map(|c| Coin { - amount: c.amount.to_string(), - denom: c.denom.clone(), - }) - .collect(), - ); - } - } - insert_message( chain_tx.hash.to_string(), index as i64, @@ -236,10 +208,6 @@ impl PostgresStorageTransaction { value, involved_addresses.clone(), chain_tx.height.into(), - wasm_sender, - wasm_contract_address, - wasm_message_type, - funds, self.inner.as_mut(), ) .await? @@ -256,14 +224,6 @@ impl PostgresStorageTransaction { } } -fn get_first_field_name(value: &Value) -> Option { - trace!("value:\n{value}"); - match value.as_object() { - Some(map) => map.keys().next().cloned(), - None => None, - } -} - #[async_trait] impl NyxdScraperTransaction for PostgresStorageTransaction { async fn commit(self) -> Result<(), NyxdScraperStorageError> { diff --git a/common/nyxd-scraper-shared/src/block_processor/types.rs b/common/nyxd-scraper-shared/src/block_processor/types.rs index cb17bb698ba..00ceeac207e 100644 --- a/common/nyxd-scraper-shared/src/block_processor/types.rs +++ b/common/nyxd-scraper-shared/src/block_processor/types.rs @@ -29,6 +29,8 @@ pub struct ParsedTransactionResponse { pub proof: Option, pub parsed_messages: HashMap, + + pub block: Block, } #[derive(Debug)] diff --git a/common/nyxd-scraper-shared/src/rpc_client.rs b/common/nyxd-scraper-shared/src/rpc_client.rs index 879b15bb6c2..81d21c28e8e 100644 --- a/common/nyxd-scraper-shared/src/rpc_client.rs +++ b/common/nyxd-scraper-shared/src/rpc_client.rs @@ -94,6 +94,7 @@ impl RpcClient { tx, proof: raw_tx.proof, parsed_messages, + block: block.block.clone(), }) } diff --git a/nym-data-observatory/Cargo.toml b/nym-data-observatory/Cargo.toml index 75d07705fe7..3445730c042 100644 --- a/nym-data-observatory/Cargo.toml +++ b/nym-data-observatory/Cargo.toml @@ -18,6 +18,7 @@ async-trait.workspace = true axum = { workspace = true, features = ["tokio"] } chrono = { workspace = true } clap = { workspace = true, features = ["cargo", "derive", "env"] } +cosmrs = { workspace = true } nym-config = { path = "../common/config" } nym-bin-common = { path = "../common/bin-common", features = ["output_format"] } nym-network-defaults = { path = "../common/network-defaults" } @@ -28,6 +29,7 @@ nyxd-scraper-shared = { path = "../common/nyxd-scraper-shared" } reqwest = { workspace = true, features = ["rustls-tls"] } schemars = { workspace = true } serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres", "time"] } thiserror = { workspace = true } time = { workspace = true } diff --git a/nym-data-observatory/migrations/0003_wasm.sql b/nym-data-observatory/migrations/0003_wasm.sql new file mode 100644 index 00000000000..8cdd5206d9d --- /dev/null +++ b/nym-data-observatory/migrations/0003_wasm.sql @@ -0,0 +1,18 @@ +CREATE TABLE wasm_execute_contract +( + sender TEXT NOT NULL, + contract_address TEXT NOT NULL, + message_type TEXT NULL, + raw_contract_message JSONB NOT NULL DEFAULT '{}'::JSONB, + funds COIN[] NOT NULL DEFAULT '{}', + fee COIN[] NOT NULL DEFAULT '{}', + executed_at TIMESTAMP NOT NULL, + height BIGINT NOT NULL, + hash TEXT NOT NULL, + message_index BIGINT NOT NULL, + memo TEXT NULL +); +CREATE INDEX execute_contract_height_index ON wasm_execute_contract (height); +CREATE INDEX execute_contract_executed_at_index ON wasm_execute_contract (executed_at); +CREATE INDEX execute_contract_message_type_index ON wasm_execute_contract (message_type); +CREATE INDEX execute_contract_sender ON wasm_execute_contract (sender); diff --git a/nym-data-observatory/migrations/0104_add_listener_failure_table.sql b/nym-data-observatory/migrations/0100_startup_info.sql similarity index 87% rename from nym-data-observatory/migrations/0104_add_listener_failure_table.sql rename to nym-data-observatory/migrations/0100_startup_info.sql index 3c80ebf473e..1e16bde072b 100644 --- a/nym-data-observatory/migrations/0104_add_listener_failure_table.sql +++ b/nym-data-observatory/migrations/0100_startup_info.sql @@ -3,7 +3,7 @@ * SPDX-License-Identifier: GPL-3.0-only */ -CREATE TABLE watcher_execution +CREATE TABLE startup_info ( start_ts TIMESTAMPTZ NOT NULL, end_ts TIMESTAMPTZ NOT NULL, diff --git a/nym-data-observatory/migrations/0102_payment_transactions.sql b/nym-data-observatory/migrations/0102_payment_transactions.sql deleted file mode 100644 index 05c0ce55490..00000000000 --- a/nym-data-observatory/migrations/0102_payment_transactions.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE TABLE payments ( - id INTEGER PRIMARY KEY, - transaction_hash TEXT NOT NULL UNIQUE, - sender_address TEXT NOT NULL, - receiver_address TEXT NOT NULL, - amount double precision NOT NULL, - timestamp bigint NOT NULL, - height bigint NOT NULL, - memo TEXT -); diff --git a/nym-data-observatory/migrations/0103_create_transactions_table.sql b/nym-data-observatory/migrations/0103_create_transactions_table.sql deleted file mode 100644 index 61f30fc0408..00000000000 --- a/nym-data-observatory/migrations/0103_create_transactions_table.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE TABLE IF NOT EXISTS transactions ( - id INTEGER PRIMARY KEY, - tx_hash TEXT NOT NULL, - height BIGINT NOT NULL, - message_index BIGINT NOT NULL, - sender TEXT NOT NULL, - recipient TEXT NOT NULL, - amount TEXT NOT NULL, - memo TEXT, - created_at DATE DEFAULT CURRENT_TIMESTAMP, - UNIQUE(tx_hash, message_index) -); \ No newline at end of file diff --git a/nym-data-observatory/src/chain_scraper/mod.rs b/nym-data-observatory/src/chain_scraper/mod.rs index 3b6b908f8d2..a1eaeeb2fcb 100644 --- a/nym-data-observatory/src/chain_scraper/mod.rs +++ b/nym-data-observatory/src/chain_scraper/mod.rs @@ -11,7 +11,7 @@ pub(crate) mod webhook; pub(crate) async fn run_chain_scraper( config: &crate::config::Config, - _connection_pool: DbPool, + connection_pool: DbPool, ) -> anyhow::Result { let websocket_url = std::env::var("NYXD_WS").expect("NYXD_WS not defined"); @@ -55,6 +55,7 @@ pub(crate) async fn run_chain_scraper( use_best_effort_start_height, }, }) + .with_msg_module(crate::modules::wasm::WasmModule::new(connection_pool)) .with_msg_module(webhook::WebhookModule::new(config.clone())?); let instance = scraper.build_and_start().await?; diff --git a/nym-data-observatory/src/chain_scraper/webhook.rs b/nym-data-observatory/src/chain_scraper/webhook.rs index 50a06596e8b..a535d11d20a 100644 --- a/nym-data-observatory/src/chain_scraper/webhook.rs +++ b/nym-data-observatory/src/chain_scraper/webhook.rs @@ -59,10 +59,9 @@ impl MsgModule for WebhookModule { for webhook in self.webhooks.clone() { let payload = payload.clone(); tokio::spawn(async move { - webhook - .invoke_webhook(&payload) - .await - .expect("webhook failed to process"); + if let Err(e) = webhook.invoke_webhook(&payload).await { + error!("webhook error: {}", e); + } }); } diff --git a/nym-data-observatory/src/cli/commands/run/mod.rs b/nym-data-observatory/src/cli/commands/run/mod.rs index eb7ee796f09..4096e2c74fe 100644 --- a/nym-data-observatory/src/cli/commands/run/mod.rs +++ b/nym-data-observatory/src/cli/commands/run/mod.rs @@ -20,7 +20,7 @@ use crate::{db, http}; pub(crate) use args::Args; use nym_task::signal::wait_for_signal; -async fn try_insert_watcher_execution_information( +async fn try_insert_startup_information( db_pool: DbPool, start: OffsetDateTime, end: OffsetDateTime, @@ -28,7 +28,7 @@ async fn try_insert_watcher_execution_information( ) { let _ = sqlx::query!( r#" - INSERT INTO watcher_execution(start_ts, end_ts, error_message) + INSERT INTO startup_info(start_ts, end_ts, error_message) VALUES ($1, $2, $3) "#, start.into(), @@ -65,13 +65,8 @@ async fn wait_for_shutdown( tasks.abort_all(); // insert execution result into the db - try_insert_watcher_execution_information( - db_pool, - start, - OffsetDateTime::now_utc(), - error_message, - ) - .await + try_insert_startup_information(db_pool, start, OffsetDateTime::now_utc(), error_message) + .await } tokio::select! { diff --git a/nym-data-observatory/src/db/mod.rs b/nym-data-observatory/src/db/mod.rs index c00fa976136..089ca6219df 100644 --- a/nym-data-observatory/src/db/mod.rs +++ b/nym-data-observatory/src/db/mod.rs @@ -1,14 +1,13 @@ use anyhow::{anyhow, Result}; -use sqlx::{migrate::Migrator, postgres::PgConnectOptions, Postgres}; +use sqlx::{postgres::PgConnectOptions, Postgres}; use std::str::FromStr; pub(crate) mod models; pub(crate) mod queries { pub mod price; + pub mod wasm; } -static _MIGRATOR: Migrator = sqlx::migrate!("./migrations"); - pub(crate) type DbPool = sqlx::Pool; pub(crate) struct Storage { @@ -23,11 +22,6 @@ impl Storage { .await .map_err(|err| anyhow!("Failed to connect to {}: {}", &connection_url, err))?; - // MIGRATOR - // .run(&pool) - // .await - // .map_err(|err| anyhow!("Failed to run migrations: {}", err))?; - Ok(Storage { pool }) } diff --git a/nym-data-observatory/src/db/queries/mod.rs b/nym-data-observatory/src/db/queries/mod.rs index e75aa1606a7..5cd63b2e1cc 100644 --- a/nym-data-observatory/src/db/queries/mod.rs +++ b/nym-data-observatory/src/db/queries/mod.rs @@ -1,5 +1,7 @@ mod price; +mod wasm; // re-exporting allows us to access all queries via `queries::bla`` pub(crate) use payments::{get_last_checked_height, insert_payment}; pub(crate) use price::{get_latest_price, insert_nym_prices}; +pub(crate) use wasm::{insert_wasm_execute}; diff --git a/nym-data-observatory/src/db/queries/wasm.rs b/nym-data-observatory/src/db/queries/wasm.rs new file mode 100644 index 00000000000..9d91f61bf83 --- /dev/null +++ b/nym-data-observatory/src/db/queries/wasm.rs @@ -0,0 +1,53 @@ +use crate::db::DbPool; +use anyhow::Result; +use nyxd_scraper_psql::models::DbCoin; +use serde_json::Value; +use time::PrimitiveDateTime; + +pub async fn insert_wasm_execute( + pool: &DbPool, + sender: String, + contract_address: String, + message_type: String, + raw_contract_message: &Value, + funds: Option>, + executed_at: PrimitiveDateTime, + height: i64, + hash: String, + message_index: i64, + memo: String, + fee: &Vec, +) -> Result<()> { + sqlx::query!( + r#" + INSERT INTO wasm_execute_contract ( + sender, + contract_address, + message_type, + raw_contract_message, + funds, + executed_at, + height, + hash, + message_index, + memo, + fee + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + "#, + sender, + contract_address, + message_type, + raw_contract_message, + &funds as _, + executed_at, + height, + hash, + message_index, + memo, + &fee as _, + ) + .execute(pool) + .await?; + + Ok(()) +} diff --git a/nym-data-observatory/src/main.rs b/nym-data-observatory/src/main.rs index 3a62b35d379..56fc6d5afe7 100644 --- a/nym-data-observatory/src/main.rs +++ b/nym-data-observatory/src/main.rs @@ -13,6 +13,7 @@ mod error; mod http; mod logging; pub mod models; +mod modules; mod price_scraper; #[tokio::main] diff --git a/nym-data-observatory/src/modules/mod.rs b/nym-data-observatory/src/modules/mod.rs new file mode 100644 index 00000000000..30d27c6b5bf --- /dev/null +++ b/nym-data-observatory/src/modules/mod.rs @@ -0,0 +1,4 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +pub(crate) mod wasm; diff --git a/nym-data-observatory/src/modules/wasm.rs b/nym-data-observatory/src/modules/wasm.rs new file mode 100644 index 00000000000..508e62c5ef5 --- /dev/null +++ b/nym-data-observatory/src/modules/wasm.rs @@ -0,0 +1,117 @@ +// Copyright 2025 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use crate::db::queries::wasm::insert_wasm_execute; +use crate::db::DbPool; +use async_trait::async_trait; +use cosmrs::proto::cosmwasm::wasm::v1::MsgExecuteContract; +use cosmrs::proto::prost::Message; +use nym_validator_client::nyxd::{Any, Name}; +use nyxd_scraper_psql::models::DbCoin; +use nyxd_scraper_psql::{ + MsgModule, NyxdScraperTransaction, ParsedTransactionResponse, ScraperError, +}; +use serde_json::Value; +use time::{OffsetDateTime, PrimitiveDateTime}; +use tracing::{error, trace}; +use utoipa::gen::serde_json; + +pub struct WasmModule { + connection_pool: DbPool, +} + +impl WasmModule { + pub fn new(connection_pool: DbPool) -> Self { + WasmModule { connection_pool } + } +} + +#[async_trait] +impl MsgModule for WasmModule { + fn type_url(&self) -> String { + MsgExecuteContract::type_url() + } + + async fn handle_msg( + &mut self, + index: usize, + msg: &Any, + tx: &ParsedTransactionResponse, + _storage_tx: &mut dyn NyxdScraperTransaction, + ) -> Result<(), ScraperError> { + let message = serde_json::to_value(tx.parsed_messages.get(&index)).unwrap_or_default(); + let value = serde_json::to_value(message.clone()).unwrap_or_default(); + let wasm_message_type = get_wasm_message_type(&value); + let fee: Vec = tx + .tx + .auth_info + .fee + .amount + .clone() + .into_iter() + .map(|x| DbCoin { + amount: x.amount.to_string(), + denom: x.denom.to_string(), + }) + .collect(); + + let offset_datetime: OffsetDateTime = tx.block.header.time.into(); + let executed_at = PrimitiveDateTime::new(offset_datetime.date(), offset_datetime.time()); + + let height = tx.height.value() as i64; + let hash = tx.hash.to_string(); + let memo = tx.tx.body.memo.clone(); + + match MsgExecuteContract::decode(msg.value.as_ref()) { + Ok(wasm_execute_msg) => { + let funds: Vec = wasm_execute_msg + .funds + .clone() + .into_iter() + .map(|x| x.into()) + .collect(); + let contract = wasm_execute_msg.contract; + let sender = wasm_execute_msg.sender; + + if let Err(err) = insert_wasm_execute( + &self.connection_pool, + sender, + contract, + wasm_message_type, + &message, + Some(funds), + executed_at, + height, + hash, + index as i64, + memo, + &fee, + ) + .await + { + error!("Error persisting wasm contract execution message: {}", err); + } + } + Err(err) => { + error!("Error decoding message: {}", err); + } + } + + Ok(()) + } +} + +fn get_first_field_name(value: Option<&Value>) -> Option { + trace!("value:\n{value:?}"); + match value { + Some(value) => match value.as_object() { + Some(map) => map.keys().next().cloned(), + None => None, + }, + None => None, + } +} + +fn get_wasm_message_type(value: &Value) -> String { + get_first_field_name(value.get("msg")).unwrap_or_default() +} From 5526c5bffd8d8c6e7fd16a29671e814ce01af200 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Fri, 31 Oct 2025 15:30:50 +0000 Subject: [PATCH 06/28] formatting and clippy --- common/nyxd-scraper-psql/src/storage/block_storage.rs | 4 ++-- common/nyxd-scraper-psql/src/storage/manager.rs | 2 +- common/nyxd-scraper-psql/src/storage/models.rs | 2 +- common/nyxd-scraper-psql/src/storage/transaction.rs | 6 +++--- common/nyxd-scraper-shared/src/block_processor/mod.rs | 2 +- .../src/cosmos_module/message_registry.rs | 4 ++-- .../nyxd-scraper-shared/src/cosmos_module/modules/auth.rs | 2 +- .../nyxd-scraper-shared/src/cosmos_module/modules/authz.rs | 2 +- .../nyxd-scraper-shared/src/cosmos_module/modules/bank.rs | 2 +- .../src/cosmos_module/modules/capability.rs | 2 +- .../src/cosmos_module/modules/consensus.rs | 2 +- .../src/cosmos_module/modules/crisis.rs | 2 +- .../src/cosmos_module/modules/distribution.rs | 2 +- .../src/cosmos_module/modules/evidence.rs | 2 +- .../src/cosmos_module/modules/feegrant.rs | 2 +- .../src/cosmos_module/modules/gov_v1.rs | 2 +- .../src/cosmos_module/modules/gov_v1beta1.rs | 2 +- .../nyxd-scraper-shared/src/cosmos_module/modules/group.rs | 2 +- .../nyxd-scraper-shared/src/cosmos_module/modules/mint.rs | 2 +- .../nyxd-scraper-shared/src/cosmos_module/modules/nft.rs | 2 +- .../src/cosmos_module/modules/params.rs | 2 +- .../src/cosmos_module/modules/slashing.rs | 2 +- .../src/cosmos_module/modules/staking.rs | 2 +- .../src/cosmos_module/modules/upgrade.rs | 2 +- .../src/cosmos_module/modules/vesting.rs | 2 +- .../nyxd-scraper-shared/src/cosmos_module/modules/wasm.rs | 4 ++-- common/nyxd-scraper-shared/src/helpers.rs | 4 ++-- common/nyxd-scraper-shared/src/lib.rs | 2 +- common/nyxd-scraper-shared/src/rpc_client.rs | 2 +- common/nyxd-scraper-shared/src/storage/mod.rs | 4 ++-- common/nyxd-scraper-sqlite/Cargo.toml | 1 + common/nyxd-scraper-sqlite/src/storage/block_storage.rs | 6 +++--- common/nyxd-scraper-sqlite/src/storage/models.rs | 2 +- common/nyxd-scraper-sqlite/src/storage/transaction.rs | 4 ++-- nym-data-observatory/src/chain_scraper/webhook.rs | 2 +- nym-data-observatory/src/cli/commands/init.rs | 2 +- nym-data-observatory/src/cli/commands/run/config.rs | 4 ++-- nym-data-observatory/src/config/mod.rs | 7 ++++--- nym-data-observatory/src/db/mod.rs | 4 ++-- nym-data-observatory/src/db/queries/price.rs | 2 +- nym-data-observatory/src/error.rs | 4 +++- nym-data-observatory/src/http/api/mod.rs | 2 +- nym-data-observatory/src/http/api/price.rs | 2 +- nym-data-observatory/src/http/state.rs | 2 +- nym-data-observatory/src/logging.rs | 2 +- nym-data-observatory/src/main.rs | 2 +- nym-data-observatory/src/models.rs | 2 +- nym-data-observatory/src/modules/wasm.rs | 4 ++-- nyx-chain-watcher/src/chain_scraper/mod.rs | 2 +- 49 files changed, 67 insertions(+), 63 deletions(-) diff --git a/common/nyxd-scraper-psql/src/storage/block_storage.rs b/common/nyxd-scraper-psql/src/storage/block_storage.rs index e87e72b1eff..49f2bab6220 100644 --- a/common/nyxd-scraper-psql/src/storage/block_storage.rs +++ b/common/nyxd-scraper-psql/src/storage/block_storage.rs @@ -4,8 +4,8 @@ use crate::error::PostgresScraperError; use crate::models::{CommitSignature, Validator}; use crate::storage::manager::{ - prune_blocks, prune_messages, prune_pre_commits, prune_transactions, update_last_pruned, - StorageManager, + StorageManager, prune_blocks, prune_messages, prune_pre_commits, prune_transactions, + update_last_pruned, }; use crate::storage::transaction::PostgresStorageTransaction; use async_trait::async_trait; diff --git a/common/nyxd-scraper-psql/src/storage/manager.rs b/common/nyxd-scraper-psql/src/storage/manager.rs index 73e3b499fce..cbe5003f828 100644 --- a/common/nyxd-scraper-psql/src/storage/manager.rs +++ b/common/nyxd-scraper-psql/src/storage/manager.rs @@ -3,8 +3,8 @@ use crate::storage::models::{CommitSignature, Validator}; use nyxd_scraper_shared::storage::helpers::log_db_operation_time; -use sqlx::types::time::PrimitiveDateTime; use sqlx::types::JsonValue; +use sqlx::types::time::PrimitiveDateTime; use sqlx::{Executor, Postgres}; use tokio::time::Instant; use tracing::{instrument, trace}; diff --git a/common/nyxd-scraper-psql/src/storage/models.rs b/common/nyxd-scraper-psql/src/storage/models.rs index 5894ed3146a..59b4f529513 100644 --- a/common/nyxd-scraper-psql/src/storage/models.rs +++ b/common/nyxd-scraper-psql/src/storage/models.rs @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 use serde::{Deserialize, Serialize}; -use sqlx::types::time::OffsetDateTime; use sqlx::FromRow; +use sqlx::types::time::OffsetDateTime; #[derive(Debug, Clone, Eq, PartialEq, Hash, FromRow)] pub struct Validator { diff --git a/common/nyxd-scraper-psql/src/storage/transaction.rs b/common/nyxd-scraper-psql/src/storage/transaction.rs index 7bc280e5669..13aec6be49e 100644 --- a/common/nyxd-scraper-psql/src/storage/transaction.rs +++ b/common/nyxd-scraper-psql/src/storage/transaction.rs @@ -8,17 +8,17 @@ use crate::storage::manager::{ update_last_processed, }; use async_trait::async_trait; -use base64::engine::general_purpose; use base64::Engine as _; +use base64::engine::general_purpose; use cosmrs::proto; +use nyxd_scraper_shared::ParsedTransactionResponse; use nyxd_scraper_shared::helpers::{ validator_consensus_address, validator_info, validator_pubkey_to_bech32, }; use nyxd_scraper_shared::storage::validators::Response; use nyxd_scraper_shared::storage::{ - validators, Block, Commit, CommitSig, NyxdScraperStorageError, NyxdScraperTransaction, + Block, Commit, CommitSig, NyxdScraperStorageError, NyxdScraperTransaction, validators, }; -use nyxd_scraper_shared::ParsedTransactionResponse; use serde_json::json; use sqlx::types::time::{OffsetDateTime, PrimitiveDateTime}; use sqlx::{Postgres, Transaction}; diff --git a/common/nyxd-scraper-shared/src/block_processor/mod.rs b/common/nyxd-scraper-shared/src/block_processor/mod.rs index 17979a7ed1f..0d30df40b78 100644 --- a/common/nyxd-scraper-shared/src/block_processor/mod.rs +++ b/common/nyxd-scraper-shared/src/block_processor/mod.rs @@ -8,7 +8,7 @@ use crate::block_requester::BlockRequest; use crate::error::ScraperError; use crate::modules::{BlockModule, MsgModule, TxModule}; use crate::rpc_client::RpcClient; -use crate::storage::{persist_block, NyxdScraperStorage, NyxdScraperTransaction}; +use crate::storage::{NyxdScraperStorage, NyxdScraperTransaction, persist_block}; use futures::StreamExt; use std::cmp::max; use std::collections::{BTreeMap, HashSet, VecDeque}; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/message_registry.rs b/common/nyxd-scraper-shared/src/cosmos_module/message_registry.rs index bacc9b499ba..45bfe786dd5 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/message_registry.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/message_registry.rs @@ -1,6 +1,7 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 +use crate::cosmos_module::CosmosModule; use crate::cosmos_module::modules::auth::Auth; use crate::cosmos_module::modules::authz::Authz; use crate::cosmos_module::modules::bank::Bank; @@ -26,11 +27,10 @@ use crate::cosmos_module::modules::staking::Staking; use crate::cosmos_module::modules::upgrade::Upgrade; use crate::cosmos_module::modules::vesting::Vesting; use crate::cosmos_module::modules::wasm::Wasm; -use crate::cosmos_module::CosmosModule; use crate::error::ScraperError; +use cosmrs::Any; use cosmrs::proto::prost::Name; use cosmrs::proto::traits::Message; -use cosmrs::Any; use serde::Serialize; use std::collections::HashMap; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/auth.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/auth.rs index f43c6b507bf..79a7da4b901 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/auth.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/auth.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::auth::v1beta1::MsgUpdateParams; pub(crate) struct Auth; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/authz.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/authz.rs index 62c3675c652..5088f7f7477 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/authz.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/authz.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::authz::v1beta1::{MsgExec, MsgGrant, MsgRevoke}; pub(crate) struct Authz; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/bank.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/bank.rs index 5f2db1cb782..b84b0bec5d7 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/bank.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/bank.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::bank::v1beta1::{ MsgMultiSend, MsgSend, MsgSetSendEnabled, MsgUpdateParams, }; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/capability.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/capability.rs index 2d0837f4332..2a8d785d758 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/capability.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/capability.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; pub(crate) struct Capability; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/consensus.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/consensus.rs index 8e40ba95b75..6d8321ddeac 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/consensus.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/consensus.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; pub(crate) struct Consensus; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/crisis.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/crisis.rs index 26653044171..0f3ace22dd2 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/crisis.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/crisis.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::crisis::v1beta1::{MsgUpdateParams, MsgVerifyInvariant}; pub(crate) struct Crisis; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/distribution.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/distribution.rs index 7e3f5e273cd..810ac75e5c6 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/distribution.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/distribution.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::distribution::v1beta1::{ MsgCommunityPoolSpend, MsgFundCommunityPool, MsgSetWithdrawAddress, MsgUpdateParams, MsgWithdrawDelegatorReward, MsgWithdrawValidatorCommission, diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/evidence.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/evidence.rs index 03532eba91f..0a370776d88 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/evidence.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/evidence.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::evidence::v1beta1::MsgSubmitEvidence; pub(crate) struct Evidence; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/feegrant.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/feegrant.rs index 6948e63198c..844ff8ec892 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/feegrant.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/feegrant.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::feegrant::v1beta1::{ MsgGrantAllowance, MsgPruneAllowances, MsgRevokeAllowance, }; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1.rs index be69613a12f..0b852db22b9 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::gov::v1::{ MsgDeposit, MsgExecLegacyContent, MsgSubmitProposal, MsgUpdateParams, MsgVote, MsgVoteWeighted, }; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1beta1.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1beta1.rs index cf64d12e0ec..25ef3e56795 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1beta1.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/gov_v1beta1.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::gov::v1beta1::{ MsgDeposit, MsgSubmitProposal, MsgVote, MsgVoteWeighted, }; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/group.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/group.rs index bddbf750910..8ea7a7cc1df 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/group.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/group.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use tracing::warn; pub(crate) struct Group; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/mint.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/mint.rs index 9cd59c50384..be6625b237f 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/mint.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/mint.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::mint::v1beta1::MsgUpdateParams; pub(crate) struct Mint; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/nft.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/nft.rs index cebfd52fba8..e4e96f57d56 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/nft.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/nft.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; pub(crate) struct Nft; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/params.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/params.rs index 909cc12ede6..70fe04783ad 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/params.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/params.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; pub(crate) struct Params; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/slashing.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/slashing.rs index 5ae5328c372..481295516f5 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/slashing.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/slashing.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; pub(crate) struct Slashing; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/staking.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/staking.rs index 44c836eb298..dc87d6cfea9 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/staking.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/staking.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::staking::v1beta1::{ MsgBeginRedelegate, MsgCancelUnbondingDelegation, MsgCreateValidator, MsgDelegate, MsgEditValidator, MsgUndelegate, MsgUpdateParams, diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/upgrade.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/upgrade.rs index 5a100f816c1..cf8174fe5ff 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/upgrade.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/upgrade.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::upgrade::v1beta1::{MsgCancelUpgrade, MsgSoftwareUpgrade}; pub(crate) struct Upgrade; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/vesting.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/vesting.rs index f690cab22bc..e518829b7f8 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/vesting.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/vesting.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::MessageRegistry; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::MessageRegistry; use cosmos_sdk_proto::cosmos::vesting::v1beta1::{ MsgCreatePeriodicVestingAccount, MsgCreatePermanentLockedAccount, MsgCreateVestingAccount, }; diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/wasm.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/wasm.rs index 38d26b8bacb..c9b7ad5bb0e 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/wasm.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/wasm.rs @@ -1,11 +1,11 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use crate::cosmos_module::message_registry::{default_proto_to_json, MessageRegistry}; use crate::cosmos_module::CosmosModule; +use crate::cosmos_module::message_registry::{MessageRegistry, default_proto_to_json}; use crate::error::ScraperError; -use base64::engine::general_purpose::STANDARD; use base64::Engine; +use base64::engine::general_purpose::STANDARD; use cosmos_sdk_proto::cosmwasm::wasm::v1::{ MsgAddCodeUploadParamsAddresses, MsgClearAdmin, MsgExecuteContract, MsgIbcCloseChannel, MsgIbcSend, MsgInstantiateContract, MsgInstantiateContract2, MsgMigrateContract, MsgPinCodes, diff --git a/common/nyxd-scraper-shared/src/helpers.rs b/common/nyxd-scraper-shared/src/helpers.rs index 2a20a2751be..54c5a1cbc00 100644 --- a/common/nyxd-scraper-shared/src/helpers.rs +++ b/common/nyxd-scraper-shared/src/helpers.rs @@ -5,8 +5,8 @@ use crate::block_processor::types::ParsedTransactionResponse; use crate::constants::{BECH32_CONESNSUS_PUBKEY_PREFIX, BECH32_CONSENSUS_ADDRESS_PREFIX}; use cosmrs::AccountId; use sha2::{Digest, Sha256}; -use tendermint::{account, PublicKey}; -use tendermint::{validator, Hash}; +use tendermint::{Hash, validator}; +use tendermint::{PublicKey, account}; use tendermint_rpc::endpoint::validators; use thiserror::Error; diff --git a/common/nyxd-scraper-shared/src/lib.rs b/common/nyxd-scraper-shared/src/lib.rs index 7d3f198b7c7..7d6b18509c3 100644 --- a/common/nyxd-scraper-shared/src/lib.rs +++ b/common/nyxd-scraper-shared/src/lib.rs @@ -15,8 +15,8 @@ pub mod storage; pub use block_processor::pruning::{PruningOptions, PruningStrategy}; pub use block_processor::types::ParsedTransactionResponse; pub use cosmos_module::{ - message_registry::{default_message_registry, MessageRegistry}, CosmosModule, + message_registry::{MessageRegistry, default_message_registry}, }; pub use cosmrs::Any; pub use modules::{BlockModule, MsgModule, TxModule}; diff --git a/common/nyxd-scraper-shared/src/rpc_client.rs b/common/nyxd-scraper-shared/src/rpc_client.rs index 81d21c28e8e..18af321ece4 100644 --- a/common/nyxd-scraper-shared/src/rpc_client.rs +++ b/common/nyxd-scraper-shared/src/rpc_client.rs @@ -6,7 +6,7 @@ use crate::block_processor::types::{ }; use crate::error::ScraperError; use crate::helpers::tx_hash; -use crate::{default_message_registry, Any, MessageRegistry}; +use crate::{Any, MessageRegistry, default_message_registry}; use futures::StreamExt; use futures::future::join3; use std::collections::{BTreeMap, HashMap}; diff --git a/common/nyxd-scraper-shared/src/storage/mod.rs b/common/nyxd-scraper-shared/src/storage/mod.rs index 97fa3ce26c1..a847f2a6bf1 100644 --- a/common/nyxd-scraper-shared/src/storage/mod.rs +++ b/common/nyxd-scraper-shared/src/storage/mod.rs @@ -6,10 +6,10 @@ use async_trait::async_trait; use thiserror::Error; use tracing::warn; -pub use crate::block_processor::types::FullBlockInformation; pub use crate::ParsedTransactionResponse; -pub use tendermint::block::{Commit, CommitSig}; +pub use crate::block_processor::types::FullBlockInformation; pub use tendermint::Block; +pub use tendermint::block::{Commit, CommitSig}; pub use tendermint_rpc::endpoint::validators; pub mod helpers; diff --git a/common/nyxd-scraper-sqlite/Cargo.toml b/common/nyxd-scraper-sqlite/Cargo.toml index 53e492b5e0c..eb10336f0bb 100644 --- a/common/nyxd-scraper-sqlite/Cargo.toml +++ b/common/nyxd-scraper-sqlite/Cargo.toml @@ -23,6 +23,7 @@ nyxd-scraper-shared = { path = "../nyxd-scraper-shared" } [build-dependencies] sqlx = { workspace = true, features = ["runtime-tokio-rustls", "sqlite", "macros", "migrate"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +anyhow.workspace = true [lints] workspace = true \ No newline at end of file diff --git a/common/nyxd-scraper-sqlite/src/storage/block_storage.rs b/common/nyxd-scraper-sqlite/src/storage/block_storage.rs index 0df04ddbabb..9a0a33d52ab 100644 --- a/common/nyxd-scraper-sqlite/src/storage/block_storage.rs +++ b/common/nyxd-scraper-sqlite/src/storage/block_storage.rs @@ -4,16 +4,16 @@ use crate::error::SqliteScraperError; use crate::models::{CommitSignature, Validator}; use crate::storage::manager::{ - prune_blocks, prune_messages, prune_pre_commits, prune_transactions, update_last_pruned, - StorageManager, + StorageManager, prune_blocks, prune_messages, prune_pre_commits, prune_transactions, + update_last_pruned, }; use crate::storage::transaction::SqliteStorageTransaction; use async_trait::async_trait; use nyxd_scraper_shared::storage::helpers::log_db_operation_time; use nyxd_scraper_shared::storage::{NyxdScraperStorage, NyxdScraperStorageError}; +use sqlx::ConnectOptions; use sqlx::sqlite::{SqliteAutoVacuum, SqliteSynchronous}; use sqlx::types::time::OffsetDateTime; -use sqlx::ConnectOptions; use std::fmt::Debug; use std::path::Path; use tokio::time::Instant; diff --git a/common/nyxd-scraper-sqlite/src/storage/models.rs b/common/nyxd-scraper-sqlite/src/storage/models.rs index 74a3dd9b1cf..6c4c16c633f 100644 --- a/common/nyxd-scraper-sqlite/src/storage/models.rs +++ b/common/nyxd-scraper-sqlite/src/storage/models.rs @@ -1,8 +1,8 @@ // Copyright 2023 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use sqlx::types::time::OffsetDateTime; use sqlx::FromRow; +use sqlx::types::time::OffsetDateTime; #[derive(Debug, Clone, Eq, PartialEq, Hash, FromRow)] pub struct Validator { diff --git a/common/nyxd-scraper-sqlite/src/storage/transaction.rs b/common/nyxd-scraper-sqlite/src/storage/transaction.rs index 76f5ecd67e9..0cd57b78c88 100644 --- a/common/nyxd-scraper-sqlite/src/storage/transaction.rs +++ b/common/nyxd-scraper-sqlite/src/storage/transaction.rs @@ -7,14 +7,14 @@ use crate::storage::manager::{ update_last_processed, }; use async_trait::async_trait; +use nyxd_scraper_shared::ParsedTransactionResponse; use nyxd_scraper_shared::helpers::{ validator_consensus_address, validator_info, validator_pubkey_to_bech32, }; use nyxd_scraper_shared::storage::validators::Response; use nyxd_scraper_shared::storage::{ - validators, Block, Commit, CommitSig, NyxdScraperStorageError, NyxdScraperTransaction, + Block, Commit, CommitSig, NyxdScraperStorageError, NyxdScraperTransaction, validators, }; -use nyxd_scraper_shared::ParsedTransactionResponse; use sqlx::{Sqlite, Transaction}; use std::ops::{Deref, DerefMut}; use tracing::{debug, trace, warn}; diff --git a/nym-data-observatory/src/chain_scraper/webhook.rs b/nym-data-observatory/src/chain_scraper/webhook.rs index a535d11d20a..34604f9f25e 100644 --- a/nym-data-observatory/src/chain_scraper/webhook.rs +++ b/nym-data-observatory/src/chain_scraper/webhook.rs @@ -11,7 +11,7 @@ use nyxd_scraper_psql::{ }; use reqwest::{Client, Url}; use tracing::{error, info}; -use utoipa::gen::serde_json; +use utoipa::r#gen::serde_json; pub struct WebhookModule { webhooks: Vec, diff --git a/nym-data-observatory/src/cli/commands/init.rs b/nym-data-observatory/src/cli/commands/init.rs index 1c1fbde3e20..f57f3fc43f6 100644 --- a/nym-data-observatory/src/cli/commands/init.rs +++ b/nym-data-observatory/src/cli/commands/init.rs @@ -4,7 +4,7 @@ use crate::cli::DEFAULT_NYM_DATA_OBSERVATORY_ID; use crate::config::data_observatory::HttpAuthenticationOptions::AuthorizationBearerToken; use crate::config::data_observatory::WebhookConfig; -use crate::config::{default_config_filepath, Config, ConfigBuilder, DataObservatoryConfig}; +use crate::config::{Config, ConfigBuilder, DataObservatoryConfig, default_config_filepath}; use crate::env::vars::*; use crate::error::NymDataObservatoryError; use nym_config::save_unformatted_config_to_file; diff --git a/nym-data-observatory/src/cli/commands/run/config.rs b/nym-data-observatory/src/cli/commands/run/config.rs index 0519b149ed6..52ec21fca14 100644 --- a/nym-data-observatory/src/cli/commands/run/config.rs +++ b/nym-data-observatory/src/cli/commands/run/config.rs @@ -1,7 +1,7 @@ -use crate::cli::commands::run::args::Args; use crate::cli::DEFAULT_NYM_DATA_OBSERVATORY_ID; +use crate::cli::commands::run::args::Args; use crate::config::data_observatory::{HttpAuthenticationOptions, WebhookConfig}; -use crate::config::{default_config_filepath, Config, ConfigBuilder, DataObservatoryConfig}; +use crate::config::{Config, ConfigBuilder, DataObservatoryConfig, default_config_filepath}; use crate::error::NymDataObservatoryError; use tracing::{info, warn}; diff --git a/nym-data-observatory/src/config/mod.rs b/nym-data-observatory/src/config/mod.rs index 3e5b46dcbc7..da75f453ea5 100644 --- a/nym-data-observatory/src/config/mod.rs +++ b/nym-data-observatory/src/config/mod.rs @@ -4,8 +4,8 @@ use crate::config::template::CONFIG_TEMPLATE; use nym_bin_common::logging::LoggingSettings; use nym_config::{ - must_get_home, read_config_from_toml_file, save_unformatted_config_to_file, NymConfigTemplate, - DEFAULT_CONFIG_DIR, DEFAULT_CONFIG_FILENAME, DEFAULT_DATA_DIR, NYM_DIR, + DEFAULT_CONFIG_DIR, DEFAULT_CONFIG_FILENAME, DEFAULT_DATA_DIR, NYM_DIR, NymConfigTemplate, + must_get_home, read_config_from_toml_file, save_unformatted_config_to_file, }; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; @@ -164,7 +164,8 @@ impl Config { if config_dir_name != DEFAULT_CONFIG_DIR { error!( "the parent directory of '{}' ({}) is not {DEFAULT_CONFIG_DIR}. currently this is not supported", - config_path.display(), config_dir_name.to_str().unwrap_or("UNKNOWN") + config_path.display(), + config_dir_name.to_str().unwrap_or("UNKNOWN") ); return Err(NymDataObservatoryError::DataDirDerivationFailure); } diff --git a/nym-data-observatory/src/db/mod.rs b/nym-data-observatory/src/db/mod.rs index 089ca6219df..70c36a7b618 100644 --- a/nym-data-observatory/src/db/mod.rs +++ b/nym-data-observatory/src/db/mod.rs @@ -1,5 +1,5 @@ -use anyhow::{anyhow, Result}; -use sqlx::{postgres::PgConnectOptions, Postgres}; +use anyhow::{Result, anyhow}; +use sqlx::{Postgres, postgres::PgConnectOptions}; use std::str::FromStr; pub(crate) mod models; diff --git a/nym-data-observatory/src/db/queries/price.rs b/nym-data-observatory/src/db/queries/price.rs index 4a6ddbeea39..493a1031d73 100644 --- a/nym-data-observatory/src/db/queries/price.rs +++ b/nym-data-observatory/src/db/queries/price.rs @@ -1,5 +1,5 @@ -use crate::db::models::{PriceHistory, PriceRecord}; use crate::db::DbPool; +use crate::db::models::{PriceHistory, PriceRecord}; use chrono::Local; use std::ops::Sub; diff --git a/nym-data-observatory/src/error.rs b/nym-data-observatory/src/error.rs index 86beec2f5f1..1dd5b302c91 100644 --- a/nym-data-observatory/src/error.rs +++ b/nym-data-observatory/src/error.rs @@ -20,7 +20,9 @@ pub enum NymDataObservatoryError { #[error("could not derive path to data directory of this nyx chain watcher")] DataDirDerivationFailure, - #[error("please provide a database connection string as an env var, cli argument or in a config file")] + #[error( + "please provide a database connection string as an env var, cli argument or in a config file" + )] DbConnectionStringMissing, // #[error("could not derive path to config directory of this nyx chain watcher")] diff --git a/nym-data-observatory/src/http/api/mod.rs b/nym-data-observatory/src/http/api/mod.rs index 4d2b84ca294..ee0fbe85634 100644 --- a/nym-data-observatory/src/http/api/mod.rs +++ b/nym-data-observatory/src/http/api/mod.rs @@ -1,5 +1,5 @@ use anyhow::anyhow; -use axum::{response::Redirect, Router}; +use axum::{Router, response::Redirect}; use tokio::net::ToSocketAddrs; use tower_http::{cors::CorsLayer, trace::TraceLayer}; use utoipa::OpenApi; diff --git a/nym-data-observatory/src/http/api/price.rs b/nym-data-observatory/src/http/api/price.rs index 1971f35f7d6..3a7829d5983 100644 --- a/nym-data-observatory/src/http/api/price.rs +++ b/nym-data-observatory/src/http/api/price.rs @@ -3,7 +3,7 @@ use crate::db::queries::price::{get_average_price, get_latest_price}; use crate::http::error::Error; use crate::http::error::HttpResult; use crate::http::state::AppState; -use axum::{extract::State, Json, Router}; +use axum::{Json, Router, extract::State}; pub(crate) fn routes() -> Router { Router::new() diff --git a/nym-data-observatory/src/http/state.rs b/nym-data-observatory/src/http/state.rs index da243731884..d3ed8acf09d 100644 --- a/nym-data-observatory/src/http/state.rs +++ b/nym-data-observatory/src/http/state.rs @@ -1,5 +1,5 @@ -use crate::db::models::CoingeckoPriceResponse; use crate::db::DbPool; +use crate::db::models::CoingeckoPriceResponse; use crate::http::models::status::Webhook; use axum::extract::FromRef; use nym_bin_common::bin_info; diff --git a/nym-data-observatory/src/logging.rs b/nym-data-observatory/src/logging.rs index 1c8cbeebb16..dfcdefaadfc 100644 --- a/nym-data-observatory/src/logging.rs +++ b/nym-data-observatory/src/logging.rs @@ -1,5 +1,5 @@ use tracing::level_filters::LevelFilter; -use tracing_subscriber::{filter::Directive, EnvFilter}; +use tracing_subscriber::{EnvFilter, filter::Directive}; pub(crate) fn setup_tracing_logger() { fn directive_checked(directive: String) -> Directive { diff --git a/nym-data-observatory/src/main.rs b/nym-data-observatory/src/main.rs index 56fc6d5afe7..0be492db19d 100644 --- a/nym-data-observatory/src/main.rs +++ b/nym-data-observatory/src/main.rs @@ -1,4 +1,4 @@ -use clap::{crate_name, crate_version, Parser}; +use clap::{Parser, crate_name, crate_version}; use nym_bin_common::bin_info_owned; use nym_bin_common::logging::maybe_print_banner; use nym_network_defaults::setup_env; diff --git a/nym-data-observatory/src/models.rs b/nym-data-observatory/src/models.rs index 44d983fac4d..692cfb4802b 100644 --- a/nym-data-observatory/src/models.rs +++ b/nym-data-observatory/src/models.rs @@ -1,7 +1,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use utoipa::gen::serde_json; use utoipa::ToSchema; +use utoipa::r#gen::serde_json; #[derive(Serialize, Deserialize, Clone, JsonSchema, ToSchema)] pub struct WebhookPayload { diff --git a/nym-data-observatory/src/modules/wasm.rs b/nym-data-observatory/src/modules/wasm.rs index 508e62c5ef5..98fe1dc8099 100644 --- a/nym-data-observatory/src/modules/wasm.rs +++ b/nym-data-observatory/src/modules/wasm.rs @@ -1,8 +1,8 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: GPL-3.0-only -use crate::db::queries::wasm::insert_wasm_execute; use crate::db::DbPool; +use crate::db::queries::wasm::insert_wasm_execute; use async_trait::async_trait; use cosmrs::proto::cosmwasm::wasm::v1::MsgExecuteContract; use cosmrs::proto::prost::Message; @@ -14,7 +14,7 @@ use nyxd_scraper_psql::{ use serde_json::Value; use time::{OffsetDateTime, PrimitiveDateTime}; use tracing::{error, trace}; -use utoipa::gen::serde_json; +use utoipa::r#gen::serde_json; pub struct WasmModule { connection_pool: DbPool, diff --git a/nyx-chain-watcher/src/chain_scraper/mod.rs b/nyx-chain-watcher/src/chain_scraper/mod.rs index f8738f195c4..c0db1b8e04d 100644 --- a/nyx-chain-watcher/src/chain_scraper/mod.rs +++ b/nyx-chain-watcher/src/chain_scraper/mod.rs @@ -53,7 +53,7 @@ pub(crate) async fn run_chain_scraper( let scraper = SqliteNyxdScraper::builder(nyxd_scraper_sqlite::Config { websocket_url, rpc_url, - database_storage: config.chain_scraper_database_path().into(), + database_storage: config.chain_scraper_database_path(), pruning_options: PruningOptions::nothing(), store_precommits: false, start_block: nyxd_scraper_sqlite::StartingBlockOpts { From c3a6ce81508a89c36fe0c3203872f7e67d91bd70 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 4 Nov 2025 14:30:44 +0000 Subject: [PATCH 07/28] copy shared migrations and add comments to ignore file to explain --- nym-data-observatory/.gitignore | 2 ++ nym-data-observatory/Cargo.toml | 3 ++- nym-data-observatory/build.rs | 28 ++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/nym-data-observatory/.gitignore b/nym-data-observatory/.gitignore index 36150a96516..c35f82d5e3b 100644 --- a/nym-data-observatory/.gitignore +++ b/nym-data-observatory/.gitignore @@ -1,2 +1,4 @@ +# ignore files that are copied in from `../common/nyxd-scraper-psql/sql_migrations/* migrations` +# and remember to name the migrations to avoid collisions and so they process in the correct string sort order 0001_metadata.sql 0002_cosmos.sql \ No newline at end of file diff --git a/nym-data-observatory/Cargo.toml b/nym-data-observatory/Cargo.toml index 3445730c042..33178009f43 100644 --- a/nym-data-observatory/Cargo.toml +++ b/nym-data-observatory/Cargo.toml @@ -45,5 +45,6 @@ utoipauto = { workspace = true } [build-dependencies] anyhow = { workspace = true } +glob = "0.3.3" tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } -sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] } +sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] } \ No newline at end of file diff --git a/nym-data-observatory/build.rs b/nym-data-observatory/build.rs index c1dcd9dcb30..e062d85bfde 100644 --- a/nym-data-observatory/build.rs +++ b/nym-data-observatory/build.rs @@ -1,7 +1,35 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 +use glob::glob; +use std::env; +use std::path::Path; fn main() { + // copy common manifest files from "../common/nyxd-scraper-psql/sql_migrations/* migrations" + println!("Copying common migrations..."); + let manifest_dir_string = env::var("CARGO_MANIFEST_DIR").unwrap(); + let common_migrations_path = + Path::new(&manifest_dir_string).join("../common/nyxd-scraper-psql/sql_migrations/"); + let output_path = Path::new(&manifest_dir_string).join("migrations"); + println!( + "output_path: {:?} (exists = {})", + output_path, + output_path.exists() + ); + let common_migrations_path = common_migrations_path.as_path(); + println!( + "common_migrations_path: {:?} (exists = {})", + common_migrations_path, + common_migrations_path.exists() + ); + for file in glob(&format!("{common_migrations_path:?}/*")) + .unwrap() + .flatten() + { + println!("- {file:?}"); + std::fs::copy(file, &output_path).unwrap(); + } + if let Ok(database_url) = std::env::var("DATABASE_URL") { println!("cargo:rustc-env=DATABASE_URL={database_url}"); } From 51b6741f3e268229178e3017c4cd559aed844b05 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 11 Nov 2025 19:34:03 +0000 Subject: [PATCH 08/28] update offline queries --- ...3c542713e7443b6f0cd4ddb0c00f7a644060.json} | 4 +- ...d1e764a35477f06dcdc2fcb1420334f07e38d.json | 70 +++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) rename nym-data-observatory/.sqlx/{query-fbf7dc2d779476fffcefafaa0a1731dfc6affe6c672df121140a5c7141f71c63.json => query-0f4f26923d4fdf4541deb1fa5e7e3c542713e7443b6f0cd4ddb0c00f7a644060.json} (50%) create mode 100644 nym-data-observatory/.sqlx/query-4b16ddeda8e6e3571836b09e63ad1e764a35477f06dcdc2fcb1420334f07e38d.json diff --git a/nym-data-observatory/.sqlx/query-fbf7dc2d779476fffcefafaa0a1731dfc6affe6c672df121140a5c7141f71c63.json b/nym-data-observatory/.sqlx/query-0f4f26923d4fdf4541deb1fa5e7e3c542713e7443b6f0cd4ddb0c00f7a644060.json similarity index 50% rename from nym-data-observatory/.sqlx/query-fbf7dc2d779476fffcefafaa0a1731dfc6affe6c672df121140a5c7141f71c63.json rename to nym-data-observatory/.sqlx/query-0f4f26923d4fdf4541deb1fa5e7e3c542713e7443b6f0cd4ddb0c00f7a644060.json index d70649da8b8..a88f36b51ac 100644 --- a/nym-data-observatory/.sqlx/query-fbf7dc2d779476fffcefafaa0a1731dfc6affe6c672df121140a5c7141f71c63.json +++ b/nym-data-observatory/.sqlx/query-0f4f26923d4fdf4541deb1fa5e7e3c542713e7443b6f0cd4ddb0c00f7a644060.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO watcher_execution(start_ts, end_ts, error_message)\n VALUES ($1, $2, $3)\n ", + "query": "\n INSERT INTO startup_info(start_ts, end_ts, error_message)\n VALUES ($1, $2, $3)\n ", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "fbf7dc2d779476fffcefafaa0a1731dfc6affe6c672df121140a5c7141f71c63" + "hash": "0f4f26923d4fdf4541deb1fa5e7e3c542713e7443b6f0cd4ddb0c00f7a644060" } diff --git a/nym-data-observatory/.sqlx/query-4b16ddeda8e6e3571836b09e63ad1e764a35477f06dcdc2fcb1420334f07e38d.json b/nym-data-observatory/.sqlx/query-4b16ddeda8e6e3571836b09e63ad1e764a35477f06dcdc2fcb1420334f07e38d.json new file mode 100644 index 00000000000..e3ec034c958 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-4b16ddeda8e6e3571836b09e63ad1e764a35477f06dcdc2fcb1420334f07e38d.json @@ -0,0 +1,70 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO wasm_execute_contract (\n sender,\n contract_address,\n message_type,\n raw_contract_message,\n funds,\n executed_at,\n height,\n hash,\n message_index,\n memo,\n fee\n ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Jsonb", + { + "Custom": { + "name": "coin[]", + "kind": { + "Array": { + "Custom": { + "name": "coin", + "kind": { + "Composite": [ + [ + "denom", + "Text" + ], + [ + "amount", + "Text" + ] + ] + } + } + } + } + } + }, + "Timestamp", + "Int8", + "Text", + "Int8", + "Text", + { + "Custom": { + "name": "coin[]", + "kind": { + "Array": { + "Custom": { + "name": "coin", + "kind": { + "Composite": [ + [ + "denom", + "Text" + ], + [ + "amount", + "Text" + ] + ] + } + } + } + } + } + } + ] + }, + "nullable": [] + }, + "hash": "4b16ddeda8e6e3571836b09e63ad1e764a35477f06dcdc2fcb1420334f07e38d" +} From 9517beecfac71c0753b5ab4ce9615bbf0b435f65 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 11 Nov 2025 19:36:17 +0000 Subject: [PATCH 09/28] change to clap args and use url::Url to parse args --- nym-data-observatory/Cargo.toml | 1 + nym-data-observatory/src/chain_scraper/mod.rs | 48 ++++++------------- nym-data-observatory/src/cli/commands/init.rs | 2 +- .../src/cli/commands/run/args.rs | 17 ++++++- nym-data-observatory/src/cli/mod.rs | 2 +- .../src/config/data_observatory.rs | 2 +- nym-data-observatory/src/config/mod.rs | 8 ++-- nym-data-observatory/src/env.rs | 3 ++ nym-data-observatory/src/error.rs | 3 ++ nym-data-observatory/src/http/models.rs | 4 +- .../src/cli/process_block.rs | 2 +- .../src/cli/process_until.rs | 2 +- nym-validator-rewarder/src/config/mod.rs | 13 +++-- .../src/config/persistence/paths.rs | 6 +-- nym-validator-rewarder/src/error.rs | 5 +- 15 files changed, 64 insertions(+), 54 deletions(-) diff --git a/nym-data-observatory/Cargo.toml b/nym-data-observatory/Cargo.toml index 33178009f43..0de46b9e4a9 100644 --- a/nym-data-observatory/Cargo.toml +++ b/nym-data-observatory/Cargo.toml @@ -38,6 +38,7 @@ tokio-util = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } tower-http = { workspace = true, features = ["cors", "trace"] } +url = { workspace = true } utoipa = { workspace = true, features = ["axum_extras", "time"] } utoipa-swagger-ui = { workspace = true, features = ["axum"] } utoipauto = { workspace = true } diff --git a/nym-data-observatory/src/chain_scraper/mod.rs b/nym-data-observatory/src/chain_scraper/mod.rs index a1eaeeb2fcb..cbc10362083 100644 --- a/nym-data-observatory/src/chain_scraper/mod.rs +++ b/nym-data-observatory/src/chain_scraper/mod.rs @@ -1,8 +1,5 @@ +use crate::cli::commands::run::Args; use crate::db::DbPool; -use crate::env::vars::{ - NYXD_SCRAPER_START_HEIGHT, NYXD_SCRAPER_UNSAFE_NUKE_DB, - NYXD_SCRAPER_USE_BEST_EFFORT_START_HEIGHT, -}; use nyxd_scraper_psql::{PostgresNyxdScraper, PruningOptions}; use std::fs; use tracing::{info, warn}; @@ -10,48 +7,31 @@ use tracing::{info, warn}; pub(crate) mod webhook; pub(crate) async fn run_chain_scraper( + args: Args, config: &crate::config::Config, connection_pool: DbPool, ) -> anyhow::Result { - let websocket_url = std::env::var("NYXD_WS").expect("NYXD_WS not defined"); + let use_best_effort_start_height = args.start_block_height.is_some(); - let rpc_url = std::env::var("NYXD").expect("NYXD not defined"); - let websocket_url = reqwest::Url::parse(&websocket_url)?; - let rpc_url = reqwest::Url::parse(&rpc_url)?; - - // why are those not part of CLI? : ( - let start_block_height = match std::env::var(NYXD_SCRAPER_START_HEIGHT).ok() { - None => None, - // blow up if passed malformed env value - Some(raw) => Some(raw.parse()?), - }; - - let use_best_effort_start_height = - match std::env::var(NYXD_SCRAPER_USE_BEST_EFFORT_START_HEIGHT).ok() { - None => false, - // blow up if passed malformed env value - Some(raw) => raw.parse()?, - }; - - let nuke_db: bool = match std::env::var(NYXD_SCRAPER_UNSAFE_NUKE_DB).ok() { - None => false, - // blow up if passed malformed env value - Some(raw) => raw.parse()?, - }; - - if nuke_db { + if args.nuke_db { warn!("☢️☢️☢️ NUKING THE SCRAPER DATABASE"); fs::remove_file(config.chain_scraper_connection_string())?; } + let database_storage = config + .chain_scraper_connection_string + .clone() + .and(args.db_connection_string) + .expect("no database connection string set in config"); + let scraper = PostgresNyxdScraper::builder(nyxd_scraper_psql::Config { - websocket_url, - rpc_url, - database_storage: config.chain_scraper_connection_string.clone(), + websocket_url: args.websocket_url, + rpc_url: args.rpc_url, + database_storage, pruning_options: PruningOptions::nothing(), store_precommits: false, start_block: nyxd_scraper_psql::StartingBlockOpts { - start_block_height, + start_block_height: args.start_block_height, use_best_effort_start_height, }, }) diff --git a/nym-data-observatory/src/cli/commands/init.rs b/nym-data-observatory/src/cli/commands/init.rs index f57f3fc43f6..4a0a2b9e928 100644 --- a/nym-data-observatory/src/cli/commands/init.rs +++ b/nym-data-observatory/src/cli/commands/init.rs @@ -28,7 +28,7 @@ pub(crate) async fn execute(args: Args) -> Result<(), NymDataObservatoryError> { .with_data_observatory_config(DataObservatoryConfig { webhooks: vec![WebhookConfig { id: DEFAULT_NYM_DATA_OBSERVATORY_ID.to_string(), - webhook_url: "https://webhook.site".to_string(), + webhook_url: url::Url::parse("https://webhook.site")?, authentication: Some(AuthorizationBearerToken { token: "1234".to_string(), }), diff --git a/nym-data-observatory/src/cli/commands/run/args.rs b/nym-data-observatory/src/cli/commands/run/args.rs index dc34af57151..6d4f71b2723 100644 --- a/nym-data-observatory/src/cli/commands/run/args.rs +++ b/nym-data-observatory/src/cli/commands/run/args.rs @@ -2,9 +2,22 @@ // SPDX-License-Identifier: GPL-3.0-only use crate::env::vars::*; +use url::Url; -#[derive(clap::Args, Debug)] +#[derive(clap::Args, Debug, Clone)] pub(crate) struct Args { + #[arg(long, env = NYXD_WS, alias = "nyxd_ws")] + pub(crate) websocket_url: Url, + + #[arg(long, env = NYXD, alias = "nyxd")] + pub(crate) rpc_url: Url, + + #[arg(long, env = NYXD_SCRAPER_START_HEIGHT)] + pub(crate) start_block_height: Option, + + #[arg(long, env = NYXD_SCRAPER_UNSAFE_NUKE_DB, default_value = "false")] + pub(crate) nuke_db: bool, + /// (Override) Postgres connection string for chain scraper history #[arg(long, env = NYM_DATA_OBSERVATORY_DB_URL, alias = "db_url")] pub(crate) db_connection_string: Option, @@ -22,7 +35,7 @@ pub(crate) struct Args { long, env = NYM_DATA_OBSERVATORY_WEBHOOK_URL )] - pub webhook_url: Option, + pub webhook_url: Option, /// (Override) Optionally, authenticate with the webhook #[clap( diff --git a/nym-data-observatory/src/cli/mod.rs b/nym-data-observatory/src/cli/mod.rs index 698bb9d293f..7aa4d99dbba 100644 --- a/nym-data-observatory/src/cli/mod.rs +++ b/nym-data-observatory/src/cli/mod.rs @@ -8,7 +8,7 @@ use clap::{Parser, Subcommand}; use nym_bin_common::bin_info; use std::sync::OnceLock; -mod commands; +pub(crate) mod commands; pub const DEFAULT_NYM_DATA_OBSERVATORY_ID: &str = "default-nym-data-observatory"; diff --git a/nym-data-observatory/src/config/data_observatory.rs b/nym-data-observatory/src/config/data_observatory.rs index 36ef39e8811..fa9341e35b6 100644 --- a/nym-data-observatory/src/config/data_observatory.rs +++ b/nym-data-observatory/src/config/data_observatory.rs @@ -9,7 +9,7 @@ pub struct DataObservatoryConfig { pub struct WebhookConfig { pub id: String, pub description: Option, - pub webhook_url: String, + pub webhook_url: url::Url, pub watch_for_chain_message_types: Vec, pub authentication: Option, } diff --git a/nym-data-observatory/src/config/mod.rs b/nym-data-observatory/src/config/mod.rs index da75f453ea5..86cd357faa5 100644 --- a/nym-data-observatory/src/config/mod.rs +++ b/nym-data-observatory/src/config/mod.rs @@ -81,7 +81,7 @@ impl ConfigBuilder { save_path: Some(self.config_path), data_observatory_config: self.data_observatory_config.unwrap_or_default(), data_dir: self.data_dir, - chain_scraper_connection_string: self.chain_scraper_connection_string, + chain_scraper_connection_string: Some(self.chain_scraper_connection_string), } } } @@ -96,7 +96,7 @@ pub struct Config { #[serde(skip)] pub(crate) data_dir: PathBuf, - pub chain_scraper_connection_string: String, + pub chain_scraper_connection_string: Option, #[serde(default)] pub data_observatory_config: DataObservatoryConfig, @@ -182,7 +182,9 @@ impl Config { } pub fn chain_scraper_connection_string(&self) -> String { - self.chain_scraper_connection_string.clone() + self.chain_scraper_connection_string + .clone() + .expect("database connection string not set") } // simple wrapper that reads config file and assigns path location diff --git a/nym-data-observatory/src/env.rs b/nym-data-observatory/src/env.rs index 45f054a8395..cff2212b57f 100644 --- a/nym-data-observatory/src/env.rs +++ b/nym-data-observatory/src/env.rs @@ -3,6 +3,9 @@ #[allow(unused)] pub mod vars { + pub const NYXD_WS: &str = "NYXD_WS"; + pub const NYXD: &str = "NYXD"; + pub const NYM_DATA_OBSERVATORY_NO_BANNER_ARG: &str = "NYM_DATA_OBSERVATORY_NO_BANNER"; pub const NYM_DATA_OBSERVATORY_CONFIG_ENV_FILE_ARG: &str = "NYM_DATA_OBSERVATORY_CONFIG_ENV_FILE_ARG"; diff --git a/nym-data-observatory/src/error.rs b/nym-data-observatory/src/error.rs index 1dd5b302c91..03d49219ea1 100644 --- a/nym-data-observatory/src/error.rs +++ b/nym-data-observatory/src/error.rs @@ -42,4 +42,7 @@ pub enum NymDataObservatoryError { #[error(transparent)] NymConfigTomlE(#[from] nym_config::error::NymConfigTomlError), + + #[error(transparent)] + UrlParseFailure(#[from] url::ParseError), } diff --git a/nym-data-observatory/src/http/models.rs b/nym-data-observatory/src/http/models.rs index c1798eb59d8..34d6d305f1d 100644 --- a/nym-data-observatory/src/http/models.rs +++ b/nym-data-observatory/src/http/models.rs @@ -8,6 +8,7 @@ pub mod status { use crate::db::models::CoingeckoPriceResponse; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; + use url::Url; use utoipa::ToSchema; #[derive(Clone, Copy, Debug, Serialize, Deserialize, schemars::JsonSchema, ToSchema)] @@ -31,7 +32,8 @@ pub mod status { pub struct Webhook { pub id: String, pub description: String, - pub webhook_url: String, + #[schema(value_type = String)] + pub webhook_url: Url, pub watched_message_types: Vec, } diff --git a/nym-validator-rewarder/src/cli/process_block.rs b/nym-validator-rewarder/src/cli/process_block.rs index aeb249076de..ba802de2750 100644 --- a/nym-validator-rewarder/src/cli/process_block.rs +++ b/nym-validator-rewarder/src/cli/process_block.rs @@ -24,7 +24,7 @@ pub(crate) async fn execute(args: Args) -> Result<(), NymRewarderError> { let config = try_load_current_config(&args.custom_config_path)?.with_override(args.config_override); - SqliteNyxdScraper::new(config.scraper_config()) + SqliteNyxdScraper::new(config.scraper_config()?) .await? .unsafe_process_single_block(args.height) .await?; diff --git a/nym-validator-rewarder/src/cli/process_until.rs b/nym-validator-rewarder/src/cli/process_until.rs index 06c3c68f870..0a34c0852f2 100644 --- a/nym-validator-rewarder/src/cli/process_until.rs +++ b/nym-validator-rewarder/src/cli/process_until.rs @@ -37,7 +37,7 @@ pub(crate) async fn execute(args: Args) -> Result<(), NymRewarderError> { let config = try_load_current_config(&args.custom_config_path)?.with_override(args.config_override); - SqliteNyxdScraper::new(config.scraper_config()) + SqliteNyxdScraper::new(config.scraper_config()?) .await? .unsafe_process_block_range(args.start_height, args.stop_height) .await?; diff --git a/nym-validator-rewarder/src/config/mod.rs b/nym-validator-rewarder/src/config/mod.rs index b3824f21ff1..146f971bada 100644 --- a/nym-validator-rewarder/src/config/mod.rs +++ b/nym-validator-rewarder/src/config/mod.rs @@ -119,18 +119,23 @@ impl Config { } } - pub fn scraper_config(&self) -> nyxd_scraper_sqlite::Config { - nyxd_scraper_sqlite::Config { + pub fn scraper_config(&self) -> Result { + let database_storage = self.storage_paths.nyxd_scraper.as_path(); + let database_storage = database_storage + .to_str() + .ok_or(NymRewarderError::ConfigError)? + .to_string(); + Ok(nyxd_scraper_sqlite::Config { websocket_url: self.nyxd_scraper.websocket_url.clone(), rpc_url: self.base.upstream_nyxd.clone(), - database_storage: self.storage_paths.nyxd_scraper.clone(), + database_storage, pruning_options: self.nyxd_scraper.pruning, store_precommits: self.nyxd_scraper.store_precommits, start_block: StartingBlockOpts { start_block_height: None, use_best_effort_start_height: true, }, - } + }) } pub fn verification_config(&self) -> ticketbook_issuance::VerificationConfig { diff --git a/nym-validator-rewarder/src/config/persistence/paths.rs b/nym-validator-rewarder/src/config/persistence/paths.rs index 47dd3bf8111..c003f900d1b 100644 --- a/nym-validator-rewarder/src/config/persistence/paths.rs +++ b/nym-validator-rewarder/src/config/persistence/paths.rs @@ -15,7 +15,7 @@ pub const DEFAULT_ED25519_PUBLIC_IDENTITY_KEY_FILENAME: &str = "ed25519_identity #[derive(Debug, Clone, Deserialize, Serialize)] pub struct ValidatorRewarderPaths { - pub nyxd_scraper: String, + pub nyxd_scraper: PathBuf, pub reward_history: PathBuf, @@ -47,9 +47,7 @@ impl Default for ValidatorRewarderPaths { fn default() -> Self { ValidatorRewarderPaths { // validator rewarder uses sqlite - nyxd_scraper: (default_data_directory().join(DEFAULT_SCRAPER_DB_FILENAME)) - .to_string_lossy() - .to_string(), + nyxd_scraper: default_data_directory().join(DEFAULT_SCRAPER_DB_FILENAME), reward_history: default_data_directory().join(DEFAULT_REWARD_HISTORY_DB_FILENAME), private_ed25519_identity_key_file: default_data_directory() .join(DEFAULT_ED25519_PRIVATE_IDENTITY_KEY_FILENAME), diff --git a/nym-validator-rewarder/src/error.rs b/nym-validator-rewarder/src/error.rs index d20ee0e81c9..f082e908edf 100644 --- a/nym-validator-rewarder/src/error.rs +++ b/nym-validator-rewarder/src/error.rs @@ -11,7 +11,7 @@ use nym_validator_client::nyxd::tx::ErrorReport; use nym_validator_client::nyxd::{AccountId, Coin}; use nyxd_scraper_sqlite::error::SqliteScraperError; use std::io; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use thiserror::Error; #[derive(Debug, Error)] @@ -25,6 +25,9 @@ pub enum NymRewarderError { #[error("failed to perform startup SQL migration: {0}")] StartupMigrationFailure(#[from] sqlx::migrate::MigrateError), + #[error("config error: database storage path invalid")] + ConfigError, + #[error( "failed to load config file using path '{}'. detailed message: {source}", path.display() )] From 0334bdc81686c28cdb656f4bfba0e98df91cdaa6 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 11 Nov 2025 19:37:38 +0000 Subject: [PATCH 10/28] tidy up README, startup info, typos --- Cargo.lock | 8 +++-- .../src/storage/transaction.rs | 2 +- .../src/cosmos_module/modules/group.rs | 30 ++++++++++--------- nym-data-observatory/README.md | 12 ++++++-- .../src/cli/commands/run/config.rs | 9 +----- .../src/cli/commands/run/mod.rs | 23 +++++++++----- nym-data-observatory/src/db/models.rs | 25 ---------------- nym-data-observatory/src/db/queries/wasm.rs | 1 + 8 files changed, 49 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28c4e7d73df..fb89147936e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2271,7 +2271,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2970,9 +2970,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "gloo-net" @@ -5747,6 +5747,7 @@ dependencies = [ "chrono", "clap", "cosmrs 0.22.0", + "glob", "nym-bin-common", "nym-config", "nym-network-defaults", @@ -5766,6 +5767,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "url", "utoipa", "utoipa-swagger-ui", "utoipauto", diff --git a/common/nyxd-scraper-psql/src/storage/transaction.rs b/common/nyxd-scraper-psql/src/storage/transaction.rs index 13aec6be49e..cfade245379 100644 --- a/common/nyxd-scraper-psql/src/storage/transaction.rs +++ b/common/nyxd-scraper-psql/src/storage/transaction.rs @@ -156,7 +156,7 @@ impl PostgresStorageTransaction { let messages = chain_tx .parsed_messages .values() - .map(|v| v.clone()) + .cloned() .collect::>(); let signer_infos = chain_tx diff --git a/common/nyxd-scraper-shared/src/cosmos_module/modules/group.rs b/common/nyxd-scraper-shared/src/cosmos_module/modules/group.rs index 8ea7a7cc1df..d78bd936aec 100644 --- a/common/nyxd-scraper-shared/src/cosmos_module/modules/group.rs +++ b/common/nyxd-scraper-shared/src/cosmos_module/modules/group.rs @@ -9,19 +9,21 @@ pub(crate) struct Group; impl CosmosModule for Group { fn register_messages(&self, _registry: &mut MessageRegistry) { - warn!("mising cosmos-sdk-proto definition for 'group::MsgCreateGroup'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgUpdateGroupMembers'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgUpdateGroupAdmin'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgUpdateGroupMetadata'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgCreateGroupWithPolicy'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgCreateGroupPolicy'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgUpdateGroupPolicyAdmin'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgUpdateGroupPolicyDecisionPolicy'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgUpdateGroupPolicyMetadata'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgSubmitProposal'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgWithdrawProposal'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgVote'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgExec'"); - warn!("mising cosmos-sdk-proto definition for 'group::MsgLeaveGroup'"); + warn!("missing cosmos-sdk-proto definition for 'group::MsgCreateGroup'"); + warn!("missing cosmos-sdk-proto definition for 'group::MsgUpdateGroupMembers'"); + warn!("missing cosmos-sdk-proto definition for 'group::MsgUpdateGroupAdmin'"); + warn!("missing cosmos-sdk-proto definition for 'group::MsgUpdateGroupMetadata'"); + warn!("missing cosmos-sdk-proto definition for 'group::MsgCreateGroupWithPolicy'"); + warn!("missing cosmos-sdk-proto definition for 'group::MsgCreateGroupPolicy'"); + warn!("missing cosmos-sdk-proto definition for 'group::MsgUpdateGroupPolicyAdmin'"); + warn!( + "missing cosmos-sdk-proto definition for 'group::MsgUpdateGroupPolicyDecisionPolicy'" + ); + warn!("missing cosmos-sdk-proto definition for 'group::MsgUpdateGroupPolicyMetadata'"); + warn!("missing cosmos-sdk-proto definition for 'group::MsgSubmitProposal'"); + warn!("missing cosmos-sdk-proto definition for 'group::MsgWithdrawProposal'"); + warn!("missing cosmos-sdk-proto definition for 'group::MsgVote'"); + warn!("missing cosmos-sdk-proto definition for 'group::MsgExec'"); + warn!("missing cosmos-sdk-proto definition for 'group::MsgLeaveGroup'"); } } diff --git a/nym-data-observatory/README.md b/nym-data-observatory/README.md index bc9b4484c72..860ee327895 100644 --- a/nym-data-observatory/README.md +++ b/nym-data-observatory/README.md @@ -61,16 +61,22 @@ GRANT ALL ON DATABASE nym_data_observatory_data TO nym_data_observatory; Then run: ``` -cargo run -- init --db_url postgres://nym_data_observatory:data-data-data@localhost/nym_data_observatory_data +cargo run -- init --db_url postgres://testuser:testpass@localhost:5433/nym_data_observatory_test ``` and then: ``` -NYM_DATA_OBSERVATORY_DB_URL=postgres://nym_data_observatory:data-data-data@localhost/nym_data_observatory_data \ +NYM_DATA_OBSERVATORY_DB_URL=postgres://testuser:testpass@localhost:5433/nym_data_observatory_test \ NYM_DATA_OBSERVATORY_WEBHOOK_URL="https://webhook.site" \ NYM_DATA_OBSERVATORY_WEBHOOK_AUTH=1234 \ -cargo run -- run +cargo run -- run --websocket-url wss://rpc.nymtech.net/websocket --rpc-url https://rpc.nymtech.net +``` + +or just: + +``` +NYM_DATA_OBSERVATORY_DB_URL=postgres://testuser:testpass@localhost:5433/nym_data_observatory_test cargo run -- run --websocket-url wss://rpc.nymtech.net/websocket --rpc-url https://rpc.nymtech.net ``` ## Troubleshooting diff --git a/nym-data-observatory/src/cli/commands/run/config.rs b/nym-data-observatory/src/cli/commands/run/config.rs index 52ec21fca14..61eb5d8d1c3 100644 --- a/nym-data-observatory/src/cli/commands/run/config.rs +++ b/nym-data-observatory/src/cli/commands/run/config.rs @@ -6,10 +6,8 @@ use crate::error::NymDataObservatoryError; use tracing::{info, warn}; pub(crate) fn get_run_config(args: Args) -> Result { - info!("{args:#?}"); - let Args { - mut watch_for_chain_message_types, + watch_for_chain_message_types, webhook_auth, webhook_url, .. @@ -21,11 +19,6 @@ pub(crate) fn get_run_config(args: Args) -> Result Result<(), NymDataObservatoryError> { let start = OffsetDateTime::now_utc(); - info!("passed arguments: {args:#?}"); - - let config = config::get_run_config(args)?; + let config = config::get_run_config(args.clone())?; let db_connection_string = config.chain_scraper_connection_string(); - info!("Config is {config:#?}"); + info!("nyxd wss: {}", args.websocket_url.to_string()); + info!("nyxd rpc: {}", args.rpc_url.to_string()); + info!("start_block_height: {:#?}", args.start_block_height); info!( - "Chain History Database path is {:?}", - std::path::Path::new(&config.chain_scraper_connection_string()).canonicalize() + "webhooks: {}", + config.data_observatory_config.webhooks.len() ); + for w in &config.data_observatory_config.webhooks { + info!( + "- {}: {} {:?}", + &w.id, + w.webhook_url.as_str(), + w.watch_for_chain_message_types + ); + } + info!("nuke_db: {}", args.nuke_db); let storage = db::Storage::init(db_connection_string).await?; let watcher_pool = storage.pool_owned(); @@ -135,7 +144,7 @@ pub(crate) async fn execute(args: Args, http_port: u16) -> Result<(), NymDataObs let config = config.clone(); async move { // this only blocks until startup sync is done; it then runs on its own set of tasks - let scraper = run_chain_scraper(&config, scraper_pool).await?; + let scraper = run_chain_scraper(args, &config, scraper_pool).await?; Ok(scraper.cancel_token()) } }); diff --git a/nym-data-observatory/src/db/models.rs b/nym-data-observatory/src/db/models.rs index c2ec1fcbcd0..31e754c7dab 100644 --- a/nym-data-observatory/src/db/models.rs +++ b/nym-data-observatory/src/db/models.rs @@ -1,6 +1,4 @@ use serde::{Deserialize, Serialize}; -use sqlx::FromRow; -use time::OffsetDateTime; use utoipa::ToSchema; #[derive(Clone, Serialize, Deserialize, Debug, ToSchema)] @@ -33,26 +31,3 @@ pub(crate) struct PriceHistory { pub(crate) gbp: f64, pub(crate) btc: f64, } - -#[derive(Serialize, Deserialize, Debug, ToSchema)] -pub(crate) struct PaymentRecord { - pub(crate) transaction_hash: String, - pub(crate) sender_address: String, - pub(crate) receiver_address: String, - pub(crate) amount: f64, - pub(crate) timestamp: i64, - pub(crate) height: i64, -} - -#[derive(Serialize, Deserialize, Debug, FromRow)] -pub(crate) struct Transaction { - pub(crate) id: i64, - pub(crate) tx_hash: String, - pub(crate) height: i64, - pub(crate) message_index: i64, - pub(crate) sender: String, - pub(crate) recipient: String, - pub(crate) amount: String, - pub(crate) memo: Option, - pub(crate) created_at: Option, -} diff --git a/nym-data-observatory/src/db/queries/wasm.rs b/nym-data-observatory/src/db/queries/wasm.rs index 9d91f61bf83..252efe69b97 100644 --- a/nym-data-observatory/src/db/queries/wasm.rs +++ b/nym-data-observatory/src/db/queries/wasm.rs @@ -4,6 +4,7 @@ use nyxd_scraper_psql::models::DbCoin; use serde_json::Value; use time::PrimitiveDateTime; +#[allow(clippy::too_many_arguments)] pub async fn insert_wasm_execute( pool: &DbPool, sender: String, From 9dcc1b41a612da2f63ea05e31a38706d7d73f5cc Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 11 Nov 2025 19:39:48 +0000 Subject: [PATCH 11/28] tidy up validator rewarder --- nym-validator-rewarder/src/error.rs | 2 +- nym-validator-rewarder/src/rewarder/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nym-validator-rewarder/src/error.rs b/nym-validator-rewarder/src/error.rs index f082e908edf..a3db95e6f68 100644 --- a/nym-validator-rewarder/src/error.rs +++ b/nym-validator-rewarder/src/error.rs @@ -11,7 +11,7 @@ use nym_validator_client::nyxd::tx::ErrorReport; use nym_validator_client::nyxd::{AccountId, Coin}; use nyxd_scraper_sqlite::error::SqliteScraperError; use std::io; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use thiserror::Error; #[derive(Debug, Error)] diff --git a/nym-validator-rewarder/src/rewarder/mod.rs b/nym-validator-rewarder/src/rewarder/mod.rs index a5d43bca058..03ca36dff02 100644 --- a/nym-validator-rewarder/src/rewarder/mod.rs +++ b/nym-validator-rewarder/src/rewarder/mod.rs @@ -187,7 +187,7 @@ impl Rewarder { info!("the block signing rewarding is running in monitor only mode"); } - let nyxd_scraper = SqliteNyxdScraper::new(config.scraper_config()).await?; + let nyxd_scraper = SqliteNyxdScraper::new(config.scraper_config()?).await?; Some(EpochSigning { nyxd_scraper, From 82d3b61d7aa400170c343ca926ee75a9938aaaa5 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 11 Nov 2025 19:46:56 +0000 Subject: [PATCH 12/28] lock file --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index fb89147936e..9605d73fa24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7859,6 +7859,7 @@ dependencies = [ name = "nyxd-scraper-sqlite" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "nyxd-scraper-shared", "sqlx", From b31c8b5cc60e0e50110035da556aff6612391f62 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 11 Nov 2025 20:43:07 +0000 Subject: [PATCH 13/28] change webhook module from msg to tx handler --- nym-data-observatory/README.md | 8 +++ nym-data-observatory/src/chain_scraper/mod.rs | 2 +- .../src/chain_scraper/webhook.rs | 72 ++++++++++--------- 3 files changed, 49 insertions(+), 33 deletions(-) diff --git a/nym-data-observatory/README.md b/nym-data-observatory/README.md index 860ee327895..5a88b7024de 100644 --- a/nym-data-observatory/README.md +++ b/nym-data-observatory/README.md @@ -79,6 +79,14 @@ or just: NYM_DATA_OBSERVATORY_DB_URL=postgres://testuser:testpass@localhost:5433/nym_data_observatory_test cargo run -- run --websocket-url wss://rpc.nymtech.net/websocket --rpc-url https://rpc.nymtech.net ``` +If you want to watch for cosmwasm messages and send to a webhook: + +``` +NYM_DATA_OBSERVATORY_WEBHOOK_URL=https://webhook.site \ +NYM_DATA_OBSERVATORY_DB_URL=postgres://testuser:testpass@localhost:5433/nym_data_observatory_test\ +cargo run -- run --websocket-url wss://rpc.nymtech.net/websocket --rpc-url https://rpc.nymtech.net --start-block-height 20966360 --watch-for-chain-message-types "/cosmwasm.wasm.v1.MsgExecuteContract" +``` + ## Troubleshooting ### SQLx Offline Mode diff --git a/nym-data-observatory/src/chain_scraper/mod.rs b/nym-data-observatory/src/chain_scraper/mod.rs index cbc10362083..a9cc9b9c216 100644 --- a/nym-data-observatory/src/chain_scraper/mod.rs +++ b/nym-data-observatory/src/chain_scraper/mod.rs @@ -36,7 +36,7 @@ pub(crate) async fn run_chain_scraper( }, }) .with_msg_module(crate::modules::wasm::WasmModule::new(connection_pool)) - .with_msg_module(webhook::WebhookModule::new(config.clone())?); + .with_tx_module(webhook::WebhookModule::new(config.clone())?); let instance = scraper.build_and_start().await?; diff --git a/nym-data-observatory/src/chain_scraper/webhook.rs b/nym-data-observatory/src/chain_scraper/webhook.rs index 34604f9f25e..fe9a4c088b9 100644 --- a/nym-data-observatory/src/chain_scraper/webhook.rs +++ b/nym-data-observatory/src/chain_scraper/webhook.rs @@ -5,9 +5,8 @@ use crate::config::data_observatory::{HttpAuthenticationOptions, WebhookConfig}; use crate::models::WebhookPayload; use anyhow::Context; use async_trait::async_trait; -use nym_validator_client::nyxd::{Any, Msg, MsgSend, Name}; use nyxd_scraper_psql::{ - MsgModule, NyxdScraperTransaction, ParsedTransactionResponse, ScraperError, + NyxdScraperTransaction, ParsedTransactionResponse, ScraperError, TxModule, }; use reqwest::{Client, Url}; use tracing::{error, info}; @@ -30,41 +29,50 @@ impl WebhookModule { } #[async_trait] -impl MsgModule for WebhookModule { - fn type_url(&self) -> String { - ::Proto::type_url() - } - - async fn handle_msg( +impl TxModule for WebhookModule { + async fn handle_tx( &mut self, - index: usize, - _msg: &Any, tx: &ParsedTransactionResponse, - _storage_tx: &mut dyn NyxdScraperTransaction, + _: &mut dyn NyxdScraperTransaction, ) -> Result<(), ScraperError> { - let message = serde_json::to_value(tx.parsed_messages.get(&index)).ok(); - - let payload = WebhookPayload { - height: tx.height.value(), - message_index: index as u64, - transaction_hash: tx.hash.to_string(), - message, - }; - - println!( - "->>>>>>>>>>>>>>>>>>>>>>>>> {}", - serde_json::to_string(&payload).unwrap() - ); - - for webhook in self.webhooks.clone() { - let payload = payload.clone(); - tokio::spawn(async move { - if let Err(e) = webhook.invoke_webhook(&payload).await { - error!("webhook error: {}", e); + for (index, msg) in &tx.parsed_messages { + if let Some(parsed_message_type_url) = tx.parsed_message_urls.get(&index) { + let payload = WebhookPayload { + height: tx.height.value(), + message_index: index.clone() as u64, + transaction_hash: tx.hash.to_string(), + message: Some(msg.clone()), + }; + + // println!( + // "->>>>>>>>>>>>>>>>>>>>>>>>> {}", + // serde_json::to_string(&payload).unwrap() + // ); + + for webhook in self.webhooks.clone() { + // if the webhook requires a type and the parsed message type doesn't match, skip + if !webhook.config.watch_for_chain_message_types.is_empty() + && !webhook + .config + .watch_for_chain_message_types + .contains(parsed_message_type_url) + { + continue; + } + + let payload = payload.clone(); + + // TODO: some excellent advice from Andrew, for another day: + // - pass a cancellation token for shutdown + // - use TaskManager and limit number of webhooks to spawn at once + tokio::spawn(async move { + if let Err(e) = webhook.invoke_webhook(&payload).await { + error!("webhook error: {}", e); + } + }); } - }); + } } - Ok(()) } } From 94c85f4dbf8a3c9d8b7f3b7d563599257104cd5c Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 11 Nov 2025 21:49:38 +0000 Subject: [PATCH 14/28] ignore profiler output --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 3c441754c6a..37bc8865732 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,5 @@ nym-api/redocly/formatted-openapi.json **/settings.sql **/enter_db.sh + +*.profraw \ No newline at end of file From 0196465da1e6e0f440b7fbd207aba32d712b0f41 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 11 Nov 2025 21:50:07 +0000 Subject: [PATCH 15/28] add missing things and make clippy happy --- common/nyxd-scraper-shared/src/block_processor/types.rs | 2 ++ common/nyxd-scraper-shared/src/rpc_client.rs | 3 +++ nym-data-observatory/src/chain_scraper/webhook.rs | 5 ++--- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/common/nyxd-scraper-shared/src/block_processor/types.rs b/common/nyxd-scraper-shared/src/block_processor/types.rs index 00ceeac207e..8bf184c0d9b 100644 --- a/common/nyxd-scraper-shared/src/block_processor/types.rs +++ b/common/nyxd-scraper-shared/src/block_processor/types.rs @@ -30,6 +30,8 @@ pub struct ParsedTransactionResponse { pub parsed_messages: HashMap, + pub parsed_message_urls: HashMap, + pub block: Block, } diff --git a/common/nyxd-scraper-shared/src/rpc_client.rs b/common/nyxd-scraper-shared/src/rpc_client.rs index 18af321ece4..3f4ee84d26e 100644 --- a/common/nyxd-scraper-shared/src/rpc_client.rs +++ b/common/nyxd-scraper-shared/src/rpc_client.rs @@ -73,6 +73,7 @@ impl RpcClient { let mut transactions = Vec::with_capacity(raw_transactions.len()); for raw_tx in raw_transactions { let mut parsed_messages = HashMap::new(); + let mut parsed_message_urls = HashMap::new(); let tx = cosmrs::Tx::from_bytes(&raw_tx.tx).map_err(|source| { ScraperError::TxParseFailure { hash: raw_tx.hash, @@ -83,6 +84,7 @@ impl RpcClient { for (index, msg) in tx.body.messages.iter().enumerate() { if let Some(value) = self.decode_or_skip(msg) { parsed_messages.insert(index, value); + parsed_message_urls.insert(index, msg.type_url.clone()); } } @@ -94,6 +96,7 @@ impl RpcClient { tx, proof: raw_tx.proof, parsed_messages, + parsed_message_urls, block: block.block.clone(), }) } diff --git a/nym-data-observatory/src/chain_scraper/webhook.rs b/nym-data-observatory/src/chain_scraper/webhook.rs index fe9a4c088b9..641f5741d70 100644 --- a/nym-data-observatory/src/chain_scraper/webhook.rs +++ b/nym-data-observatory/src/chain_scraper/webhook.rs @@ -10,7 +10,6 @@ use nyxd_scraper_psql::{ }; use reqwest::{Client, Url}; use tracing::{error, info}; -use utoipa::r#gen::serde_json; pub struct WebhookModule { webhooks: Vec, @@ -36,10 +35,10 @@ impl TxModule for WebhookModule { _: &mut dyn NyxdScraperTransaction, ) -> Result<(), ScraperError> { for (index, msg) in &tx.parsed_messages { - if let Some(parsed_message_type_url) = tx.parsed_message_urls.get(&index) { + if let Some(parsed_message_type_url) = tx.parsed_message_urls.get(index) { let payload = WebhookPayload { height: tx.height.value(), - message_index: index.clone() as u64, + message_index: *index as u64, transaction_hash: tx.hash.to_string(), message: Some(msg.clone()), }; From c9676de462c614dc8df7728afdf0cec443f12dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Wed, 12 Nov 2025 08:39:55 +0000 Subject: [PATCH 16/28] updated cosmrs version used by the nym wallet --- Cargo.lock | 56 ++++++-------------------- nym-wallet/Cargo.lock | 42 +++---------------- nym-wallet/nym-wallet-types/Cargo.toml | 4 +- nym-wallet/src-tauri/Cargo.toml | 4 +- 4 files changed, 23 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9605d73fa24..e60ae9c5def 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1408,16 +1408,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "cosmos-sdk-proto" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "462e1f6a8e005acc8835d32d60cbd7973ed65ea2a8d8473830e675f050956427" -dependencies = [ - "prost", - "tendermint-proto", -] - [[package]] name = "cosmos-sdk-proto" version = "0.27.0" @@ -1431,26 +1421,6 @@ dependencies = [ "tonic 0.13.1", ] -[[package]] -name = "cosmrs" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1394c263335da09e8ba8c4b2c675d804e3e0deb44cce0866a5f838d3ddd43d02" -dependencies = [ - "bip32", - "cosmos-sdk-proto 0.26.1", - "ecdsa", - "eyre", - "k256", - "rand_core 0.6.4", - "serde", - "serde_json", - "signature", - "subtle-encoding", - "tendermint", - "thiserror 1.0.69", -] - [[package]] name = "cosmrs" version = "0.22.0" @@ -1458,7 +1428,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34e74fa7a22930fe0579bef560f2d64b78415d4c47b9dd976c0635136809471d" dependencies = [ "bip32", - "cosmos-sdk-proto 0.27.0", + "cosmos-sdk-proto", "ecdsa", "eyre", "k256", @@ -3615,7 +3585,7 @@ checksum = "a650b51e384e54264b53974feb38e95e37aac70f7f2f9c07eb8022fe15eb8e20" dependencies = [ "base64 0.22.1", "bytes", - "cosmos-sdk-proto 0.27.0", + "cosmos-sdk-proto", "flex-error", "ics23", "informalsystems-pbjson", @@ -4955,7 +4925,7 @@ name = "nym-api-requests" version = "0.1.0" dependencies = [ "bs58", - "cosmrs 0.22.0", + "cosmrs", "cosmwasm-std", "ecdsa", "hex", @@ -5126,7 +5096,7 @@ dependencies = [ "clap", "colored", "comfy-table", - "cosmrs 0.22.0", + "cosmrs", "cosmwasm-std", "csv", "cw-utils", @@ -5292,7 +5262,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "cosmrs 0.22.0", + "cosmrs", "nym-crypto", "nym-gateway-requests", "serde", @@ -5667,7 +5637,7 @@ version = "0.1.0" dependencies = [ "bincode", "bls12_381", - "cosmrs 0.22.0", + "cosmrs", "log", "nym-api-requests", "nym-credentials-interface", @@ -5746,7 +5716,7 @@ dependencies = [ "axum", "chrono", "clap", - "cosmrs 0.22.0", + "cosmrs", "glob", "nym-bin-common", "nym-config", @@ -7429,7 +7399,7 @@ name = "nym-types" version = "1.0.0" dependencies = [ "base64 0.22.1", - "cosmrs 0.22.0", + "cosmrs", "cosmwasm-std", "eyre", "hmac", @@ -7483,7 +7453,7 @@ dependencies = [ "bip32", "bip39", "colored", - "cosmrs 0.22.0", + "cosmrs", "cosmwasm-std", "cw-controllers", "cw-utils", @@ -7631,7 +7601,7 @@ dependencies = [ name = "nym-wallet-types" version = "1.0.0" dependencies = [ - "cosmrs 0.21.1", + "cosmrs", "cosmwasm-std", "hex-literal", "nym-config", @@ -7816,7 +7786,7 @@ version = "0.1.0" dependencies = [ "async-trait", "base64 0.22.1", - "cosmrs 0.22.0", + "cosmrs", "itertools 0.14.0", "nyxd-scraper-shared", "serde", @@ -7834,8 +7804,8 @@ dependencies = [ "async-trait", "base64 0.22.1", "const_format", - "cosmos-sdk-proto 0.27.0", - "cosmrs 0.22.0", + "cosmos-sdk-proto", + "cosmrs", "eyre", "futures", "humantime", diff --git a/nym-wallet/Cargo.lock b/nym-wallet/Cargo.lock index e539f36c598..a7bd151e9de 100644 --- a/nym-wallet/Cargo.lock +++ b/nym-wallet/Cargo.lock @@ -11,7 +11,7 @@ dependencies = [ "bip39", "cfg-if", "colored 2.2.0", - "cosmrs 0.21.1", + "cosmrs", "cosmwasm-std", "dirs 4.0.0", "dotenvy", @@ -1239,16 +1239,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cosmos-sdk-proto" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "462e1f6a8e005acc8835d32d60cbd7973ed65ea2a8d8473830e675f050956427" -dependencies = [ - "prost", - "tendermint-proto", -] - [[package]] name = "cosmos-sdk-proto" version = "0.27.0" @@ -1259,26 +1249,6 @@ dependencies = [ "tendermint-proto", ] -[[package]] -name = "cosmrs" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1394c263335da09e8ba8c4b2c675d804e3e0deb44cce0866a5f838d3ddd43d02" -dependencies = [ - "bip32", - "cosmos-sdk-proto 0.26.1", - "ecdsa", - "eyre", - "k256", - "rand_core 0.6.4", - "serde", - "serde_json", - "signature", - "subtle-encoding", - "tendermint", - "thiserror 1.0.69", -] - [[package]] name = "cosmrs" version = "0.22.0" @@ -1286,7 +1256,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34e74fa7a22930fe0579bef560f2d64b78415d4c47b9dd976c0635136809471d" dependencies = [ "bip32", - "cosmos-sdk-proto 0.27.0", + "cosmos-sdk-proto", "ecdsa", "eyre", "k256", @@ -4186,7 +4156,7 @@ name = "nym-api-requests" version = "0.1.0" dependencies = [ "bs58", - "cosmrs 0.22.0", + "cosmrs", "cosmwasm-std", "ecdsa", "hex", @@ -4598,7 +4568,7 @@ name = "nym-types" version = "1.0.0" dependencies = [ "base64 0.22.1", - "cosmrs 0.22.0", + "cosmrs", "cosmwasm-std", "eyre", "hmac", @@ -4648,7 +4618,7 @@ dependencies = [ "bip32", "bip39", "colored 2.2.0", - "cosmrs 0.22.0", + "cosmrs", "cosmwasm-std", "cw-controllers", "cw-utils", @@ -4718,7 +4688,7 @@ dependencies = [ name = "nym-wallet-types" version = "1.0.0" dependencies = [ - "cosmrs 0.21.1", + "cosmrs", "cosmwasm-std", "hex-literal", "nym-config", diff --git a/nym-wallet/nym-wallet-types/Cargo.toml b/nym-wallet/nym-wallet-types/Cargo.toml index 9b869ec93f4..1ebcaa9df81 100644 --- a/nym-wallet/nym-wallet-types/Cargo.toml +++ b/nym-wallet/nym-wallet-types/Cargo.toml @@ -14,8 +14,8 @@ strum_macros = "0.27.2" ts-rs = "10.0.0" -cosmwasm-std = "2.2.1" -cosmrs = "=0.21.1" +cosmwasm-std = "=2.2.2" +cosmrs = { version = "0.22.0" } nym-config = { path = "../../common/config" } nym-network-defaults = { path = "../../common/network-defaults" } diff --git a/nym-wallet/src-tauri/Cargo.toml b/nym-wallet/src-tauri/Cargo.toml index 7ea924a9e88..63279911070 100644 --- a/nym-wallet/src-tauri/Cargo.toml +++ b/nym-wallet/src-tauri/Cargo.toml @@ -53,8 +53,8 @@ base64 = "0.13" zeroize = { version = "1.5", features = ["zeroize_derive", "serde"] } plist = "1.6.0" -cosmwasm-std = "2.2.1" -cosmrs = { version = "0.21.0" } +cosmwasm-std = "=2.2.2" +cosmrs = { version = "0.22.0" } nym-node-requests = { path = "../../nym-node/nym-node-requests" } nym-validator-client = { path = "../../common/client-libs/validator-client" } From f53ac2b08f0eef04940974017ef1563330b72e1b Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 12 Nov 2025 11:15:04 +0000 Subject: [PATCH 17/28] add glob to workspace dependencies --- Cargo.toml | 1 + nym-data-observatory/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 28f7f6c3194..85e43d76e2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -266,6 +266,7 @@ futures = "0.3.31" futures-util = "0.3" generic-array = "0.14.7" getrandom = "0.2.10" +glob = "0.3" handlebars = "3.5.5" hex = "0.4.3" hickory-resolver = "0.25" diff --git a/nym-data-observatory/Cargo.toml b/nym-data-observatory/Cargo.toml index 0de46b9e4a9..8ecbe74e0ac 100644 --- a/nym-data-observatory/Cargo.toml +++ b/nym-data-observatory/Cargo.toml @@ -46,6 +46,6 @@ utoipauto = { workspace = true } [build-dependencies] anyhow = { workspace = true } -glob = "0.3.3" +glob = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] } \ No newline at end of file From 9bef69d9670fe0e03792791e8d8b20f3d5bc3809 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 12 Nov 2025 11:17:17 +0000 Subject: [PATCH 18/28] rename migration files --- nym-data-observatory/migrations/{0003_wasm.sql => 0103_wasm.sql} | 0 .../migrations/{0100_startup_info.sql => 1000_startup_info.sql} | 0 .../migrations/{0101_price_data.sql => 1101_price_data.sql} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename nym-data-observatory/migrations/{0003_wasm.sql => 0103_wasm.sql} (100%) rename nym-data-observatory/migrations/{0100_startup_info.sql => 1000_startup_info.sql} (100%) rename nym-data-observatory/migrations/{0101_price_data.sql => 1101_price_data.sql} (100%) diff --git a/nym-data-observatory/migrations/0003_wasm.sql b/nym-data-observatory/migrations/0103_wasm.sql similarity index 100% rename from nym-data-observatory/migrations/0003_wasm.sql rename to nym-data-observatory/migrations/0103_wasm.sql diff --git a/nym-data-observatory/migrations/0100_startup_info.sql b/nym-data-observatory/migrations/1000_startup_info.sql similarity index 100% rename from nym-data-observatory/migrations/0100_startup_info.sql rename to nym-data-observatory/migrations/1000_startup_info.sql diff --git a/nym-data-observatory/migrations/0101_price_data.sql b/nym-data-observatory/migrations/1101_price_data.sql similarity index 100% rename from nym-data-observatory/migrations/0101_price_data.sql rename to nym-data-observatory/migrations/1101_price_data.sql From dc78f979e1ee5a7965afd97f6964f9ff7bc922f2 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 12 Nov 2025 11:35:58 +0000 Subject: [PATCH 19/28] remove copying from shared migrations --- Cargo.lock | 1 - Cargo.toml | 1 - nym-data-observatory/.gitignore | 4 ---- nym-data-observatory/Cargo.toml | 1 - nym-data-observatory/build.rs | 28 ---------------------------- 5 files changed, 35 deletions(-) delete mode 100644 nym-data-observatory/.gitignore diff --git a/Cargo.lock b/Cargo.lock index e60ae9c5def..19376eb2b89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5717,7 +5717,6 @@ dependencies = [ "chrono", "clap", "cosmrs", - "glob", "nym-bin-common", "nym-config", "nym-network-defaults", diff --git a/Cargo.toml b/Cargo.toml index 85e43d76e2b..28f7f6c3194 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -266,7 +266,6 @@ futures = "0.3.31" futures-util = "0.3" generic-array = "0.14.7" getrandom = "0.2.10" -glob = "0.3" handlebars = "3.5.5" hex = "0.4.3" hickory-resolver = "0.25" diff --git a/nym-data-observatory/.gitignore b/nym-data-observatory/.gitignore deleted file mode 100644 index c35f82d5e3b..00000000000 --- a/nym-data-observatory/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# ignore files that are copied in from `../common/nyxd-scraper-psql/sql_migrations/* migrations` -# and remember to name the migrations to avoid collisions and so they process in the correct string sort order -0001_metadata.sql -0002_cosmos.sql \ No newline at end of file diff --git a/nym-data-observatory/Cargo.toml b/nym-data-observatory/Cargo.toml index 8ecbe74e0ac..9f4dfd225ab 100644 --- a/nym-data-observatory/Cargo.toml +++ b/nym-data-observatory/Cargo.toml @@ -46,6 +46,5 @@ utoipauto = { workspace = true } [build-dependencies] anyhow = { workspace = true } -glob = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] } \ No newline at end of file diff --git a/nym-data-observatory/build.rs b/nym-data-observatory/build.rs index e062d85bfde..c1dcd9dcb30 100644 --- a/nym-data-observatory/build.rs +++ b/nym-data-observatory/build.rs @@ -1,35 +1,7 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use glob::glob; -use std::env; -use std::path::Path; fn main() { - // copy common manifest files from "../common/nyxd-scraper-psql/sql_migrations/* migrations" - println!("Copying common migrations..."); - let manifest_dir_string = env::var("CARGO_MANIFEST_DIR").unwrap(); - let common_migrations_path = - Path::new(&manifest_dir_string).join("../common/nyxd-scraper-psql/sql_migrations/"); - let output_path = Path::new(&manifest_dir_string).join("migrations"); - println!( - "output_path: {:?} (exists = {})", - output_path, - output_path.exists() - ); - let common_migrations_path = common_migrations_path.as_path(); - println!( - "common_migrations_path: {:?} (exists = {})", - common_migrations_path, - common_migrations_path.exists() - ); - for file in glob(&format!("{common_migrations_path:?}/*")) - .unwrap() - .flatten() - { - println!("- {file:?}"); - std::fs::copy(file, &output_path).unwrap(); - } - if let Ok(database_url) = std::env::var("DATABASE_URL") { println!("cargo:rustc-env=DATABASE_URL={database_url}"); } From c2e82a0ebf7dc46b5b2c41e69dcbfa809b390fd1 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 12 Nov 2025 11:36:29 +0000 Subject: [PATCH 20/28] duplicate shared migrations to keep things simple --- .../migrations/0001_metadata.sql | 10 ++ .../migrations/0002_cosmos.sql | 131 ++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 nym-data-observatory/migrations/0001_metadata.sql create mode 100644 nym-data-observatory/migrations/0002_cosmos.sql diff --git a/nym-data-observatory/migrations/0001_metadata.sql b/nym-data-observatory/migrations/0001_metadata.sql new file mode 100644 index 00000000000..43070210c4a --- /dev/null +++ b/nym-data-observatory/migrations/0001_metadata.sql @@ -0,0 +1,10 @@ +/* + * Copyright 2023 - Nym Technologies SA + * SPDX-License-Identifier: Apache-2.0 + */ + +CREATE TABLE METADATA +( + id INTEGER PRIMARY KEY CHECK (id = 0), + last_processed_height BIGINT NOT NULL +); \ No newline at end of file diff --git a/nym-data-observatory/migrations/0002_cosmos.sql b/nym-data-observatory/migrations/0002_cosmos.sql new file mode 100644 index 00000000000..d33b75fae9d --- /dev/null +++ b/nym-data-observatory/migrations/0002_cosmos.sql @@ -0,0 +1,131 @@ +CREATE TABLE validator +( + consensus_address TEXT NOT NULL PRIMARY KEY, /* Validator consensus address */ + consensus_pubkey TEXT NOT NULL UNIQUE /* Validator consensus public key */ +); + +CREATE TABLE pre_commit +( + validator_address TEXT NOT NULL REFERENCES validator (consensus_address), + height BIGINT NOT NULL, + timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL, + voting_power BIGINT NOT NULL, + proposer_priority BIGINT NOT NULL, + UNIQUE (validator_address, timestamp) +); +CREATE INDEX pre_commit_validator_address_index ON pre_commit (validator_address); +CREATE INDEX pre_commit_height_index ON pre_commit (height); + +CREATE TABLE block +( + height BIGINT UNIQUE PRIMARY KEY, + hash TEXT NOT NULL UNIQUE, + num_txs INTEGER DEFAULT 0, + total_gas BIGINT DEFAULT 0, + proposer_address TEXT REFERENCES validator (consensus_address), + timestamp TIMESTAMP WITHOUT TIME ZONE NOT NULL +); +CREATE INDEX block_height_index ON block (height); +CREATE INDEX block_hash_index ON block (hash); +CREATE INDEX block_proposer_address_index ON block (proposer_address); +ALTER TABLE block + SET ( + autovacuum_vacuum_scale_factor = 0, + autovacuum_analyze_scale_factor = 0, + autovacuum_vacuum_threshold = 10000, + autovacuum_analyze_threshold = 10000 + ); + +CREATE TABLE transaction +( + hash TEXT NOT NULL, + height BIGINT NOT NULL REFERENCES block (height), + "index" INTEGER NOT NULL, -- <<<=== not present in original bdjuno table, but it's quite useful + success BOOLEAN NOT NULL, + + /* Body */ + messages JSONB NOT NULL DEFAULT '[]'::JSONB, + memo TEXT, + signatures TEXT[] NOT NULL, + + /* AuthInfo */ + signer_infos JSONB NOT NULL DEFAULT '[]'::JSONB, + fee JSONB NOT NULL DEFAULT '{}'::JSONB, + + /* Tx response */ + gas_wanted BIGINT DEFAULT 0, + gas_used BIGINT DEFAULT 0, + raw_log TEXT, + logs JSONB, + + CONSTRAINT unique_tx UNIQUE (hash) +); +CREATE INDEX transaction_hash_index ON transaction (hash); +CREATE INDEX transaction_height_index ON transaction (height); + +CREATE TYPE COIN AS +( + denom TEXT, + amount TEXT +); + +CREATE TABLE message +( + transaction_hash TEXT NOT NULL, + index BIGINT NOT NULL, + type TEXT NOT NULL, + value JSONB NOT NULL, + involved_accounts_addresses TEXT[] NOT NULL, + height BIGINT NOT NULL, + + funds COIN[] DEFAULT '{}', + + wasm_sender TEXT, + wasm_contract_address TEXT, + wasm_message_type TEXT, + + FOREIGN KEY (transaction_hash) REFERENCES transaction (hash), + CONSTRAINT unique_message_per_tx UNIQUE (transaction_hash, index) +); +CREATE INDEX message_transaction_hash_index ON message (transaction_hash); +CREATE INDEX message_type_index ON message (type); +CREATE INDEX message_involved_accounts_index ON message USING GIN (involved_accounts_addresses); +CREATE INDEX message_wasm_contract_message_type_index ON message (wasm_message_type); + +/** + * This function is used to find all the utils that involve any of the given addresses and have + * type that is one of the specified types. + */ +CREATE FUNCTION messages_by_address( + addresses TEXT[], + types TEXT[], + "limit" BIGINT = 100, + "offset" BIGINT = 0) + RETURNS SETOF message AS +$$ +SELECT * +FROM message +WHERE (cardinality(types) = 0 OR type = ANY (types)) + AND addresses && involved_accounts_addresses +ORDER BY height DESC +LIMIT "limit" OFFSET "offset" +$$ LANGUAGE sql STABLE; + +CREATE FUNCTION messages_by_type( + types text[], + "limit" bigint DEFAULT 100, + "offset" bigint DEFAULT 0) + RETURNS SETOF message AS +$$ +SELECT * +FROM message +WHERE (cardinality(types) = 0 OR type = ANY (types)) +ORDER BY height DESC +LIMIT "limit" OFFSET "offset" +$$ LANGUAGE sql STABLE; + +CREATE TABLE pruning +( + last_pruned_height BIGINT NOT NULL +); + From 67b69f655fd8457cb3d43100b1fa99244970c64f Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 12 Nov 2025 15:18:37 +0000 Subject: [PATCH 21/28] add check for manual migration sync that will fail on `cargo build` in CI --- Cargo.lock | 1 + Cargo.toml | 1 + nym-data-observatory/Cargo.toml | 1 + nym-data-observatory/build.rs | 53 ++++++++++++++++++++++++++++++++- 4 files changed, 55 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 19376eb2b89..e60ae9c5def 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5717,6 +5717,7 @@ dependencies = [ "chrono", "clap", "cosmrs", + "glob", "nym-bin-common", "nym-config", "nym-network-defaults", diff --git a/Cargo.toml b/Cargo.toml index 28f7f6c3194..85e43d76e2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -266,6 +266,7 @@ futures = "0.3.31" futures-util = "0.3" generic-array = "0.14.7" getrandom = "0.2.10" +glob = "0.3" handlebars = "3.5.5" hex = "0.4.3" hickory-resolver = "0.25" diff --git a/nym-data-observatory/Cargo.toml b/nym-data-observatory/Cargo.toml index 9f4dfd225ab..8ecbe74e0ac 100644 --- a/nym-data-observatory/Cargo.toml +++ b/nym-data-observatory/Cargo.toml @@ -46,5 +46,6 @@ utoipauto = { workspace = true } [build-dependencies] anyhow = { workspace = true } +glob = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] } \ No newline at end of file diff --git a/nym-data-observatory/build.rs b/nym-data-observatory/build.rs index c1dcd9dcb30..6b2122d7482 100644 --- a/nym-data-observatory/build.rs +++ b/nym-data-observatory/build.rs @@ -1,8 +1,59 @@ // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 +use glob::glob; +use std::env; +use std::path::Path; -fn main() { +fn main() -> anyhow::Result<()> { + // check if migrations in "../common/nyxd-scraper-psql/sql_migrations/* are in "nym-data-observatory/migrations" + println!("Checking common migrations..."); + let manifest_dir_string = env::var("CARGO_MANIFEST_DIR").unwrap(); + let common_migrations_path = Path::new(&manifest_dir_string) + .join("../common/nyxd-scraper-psql/sql_migrations/") + .canonicalize()?; + let output_path = Path::new(&manifest_dir_string) + .join("migrations") + .canonicalize()?; + println!( + "output_path: {:?} (exists = {})", + output_path, + output_path.exists() + ); + let common_migrations_path = common_migrations_path.as_path(); + println!( + "common_migrations_path: {:?} (exists = {})", + common_migrations_path, + common_migrations_path.exists() + ); + for file in glob(&format!("{}/*", common_migrations_path.to_str().unwrap())) + .unwrap() + .flatten() + { + println!("- checking if {file:?} exists in nym-data-observatory/migrations directory..."); + let filename = file + .as_path() + .file_name() + .expect("migration filename is found"); + let filename = output_path.join(filename); + println!( + "- {} {file:?} => {filename:?} (exists = {})", + if filename.exists() { "✅" } else { "❌" }, + filename.exists() + ); + + if !filename.exists() { + anyhow::bail!( + "migration {file:?} does not exist in nym-data-observatory/migrations directory, please check and copy it" + ); + } + } + + // sqlx if let Ok(database_url) = std::env::var("DATABASE_URL") { println!("cargo:rustc-env=DATABASE_URL={database_url}"); } + + println!("✅ done"); + + Ok(()) } From d455620789ebf948d2ed0f5d8f31a89c92937c4a Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 12 Nov 2025 16:06:12 +0000 Subject: [PATCH 22/28] build.rs checks data observatory migrations have content of all shared scraper migrations and errors on changes or new files --- Cargo.lock | 1 + nym-data-observatory/Cargo.toml | 1 + nym-data-observatory/build.rs | 54 +++++++++++++++++++++++++-------- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e60ae9c5def..e5659b47ac8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5714,6 +5714,7 @@ dependencies = [ "anyhow", "async-trait", "axum", + "blake3", "chrono", "clap", "cosmrs", diff --git a/nym-data-observatory/Cargo.toml b/nym-data-observatory/Cargo.toml index 8ecbe74e0ac..d23b40f20de 100644 --- a/nym-data-observatory/Cargo.toml +++ b/nym-data-observatory/Cargo.toml @@ -46,6 +46,7 @@ utoipauto = { workspace = true } [build-dependencies] anyhow = { workspace = true } +blake3 = { workspace = true } glob = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] } \ No newline at end of file diff --git a/nym-data-observatory/build.rs b/nym-data-observatory/build.rs index 6b2122d7482..a56d73fe37b 100644 --- a/nym-data-observatory/build.rs +++ b/nym-data-observatory/build.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; // Copyright 2025 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 use glob::glob; @@ -25,29 +26,56 @@ fn main() -> anyhow::Result<()> { common_migrations_path, common_migrations_path.exists() ); + + // hash contents of files in common migrations + let mut common_migrations_hashes = HashMap::new(); for file in glob(&format!("{}/*", common_migrations_path.to_str().unwrap())) .unwrap() .flatten() { - println!("- checking if {file:?} exists in nym-data-observatory/migrations directory..."); - let filename = file - .as_path() - .file_name() - .expect("migration filename is found"); - let filename = output_path.join(filename); + let hash = blake3::hash(std::fs::read(&file)?.as_slice()); + common_migrations_hashes.insert(hash, file); + } + + // hash contents of files in data observatory migrations + let mut data_observatory_migrations_hashes = HashMap::new(); + for file in glob(&format!("{}/*", output_path.to_str().unwrap())) + .unwrap() + .flatten() + { + let hash = blake3::hash(std::fs::read(&file)?.as_slice()); + data_observatory_migrations_hashes.insert(hash, file); + } + + let mut errors = vec![]; + + for entry in common_migrations_hashes { + println!( + "- checking if {:?} exists in nym-data-observatory/migrations directory...", + entry.1 + ); + let res = data_observatory_migrations_hashes.get(&entry.0); + let res_path = res.and_then(|r| r.to_str()).unwrap_or("(not found)"); println!( - "- {} {file:?} => {filename:?} (exists = {})", - if filename.exists() { "✅" } else { "❌" }, - filename.exists() + "- {} {} => {res_path} (content matches = {})", + if res.is_some() { "✅" } else { "❌" }, + entry.1.as_path().to_str().unwrap(), + res.is_some() ); - if !filename.exists() { - anyhow::bail!( - "migration {file:?} does not exist in nym-data-observatory/migrations directory, please check and copy it" - ); + if res.is_none() { + errors.push(format!("- {:?}", entry.1.as_path())); } } + // show all errors + if !errors.is_empty() { + anyhow::bail!( + "the following migrations have changed or do not exist in nym-data-observatory/migrations directory, please check and copy them:\n{}", + errors.join("\n") + ); + } + // sqlx if let Ok(database_url) = std::env::var("DATABASE_URL") { println!("cargo:rustc-env=DATABASE_URL={database_url}"); From 3ea31075fe00ec9c010d77552332121516c9c19d Mon Sep 17 00:00:00 2001 From: benedettadavico Date: Mon, 24 Nov 2025 11:03:22 +0100 Subject: [PATCH 23/28] update runner --- .github/workflows/push-data-observatory.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-data-observatory.yaml b/.github/workflows/push-data-observatory.yaml index 201708d3bd8..eb937a04d65 100644 --- a/.github/workflows/push-data-observatory.yaml +++ b/.github/workflows/push-data-observatory.yaml @@ -8,7 +8,7 @@ env: jobs: build-container: - runs-on: arc-ubuntu-22.04-dind + runs-on: arc-linux-latest-dind steps: - name: Login to Harbor uses: docker/login-action@v3 From bce7871c418a6b700fd8a3ae794bf4e875c79625 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 2 Dec 2025 12:17:17 +0000 Subject: [PATCH 24/28] add reset target to make file --- common/nyxd-scraper-psql/Makefile | 3 +++ common/nyxd-scraper-psql/README.md | 1 + nym-data-observatory/Makefile | 3 +++ 3 files changed, 7 insertions(+) diff --git a/common/nyxd-scraper-psql/Makefile b/common/nyxd-scraper-psql/Makefile index 67eda86cd1f..408dfb2fbcf 100644 --- a/common/nyxd-scraper-psql/Makefile +++ b/common/nyxd-scraper-psql/Makefile @@ -23,6 +23,9 @@ dev-db: test-db-up test-db-wait test-db-migrate ## Start PostgreSQL for developm @echo "PostgreSQL is running on port 5433" @echo "Connection string: $(TEST_DATABASE_URL)" +.PHONY: dev-db-restart +dev-db-restart: clean-db dev-db + # --- Docker Compose Targets --- .PHONY: test-db-up test-db-up: ## Start the PostgreSQL test database in the background diff --git a/common/nyxd-scraper-psql/README.md b/common/nyxd-scraper-psql/README.md index bfa3e04fe21..d2946ec4161 100644 --- a/common/nyxd-scraper-psql/README.md +++ b/common/nyxd-scraper-psql/README.md @@ -50,6 +50,7 @@ make build-pg # Build with PostgreSQL make psql # Connect to running PostgreSQL make clean # Clean build artifacts make clean-db # Stop database and clean volumes +make dev-db-restart # Stop database, clean volumes, rebuild test database and restart ``` ## Environment Variables diff --git a/nym-data-observatory/Makefile b/nym-data-observatory/Makefile index 30d5936a107..017263df031 100644 --- a/nym-data-observatory/Makefile +++ b/nym-data-observatory/Makefile @@ -23,6 +23,9 @@ dev-db: test-db-up test-db-wait test-db-migrate ## Start PostgreSQL for developm @echo "PostgreSQL is running on port 5433" @echo "Connection string: $(TEST_DATABASE_URL)" +.PHONY: dev-db-restart +dev-db-restart: clean-db dev-db + # --- Docker Compose Targets --- .PHONY: test-db-up test-db-up: ## Start the PostgreSQL test database in the background From e05d1f50c7410fc1cab9ff441dd54f35cc8e25fd Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 2 Dec 2025 12:16:56 +0000 Subject: [PATCH 25/28] process events and logs --- .../sql_migrations/0002_cosmos.sql | 1 + .../nyxd-scraper-psql/src/storage/manager.rs | 9 ++++++--- .../src/storage/transaction.rs | 18 +++++++++++++----- .../migrations/0002_cosmos.sql | 1 + 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql b/common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql index d33b75fae9d..e0241f835cd 100644 --- a/common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql +++ b/common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql @@ -57,6 +57,7 @@ CREATE TABLE transaction gas_used BIGINT DEFAULT 0, raw_log TEXT, logs JSONB, + events JSONB, CONSTRAINT unique_tx UNIQUE (hash) ); diff --git a/common/nyxd-scraper-psql/src/storage/manager.rs b/common/nyxd-scraper-psql/src/storage/manager.rs index cbe5003f828..b0bf11f2f5a 100644 --- a/common/nyxd-scraper-psql/src/storage/manager.rs +++ b/common/nyxd-scraper-psql/src/storage/manager.rs @@ -345,6 +345,7 @@ pub(crate) async fn insert_transaction<'a, E>( gas_used: i64, raw_log: String, logs: JsonValue, + events: JsonValue, executor: E, ) -> Result<(), sqlx::Error> where @@ -356,8 +357,8 @@ where sqlx::query!( r#" INSERT INTO transaction - (hash, height, index, success, messages, memo, signatures, signer_infos, fee, gas_wanted, gas_used, raw_log, logs) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + (hash, height, index, success, messages, memo, signatures, signer_infos, fee, gas_wanted, gas_used, raw_log, logs, events) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT (hash) DO UPDATE SET height = excluded.height, index = excluded.index, @@ -370,7 +371,8 @@ where gas_wanted = excluded.gas_wanted, gas_used = excluded.gas_used, raw_log = excluded.raw_log, - logs = excluded.logs + logs = excluded.logs, + events = excluded.events "#, hash, height, @@ -385,6 +387,7 @@ where gas_used, raw_log, logs, + events, ) .execute(executor) .await?; diff --git a/common/nyxd-scraper-psql/src/storage/transaction.rs b/common/nyxd-scraper-psql/src/storage/transaction.rs index cfade245379..bdddf5a1201 100644 --- a/common/nyxd-scraper-psql/src/storage/transaction.rs +++ b/common/nyxd-scraper-psql/src/storage/transaction.rs @@ -23,7 +23,7 @@ use serde_json::json; use sqlx::types::time::{OffsetDateTime, PrimitiveDateTime}; use sqlx::{Postgres, Transaction}; use std::ops::{Deref, DerefMut}; -use tracing::{debug, trace, warn}; +use tracing::{debug, error, trace, warn}; pub struct PostgresStorageTransaction { pub(super) inner: Transaction<'static, Postgres>, @@ -167,10 +167,17 @@ impl PostgresStorageTransaction { .map(|info| proto::cosmos::tx::v1beta1::SignerInfo::from(info.clone())) .collect::>(); + let hash = chain_tx.hash.to_string(); + let height = chain_tx.height.into(); + let index = chain_tx.index as i32; + + let log = serde_json::to_value(chain_tx.tx_result.log.clone()).map_err(|e| error!(hash, height, index, "Failed to parse logs: {e}")).unwrap_or_default(); + let events = &chain_tx.tx_result.events; + insert_transaction( - chain_tx.hash.to_string(), - chain_tx.height.into(), - chain_tx.index as i32, + hash, + height, + index, chain_tx.tx_result.code.is_ok(), serde_json::Value::Array(messages), chain_tx.tx.body.memo.clone(), @@ -180,7 +187,8 @@ impl PostgresStorageTransaction { chain_tx.tx_result.gas_wanted, chain_tx.tx_result.gas_used, chain_tx.tx_result.log.clone(), - json!("null"), + json!(log), + json!(events), self.inner.as_mut(), ) .await?; diff --git a/nym-data-observatory/migrations/0002_cosmos.sql b/nym-data-observatory/migrations/0002_cosmos.sql index d33b75fae9d..e0241f835cd 100644 --- a/nym-data-observatory/migrations/0002_cosmos.sql +++ b/nym-data-observatory/migrations/0002_cosmos.sql @@ -57,6 +57,7 @@ CREATE TABLE transaction gas_used BIGINT DEFAULT 0, raw_log TEXT, logs JSONB, + events JSONB, CONSTRAINT unique_tx UNIQUE (hash) ); From 2c989071ef73263772c53f328c955f1b4e5dd71b Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 2 Dec 2025 11:29:03 +0000 Subject: [PATCH 26/28] migrations - remove unnecessary columns --- common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql | 5 ----- nym-data-observatory/migrations/0002_cosmos.sql | 5 ----- 2 files changed, 10 deletions(-) diff --git a/common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql b/common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql index e0241f835cd..00440004ada 100644 --- a/common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql +++ b/common/nyxd-scraper-psql/sql_migrations/0002_cosmos.sql @@ -81,17 +81,12 @@ CREATE TABLE message funds COIN[] DEFAULT '{}', - wasm_sender TEXT, - wasm_contract_address TEXT, - wasm_message_type TEXT, - FOREIGN KEY (transaction_hash) REFERENCES transaction (hash), CONSTRAINT unique_message_per_tx UNIQUE (transaction_hash, index) ); CREATE INDEX message_transaction_hash_index ON message (transaction_hash); CREATE INDEX message_type_index ON message (type); CREATE INDEX message_involved_accounts_index ON message USING GIN (involved_accounts_addresses); -CREATE INDEX message_wasm_contract_message_type_index ON message (wasm_message_type); /** * This function is used to find all the utils that involve any of the given addresses and have diff --git a/nym-data-observatory/migrations/0002_cosmos.sql b/nym-data-observatory/migrations/0002_cosmos.sql index e0241f835cd..00440004ada 100644 --- a/nym-data-observatory/migrations/0002_cosmos.sql +++ b/nym-data-observatory/migrations/0002_cosmos.sql @@ -81,17 +81,12 @@ CREATE TABLE message funds COIN[] DEFAULT '{}', - wasm_sender TEXT, - wasm_contract_address TEXT, - wasm_message_type TEXT, - FOREIGN KEY (transaction_hash) REFERENCES transaction (hash), CONSTRAINT unique_message_per_tx UNIQUE (transaction_hash, index) ); CREATE INDEX message_transaction_hash_index ON message (transaction_hash); CREATE INDEX message_type_index ON message (type); CREATE INDEX message_involved_accounts_index ON message USING GIN (involved_accounts_addresses); -CREATE INDEX message_wasm_contract_message_type_index ON message (wasm_message_type); /** * This function is used to find all the utils that involve any of the given addresses and have From d2e9f1cc1ac6d31a08452dc25baeea1e1118975d Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Tue, 2 Dec 2025 12:52:36 +0000 Subject: [PATCH 27/28] update offline queries --- ...749e457f8cd4ba3d7d906a7ab3bf5b4e7dc9c.json | 27 ++++++++++++++++++ ...f3cb2eed5b5d35391fd30a4d5f44f2e2178b7.json | 26 ----------------- ...c57402119ec7c3aae731b0da831327301466f.json | 2 +- ...9ce71582635df47f52dcf3fd1df4e7be6b96d.json | 2 +- ...890bbf6150ab394c72783114340d4def5f9ef.json | 2 +- ...749e457f8cd4ba3d7d906a7ab3bf5b4e7dc9c.json | 27 ++++++++++++++++++ ...03b5ee5b27879b0026bb0480b3f2722318a75.json | 15 ++++++++++ ...16422b1f723d0a316314b50c43c8b29f8891d.json | 14 ++++++++++ ...a93e944b0d44ed1f7c1036f306e34372da11c.json | 20 +++++++++++++ ...c57402119ec7c3aae731b0da831327301466f.json | 14 ++++++++++ ...a70635028f392fe794d6131827b083e1755e1.json | 14 ++++++++++ ...3f419a849d4ec45af40b052a4cbf09b44f3ec.json | 20 +++++++++++++ ...5b67fd9281ce1de0653efa53b9d9b93cf335d.json | 14 ++++++++++ ...7a22f0444fbc679db1c06b651fb8b5538b278.json | 18 ++++++++++++ ...ed56b6e270ce186f0e49528865d1924343b78.json | 19 +++++++++++++ ...19cc462d04222fb20ad76de2a40f3f4f8fe15.json | 22 +++++++++++++++ ...6b2e2d6a9ad4b225c4c883aafc4e9f0428008.json | 22 +++++++++++++++ ...e77b7bc0a1af315ffd42c3e68156d6e4ace70.json | 24 ++++++++++++++++ ...ab7bd82ecd68041aa932a56c8ce09623251e4.json | 28 +++++++++++++++++++ ...9ce71582635df47f52dcf3fd1df4e7be6b96d.json | 20 +++++++++++++ ...890bbf6150ab394c72783114340d4def5f9ef.json | 19 +++++++++++++ ...3cf00236490b86779559d84740ec18bcfa3a9.json | 14 ++++++++++ ...13d0598fd856aa019a0cbbae12d7cafb4672f.json | 14 ++++++++++ 23 files changed, 368 insertions(+), 29 deletions(-) create mode 100644 common/nyxd-scraper-psql/.sqlx/query-08f4e54ac24fccd54f4208797b3749e457f8cd4ba3d7d906a7ab3bf5b4e7dc9c.json delete mode 100644 common/nyxd-scraper-psql/.sqlx/query-1e344c1dff8b98eb0eb2e530e28f3cb2eed5b5d35391fd30a4d5f44f2e2178b7.json create mode 100644 nym-data-observatory/.sqlx/query-08f4e54ac24fccd54f4208797b3749e457f8cd4ba3d7d906a7ab3bf5b4e7dc9c.json create mode 100644 nym-data-observatory/.sqlx/query-0d3709efacf763b06bf14803bb803b5ee5b27879b0026bb0480b3f2722318a75.json create mode 100644 nym-data-observatory/.sqlx/query-1c2fb0e9ffceca21ef8dbea19b116422b1f723d0a316314b50c43c8b29f8891d.json create mode 100644 nym-data-observatory/.sqlx/query-2561fb016951ea4cd29e43fb9a4a93e944b0d44ed1f7c1036f306e34372da11c.json create mode 100644 nym-data-observatory/.sqlx/query-2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f.json create mode 100644 nym-data-observatory/.sqlx/query-36ba5941aca6e7b604a10b8b0aba70635028f392fe794d6131827b083e1755e1.json create mode 100644 nym-data-observatory/.sqlx/query-3bdf81a9db6075f6f77224c30553f419a849d4ec45af40b052a4cbf09b44f3ec.json create mode 100644 nym-data-observatory/.sqlx/query-52c27143720ddfdfd0f5644b60f5b67fd9281ce1de0653efa53b9d9b93cf335d.json create mode 100644 nym-data-observatory/.sqlx/query-62e14613f5ffe692346a79086857a22f0444fbc679db1c06b651fb8b5538b278.json create mode 100644 nym-data-observatory/.sqlx/query-64a484fd46d8ec46797f944a4cced56b6e270ce186f0e49528865d1924343b78.json create mode 100644 nym-data-observatory/.sqlx/query-7e82426f5dbcadf1631ba1a806e19cc462d04222fb20ad76de2a40f3f4f8fe15.json create mode 100644 nym-data-observatory/.sqlx/query-9455331f9be5a3be28e2bd399a36b2e2d6a9ad4b225c4c883aafc4e9f0428008.json create mode 100644 nym-data-observatory/.sqlx/query-bc7795e58ce71893c3f32a19db8e77b7bc0a1af315ffd42c3e68156d6e4ace70.json create mode 100644 nym-data-observatory/.sqlx/query-be43d4873911deca784b7be0531ab7bd82ecd68041aa932a56c8ce09623251e4.json create mode 100644 nym-data-observatory/.sqlx/query-c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d.json create mode 100644 nym-data-observatory/.sqlx/query-cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef.json create mode 100644 nym-data-observatory/.sqlx/query-cdba9b267f143c8a8c6c3d6ed713cf00236490b86779559d84740ec18bcfa3a9.json create mode 100644 nym-data-observatory/.sqlx/query-d89558c37c51e8e6b1e6a9d5a2b13d0598fd856aa019a0cbbae12d7cafb4672f.json diff --git a/common/nyxd-scraper-psql/.sqlx/query-08f4e54ac24fccd54f4208797b3749e457f8cd4ba3d7d906a7ab3bf5b4e7dc9c.json b/common/nyxd-scraper-psql/.sqlx/query-08f4e54ac24fccd54f4208797b3749e457f8cd4ba3d7d906a7ab3bf5b4e7dc9c.json new file mode 100644 index 00000000000..cc5863fd0ea --- /dev/null +++ b/common/nyxd-scraper-psql/.sqlx/query-08f4e54ac24fccd54f4208797b3749e457f8cd4ba3d7d906a7ab3bf5b4e7dc9c.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO transaction\n (hash, height, index, success, messages, memo, signatures, signer_infos, fee, gas_wanted, gas_used, raw_log, logs, events)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)\n ON CONFLICT (hash) DO UPDATE\n SET height = excluded.height,\n index = excluded.index,\n success = excluded.success,\n messages = excluded.messages,\n memo = excluded.memo,\n signatures = excluded.signatures,\n signer_infos = excluded.signer_infos,\n fee = excluded.fee,\n gas_wanted = excluded.gas_wanted,\n gas_used = excluded.gas_used,\n raw_log = excluded.raw_log,\n logs = excluded.logs,\n events = excluded.events\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int4", + "Bool", + "Jsonb", + "Text", + "TextArray", + "Jsonb", + "Jsonb", + "Int8", + "Int8", + "Text", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "08f4e54ac24fccd54f4208797b3749e457f8cd4ba3d7d906a7ab3bf5b4e7dc9c" +} diff --git a/common/nyxd-scraper-psql/.sqlx/query-1e344c1dff8b98eb0eb2e530e28f3cb2eed5b5d35391fd30a4d5f44f2e2178b7.json b/common/nyxd-scraper-psql/.sqlx/query-1e344c1dff8b98eb0eb2e530e28f3cb2eed5b5d35391fd30a4d5f44f2e2178b7.json deleted file mode 100644 index 8c9fee0b4a8..00000000000 --- a/common/nyxd-scraper-psql/.sqlx/query-1e344c1dff8b98eb0eb2e530e28f3cb2eed5b5d35391fd30a4d5f44f2e2178b7.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO transaction\n (hash, height, index, success, messages, memo, signatures, signer_infos, fee, gas_wanted, gas_used, raw_log, logs)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)\n ON CONFLICT (hash) DO UPDATE\n SET height = excluded.height,\n index = excluded.index,\n success = excluded.success,\n messages = excluded.messages,\n memo = excluded.memo,\n signatures = excluded.signatures,\n signer_infos = excluded.signer_infos,\n fee = excluded.fee,\n gas_wanted = excluded.gas_wanted,\n gas_used = excluded.gas_used,\n raw_log = excluded.raw_log,\n logs = excluded.logs\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Int4", - "Bool", - "Json", - "Text", - "TextArray", - "Jsonb", - "Jsonb", - "Int8", - "Int8", - "Text", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "1e344c1dff8b98eb0eb2e530e28f3cb2eed5b5d35391fd30a4d5f44f2e2178b7" -} diff --git a/common/nyxd-scraper-psql/.sqlx/query-2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f.json b/common/nyxd-scraper-psql/.sqlx/query-2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f.json index 7efbd0abe8c..b97ea34d16a 100644 --- a/common/nyxd-scraper-psql/.sqlx/query-2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f.json +++ b/common/nyxd-scraper-psql/.sqlx/query-2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f.json @@ -5,7 +5,7 @@ "columns": [], "parameters": { "Left": [ - "Int4" + "Int8" ] }, "nullable": [] diff --git a/common/nyxd-scraper-psql/.sqlx/query-c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d.json b/common/nyxd-scraper-psql/.sqlx/query-c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d.json index 15c137540ce..9bf3eaf97be 100644 --- a/common/nyxd-scraper-psql/.sqlx/query-c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d.json +++ b/common/nyxd-scraper-psql/.sqlx/query-c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d.json @@ -6,7 +6,7 @@ { "ordinal": 0, "name": "last_processed_height", - "type_info": "Int4" + "type_info": "Int8" } ], "parameters": { diff --git a/common/nyxd-scraper-psql/.sqlx/query-cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef.json b/common/nyxd-scraper-psql/.sqlx/query-cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef.json index b5fd43d8d72..5c0da1448a3 100644 --- a/common/nyxd-scraper-psql/.sqlx/query-cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef.json +++ b/common/nyxd-scraper-psql/.sqlx/query-cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef.json @@ -8,7 +8,7 @@ "Text", "Int8", "Text", - "Json", + "Jsonb", "TextArray", "Int8" ] diff --git a/nym-data-observatory/.sqlx/query-08f4e54ac24fccd54f4208797b3749e457f8cd4ba3d7d906a7ab3bf5b4e7dc9c.json b/nym-data-observatory/.sqlx/query-08f4e54ac24fccd54f4208797b3749e457f8cd4ba3d7d906a7ab3bf5b4e7dc9c.json new file mode 100644 index 00000000000..cc5863fd0ea --- /dev/null +++ b/nym-data-observatory/.sqlx/query-08f4e54ac24fccd54f4208797b3749e457f8cd4ba3d7d906a7ab3bf5b4e7dc9c.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO transaction\n (hash, height, index, success, messages, memo, signatures, signer_infos, fee, gas_wanted, gas_used, raw_log, logs, events)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)\n ON CONFLICT (hash) DO UPDATE\n SET height = excluded.height,\n index = excluded.index,\n success = excluded.success,\n messages = excluded.messages,\n memo = excluded.memo,\n signatures = excluded.signatures,\n signer_infos = excluded.signer_infos,\n fee = excluded.fee,\n gas_wanted = excluded.gas_wanted,\n gas_used = excluded.gas_used,\n raw_log = excluded.raw_log,\n logs = excluded.logs,\n events = excluded.events\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int4", + "Bool", + "Jsonb", + "Text", + "TextArray", + "Jsonb", + "Jsonb", + "Int8", + "Int8", + "Text", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "08f4e54ac24fccd54f4208797b3749e457f8cd4ba3d7d906a7ab3bf5b4e7dc9c" +} diff --git a/nym-data-observatory/.sqlx/query-0d3709efacf763b06bf14803bb803b5ee5b27879b0026bb0480b3f2722318a75.json b/nym-data-observatory/.sqlx/query-0d3709efacf763b06bf14803bb803b5ee5b27879b0026bb0480b3f2722318a75.json new file mode 100644 index 00000000000..36ba8bb96b3 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-0d3709efacf763b06bf14803bb803b5ee5b27879b0026bb0480b3f2722318a75.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO validator (consensus_address, consensus_pubkey)\n VALUES ($1, $2)\n ON CONFLICT DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0d3709efacf763b06bf14803bb803b5ee5b27879b0026bb0480b3f2722318a75" +} diff --git a/nym-data-observatory/.sqlx/query-1c2fb0e9ffceca21ef8dbea19b116422b1f723d0a316314b50c43c8b29f8891d.json b/nym-data-observatory/.sqlx/query-1c2fb0e9ffceca21ef8dbea19b116422b1f723d0a316314b50c43c8b29f8891d.json new file mode 100644 index 00000000000..2e10a89220b --- /dev/null +++ b/nym-data-observatory/.sqlx/query-1c2fb0e9ffceca21ef8dbea19b116422b1f723d0a316314b50c43c8b29f8891d.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM pre_commit WHERE height < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1c2fb0e9ffceca21ef8dbea19b116422b1f723d0a316314b50c43c8b29f8891d" +} diff --git a/nym-data-observatory/.sqlx/query-2561fb016951ea4cd29e43fb9a4a93e944b0d44ed1f7c1036f306e34372da11c.json b/nym-data-observatory/.sqlx/query-2561fb016951ea4cd29e43fb9a4a93e944b0d44ed1f7c1036f306e34372da11c.json new file mode 100644 index 00000000000..0d1b70f8cce --- /dev/null +++ b/nym-data-observatory/.sqlx/query-2561fb016951ea4cd29e43fb9a4a93e944b0d44ed1f7c1036f306e34372da11c.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT height\n FROM block\n ORDER BY height ASC\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "height", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "2561fb016951ea4cd29e43fb9a4a93e944b0d44ed1f7c1036f306e34372da11c" +} diff --git a/nym-data-observatory/.sqlx/query-2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f.json b/nym-data-observatory/.sqlx/query-2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f.json new file mode 100644 index 00000000000..b97ea34d16a --- /dev/null +++ b/nym-data-observatory/.sqlx/query-2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE metadata SET last_processed_height = GREATEST(last_processed_height, $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "2679cdf11fa66c7920678cde860c57402119ec7c3aae731b0da831327301466f" +} diff --git a/nym-data-observatory/.sqlx/query-36ba5941aca6e7b604a10b8b0aba70635028f392fe794d6131827b083e1755e1.json b/nym-data-observatory/.sqlx/query-36ba5941aca6e7b604a10b8b0aba70635028f392fe794d6131827b083e1755e1.json new file mode 100644 index 00000000000..dede45475e4 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-36ba5941aca6e7b604a10b8b0aba70635028f392fe794d6131827b083e1755e1.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE pruning SET last_pruned_height = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "36ba5941aca6e7b604a10b8b0aba70635028f392fe794d6131827b083e1755e1" +} diff --git a/nym-data-observatory/.sqlx/query-3bdf81a9db6075f6f77224c30553f419a849d4ec45af40b052a4cbf09b44f3ec.json b/nym-data-observatory/.sqlx/query-3bdf81a9db6075f6f77224c30553f419a849d4ec45af40b052a4cbf09b44f3ec.json new file mode 100644 index 00000000000..e638bce9220 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-3bdf81a9db6075f6f77224c30553f419a849d4ec45af40b052a4cbf09b44f3ec.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT last_pruned_height FROM pruning\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_pruned_height", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "3bdf81a9db6075f6f77224c30553f419a849d4ec45af40b052a4cbf09b44f3ec" +} diff --git a/nym-data-observatory/.sqlx/query-52c27143720ddfdfd0f5644b60f5b67fd9281ce1de0653efa53b9d9b93cf335d.json b/nym-data-observatory/.sqlx/query-52c27143720ddfdfd0f5644b60f5b67fd9281ce1de0653efa53b9d9b93cf335d.json new file mode 100644 index 00000000000..58af4f89c42 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-52c27143720ddfdfd0f5644b60f5b67fd9281ce1de0653efa53b9d9b93cf335d.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM message WHERE height < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "52c27143720ddfdfd0f5644b60f5b67fd9281ce1de0653efa53b9d9b93cf335d" +} diff --git a/nym-data-observatory/.sqlx/query-62e14613f5ffe692346a79086857a22f0444fbc679db1c06b651fb8b5538b278.json b/nym-data-observatory/.sqlx/query-62e14613f5ffe692346a79086857a22f0444fbc679db1c06b651fb8b5538b278.json new file mode 100644 index 00000000000..a7c102469df --- /dev/null +++ b/nym-data-observatory/.sqlx/query-62e14613f5ffe692346a79086857a22f0444fbc679db1c06b651fb8b5538b278.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO pre_commit (validator_address, height, timestamp, voting_power, proposer_priority)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (validator_address, timestamp) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Timestamp", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "62e14613f5ffe692346a79086857a22f0444fbc679db1c06b651fb8b5538b278" +} diff --git a/nym-data-observatory/.sqlx/query-64a484fd46d8ec46797f944a4cced56b6e270ce186f0e49528865d1924343b78.json b/nym-data-observatory/.sqlx/query-64a484fd46d8ec46797f944a4cced56b6e270ce186f0e49528865d1924343b78.json new file mode 100644 index 00000000000..08983f2af9f --- /dev/null +++ b/nym-data-observatory/.sqlx/query-64a484fd46d8ec46797f944a4cced56b6e270ce186f0e49528865d1924343b78.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO block (height, hash, num_txs, total_gas, proposer_address, timestamp)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Int4", + "Int8", + "Text", + "Timestamp" + ] + }, + "nullable": [] + }, + "hash": "64a484fd46d8ec46797f944a4cced56b6e270ce186f0e49528865d1924343b78" +} diff --git a/nym-data-observatory/.sqlx/query-7e82426f5dbcadf1631ba1a806e19cc462d04222fb20ad76de2a40f3f4f8fe15.json b/nym-data-observatory/.sqlx/query-7e82426f5dbcadf1631ba1a806e19cc462d04222fb20ad76de2a40f3f4f8fe15.json new file mode 100644 index 00000000000..3a60c573ed8 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-7e82426f5dbcadf1631ba1a806e19cc462d04222fb20ad76de2a40f3f4f8fe15.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT height\n FROM block\n WHERE timestamp < $1\n ORDER BY timestamp DESC\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "height", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Timestamp" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7e82426f5dbcadf1631ba1a806e19cc462d04222fb20ad76de2a40f3f4f8fe15" +} diff --git a/nym-data-observatory/.sqlx/query-9455331f9be5a3be28e2bd399a36b2e2d6a9ad4b225c4c883aafc4e9f0428008.json b/nym-data-observatory/.sqlx/query-9455331f9be5a3be28e2bd399a36b2e2d6a9ad4b225c4c883aafc4e9f0428008.json new file mode 100644 index 00000000000..309aa81d9c7 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-9455331f9be5a3be28e2bd399a36b2e2d6a9ad4b225c4c883aafc4e9f0428008.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT height\n FROM block\n WHERE timestamp > $1\n ORDER BY timestamp\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "height", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Timestamp" + ] + }, + "nullable": [ + false + ] + }, + "hash": "9455331f9be5a3be28e2bd399a36b2e2d6a9ad4b225c4c883aafc4e9f0428008" +} diff --git a/nym-data-observatory/.sqlx/query-bc7795e58ce71893c3f32a19db8e77b7bc0a1af315ffd42c3e68156d6e4ace70.json b/nym-data-observatory/.sqlx/query-bc7795e58ce71893c3f32a19db8e77b7bc0a1af315ffd42c3e68156d6e4ace70.json new file mode 100644 index 00000000000..caca484b94d --- /dev/null +++ b/nym-data-observatory/.sqlx/query-bc7795e58ce71893c3f32a19db8e77b7bc0a1af315ffd42c3e68156d6e4ace70.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT COUNT(*) as count FROM pre_commit\n WHERE\n validator_address = $1\n AND height >= $2\n AND height <= $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bc7795e58ce71893c3f32a19db8e77b7bc0a1af315ffd42c3e68156d6e4ace70" +} diff --git a/nym-data-observatory/.sqlx/query-be43d4873911deca784b7be0531ab7bd82ecd68041aa932a56c8ce09623251e4.json b/nym-data-observatory/.sqlx/query-be43d4873911deca784b7be0531ab7bd82ecd68041aa932a56c8ce09623251e4.json new file mode 100644 index 00000000000..f1df706371b --- /dev/null +++ b/nym-data-observatory/.sqlx/query-be43d4873911deca784b7be0531ab7bd82ecd68041aa932a56c8ce09623251e4.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT * FROM validator\n WHERE EXISTS (\n SELECT 1 FROM pre_commit\n WHERE height = $1\n AND pre_commit.validator_address = validator.consensus_address\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "consensus_address", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "consensus_pubkey", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "be43d4873911deca784b7be0531ab7bd82ecd68041aa932a56c8ce09623251e4" +} diff --git a/nym-data-observatory/.sqlx/query-c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d.json b/nym-data-observatory/.sqlx/query-c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d.json new file mode 100644 index 00000000000..9bf3eaf97be --- /dev/null +++ b/nym-data-observatory/.sqlx/query-c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT last_processed_height FROM metadata\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_processed_height", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "c88d07fecc3f33deaa6e93db1469ce71582635df47f52dcf3fd1df4e7be6b96d" +} diff --git a/nym-data-observatory/.sqlx/query-cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef.json b/nym-data-observatory/.sqlx/query-cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef.json new file mode 100644 index 00000000000..5c0da1448a3 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO message(transaction_hash, index, type, value, involved_accounts_addresses, height)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (transaction_hash, index) DO UPDATE\n SET height = excluded.height,\n type = excluded.type,\n value = excluded.value,\n involved_accounts_addresses = excluded.involved_accounts_addresses\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Text", + "Jsonb", + "TextArray", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cc0ae74082d7d8a89f2d3364676890bbf6150ab394c72783114340d4def5f9ef" +} diff --git a/nym-data-observatory/.sqlx/query-cdba9b267f143c8a8c6c3d6ed713cf00236490b86779559d84740ec18bcfa3a9.json b/nym-data-observatory/.sqlx/query-cdba9b267f143c8a8c6c3d6ed713cf00236490b86779559d84740ec18bcfa3a9.json new file mode 100644 index 00000000000..2ae11a8fbb4 --- /dev/null +++ b/nym-data-observatory/.sqlx/query-cdba9b267f143c8a8c6c3d6ed713cf00236490b86779559d84740ec18bcfa3a9.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM block WHERE height < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cdba9b267f143c8a8c6c3d6ed713cf00236490b86779559d84740ec18bcfa3a9" +} diff --git a/nym-data-observatory/.sqlx/query-d89558c37c51e8e6b1e6a9d5a2b13d0598fd856aa019a0cbbae12d7cafb4672f.json b/nym-data-observatory/.sqlx/query-d89558c37c51e8e6b1e6a9d5a2b13d0598fd856aa019a0cbbae12d7cafb4672f.json new file mode 100644 index 00000000000..1970629169b --- /dev/null +++ b/nym-data-observatory/.sqlx/query-d89558c37c51e8e6b1e6a9d5a2b13d0598fd856aa019a0cbbae12d7cafb4672f.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM transaction WHERE height < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "d89558c37c51e8e6b1e6a9d5a2b13d0598fd856aa019a0cbbae12d7cafb4672f" +} From 970db18e8fd552571ddfbf09209e80ad27e0d643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C4=99drzej=20Stuczy=C5=84ski?= Date: Tue, 2 Dec 2025 14:19:13 +0000 Subject: [PATCH 28/28] chore: run cargo fmt --- common/nyxd-scraper-psql/src/storage/transaction.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/nyxd-scraper-psql/src/storage/transaction.rs b/common/nyxd-scraper-psql/src/storage/transaction.rs index bdddf5a1201..e0d7a6d86c7 100644 --- a/common/nyxd-scraper-psql/src/storage/transaction.rs +++ b/common/nyxd-scraper-psql/src/storage/transaction.rs @@ -171,7 +171,9 @@ impl PostgresStorageTransaction { let height = chain_tx.height.into(); let index = chain_tx.index as i32; - let log = serde_json::to_value(chain_tx.tx_result.log.clone()).map_err(|e| error!(hash, height, index, "Failed to parse logs: {e}")).unwrap_or_default(); + let log = serde_json::to_value(chain_tx.tx_result.log.clone()) + .map_err(|e| error!(hash, height, index, "Failed to parse logs: {e}")) + .unwrap_or_default(); let events = &chain_tx.tx_result.events; insert_transaction(