Merge #1957: feat(electrum): optimize merkle proof validation with batching

156cbab test(electrum): Improve benchmark (志宇)
4ea5ea6 feat(electrum): batch `transaction.get_merkle` calls via `batch_call` (Wei Chen)
ec4fd97 feat(electrum): batched `Header`s and `script_get_history` (Wei Chen)
f21a21d test(electrum): add `criterion` benchmark for `sync` (Wei Chen)
b57768d fix(electrum): improve tx validation and gap limit scanning (keerthi)
7a18cad feat(electrum): optimize merkle proof validation with batching (Wei Chen)

Pull request description:

  Replaces #1908, originally authored by @Keerthi421.
  Fixes #1891.

  ### Description

  This PR optimizes `sync`/`full_scan` performance by batching and caching key RPC calls to slash network round-trips and eliminate redundant work.

  Key improvements:

  * Gather all `blockchain.transaction.get_merkle` calls into a single `batch_call` request.
  * Use `batch_script_get_history` instead of many individual `script_get_history` calls.
  * Use `batch_block_header` to fetch all needed block headers in one call rather than repeatedly calling `block_header`.
  * Introduce a cache of transaction anchors to skip re-validating already confirmed transactions.

  #### Anchor Caching Performance Improvements

  Results suggest a significant speed up with a warmed up cache. Tested on local Electrum server with:
  ```
  $ cargo bench -p bdk_electrum --bench test_sync
  ```

  Results before this PR (https://github.com/LagginTimes/bdk/tree/1957-master-branch):

  ```
  sync_with_electrum      time:   [1.3702 s 1.3732 s 1.3852 s]
  ```

  Results after this PR:

  ```
  sync_with_electrum      time:   [851.31 ms 853.26 ms 856.23 ms]
  ```
  #### Batch Call Performance Improvements

  No persisted data was carried over between runs, so each test started with cold caches and measured only raw batching performance. Tested with`example_electrum` out of https://github.com/LagginTimes/bdk/tree/example_electrum_timing with the following parameters:

  ```
  $ example_electrum init "tr([62f3f3af/86'/1'/0']tpubDD4Kse29e47rSP5paSuNPhWnGMcdEDAuiG42LEd5yaRDN2CFApWiLTAzxQSLS7MpvxrpxvRJBVcjhVPRk7gec4iWfwvLrEhns1LA4h7i3c2/0/*)#cn4sudyq"
  $ example_electrum scan tcp://signet-electrumx.wakiyamap.dev:50001
  ```

  Results before this PR:

  ```
  FULL_SCAN TIME: 8.145874476s
  ```

  Results after this PR (using this PR's [`bdk_electrum_client.rs`](https://github.com/bitcoindevkit/bdk/blob/70495e2010541acbb5d62f9b5692de20924ac53f/crates/electrum/src/bdk_electrum_client.rs)):

  ```
  FULL_SCAN TIME: 2.594050112s
  ```

  ### Changelog notice

  * Add transaction anchor cache to prevent redundant network calls.
  * Batch Merkle proof, script history, and header requests.

  ### Checklists

  #### All Submissions:

  * [x] I've signed all my commits
  * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
  * [x] I ran `cargo fmt` and `cargo clippy` before committing

  #### New Features:

  * [x] I've added tests for the new feature
  * [x] I've added docs for the new feature

  #### Bugfixes:

  * [ ] This pull request breaks the existing API
  * [x] I've added tests to reproduce the issue which are now passing
  * [x] I'm linking the issue being fixed by this PR

ACKs for top commit:
  oleonardolima:
    tACK 156cbab
  evanlinjin:
    ACK 156cbab

Tree-SHA512: dc7dc1d7de938223cc03293d8bb8ae12c8799c7ec8ba8c7faec5cf2076c96a1b1e50b406cbcc90cbd6cbe7a311c0c11dd036691c03ed067c469a26260903993b

Author: evanlinjin

crates/electrum/Cargo.toml

@@ -15,10 +15,12 @@ workspace = true
 [dependencies]
 bdk_core = { path = "../core", version = "0.6.0" }
 electrum-client = { version = "0.23.1", features = [ "proxy" ], default-features = false }
+serde_json = "1.0"
 
 [dev-dependencies]
 bdk_testenv = { path = "../testenv" }
 bdk_chain = { path = "../chain" }
+criterion = { version = "0.2" }
 
 [features]
 default = ["use-rustls"]
@@ -29,3 +31,7 @@ use-openssl = ["electrum-client/use-openssl"]
 [[test]]
 name = "test_electrum"
 required-features = ["use-rustls"]
+
+[[bench]]
+name = "test_sync"
+harness = false

crates/electrum/benches/test_sync.rs

@@ -0,0 +1,118 @@
+use bdk_chain::bitcoin::{Address, Amount, ScriptBuf};
+use bdk_core::{
+    bitcoin::{
+        consensus::WriteExt,
+        hashes::Hash,
+        key::{Secp256k1, UntweakedPublicKey},
+        Network, TapNodeHash,
+    },
+    spk_client::SyncRequest,
+    CheckPoint,
+};
+use bdk_electrum::BdkElectrumClient;
+use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv};
+use criterion::{criterion_group, criterion_main, Criterion};
+use electrum_client::ElectrumApi;
+use std::{collections::BTreeSet, time::Duration};
+
+// Batch size for `sync_with_electrum`.
+const BATCH_SIZE: usize = 100;
+
+pub fn get_test_spk(i: usize) -> ScriptBuf {
+    const PK_BYTES: &[u8] = &[
+        12, 244, 72, 4, 163, 4, 211, 81, 159, 82, 153, 123, 125, 74, 142, 40, 55, 237, 191, 231,
+        31, 114, 89, 165, 83, 141, 8, 203, 93, 240, 53, 101,
+    ];
+    let secp = Secp256k1::new();
+    let pk = UntweakedPublicKey::from_slice(PK_BYTES).expect("Must be valid PK");
+    let mut engine = TapNodeHash::engine();
+    engine.emit_u64(i as u64).expect("must emit");
+    ScriptBuf::new_p2tr(&secp, pk, Some(TapNodeHash::from_engine(engine)))
+}
+
+fn sync_with_electrum<E: ElectrumApi>(
+    client: &BdkElectrumClient<E>,
+    spks: &[ScriptBuf],
+    chain_tip: &CheckPoint,
+) -> anyhow::Result<()> {
+    let update = client.sync(
+        SyncRequest::builder()
+            .chain_tip(chain_tip.clone())
+            .spks(spks.iter().cloned()),
+        BATCH_SIZE,
+        true,
+    )?;
+
+    assert!(
+        !update.tx_update.txs.is_empty(),
+        "expected some transactions from sync, but got none"
+    );
+
+    Ok(())
+}
+
+pub fn test_sync_performance(c: &mut Criterion) {
+    let env = TestEnv::new().unwrap();
+
+    const NUM_BLOCKS: usize = 100;
+    let mut spks = Vec::with_capacity(NUM_BLOCKS);
+
+    // Mine some blocks and send transactions.
+    env.mine_blocks(101, None).unwrap();
+
+    // Scatter UTXOs across many blocks.
+    for i in 0..NUM_BLOCKS {
+        let spk = get_test_spk(i);
+        let addr = Address::from_script(&spk, Network::Regtest).unwrap();
+        env.send(&addr, Amount::from_sat(10_000)).unwrap();
+        env.mine_blocks(1, None).unwrap();
+
+        spks.push(spk);
+    }
+    let _ = env.wait_until_electrum_sees_block(Duration::from_secs(6));
+    assert_eq!(
+        spks.iter().cloned().collect::<BTreeSet<_>>().len(),
+        spks.len(),
+        "all spks must be unique",
+    );
+
+    // Setup receiver.
+    let genesis_cp = CheckPoint::new(bdk_core::BlockId {
+        height: 0,
+        hash: env.bitcoind.client.get_block_hash(0).unwrap(),
+    });
+
+    {
+        let electrum_client =
+            electrum_client::Client::new(env.electrsd.electrum_url.as_str()).unwrap();
+        let spks = spks.clone();
+        let genesis_cp = genesis_cp.clone();
+        c.bench_function("sync_with_electrum", move |b| {
+            b.iter(|| {
+                sync_with_electrum(
+                    &BdkElectrumClient::new(&electrum_client),
+                    &spks,
+                    &genesis_cp,
+                )
+                .expect("must not error")
+            })
+        });
+    }
+
+    {
+        let client = BdkElectrumClient::new(
+            electrum_client::Client::new(env.electrsd.electrum_url.as_str()).unwrap(),
+        );
+        c.bench_function("sync_with_electrum_cached", move |b| {
+            b.iter(|| sync_with_electrum(&client, &spks, &genesis_cp).expect("must not error"))
+        });
+    }
+}
+
+criterion_group! {
+    name = benches;
+    config = Criterion::default()
+        .sample_size(10);
+    targets = test_sync_performance
+}
+criterion_main!(benches);