Merge #2000: FilterIter API redesign

6aa81fe test(bitcoind): Test 6-block reorg to `filter_iter_detects_reorgs` test (志宇)
4b295ee docs(bip158): Update `FilterIter` documentation (valued mammal)
2181c4b feat(rpc)!: `FilterIter` API redesign (valued mammal)

Pull request description:

  Previously `FilterIter` did not detect or handle reorgs between `next` calls, meaning that if a reorg occurred, we might process blocks from a stale fork potentially resulting in an invalid wallet state. This PR aims to fix that by adding logic to explicitly check for and respond to a reorg on every call to `next`.

  ### Notes to the reviewers

  The old implementation required storing block IDs of scanned blocks before creating a checkpoint update, but because the interface was split across different methods, it introduced a timing risk between method calls which, when we consider the possibility of reorgs, made the implementation somewhat brittle.

  To address this, we make sure that 1) Finding the start block and 2) Updating the internal checkpoint are directly tied to the logic of `next`. Since the checkpoint in practice is derived from a clone of the local chain, this ensures that the checkpoint returned by `next` can always find a connection point with the receiver. Additionally we now emit a checkpoint at every height to ensure that any "must-include" heights are not missing.

  For example usage see `examples/filter_iter.rs`

  fixes #1848

  ### Changelog notice

      Fixed
      - `FilterIter` now handles reorgs to ensure consistency of the header chain.

      Changed
      - `Event` is now a struct instead of enum

      Added
      - `FilterIter::new` constructor that takes as input a reference to the RPC client, checkpoint, and a list of SPKs.
      - `Error::ReorgDepthExceeded` variant
      - `Error::TryFromInt` variant

      Removed
      - `FilterIter::new_with_height`
      - `FilterIter::new_with_checkpoint`
      - `EventInner` type
      - `FilterIter::get_tip`
      - `FilterIter::chain_update`
      - `Error::NoScripts` variant

  ### Checklists

  #### All Submissions:

  * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)

  #### New Features:

  * [x] I've added tests for the new feature
  * [x] I've added docs for the new feature

  #### Bugfixes:

  * [x] This pull request breaks the existing API
  * [x] I've added tests to reproduce the issue which are now passing
  * [x] I'm linking the issue being fixed by this PR

ACKs for top commit:
  ValuedMammal:
    ACK 6aa81fe
  evanlinjin:
    ACK 6aa81fe
  oleonardolima:
    tACK 6aa81fe
  LagginTimes:
    ACK 6aa81fe

Tree-SHA512: 4aebec0184f9cb0ba601811409cf782ab3ad1536bb2dd1f177b2263df5fe99739a675734aa00b28af1dd1bce77071ac18ac7f2c3e849cbd876a3639981af390a

Author: evanlinjin

crates/bitcoind_rpc/examples/filter_iter.rs

@@ -1,29 +1,28 @@
-#![allow(clippy::print_stdout)]
+#![allow(clippy::print_stdout, clippy::print_stderr)]
 use std::time::Instant;
 
 use anyhow::Context;
-use bdk_bitcoind_rpc::bip158::{Event, EventInner, FilterIter};
+use bdk_bitcoind_rpc::bip158::{Event, FilterIter};
 use bdk_chain::bitcoin::{constants::genesis_block, secp256k1::Secp256k1, Network};
 use bdk_chain::indexer::keychain_txout::KeychainTxOutIndex;
 use bdk_chain::local_chain::LocalChain;
 use bdk_chain::miniscript::Descriptor;
 use bdk_chain::{BlockId, ConfirmationBlockTime, IndexedTxGraph, SpkIterator};
 use bdk_testenv::anyhow;
-use bitcoin::Address;
 
 // This example shows how BDK chain and tx-graph structures are updated using compact
 // filters syncing. Assumes a connection can be made to a bitcoin node via environment
 // variables `RPC_URL` and `RPC_COOKIE`.
 
 // Usage: `cargo run -p bdk_bitcoind_rpc --example filter_iter`
 
-const EXTERNAL: &str = "tr([7d94197e]tprv8ZgxMBicQKsPe1chHGzaa84k1inY2nAXUL8iPSyWESPrEst4E5oCFXhPATqj5fvw34LDknJz7rtXyEC4fKoXryUdc9q87pTTzfQyv61cKdE/86'/1'/0'/0/*)#uswl2jj7";
-const INTERNAL: &str = "tr([7d94197e]tprv8ZgxMBicQKsPe1chHGzaa84k1inY2nAXUL8iPSyWESPrEst4E5oCFXhPATqj5fvw34LDknJz7rtXyEC4fKoXryUdc9q87pTTzfQyv61cKdE/86'/1'/0'/1/*)#dyt7h8zx";
+const EXTERNAL: &str = "tr([83737d5e/86'/1'/0']tpubDDR5GgtoxS8fJyjjvdahN4VzV5DV6jtbcyvVXhEKq2XtpxjxBXmxH3r8QrNbQqHg4bJM1EGkxi7Pjfkgnui9jQWqS7kxHvX6rhUeriLDKxz/0/*)";
+const INTERNAL: &str = "tr([83737d5e/86'/1'/0']tpubDDR5GgtoxS8fJyjjvdahN4VzV5DV6jtbcyvVXhEKq2XtpxjxBXmxH3r8QrNbQqHg4bJM1EGkxi7Pjfkgnui9jQWqS7kxHvX6rhUeriLDKxz/1/*)";
 const SPK_COUNT: u32 = 25;
 const NETWORK: Network = Network::Signet;
 
-const START_HEIGHT: u32 = 170_000;
-const START_HASH: &str = "00000041c812a89f084f633e4cf47e819a2f6b1c0a15162355a930410522c99d";
+const START_HEIGHT: u32 = 205_000;
+const START_HASH: &str = "0000002bd0f82f8c0c0f1e19128f84c938763641dba85c44bdb6aed1678d16cb";
 
 fn main() -> anyhow::Result<()> {
     // Setup receiving chain and graph structures.
@@ -52,37 +51,22 @@ fn main() -> anyhow::Result<()> {
     let rpc_client =
         bitcoincore_rpc::Client::new(&url, bitcoincore_rpc::Auth::CookieFile(cookie.into()))?;
 
-    // Initialize block emitter
-    let cp = chain.tip();
-    let start_height = cp.height();
-    let mut emitter = FilterIter::new_with_checkpoint(&rpc_client, cp);
+    // Initialize `FilterIter`
+    let mut spks = vec![];
     for (_, desc) in graph.index.keychains() {
-        let spks = SpkIterator::new_with_range(desc, 0..SPK_COUNT).map(|(_, spk)| spk);
-        emitter.add_spks(spks);
+        spks.extend(SpkIterator::new_with_range(desc, 0..SPK_COUNT).map(|(_, s)| s));
     }
+    let iter = FilterIter::new(&rpc_client, chain.tip(), spks);
 
     let start = Instant::now();
 
-    // Sync
-    if let Some(tip) = emitter.get_tip()? {
-        let blocks_to_scan = tip.height - start_height;
-
-        for event in emitter.by_ref() {
-            let event = event?;
-            let curr = event.height();
-            // apply relevant blocks
-            if let Event::Block(EventInner { height, ref block }) = event {
-                let _ = graph.apply_block_relevant(block, height);
-                println!("Matched block {curr}");
-            }
-            if curr % 1000 == 0 {
-                let progress = (curr - start_height) as f32 / blocks_to_scan as f32;
-                println!("[{:.2}%]", progress * 100.0);
-            }
-        }
-        // update chain
-        if let Some(cp) = emitter.chain_update() {
-            let _ = chain.apply_update(cp)?;
+    for res in iter {
+        let Event { cp, block } = res?;
+        let height = cp.height();
+        let _ = chain.apply_update(cp)?;
+        if let Some(block) = block {
+            let _ = graph.apply_block_relevant(&block, height);
+            println!("Matched block {height}");
         }
     }
 
@@ -105,9 +89,18 @@ fn main() -> anyhow::Result<()> {
         }
     }
 
-    let unused_spk = graph.index.reveal_next_spk("external").unwrap().0 .1;
-    let unused_address = Address::from_script(&unused_spk, NETWORK)?;
-    println!("Next external address: {unused_address}");
+    for canon_tx in graph.graph().list_canonical_txs(
+        &chain,
+        chain.tip().block_id(),
+        bdk_chain::CanonicalizationParams::default(),
+    ) {
+        if !canon_tx.chain_position.is_confirmed() {
+            eprintln!(
+                "ERROR: canonical tx should be confirmed {}",
+                canon_tx.tx_node.txid
+            );
+        }
+    }
 
     Ok(())
 }