Merge #1963: Persist spks derived from KeychainTxOutIndex

62767f0 fix(rusqlite_impl): Fix derived spks create table statement (valued mammal)
603f133 docs(chain): Adds docs for persisting spks in `KeychainTxOutIndex` (志宇)
3126cd2 feat(chain)!: Clean up ergonomics of `IndexedTxGraph` (志宇)
a055581 feat(chain): `KeychainTxOutIndex`: Make spk cache optional (志宇)
19e3b5d feat(chain): Add `KeychainTxOutIndex::from_changeset` constructor (志宇)
d761265 feat(chain): `KeychainTxOutIndex`: Debug build checks (志宇)
76875e7 fix(chain)!: API and logical issues in `KeychainTxOutIndex` (志宇)
d299dae feat(chain)!: Persist spks derived from `KeychainTxOutIndex` (志宇)

Pull request description:

  Replaces #1960
  Fixes #1964

  ### Description

  Users with large wallet and/or complex descriptors may experience slow startup of `KeychainTxOutIndex`. This PR addresses this problem by providing the option to persist derived spks so that they no longer need to be re-derived on startup.

  The `IndexedTxGraph` API has been changed for better ergonomics.

  Compared to #1960, this is a more longterm solution that does not depend on multi-threading logic.

  ### Changelog notice

  ```md
  Changed
    - `KeychainTxOutIndex::new` to take in an additional parameter `persist_spks` to control whether derived spks are cached and persisted across restarts. The default of `persist_spks` is false.
    - `KeychainTxOutIndex` methods (`lookahead_to_target, `next_unused_spk`, `reveal_next_spk`) now return changesets as they may derive spks to be persisted.
    - The `InsertDescriptorError` type now wraps descriptors in `Box` to reduce enum size.

  Added
    - `spk_cache` field to `indexer::keychain_txout::ChangeSet` which persists derived spks.
    - `IndexedTxGraph::from_changeset` - allows constructing from `indexed_tx_graph::ChangeSet` and only indexing when ready.
    - `IndexedTxGraph::reindex` method.

  Fixed
    - `KeychainTxOutIndex::reveal_to_target` so that it actually returns `None` if the `keychain` does not exist.
  ```

  ### Checklists

  #### All Submissions:

  * [x] I've signed all my commits
  * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
  * [x] I ran `cargo +nightly fmt` and `cargo clippy` before committing

  #### New Features:

  * [ ] I've added tests for the new feature
  * [x] I've added docs for the new feature

ACKs for top commit:
  ValuedMammal:
    ACK 62767f0
  notmandatory:
    ACK 62767f0

Tree-SHA512: dc1aa4308ffcc6d121e0d7a1ca4ff9f641ed5db63204747fde47ac02e45dae9b65da95554541705a98b69e59f741c043485a26db736966417061a4c9d220ba29

Author: ValuedMammal

crates/bitcoind_rpc/examples/filter_iter.rs

@@ -31,6 +31,7 @@ fn main() -> anyhow::Result<()> {
     let (descriptor, _) = Descriptor::parse_descriptor(&secp, EXTERNAL)?;
     let (change_descriptor, _) = Descriptor::parse_descriptor(&secp, INTERNAL)?;
     let (mut chain, _) = LocalChain::from_genesis_hash(genesis_block(NETWORK).block_hash());
+
     let mut graph = IndexedTxGraph::<ConfirmationBlockTime, KeychainTxOutIndex<&str>>::new({
         let mut index = KeychainTxOutIndex::default();
         index.insert_descriptor("external", descriptor.clone())?;

crates/chain/benches/canonicalization.rs

@@ -82,7 +82,7 @@ fn setup<F: Fn(&mut KeychainTxGraph, &LocalChain)>(f: F) -> (KeychainTxGraph, Lo
 
     let (desc, _) =
         <Descriptor<DescriptorPublicKey>>::parse_descriptor(&Secp256k1::new(), DESC).unwrap();
-    let mut index = KeychainTxOutIndex::new(10);
+    let mut index = KeychainTxOutIndex::new(10, true);
     index.insert_descriptor((), desc).unwrap();
     let mut tx_graph = KeychainTxGraph::new(index);
 

crates/chain/src/indexed_tx_graph.rs

@@ -15,12 +15,14 @@ use crate::{
     Anchor, BlockId, ChainOracle, Indexer, Merge, TxPosInBlock,
 };
 
-/// The [`IndexedTxGraph`] combines a [`TxGraph`] and an [`Indexer`] implementation.
+/// A [`TxGraph<A>`] paired with an indexer `I`, enforcing that every insertion into the graph is
+/// simultaneously fed through the indexer.
 ///
-/// It ensures that [`TxGraph`] and [`Indexer`] are updated atomically.
+/// This guarantees that `tx_graph` and `index` remain in sync: any transaction or floating txout
+/// you add to `tx_graph` has already been processed by `index`.
 #[derive(Debug, Clone)]
 pub struct IndexedTxGraph<A, I> {
-    /// Transaction index.
+    /// The indexer used for filtering transactions and floating txouts that we are interested in.
     pub index: I,
     graph: TxGraph<A>,
 }
@@ -35,14 +37,6 @@ impl<A, I: Default> Default for IndexedTxGraph<A, I> {
 }
 
 impl<A, I> IndexedTxGraph<A, I> {
-    /// Construct a new [`IndexedTxGraph`] with a given `index`.
-    pub fn new(index: I) -> Self {
-        Self {
-            index,
-            graph: TxGraph::default(),
-        }
-    }
-
     /// Get a reference of the internal transaction graph.
     pub fn graph(&self) -> &TxGraph<A> {
         &self.graph
@@ -79,6 +73,87 @@ impl<A: Anchor, I: Indexer> IndexedTxGraph<A, I>
 where
     I::ChangeSet: Default + Merge,
 {
+    /// Create a new, empty [`IndexedTxGraph`].
+    ///
+    /// The underlying `TxGraph` is initialized with `TxGraph::default()`, and the provided
+    /// `index`er is used as‐is (since there are no existing transactions to process).
+    pub fn new(index: I) -> Self {
+        Self {
+            index,
+            graph: TxGraph::default(),
+        }
+    }
+
+    /// Reconstruct an [`IndexedTxGraph`] from persisted graph + indexer state.
+    ///
+    /// 1. Rebuilds the `TxGraph` from `changeset.tx_graph`.
+    /// 2. Calls your `indexer_from_changeset` closure on `changeset.indexer` to restore any state
+    ///    your indexer needs beyond its raw changeset.
+    /// 3. Runs a full `.reindex()`, returning its `ChangeSet` to describe any additional updates
+    ///    applied.
+    ///
+    /// # Errors
+    ///
+    /// Returns `Err(E)` if `indexer_from_changeset` fails.
+    ///
+    /// # Examples
+    ///
+    /// ```rust,no_run
+    /// use bdk_chain::IndexedTxGraph;
+    /// # use bdk_chain::indexed_tx_graph::ChangeSet;
+    /// # use bdk_chain::indexer::keychain_txout::{KeychainTxOutIndex, DEFAULT_LOOKAHEAD};
+    /// # use bdk_core::BlockId;
+    /// # use bdk_testenv::anyhow;
+    /// # use miniscript::{Descriptor, DescriptorPublicKey};
+    /// # use std::str::FromStr;
+    /// # let persisted_changeset = ChangeSet::<BlockId, _>::default();
+    /// # let persisted_desc = Some(Descriptor::<DescriptorPublicKey>::from_str("")?);
+    /// # let persisted_change_desc = Some(Descriptor::<DescriptorPublicKey>::from_str("")?);
+    ///
+    /// let (graph, reindex_cs) =
+    ///     IndexedTxGraph::from_changeset(persisted_changeset, move |idx_cs| -> anyhow::Result<_> {
+    ///         // e.g. KeychainTxOutIndex needs descriptors that weren’t in its CS
+    ///         let mut idx = KeychainTxOutIndex::from_changeset(DEFAULT_LOOKAHEAD, true, idx_cs);
+    ///         if let Some(desc) = persisted_desc {
+    ///             idx.insert_descriptor("external", desc)?;
+    ///         }
+    ///         if let Some(desc) = persisted_change_desc {
+    ///             idx.insert_descriptor("internal", desc)?;
+    ///         }
+    ///         Ok(idx)
+    ///     })?;
+    /// # Ok::<(), anyhow::Error>(())
+    /// ```
+    pub fn from_changeset<F, E>(
+        changeset: ChangeSet<A, I::ChangeSet>,
+        indexer_from_changeset: F,
+    ) -> Result<(Self, ChangeSet<A, I::ChangeSet>), E>
+    where
+        F: FnOnce(I::ChangeSet) -> Result<I, E>,
+    {
+        let graph = TxGraph::<A>::from_changeset(changeset.tx_graph);
+        let index = indexer_from_changeset(changeset.indexer)?;
+        let mut out = Self { graph, index };
+        let out_changeset = out.reindex();
+        Ok((out, out_changeset))
+    }
+
+    /// Synchronizes the indexer to reflect every entry in the transaction graph.
+    ///
+    /// Iterates over **all** full transactions and floating outputs in `self.graph`, passing each
+    /// into `self.index`. Any indexer-side changes produced (via `index_tx` or `index_txout`) are
+    /// merged into a fresh `ChangeSet`, which is then returned.
+    pub fn reindex(&mut self) -> ChangeSet<A, I::ChangeSet> {
+        let mut changeset = ChangeSet::<A, I::ChangeSet>::default();
+        for tx in self.graph.full_txs() {
+            changeset.indexer.merge(self.index.index_tx(&tx));
+        }
+        for (op, txout) in self.graph.floating_txouts() {
+            changeset.indexer.merge(self.index.index_txout(op, txout));
+        }
+        changeset
+    }
+
     fn index_tx_graph_changeset(
         &mut self,
         tx_graph_changeset: &tx_graph::ChangeSet<A>,
@@ -443,6 +518,12 @@ impl<A, IA: Default> From<tx_graph::ChangeSet<A>> for ChangeSet<A, IA> {
     }
 }
 
+impl<A, IA> From<(tx_graph::ChangeSet<A>, IA)> for ChangeSet<A, IA> {
+    fn from((tx_graph, indexer): (tx_graph::ChangeSet<A>, IA)) -> Self {
+        Self { tx_graph, indexer }
+    }
+}
+
 #[cfg(feature = "miniscript")]
 impl<A> From<crate::keychain_txout::ChangeSet> for ChangeSet<A, crate::keychain_txout::ChangeSet> {
     fn from(indexer: crate::keychain_txout::ChangeSet) -> Self {