From 1e96dc9e31afdfc69f81f5c509766e43e9ed5b04 Mon Sep 17 00:00:00 2001
From: Elias Rohrer <dev@tnull.de>
Date: Tue, 11 Nov 2025 12:54:04 +0100
Subject: [PATCH 01/11] Introduce schema versioning

We introuce an `enum VssSchemaVersion` that will allow us to discern
differnent behaviors based on the schema version based on a backwards
compatible manner.
---
 src/io/vss_store.rs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs
index 0e7d0872a..b42924ead 100644
--- a/src/io/vss_store.rs
+++ b/src/io/vss_store.rs
@@ -44,6 +44,13 @@ type CustomRetryPolicy = FilteredRetryPolicy<
 	Box<dyn Fn(&VssError) -> bool + 'static + Send + Sync>,
 >;
 
+enum VssSchemaVersion {
+	// The initial schema version.
+	// This used an empty `aad` and unobfuscated `primary_namespace`/`secondary_namespace`s in the
+	// stored key.
+	V0,
+}
+
 // We set this to a small number of threads that would still allow to make some progress if one
 // would hit a blocking case
 const INTERNAL_RUNTIME_WORKERS: usize = 2;

From 9019366383bad7416441bec432c27d456a839909 Mon Sep 17 00:00:00 2001
From: Elias Rohrer <dev@tnull.de>
Date: Fri, 22 Aug 2025 11:06:53 +0200
Subject: [PATCH 02/11] Use obf. key as `aad` for `StorableBuilder` and
 obfuscate namespaces

We bump our `vss-client` dependency to include the changes to the
`StorableBuilder` interface.

Previously, we the `vss-client` didn't allow to set `ChaCha20Poly1305RFC`'s `aad` field,
which had the `tag` not commit to any particular key. This would allow a
malicious VSS provider to substitute blobs stored under a different key
without the client noticing.

Here, we now set the `aad` field to the key under which the `Storable`
will be stored, ensuring that the retrieved data was originally stored
under the key we expected, if `VssSchemaVersion::V1` is set.

We also now obfuscate primary and secondary namespaces in the persisted
keys, if `VssSchemaVersion::V1` is set.

We also account for `StorableBuilder` now taking `data_decryption_key`
by reference on `build`/`deconstruct`.
---
 Cargo.toml          |  3 +-
 src/io/vss_store.rs | 89 +++++++++++++++++++++++++++++++--------------
 2 files changed, 64 insertions(+), 28 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 544dfca08..a2634e330 100755
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -65,7 +65,8 @@ serde = { version = "1.0.210", default-features = false, features = ["std", "der
 serde_json = { version = "1.0.128", default-features = false, features = ["std"] }
 log = { version = "0.4.22", default-features = false, features = ["std"]}
 
-vss-client = "0.3"
+#vss-client = "0.3"
+vss-client = { git = "https://github.com/tnull/vss-rust-client", rev = "03ca9d99f70387aabec225020e46434cda8d18ff" }
 prost = { version = "0.11.6", default-features = false}
 
 [target.'cfg(windows)'.dependencies]
diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs
index b42924ead..1cc881572 100644
--- a/src/io/vss_store.rs
+++ b/src/io/vss_store.rs
@@ -44,11 +44,15 @@ type CustomRetryPolicy = FilteredRetryPolicy<
 	Box<dyn Fn(&VssError) -> bool + 'static + Send + Sync>,
 >;
 
+#[derive(Debug, PartialEq)]
 enum VssSchemaVersion {
 	// The initial schema version.
 	// This used an empty `aad` and unobfuscated `primary_namespace`/`secondary_namespace`s in the
 	// stored key.
 	V0,
+	// The second deployed schema version.
+	// Here we started to obfuscate the primary and secondary namespaces and the obfuscated `store_key` (`obfuscate(primary_namespace)#obfuscate(secondary_namespace)#obfuscate(key)`) is now used as `aad` for encryption, ensuring that the encrypted blobs commit to the key they're stored under.
+	V1,
 }
 
 // We set this to a small number of threads that would still allow to make some progress if one
@@ -321,9 +325,10 @@ impl Drop for VssStore {
 }
 
 struct VssStoreInner {
+	schema_version: VssSchemaVersion,
 	client: VssClient<CustomRetryPolicy>,
 	store_id: String,
-	storable_builder: StorableBuilder<RandEntropySource>,
+	data_encryption_key: [u8; 32],
 	key_obfuscator: KeyObfuscator,
 	// Per-key locks that ensures that we don't have concurrent writes to the same namespace/key.
 	// The lock also encapsulates the latest written version per key.
@@ -335,10 +340,10 @@ impl VssStoreInner {
 		base_url: String, store_id: String, vss_seed: [u8; 32],
 		header_provider: Arc<dyn VssHeaderProvider>,
 	) -> Self {
+		let schema_version = VssSchemaVersion::V0;
 		let (data_encryption_key, obfuscation_master_key) =
 			derive_data_encryption_and_obfuscation_keys(&vss_seed);
 		let key_obfuscator = KeyObfuscator::new(obfuscation_master_key);
-		let storable_builder = StorableBuilder::new(data_encryption_key, RandEntropySource);
 		let retry_policy = ExponentialBackoffRetryPolicy::new(Duration::from_millis(10))
 			.with_max_attempts(10)
 			.with_max_total_delay(Duration::from_secs(15))
@@ -354,7 +359,7 @@ impl VssStoreInner {
 
 		let client = VssClient::new_with_headers(base_url, retry_policy, header_provider);
 		let locks = Mutex::new(HashMap::new());
-		Self { client, store_id, storable_builder, key_obfuscator, locks }
+		Self { schema_version, client, store_id, data_encryption_key, key_obfuscator, locks }
 	}
 
 	fn get_inner_lock_ref(&self, locking_key: String) -> Arc<tokio::sync::Mutex<u64>> {
@@ -365,11 +370,35 @@ impl VssStoreInner {
 	fn build_obfuscated_key(
 		&self, primary_namespace: &str, secondary_namespace: &str, key: &str,
 	) -> String {
-		let obfuscated_key = self.key_obfuscator.obfuscate(key);
-		if primary_namespace.is_empty() {
-			obfuscated_key
+		if self.schema_version == VssSchemaVersion::V1 {
+			let obfuscated_primary_namepace = self.key_obfuscator.obfuscate(primary_namespace);
+			let obfuscated_secondary_namepace = self.key_obfuscator.obfuscate(secondary_namespace);
+			let obfuscated_key = self.key_obfuscator.obfuscate(key);
+			format!(
+				"{}#{}#{}",
+				obfuscated_primary_namepace, obfuscated_secondary_namepace, obfuscated_key
+			)
+		} else {
+			// Default to V0 schema
+			let obfuscated_key = self.key_obfuscator.obfuscate(key);
+			if primary_namespace.is_empty() {
+				obfuscated_key
+			} else {
+				format!("{}#{}#{}", primary_namespace, secondary_namespace, obfuscated_key)
+			}
+		}
+	}
+
+	fn build_obfuscated_prefix(
+		&self, primary_namespace: &str, secondary_namespace: &str,
+	) -> String {
+		if self.schema_version == VssSchemaVersion::V1 {
+			let obfuscated_primary_namepace = self.key_obfuscator.obfuscate(primary_namespace);
+			let obfuscated_secondary_namepace = self.key_obfuscator.obfuscate(secondary_namespace);
+			format!("{}#{}", obfuscated_primary_namepace, obfuscated_secondary_namepace,)
 		} else {
-			format!("{}#{}#{}", primary_namespace, secondary_namespace, obfuscated_key)
+			// Default to V0 schema
+			format!("{}#{}", primary_namespace, secondary_namespace)
 		}
 	}
 
@@ -390,7 +419,7 @@ impl VssStoreInner {
 	) -> io::Result<Vec<String>> {
 		let mut page_token = None;
 		let mut keys = vec![];
-		let key_prefix = format!("{}#{}", primary_namespace, secondary_namespace);
+		let key_prefix = self.build_obfuscated_prefix(primary_namespace, secondary_namespace);
 		while page_token != Some("".to_string()) {
 			let request = ListKeyVersionsRequest {
 				store_id: self.store_id.clone(),
@@ -420,9 +449,8 @@ impl VssStoreInner {
 	) -> io::Result<Vec<u8>> {
 		check_namespace_key_validity(&primary_namespace, &secondary_namespace, Some(&key), "read")?;
 
-		let obfuscated_key =
-			self.build_obfuscated_key(&primary_namespace, &secondary_namespace, &key);
-		let request = GetObjectRequest { store_id: self.store_id.clone(), key: obfuscated_key };
+		let store_key = self.build_obfuscated_key(&primary_namespace, &secondary_namespace, &key);
+		let request = GetObjectRequest { store_id: self.store_id.clone(), key: store_key.clone() };
 		let resp = self.client.get_object(&request).await.map_err(|e| {
 			let msg = format!(
 				"Failed to read from key {}/{}/{}: {}",
@@ -444,7 +472,11 @@ impl VssStoreInner {
 			Error::new(ErrorKind::Other, msg)
 		})?;
 
-		Ok(self.storable_builder.deconstruct(storable)?.0)
+		let storable_builder = StorableBuilder::new(RandEntropySource);
+		let aad =
+			if self.schema_version == VssSchemaVersion::V1 { store_key.as_bytes() } else { &[] };
+		let decrypted = storable_builder.deconstruct(storable, &self.data_encryption_key, aad)?.0;
+		Ok(decrypted)
 	}
 
 	async fn write_internal(
@@ -458,22 +490,25 @@ impl VssStoreInner {
 			"write",
 		)?;
 
-		self.execute_locked_write(inner_lock_ref, locking_key, version, async move || {
-			let obfuscated_key =
-				self.build_obfuscated_key(&primary_namespace, &secondary_namespace, &key);
-			let vss_version = -1;
-			let storable = self.storable_builder.build(buf, vss_version);
-			let request = PutObjectRequest {
-				store_id: self.store_id.clone(),
-				global_version: None,
-				transaction_items: vec![KeyValue {
-					key: obfuscated_key,
-					version: vss_version,
-					value: storable.encode_to_vec(),
-				}],
-				delete_items: vec![],
-			};
+		let store_key = self.build_obfuscated_key(&primary_namespace, &secondary_namespace, &key);
+		let vss_version = -1;
+		let storable_builder = StorableBuilder::new(RandEntropySource);
+		let aad =
+			if self.schema_version == VssSchemaVersion::V1 { store_key.as_bytes() } else { &[] };
+		let storable =
+			storable_builder.build(buf.to_vec(), vss_version, &self.data_encryption_key, aad);
+		let request = PutObjectRequest {
+			store_id: self.store_id.clone(),
+			global_version: None,
+			transaction_items: vec![KeyValue {
+				key: store_key,
+				version: vss_version,
+				value: storable.encode_to_vec(),
+			}],
+			delete_items: vec![],
+		};
 
+		self.execute_locked_write(inner_lock_ref, locking_key, version, async move || {
 			self.client.put_object(&request).await.map_err(|e| {
 				let msg = format!(
 					"Failed to write to key {}/{}/{}: {}",

From 35a9306ed76445dbc076331b7ba5d75a7faecb01 Mon Sep 17 00:00:00 2001
From: Elias Rohrer <dev@tnull.de>
Date: Tue, 11 Nov 2025 13:35:34 +0100
Subject: [PATCH 03/11] Prefactor: move client construction out to `VssStore`

While having it in `VssStoreInner` makes more sense, we now opt to
construt the client (soon, clients) in `VssStore` and then hand it down
to `VssStoreInner`. That will allow us to use the client once for
checking the schema version before actually instantiating
`VssStoreInner`.
---
 src/io/vss_store.rs | 50 ++++++++++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 21 deletions(-)

diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs
index 1cc881572..c7b290dad 100644
--- a/src/io/vss_store.rs
+++ b/src/io/vss_store.rs
@@ -81,7 +81,6 @@ impl VssStore {
 		base_url: String, store_id: String, vss_seed: [u8; 32],
 		header_provider: Arc<dyn VssHeaderProvider>, runtime: Arc<Runtime>,
 	) -> Self {
-		let inner = Arc::new(VssStoreInner::new(base_url, store_id, vss_seed, header_provider));
 		let next_version = AtomicU64::new(1);
 		let internal_runtime = Some(
 			tokio::runtime::Builder::new_multi_thread()
@@ -97,6 +96,33 @@ impl VssStore {
 				.unwrap(),
 		);
 
+		let schema_version = VssSchemaVersion::V0;
+		let (data_encryption_key, obfuscation_master_key) =
+			derive_data_encryption_and_obfuscation_keys(&vss_seed);
+		let key_obfuscator = KeyObfuscator::new(obfuscation_master_key);
+		let retry_policy = ExponentialBackoffRetryPolicy::new(Duration::from_millis(10))
+			.with_max_attempts(10)
+			.with_max_total_delay(Duration::from_secs(15))
+			.with_max_jitter(Duration::from_millis(10))
+			.skip_retry_on_error(Box::new(|e: &VssError| {
+				matches!(
+					e,
+					VssError::NoSuchKeyError(..)
+						| VssError::InvalidRequestError(..)
+						| VssError::ConflictError(..)
+				)
+			}) as _);
+
+		let client = VssClient::new_with_headers(base_url, retry_policy, header_provider);
+
+		let inner = Arc::new(VssStoreInner::new(
+			schema_version,
+			client,
+			store_id,
+			data_encryption_key,
+			key_obfuscator,
+		));
+
 		Self { inner, next_version, runtime, internal_runtime }
 	}
 
@@ -337,27 +363,9 @@ struct VssStoreInner {
 
 impl VssStoreInner {
 	pub(crate) fn new(
-		base_url: String, store_id: String, vss_seed: [u8; 32],
-		header_provider: Arc<dyn VssHeaderProvider>,
+		schema_version: VssSchemaVersion, client: VssClient<CustomRetryPolicy>, store_id: String,
+		data_encryption_key: [u8; 32], key_obfuscator: KeyObfuscator,
 	) -> Self {
-		let schema_version = VssSchemaVersion::V0;
-		let (data_encryption_key, obfuscation_master_key) =
-			derive_data_encryption_and_obfuscation_keys(&vss_seed);
-		let key_obfuscator = KeyObfuscator::new(obfuscation_master_key);
-		let retry_policy = ExponentialBackoffRetryPolicy::new(Duration::from_millis(10))
-			.with_max_attempts(10)
-			.with_max_total_delay(Duration::from_secs(15))
-			.with_max_jitter(Duration::from_millis(10))
-			.skip_retry_on_error(Box::new(|e: &VssError| {
-				matches!(
-					e,
-					VssError::NoSuchKeyError(..)
-						| VssError::InvalidRequestError(..)
-						| VssError::ConflictError(..)
-				)
-			}) as _);
-
-		let client = VssClient::new_with_headers(base_url, retry_policy, header_provider);
 		let locks = Mutex::new(HashMap::new());
 		Self { schema_version, client, store_id, data_encryption_key, key_obfuscator, locks }
 	}

From 27663bf73ce3680c7ff7f9797b8d469f5ec7ec7b Mon Sep 17 00:00:00 2001
From: Elias Rohrer <dev@tnull.de>
Date: Thu, 6 Nov 2025 12:22:17 +0100
Subject: [PATCH 04/11] Account for `vss-client` being renamed to
 `vss-client-ng`

---
 Cargo.toml          |  4 ++--
 src/builder.rs      |  2 +-
 src/ffi/types.rs    |  2 +-
 src/io/vss_store.rs | 16 ++++++++--------
 src/lib.rs          |  2 +-
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index a2634e330..5d605c983 100755
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -65,8 +65,8 @@ serde = { version = "1.0.210", default-features = false, features = ["std", "der
 serde_json = { version = "1.0.128", default-features = false, features = ["std"] }
 log = { version = "0.4.22", default-features = false, features = ["std"]}
 
-#vss-client = "0.3"
-vss-client = { git = "https://github.com/tnull/vss-rust-client", rev = "03ca9d99f70387aabec225020e46434cda8d18ff" }
+#vss-client-ng = "0.3"
+vss-client-ng = { git = "https://github.com/tnull/vss-client", rev = "98ac5e171ef1ab970bbc58cd995ffc1e615421d1" }
 prost = { version = "0.11.6", default-features = false}
 
 [target.'cfg(windows)'.dependencies]
diff --git a/src/builder.rs b/src/builder.rs
index c0e39af7a..bed325db6 100644
--- a/src/builder.rs
+++ b/src/builder.rs
@@ -39,7 +39,7 @@ use lightning::util::persist::{
 use lightning::util::ser::ReadableArgs;
 use lightning::util::sweep::OutputSweeper;
 use lightning_persister::fs_store::FilesystemStore;
-use vss_client::headers::{FixedHeaders, LnurlAuthToJwtProvider, VssHeaderProvider};
+use vss_client_ng::headers::{FixedHeaders, LnurlAuthToJwtProvider, VssHeaderProvider};
 
 use crate::chain::ChainSource;
 use crate::config::{
diff --git a/src/ffi/types.rs b/src/ffi/types.rs
index 3c88a665f..e99cdc230 100644
--- a/src/ffi/types.rs
+++ b/src/ffi/types.rs
@@ -40,7 +40,7 @@ pub use lightning_liquidity::lsps1::msgs::{
 };
 pub use lightning_types::payment::{PaymentHash, PaymentPreimage, PaymentSecret};
 pub use lightning_types::string::UntrustedString;
-pub use vss_client::headers::{VssHeaderProvider, VssHeaderProviderError};
+pub use vss_client_ng::headers::{VssHeaderProvider, VssHeaderProviderError};
 
 use crate::builder::sanitize_alias;
 pub use crate::config::{
diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs
index c7b290dad..498ca5ec3 100644
--- a/src/io/vss_store.rs
+++ b/src/io/vss_store.rs
@@ -20,19 +20,19 @@ use lightning::io::{self, Error, ErrorKind};
 use lightning::util::persist::{KVStore, KVStoreSync};
 use prost::Message;
 use rand::RngCore;
-use vss_client::client::VssClient;
-use vss_client::error::VssError;
-use vss_client::headers::VssHeaderProvider;
-use vss_client::types::{
+use vss_client_ng::client::VssClient;
+use vss_client_ng::error::VssError;
+use vss_client_ng::headers::VssHeaderProvider;
+use vss_client_ng::types::{
 	DeleteObjectRequest, GetObjectRequest, KeyValue, ListKeyVersionsRequest, PutObjectRequest,
 	Storable,
 };
-use vss_client::util::key_obfuscator::KeyObfuscator;
-use vss_client::util::retry::{
+use vss_client_ng::util::key_obfuscator::KeyObfuscator;
+use vss_client_ng::util::retry::{
 	ExponentialBackoffRetryPolicy, FilteredRetryPolicy, JitteredRetryPolicy,
 	MaxAttemptsRetryPolicy, MaxTotalDelayRetryPolicy, RetryPolicy,
 };
-use vss_client::util::storable_builder::{EntropySource, StorableBuilder};
+use vss_client_ng::util::storable_builder::{EntropySource, StorableBuilder};
 
 use crate::io::utils::check_namespace_key_validity;
 use crate::runtime::Runtime;
@@ -656,7 +656,7 @@ mod tests {
 
 	use rand::distr::Alphanumeric;
 	use rand::{rng, Rng, RngCore};
-	use vss_client::headers::FixedHeaders;
+	use vss_client_ng::headers::FixedHeaders;
 
 	use super::*;
 	use crate::io::test_utils::do_read_write_remove_list_persist;
diff --git a/src/lib.rs b/src/lib.rs
index 701a14dde..3a4c1df9e 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -159,7 +159,7 @@ pub use types::{
 };
 pub use {
 	bip39, bitcoin, lightning, lightning_invoice, lightning_liquidity, lightning_types, tokio,
-	vss_client,
+	vss_client_ng,
 };
 
 use crate::scoring::setup_background_pathfinding_scores_sync;

From cf84c0e0bd0821769fcd62f1b7d2efe9c64cd645 Mon Sep 17 00:00:00 2001
From: Elias Rohrer <dev@tnull.de>
Date: Mon, 3 Nov 2025 13:15:36 +0100
Subject: [PATCH 05/11] Only use internal runtime in `VssStore`

We previously attempted to drop the internal runtime from `VssStore`,
resulting into blocking behavior. While we recently made changes that
improved our situation (having VSS CI pass again pretty reliably), we
just ran into yet another case where the VSS CI hung (cf.
https://github.com/lightningdevkit/vss-server/actions/runs/19023212819/job/54322173817?pr=59).

Here we attempt to restore even more of the original pre-
ab3d78d1ecd05a755c836915284e5ca60c65692a / #623 behavior to get rid of
the reappearing blocking behavior, i.e., only use the internal runtime
in `VssStore`.
---
 src/builder.rs      |  3 +-
 src/io/vss_store.rs | 78 +++++++++++++++++++++------------------------
 2 files changed, 37 insertions(+), 44 deletions(-)

diff --git a/src/builder.rs b/src/builder.rs
index bed325db6..9382d2ccc 100644
--- a/src/builder.rs
+++ b/src/builder.rs
@@ -731,8 +731,7 @@ impl NodeBuilder {
 
 		let vss_seed_bytes: [u8; 32] = vss_xprv.private_key.secret_bytes();
 
-		let vss_store =
-			VssStore::new(vss_url, store_id, vss_seed_bytes, header_provider, Arc::clone(&runtime));
+		let vss_store = VssStore::new(vss_url, store_id, vss_seed_bytes, header_provider);
 		build_with_store_internal(
 			config,
 			self.chain_data_source_config.as_ref(),
diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs
index 498ca5ec3..97f6ff9c7 100644
--- a/src/io/vss_store.rs
+++ b/src/io/vss_store.rs
@@ -35,7 +35,6 @@ use vss_client_ng::util::retry::{
 use vss_client_ng::util::storable_builder::{EntropySource, StorableBuilder};
 
 use crate::io::utils::check_namespace_key_validity;
-use crate::runtime::Runtime;
 
 type CustomRetryPolicy = FilteredRetryPolicy<
 	JitteredRetryPolicy<
@@ -66,7 +65,6 @@ pub struct VssStore {
 	// Version counter to ensure that writes are applied in the correct order. It is assumed that read and list
 	// operations aren't sensitive to the order of execution.
 	next_version: AtomicU64,
-	runtime: Arc<Runtime>,
 	// A VSS-internal runtime we use to avoid any deadlocks we could hit when waiting on a spawned
 	// blocking task to finish while the blocked thread had acquired the reactor. In particular,
 	// this works around a previously-hit case where a concurrent call to
@@ -79,7 +77,7 @@ pub struct VssStore {
 impl VssStore {
 	pub(crate) fn new(
 		base_url: String, store_id: String, vss_seed: [u8; 32],
-		header_provider: Arc<dyn VssHeaderProvider>, runtime: Arc<Runtime>,
+		header_provider: Arc<dyn VssHeaderProvider>,
 	) -> Self {
 		let next_version = AtomicU64::new(1);
 		let internal_runtime = Some(
@@ -123,7 +121,7 @@ impl VssStore {
 			key_obfuscator,
 		));
 
-		Self { inner, next_version, runtime, internal_runtime }
+		Self { inner, next_version, internal_runtime }
 	}
 
 	// Same logic as for the obfuscated keys below, but just for locking, using the plaintext keys
@@ -170,13 +168,14 @@ impl KVStoreSync for VssStore {
 			async move { inner.read_internal(primary_namespace, secondary_namespace, key).await };
 		// TODO: We could drop the timeout here once we ensured vss-client's Retry logic always
 		// times out.
-		let spawned_fut = internal_runtime.spawn(async move {
-			tokio::time::timeout(VSS_IO_TIMEOUT, fut).await.map_err(|_| {
-				let msg = "VssStore::read timed out";
-				Error::new(ErrorKind::Other, msg)
-			})
-		});
-		self.runtime.block_on(spawned_fut).expect("We should always finish")?
+		tokio::task::block_in_place(move || {
+			internal_runtime.block_on(async move {
+				tokio::time::timeout(VSS_IO_TIMEOUT, fut).await.map_err(|_| {
+					let msg = "VssStore::read timed out";
+					Error::new(ErrorKind::Other, msg)
+				})
+			})?
+		})
 	}
 
 	fn write(
@@ -208,13 +207,14 @@ impl KVStoreSync for VssStore {
 		};
 		// TODO: We could drop the timeout here once we ensured vss-client's Retry logic always
 		// times out.
-		let spawned_fut = internal_runtime.spawn(async move {
-			tokio::time::timeout(VSS_IO_TIMEOUT, fut).await.map_err(|_| {
-				let msg = "VssStore::write timed out";
-				Error::new(ErrorKind::Other, msg)
-			})
-		});
-		self.runtime.block_on(spawned_fut).expect("We should always finish")?
+		tokio::task::block_in_place(move || {
+			internal_runtime.block_on(async move {
+				tokio::time::timeout(VSS_IO_TIMEOUT, fut).await.map_err(|_| {
+					let msg = "VssStore::write timed out";
+					Error::new(ErrorKind::Other, msg)
+				})
+			})?
+		})
 	}
 
 	fn remove(
@@ -245,13 +245,14 @@ impl KVStoreSync for VssStore {
 		};
 		// TODO: We could drop the timeout here once we ensured vss-client's Retry logic always
 		// times out.
-		let spawned_fut = internal_runtime.spawn(async move {
-			tokio::time::timeout(VSS_IO_TIMEOUT, fut).await.map_err(|_| {
-				let msg = "VssStore::remove timed out";
-				Error::new(ErrorKind::Other, msg)
-			})
-		});
-		self.runtime.block_on(spawned_fut).expect("We should always finish")?
+		tokio::task::block_in_place(move || {
+			internal_runtime.block_on(async move {
+				tokio::time::timeout(VSS_IO_TIMEOUT, fut).await.map_err(|_| {
+					let msg = "VssStore::remove timed out";
+					Error::new(ErrorKind::Other, msg)
+				})
+			})?
+		})
 	}
 
 	fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result<Vec<String>> {
@@ -266,13 +267,14 @@ impl KVStoreSync for VssStore {
 		let fut = async move { inner.list_internal(primary_namespace, secondary_namespace).await };
 		// TODO: We could drop the timeout here once we ensured vss-client's Retry logic always
 		// times out.
-		let spawned_fut = internal_runtime.spawn(async move {
-			tokio::time::timeout(VSS_IO_TIMEOUT, fut).await.map_err(|_| {
-				let msg = "VssStore::list timed out";
-				Error::new(ErrorKind::Other, msg)
-			})
-		});
-		self.runtime.block_on(spawned_fut).expect("We should always finish")?
+		tokio::task::block_in_place(move || {
+			internal_runtime.block_on(async move {
+				tokio::time::timeout(VSS_IO_TIMEOUT, fut).await.map_err(|_| {
+					let msg = "VssStore::list timed out";
+					Error::new(ErrorKind::Other, msg)
+				})
+			})?
+		})
 	}
 }
 
@@ -660,7 +662,6 @@ mod tests {
 
 	use super::*;
 	use crate::io::test_utils::do_read_write_remove_list_persist;
-	use crate::logger::Logger;
 
 	#[test]
 	fn vss_read_write_remove_list_persist() {
@@ -670,11 +671,7 @@ mod tests {
 		let mut vss_seed = [0u8; 32];
 		rng.fill_bytes(&mut vss_seed);
 		let header_provider = Arc::new(FixedHeaders::new(HashMap::new()));
-		let logger = Arc::new(Logger::new_log_facade());
-		let runtime = Arc::new(Runtime::new(logger).unwrap());
-		let vss_store =
-			VssStore::new(vss_base_url, rand_store_id, vss_seed, header_provider, runtime);
-
+		let vss_store = VssStore::new(vss_base_url, rand_store_id, vss_seed, header_provider);
 		do_read_write_remove_list_persist(&vss_store);
 	}
 
@@ -686,10 +683,7 @@ mod tests {
 		let mut vss_seed = [0u8; 32];
 		rng.fill_bytes(&mut vss_seed);
 		let header_provider = Arc::new(FixedHeaders::new(HashMap::new()));
-		let logger = Arc::new(Logger::new_log_facade());
-		let runtime = Arc::new(Runtime::new(logger).unwrap());
-		let vss_store =
-			VssStore::new(vss_base_url, rand_store_id, vss_seed, header_provider, runtime);
+		let vss_store = VssStore::new(vss_base_url, rand_store_id, vss_seed, header_provider);
 
 		do_read_write_remove_list_persist(&vss_store);
 		drop(vss_store)

From 53532d009917c92175f224a01333c65ec21ed3db Mon Sep 17 00:00:00 2001
From: Elias Rohrer <dev@tnull.de>
Date: Thu, 6 Nov 2025 12:39:51 +0100
Subject: [PATCH 06/11] Drop redundant `tokio::timeout`s for VSS IO

Now that we rely on `reqwest` v0.12.* retry logic as well as client-side
timeouts, we can address the remaining TODOs here and simply drop the
redundant `tokio::timeout`s we previously added as a safeguard to
blocking tasks (even though in the worst cases we saw they never
actually fired).
---
 src/io/vss_store.rs | 45 ++++-----------------------------------------
 1 file changed, 4 insertions(+), 41 deletions(-)

diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs
index 97f6ff9c7..b0a3ca253 100644
--- a/src/io/vss_store.rs
+++ b/src/io/vss_store.rs
@@ -57,7 +57,6 @@ enum VssSchemaVersion {
 // We set this to a small number of threads that would still allow to make some progress if one
 // would hit a blocking case
 const INTERNAL_RUNTIME_WORKERS: usize = 2;
-const VSS_IO_TIMEOUT: Duration = Duration::from_secs(5);
 
 /// A [`KVStoreSync`] implementation that writes to and reads from a [VSS](https://github.com/lightningdevkit/vss-server/blob/main/README.md) backend.
 pub struct VssStore {
@@ -166,16 +165,7 @@ impl KVStoreSync for VssStore {
 		let inner = Arc::clone(&self.inner);
 		let fut =
 			async move { inner.read_internal(primary_namespace, secondary_namespace, key).await };
-		// TODO: We could drop the timeout here once we ensured vss-client's Retry logic always
-		// times out.
-		tokio::task::block_in_place(move || {
-			internal_runtime.block_on(async move {
-				tokio::time::timeout(VSS_IO_TIMEOUT, fut).await.map_err(|_| {
-					let msg = "VssStore::read timed out";
-					Error::new(ErrorKind::Other, msg)
-				})
-			})?
-		})
+		tokio::task::block_in_place(move || internal_runtime.block_on(fut))
 	}
 
 	fn write(
@@ -205,16 +195,7 @@ impl KVStoreSync for VssStore {
 				)
 				.await
 		};
-		// TODO: We could drop the timeout here once we ensured vss-client's Retry logic always
-		// times out.
-		tokio::task::block_in_place(move || {
-			internal_runtime.block_on(async move {
-				tokio::time::timeout(VSS_IO_TIMEOUT, fut).await.map_err(|_| {
-					let msg = "VssStore::write timed out";
-					Error::new(ErrorKind::Other, msg)
-				})
-			})?
-		})
+		tokio::task::block_in_place(move || internal_runtime.block_on(fut))
 	}
 
 	fn remove(
@@ -243,16 +224,7 @@ impl KVStoreSync for VssStore {
 				)
 				.await
 		};
-		// TODO: We could drop the timeout here once we ensured vss-client's Retry logic always
-		// times out.
-		tokio::task::block_in_place(move || {
-			internal_runtime.block_on(async move {
-				tokio::time::timeout(VSS_IO_TIMEOUT, fut).await.map_err(|_| {
-					let msg = "VssStore::remove timed out";
-					Error::new(ErrorKind::Other, msg)
-				})
-			})?
-		})
+		tokio::task::block_in_place(move || internal_runtime.block_on(fut))
 	}
 
 	fn list(&self, primary_namespace: &str, secondary_namespace: &str) -> io::Result<Vec<String>> {
@@ -265,16 +237,7 @@ impl KVStoreSync for VssStore {
 		let secondary_namespace = secondary_namespace.to_string();
 		let inner = Arc::clone(&self.inner);
 		let fut = async move { inner.list_internal(primary_namespace, secondary_namespace).await };
-		// TODO: We could drop the timeout here once we ensured vss-client's Retry logic always
-		// times out.
-		tokio::task::block_in_place(move || {
-			internal_runtime.block_on(async move {
-				tokio::time::timeout(VSS_IO_TIMEOUT, fut).await.map_err(|_| {
-					let msg = "VssStore::list timed out";
-					Error::new(ErrorKind::Other, msg)
-				})
-			})?
-		})
+		tokio::task::block_in_place(move || internal_runtime.block_on(fut))
 	}
 }
 

From 2e7d7106325452c4442ee6612cf9251e998eb480 Mon Sep 17 00:00:00 2001
From: Elias Rohrer <dev@tnull.de>
Date: Mon, 10 Nov 2025 16:31:05 +0100
Subject: [PATCH 07/11] Bump retries and timeouts considerably

---
 src/io/vss_store.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs
index b0a3ca253..5614140c3 100644
--- a/src/io/vss_store.rs
+++ b/src/io/vss_store.rs
@@ -98,9 +98,9 @@ impl VssStore {
 			derive_data_encryption_and_obfuscation_keys(&vss_seed);
 		let key_obfuscator = KeyObfuscator::new(obfuscation_master_key);
 		let retry_policy = ExponentialBackoffRetryPolicy::new(Duration::from_millis(10))
-			.with_max_attempts(10)
-			.with_max_total_delay(Duration::from_secs(15))
-			.with_max_jitter(Duration::from_millis(10))
+			.with_max_attempts(100)
+			.with_max_total_delay(Duration::from_secs(60))
+			.with_max_jitter(Duration::from_millis(50))
 			.skip_retry_on_error(Box::new(|e: &VssError| {
 				matches!(
 					e,

From b4254e36ec1ae27179ca22975ca2b357846706e7 Mon Sep 17 00:00:00 2001
From: Elias Rohrer <dev@tnull.de>
Date: Mon, 10 Nov 2025 16:44:26 +0100
Subject: [PATCH 08/11] Introduce two separate `VssClient`s for async/blocking
 contexts

To avoid any blocking cross-runtime behavior that could arise from
reusing a single client's TCP connections in different runtime contexts,
we here split out the `VssStore` behavior to use one dedicated
`VssClient` per context. I.e., we're now using two
connections/connection pools and make sure only the `blocking_client` is
used in `KVStoreSync` contexts, and `async_client` in `KVStore`
contexts.
---
 src/io/vss_store.rs | 121 ++++++++++++++++++++++++++++++--------------
 1 file changed, 84 insertions(+), 37 deletions(-)

diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs
index 5614140c3..3c0d425d7 100644
--- a/src/io/vss_store.rs
+++ b/src/io/vss_store.rs
@@ -97,24 +97,22 @@ impl VssStore {
 		let (data_encryption_key, obfuscation_master_key) =
 			derive_data_encryption_and_obfuscation_keys(&vss_seed);
 		let key_obfuscator = KeyObfuscator::new(obfuscation_master_key);
-		let retry_policy = ExponentialBackoffRetryPolicy::new(Duration::from_millis(10))
-			.with_max_attempts(100)
-			.with_max_total_delay(Duration::from_secs(60))
-			.with_max_jitter(Duration::from_millis(50))
-			.skip_retry_on_error(Box::new(|e: &VssError| {
-				matches!(
-					e,
-					VssError::NoSuchKeyError(..)
-						| VssError::InvalidRequestError(..)
-						| VssError::ConflictError(..)
-				)
-			}) as _);
 
-		let client = VssClient::new_with_headers(base_url, retry_policy, header_provider);
+		let sync_retry_policy = retry_policy();
+		let blocking_client = VssClient::new_with_headers(
+			base_url.clone(),
+			sync_retry_policy,
+			header_provider.clone(),
+		);
+
+		let async_retry_policy = retry_policy();
+		let async_client =
+			VssClient::new_with_headers(base_url, async_retry_policy, header_provider);
 
 		let inner = Arc::new(VssStoreInner::new(
 			schema_version,
-			client,
+			blocking_client,
+			async_client,
 			store_id,
 			data_encryption_key,
 			key_obfuscator,
@@ -163,8 +161,11 @@ impl KVStoreSync for VssStore {
 		let secondary_namespace = secondary_namespace.to_string();
 		let key = key.to_string();
 		let inner = Arc::clone(&self.inner);
-		let fut =
-			async move { inner.read_internal(primary_namespace, secondary_namespace, key).await };
+		let fut = async move {
+			inner
+				.read_internal(&inner.blocking_client, primary_namespace, secondary_namespace, key)
+				.await
+		};
 		tokio::task::block_in_place(move || internal_runtime.block_on(fut))
 	}
 
@@ -185,6 +186,7 @@ impl KVStoreSync for VssStore {
 		let fut = async move {
 			inner
 				.write_internal(
+					&inner.blocking_client,
 					inner_lock_ref,
 					locking_key,
 					version,
@@ -215,6 +217,7 @@ impl KVStoreSync for VssStore {
 		let fut = async move {
 			inner
 				.remove_internal(
+					&inner.blocking_client,
 					inner_lock_ref,
 					locking_key,
 					version,
@@ -236,7 +239,11 @@ impl KVStoreSync for VssStore {
 		let primary_namespace = primary_namespace.to_string();
 		let secondary_namespace = secondary_namespace.to_string();
 		let inner = Arc::clone(&self.inner);
-		let fut = async move { inner.list_internal(primary_namespace, secondary_namespace).await };
+		let fut = async move {
+			inner
+				.list_internal(&inner.blocking_client, primary_namespace, secondary_namespace)
+				.await
+		};
 		tokio::task::block_in_place(move || internal_runtime.block_on(fut))
 	}
 }
@@ -249,9 +256,11 @@ impl KVStore for VssStore {
 		let secondary_namespace = secondary_namespace.to_string();
 		let key = key.to_string();
 		let inner = Arc::clone(&self.inner);
-		Box::pin(
-			async move { inner.read_internal(primary_namespace, secondary_namespace, key).await },
-		)
+		Box::pin(async move {
+			inner
+				.read_internal(&inner.async_client, primary_namespace, secondary_namespace, key)
+				.await
+		})
 	}
 	fn write(
 		&self, primary_namespace: &str, secondary_namespace: &str, key: &str, buf: Vec<u8>,
@@ -265,6 +274,7 @@ impl KVStore for VssStore {
 		Box::pin(async move {
 			inner
 				.write_internal(
+					&inner.async_client,
 					inner_lock_ref,
 					locking_key,
 					version,
@@ -288,6 +298,7 @@ impl KVStore for VssStore {
 		Box::pin(async move {
 			inner
 				.remove_internal(
+					&inner.async_client,
 					inner_lock_ref,
 					locking_key,
 					version,
@@ -304,7 +315,9 @@ impl KVStore for VssStore {
 		let primary_namespace = primary_namespace.to_string();
 		let secondary_namespace = secondary_namespace.to_string();
 		let inner = Arc::clone(&self.inner);
-		Box::pin(async move { inner.list_internal(primary_namespace, secondary_namespace).await })
+		Box::pin(async move {
+			inner.list_internal(&inner.async_client, primary_namespace, secondary_namespace).await
+		})
 	}
 }
 
@@ -317,7 +330,10 @@ impl Drop for VssStore {
 
 struct VssStoreInner {
 	schema_version: VssSchemaVersion,
-	client: VssClient<CustomRetryPolicy>,
+	blocking_client: VssClient<CustomRetryPolicy>,
+	// A secondary client that will only be used for async persistence via `KVStore`, to ensure TCP
+	// connections aren't shared between our outer and the internal runtime.
+	async_client: VssClient<CustomRetryPolicy>,
 	store_id: String,
 	data_encryption_key: [u8; 32],
 	key_obfuscator: KeyObfuscator,
@@ -328,11 +344,20 @@ struct VssStoreInner {
 
 impl VssStoreInner {
 	pub(crate) fn new(
-		schema_version: VssSchemaVersion, client: VssClient<CustomRetryPolicy>, store_id: String,
+		schema_version: VssSchemaVersion, blocking_client: VssClient<CustomRetryPolicy>,
+		async_client: VssClient<CustomRetryPolicy>, store_id: String,
 		data_encryption_key: [u8; 32], key_obfuscator: KeyObfuscator,
 	) -> Self {
 		let locks = Mutex::new(HashMap::new());
-		Self { schema_version, client, store_id, data_encryption_key, key_obfuscator, locks }
+		Self {
+			schema_version,
+			blocking_client,
+			async_client,
+			store_id,
+			data_encryption_key,
+			key_obfuscator,
+			locks,
+		}
 	}
 
 	fn get_inner_lock_ref(&self, locking_key: String) -> Arc<tokio::sync::Mutex<u64>> {
@@ -388,7 +413,8 @@ impl VssStoreInner {
 	}
 
 	async fn list_all_keys(
-		&self, primary_namespace: &str, secondary_namespace: &str,
+		&self, client: &VssClient<CustomRetryPolicy>, primary_namespace: &str,
+		secondary_namespace: &str,
 	) -> io::Result<Vec<String>> {
 		let mut page_token = None;
 		let mut keys = vec![];
@@ -401,7 +427,7 @@ impl VssStoreInner {
 				page_size: None,
 			};
 
-			let response = self.client.list_key_versions(&request).await.map_err(|e| {
+			let response = client.list_key_versions(&request).await.map_err(|e| {
 				let msg = format!(
 					"Failed to list keys in {}/{}: {}",
 					primary_namespace, secondary_namespace, e
@@ -418,13 +444,14 @@ impl VssStoreInner {
 	}
 
 	async fn read_internal(
-		&self, primary_namespace: String, secondary_namespace: String, key: String,
+		&self, client: &VssClient<CustomRetryPolicy>, primary_namespace: String,
+		secondary_namespace: String, key: String,
 	) -> io::Result<Vec<u8>> {
 		check_namespace_key_validity(&primary_namespace, &secondary_namespace, Some(&key), "read")?;
 
 		let store_key = self.build_obfuscated_key(&primary_namespace, &secondary_namespace, &key);
 		let request = GetObjectRequest { store_id: self.store_id.clone(), key: store_key.clone() };
-		let resp = self.client.get_object(&request).await.map_err(|e| {
+		let resp = client.get_object(&request).await.map_err(|e| {
 			let msg = format!(
 				"Failed to read from key {}/{}/{}: {}",
 				primary_namespace, secondary_namespace, key, e
@@ -453,8 +480,9 @@ impl VssStoreInner {
 	}
 
 	async fn write_internal(
-		&self, inner_lock_ref: Arc<tokio::sync::Mutex<u64>>, locking_key: String, version: u64,
-		primary_namespace: String, secondary_namespace: String, key: String, buf: Vec<u8>,
+		&self, client: &VssClient<CustomRetryPolicy>, inner_lock_ref: Arc<tokio::sync::Mutex<u64>>,
+		locking_key: String, version: u64, primary_namespace: String, secondary_namespace: String,
+		key: String, buf: Vec<u8>,
 	) -> io::Result<()> {
 		check_namespace_key_validity(
 			&primary_namespace,
@@ -482,7 +510,7 @@ impl VssStoreInner {
 		};
 
 		self.execute_locked_write(inner_lock_ref, locking_key, version, async move || {
-			self.client.put_object(&request).await.map_err(|e| {
+			client.put_object(&request).await.map_err(|e| {
 				let msg = format!(
 					"Failed to write to key {}/{}/{}: {}",
 					primary_namespace, secondary_namespace, key, e
@@ -496,8 +524,9 @@ impl VssStoreInner {
 	}
 
 	async fn remove_internal(
-		&self, inner_lock_ref: Arc<tokio::sync::Mutex<u64>>, locking_key: String, version: u64,
-		primary_namespace: String, secondary_namespace: String, key: String,
+		&self, client: &VssClient<CustomRetryPolicy>, inner_lock_ref: Arc<tokio::sync::Mutex<u64>>,
+		locking_key: String, version: u64, primary_namespace: String, secondary_namespace: String,
+		key: String,
 	) -> io::Result<()> {
 		check_namespace_key_validity(
 			&primary_namespace,
@@ -514,7 +543,7 @@ impl VssStoreInner {
 				key_value: Some(KeyValue { key: obfuscated_key, version: -1, value: vec![] }),
 			};
 
-			self.client.delete_object(&request).await.map_err(|e| {
+			client.delete_object(&request).await.map_err(|e| {
 				let msg = format!(
 					"Failed to delete key {}/{}/{}: {}",
 					primary_namespace, secondary_namespace, key, e
@@ -528,12 +557,15 @@ impl VssStoreInner {
 	}
 
 	async fn list_internal(
-		&self, primary_namespace: String, secondary_namespace: String,
+		&self, client: &VssClient<CustomRetryPolicy>, primary_namespace: String,
+		secondary_namespace: String,
 	) -> io::Result<Vec<String>> {
 		check_namespace_key_validity(&primary_namespace, &secondary_namespace, None, "list")?;
 
-		let keys =
-			self.list_all_keys(&primary_namespace, &secondary_namespace).await.map_err(|e| {
+		let keys = self
+			.list_all_keys(client, &primary_namespace, &secondary_namespace)
+			.await
+			.map_err(|e| {
 				let msg = format!(
 					"Failed to retrieve keys in namespace: {}/{} : {}",
 					primary_namespace, secondary_namespace, e
@@ -602,6 +634,21 @@ fn derive_data_encryption_and_obfuscation_keys(vss_seed: &[u8; 32]) -> ([u8; 32]
 	(k1, k2)
 }
 
+fn retry_policy() -> CustomRetryPolicy {
+	ExponentialBackoffRetryPolicy::new(Duration::from_millis(10))
+		.with_max_attempts(100)
+		.with_max_total_delay(Duration::from_secs(60))
+		.with_max_jitter(Duration::from_millis(50))
+		.skip_retry_on_error(Box::new(|e: &VssError| {
+			matches!(
+				e,
+				VssError::NoSuchKeyError(..)
+					| VssError::InvalidRequestError(..)
+					| VssError::ConflictError(..)
+			)
+		}) as _)
+}
+
 /// A source for generating entropy/randomness using [`rand`].
 pub(crate) struct RandEntropySource;
 

From 448616ed08786b95017b247f94b6139994dcb46b Mon Sep 17 00:00:00 2001
From: Elias Rohrer <dev@tnull.de>
Date: Wed, 12 Nov 2025 09:47:27 +0100
Subject: [PATCH 09/11] Bump `vss-client-ng` dependency to `v0.4.0`

We bump our `vss-client-ng` dependency to the latest release.
---
 Cargo.toml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 5d605c983..d02b88bbf 100755
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -65,8 +65,7 @@ serde = { version = "1.0.210", default-features = false, features = ["std", "der
 serde_json = { version = "1.0.128", default-features = false, features = ["std"] }
 log = { version = "0.4.22", default-features = false, features = ["std"]}
 
-#vss-client-ng = "0.3"
-vss-client-ng = { git = "https://github.com/tnull/vss-client", rev = "98ac5e171ef1ab970bbc58cd995ffc1e615421d1" }
+vss-client-ng = "0.4.0"
 prost = { version = "0.11.6", default-features = false}
 
 [target.'cfg(windows)'.dependencies]
@@ -152,3 +151,6 @@ harness = false
 #lightning-transaction-sync = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "21e9a9c0ef80021d0669f2a366f55d08ba8d9b03" }
 #lightning-liquidity = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "21e9a9c0ef80021d0669f2a366f55d08ba8d9b03" }
 #lightning-macros = { git = "https://github.com/lightningdevkit/rust-lightning", rev = "21e9a9c0ef80021d0669f2a366f55d08ba8d9b03" }
+
+#vss-client-ng = { path = "../vss-client" }
+#vss-client-ng = { git = "https://github.com/lightningdevkit/vss-client", branch = "main" }

From 85fd473c51acf7cc385a0cab219462b5e72f6538 Mon Sep 17 00:00:00 2001
From: Elias Rohrer <dev@tnull.de>
Date: Wed, 12 Nov 2025 11:31:12 +0100
Subject: [PATCH 10/11] Determine VSS schema version at startup

Since we just made some breaking changes to how exactly we persist data
via VSS (now using an `aad` that commits to the key and also obfuscating
namespaces), we have to detect which schema version we're on to ensure
backwards compatibility.

To this end, we here start reading a persisted `vss_schema_version` key
in `VssStore::new`. If it is present, we just return the encoded value
(right now that can only be V1). If it is not present, it can either
mean we run for the first time *or* we're on V0, which we determine
checking if anything related to the `bdk_wallet` descriptors are present
in the store. If we're running for the first time, we also persist the
schema version to save us these rather inefficient steps on following
startups.
---
 src/builder.rs      |   7 +-
 src/io/vss_store.rs | 171 +++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 159 insertions(+), 19 deletions(-)

diff --git a/src/builder.rs b/src/builder.rs
index 9382d2ccc..a15697c91 100644
--- a/src/builder.rs
+++ b/src/builder.rs
@@ -731,7 +731,12 @@ impl NodeBuilder {
 
 		let vss_seed_bytes: [u8; 32] = vss_xprv.private_key.secret_bytes();
 
-		let vss_store = VssStore::new(vss_url, store_id, vss_seed_bytes, header_provider);
+		let vss_store =
+			VssStore::new(vss_url, store_id, vss_seed_bytes, header_provider).map_err(|e| {
+				log_error!(logger, "Failed to setup VSS store: {}", e);
+				BuildError::KVStoreSetupFailed
+			})?;
+
 		build_with_store_internal(
 			config,
 			self.chain_data_source_config.as_ref(),
diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs
index 3c0d425d7..73c49b779 100644
--- a/src/io/vss_store.rs
+++ b/src/io/vss_store.rs
@@ -16,8 +16,10 @@ use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
 use bitcoin::hashes::{sha256, Hash, HashEngine, Hmac, HmacEngine};
+use lightning::impl_writeable_tlv_based_enum;
 use lightning::io::{self, Error, ErrorKind};
 use lightning::util::persist::{KVStore, KVStoreSync};
+use lightning::util::ser::{Readable, Writeable};
 use prost::Message;
 use rand::RngCore;
 use vss_client_ng::client::VssClient;
@@ -54,6 +56,13 @@ enum VssSchemaVersion {
 	V1,
 }
 
+impl_writeable_tlv_based_enum!(VssSchemaVersion,
+	(0, V0) => {},
+	(1, V1) => {},
+);
+
+const VSS_SCHEMA_VERSION_KEY: &str = "vss_schema_version";
+
 // We set this to a small number of threads that would still allow to make some progress if one
 // would hit a blocking case
 const INTERNAL_RUNTIME_WORKERS: usize = 2;
@@ -77,23 +86,20 @@ impl VssStore {
 	pub(crate) fn new(
 		base_url: String, store_id: String, vss_seed: [u8; 32],
 		header_provider: Arc<dyn VssHeaderProvider>,
-	) -> Self {
+	) -> io::Result<Self> {
 		let next_version = AtomicU64::new(1);
-		let internal_runtime = Some(
-			tokio::runtime::Builder::new_multi_thread()
-				.enable_all()
-				.thread_name_fn(|| {
-					static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0);
-					let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst);
-					format!("ldk-node-vss-runtime-{}", id)
-				})
-				.worker_threads(INTERNAL_RUNTIME_WORKERS)
-				.max_blocking_threads(INTERNAL_RUNTIME_WORKERS)
-				.build()
-				.unwrap(),
-		);
+		let internal_runtime = tokio::runtime::Builder::new_multi_thread()
+			.enable_all()
+			.thread_name_fn(|| {
+				static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0);
+				let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst);
+				format!("ldk-node-vss-runtime-{}", id)
+			})
+			.worker_threads(INTERNAL_RUNTIME_WORKERS)
+			.max_blocking_threads(INTERNAL_RUNTIME_WORKERS)
+			.build()
+			.unwrap();
 
-		let schema_version = VssSchemaVersion::V0;
 		let (data_encryption_key, obfuscation_master_key) =
 			derive_data_encryption_and_obfuscation_keys(&vss_seed);
 		let key_obfuscator = KeyObfuscator::new(obfuscation_master_key);
@@ -105,6 +111,21 @@ impl VssStore {
 			header_provider.clone(),
 		);
 
+		let runtime_handle = internal_runtime.handle();
+		let schema_store_id = store_id.clone();
+		let schema_key_obfuscator = KeyObfuscator::new(obfuscation_master_key);
+		let (schema_version, blocking_client) = tokio::task::block_in_place(move || {
+			runtime_handle.block_on(async {
+				determine_and_write_schema_version(
+					blocking_client,
+					schema_store_id,
+					data_encryption_key,
+					schema_key_obfuscator,
+				)
+				.await
+			})
+		})?;
+
 		let async_retry_policy = retry_policy();
 		let async_client =
 			VssClient::new_with_headers(base_url, async_retry_policy, header_provider);
@@ -118,7 +139,7 @@ impl VssStore {
 			key_obfuscator,
 		));
 
-		Self { inner, next_version, internal_runtime }
+		Ok(Self { inner, next_version, internal_runtime: Some(internal_runtime) })
 	}
 
 	// Same logic as for the obfuscated keys below, but just for locking, using the plaintext keys
@@ -649,6 +670,118 @@ fn retry_policy() -> CustomRetryPolicy {
 		}) as _)
 }
 
+// FIXME: This returns the used client as currently `VssClient`'s `RetryPolicy`s aren't `Clone`. So
+// we're forced to take the owned client and return it to be able to reuse the same connection
+// later.
+async fn determine_and_write_schema_version(
+	client: VssClient<CustomRetryPolicy>, store_id: String, data_encryption_key: [u8; 32],
+	key_obfuscator: KeyObfuscator,
+) -> io::Result<(VssSchemaVersion, VssClient<CustomRetryPolicy>)> {
+	// Build the obfuscated `vss_schema_version` key.
+	let obfuscated_primary_namepace = key_obfuscator.obfuscate("");
+	let obfuscated_secondary_namepace = key_obfuscator.obfuscate("");
+	let obfuscated_key = key_obfuscator.obfuscate(VSS_SCHEMA_VERSION_KEY);
+	let store_key = format!(
+		"{}#{}#{}",
+		obfuscated_primary_namepace, obfuscated_secondary_namepace, obfuscated_key
+	);
+
+	// Try to read the stored schema version.
+	let request = GetObjectRequest { store_id: store_id.clone(), key: store_key.clone() };
+	let resp = match client.get_object(&request).await {
+		Ok(resp) => Some(resp),
+		Err(VssError::NoSuchKeyError(..)) => {
+			// The value is not set.
+			None
+		},
+		Err(e) => {
+			let msg = format!("Failed to read schema version: {}", e);
+			return Err(Error::new(ErrorKind::Other, msg));
+		},
+	};
+
+	if let Some(resp) = resp {
+		// The schema version was present, so just decrypt the stored data.
+
+		// unwrap safety: resp.value must be always present for a non-erroneous VSS response, otherwise
+		// it is an API-violation which is converted to [`VssError::InternalServerError`] in [`VssClient`]
+		let storable = Storable::decode(&resp.value.unwrap().value[..]).map_err(|e| {
+			let msg = format!("Failed to decode schema version: {}", e);
+			Error::new(ErrorKind::Other, msg)
+		})?;
+
+		let storable_builder = StorableBuilder::new(RandEntropySource);
+		// Schema version was added starting with V1, so if set at all, we use the key as `aad`
+		let aad = store_key.as_bytes();
+		let decrypted = storable_builder
+			.deconstruct(storable, &data_encryption_key, aad)
+			.map_err(|e| {
+				let msg = format!("Failed to decode schema version: {}", e);
+				Error::new(ErrorKind::Other, msg)
+			})?
+			.0;
+
+		let schema_version: VssSchemaVersion = Readable::read(&mut io::Cursor::new(decrypted))
+			.map_err(|e| {
+				let msg = format!("Failed to decode schema version: {}", e);
+				Error::new(ErrorKind::Other, msg)
+			})?;
+		Ok((schema_version, client))
+	} else {
+		// The schema version wasn't present, this either means we're running for the first time *or* it's V0 pre-migration (predating writing of the schema version).
+
+		// Check if any `bdk_wallet` data was written by listing keys under the respective
+		// (unobfuscated) prefix.
+		const V0_BDK_WALLET_PREFIX: &str = "bdk_wallet#";
+		let request = ListKeyVersionsRequest {
+			store_id: store_id.clone(),
+			key_prefix: Some(V0_BDK_WALLET_PREFIX.to_string()),
+			page_token: None,
+			page_size: None,
+		};
+
+		let response = client.list_key_versions(&request).await.map_err(|e| {
+			let msg = format!("Failed to determine schema version: {}", e);
+			Error::new(ErrorKind::Other, msg)
+		})?;
+
+		let wallet_data_present = !response.key_versions.is_empty();
+		if wallet_data_present {
+			// If the wallet data is present, it means we're not running for the first time.
+			Ok((VssSchemaVersion::V0, client))
+		} else {
+			// We're running for the first time, write the schema version to save unnecessary IOps
+			// on future startup.
+			let schema_version = VssSchemaVersion::V1;
+			let encoded_version = schema_version.encode();
+
+			let storable_builder = StorableBuilder::new(RandEntropySource);
+			let vss_version = -1;
+			let aad = store_key.as_bytes();
+			let storable =
+				storable_builder.build(encoded_version, vss_version, &data_encryption_key, aad);
+
+			let request = PutObjectRequest {
+				store_id,
+				global_version: None,
+				transaction_items: vec![KeyValue {
+					key: store_key,
+					version: vss_version,
+					value: storable.encode_to_vec(),
+				}],
+				delete_items: vec![],
+			};
+
+			client.put_object(&request).await.map_err(|e| {
+				let msg = format!("Failed to write schema version: {}", e);
+				Error::new(ErrorKind::Other, msg)
+			})?;
+
+			Ok((schema_version, client))
+		}
+	}
+}
+
 /// A source for generating entropy/randomness using [`rand`].
 pub(crate) struct RandEntropySource;
 
@@ -681,7 +814,8 @@ mod tests {
 		let mut vss_seed = [0u8; 32];
 		rng.fill_bytes(&mut vss_seed);
 		let header_provider = Arc::new(FixedHeaders::new(HashMap::new()));
-		let vss_store = VssStore::new(vss_base_url, rand_store_id, vss_seed, header_provider);
+		let vss_store =
+			VssStore::new(vss_base_url, rand_store_id, vss_seed, header_provider).unwrap();
 		do_read_write_remove_list_persist(&vss_store);
 	}
 
@@ -693,7 +827,8 @@ mod tests {
 		let mut vss_seed = [0u8; 32];
 		rng.fill_bytes(&mut vss_seed);
 		let header_provider = Arc::new(FixedHeaders::new(HashMap::new()));
-		let vss_store = VssStore::new(vss_base_url, rand_store_id, vss_seed, header_provider);
+		let vss_store =
+			VssStore::new(vss_base_url, rand_store_id, vss_seed, header_provider).unwrap();
 
 		do_read_write_remove_list_persist(&vss_store);
 		drop(vss_store)

From 795e0f2bc8625ced096a1d3319a531b958c49cb5 Mon Sep 17 00:00:00 2001
From: Elias Rohrer <dev@tnull.de>
Date: Wed, 12 Nov 2025 20:02:57 +0100
Subject: [PATCH 11/11] TRY

---
 src/io/vss_store.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/io/vss_store.rs b/src/io/vss_store.rs
index 73c49b779..f34a303c3 100644
--- a/src/io/vss_store.rs
+++ b/src/io/vss_store.rs
@@ -513,6 +513,8 @@ impl VssStoreInner {
 		)?;
 
 		let store_key = self.build_obfuscated_key(&primary_namespace, &secondary_namespace, &key);
+		let cnt = store_key.chars().count();
+		let len = store_key.len();
 		let vss_version = -1;
 		let storable_builder = StorableBuilder::new(RandEntropySource);
 		let aad =
@@ -533,8 +535,8 @@ impl VssStoreInner {
 		self.execute_locked_write(inner_lock_ref, locking_key, version, async move || {
 			client.put_object(&request).await.map_err(|e| {
 				let msg = format!(
-					"Failed to write to key {}/{}/{}: {}",
-					primary_namespace, secondary_namespace, key, e
+					"Failed to write to key {}/{}/{}, obfs len {}, cnt {}: {}",
+					primary_namespace, secondary_namespace, key, len, cnt, e
 				);
 				Error::new(ErrorKind::Other, msg)
 			})?;