From 7914128ab5a9bdba8e3e9b2807d1b211338a6056 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Tue, 28 Oct 2025 17:05:18 +0100 Subject: [PATCH 01/30] Implement Cell Dissemination --- Cargo.lock | 151 ++++---- Cargo.toml | 2 +- beacon_node/beacon_chain/src/beacon_chain.rs | 75 ++-- .../src/data_availability_checker.rs | 99 ++++- .../overflow_lru_cache.rs | 116 ++++-- .../src/data_column_verification.rs | 361 ++++++++++++----- .../fetch_blobs/fetch_blobs_beacon_adapter.rs | 17 + .../beacon_chain/src/fetch_blobs/mod.rs | 65 ++-- beacon_node/beacon_chain/src/kzg_utils.rs | 155 +++++++- beacon_node/beacon_chain/src/metrics.rs | 21 + beacon_node/beacon_chain/src/test_utils.rs | 2 +- beacon_node/beacon_chain/tests/events.rs | 2 +- beacon_node/beacon_processor/src/lib.rs | 16 +- beacon_node/execution_layer/src/engine_api.rs | 1 + .../execution_layer/src/engine_api/http.rs | 16 + .../src/engine_api/json_structures.rs | 3 + beacon_node/execution_layer/src/lib.rs | 19 +- .../execution_layer/src/test_utils/mod.rs | 1 + beacon_node/http_api/src/publish_blocks.rs | 14 +- beacon_node/lighthouse_network/Cargo.toml | 7 +- .../lighthouse_network/src/service/mod.rs | 60 ++- .../lighthouse_network/src/types/mod.rs | 1 + .../lighthouse_network/src/types/partial.rs | 123 ++++++ .../lighthouse_network/src/types/pubsub.rs | 64 ++- .../lighthouse_network/src/types/topics.rs | 6 + beacon_node/lighthouse_tracing/src/lib.rs | 1 + beacon_node/network/src/lib.rs | 1 + beacon_node/network/src/metrics.rs | 40 ++ .../gossip_methods.rs | 303 ++++++++++++++- .../src/network_beacon_processor/mod.rs | 59 ++- .../network/src/partial_data_column_cache.rs | 76 ++++ beacon_node/network/src/router.rs | 15 + beacon_node/network/src/service.rs | 2 +- consensus/types/src/das_column.rs | 105 +++++ consensus/types/src/data_column_sidecar.rs | 92 ++++- consensus/types/src/lib.rs | 2 + .../types/src/partial_data_column_sidecar.rs | 364 ++++++++++++++++++ testing/ef_tests/src/cases/fork_choice.rs | 11 +- 38 files changed, 2137 insertions(+), 331 deletions(-) create mode 100644 beacon_node/lighthouse_network/src/types/partial.rs create mode 100644 beacon_node/network/src/partial_data_column_cache.rs create mode 100644 consensus/types/src/das_column.rs create mode 100644 consensus/types/src/partial_data_column_sidecar.rs diff --git a/Cargo.lock b/Cargo.lock index 1efb1fbc706..4827786a167 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2420,7 +2420,7 @@ dependencies = [ "parking_lot 0.12.3", "rand 0.8.5", "smallvec", - "socket2", + "socket2 0.5.8", "tokio", "tracing", "uint 0.10.0", @@ -4156,7 +4156,7 @@ dependencies = [ "once_cell", "rand 0.9.0", "ring", - "socket2", + "socket2 0.5.8", "thiserror 2.0.12", "tinyvec", "tokio", @@ -4396,7 +4396,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.5.8", "tokio", "tower-service", "tracing", @@ -4477,7 +4477,7 @@ dependencies = [ "http-body 1.0.1", "hyper 1.6.0", "pin-project-lite", - "socket2", + "socket2 0.5.8", "tokio", "tower-service", "tracing", @@ -4865,7 +4865,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2", + "socket2 0.5.8", "widestring 1.1.0", "windows-sys 0.48.0", "winreg", @@ -5161,9 +5161,8 @@ dependencies = [ [[package]] name = "libp2p" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce71348bf5838e46449ae240631117b487073d5f347c06d434caddcb91dceb5a" +version = "0.56.1" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "bytes", "either", @@ -5174,6 +5173,7 @@ dependencies = [ "libp2p-connection-limits", "libp2p-core", "libp2p-dns", + "libp2p-gossipsub", "libp2p-identify", "libp2p-identity", "libp2p-mdns", @@ -5194,8 +5194,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16ccf824ee859ca83df301e1c0205270206223fd4b1f2e512a693e1912a8f4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5205,8 +5204,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18b8b607cf3bfa2f8c57db9c7d8569a315d5cc0a282e6bfd5ebfc0a9840b2a0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5216,8 +5214,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d28e2d2def7c344170f5c6450c0dbe3dfef655610dbfde2f6ac28a527abbe36" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "either", "fnv", @@ -5234,15 +5231,14 @@ dependencies = [ "rw-stream-sink", "thiserror 2.0.12", "tracing", - "unsigned-varint 0.8.0", + "unsigned-varint", "web-time", ] [[package]] name = "libp2p-dns" version = "0.44.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b770c1c8476736ca98c578cba4b505104ff8e842c2876b528925f9766379f9a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "async-trait", "futures", @@ -5257,7 +5253,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/sigp/rust-libp2p.git?rev=5acdf89a65d64098f9346efa5769e57bcd19dea9#5acdf89a65d64098f9346efa5769e57bcd19dea9" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "async-channel 2.3.1", "asynchronous-codec", @@ -5287,8 +5283,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ab792a8b68fdef443a62155b01970c81c3aadab5e659621b063ef252a8e65e8" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "asynchronous-codec", "either", @@ -5330,8 +5325,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66872d0f1ffcded2788683f76931be1c52e27f343edb93bc6d0bcd8887be443" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "hickory-proto", @@ -5341,16 +5335,15 @@ dependencies = [ "libp2p-swarm", "rand 0.8.5", "smallvec", - "socket2", + "socket2 0.6.1", "tokio", "tracing", ] [[package]] name = "libp2p-metrics" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "805a555148522cb3414493a5153451910cb1a146c53ffbf4385708349baf62b7" +version = "0.17.1" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "libp2p-core", @@ -5364,9 +5357,8 @@ dependencies = [ [[package]] name = "libp2p-mplex" -version = "0.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aaa6fee3722e355443058472fc4705d78681bc2d8e447a0bdeb3fecf40cd197" +version = "0.43.1" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "asynchronous-codec", "bytes", @@ -5378,14 +5370,13 @@ dependencies = [ "rand 0.8.5", "smallvec", "tracing", - "unsigned-varint 0.8.0", + "unsigned-varint", ] [[package]] name = "libp2p-noise" version = "0.46.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc73eacbe6462a0eb92a6527cac6e63f02026e5407f8831bde8293f19217bfbf" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "asynchronous-codec", "bytes", @@ -5407,8 +5398,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e659439578fc6d305da8303834beb9d62f155f40e7f5b9d81c9f2b2c69d1926" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "asynchronous-codec", "bytes", @@ -5423,8 +5413,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dc448b2de9f4745784e3751fe8bc6c473d01b8317edd5ababcb0dec803d843f" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "futures-timer", @@ -5436,7 +5425,7 @@ dependencies = [ "rand 0.8.5", "ring", "rustls 0.23.23", - "socket2", + "socket2 0.6.1", "thiserror 2.0.12", "tokio", "tracing", @@ -5445,17 +5434,16 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aa762e5215919a34e31c35d4b18bf2e18566ecab7f8a3d39535f4a3068f8b62" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "either", "fnv", "futures", "futures-timer", + "hashlink 0.10.0", "libp2p-core", "libp2p-identity", "libp2p-swarm-derive", - "lru", "multistream-select", "rand 0.8.5", "smallvec", @@ -5467,8 +5455,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd297cf53f0cb3dee4d2620bb319ae47ef27c702684309f682bdb7e55a18ae9c" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "heck 0.5.0", "quote", @@ -5478,15 +5465,14 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65b4e030c52c46c8d01559b2b8ca9b7c4185f10576016853129ca1fe5cd1a644" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "futures-timer", "if-watch", "libc", "libp2p-core", - "socket2", + "socket2 0.6.1", "tokio", "tracing", ] @@ -5494,8 +5480,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96ff65a82e35375cbc31ebb99cacbbf28cb6c4fefe26bf13756ddcf708d40080" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "futures-rustls", @@ -5512,9 +5497,8 @@ dependencies = [ [[package]] name = "libp2p-upnp" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4757e65fe69399c1a243bbb90ec1ae5a2114b907467bf09f3575e899815bb8d3" +version = "0.6.0" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "futures-timer", @@ -5528,8 +5512,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f15df094914eb4af272acf9adaa9e287baa269943f32ea348ba29cfb9bfc60d8" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "either", "futures", @@ -5673,7 +5656,7 @@ dependencies = [ "tracing", "tracing-subscriber", "types", - "unsigned-varint 0.8.0", + "unsigned-varint", ] [[package]] @@ -6174,7 +6157,7 @@ dependencies = [ "percent-encoding", "serde", "static_assertions", - "unsigned-varint 0.8.0", + "unsigned-varint", "url", ] @@ -6196,21 +6179,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b430e7953c29dd6a09afc29ff0bb69c6e306329ee6794700aee27b76a1aea8d" dependencies = [ "core2", - "unsigned-varint 0.8.0", + "unsigned-varint", ] [[package]] name = "multistream-select" version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0df8e5eec2298a62b326ee4f0d7fe1a6b90a09dfcf9df37b38f947a8c42f19" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "bytes", "futures", - "log", "pin-project", "smallvec", - "unsigned-varint 0.7.2", + "tracing", + "unsigned-varint", ] [[package]] @@ -7319,9 +7301,9 @@ dependencies = [ [[package]] name = "prometheus-client" -version = "0.23.1" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf41c1a7c32ed72abe5082fb19505b969095c12da9f5732a4bc9878757fd087c" +checksum = "e4500adecd7af8e0e9f4dbce15cfee07ce913fbf6ad605cc468b83f2d531ee94" dependencies = [ "dtoa", "itoa", @@ -7331,9 +7313,9 @@ dependencies = [ [[package]] name = "prometheus-client-derive-encode" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" +checksum = "9adf1691c04c0a5ff46ff8f262b58beb07b0dbb61f96f9f54f6cbd82106ed87f" dependencies = [ "proc-macro2", "quote", @@ -7452,14 +7434,13 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15a0580ab32b169745d7a39db2ba969226ca16738931be152a3209b409de2474" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "asynchronous-codec", "bytes", "quick-protobuf", - "thiserror 1.0.69", - "unsigned-varint 0.8.0", + "thiserror 2.0.12", + "unsigned-varint", ] [[package]] @@ -7497,7 +7478,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.23", - "socket2", + "socket2 0.5.8", "thiserror 2.0.12", "tokio", "tracing", @@ -7532,7 +7513,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.5.8", "tracing", "windows-sys 0.59.0", ] @@ -8266,8 +8247,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8c9026ff5d2f23da5e45bbc283f156383001bfb09c4e44256d02c1a685fe9a1" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" dependencies = [ "futures", "pin-project", @@ -8911,6 +8891,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + [[package]] name = "spin" version = "0.9.8" @@ -9499,7 +9489,7 @@ dependencies = [ "parking_lot 0.12.3", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.8", "tokio-macros", "tracing", "windows-sys 0.52.0", @@ -9633,7 +9623,7 @@ dependencies = [ "percent-encoding", "pin-project", "prost", - "socket2", + "socket2 0.5.8", "tokio", "tokio-stream", "tower 0.4.13", @@ -10037,12 +10027,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" -[[package]] -name = "unsigned-varint" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" - [[package]] name = "unsigned-varint" version = "0.8.0" @@ -10817,6 +10801,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", +] + [[package]] name = "windows-targets" version = "0.42.2" diff --git a/Cargo.toml b/Cargo.toml index ae84d645bb9..96f9e2664f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -163,7 +163,7 @@ fs2 = "0.4" futures = "0.3" genesis = { path = "beacon_node/genesis" } # This is tracking the sigp-gossipsub branch on sigp/rust-libp2p commit: Aug 20 2025 -gossipsub = { package = "libp2p-gossipsub", git = "https://github.com/sigp/rust-libp2p.git", rev = "5acdf89a65d64098f9346efa5769e57bcd19dea9", "features" = ["metrics"] } +gossipsub = { package = "libp2p-gossipsub", git = "https://github.com/jxs/rust-libp2p.git", branch = "gossipsub-partial-messages", features = ["partial_messages", "metrics"] } graffiti_file = { path = "validator_client/graffiti_file" } hashlink = "0.9.0" health_metrics = { path = "common/health_metrics" } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 3e02baf9017..6a8d5af5293 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -24,7 +24,7 @@ use crate::chain_config::ChainConfig; use crate::custody_context::CustodyContextSsz; use crate::data_availability_checker::{ Availability, AvailabilityCheckError, AvailableBlock, AvailableBlockData, - DataAvailabilityChecker, DataColumnReconstructionResult, + DataAvailabilityChecker, DataColumnReconstructionResult, MergedData, }; use crate::data_column_verification::{GossipDataColumnError, GossipVerifiedDataColumn}; use crate::early_attester_cache::EarlyAttesterCache; @@ -130,7 +130,9 @@ use tokio_stream::Stream; use tracing::{Span, debug, debug_span, error, info, info_span, instrument, trace, warn}; use tree_hash::TreeHash; use types::blob_sidecar::FixedBlobSidecarList; +use types::das_column::DasColumn; use types::data_column_sidecar::ColumnIndex; +use types::partial_data_column_sidecar::VerifiablePartialDataColumn; use types::payload::BlockProductionVersion; use types::*; @@ -2194,7 +2196,8 @@ impl BeaconChain { self: &Arc, data_column_sidecar: Arc>, subnet_id: DataColumnSubnetId, - ) -> Result, GossipDataColumnError> { + ) -> Result>, GossipDataColumnError> + { metrics::inc_counter(&metrics::DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); let _timer = metrics::start_timer(&metrics::DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); GossipVerifiedDataColumn::new(data_column_sidecar, subnet_id, self).inspect(|_| { @@ -2202,6 +2205,22 @@ impl BeaconChain { }) } + #[instrument(skip_all, level = "trace")] + pub fn verify_partial_data_column_sidecar_for_gossip( + self: &Arc, + data_column_sidecar: Arc>, + ) -> Result< + GossipVerifiedDataColumn>, + GossipDataColumnError, + > { + metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_REQUESTS); + let _timer = + metrics::start_timer(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES); + GossipVerifiedDataColumn::new_partial(data_column_sidecar, self).inspect(|_| { + metrics::inc_counter(&metrics::PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_SUCCESSES); + }) + } + #[instrument(skip_all, level = "trace")] pub fn verify_blob_sidecar_for_gossip( self: &Arc, @@ -3042,10 +3061,11 @@ impl BeaconChain { /// Cache the data columns in the processing cache, process it, then evict it from the cache if it was /// imported or errors. #[instrument(skip_all, level = "debug")] - pub async fn process_gossip_data_columns( + pub async fn process_gossip_data_columns>( self: &Arc, - data_columns: Vec>, + data_columns: Vec>, publish_fn: impl FnOnce() -> Result<(), BlockError>, + data_publish_fn: impl FnOnce(MergedData), ) -> Result { let Ok((slot, block_root)) = data_columns .iter() @@ -3068,16 +3088,18 @@ impl BeaconChain { return Err(BlockError::DuplicateFullyImported(block_root)); } - self.emit_sse_data_column_sidecar_events( - &block_root, - data_columns.iter().map(|column| column.as_data_column()), - ); + // TODO(dknopik): impl SSE + //self.emit_sse_data_column_sidecar_events( + // &block_root, + // data_columns.iter().map(|column| column.as_data_column()), + //); self.check_gossip_data_columns_availability_and_import( slot, block_root, data_columns, publish_fn, + data_publish_fn, ) .await } @@ -3143,11 +3165,12 @@ impl BeaconChain { EngineGetBlobsOutput::Blobs(blobs) => { self.emit_sse_blob_sidecar_events(&block_root, blobs.iter().map(|b| b.as_blob())); } - EngineGetBlobsOutput::CustodyColumns(columns) => { - self.emit_sse_data_column_sidecar_events( - &block_root, - columns.iter().map(|column| column.as_data_column()), - ); + EngineGetBlobsOutput::CustodyColumns(_columns) => { + // TODO(dknopik): impl SSE + //self.emit_sse_data_column_sidecar_events( + // &block_root, + // columns.iter().map(|column| column.as_data_column()), + //); } } @@ -3543,22 +3566,23 @@ impl BeaconChain { /// Checks if the provided data column can make any cached blocks available, and imports immediately /// if so, otherwise caches the data column in the data availability checker. - async fn check_gossip_data_columns_availability_and_import( + async fn check_gossip_data_columns_availability_and_import>( self: &Arc, slot: Slot, block_root: Hash256, - data_columns: Vec>, + data_columns: Vec>, publish_fn: impl FnOnce() -> Result<(), BlockError>, + data_publish_fn: impl FnOnce(MergedData), ) -> Result { if let Some(slasher) = self.slasher.as_ref() { - for data_colum in &data_columns { - slasher.accept_block_header(data_colum.signed_block_header()); + for header in data_columns.iter().filter_map(|c| c.signed_block_header()) { + slasher.accept_block_header(header.clone()); } } let availability = self .data_availability_checker - .put_gossip_verified_data_columns(block_root, slot, data_columns)?; + .put_gossip_verified_data_columns(block_root, slot, data_columns, data_publish_fn)?; self.process_availability(slot, availability, publish_fn) .await @@ -3635,7 +3659,7 @@ impl BeaconChain { data_columns.iter().map(|c| c.as_data_column()), )?; self.data_availability_checker - .put_kzg_verified_custody_data_columns(block_root, data_columns)? + .put_kzg_verified_custody_data_columns(block_root, data_columns, |_| ())? } }; @@ -3668,10 +3692,13 @@ impl BeaconChain { .await } - fn check_data_column_sidecar_header_signature_and_slashability<'a>( + fn check_data_column_sidecar_header_signature_and_slashability< + 'a, + C: DasColumn + 'a, + >( self: &Arc, block_root: Hash256, - custody_columns: impl IntoIterator>, + custody_columns: impl IntoIterator, ) -> Result<(), BlockError> { let mut slashable_cache = self.observed_slashable.write(); // Process all unique block headers - previous logic assumed all headers were identical and @@ -3679,13 +3706,13 @@ impl BeaconChain { // from RPC. for header in custody_columns .into_iter() - .map(|c| c.signed_block_header.clone()) + .filter_map(|c| c.signed_block_header()) .unique() { // Return an error if *any* header signature is invalid, we do not want to import this // list of blobs into the DA checker. However, we will process any valid headers prior // to the first invalid header in the slashable cache & slasher. - verify_header_signature::(self, &header)?; + verify_header_signature::(self, header)?; slashable_cache .observe_slashable( @@ -3695,7 +3722,7 @@ impl BeaconChain { ) .map_err(|e| BlockError::BeaconChainError(Box::new(e.into())))?; if let Some(slasher) = self.slasher.as_ref() { - slasher.accept_block_header(header); + slasher.accept_block_header(header.clone()); } } Ok(()) diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 4dbc634b241..8807d8ad980 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -39,7 +39,9 @@ use crate::metrics::{ }; use crate::observed_data_sidecars::ObservationStrategy; pub use error::{Error as AvailabilityCheckError, ErrorCategory as AvailabilityCheckErrorCategory}; +use types::das_column::{ColumnComparison, DasColumn}; use types::non_zero_usize::new_non_zero_usize; +use types::partial_data_column_sidecar::VerifiablePartialDataColumn; /// The LRU Cache stores `PendingComponents`, which store block and its associated blob data: /// @@ -179,18 +181,40 @@ impl DataAvailabilityChecker { }) } - /// Check if the exact data column is in the availability cache. - pub fn is_data_column_cached( + /// Check if the (potentially partial) data column is in the availability cache. + /// Returns None if this is not checkable due to conflicting data, and a vec of missing cells + /// otherwise. + pub fn determine_missing_cells>( &self, block_root: &Hash256, - data_column: &DataColumnSidecar, - ) -> bool { + data_column: &C, + ) -> Option> { + fn do_compare, C2: DasColumn>( + cached: &C1, + data_column: &C2, + ) -> Option> { + match cached.compare(data_column) { + ColumnComparison::Equal => Some(vec![]), + ColumnComparison::MissingCells { missing_in_lhs, .. } => Some(missing_in_lhs), + _ => None, + } + } + self.availability_cache .peek_pending_components(block_root, |components| { - components.is_some_and(|components| { - let cached_column_opt = components.get_cached_data_column(data_column.index); - cached_column_opt.is_some_and(|cached| *cached == *data_column) - }) + if let Some(components) = components { + if let Some(cached_column) = + components.get_cached_data_column(data_column.index()) + { + return do_compare(cached_column.as_ref(), data_column); + } + if let Some(cached_column) = + components.get_cached_partial_data_column(data_column.index()) + { + return do_compare(cached_column.as_ref(), data_column); + } + } + Some(data_column.cells_present().collect()) }) } @@ -266,8 +290,11 @@ impl DataAvailabilityChecker { .map(KzgVerifiedCustodyDataColumn::from_asserted_custody) .collect::>(); - self.availability_cache - .put_kzg_verified_data_columns(block_root, verified_custody_columns) + self.availability_cache.put_kzg_verified_data_columns( + block_root, + verified_custody_columns, + |_| (), + ) } /// Check if we've cached other blobs for this block. If it completes a set and we also @@ -306,12 +333,14 @@ impl DataAvailabilityChecker { #[instrument(skip_all, level = "trace")] pub fn put_gossip_verified_data_columns< O: ObservationStrategy, - I: IntoIterator>, + C: DasColumn, + I: IntoIterator>, >( &self, block_root: Hash256, slot: Slot, data_columns: I, + publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let epoch = slot.epoch(T::EthSpec::slots_per_epoch()); let sampling_columns = self @@ -323,20 +352,28 @@ impl DataAvailabilityChecker { .map(|c| KzgVerifiedCustodyDataColumn::from_asserted_custody(c.into_inner())) .collect::>(); - self.availability_cache - .put_kzg_verified_data_columns(block_root, custody_columns) + self.availability_cache.put_kzg_verified_data_columns( + block_root, + custody_columns, + publish_fn, + ) } #[instrument(skip_all, level = "trace")] pub fn put_kzg_verified_custody_data_columns< - I: IntoIterator>, + I: IntoIterator>, + C: DasColumn, >( &self, block_root: Hash256, custody_columns: I, + publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { - self.availability_cache - .put_kzg_verified_data_columns(block_root, custody_columns) + self.availability_cache.put_kzg_verified_data_columns( + block_root, + custody_columns, + publish_fn, + ) } /// Check if we have all the blobs for a block. Returns `Availability` which has information @@ -651,7 +688,11 @@ impl DataAvailabilityChecker { ); self.availability_cache - .put_kzg_verified_data_columns(*block_root, data_columns_to_import_and_publish.clone()) + .put_kzg_verified_data_columns( + *block_root, + data_columns_to_import_and_publish.clone(), + |_| (), + ) .map(|availability| { DataColumnReconstructionResult::Success(( availability, @@ -861,6 +902,20 @@ impl MaybeAvailableBlock { } } +pub struct MergedData { + pub completed_columns: Vec>>, + pub updated_partials: Vec>>, +} + +impl MergedData { + pub fn empty() -> Self { + MergedData { + completed_columns: vec![], + updated_partials: vec![], + } + } +} + #[cfg(test)] mod test { use super::*; @@ -878,7 +933,9 @@ mod test { use std::time::Duration; use store::HotColdDB; use types::data_column_sidecar::DataColumn; - use types::{ChainSpec, ColumnIndex, EthSpec, ForkName, MainnetEthSpec, Slot}; + use types::{ + ChainSpec, ColumnIndex, DataColumnSidecar, EthSpec, ForkName, MainnetEthSpec, Slot, + }; type E = MainnetEthSpec; type T = EphemeralHarnessType; @@ -1011,10 +1068,10 @@ mod test { let gossip_columns = data_columns .into_iter() .filter(|d| requested_columns.contains(&d.index)) - .map(GossipVerifiedDataColumn::::__new_for_testing) + .map(GossipVerifiedDataColumn::>::__new_for_testing) .collect::>(); da_checker - .put_gossip_verified_data_columns(block_root, cgc_change_slot, gossip_columns) + .put_gossip_verified_data_columns(block_root, cgc_change_slot, gossip_columns, |_| ()) .expect("should put gossip custody columns"); // THEN the sampling size for the end slot of the same epoch remains unchanged @@ -1141,7 +1198,7 @@ mod test { da_checker .availability_cache - .put_kzg_verified_data_columns(block_root, custody_columns) + .put_kzg_verified_data_columns(block_root, custody_columns, |_| ()) .expect("should put custody columns"); // Try reconstrucing diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index b842a1a3f99..c5826afb5ee 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -1,5 +1,5 @@ -use super::AvailableBlockData; use super::state_lru_cache::{DietAvailabilityPendingExecutedBlock, StateLRUCache}; +use super::{AvailableBlockData, MergedData}; use crate::CustodyContext; use crate::beacon_chain::BeaconStore; use crate::blob_verification::KzgVerifiedBlob; @@ -18,6 +18,8 @@ use std::sync::Arc; use tracing::{Span, debug, debug_span}; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::BlobIdentifier; +use types::das_column::DasColumn; +use types::partial_data_column_sidecar::VerifiablePartialDataColumn; use types::{ BlobSidecar, BlockImportSource, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, Epoch, EthSpec, Hash256, RuntimeFixedVector, RuntimeVariableList, @@ -73,7 +75,10 @@ impl CachedBlock { pub struct PendingComponents { pub block_root: Hash256, pub verified_blobs: RuntimeFixedVector>>, - pub verified_data_columns: Vec>, + // TODO(dknopik): four options: two fields, one field containing an enum, converting everything to partial, or refactor for cell level storage + pub verified_partial_columns: + Vec>>, + pub verified_data_columns: Vec>>, pub block: Option>, pub reconstruction_started: bool, span: Span, @@ -93,7 +98,7 @@ impl PendingComponents { }) } - /// Returns an immutable reference to the cached data column. + /// Returns an immutable reference to the full cached data column. pub fn get_cached_data_column( &self, data_column_index: u64, @@ -104,6 +109,17 @@ impl PendingComponents { .map(|d| d.clone_arc()) } + /// Returns an immutable reference to the partial cached data column. + pub fn get_cached_partial_data_column( + &self, + data_column_index: u64, + ) -> Option>> { + self.verified_partial_columns + .iter() + .find(|d| d.index() == data_column_index) + .map(|d| d.clone_arc()) + } + /// Returns a mutable reference to the fixed vector of cached blobs. pub fn get_cached_blobs_mut(&mut self) -> &mut RuntimeFixedVector>> { &mut self.verified_blobs @@ -186,17 +202,56 @@ impl PendingComponents { } /// Merges a given set of data columns into the cache. - fn merge_data_columns>>( + fn merge_data_columns< + I: IntoIterator>, + C: DasColumn, + >( &mut self, kzg_verified_data_columns: I, - ) -> Result<(), AvailabilityCheckError> { + ) -> Result, AvailabilityCheckError> { + let mut merged_data = MergedData::empty(); + for data_column in kzg_verified_data_columns { - if self.get_cached_data_column(data_column.index()).is_none() { - self.verified_data_columns.push(data_column); + if self.get_cached_data_column(data_column.index()).is_some() { + // already have the full column + continue; + } + + if let Some(full) = data_column.try_as_full() { + self.verified_partial_columns + .retain(|col| col.index() != full.index()); + self.verified_data_columns.push(full); + continue; + } + + let partial = data_column.into_partial(); + if let Some((idx, cached_partial)) = self + .verified_partial_columns + .iter_mut() + .enumerate() + .find(|d| d.1.index() == partial.index()) + { + let did_merge = cached_partial.merge(&partial); + if did_merge { + if let Some(block) = &self.block + && let Some(full) = cached_partial.try_upgrade_full(block.as_block()) + { + merged_data.completed_columns.push(full.clone_arc()); + self.verified_data_columns.push(full); + self.verified_partial_columns.remove(idx); + } else { + merged_data + .updated_partials + .push(cached_partial.clone_arc()); + } + } + } else { + merged_data.updated_partials.push(partial.clone_arc()); + self.verified_partial_columns.push(partial); } } - Ok(()) + Ok(merged_data) } /// Inserts a new block and revalidates the existing blobs against it. @@ -247,7 +302,12 @@ impl PendingComponents { let data_columns = self .verified_data_columns .iter() - .map(|d| d.clone().into_inner()) + .filter_map(|d| { + d.clone() + .into_inner() + .as_full(Some(block.as_block())) + .map(|c| Arc::new(c.into_owned())) + }) .collect::>(); Some(AvailableBlockData::DataColumns(data_columns)) } @@ -340,6 +400,7 @@ impl PendingComponents { block_root, verified_blobs: RuntimeFixedVector::new(vec![None; max_len]), verified_data_columns: vec![], + verified_partial_columns: vec![], block: None, reconstruction_started: false, span, @@ -412,7 +473,7 @@ pub struct DataAvailabilityCheckerInner { // the current usage, as it's deconstructed immediately. #[allow(clippy::large_enum_variant)] pub(crate) enum ReconstructColumnsDecision { - Yes(Vec>), + Yes(Vec>>), No(&'static str), } @@ -514,7 +575,7 @@ impl DataAvailabilityCheckerInner { *blob_opt = Some(blob); } } - let pending_components = + let (pending_components, ()) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.merge_blobs(fixed_blobs); Ok(()) @@ -533,17 +594,21 @@ impl DataAvailabilityCheckerInner { #[allow(clippy::type_complexity)] pub fn put_kzg_verified_data_columns< - I: IntoIterator>, + I: IntoIterator>, + C: DasColumn, >( &self, block_root: Hash256, kzg_verified_data_columns: I, + publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let mut kzg_verified_data_columns = kzg_verified_data_columns.into_iter().peekable(); - let Some(epoch) = kzg_verified_data_columns - .peek() - .map(|verified_blob| verified_blob.as_data_column().epoch()) - else { + let Some(epoch) = kzg_verified_data_columns.peek().map(|verified_blob| { + verified_blob + .as_data_column() + .slot() + .epoch(T::EthSpec::slots_per_epoch()) + }) else { // No columns are processed. This can occur if all received columns were filtered out // before this point, e.g. due to a CGC change that caused extra columns to be downloaded // // before the new CGC took effect. @@ -551,11 +616,13 @@ impl DataAvailabilityCheckerInner { return Ok(Availability::MissingComponents(block_root)); }; - let pending_components = + let (pending_components, merged_data) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.merge_data_columns(kzg_verified_data_columns) })?; + publish_fn(merged_data); + let num_expected_columns = self .custody_context .num_of_data_columns_to_sample(epoch, &self.spec); @@ -610,23 +677,23 @@ impl DataAvailabilityCheckerInner { /// /// Once the update is complete, the write lock is downgraded and a read guard with a /// reference of the updated `PendingComponents` is returned. - fn update_or_insert_pending_components( + fn update_or_insert_pending_components( &self, block_root: Hash256, epoch: Epoch, update_fn: F, - ) -> Result>, AvailabilityCheckError> + ) -> Result<(MappedRwLockReadGuard<'_, PendingComponents>, R), AvailabilityCheckError> where - F: FnOnce(&mut PendingComponents) -> Result<(), AvailabilityCheckError>, + F: FnOnce(&mut PendingComponents) -> Result, { let mut write_lock = self.critical.write(); - { + let ret = { let pending_components = write_lock.get_or_insert_mut(block_root, || { PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize) }); update_fn(pending_components)? - } + }; RwLockReadGuard::try_map(RwLockWriteGuard::downgrade(write_lock), |cache| { cache.peek(&block_root) @@ -634,6 +701,7 @@ impl DataAvailabilityCheckerInner { .map_err(|_| { AvailabilityCheckError::Unexpected("pending components should exist".to_string()) }) + .map(|guard| (guard, ret)) } /// Check whether data column reconstruction should be attempted. @@ -703,7 +771,7 @@ impl DataAvailabilityCheckerInner { source: BlockImportSource, ) -> Result<(), AvailabilityCheckError> { let epoch = block.epoch(); - let pending_components = + let (pending_components, ()) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.insert_pre_execution_block(block, source); Ok(()) @@ -745,7 +813,7 @@ impl DataAvailabilityCheckerInner { .state_cache .register_pending_executed_block(executed_block); - let pending_components = + let (pending_components, ()) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.merge_block(diet_executed_block); Ok(()) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 07f85b045ab..7c4f558a168 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -15,10 +15,12 @@ use std::iter; use std::marker::PhantomData; use std::sync::Arc; use tracing::{debug, instrument}; +use types::das_column::DasColumn; use types::data_column_sidecar::ColumnIndex; +use types::partial_data_column_sidecar::{DanglingPartialDataColumn, VerifiablePartialDataColumn}; use types::{ BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256, - SignedBeaconBlockHeader, Slot, + SignedBeaconBlock, SignedBeaconBlockHeader, Slot, }; /// An error occurred while validating a gossip data column. @@ -187,13 +189,19 @@ impl From for GossipDataColumnError { /// A wrapper around a `DataColumnSidecar` that indicates it has been approved for re-gossiping on /// the p2p network. #[derive(Debug)] -pub struct GossipVerifiedDataColumn { +pub struct GossipVerifiedDataColumn< + T: BeaconChainTypes, + C: DasColumn, + O: ObservationStrategy = Observe, +> { block_root: Hash256, - data_column: KzgVerifiedDataColumn, + data_column: KzgVerifiedDataColumn, _phantom: PhantomData, } -impl Clone for GossipVerifiedDataColumn { +impl, O: ObservationStrategy> Clone + for GossipVerifiedDataColumn +{ fn clone(&self) -> Self { Self { block_root: self.block_root, @@ -203,23 +211,69 @@ impl Clone for GossipVerifiedDataCo } } -impl GossipVerifiedDataColumn { +impl, O: ObservationStrategy> + GossipVerifiedDataColumn +{ + /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for testing ONLY. + pub fn __new_for_testing(column_sidecar: Arc) -> Self { + Self { + block_root: column_sidecar.block_root(), + data_column: KzgVerifiedDataColumn::__new_for_testing(column_sidecar), + _phantom: Default::default(), + } + } + + pub fn as_data_column(&self) -> &C { + self.data_column.as_data_column() + } + + /// This is cheap as we're calling clone on an Arc + pub fn clone_data_column(&self) -> Arc { + self.data_column.clone_data_column() + } + + pub fn block_root(&self) -> Hash256 { + self.block_root + } + + pub fn slot(&self) -> Slot { + self.data_column.data.slot() + } + + pub fn index(&self) -> ColumnIndex { + self.data_column.index() + } + + pub fn signed_block_header(&self) -> Option<&SignedBeaconBlockHeader> { + self.data_column.data.signed_block_header() + } + + pub fn into_inner(self) -> KzgVerifiedDataColumn { + self.data_column + } +} + +impl + GossipVerifiedDataColumn, O> +{ pub fn new( column_sidecar: Arc>, subnet_id: DataColumnSubnetId, chain: &BeaconChain, ) -> Result { - let header = column_sidecar.signed_block_header.clone(); + let header = column_sidecar.signed_block_header().cloned(); // We only process slashing info if the gossip verification failed // since we do not process the data column any further in that case. - validate_data_column_sidecar_for_gossip::(column_sidecar, subnet_id, chain).map_err( - |e| { + validate_data_column_sidecar_for_gossip(column_sidecar, subnet_id, chain).map_err(|e| { + if let Some(header) = header { process_block_slash_info::<_, GossipDataColumnError>( chain, BlockSlashInfo::from_early_error_data_column(header, e), ) - }, - ) + } else { + e + } + }) } /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for block production ONLY. @@ -238,9 +292,12 @@ impl GossipVerifiedDataColumn // In this case, we should accept it for gossip propagation. verify_is_unknown_sidecar(chain, &column_sidecar)?; + // TODO(dknopik): is proper handling of none? if chain .data_availability_checker - .is_data_column_cached(&column_sidecar.block_root(), &column_sidecar) + .determine_missing_cells(&column_sidecar.block_root(), column_sidecar.as_ref()) + .ok_or_else(|| GossipDataColumnError::UnexpectedDataColumn)? + .is_empty() { // Observe this data column so we don't process it again. if O::observe() { @@ -255,97 +312,76 @@ impl GossipVerifiedDataColumn _phantom: Default::default(), }) } +} - /// Create a `GossipVerifiedDataColumn` from `DataColumnSidecar` for testing ONLY. - pub fn __new_for_testing(column_sidecar: Arc>) -> Self { - Self { - block_root: column_sidecar.block_root(), - data_column: KzgVerifiedDataColumn::__new_for_testing(column_sidecar), - _phantom: Default::default(), - } - } - - pub fn as_data_column(&self) -> &DataColumnSidecar { - self.data_column.as_data_column() - } - - /// This is cheap as we're calling clone on an Arc - pub fn clone_data_column(&self) -> Arc> { - self.data_column.clone_data_column() - } - - pub fn block_root(&self) -> Hash256 { - self.block_root - } - - pub fn slot(&self) -> Slot { - self.data_column.data.slot() - } - - pub fn index(&self) -> ColumnIndex { - self.data_column.data.index - } - - pub fn signed_block_header(&self) -> SignedBeaconBlockHeader { - self.data_column.data.signed_block_header.clone() - } - - pub fn into_inner(self) -> KzgVerifiedDataColumn { - self.data_column +impl + GossipVerifiedDataColumn, O> +{ + pub fn new_partial( + column_sidecar: Arc>, + chain: &BeaconChain, + ) -> Result { + validate_partial_data_column_sidecar_for_gossip(column_sidecar, chain) } } /// Wrapper over a `DataColumnSidecar` for which we have completed kzg verification. -#[derive(Debug, Derivative, Clone, Encode, Decode)] +#[derive(Debug, Derivative, Clone)] #[derivative(PartialEq, Eq)] -#[ssz(struct_behaviour = "transparent")] -pub struct KzgVerifiedDataColumn { - data: Arc>, +pub struct KzgVerifiedDataColumn> { + data: Arc, + _phantom: PhantomData, } -impl KzgVerifiedDataColumn { - pub fn new( - data_column: Arc>, - kzg: &Kzg, - ) -> Result, KzgError)> { +impl> KzgVerifiedDataColumn { + pub fn new(data_column: Arc, kzg: &Kzg) -> Result, KzgError)> { verify_kzg_for_data_column(data_column, kzg) } /// Mark a data column as KZG verified. Caller must ONLY use this on columns constructed /// from EL blobs. - pub fn from_execution_verified(data_column: Arc>) -> Self { - Self { data: data_column } + pub fn from_execution_verified(data_column: Arc) -> Self { + Self { + data: data_column, + _phantom: PhantomData, + } } /// Create a `KzgVerifiedDataColumn` from `DataColumnSidecar` for testing ONLY. - pub(crate) fn __new_for_testing(data_column: Arc>) -> Self { - Self { data: data_column } + pub(crate) fn __new_for_testing(data_column: Arc) -> Self { + Self { + data: data_column, + _phantom: PhantomData, + } } pub fn from_batch_with_scoring( - data_columns: Vec>>, + data_columns: Vec>, kzg: &Kzg, ) -> Result, (Option, KzgError)> { verify_kzg_for_data_column_list(data_columns.iter(), kzg)?; Ok(data_columns .into_iter() - .map(|column| Self { data: column }) + .map(|column| Self { + data: column, + _phantom: PhantomData, + }) .collect()) } - pub fn to_data_column(self) -> Arc> { + pub fn to_data_column(self) -> Arc { self.data } - pub fn as_data_column(&self) -> &DataColumnSidecar { + pub fn as_data_column(&self) -> &C { &self.data } /// This is cheap as we're calling clone on an Arc - pub fn clone_data_column(&self) -> Arc> { + pub fn clone_data_column(&self) -> Arc { self.data.clone() } pub fn index(&self) -> ColumnIndex { - self.data.index + self.data.index() } } @@ -383,22 +419,70 @@ impl CustodyDataColumn { } /// Data column that we must custody and has completed kzg verification -#[derive(Debug, Derivative, Clone, Encode, Decode)] +#[derive(Debug, Derivative, Clone)] #[derivative(PartialEq, Eq)] -#[ssz(struct_behaviour = "transparent")] -pub struct KzgVerifiedCustodyDataColumn { - data: Arc>, +pub struct KzgVerifiedCustodyDataColumn> { + data: Arc, + _phantom: PhantomData, } -impl KzgVerifiedCustodyDataColumn { +impl> KzgVerifiedCustodyDataColumn { /// Mark a column as custody column. Caller must ensure that our current custody requirements /// include this column - pub fn from_asserted_custody(kzg_verified: KzgVerifiedDataColumn) -> Self { + pub fn from_asserted_custody(kzg_verified: KzgVerifiedDataColumn) -> Self { Self { data: kzg_verified.to_data_column(), + _phantom: PhantomData, } } + pub fn into_inner(self) -> Arc { + self.data + } + + pub fn as_data_column(&self) -> &C { + &self.data + } + pub fn clone_arc(&self) -> Arc { + self.data.clone() + } + pub fn index(&self) -> ColumnIndex { + self.data.index() + } + + // TODO(dknopik): below three functions are baaaad. They clone in far too many cases + + pub fn try_as_full(&self) -> Option>> { + self.data + .as_full(None) + .map(|full| KzgVerifiedCustodyDataColumn { + data: Arc::new(full.into_owned()), + _phantom: PhantomData, + }) + } + + pub fn try_upgrade_full( + &self, + block: &SignedBeaconBlock, + ) -> Option>> { + self.data + .as_full(Some(block)) + .map(|full| KzgVerifiedCustodyDataColumn { + data: Arc::new(full.into_owned()), + _phantom: PhantomData, + }) + } + + pub fn into_partial(self) -> KzgVerifiedCustodyDataColumn> { + let column = Arc::try_unwrap(self.data).unwrap_or_else(|column| (*column).clone()); + KzgVerifiedCustodyDataColumn { + data: Arc::new(column.into_partial()), + _phantom: PhantomData, + } + } +} + +impl KzgVerifiedCustodyDataColumn> { /// Verify a column already marked as custody column pub fn new( data_column: CustodyDataColumn, @@ -407,6 +491,7 @@ impl KzgVerifiedCustodyDataColumn { verify_kzg_for_data_column(data_column.clone_arc(), kzg)?; Ok(Self { data: data_column.data, + _phantom: PhantomData, }) } @@ -414,7 +499,7 @@ impl KzgVerifiedCustodyDataColumn { kzg: &Kzg, partial_set_of_columns: &[Self], spec: &ChainSpec, - ) -> Result>, KzgError> { + ) -> Result, KzgError> { let all_data_columns = reconstruct_data_columns( kzg, partial_set_of_columns @@ -427,23 +512,45 @@ impl KzgVerifiedCustodyDataColumn { Ok(all_data_columns .into_iter() .map(|data| { - KzgVerifiedCustodyDataColumn::from_asserted_custody(KzgVerifiedDataColumn { data }) + KzgVerifiedCustodyDataColumn::from_asserted_custody(KzgVerifiedDataColumn { + data, + _phantom: PhantomData, + }) }) .collect::>()) } +} - pub fn into_inner(self) -> Arc> { - self.data - } +impl KzgVerifiedCustodyDataColumn> { + pub fn merge(&mut self, other: &Self) -> bool { + let to_be_added = other + .data + .column + .sidecar + .cells_present_bitmap + .difference(&self.data.column.sidecar.cells_present_bitmap); + if to_be_added.is_zero() { + return false; + }; - pub fn as_data_column(&self) -> &DataColumnSidecar { - &self.data - } - pub fn clone_arc(&self) -> Arc> { - self.data.clone() - } - pub fn index(&self) -> ColumnIndex { - self.data.index + let Some(merged) = self.data.column.sidecar.merge(&other.data.column.sidecar) else { + return false; + }; + + *self = Self { + data: Arc::new(VerifiablePartialDataColumn { + column: Arc::new(DanglingPartialDataColumn { + block_root: self.data.column.block_root, + index: self.data.column.index, + sidecar: merged, + }), + kzg_commitments: self.data.kzg_commitments.clone(), + slot: self.data.slot, + }), + _phantom: PhantomData, + }; + + true } } @@ -451,13 +558,16 @@ impl KzgVerifiedCustodyDataColumn { /// /// Returns an error if the kzg verification check fails. #[instrument(skip_all, level = "debug")] -pub fn verify_kzg_for_data_column( - data_column: Arc>, +pub fn verify_kzg_for_data_column>( + data_column: Arc, kzg: &Kzg, -) -> Result, (Option, KzgError)> { +) -> Result, (Option, KzgError)> { let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_SINGLE_TIMES); validate_data_columns(kzg, iter::once(&data_column))?; - Ok(KzgVerifiedDataColumn { data: data_column }) + Ok(KzgVerifiedDataColumn { + data: data_column, + _phantom: PhantomData, + }) } /// Complete kzg verification for a list of `DataColumnSidecar`s. @@ -465,12 +575,14 @@ pub fn verify_kzg_for_data_column( /// /// Note: This function should be preferred over calling `verify_kzg_for_data_column` /// in a loop since this function kzg verifies a list of data columns more efficiently. -pub fn verify_kzg_for_data_column_list<'a, E: EthSpec, I>( +pub fn verify_kzg_for_data_column_list<'a, E: EthSpec, I, A, C>( data_column_iter: I, kzg: &'a Kzg, ) -> Result<(), (Option, KzgError)> where - I: Iterator>> + Clone, + I: Iterator + Clone, + A: AsRef + 'a, + C: DasColumn + 'a, { let _timer = metrics::start_timer(&metrics::KZG_VERIFICATION_DATA_COLUMN_BATCH_TIMES); validate_data_columns(kzg, data_column_iter)?; @@ -482,7 +594,7 @@ pub fn validate_data_column_sidecar_for_gossip>, subnet: DataColumnSubnetId, chain: &BeaconChain, -) -> Result, GossipDataColumnError> { +) -> Result, O>, GossipDataColumnError> { let column_slot = data_column.slot(); verify_data_column_sidecar(&data_column, &chain.spec)?; verify_index_matches_subnet(&data_column, subnet, &chain.spec)?; @@ -491,13 +603,16 @@ pub fn validate_data_column_sidecar_for_gossip( + data_column: Arc>, + chain: &BeaconChain, +) -> Result< + GossipVerifiedDataColumn, O>, + GossipDataColumnError, +> { + // TODO(dknopik): This is kinda underspecified, just slap everything in here that *could* apply + let column_slot = data_column.slot(); + verify_sidecar_not_from_future_slot(chain, column_slot)?; + verify_slot_greater_than_latest_finalized_slot(chain, column_slot)?; + + let missing_cells = chain + .data_availability_checker + .determine_missing_cells(&data_column.block_root(), data_column.as_ref()) + .ok_or_else(|| GossipDataColumnError::UnexpectedDataColumn)?; + if missing_cells.is_empty() { + return Err(GossipDataColumnError::PriorKnownUnpublished); + } + + let filtered_column = Arc::new( + data_column + .clone_filter(|idx| missing_cells.contains(&idx)) + .ok_or_else(|| GossipDataColumnError::UnexpectedDataColumn)?, + ); + + // We do not have to check block related data here, as we create the verifiable column from + // gossip accepted block + + let kzg = &chain.kzg; + let kzg_verified_data_column = verify_kzg_for_data_column(filtered_column, kzg) + .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; + + // TODO(dknopik): observe?! I do not think so, as publishing works differently anyway... + // Basically we publish if we update our internal view, but this is merged elsewhere + //if O::observe() { + // observe_gossip_data_column(&data_column, chain)?; + //} + + Ok(GossipVerifiedDataColumn { + block_root: data_column.block_root(), + data_column: kzg_verified_data_column, + _phantom: PhantomData, + }) +} + /// Verify if the data column sidecar is valid. fn verify_data_column_sidecar( data_column: &DataColumnSidecar, @@ -851,7 +1016,7 @@ mod test { harness.advance_slot(); let verify_fn = |column_sidecar: DataColumnSidecar| { - GossipVerifiedDataColumn::<_>::new_for_block_publishing( + GossipVerifiedDataColumn::<_, _>::new_for_block_publishing( column_sidecar.into(), &harness.chain, ) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs index 9526921da73..4e98e81df24 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs @@ -51,6 +51,7 @@ impl FetchBlobsBeaconAdapter { .map_err(FetchEngineBlobError::RequestFailed) } + // TODO(dknopik): fall back to this pub(crate) async fn get_blobs_v2( &self, versioned_hashes: Vec, @@ -67,6 +68,22 @@ impl FetchBlobsBeaconAdapter { .map_err(FetchEngineBlobError::RequestFailed) } + pub(crate) async fn get_blobs_v3( + &self, + versioned_hashes: Vec, + ) -> Result>>>, FetchEngineBlobError> { + let execution_layer = self + .chain + .execution_layer + .as_ref() + .ok_or(FetchEngineBlobError::ExecutionLayerMissing)?; + + execution_layer + .get_blobs_v3(versioned_hashes) + .await + .map_err(FetchEngineBlobError::RequestFailed) + } + pub(crate) fn blobs_known_for_proposal( &self, proposer: u64, diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 4c6b2d10a95..55f43289900 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -17,7 +17,7 @@ use crate::block_verification_types::AsBlock; use crate::data_column_verification::{KzgVerifiedCustodyDataColumn, KzgVerifiedDataColumn}; #[cfg_attr(test, double)] use crate::fetch_blobs::fetch_blobs_beacon_adapter::FetchBlobsBeaconAdapter; -use crate::kzg_utils::blobs_to_data_column_sidecars; +use crate::kzg_utils::blobs_to_partial_data_columns; use crate::observed_block_producers::ProposalKey; use crate::validator_monitor::timestamp_now; use crate::{ @@ -32,9 +32,11 @@ use mockall_double::double; use ssz_types::FixedVector; use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::sync::Arc; -use tracing::{Span, debug, instrument, warn}; +use tracing::{Span, debug, instrument}; use types::blob_sidecar::BlobSidecarError; +use types::das_column::DasColumn; use types::data_column_sidecar::DataColumnSidecarError; +use types::partial_data_column_sidecar::VerifiablePartialDataColumn; use types::{ BeaconStateError, Blob, BlobSidecar, ColumnIndex, EthSpec, FullPayload, Hash256, KzgProofs, SignedBeaconBlock, SignedBeaconBlockHeader, VersionedHash, @@ -47,7 +49,9 @@ use types::{ pub enum EngineGetBlobsOutput { Blobs(Vec>), /// A filtered list of custody data columns to be imported into the `DataAvailabilityChecker`. - CustodyColumns(Vec>), + CustodyColumns( + Vec>>, + ), } #[derive(Debug)] @@ -246,9 +250,10 @@ async fn fetch_and_process_blobs_v2( let num_expected_blobs = versioned_hashes.len(); metrics::observe(&metrics::BLOBS_FROM_EL_EXPECTED, num_expected_blobs as f64); + // TODO(dknopik): implement fallback to get_blobs_v2 debug!(num_expected_blobs, "Fetching blobs from the EL"); let response = chain_adapter - .get_blobs_v2(versioned_hashes) + .get_blobs_v3(versioned_hashes) .await .inspect_err(|_| { inc_counter(&metrics::BLOBS_FROM_EL_ERROR_TOTAL); @@ -260,30 +265,15 @@ async fn fetch_and_process_blobs_v2( return Ok(None); }; - let (blobs, proofs): (Vec<_>, Vec<_>) = blobs_and_proofs + let blobs_and_proofs: Vec<_> = blobs_and_proofs .into_iter() - .map(|blob_and_proof| { - let BlobAndProofV2 { blob, proofs } = blob_and_proof; - (blob, proofs) - }) - .unzip(); + .map(|blob_and_proof| blob_and_proof.map(|BlobAndProofV2 { blob, proofs }| (blob, proofs))) + .collect(); - let num_fetched_blobs = blobs.len(); + let num_fetched_blobs = blobs_and_proofs.len(); metrics::observe(&metrics::BLOBS_FROM_EL_RECEIVED, num_fetched_blobs as f64); - if num_fetched_blobs != num_expected_blobs { - // This scenario is not supposed to happen if the EL is spec compliant. - // It should either return all requested blobs or none, but NOT partial responses. - // If we attempt to compute columns with partial blobs, we'd end up with invalid columns. - warn!( - num_fetched_blobs, - num_expected_blobs, "The EL did not return all requested blobs" - ); - inc_counter(&metrics::BLOBS_FROM_EL_MISS_TOTAL); - return Ok(None); - } - - debug!(num_fetched_blobs, "All expected blobs received from the EL"); + debug!(num_fetched_blobs, "Blobs received from the EL"); inc_counter(&metrics::BLOBS_FROM_EL_HIT_TOTAL); if chain_adapter.fork_choice_contains_block(&block_root) { @@ -300,8 +290,7 @@ async fn fetch_and_process_blobs_v2( &chain_adapter, block_root, block.clone(), - blobs, - proofs, + blobs_and_proofs, custody_columns_indices, ) .await?; @@ -337,10 +326,12 @@ async fn compute_custody_columns_to_import( chain_adapter: &Arc>, block_root: Hash256, block: Arc>>, - blobs: Vec>, - proofs: Vec>, + blobs_and_proofs: Vec, KzgProofs)>>, custody_columns_indices: &[ColumnIndex], -) -> Result>, FetchEngineBlobError> { +) -> Result< + Vec>>, + FetchEngineBlobError, +> { let kzg = chain_adapter.kzg().clone(); let spec = chain_adapter.spec().clone(); let chain_adapter_cloned = chain_adapter.clone(); @@ -353,13 +344,19 @@ async fn compute_custody_columns_to_import( let _guard = current_span.enter(); let mut timer = metrics::start_timer_vec( &metrics::DATA_COLUMN_SIDECAR_COMPUTATION, - &[&blobs.len().to_string()], + &[&blobs_and_proofs.len().to_string()], ); - let blob_refs = blobs.iter().collect::>(); - let cell_proofs = proofs.into_iter().flatten().collect(); + let blob_and_cell_refs = blobs_and_proofs + .iter() + .map(|option| { + option + .as_ref() + .map(|(blob, proofs)| (blob, proofs.as_ref())) + }) + .collect::>(); let data_columns_result = - blobs_to_data_column_sidecars(&blob_refs, cell_proofs, &block, &kzg, &spec) + blobs_to_partial_data_columns(blob_and_cell_refs, &block, &kzg, &spec) .discard_timer_on_break(&mut timer); drop(timer); @@ -370,7 +367,7 @@ async fn compute_custody_columns_to_import( .map(|data_columns| { data_columns .into_iter() - .filter(|col| custody_columns_indices.contains(&col.index)) + .filter(|col| custody_columns_indices.contains(&col.index())) .map(|col| { KzgVerifiedCustodyDataColumn::from_asserted_custody( KzgVerifiedDataColumn::from_execution_verified(col), diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 382775ab50f..1661c7fe605 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -7,7 +7,11 @@ use ssz_types::{FixedVector, VariableList}; use std::sync::Arc; use tracing::instrument; use types::beacon_block_body::KzgCommitments; +use types::das_column::DasColumn; use types::data_column_sidecar::{Cell, DataColumn, DataColumnSidecarError}; +use types::partial_data_column_sidecar::{ + CellBitmap, DanglingPartialDataColumn, PartialDataColumnSidecar, VerifiablePartialDataColumn, +}; use types::{ Blob, BlobSidecar, BlobSidecarList, ChainSpec, DataColumnSidecar, DataColumnSidecarList, EthSpec, Hash256, KzgCommitment, KzgProof, SignedBeaconBlock, SignedBeaconBlockHeader, @@ -46,12 +50,14 @@ pub fn validate_blob( } /// Validate a batch of `DataColumnSidecar`. -pub fn validate_data_columns<'a, E: EthSpec, I>( +pub fn validate_data_columns<'a, E: EthSpec, I, A, C>( kzg: &Kzg, data_column_iter: I, ) -> Result<(), (Option, KzgError)> where - I: Iterator>> + Clone, + I: Iterator + Clone, + A: AsRef + 'a, + C: DasColumn + 'a, { let mut cells = Vec::new(); let mut proofs = Vec::new(); @@ -59,22 +65,23 @@ where let mut commitments = Vec::new(); for data_column in data_column_iter { - let col_index = data_column.index; + let data_column = data_column.as_ref(); + let col_index = data_column.index(); - if data_column.column.is_empty() { + if data_column.column().is_empty() { return Err((Some(col_index), KzgError::KzgVerificationFailed)); } - for cell in &data_column.column { + for cell in data_column.column() { cells.push(ssz_cell_to_crypto_cell::(cell).map_err(|e| (Some(col_index), e))?); column_indices.push(col_index); } - for &proof in &data_column.kzg_proofs { + for &proof in data_column.kzg_proofs() { proofs.push(Bytes48::from(proof)); } - for &commitment in &data_column.kzg_commitments { + for &commitment in data_column.kzg_commitments() { commitments.push(Bytes48::from(commitment)); } @@ -217,6 +224,58 @@ pub fn blobs_to_data_column_sidecars( .map_err(DataColumnSidecarError::BuildSidecarFailed) } +/// Build data column sidecars from a signed beacon block and its blobs. +#[instrument(skip_all, level = "debug", fields(blob_count = blobs_and_proofs.len()))] +pub fn blobs_to_partial_data_columns( + blobs_and_proofs: Vec, &[KzgProof])>>, + block: &SignedBeaconBlock, + kzg: &Kzg, + spec: &ChainSpec, +) -> Result>>, DataColumnSidecarError> { + if blobs_and_proofs.is_empty() { + return Ok(vec![]); + } + + let kzg_commitments = block + .message() + .body() + .blob_kzg_commitments() + .map_err(|_err| DataColumnSidecarError::PreDeneb)?; + let signed_block_header = block.signed_block_header(); + + let blob_cells_and_proofs_vec = blobs_and_proofs + .into_par_iter() + .map(|maybe_blob_and_proofs| { + let Some((blob, proofs)) = maybe_blob_and_proofs else { + return Ok(None); + }; + + let blob = blob.as_ref().try_into().map_err(|e| { + KzgError::InconsistentArrayLength(format!( + "blob should have a guaranteed size due to FixedVector: {e:?}" + )) + })?; + + kzg.compute_cells(blob).and_then(|cells| { + let proofs = proofs.try_into().map_err(|e| { + KzgError::InconsistentArrayLength(format!( + "proof chunks should have exactly `number_of_columns` proofs: {e:?}" + )) + })?; + Ok(Some((cells, proofs))) + }) + }) + .collect::, KzgError>>()?; + + build_partial_data_columns( + kzg_commitments.clone(), + signed_block_header, + blob_cells_and_proofs_vec, + spec, + ) + .map_err(DataColumnSidecarError::BuildSidecarFailed) +} + pub fn compute_cells(blobs: &[&Blob], kzg: &Kzg) -> Result, KzgError> { let cells_vec = blobs .into_par_iter() @@ -295,6 +354,86 @@ pub(crate) fn build_data_column_sidecars( Ok(sidecars) } +pub(crate) fn build_partial_data_columns( + kzg_commitments: KzgCommitments, + signed_block_header: SignedBeaconBlockHeader, + blob_cells_and_proofs_vec: Vec>, + spec: &ChainSpec, +) -> Result>>, String> { + let number_of_columns = E::number_of_columns(); + let max_blobs_per_block = spec + .max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch())) + as usize; + let mut bitmap = + CellBitmap::::with_capacity(blob_cells_and_proofs_vec.len()).map_err(|_| { + format!( + "Exceeded max committment count: {} (got {})", + E::max_blob_commitments_per_block(), + blob_cells_and_proofs_vec.len() + ) + })?; + let mut columns = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns]; + let mut column_kzg_proofs = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns]; + + for (idx, maybe_cells_and_proofs) in blob_cells_and_proofs_vec.into_iter().enumerate() { + let Some((blob_cells, blob_cell_proofs)) = maybe_cells_and_proofs else { + continue; + }; + + bitmap + .set(idx, true) + .expect("bitmap constructed from iterator length above"); + + // we iterate over each column, and we construct the column from "top to bottom", + // pushing on the cell and the corresponding proof at each column index. we do this for + // each blob (i.e. the outer loop). + for col in 0..number_of_columns { + let cell = blob_cells + .get(col) + .ok_or(format!("Missing blob cell at index {col}"))?; + let cell: Vec = cell.to_vec(); + let cell = Cell::::from(cell); + + let proof = blob_cell_proofs + .get(col) + .ok_or(format!("Missing blob cell KZG proof at index {col}"))?; + + let column = columns + .get_mut(col) + .ok_or(format!("Missing data column at index {col}"))?; + let column_proofs = column_kzg_proofs + .get_mut(col) + .ok_or(format!("Missing data column proofs at index {col}"))?; + + column.push(cell); + column_proofs.push(*proof); + } + } + + let sidecars: Vec>> = columns + .into_iter() + .zip(column_kzg_proofs) + .enumerate() + .map(|(index, (col, proofs))| { + Arc::new(VerifiablePartialDataColumn { + column: Arc::new(DanglingPartialDataColumn { + block_root: signed_block_header.message.canonical_root(), + index: index as u64, + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: bitmap.clone(), + column: DataColumn::::from(col), + kzg_proofs: VariableList::from(proofs), + }, + }), + kzg_commitments: kzg_commitments.clone(), + slot: signed_block_header.message.slot, + }) + }) + .collect(); + + Ok(sidecars) +} + /// Reconstruct blobs from a subset of data column sidecars (requires at least 50%). /// /// If `blob_indices_opt` is `None`, this function attempts to reconstruct all blobs associated @@ -471,7 +610,7 @@ mod test { blobs_to_data_column_sidecars(&blob_refs, proofs.to_vec(), &signed_block, kzg, spec) .unwrap(); - let result = validate_data_columns::(kzg, column_sidecars.iter()); + let result = validate_data_columns(kzg, column_sidecars.iter()); assert!(result.is_ok()); } diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 8f1da7b67b7..6ca2af01589 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1662,6 +1662,27 @@ pub static DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES: LazyLock> = + LazyLock::new(|| { + try_create_int_counter( + "beacon_partial_data_column_sidecar_processing_requests_total", + "Count of all partial data column sidecars submitted for processing", + ) + }); +pub static PARTIAL_DATA_COLUMN_SIDECAR_PROCESSING_SUCCESSES: LazyLock> = + LazyLock::new(|| { + try_create_int_counter( + "beacon_partial_data_column_sidecar_processing_successes_total", + "Number of partial data column sidecars verified for gossip", + ) + }); +pub static PARTIAL_DATA_COLUMN_SIDECAR_GOSSIP_VERIFICATION_TIMES: LazyLock> = + LazyLock::new(|| { + try_create_histogram( + "beacon_partial_data_column_sidecar_gossip_verification_seconds", + "Full runtime of partial data column sidecars gossip verification", + ) + }); pub static BLOBS_FROM_EL_HIT_TOTAL: LazyLock> = LazyLock::new(|| { try_create_int_counter( diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 38797d0264d..639b933795f 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -3206,7 +3206,7 @@ where if !verified_columns.is_empty() { self.chain - .process_gossip_data_columns(verified_columns, || Ok(())) + .process_gossip_data_columns(verified_columns, || Ok(()), |_| ()) .await .unwrap(); } diff --git a/beacon_node/beacon_chain/tests/events.rs b/beacon_node/beacon_chain/tests/events.rs index 466058eea38..d3e8af7a01e 100644 --- a/beacon_node/beacon_chain/tests/events.rs +++ b/beacon_node/beacon_chain/tests/events.rs @@ -79,7 +79,7 @@ async fn data_column_sidecar_event_on_process_gossip_data_column() { let _ = harness .chain - .process_gossip_data_columns(vec![gossip_verified_data_column], || Ok(())) + .process_gossip_data_columns(vec![gossip_verified_data_column], || Ok(()), |_| ()) .await .unwrap(); diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 28ed0cca913..c3dfd0a9126 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -121,6 +121,7 @@ pub struct BeaconProcessorQueueLengths { gossip_block_queue: usize, gossip_blob_queue: usize, gossip_data_column_queue: usize, + gossip_partial_data_column_queue: usize, delayed_block_queue: usize, status_queue: usize, bbrange_queue: usize, @@ -187,6 +188,7 @@ impl BeaconProcessorQueueLengths { gossip_block_queue: 1024, gossip_blob_queue: 1024, gossip_data_column_queue: 1024, + gossip_partial_data_column_queue: 1024, delayed_block_queue: 1024, status_queue: 1024, bbrange_queue: 1024, @@ -579,6 +581,7 @@ pub enum Work { GossipBlock(AsyncFn), GossipBlobSidecar(AsyncFn), GossipDataColumnSidecar(AsyncFn), + GossipPartialDataColumnSidecar(AsyncFn), DelayedImportBlock { beacon_block_slot: Slot, beacon_block_root: Hash256, @@ -641,6 +644,7 @@ pub enum WorkType { GossipBlock, GossipBlobSidecar, GossipDataColumnSidecar, + GossipPartialDataColumnSidecar, DelayedImportBlock, GossipVoluntaryExit, GossipProposerSlashing, @@ -688,6 +692,7 @@ impl Work { Work::GossipBlock(_) => WorkType::GossipBlock, Work::GossipBlobSidecar(_) => WorkType::GossipBlobSidecar, Work::GossipDataColumnSidecar(_) => WorkType::GossipDataColumnSidecar, + Work::GossipPartialDataColumnSidecar(_) => WorkType::GossipPartialDataColumnSidecar, Work::DelayedImportBlock { .. } => WorkType::DelayedImportBlock, Work::GossipVoluntaryExit(_) => WorkType::GossipVoluntaryExit, Work::GossipProposerSlashing(_) => WorkType::GossipProposerSlashing, @@ -873,6 +878,8 @@ impl BeaconProcessor { let mut gossip_block_queue = FifoQueue::new(queue_lengths.gossip_block_queue); let mut gossip_blob_queue = FifoQueue::new(queue_lengths.gossip_blob_queue); let mut gossip_data_column_queue = FifoQueue::new(queue_lengths.gossip_data_column_queue); + let mut gossip_partial_data_column_queue = + FifoQueue::new(queue_lengths.gossip_partial_data_column_queue); let mut delayed_block_queue = FifoQueue::new(queue_lengths.delayed_block_queue); let mut status_queue = FifoQueue::new(queue_lengths.status_queue); @@ -1325,6 +1332,9 @@ impl BeaconProcessor { Work::GossipDataColumnSidecar { .. } => { gossip_data_column_queue.push(work, work_id) } + Work::GossipPartialDataColumnSidecar { .. } => { + gossip_partial_data_column_queue.push(work, work_id) + } Work::DelayedImportBlock { .. } => { delayed_block_queue.push(work, work_id) } @@ -1416,6 +1426,9 @@ impl BeaconProcessor { WorkType::GossipBlock => gossip_block_queue.len(), WorkType::GossipBlobSidecar => gossip_blob_queue.len(), WorkType::GossipDataColumnSidecar => gossip_data_column_queue.len(), + WorkType::GossipPartialDataColumnSidecar => { + gossip_partial_data_column_queue.len() + } WorkType::DelayedImportBlock => delayed_block_queue.len(), WorkType::GossipVoluntaryExit => gossip_voluntary_exit_queue.len(), WorkType::GossipProposerSlashing => gossip_proposer_slashing_queue.len(), @@ -1591,7 +1604,8 @@ impl BeaconProcessor { Work::IgnoredRpcBlock { process_fn } => task_spawner.spawn_blocking(process_fn), Work::GossipBlock(work) | Work::GossipBlobSidecar(work) - | Work::GossipDataColumnSidecar(work) => task_spawner.spawn_async(async move { + | Work::GossipDataColumnSidecar(work) + | Work::GossipPartialDataColumnSidecar(work) => task_spawner.spawn_async(async move { work.await; }), Work::BlobsByRangeRequest(process_fn) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 98da7dbf2c7..9e3b984b969 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -592,6 +592,7 @@ pub struct EngineCapabilities { pub get_client_version_v1: bool, pub get_blobs_v1: bool, pub get_blobs_v2: bool, + pub get_blobs_v3: bool, } impl EngineCapabilities { diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index bc927e19b41..d85f5965c77 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -61,6 +61,7 @@ pub const ENGINE_GET_CLIENT_VERSION_TIMEOUT: Duration = Duration::from_secs(1); pub const ENGINE_GET_BLOBS_V1: &str = "engine_getBlobsV1"; pub const ENGINE_GET_BLOBS_V2: &str = "engine_getBlobsV2"; +pub const ENGINE_GET_BLOBS_V3: &str = "engine_getBlobsV3"; pub const ENGINE_GET_BLOBS_TIMEOUT: Duration = Duration::from_secs(1); /// This error is returned during a `chainId` call by Geth. @@ -736,6 +737,20 @@ impl HttpJsonRpc { .await } + pub async fn get_blobs_v3( + &self, + versioned_hashes: Vec, + ) -> Result>>, Error> { + let params = json!([versioned_hashes]); + + self.rpc_request( + ENGINE_GET_BLOBS_V3, + params, + ENGINE_GET_BLOBS_TIMEOUT * self.execution_timeout_multiplier, + ) + .await + } + pub async fn get_block_by_number( &self, query: BlockByNumberQuery<'_>, @@ -1186,6 +1201,7 @@ impl HttpJsonRpc { get_client_version_v1: capabilities.contains(ENGINE_GET_CLIENT_VERSION_V1), get_blobs_v1: capabilities.contains(ENGINE_GET_BLOBS_V1), get_blobs_v2: capabilities.contains(ENGINE_GET_BLOBS_V2), + get_blobs_v3: capabilities.contains(ENGINE_GET_BLOBS_V3), }) } diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 33decd4ec86..8b8fe228956 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -812,6 +812,9 @@ pub struct BlobAndProof { pub proofs: KzgProofs, } +/// A BlobAndProofV3 is just a BlobAndProofV2 that may also be `null` if unknown by the EL. +pub type BlobAndProofV3 = Option>; + #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct JsonForkchoiceStateV1 { diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index a5fa0f34158..09a130dfdb3 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -4,7 +4,7 @@ //! This crate only provides useful functionality for "The Merge", it does not provide any of the //! deposit-contract functionality that the `beacon_node/eth1` crate already provides. -use crate::json_structures::{BlobAndProofV1, BlobAndProofV2}; +use crate::json_structures::{BlobAndProofV1, BlobAndProofV2, BlobAndProofV3}; use crate::payload_cache::PayloadCache; use arc_swap::ArcSwapOption; use auth::{Auth, JwtKey, strip_prefix}; @@ -1895,6 +1895,23 @@ impl ExecutionLayer { } } + pub async fn get_blobs_v3( + &self, + query: Vec, + ) -> Result>>, Error> { + let capabilities = self.get_engine_capabilities(None).await?; + + if capabilities.get_blobs_v3 { + self.engine() + .request(|engine| async move { engine.api.get_blobs_v3(query).await }) + .await + .map_err(Box::new) + .map_err(Error::EngineError) + } else { + Err(Error::GetBlobsNotSupported) + } + } + pub async fn get_block_by_number( &self, query: BlockByNumberQuery<'_>, diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 712c773dda0..622132a0a8d 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -58,6 +58,7 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { get_client_version_v1: true, get_blobs_v1: true, get_blobs_v2: true, + get_blobs_v3: true, }; pub static DEFAULT_CLIENT_VERSION: LazyLock = diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 05a4a4b7a4a..a1412debc28 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -29,8 +29,9 @@ use tracing::{Span, debug, debug_span, error, info, instrument, warn}; use tree_hash::TreeHash; use types::{ AbstractExecPayload, BeaconBlockRef, BlobSidecar, BlobsList, BlockImportSource, - DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, - FullPayloadBellatrix, Hash256, KzgProofs, SignedBeaconBlock, SignedBlindedBeaconBlock, + DataColumnSidecar, DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, + FullPayload, FullPayloadBellatrix, Hash256, KzgProofs, SignedBeaconBlock, + SignedBlindedBeaconBlock, }; use warp::http::StatusCode; use warp::{Rejection, Reply, reply::Response}; @@ -247,7 +248,8 @@ pub async fn publish_block>( // Importing the columns could trigger block import and network publication in the case // where the block was already seen on gossip. if let Err(e) = - Box::pin(chain.process_gossip_data_columns(sampling_columns, publish_fn)).await + Box::pin(chain.process_gossip_data_columns(sampling_columns, publish_fn, |_| ())) + .await { let msg = format!("Invalid data column: {e}"); return if let BroadcastValidation::Gossip = validation_level { @@ -347,7 +349,7 @@ pub async fn publish_block>( type BuildDataSidecarTaskResult = Result< ( Vec>>, - Vec>, + Vec::EthSpec>>>, ), Rejection, >; @@ -405,7 +407,7 @@ fn build_data_columns( block: &SignedBeaconBlock>, blobs: BlobsList, kzg_cell_proofs: KzgProofs, -) -> Result>, Rejection> { +) -> Result>>, Rejection> { let slot = block.slot(); let data_column_sidecars = build_blob_data_column_sidecars(chain, block, blobs, kzg_cell_proofs).map_err(|e| { @@ -499,7 +501,7 @@ fn publish_blob_sidecars( fn publish_column_sidecars( sender_clone: &UnboundedSender>, - data_column_sidecars: &[GossipVerifiedDataColumn], + data_column_sidecars: &[GossipVerifiedDataColumn>], chain: &BeaconChain, ) -> Result<(), BlockError> { let malicious_withhold_count = chain.config.malicious_withhold_count; diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml index 7e69f6770bf..f7853e2412f 100644 --- a/beacon_node/lighthouse_network/Cargo.toml +++ b/beacon_node/lighthouse_network/Cargo.toml @@ -24,7 +24,7 @@ futures = { workspace = true } gossipsub = { workspace = true } hex = { workspace = true } itertools = { workspace = true } -libp2p-mplex = "0.43" +libp2p-mplex = { git = "https://github.com/jxs/rust-libp2p.git", branch = "gossipsub-partial-messages" } lighthouse_version = { workspace = true } local-ip-address = "0.6" logging = { workspace = true } @@ -33,7 +33,7 @@ lru_cache = { workspace = true } metrics = { workspace = true } network_utils = { workspace = true } parking_lot = { workspace = true } -prometheus-client = "0.23.0" +prometheus-client = "0.24.0" rand = { workspace = true } regex = { workspace = true } serde = { workspace = true } @@ -52,7 +52,8 @@ types = { workspace = true } unsigned-varint = { version = "0.8", features = ["codec"] } [dependencies.libp2p] -version = "0.56" +git = "https://github.com/jxs/rust-libp2p.git" +branch = "gossipsub-partial-messages" default-features = false features = [ "identify", diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index ea2c53a07fe..64120b38f2a 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -22,7 +22,7 @@ use crate::{Enr, NetworkGlobals, PubsubMessage, TopicHash, metrics}; use api_types::{AppRequestId, Response}; use futures::stream::StreamExt; use gossipsub::{ - IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, PublishError, + Event, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, PublishError, TopicScoreParams, }; use gossipsub_scoring_parameters::{PeerScoreSettings, lighthouse_gossip_thresholds}; @@ -805,9 +805,10 @@ impl Network { .write() .insert(topic.clone()); + let partial = topic.kind().supports_partial_messages(); let topic: Topic = topic.into(); - match self.gossipsub_mut().subscribe(&topic) { + match self.gossipsub_mut().subscribe(&topic, partial) { Err(e) => { warn!(%topic, error = ?e, "Failed to subscribe to topic"); false @@ -838,11 +839,8 @@ impl Network { pub fn publish(&mut self, messages: Vec>) { for message in messages { for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) { - let message_data = message.encode(GossipEncoding::default()); - if let Err(e) = self - .gossipsub_mut() - .publish(Topic::from(topic.clone()), message_data.clone()) - { + let result = message.publish(self.gossipsub_mut(), topic.clone().into()); + if let Err(e) = result { match e { PublishError::Duplicate => { debug!( @@ -880,7 +878,8 @@ impl Network { } if let PublishError::NoPeersSubscribedToTopic = e { - self.gossip_cache.insert(topic, message_data); + // TODO(dknopik): fix gossip cache + //self.gossip_cache.insert(gossip_topic, message); } } } @@ -1291,6 +1290,44 @@ impl Network { } } } + Event::Partial { + topic_id, + propagation_source, + group_id, + message, + metadata: _, + } => { + // TODO(dknopik): take a look at the metadata and publish if we have something for them + // maybe by reusing the gossip cache?! + + if let Some(message) = message { + match PubsubMessage::decode_partial(&topic_id, &group_id, &message) { + Err(error) => { + debug!( + topic = ?topic_id, + error, + "Could not decode gossipsub partial message" + ); + //reject the message + // TODO(dknopik): implement when ready in libp2p + //self.gossipsub_mut().report_message_validation_result( + // &todo!(), + // &propagation_source, + // MessageAcceptance::Reject, + //); + } + Ok(message) => { + // Notify the network + return Some(NetworkEvent::PubsubMessage { + id: MessageId::new(&[]), // TODO(dknopik): waht to send + source: propagation_source, + topic: topic_id, + message, + }); + } + } + } + } gossipsub::Event::Subscribed { peer_id, topic } => { if let Ok(topic) = GossipTopic::decode(topic.as_str()) { if let Some(subnet_id) = topic.subnet_id() { @@ -1764,7 +1801,10 @@ impl Network { fn inject_upnp_event(&mut self, event: libp2p::upnp::Event) { match event { - libp2p::upnp::Event::NewExternalAddr(addr) => { + libp2p::upnp::Event::NewExternalAddr { + external_addr: addr, + .. + } => { info!(%addr, "UPnP route established"); let mut iter = addr.iter(); let is_ip6 = { @@ -1794,7 +1834,7 @@ impl Network { } } } - libp2p::upnp::Event::ExpiredExternalAddr(_) => {} + libp2p::upnp::Event::ExpiredExternalAddr { .. } => {} libp2p::upnp::Event::GatewayNotFound => { info!("UPnP not available"); } diff --git a/beacon_node/lighthouse_network/src/types/mod.rs b/beacon_node/lighthouse_network/src/types/mod.rs index 3f57406fc78..748df3cb15c 100644 --- a/beacon_node/lighthouse_network/src/types/mod.rs +++ b/beacon_node/lighthouse_network/src/types/mod.rs @@ -1,4 +1,5 @@ mod globals; +mod partial; mod pubsub; mod subnet; mod topics; diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs new file mode 100644 index 00000000000..3a165127e1e --- /dev/null +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -0,0 +1,123 @@ +use gossipsub::partial::{Metadata, PublishAction}; +use gossipsub::{Partial, PartialMessageError}; +use ssz::{Decode, Encode}; +use std::fmt::Debug; +use std::sync::Arc; +use types::EthSpec; +use types::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataColumn}; + +#[derive(Debug, Clone, PartialEq)] +pub struct PartialDataColumnSidecarMessage { + pub partial_column: Arc>, + pub send_eager: Option>, +} + +impl From> for PartialDataColumnSidecarMessage { + fn from(value: DanglingPartialDataColumn) -> Self { + Self { + partial_column: Arc::new(value), + send_eager: None, + } + } +} + +impl From>> for PartialDataColumnSidecarMessage { + fn from(value: Arc>) -> Self { + Self { + partial_column: value, + send_eager: None, + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct SendEager { + /// The encoded message to send eagerly, i.e. when we have no metadata for that peer. + data: Vec, + /// The metadata to associate with a peer after sending it the eager message. + metadata: CellBitmapMetadata, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CellBitmapMetadata { + bitmap: CellBitmap, + encoded: Vec, +} + +impl Metadata for CellBitmapMetadata { + fn as_slice(&self) -> &[u8] { + &self.encoded + } + + fn update(&mut self, data: &[u8]) -> Result { + let data = CellBitmap::::from_ssz_bytes(data) + .map_err(|_| PartialMessageError::InvalidFormat)?; + if data.len() != self.bitmap.len() { + return Err(PartialMessageError::OutOfRange); + } + let new_bitmap = self.bitmap.union(&data); + if self.bitmap == new_bitmap { + return Ok(false); + } + self.bitmap = new_bitmap; + self.encoded = self.bitmap.as_ssz_bytes(); + Ok(true) + } +} + +impl From> for CellBitmapMetadata { + fn from(value: CellBitmap) -> Self { + Self { + encoded: value.as_ssz_bytes(), + bitmap: value, + } + } +} + +impl Partial for PartialDataColumnSidecarMessage { + fn group_id(&self) -> impl AsRef<[u8]> { + &self.partial_column.block_root + } + + fn parts_metadata(&self) -> impl AsRef<[u8]> { + self.partial_column + .sidecar + .cells_present_bitmap + .as_ssz_bytes() + } + + fn partial_message_bytes_from_metadata( + &self, + metadata: Option>, + ) -> Result { + match metadata { + None => { + // Send the eager message if any + match &self.send_eager { + None => Ok(PublishAction::NothingToSend), + Some(send) => Ok(PublishAction::Send { + message: send.data.clone(), + metadata: Box::new(send.metadata.clone()), + }), + } + } + Some(metadata) => { + let peer_has = CellBitmap::::from_ssz_bytes(metadata.as_ref()) + .map_err(|_| PartialMessageError::InvalidFormat)?; + if peer_has == self.partial_column.sidecar.cells_present_bitmap { + return Ok(PublishAction::SameMetadata); + } + + let Some(send) = self.partial_column.sidecar.with_missing_cells(&peer_has) else { + return Ok(PublishAction::NothingToSend); + }; + + let new_metadata = peer_has.union(&send.cells_present_bitmap); + Ok(PublishAction::Send { + message: send.as_ssz_bytes().to_vec(), + metadata: Box::new(CellBitmapMetadata::::from(new_metadata)), + }) + } + } + } +} diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 72f2873def9..a784eae2e9a 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -1,14 +1,17 @@ //! Handles the encoding and decoding of pubsub messages. -use crate::TopicHash; +use crate::types::partial::PartialDataColumnSidecarMessage; use crate::types::{GossipEncoding, GossipKind, GossipTopic}; +use crate::{Gossipsub, TopicHash}; +use gossipsub::{IdentTopic, PublishError}; use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; +use types::partial_data_column_sidecar::{DanglingPartialDataColumn, PartialDataColumnSidecar}; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, - DataColumnSidecar, DataColumnSubnetId, EthSpec, ForkContext, ForkName, + DataColumnSidecar, DataColumnSubnetId, EthSpec, ForkContext, ForkName, Hash256, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedAggregateAndProofBase, SignedAggregateAndProofElectra, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, @@ -26,6 +29,8 @@ pub enum PubsubMessage { BlobSidecar(Box<(u64, Arc>)>), /// Gossipsub message providing notification of a [`DataColumnSidecar`] along with the subnet id where it was received. DataColumnSidecar(Box<(DataColumnSubnetId, Arc>)>), + /// Gossipsub message providing notification of a [`PartialDataColumnSidecar`] along with the subnet id where it was received. + PartialDataColumnSidecar(Box<(DataColumnSubnetId, PartialDataColumnSidecarMessage)>), // TODO(dknopik): review `Arc` situation /// Gossipsub message providing notification of a Aggregate attestation and associated proof. AggregateAndProofAttestation(Box>), /// Gossipsub message providing notification of a `SingleAttestation` with its subnet id. @@ -135,6 +140,9 @@ impl PubsubMessage { PubsubMessage::DataColumnSidecar(column_sidecar_data) => { GossipKind::DataColumnSidecar(column_sidecar_data.0) } + PubsubMessage::PartialDataColumnSidecar(partial_column_sidecar_data) => { + GossipKind::DataColumnSidecar(partial_column_sidecar_data.0) + } PubsubMessage::AggregateAndProofAttestation(_) => GossipKind::BeaconAggregateAndProof, PubsubMessage::Attestation(attestation_data) => { GossipKind::Attestation(attestation_data.0) @@ -392,17 +400,49 @@ impl PubsubMessage { } } + pub fn decode_partial(topic: &TopicHash, group: &[u8], data: &[u8]) -> Result { + match GossipTopic::decode(topic.as_str()) { + Err(_) => Err(format!("Unknown gossipsub topic: {:?}", topic)), + Ok(gossip_topic) => match gossip_topic.kind() { + GossipKind::DataColumnSidecar(id) => { + let block_root = Hash256::from_ssz_bytes(group) + .map_err(|e| format!("Error decoding group: {:?}", e))?; + let sidecar = PartialDataColumnSidecar::from_ssz_bytes(data) + .map_err(|e| format!("Error decoding sidecar: {:?}", e))?; + let data_column = DanglingPartialDataColumn { + block_root, + // Partial messages are spec'd under the assumption that there is one column per subnet. + index: **id, + sidecar, + }; + Ok(Self::PartialDataColumnSidecar(Box::new(( + *id, + data_column.into(), + )))) + } + other => Err(format!("Partial message unsupported for topic: {other}")), + }, + } + } + /// Encodes a `PubsubMessage` based on the topic encodings. The first known encoding is used. If /// no encoding is known, and error is returned. - pub fn encode(&self, _encoding: GossipEncoding) -> Vec { + pub fn publish( + &self, + gossipsub: &mut Gossipsub, + topic: IdentTopic, + ) -> Result<(), PublishError> { // Currently do not employ encoding strategies based on the topic. All messages are ssz // encoded. // Also note, that the compression is handled by the `SnappyTransform` struct. Gossipsub will compress the // messages for us. - match &self { + let bytes = match &self { PubsubMessage::BeaconBlock(data) => data.as_ssz_bytes(), PubsubMessage::BlobSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::DataColumnSidecar(data) => data.1.as_ssz_bytes(), + PubsubMessage::PartialDataColumnSidecar(data) => { + return gossipsub.publish_partial(topic, &data.1); + } PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(), PubsubMessage::ProposerSlashing(data) => data.as_ssz_bytes(), @@ -413,7 +453,8 @@ impl PubsubMessage { PubsubMessage::BlsToExecutionChange(data) => data.as_ssz_bytes(), PubsubMessage::LightClientFinalityUpdate(data) => data.as_ssz_bytes(), PubsubMessage::LightClientOptimisticUpdate(data) => data.as_ssz_bytes(), - } + }; + gossipsub.publish(topic, bytes).map(|_| ()) } } @@ -438,6 +479,19 @@ impl std::fmt::Display for PubsubMessage { data.1.slot(), data.1.index, ), + PubsubMessage::PartialDataColumnSidecar(data) => write!( + f, + "PartialDataColumnSidecar: group: {}, column index: {}, cells: {}", + data.1.partial_column.block_root, + data.1.partial_column.block_root, + hex::encode( + data.1 + .partial_column + .sidecar + .cells_present_bitmap + .as_slice() + ), + ), PubsubMessage::AggregateAndProofAttestation(att) => write!( f, "Aggregate and Proof: slot: {}, index: {:?}, aggregator_index: {}", diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index b22adfbc487..fcc6bb9051d 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -178,6 +178,12 @@ pub enum GossipKind { LightClientOptimisticUpdate, } +impl GossipKind { + pub fn supports_partial_messages(&self) -> bool { + matches!(self, GossipKind::DataColumnSidecar(_)) + } +} + impl std::fmt::Display for GossipKind { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/beacon_node/lighthouse_tracing/src/lib.rs b/beacon_node/lighthouse_tracing/src/lib.rs index 56dccadaa94..154c5dbf6fe 100644 --- a/beacon_node/lighthouse_tracing/src/lib.rs +++ b/beacon_node/lighthouse_tracing/src/lib.rs @@ -12,6 +12,7 @@ pub const SPAN_PUBLISH_BLOCK: &str = "publish_block"; pub const SPAN_PENDING_COMPONENTS: &str = "pending_components"; /// Gossip methods root spans +pub const SPAN_PROCESS_GOSSIP_PARTIAL_DATA_COLUMN: &str = "process_gossip_partial_data_column"; pub const SPAN_PROCESS_GOSSIP_DATA_COLUMN: &str = "process_gossip_data_column"; pub const SPAN_PROCESS_GOSSIP_BLOB: &str = "process_gossip_blob"; pub const SPAN_PROCESS_GOSSIP_BLOCK: &str = "process_gossip_block"; diff --git a/beacon_node/network/src/lib.rs b/beacon_node/network/src/lib.rs index 2a7fedb53e9..db2fd876287 100644 --- a/beacon_node/network/src/lib.rs +++ b/beacon_node/network/src/lib.rs @@ -4,6 +4,7 @@ pub mod service; mod metrics; mod nat; mod network_beacon_processor; +mod partial_data_column_cache; mod persisted_dht; mod router; mod status; diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index cea06a28c86..efaae60267f 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -143,6 +143,22 @@ pub static BEACON_PROCESSOR_GOSSIP_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL: LazyLock< "Total number of gossip data column sidecar verified for propagation.", ) }); +pub static BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL: LazyLock< + Result, +> = LazyLock::new(|| { + try_create_int_counter( + "beacon_processor_gossip_partial_data_column_verified_total", + "Total number of gossip partial data column sidecar verified for propagation.", + ) +}); +pub static BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_CACHED_TOTAL: LazyLock< + Result, +> = LazyLock::new(|| { + try_create_int_counter( + "beacon_processor_gossip_partial_data_column_cached_total", + "Total number of gossip partial data column sidecars received before their block.", + ) +}); // Gossip Exits. pub static BEACON_PROCESSOR_EXIT_VERIFIED_TOTAL: LazyLock> = LazyLock::new(|| { @@ -560,6 +576,16 @@ pub static BEACON_DATA_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME: LazyLo decimal_buckets(-3, -1), ) }); +pub static BEACON_PARTIAL_DATA_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME: LazyLock< + Result, +> = LazyLock::new(|| { + try_create_histogram_with_buckets( + "beacon_partial_data_column_gossip_propagation_verification_delay_time", + "Duration between when the partial data column sidecar is received over gossip and when it is verified for propagation.", + // [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] + decimal_buckets(-3, -1), + ) +}); pub static BEACON_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME: LazyLock> = LazyLock::new(|| { try_create_histogram_with_buckets( @@ -574,6 +600,20 @@ pub static BEACON_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME: LazyLock> = + LazyLock::new(|| { + try_create_histogram_with_buckets( + "beacon_partial_data_column_gossip_slot_start_delay_time", + "Duration between when the partial data column sidecar is received over gossip and the start of the slot it belongs to.", + // Create a custom bucket list for greater granularity in block delay + Ok(vec![ + 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 5.0, + 6.0, 7.0, 8.0, 9.0, 10.0, 15.0, 20.0, + ]), // NOTE: Previous values, which we may want to switch back to. + // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] + //decimal_buckets(-1,2) + ) + }); pub static BEACON_BLOB_DELAY_GOSSIP_VERIFICATION: LazyLock> = LazyLock::new( || { diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index fa6b5fd2434..f3554a45954 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -6,11 +6,12 @@ use crate::{ }; use beacon_chain::blob_verification::{GossipBlobError, GossipVerifiedBlob}; use beacon_chain::block_verification_types::AsBlock; +use beacon_chain::data_availability_checker::MergedData; use beacon_chain::data_column_verification::{GossipDataColumnError, GossipVerifiedDataColumn}; use beacon_chain::store::Error; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, ForkChoiceError, - GossipVerifiedBlock, NotifyExecutionLayer, + GossipVerifiedBlock, IntoExecutionPendingBlock, NotifyExecutionLayer, attestation_verification::{self, Error as AttnError, VerifiedAttestation}, data_availability_checker::AvailabilityCheckErrorCategory, light_client_finality_update_verification::Error as LightClientFinalityUpdateError, @@ -20,9 +21,12 @@ use beacon_chain::{ validator_monitor::{get_block_delay_ms, get_slot_delay_ms}, }; use beacon_processor::{Work, WorkEvent}; -use lighthouse_network::{Client, MessageAcceptance, MessageId, PeerAction, PeerId, ReportSource}; +use lighthouse_network::{ + Client, MessageAcceptance, MessageId, PeerAction, PeerId, PubsubMessage, ReportSource, +}; use lighthouse_tracing::{ SPAN_PROCESS_GOSSIP_BLOB, SPAN_PROCESS_GOSSIP_BLOCK, SPAN_PROCESS_GOSSIP_DATA_COLUMN, + SPAN_PROCESS_GOSSIP_PARTIAL_DATA_COLUMN, }; use logging::crit; use operation_pool::ReceivedPreCapella; @@ -51,6 +55,9 @@ use beacon_processor::{ ReprocessQueueMessage, }, }; +use store::DatabaseBlock; +use types::das_column::DasColumn; +use types::partial_data_column_sidecar::{DanglingPartialDataColumn, VerifiablePartialDataColumn}; /// Set to `true` to introduce stricter penalties for peers who send some types of late consensus /// messages. @@ -768,6 +775,227 @@ impl NetworkBeaconProcessor { } } + #[instrument( + name = SPAN_PROCESS_GOSSIP_PARTIAL_DATA_COLUMN, + parent = None, + level = "debug", + skip_all, + fields(block_root = ?column_sidecar.block_root, index = column_sidecar.index), + )] + pub async fn process_gossip_dangling_partial_data_column_sidecar( + self: &Arc, + peer_id: PeerId, + _subnet_id: DataColumnSubnetId, + column_sidecar: Arc>, + seen_duration: Duration, + ) { + let partial_column = match self + .chain + .store + .try_get_full_block(&column_sidecar.block_root) + { + Ok(Some(DatabaseBlock::Full(block))) => { + Some(VerifiablePartialDataColumn::from_dangling_and_block( + column_sidecar.clone(), + &block, + )) + } + Ok(Some(DatabaseBlock::Blinded(block))) => { + Some(VerifiablePartialDataColumn::from_dangling_and_block( + column_sidecar.clone(), + &block, + )) + } + Ok(None) => None, + Err(err) => { + warn!(?err, "Error getting block for partial data column"); + None + } + }; + + match partial_column { + Some(Ok(partial_column)) => { + debug!("Received partial while having block"); + self.process_gossip_verifiable_partial_data_column_sidecar( + peer_id, + Arc::new(partial_column), + seen_duration, + ) + .await; + } + Some(Err(err)) => { + warn!(?err, "Error creating verifiable partial data column"); + } + None => { + debug!("Received partial while not having block"); + metrics::inc_counter( + &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_CACHED_TOTAL, + ); + self.partial_data_column_cache.lock().insert( + column_sidecar, + peer_id, + seen_duration, + ); + } + } + } + + #[instrument( + name = SPAN_PROCESS_GOSSIP_PARTIAL_DATA_COLUMN, + parent = None, + level = "debug", + skip_all, + fields(slot = %column_sidecar.slot(), block_root = ?column_sidecar.block_root(), index = column_sidecar.index()), + )] + pub async fn process_gossip_verifiable_partial_data_column_sidecar( + self: &Arc, + peer_id: PeerId, + column_sidecar: Arc>, + seen_duration: Duration, + ) { + let slot = column_sidecar.slot(); + let block_root = column_sidecar.block_root(); + let index = column_sidecar.index(); + let delay = get_slot_delay_ms(seen_duration, slot, &self.chain.slot_clock); + // Log metrics to track delay from other nodes on the network. + metrics::observe_duration( + &metrics::BEACON_PARTIAL_DATA_COLUMN_GOSSIP_SLOT_START_DELAY_TIME, + delay, + ); + match self + .chain + .verify_partial_data_column_sidecar_for_gossip(column_sidecar.clone()) + { + Ok(gossip_verified_data_column) => { + metrics::inc_counter( + &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_VERIFIED_TOTAL, + ); + + debug!( + %slot, + %block_root, + %index, + "Successfully verified gossip partial data column sidecar" + ); + + // TODO(dknopik): wait for joao's validation result impl + //self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Accept); + + // Log metrics to keep track of propagation delay times. + if let Some(duration) = SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .and_then(|now| now.checked_sub(seen_duration)) + { + metrics::observe_duration( + &metrics::BEACON_PARTIAL_DATA_COLUMN_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME, + duration, + ); + } + + self.process_gossip_verified_data_column( + peer_id, + gossip_verified_data_column, + seen_duration, + ) + .await + } + Err(err) => { + match err { + GossipDataColumnError::PriorKnownUnpublished => { + debug!( + %slot, + %block_root, + %index, + "Gossip data column already processed via the EL. Accepting the column sidecar without re-processing." + ); + // TODO(dknopik): Joao + //self.propagate_validation_result( + // message_id, + // peer_id, + // MessageAcceptance::Accept, + //); + } + // ParentUnknown can't really happen, so we treat it like an internal error + GossipDataColumnError::ParentUnknown { .. } + | GossipDataColumnError::PubkeyCacheTimeout + | GossipDataColumnError::BeaconChainError(_) => { + crit!( + error = ?err, + "Internal error when verifying column sidecar" + ) + } + GossipDataColumnError::ProposalSignatureInvalid + | GossipDataColumnError::UnknownValidator(_) + | GossipDataColumnError::ProposerIndexMismatch { .. } + | GossipDataColumnError::IsNotLaterThanParent { .. } + | GossipDataColumnError::InvalidSubnetId { .. } + | GossipDataColumnError::InvalidInclusionProof + | GossipDataColumnError::InvalidKzgProof { .. } + | GossipDataColumnError::UnexpectedDataColumn + | GossipDataColumnError::InvalidColumnIndex(_) + | GossipDataColumnError::MaxBlobsPerBlockExceeded { .. } + | GossipDataColumnError::InconsistentCommitmentsLength { .. } + | GossipDataColumnError::InconsistentProofsLength { .. } + | GossipDataColumnError::NotFinalizedDescendant { .. } => { + debug!( + error = ?err, + %slot, + %block_root, + %index, + "Could not verify column sidecar for gossip. Rejecting the column sidecar" + ); + // Prevent recurring behaviour by penalizing the peer slightly. + self.gossip_penalize_peer( + peer_id, + PeerAction::LowToleranceError, + "gossip_data_column_low", + ); + // TODO(dknopik): Joao + //self.propagate_validation_result( + // message_id, + // peer_id, + // MessageAcceptance::Reject, + //); + } + GossipDataColumnError::PriorKnown { .. } => { + // Data column is available via either the EL or reconstruction. + // Do not penalise the peer. + // Gossip filter should filter any duplicates received after this. + debug!( + %slot, + %block_root, + %index, + "Received already available column sidecar. Ignoring the column sidecar" + ) + } + GossipDataColumnError::FutureSlot { .. } + | GossipDataColumnError::PastFinalizedSlot { .. } => { + debug!( + error = ?err, + %slot, + %block_root, + %index, + "Could not verify column sidecar for gossip. Ignoring the column sidecar" + ); + // Prevent recurring behaviour by penalizing the peer slightly. + self.gossip_penalize_peer( + peer_id, + PeerAction::HighToleranceError, + "gossip_data_column_high", + ); + // TODO(dknopik): Joao + //self.propagate_validation_result( + // message_id, + // peer_id, + // MessageAcceptance::Ignore, + //); + } + } + } + } + } + #[allow(clippy::too_many_arguments)] #[instrument( name = SPAN_PROCESS_GOSSIP_BLOB, @@ -1014,10 +1242,10 @@ impl NetworkBeaconProcessor { } } - async fn process_gossip_verified_data_column( + async fn process_gossip_verified_data_column>( self: &Arc, peer_id: PeerId, - verified_data_column: GossipVerifiedDataColumn, + verified_data_column: GossipVerifiedDataColumn, // This value is not used presently, but it might come in handy for debugging. _seen_duration: Duration, ) { @@ -1026,9 +1254,45 @@ impl NetworkBeaconProcessor { let data_column_slot = verified_data_column.slot(); let data_column_index = verified_data_column.index(); + let cloned_self = self.clone(); + let data_publish_fn = move |merged_data: MergedData| { + debug!( + partial = merged_data.updated_partials.len(), + full = merged_data.completed_columns.len(), + "Sending merged data" + ); + let messages: Vec<_> = merged_data + .updated_partials + .into_iter() + .map(|partial| { + PubsubMessage::PartialDataColumnSidecar(Box::new(( + DataColumnSubnetId::from_column_index( + partial.index(), + &cloned_self.chain.spec, + ), + partial.column.clone().into(), + ))) + }) + .chain(merged_data.completed_columns.into_iter().flat_map(|full| { + let subnet = + DataColumnSubnetId::from_column_index(full.index, &self.chain.spec); + [ + PubsubMessage::PartialDataColumnSidecar(Box::new(( + subnet, + (*full).clone().into_partial().column.clone().into(), + ))), + PubsubMessage::DataColumnSidecar(Box::new((subnet, full))), + ] + })) + .collect(); + if !messages.is_empty() { + cloned_self.send_network_message(NetworkMessage::Publish { messages }) + } + }; + let result = self .chain - .process_gossip_data_columns(vec![verified_data_column], || Ok(())) + .process_gossip_data_columns(vec![verified_data_column], || Ok(()), data_publish_fn) .await; register_process_result_metrics(&result, metrics::BlockSource::Gossip, "data_column"); @@ -1166,6 +1430,35 @@ impl NetworkBeaconProcessor { Span::current().record("block_root", block_root.to_string()); if let Some(handle) = duplicate_cache.check_and_insert(block_root) { + // First, get the partial columns that have been waiting. Do this first in order to gossip them quickly (i guess?) TODO(dknopik): check if good + let partials = { + let mut partial_cache = self.partial_data_column_cache.lock(); + let partials = partial_cache.get_for_block(block_root); + // Opportunistically clean the cache as we are holding the lock anyway. TODO(dknopik): do somewhere else + partial_cache.clean(); + partials + }; + + for cached_partial in partials { + // TODO(dknopik): Do this in parallel (queueing it maybe) + match VerifiablePartialDataColumn::from_dangling_and_block( + cached_partial.sidecar, + gossip_verified_block.block(), + ) { + Ok(partial_data_column) => { + self.process_gossip_verifiable_partial_data_column_sidecar( + cached_partial.peer_id, + Arc::new(partial_data_column), + cached_partial.seen_duration, + ) + .await; + } + Err(err) => { + warn!(?err, "Unable to process cached partial column") + } + } + } + self.process_gossip_verified_block( peer_id, gossip_verified_block, diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 7441e928712..3310b9c72ec 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -1,3 +1,4 @@ +use crate::partial_data_column_cache::PartialDataColumnCache; use crate::sync::manager::BlockProcessType; use crate::{service::NetworkMessage, sync::manager::SyncMessage}; use beacon_chain::blob_verification::{GossipBlobError, observe_gossip_blob}; @@ -11,6 +12,7 @@ use beacon_processor::{ BeaconProcessorSend, DuplicateCache, GossipAggregatePackage, GossipAttestationPackage, Work, WorkEvent as BeaconWorkEvent, }; +use futures::FutureExt; use lighthouse_network::rpc::InboundRequestId; use lighthouse_network::rpc::methods::{ BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, DataColumnsByRootRequest, @@ -21,6 +23,7 @@ use lighthouse_network::{ Client, MessageId, NetworkGlobals, PeerId, PubsubMessage, rpc::{BlocksByRangeRequest, BlocksByRootRequest, LightClientBootstrapRequest, StatusMessage}, }; +use parking_lot::Mutex; use rand::prelude::SliceRandom; use std::path::PathBuf; use std::sync::Arc; @@ -28,6 +31,7 @@ use std::time::Duration; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, error::TrySendError}; use tracing::{debug, error, instrument, trace, warn}; +use types::partial_data_column_sidecar::DanglingPartialDataColumn; use types::*; pub use sync_methods::ChainSegmentProcessId; @@ -55,6 +59,7 @@ pub enum InvalidBlockStorage { pub struct NetworkBeaconProcessor { pub beacon_processor_send: BeaconProcessorSend, pub duplicate_cache: DuplicateCache, + pub partial_data_column_cache: Mutex>, pub chain: Arc>, pub network_tx: mpsc::UnboundedSender>, pub sync_tx: mpsc::UnboundedSender>, @@ -249,6 +254,32 @@ impl NetworkBeaconProcessor { }) } + /// Create a new `Work` event for some partial data column sidecar. + pub fn send_gossip_partial_data_column_sidecar( + self: &Arc, + peer_id: PeerId, + subnet_id: DataColumnSubnetId, + column_sidecar: Arc>, + seen_timestamp: Duration, + ) -> Result<(), Error> { + let processor = self.clone(); + let process_fn = async move { + processor + .process_gossip_dangling_partial_data_column_sidecar( + peer_id, + subnet_id, + column_sidecar, + seen_timestamp, + ) + .await + }; + + self.try_send(BeaconWorkEvent { + drop_during_sync: false, + work: Work::GossipPartialDataColumnSidecar(Box::pin(process_fn)), + }) + } + /// Create a new `Work` event for some sync committee signature. pub fn send_gossip_sync_signature( self: &Arc, @@ -762,6 +793,7 @@ impl NetworkBeaconProcessor { let epoch = block.slot().epoch(T::EthSpec::slots_per_epoch()); let custody_columns = self.chain.sampling_columns_for_epoch(epoch); let self_cloned = self.clone(); + let block_cloned = block.clone(); let publish_fn = move |blobs_or_data_column| { if publish_blobs { match blobs_or_data_column { @@ -772,10 +804,33 @@ impl NetworkBeaconProcessor { ); } EngineGetBlobsOutput::CustodyColumns(columns) => { + // Gradually publish any full columns self_cloned.publish_data_columns_gradually( - columns.into_iter().map(|c| c.clone_arc()).collect(), + columns + .iter() + .flat_map(|c| { + c.clone_arc() + .as_full(Some(&block_cloned)) + .map(|block| Arc::new(block.into_owned())) + }) + .collect(), block_root, ); + // "Publish" all columns as partial without eager send + self_cloned.send_network_message(NetworkMessage::Publish { + messages: columns + .into_iter() + .map(|c| { + PubsubMessage::PartialDataColumnSidecar(Box::new(( + DataColumnSubnetId::from_column_index( + c.index(), + &self_cloned.chain.spec, + ), + c.into_partial().into_inner().column.clone().into(), + ))) + }) + .collect(), + }) } }; } @@ -1036,6 +1091,7 @@ impl NetworkBeaconProcessor { } } +use types::das_column::DasColumn; #[cfg(test)] use { beacon_chain::builder::Witness, beacon_processor::BeaconProcessorChannels, @@ -1068,6 +1124,7 @@ impl NetworkBeaconProcessor> { let network_beacon_processor = Self { beacon_processor_send: beacon_processor_tx, duplicate_cache: DuplicateCache::default(), + partial_data_column_cache: Mutex::new(PartialDataColumnCache::new()), chain, network_tx, sync_tx, diff --git a/beacon_node/network/src/partial_data_column_cache.rs b/beacon_node/network/src/partial_data_column_cache.rs new file mode 100644 index 00000000000..07f7cccabcc --- /dev/null +++ b/beacon_node/network/src/partial_data_column_cache.rs @@ -0,0 +1,76 @@ +use lighthouse_network::PeerId; +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use types::partial_data_column_sidecar::DanglingPartialDataColumn; +use types::{EthSpec, Hash256}; + +const BLOCK_LIMIT: usize = 16; +const SIDECAR_PER_BLOCK_LIMIT: usize = 16; +const EXPIRATION_TIME: Duration = Duration::from_secs(24); + +/// Really dumb hacky implementation of a cache for partial data column sidecars. +/// +/// The issue is that (with the current spec draft) we have to match up the dangling partial data +/// columns sidecars with the corresponding block. Of course, usually we have to take great care to +/// not be exploitable, but this cache design (for now) assumes no malicious behaviour. +/// +/// Do not take anything in this file as an actual implementation suggestion, it is just a hack +/// while we discuss the spec! +pub struct PartialDataColumnCache { + per_block: HashMap>>, +} + +pub struct CachedPartial { + pub sidecar: Arc>, + pub peer_id: PeerId, + pub seen_duration: Duration, +} + +impl PartialDataColumnCache { + pub fn new() -> Self { + Self { + per_block: HashMap::new(), + } + } + + pub fn insert( + &mut self, + sidecar: Arc>, + peer_id: PeerId, + seen_duration: Duration, + ) { + let len = self.per_block.len(); + let entry = self.per_block.entry(sidecar.block_root); + if matches!(entry, Entry::Vacant(_)) && len >= BLOCK_LIMIT { + return; + } + + let sidecars = entry.or_default(); + if sidecars.len() < SIDECAR_PER_BLOCK_LIMIT { + sidecars.push(CachedPartial { + sidecar, + peer_id, + seen_duration, + }); + } + } + + pub fn get_for_block(&mut self, block_root: Hash256) -> Vec> { + self.per_block.remove(&block_root).unwrap_or_default() + } + + pub fn clean(&mut self) { + if let Ok(now) = SystemTime::now().duration_since(UNIX_EPOCH) { + self.per_block.retain(|_, sidecars| { + sidecars + .iter() + .map(|cached| cached.seen_duration) + .max() + .map(|last_seen| now.saturating_sub(last_seen) < EXPIRATION_TIME) + .unwrap_or(false) + }); + } + } +} diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 60fe094bb7c..30ccb4319ae 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -6,6 +6,7 @@ #![allow(clippy::unit_arg)] use crate::network_beacon_processor::{InvalidBlockStorage, NetworkBeaconProcessor}; +use crate::partial_data_column_cache::PartialDataColumnCache; use crate::service::NetworkMessage; use crate::status::status_message; use crate::sync::SyncMessage; @@ -19,6 +20,7 @@ use lighthouse_network::{ }; use logging::TimeLatch; use logging::crit; +use parking_lot::Mutex; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; @@ -97,6 +99,7 @@ impl Router { let network_beacon_processor = NetworkBeaconProcessor { beacon_processor_send, duplicate_cache: DuplicateCache::default(), + partial_data_column_cache: Mutex::new(PartialDataColumnCache::new()), chain: beacon_chain.clone(), network_tx: network_send.clone(), sync_tx: sync_send.clone(), @@ -384,6 +387,18 @@ impl Router { ), ) } + PubsubMessage::PartialDataColumnSidecar(data) => { + let (subnet_id, column_sidecar) = *data; + self.handle_beacon_processor_send_result( + self.network_beacon_processor + .send_gossip_partial_data_column_sidecar( + peer_id, + subnet_id, + column_sidecar.partial_column, + timestamp_now(), + ), + ) + } PubsubMessage::VoluntaryExit(exit) => { debug!(%peer_id, "Received a voluntary exit"); self.handle_beacon_processor_send_result( diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index 4bd649ba824..dd9448983c6 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -534,7 +534,7 @@ impl NetworkService { let subnet_id = subnet_and_attestation.0; let attestation = &subnet_and_attestation.1; // checks if we have an aggregator for the slot. If so, we should process - // the attestation, else we just just propagate the Attestation. + // the attestation, else we just propagate the Attestation. let should_process = self.subnet_service.should_process_attestation( Subnet::Attestation(subnet_id), &attestation.data, diff --git a/consensus/types/src/das_column.rs b/consensus/types/src/das_column.rs new file mode 100644 index 00000000000..3e25576fdac --- /dev/null +++ b/consensus/types/src/das_column.rs @@ -0,0 +1,105 @@ +//! A PeerDAS data column. May or may not contain all cells. This is not necessarily implementable +//! for a beacon spec container, more as more data might be required, such as the partial message +//! group id alongside a partial message. + +use crate::beacon_block_body::KzgCommitments; +use crate::data_column_sidecar::{Cell, DataColumn}; +use crate::partial_data_column_sidecar::VerifiablePartialDataColumn; +use crate::{ + ColumnIndex, DataColumnSidecar, EthSpec, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, + Slot, +}; +use kzg::{KzgCommitment, KzgProof}; +use ssz_types::VariableList; +use std::borrow::Cow; + +// TODO(dknopik): Name good? +// TODO(dknopik): Maybe move to unified cell storage? +// TODO(dknopik): Move generic parameter to associated type? +pub trait DasColumn: Clone { + fn slot(&self) -> Slot; + fn index(&self) -> ColumnIndex; + fn cell_count_total(&self) -> usize; + fn cells_present(&self) -> impl Iterator; + fn column(&self) -> &DataColumn; + fn kzg_proofs(&self) -> &VariableList; + fn kzg_commitments(&self) -> &KzgCommitments; + fn block_root(&self) -> Hash256; + fn signed_block_header(&self) -> Option<&SignedBeaconBlockHeader>; + fn into_partial(self) -> VerifiablePartialDataColumn; + + /// Convert this column into a full data column (e.g. for gossip). Note that this is potentially + /// expensive. + fn as_full( + &self, + header: Option<&SignedBeaconBlock>, + ) -> Option>>; + + fn iter(&self) -> impl Iterator>>; + + fn compare>(&self, rhs: &C) -> ColumnComparison { + if self.slot() == rhs.slot() + && self.index() == rhs.index() + && self.block_root() == rhs.block_root() + { + return ColumnComparison::DifferentColumns; + } + + if self.cell_count_total() != rhs.cell_count_total() { + return ColumnComparison::DataConflict; + } + + let mut missing_in_rhs = vec![]; + let mut missing_in_lhs = vec![]; + for (index, (lhs, rhs)) in self.iter().zip(rhs.iter()).enumerate() { + match (lhs, rhs) { + (None, None) => {} + (Some(_), None) => missing_in_rhs.push(index), + (None, Some(_)) => missing_in_lhs.push(index), + (Some(lhs), Some(rhs)) => { + if lhs != rhs { + return ColumnComparison::DataConflict; + } + } + } + } + if missing_in_rhs.is_empty() && missing_in_lhs.is_empty() { + return ColumnComparison::Equal; + } + + ColumnComparison::MissingCells { + missing_in_lhs, + missing_in_rhs, + } + } +} + +pub enum ColumnComparison { + DifferentColumns, + DataConflict, + MissingCells { + missing_in_lhs: Vec, + missing_in_rhs: Vec, + }, + Equal, +} + +impl PartialEq> for VerifiablePartialDataColumn { + fn eq(&self, other: &DataColumnSidecar) -> bool { + // Slight optimisation: Can only be the same if `self` is fully present + self.column.sidecar.is_complete() + && self.slot() == other.slot() + && self.index() == other.index() + && self.block_root() == other.block_root() + && self.kzg_commitments() == other.kzg_commitments() + && self.column() == other.column() + && self.kzg_proofs() == other.kzg_proofs() + } +} + +#[derive(Clone, Copy, PartialEq, Debug)] +pub struct CellWithMetadata<'a, E: EthSpec> { + pub cell: &'a Cell, + pub proof: &'a KzgProof, + pub commitment: &'a KzgCommitment, +} diff --git a/consensus/types/src/data_column_sidecar.rs b/consensus/types/src/data_column_sidecar.rs index 2272b1695c9..725425c44e8 100644 --- a/consensus/types/src/data_column_sidecar.rs +++ b/consensus/types/src/data_column_sidecar.rs @@ -1,10 +1,14 @@ use crate::beacon_block_body::{BLOB_KZG_COMMITMENTS_INDEX, KzgCommitments}; -use crate::context_deserialize; +use crate::das_column::{CellWithMetadata, DasColumn}; +use crate::partial_data_column_sidecar::{ + CellBitmap, DanglingPartialDataColumn, PartialDataColumnSidecar, VerifiablePartialDataColumn, +}; use crate::test_utils::TestRandom; use crate::{ BeaconBlockHeader, BeaconStateError, Epoch, EthSpec, ForkName, Hash256, SignedBeaconBlockHeader, Slot, }; +use crate::{SignedBeaconBlock, context_deserialize}; use bls::Signature; use derivative::Derivative; use kzg::Error as KzgError; @@ -16,13 +20,14 @@ use ssz::Encode; use ssz_derive::{Decode, Encode}; use ssz_types::Error as SszError; use ssz_types::{FixedVector, VariableList}; +use std::borrow::Cow; use std::sync::Arc; use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; pub type ColumnIndex = u64; -pub type Cell = FixedVector::BytesPerCell>; +pub type Cell = FixedVector::BytesPerCell>; // TODO(dknopik): Arc<[u8; E::BytesPerCell]> ??? cell level arcing acors the codebase seems reasonable pub type DataColumn = VariableList, ::MaxBlobCommitmentsPerBlock>; /// Identifies a set of data columns associated with a specific beacon block. @@ -169,3 +174,86 @@ impl From for DataColumnSidecarError { Self::SszError(e) } } + +impl DasColumn for DataColumnSidecar { + fn slot(&self) -> Slot { + self.slot() + } + + fn index(&self) -> ColumnIndex { + self.index + } + + fn cell_count_total(&self) -> usize { + self.column.len() + } + + fn cells_present(&self) -> impl Iterator { + 0..self.cell_count_total() + } + + fn column(&self) -> &DataColumn { + &self.column + } + + fn kzg_proofs(&self) -> &VariableList { + &self.kzg_proofs + } + + fn kzg_commitments(&self) -> &KzgCommitments { + &self.kzg_commitments + } + + fn block_root(&self) -> Hash256 { + self.block_root() + } + + fn signed_block_header(&self) -> Option<&SignedBeaconBlockHeader> { + Some(&self.signed_block_header) + } + + fn into_partial(self) -> VerifiablePartialDataColumn { + let mut bitmap = CellBitmap::::with_capacity(self.cell_count_total()) + .expect("our column has the same bound"); + for idx in 0..self.cell_count_total() { + bitmap + .set(idx, true) + .expect("The correct size is initialized right above"); + } + + VerifiablePartialDataColumn { + slot: self.slot(), + column: Arc::new(DanglingPartialDataColumn { + block_root: self.block_root(), + index: self.index(), + sidecar: PartialDataColumnSidecar { + cells_present_bitmap: bitmap, + column: self.column, + kzg_proofs: self.kzg_proofs, + }, + }), + kzg_commitments: self.kzg_commitments, + } + } + + fn as_full( + &self, + _block: Option<&SignedBeaconBlock>, + ) -> Option>> { + Some(Cow::Borrowed(self)) + } + + fn iter(&self) -> impl Iterator>> { + self.column + .iter() + .zip(self.kzg_commitments.iter()) + .zip(self.kzg_proofs.iter()) + .map(|((cell, commitment), proof)| { + Some(CellWithMetadata { + cell, + commitment, + proof, + }) + }) + } +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 8e83fed1d9a..eb8726a6963 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -104,11 +104,13 @@ pub mod slot_data; pub mod sqlite; pub mod blob_sidecar; +pub mod das_column; pub mod data_column_custody_group; pub mod data_column_sidecar; pub mod data_column_subnet_id; pub mod light_client_header; pub mod non_zero_usize; +pub mod partial_data_column_sidecar; pub mod runtime_fixed_vector; pub mod runtime_var_list; diff --git a/consensus/types/src/partial_data_column_sidecar.rs b/consensus/types/src/partial_data_column_sidecar.rs new file mode 100644 index 00000000000..e0a87d6cf02 --- /dev/null +++ b/consensus/types/src/partial_data_column_sidecar.rs @@ -0,0 +1,364 @@ +use crate::beacon_block_body::KzgCommitments; +use crate::das_column::{CellWithMetadata, DasColumn}; +use crate::data_column_sidecar::{Cell, DataColumn}; +use crate::test_utils::TestRandom; +use crate::{AbstractExecPayload, ColumnIndex, DataColumnSidecar, context_deserialize}; +use crate::{EthSpec, ForkName, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, Slot}; +use derivative::Derivative; +use kzg::KzgProof; +use serde::{Deserialize, Serialize}; +use ssz::{BitList, Encode}; +use ssz_derive::{Decode, Encode}; +use ssz_types::VariableList; +use std::borrow::Cow; +use std::sync::Arc; +use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::TreeHash; + +pub type CellBitmap = BitList<::MaxBlobCommitmentsPerBlock>; + +#[cfg_attr( + feature = "arbitrary", + derive(arbitrary::Arbitrary), + arbitrary(bound = "E: EthSpec") +)] +#[derive( + Debug, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom, Derivative, +)] +#[serde(bound = "E: EthSpec")] +#[derivative(PartialEq, Eq, Hash(bound = "E: EthSpec"))] +#[context_deserialize(ForkName)] +pub struct PartialDataColumnSidecar { + pub cells_present_bitmap: CellBitmap, + #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] + pub column: DataColumn, + pub kzg_proofs: VariableList, +} + +impl PartialDataColumnSidecar { + pub fn min_size() -> usize { + // min size is one cell + Self { + cells_present_bitmap: BitList::with_capacity(1).unwrap(), + column: VariableList::new(vec![Cell::::default()]).unwrap(), + kzg_proofs: VariableList::new(vec![KzgProof::empty()]).unwrap(), + } + .as_ssz_bytes() + .len() + } + + pub fn size(present_blobs: usize, block_blobs: usize) -> usize { + // min size is one cell + Self { + cells_present_bitmap: BitList::with_capacity(block_blobs).unwrap(), + column: VariableList::new(vec![Cell::::default(); present_blobs]).unwrap(), + kzg_proofs: VariableList::new(vec![KzgProof::empty(); present_blobs]).unwrap(), + } + .as_ssz_bytes() + .len() + } + + pub fn max_size(max_blobs_per_block: usize) -> usize { + Self { + cells_present_bitmap: BitList::with_capacity(max_blobs_per_block).unwrap(), + column: VariableList::new(vec![Cell::::default(); max_blobs_per_block]).unwrap(), + kzg_proofs: VariableList::new(vec![KzgProof::empty(); max_blobs_per_block]).unwrap(), + } + .as_ssz_bytes() + .len() + } + + pub fn is_complete(&self) -> bool { + self.cells_present_bitmap.iter().all(|bit| bit) + } + + pub fn with_missing_cells(&self, bitmap: &CellBitmap) -> Option { + if self.cells_present_bitmap.len() != bitmap.len() { + return None; + } + self.clone_filter(|idx| !bitmap.get(idx).expect("Bounds checked above")) + } + + /// Creates a new partial data column sidecar containing only the blob indices for which the + /// passed closure returns `true` and were present in `self`. Will return `None` if there is no + /// overlap. + pub fn clone_filter(&self, filter: F) -> Option + where + F: Fn(usize) -> bool, + { + let mut new_bitmap = self.cells_present_bitmap.clone(); + let mut new_column = VariableList::default(); + let mut new_proofs = VariableList::default(); + let mut column_idx = 0; + + for (blob_idx, present) in self.cells_present_bitmap.iter().enumerate() { + if present { + if filter(blob_idx) { + // Keep this cell + let cell = self.column.get(column_idx)?; + new_column + .push(cell.clone()) + .expect("Has same capacity as existing column"); + let proof = self.kzg_proofs.get(column_idx)?; + new_proofs + .push(*proof) + .expect("Has same capacity as existing column"); + } else { + // Mark as not present + new_bitmap + .set(blob_idx, false) + .expect("Within bounds due to clone above"); + } + column_idx = column_idx + .checked_add(1) + .expect("Will not have more cells than 2^64 - 1"); + } + } + + if new_column.is_empty() { + return None; + } + + Some(Self { + cells_present_bitmap: new_bitmap, + column: new_column, + kzg_proofs: new_proofs, + }) + } + + pub fn merge(&self, other: &Self) -> Option { + let new_bitmap = self.cells_present_bitmap.union(&other.cells_present_bitmap); + let mut new_column = VariableList::default(); + let mut new_proofs = VariableList::default(); + let mut self_cell_idx = 0usize; + let mut other_cell_idx = 0usize; + + for presence_bits in self + .cells_present_bitmap + .iter() + .zip(other.cells_present_bitmap.iter()) + { + match presence_bits { + (false, false) => {} + (true, other) => { + new_column + .push(self.column.get(self_cell_idx)?.clone()) + .expect("Has same capacity"); + new_proofs + .push(*self.kzg_proofs.get(self_cell_idx)?) + .expect("Has same capacity"); + self_cell_idx = self_cell_idx + .checked_add(1) + .expect("Will not have more cells than 2^64 - 1"); + if other { + other_cell_idx = other_cell_idx + .checked_add(1) + .expect("Will not have more cells than 2^64 - 1"); + } + } + (false, true) => { + new_column + .push(other.column.get(other_cell_idx)?.clone()) + .expect("Has same capacity"); + new_proofs + .push(*other.kzg_proofs.get(other_cell_idx)?) + .expect("Has same capacity"); + other_cell_idx = other_cell_idx + .checked_add(1) + .expect("Will not have more cells than 2^64 - 1"); + } + } + } + + Some(Self { + cells_present_bitmap: new_bitmap, + column: new_column, + kzg_proofs: new_proofs, + }) + } +} + +// TODO(dknopik): More specific error cases - e.g. internal inconsistency +pub struct MissingCellError; + +impl From> for PartialDataColumnSidecar { + fn from(value: DataColumnSidecar) -> Self { + // Create a bitmap with all cells marked as present + let mut cells_present_bitmap = BitList::with_capacity(value.column.len()) + .expect("Bitmap and cell list are both bounded by `MaxBlobCommitmentsPerBlock`"); + for idx in 0..value.column.len() { + cells_present_bitmap.set(idx, true).expect( + "Bitmap was created with column length, so we should be able to push a value", + ); + } + + Self { + cells_present_bitmap, + column: value.column, + kzg_proofs: value.kzg_proofs, + } + } +} + +// TODO(dknopik): Name? +#[derive(Debug, Clone, PartialEq)] +pub struct DanglingPartialDataColumn { + pub block_root: Hash256, + pub index: ColumnIndex, + pub sidecar: PartialDataColumnSidecar, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct VerifiablePartialDataColumn { + pub column: Arc>, + pub kzg_commitments: KzgCommitments, + pub slot: Slot, +} + +// TODO(dknopik): Is there an existing approriate error type? +#[derive(Debug)] +pub enum PartialDataColumnMatchingError { + MismatchingBlock, + InvalidForkBlock, +} + +impl VerifiablePartialDataColumn { + pub fn from_dangling_and_block>( + column: Arc>, + block: &SignedBeaconBlock, + ) -> Result { + if column.block_root != block.canonical_root() { + return Err(PartialDataColumnMatchingError::MismatchingBlock); + } + + let kzg_commitments = block + .message() + .body() + .blob_kzg_commitments() + .map_err(|_| PartialDataColumnMatchingError::InvalidForkBlock)? + .clone(); + + Ok(VerifiablePartialDataColumn { + column, + kzg_commitments, + slot: block.slot(), + }) + } + + pub fn clone_filter(&self, filter: F) -> Option + where + F: Fn(usize) -> bool, + { + Some(VerifiablePartialDataColumn { + column: Arc::new(DanglingPartialDataColumn { + sidecar: self.column.sidecar.clone_filter(filter)?, + block_root: self.column.block_root, + index: self.column.index, + }), + kzg_commitments: self.kzg_commitments.clone(), + slot: self.slot, + }) + } +} + +impl DasColumn for VerifiablePartialDataColumn { + fn slot(&self) -> Slot { + self.slot + } + + fn index(&self) -> ColumnIndex { + self.column.index + } + + fn cell_count_total(&self) -> usize { + self.column.sidecar.cells_present_bitmap.len() + } + + fn cells_present(&self) -> impl Iterator { + self.column + .sidecar + .cells_present_bitmap + .iter() + .enumerate() + .filter_map(|(idx, bit)| bit.then_some(idx)) + } + + fn column(&self) -> &DataColumn { + &self.column.sidecar.column + } + + fn kzg_proofs(&self) -> &VariableList { + &self.column.sidecar.kzg_proofs + } + + fn kzg_commitments(&self) -> &KzgCommitments { + &self.kzg_commitments + } + + fn block_root(&self) -> Hash256 { + self.column.block_root.tree_hash_root() + } + + fn signed_block_header(&self) -> Option<&SignedBeaconBlockHeader> { + None + } + + fn into_partial(self) -> VerifiablePartialDataColumn { + self + } + + fn as_full( + &self, + block: Option<&SignedBeaconBlock>, + ) -> Option>> { + // we definitely require the block + let block = block?; + + // we need to have all columns + if !self.column.sidecar.is_complete() { + return None; + } + + // we need to have the correct amount of everything + let expected = self.column.sidecar.cells_present_bitmap.len(); + if self.column.sidecar.column.len() != expected + || self.column.sidecar.kzg_proofs.len() != expected + || self.kzg_commitments.len() != expected + { + return None; + } + + let (signed_block_header, kzg_commitments_inclusion_proof) = + block.signed_block_header_and_kzg_commitments_proof().ok()?; + Some(Cow::Owned(DataColumnSidecar { + kzg_commitments_inclusion_proof, + index: self.column.index, + column: self.column.sidecar.column.clone(), + kzg_commitments: self.kzg_commitments.clone(), + kzg_proofs: self.column.sidecar.kzg_proofs.clone(), + signed_block_header, + })) + } + + fn iter(&self) -> impl Iterator>> { + let sidecar = &self.column.sidecar; + let mut present_iterator = sidecar + .column + .iter() + .zip(self.kzg_commitments.iter()) + .zip(sidecar.kzg_proofs.iter()) + .map(|((cell, commitment), proof)| CellWithMetadata { + cell, + commitment, + proof, + }); + sidecar.cells_present_bitmap.iter().map(move |present| { + if present { + present_iterator.next() + } else { + None + } + }) + } +} diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 47b99023455..0163c6b779e 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -530,11 +530,12 @@ impl Tester { }) .collect(); - let result = self.block_on_dangerous( - self.harness - .chain - .process_gossip_data_columns(gossip_verified_data_columns, || Ok(())), - )?; + let result = + self.block_on_dangerous(self.harness.chain.process_gossip_data_columns( + gossip_verified_data_columns, + || Ok(()), + |_| (), + ))?; if valid { assert!(result.is_ok()); } From 303c6387e76a662634985f3d7bae13dcb0172ebc Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 14:21:27 +0100 Subject: [PATCH 02/30] fix compilation --- beacon_node/beacon_chain/src/kzg_utils.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 64183e7154a..49dbed69479 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -397,7 +397,8 @@ pub(crate) fn build_partial_data_columns( .get(col) .ok_or(format!("Missing blob cell at index {col}"))?; let cell: Vec = cell.to_vec(); - let cell = Cell::::from(cell); + let cell = + Cell::::try_from(cell).map_err(|e| format!("BytesPerCell exceeded: {e:?}"))?; let proof = blob_cell_proofs .get(col) @@ -410,33 +411,35 @@ pub(crate) fn build_partial_data_columns( .get_mut(col) .ok_or(format!("Missing data column proofs at index {col}"))?; - column.push(cell); + column.push(cell.clone()); column_proofs.push(*proof); } } - let sidecars: Vec>> = columns + let sidecars: Result>>, String> = columns .into_iter() .zip(column_kzg_proofs) .enumerate() .map(|(index, (col, proofs))| { - Arc::new(VerifiablePartialDataColumn { + Ok(Arc::new(VerifiablePartialDataColumn { column: Arc::new(DanglingPartialDataColumn { block_root: signed_block_header.message.canonical_root(), index: index as u64, sidecar: PartialDataColumnSidecar { cells_present_bitmap: bitmap.clone(), - column: DataColumn::::from(col), - kzg_proofs: VariableList::from(proofs), + column: DataColumn::::try_from(col) + .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, + kzg_proofs: VariableList::try_from(proofs) + .map_err(|e| format!("MaxBlobCommitmentsPerBlock exceeded: {e:?}"))?, }, }), kzg_commitments: kzg_commitments.clone(), slot: signed_block_header.message.slot, - }) + })) }) .collect(); - Ok(sidecars) + sidecars } /// Reconstruct blobs from a subset of data column sidecars (requires at least 50%). From ad850598caffa081c35b818e4719d25fb3210ad9 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 14:41:11 +0100 Subject: [PATCH 03/30] fix some lints --- .../overflow_lru_cache.rs | 7 +++++-- .../beacon_chain/src/data_column_verification.rs | 3 +++ .../src/fetch_blobs/fetch_blobs_beacon_adapter.rs | 4 ++-- beacon_node/beacon_chain/src/fetch_blobs/mod.rs | 15 +++++---------- beacon_node/http_api/src/publish_blocks.rs | 6 ++++-- .../lighthouse_network/src/service/utils.rs | 2 +- .../network/src/network_beacon_processor/mod.rs | 1 - 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 5a5f0350a5d..5a3c8fb4423 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -645,7 +645,7 @@ impl DataAvailabilityCheckerInner { fn check_availability_and_cache_components( &self, block_root: Hash256, - pending_components: MappedRwLockReadGuard<'_, PendingComponents>, + pending_components: ComponentsLock<'_, T>, num_expected_columns_opt: Option, ) -> Result, AvailabilityCheckError> { if let Some(available_block) = pending_components.make_available( @@ -682,7 +682,7 @@ impl DataAvailabilityCheckerInner { block_root: Hash256, epoch: Epoch, update_fn: F, - ) -> Result<(MappedRwLockReadGuard<'_, PendingComponents>, R), AvailabilityCheckError> + ) -> Result<(ComponentsLock<'_, T>, R), AvailabilityCheckError> where F: FnOnce(&mut PendingComponents) -> Result, { @@ -887,6 +887,9 @@ impl DataAvailabilityCheckerInner { } } +type ComponentsLock<'a, T> = + MappedRwLockReadGuard<'a, PendingComponents<::EthSpec>>; + #[cfg(test)] mod test { use super::*; diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index d8077f93854..b9e6d686426 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -325,6 +325,9 @@ impl } } +pub type GossipVerifiedFullDataColumn = + GossipVerifiedDataColumn::EthSpec>>; + /// Wrapper over a `DataColumnSidecar` for which we have completed kzg verification. #[derive(Debug, Derivative, Clone)] #[derivative(PartialEq, Eq)] diff --git a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs index 4e98e81df24..98de21b2fbd 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs @@ -1,7 +1,7 @@ use crate::fetch_blobs::{EngineGetBlobsOutput, FetchEngineBlobError}; use crate::observed_block_producers::ProposalKey; use crate::{AvailabilityProcessingStatus, BeaconChain, BeaconChainTypes}; -use execution_layer::json_structures::{BlobAndProofV1, BlobAndProofV2}; +use execution_layer::json_structures::{BlobAndProofV1, BlobAndProofV2, BlobAndProofV3}; use kzg::Kzg; #[cfg(test)] use mockall::automock; @@ -71,7 +71,7 @@ impl FetchBlobsBeaconAdapter { pub(crate) async fn get_blobs_v3( &self, versioned_hashes: Vec, - ) -> Result>>>, FetchEngineBlobError> { + ) -> Result>>, FetchEngineBlobError> { let execution_layer = self .chain .execution_layer diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 55f43289900..560887591f4 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -25,7 +25,7 @@ use crate::{ metrics, }; use execution_layer::Error as ExecutionLayerError; -use execution_layer::json_structures::{BlobAndProofV1, BlobAndProofV2}; +use execution_layer::json_structures::{BlobAndProofV1, BlobAndProofV2, BlobAndProofV3}; use metrics::{TryExt, inc_counter}; #[cfg(test)] use mockall_double::double; @@ -38,8 +38,8 @@ use types::das_column::DasColumn; use types::data_column_sidecar::DataColumnSidecarError; use types::partial_data_column_sidecar::VerifiablePartialDataColumn; use types::{ - BeaconStateError, Blob, BlobSidecar, ColumnIndex, EthSpec, FullPayload, Hash256, KzgProofs, - SignedBeaconBlock, SignedBeaconBlockHeader, VersionedHash, + BeaconStateError, BlobSidecar, ColumnIndex, EthSpec, FullPayload, Hash256, SignedBeaconBlock, + SignedBeaconBlockHeader, VersionedHash, }; /// Result from engine get blobs to be passed onto `DataAvailabilityChecker` and published to the @@ -265,11 +265,6 @@ async fn fetch_and_process_blobs_v2( return Ok(None); }; - let blobs_and_proofs: Vec<_> = blobs_and_proofs - .into_iter() - .map(|blob_and_proof| blob_and_proof.map(|BlobAndProofV2 { blob, proofs }| (blob, proofs))) - .collect(); - let num_fetched_blobs = blobs_and_proofs.len(); metrics::observe(&metrics::BLOBS_FROM_EL_RECEIVED, num_fetched_blobs as f64); @@ -326,7 +321,7 @@ async fn compute_custody_columns_to_import( chain_adapter: &Arc>, block_root: Hash256, block: Arc>>, - blobs_and_proofs: Vec, KzgProofs)>>, + blobs_and_proofs: Vec>, custody_columns_indices: &[ColumnIndex], ) -> Result< Vec>>, @@ -352,7 +347,7 @@ async fn compute_custody_columns_to_import( .map(|option| { option .as_ref() - .map(|(blob, proofs)| (blob, proofs.as_ref())) + .map(|BlobAndProofV2 { blob, proofs }| (blob, proofs.as_ref())) }) .collect::>(); let data_columns_result = diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index a1412debc28..2c97aba5e70 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -3,7 +3,9 @@ use std::future::Future; use beacon_chain::blob_verification::{GossipBlobError, GossipVerifiedBlob}; use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; -use beacon_chain::data_column_verification::GossipVerifiedDataColumn; +use beacon_chain::data_column_verification::{ + GossipVerifiedDataColumn, GossipVerifiedFullDataColumn, +}; use beacon_chain::validator_monitor::{get_block_delay_ms, timestamp_now}; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, @@ -407,7 +409,7 @@ fn build_data_columns( block: &SignedBeaconBlock>, blobs: BlobsList, kzg_cell_proofs: KzgProofs, -) -> Result>>, Rejection> { +) -> Result>, Rejection> { let slot = block.slot(); let data_column_sidecars = build_blob_data_column_sidecars(chain, block, blobs, kzg_cell_proofs).map_err(|e| { diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index a0026837e37..63f22be5e2c 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -41,7 +41,7 @@ pub fn build_transport( quic_support: bool, ) -> std::io::Result { // mplex config - let mut mplex_config = libp2p_mplex::MplexConfig::new(); + let mut mplex_config = libp2p_mplex::Config::new(); mplex_config.set_max_buffer_size(256); mplex_config.set_max_buffer_behaviour(libp2p_mplex::MaxBufferBehaviour::Block); diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 3310b9c72ec..7bb4884c851 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -12,7 +12,6 @@ use beacon_processor::{ BeaconProcessorSend, DuplicateCache, GossipAggregatePackage, GossipAttestationPackage, Work, WorkEvent as BeaconWorkEvent, }; -use futures::FutureExt; use lighthouse_network::rpc::InboundRequestId; use lighthouse_network::rpc::methods::{ BlobsByRangeRequest, BlobsByRootRequest, DataColumnsByRangeRequest, DataColumnsByRootRequest, From 7b8cb0d99a9c22a9d2de9a81eeb257705a869681 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 15:05:28 +0100 Subject: [PATCH 04/30] impl get_blobs_v2 fallback --- .../fetch_blobs/fetch_blobs_beacon_adapter.rs | 15 ++++++++- .../beacon_chain/src/fetch_blobs/mod.rs | 31 +++++++++++++------ 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs index 98de21b2fbd..e95cc9dfe13 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/fetch_blobs_beacon_adapter.rs @@ -51,7 +51,6 @@ impl FetchBlobsBeaconAdapter { .map_err(FetchEngineBlobError::RequestFailed) } - // TODO(dknopik): fall back to this pub(crate) async fn get_blobs_v2( &self, versioned_hashes: Vec, @@ -138,4 +137,18 @@ impl FetchBlobsBeaconAdapter { .fork_choice_read_lock() .contains_block(block_root) } + + pub(crate) async fn supports_get_blobs_v3(&self) -> Result { + let execution_layer = self + .chain + .execution_layer + .as_ref() + .ok_or(FetchEngineBlobError::ExecutionLayerMissing)?; + + execution_layer + .get_engine_capabilities(None) + .await + .map_err(FetchEngineBlobError::RequestFailed) + .map(|caps| caps.get_blobs_v3) + } } diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 560887591f4..fb70cbc8e89 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -124,7 +124,7 @@ async fn fetch_and_process_engine_blobs_inner( .spec() .is_peer_das_enabled_for_epoch(block.epoch()) { - fetch_and_process_blobs_v2( + fetch_and_process_blobs_v2_or_v3( chain_adapter, block_root, block, @@ -239,7 +239,7 @@ async fn fetch_and_process_blobs_v1( } #[instrument(skip_all, level = "debug")] -async fn fetch_and_process_blobs_v2( +async fn fetch_and_process_blobs_v2_or_v3( chain_adapter: FetchBlobsBeaconAdapter, block_root: Hash256, block: Arc>, @@ -250,14 +250,25 @@ async fn fetch_and_process_blobs_v2( let num_expected_blobs = versioned_hashes.len(); metrics::observe(&metrics::BLOBS_FROM_EL_EXPECTED, num_expected_blobs as f64); - // TODO(dknopik): implement fallback to get_blobs_v2 - debug!(num_expected_blobs, "Fetching blobs from the EL"); - let response = chain_adapter - .get_blobs_v3(versioned_hashes) - .await - .inspect_err(|_| { - inc_counter(&metrics::BLOBS_FROM_EL_ERROR_TOTAL); - })?; + + let response = if chain_adapter.supports_get_blobs_v3().await? { + debug!(num_expected_blobs, "Fetching available blobs from the EL"); + chain_adapter + .get_blobs_v3(versioned_hashes) + .await + .inspect_err(|_| { + inc_counter(&metrics::BLOBS_FROM_EL_ERROR_TOTAL); + })? + } else { + debug!(num_expected_blobs, "Fetching all blobs from the EL"); + chain_adapter + .get_blobs_v2(versioned_hashes) + .await + .inspect_err(|_| { + inc_counter(&metrics::BLOBS_FROM_EL_ERROR_TOTAL); + })? + .map(|vec| vec.into_iter().map(Some).collect()) + }; let Some(blobs_and_proofs) = response else { debug!(num_expected_blobs, "No blobs fetched from the EL"); From efa9366d4e3d4c3339455643693df81f676ffc70 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 15:46:47 +0100 Subject: [PATCH 05/30] update gossipsub --- Cargo.lock | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e638a561ab8..64430c3cf72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5165,7 +5165,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "bytes", "either", @@ -5197,7 +5197,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5207,7 +5207,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5217,7 +5217,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "either", "fnv", @@ -5241,7 +5241,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "async-trait", "futures", @@ -5256,7 +5256,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "async-channel 2.3.1", "asynchronous-codec", @@ -5286,7 +5286,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "asynchronous-codec", "either", @@ -5328,7 +5328,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "hickory-proto", @@ -5346,7 +5346,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "libp2p-core", @@ -5361,7 +5361,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "asynchronous-codec", "bytes", @@ -5379,7 +5379,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "asynchronous-codec", "bytes", @@ -5401,7 +5401,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "asynchronous-codec", "bytes", @@ -5416,7 +5416,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "futures-timer", @@ -5437,7 +5437,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "either", "fnv", @@ -5458,7 +5458,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "heck 0.5.0", "quote", @@ -5468,7 +5468,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "futures-timer", @@ -5483,7 +5483,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "futures-rustls", @@ -5501,7 +5501,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "futures-timer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "either", "futures", @@ -6188,7 +6188,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "bytes", "futures", @@ -7437,7 +7437,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "asynchronous-codec", "bytes", @@ -8250,7 +8250,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#13d61c7718a4ac9efa8cae18f6c68857df628353" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" dependencies = [ "futures", "pin-project", From 56f7a1f8bb3778ebeaedfb904c1c5918f356e820 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 15:48:07 +0100 Subject: [PATCH 06/30] remove redundant log --- beacon_node/beacon_chain/src/fetch_blobs/mod.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index fb70cbc8e89..ea77b9eb9d4 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -115,11 +115,6 @@ async fn fetch_and_process_engine_blobs_inner( return Ok(None); }; - debug!( - num_expected_blobs = versioned_hashes.len(), - "Fetching blobs from the EL" - ); - if chain_adapter .spec() .is_peer_das_enabled_for_epoch(block.epoch()) From 2a721d72a75d0cc2aed5d10516f580e6e67b86f0 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 16:00:09 +0100 Subject: [PATCH 07/30] try to upgrade column using block immediately --- .../overflow_lru_cache.rs | 4 ++-- .../src/data_column_verification.rs | 18 +++--------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 5a3c8fb4423..e1cc6eac7c1 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -217,7 +217,7 @@ impl PendingComponents { continue; } - if let Some(full) = data_column.try_as_full() { + if let Some(full) = data_column.try_as_full(self.block.as_ref().map(|b| b.as_block())) { self.verified_partial_columns .retain(|col| col.index() != full.index()); self.verified_data_columns.push(full); @@ -234,7 +234,7 @@ impl PendingComponents { let did_merge = cached_partial.merge(&partial); if did_merge { if let Some(block) = &self.block - && let Some(full) = cached_partial.try_upgrade_full(block.as_block()) + && let Some(full) = cached_partial.try_as_full(Some(block.as_block())) { merged_data.completed_columns.push(full.clone_arc()); self.verified_data_columns.push(full); diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index b9e6d686426..643916e3210 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -453,23 +453,11 @@ impl> KzgVerifiedCustodyDataColumn { self.data.index() } - // TODO(dknopik): below three functions are baaaad. They clone in far too many cases + // TODO(dknopik): below two functions are baaaad. They clone in far too many cases - pub fn try_as_full(&self) -> Option>> { + pub fn try_as_full(&self, block: Option<&SignedBeaconBlock>) -> Option>> { self.data - .as_full(None) - .map(|full| KzgVerifiedCustodyDataColumn { - data: Arc::new(full.into_owned()), - _phantom: PhantomData, - }) - } - - pub fn try_upgrade_full( - &self, - block: &SignedBeaconBlock, - ) -> Option>> { - self.data - .as_full(Some(block)) + .as_full(block) .map(|full| KzgVerifiedCustodyDataColumn { data: Arc::new(full.into_owned()), _phantom: PhantomData, From 57094088ace561caf90ddb56467f0ed1c57ca506 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 16:31:58 +0100 Subject: [PATCH 08/30] publish data after block availability --- beacon_node/beacon_chain/src/beacon_chain.rs | 10 +++- .../src/data_availability_checker.rs | 13 +++-- .../overflow_lru_cache.rs | 51 ++++++++++--------- .../src/data_column_verification.rs | 5 +- beacon_node/beacon_chain/src/test_utils.rs | 2 + beacon_node/http_api/src/publish_blocks.rs | 39 ++++++++++++++ .../gossip_methods.rs | 1 + .../network_beacon_processor/sync_methods.rs | 1 + beacon_node/network/src/sync/tests/lookups.rs | 2 +- beacon_node/network/src/sync/tests/range.rs | 1 + testing/ef_tests/src/cases/fork_choice.rs | 2 + 11 files changed, 93 insertions(+), 34 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 0d6d0e30a2c..11694acb24c 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2929,6 +2929,7 @@ impl BeaconChain { notify_execution_layer, BlockImportSource::RangeSync, || Ok(()), + |_| (), ) .await { @@ -3360,6 +3361,7 @@ impl BeaconChain { notify_execution_layer: NotifyExecutionLayer, block_source: BlockImportSource, publish_fn: impl FnOnce() -> Result<(), BlockError>, + data_publish_fn: impl FnOnce(MergedData), ) -> Result { let block_slot = unverified_block.block().slot(); @@ -3429,7 +3431,8 @@ impl BeaconChain { self.import_available_block(Box::new(block)).await } ExecutedBlock::AvailabilityPending(block) => { - self.check_block_availability_and_import(block).await + self.check_block_availability_and_import(block, data_publish_fn) + .await } } }; @@ -3539,9 +3542,12 @@ impl BeaconChain { async fn check_block_availability_and_import( self: &Arc, block: AvailabilityPendingExecutedBlock, + data_publish_fn: impl FnOnce(MergedData), ) -> Result { let slot = block.block.slot(); - let availability = self.data_availability_checker.put_executed_block(block)?; + let availability = self + .data_availability_checker + .put_executed_block(block, data_publish_fn)?; self.process_availability(slot, availability, || Ok(())) .await } diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 9c8172ebb0c..b43e0826f7b 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -339,7 +339,7 @@ impl DataAvailabilityChecker { block_root: Hash256, slot: Slot, data_columns: I, - publish_fn: impl FnOnce(MergedData), + data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let epoch = slot.epoch(T::EthSpec::slots_per_epoch()); let sampling_columns = self @@ -354,7 +354,7 @@ impl DataAvailabilityChecker { self.availability_cache.put_kzg_verified_data_columns( block_root, custody_columns, - publish_fn, + data_publish_fn, ) } @@ -366,12 +366,12 @@ impl DataAvailabilityChecker { &self, block_root: Hash256, custody_columns: I, - publish_fn: impl FnOnce(MergedData), + data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { self.availability_cache.put_kzg_verified_data_columns( block_root, custody_columns, - publish_fn, + data_publish_fn, ) } @@ -380,8 +380,10 @@ impl DataAvailabilityChecker { pub fn put_executed_block( &self, executed_block: AvailabilityPendingExecutedBlock, + data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { - self.availability_cache.put_executed_block(executed_block) + self.availability_cache + .put_executed_block(executed_block, data_publish_fn) } /// Inserts a pre-execution block into the cache. @@ -901,6 +903,7 @@ impl MaybeAvailableBlock { } } +#[must_use = "Publish the data within"] pub struct MergedData { pub completed_columns: Vec>>, pub updated_partials: Vec>>, diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index e1cc6eac7c1..ea7e6295b6f 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -13,6 +13,7 @@ use lighthouse_tracing::SPAN_PENDING_COMPONENTS; use lru::LruCache; use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; use std::cmp::Ordering; +use std::mem; use std::num::NonZeroUsize; use std::sync::Arc; use tracing::{Span, debug, debug_span}; @@ -202,13 +203,11 @@ impl PendingComponents { } /// Merges a given set of data columns into the cache. - fn merge_data_columns< + fn merge_data_columns(&mut self, kzg_verified_data_columns: I) -> MergedData + where I: IntoIterator>, C: DasColumn, - >( - &mut self, - kzg_verified_data_columns: I, - ) -> Result, AvailabilityCheckError> { + { let mut merged_data = MergedData::empty(); for data_column in kzg_verified_data_columns { @@ -251,16 +250,18 @@ impl PendingComponents { } } - Ok(merged_data) + merged_data } /// Inserts a new block and revalidates the existing blobs against it. /// /// Blobs that don't match the new block's commitments are evicted. - pub fn merge_block(&mut self, block: DietAvailabilityPendingExecutedBlock) { + pub fn merge_block(&mut self, block: DietAvailabilityPendingExecutedBlock) -> MergedData { self.insert_executed_block(block); let reinsert = self.get_cached_blobs_mut().take(); self.merge_blobs(reinsert); + let reinsert = mem::take(&mut self.verified_partial_columns); + self.merge_data_columns(reinsert) } /// Returns Some if the block has received all its required data for import. The return value @@ -578,7 +579,6 @@ impl DataAvailabilityCheckerInner { let (pending_components, ()) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.merge_blobs(fixed_blobs); - Ok(()) })?; pending_components.span.in_scope(|| { @@ -600,7 +600,7 @@ impl DataAvailabilityCheckerInner { &self, block_root: Hash256, kzg_verified_data_columns: I, - publish_fn: impl FnOnce(MergedData), + data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let mut kzg_verified_data_columns = kzg_verified_data_columns.into_iter().peekable(); let Some(epoch) = kzg_verified_data_columns.peek().map(|verified_blob| { @@ -621,7 +621,7 @@ impl DataAvailabilityCheckerInner { pending_components.merge_data_columns(kzg_verified_data_columns) })?; - publish_fn(merged_data); + data_publish_fn(merged_data); let num_expected_columns = self .custody_context @@ -684,7 +684,7 @@ impl DataAvailabilityCheckerInner { update_fn: F, ) -> Result<(ComponentsLock<'_, T>, R), AvailabilityCheckError> where - F: FnOnce(&mut PendingComponents) -> Result, + F: FnOnce(&mut PendingComponents) -> R, { let mut write_lock = self.critical.write(); @@ -692,7 +692,7 @@ impl DataAvailabilityCheckerInner { let pending_components = write_lock.get_or_insert_mut(block_root, || { PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize) }); - update_fn(pending_components)? + update_fn(pending_components) }; RwLockReadGuard::try_map(RwLockWriteGuard::downgrade(write_lock), |cache| { @@ -774,7 +774,6 @@ impl DataAvailabilityCheckerInner { let (pending_components, ()) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { pending_components.insert_pre_execution_block(block, source); - Ok(()) })?; let num_expected_columns_opt = self.get_num_expected_columns(epoch); @@ -804,6 +803,7 @@ impl DataAvailabilityCheckerInner { pub fn put_executed_block( &self, executed_block: AvailabilityPendingExecutedBlock, + data_publish_fn: impl FnOnce(MergedData), ) -> Result, AvailabilityCheckError> { let epoch = executed_block.as_block().epoch(); let block_root = executed_block.import_data.block_root; @@ -813,12 +813,13 @@ impl DataAvailabilityCheckerInner { .state_cache .register_pending_executed_block(executed_block); - let (pending_components, ()) = + let (pending_components, merged_data) = self.update_or_insert_pending_components(block_root, epoch, |pending_components| { - pending_components.merge_block(diet_executed_block); - Ok(()) + pending_components.merge_block(diet_executed_block) })?; + data_publish_fn(merged_data); + let num_expected_columns_opt = self.get_num_expected_columns(epoch); pending_components.span.in_scope(|| { @@ -1125,7 +1126,7 @@ mod test { ); assert!(cache.critical.read().is_empty(), "cache should be empty"); let availability = cache - .put_executed_block(pending_block) + .put_executed_block(pending_block, |_| ()) .expect("should put block"); if blobs_expected == 0 { assert!( @@ -1192,7 +1193,7 @@ mod test { ); } let availability = cache - .put_executed_block(pending_block) + .put_executed_block(pending_block, |_| ()) .expect("should put block"); assert!( matches!(availability, Availability::Available(_)), @@ -1254,7 +1255,7 @@ mod test { // put the block in the cache let availability = cache - .put_executed_block(pending_block) + .put_executed_block(pending_block, |_| ()) .expect("should put block"); // grab the diet block from the cache for later testing @@ -1449,7 +1450,7 @@ mod pending_components_tests { setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); let mut cache = >::empty(block_root, max_len); - cache.merge_block(block_commitments); + let _ = cache.merge_block(block_commitments); cache.merge_blobs(random_blobs); cache.merge_blobs(blobs); @@ -1464,7 +1465,7 @@ mod pending_components_tests { let block_root = Hash256::zero(); let mut cache = >::empty(block_root, max_len); cache.merge_blobs(random_blobs); - cache.merge_block(block_commitments); + let _ = cache.merge_block(block_commitments); cache.merge_blobs(blobs); assert_cache_consistent(cache, max_len); @@ -1480,7 +1481,7 @@ mod pending_components_tests { let mut cache = >::empty(block_root, max_len); cache.merge_blobs(random_blobs); cache.merge_blobs(blobs); - cache.merge_block(block_commitments); + let _ = cache.merge_block(block_commitments); assert_empty_blob_cache(cache); } @@ -1493,7 +1494,7 @@ mod pending_components_tests { let block_root = Hash256::zero(); let mut cache = >::empty(block_root, max_len); - cache.merge_block(block_commitments); + let _ = cache.merge_block(block_commitments); cache.merge_blobs(blobs); cache.merge_blobs(random_blobs); @@ -1509,7 +1510,7 @@ mod pending_components_tests { let block_root = Hash256::zero(); let mut cache = >::empty(block_root, max_len); cache.merge_blobs(blobs); - cache.merge_block(block_commitments); + let _ = cache.merge_block(block_commitments); cache.merge_blobs(random_blobs); assert_cache_consistent(cache, max_len); @@ -1525,7 +1526,7 @@ mod pending_components_tests { let mut cache = >::empty(block_root, max_len); cache.merge_blobs(blobs); cache.merge_blobs(random_blobs); - cache.merge_block(block_commitments); + let _ = cache.merge_block(block_commitments); assert_cache_consistent(cache, max_len); } diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 643916e3210..cdfd6957539 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -455,7 +455,10 @@ impl> KzgVerifiedCustodyDataColumn { // TODO(dknopik): below two functions are baaaad. They clone in far too many cases - pub fn try_as_full(&self, block: Option<&SignedBeaconBlock>) -> Option>> { + pub fn try_as_full( + &self, + block: Option<&SignedBeaconBlock>, + ) -> Option>> { self.data .as_full(block) .map(|full| KzgVerifiedCustodyDataColumn { diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 4e1d0002087..58e0a62680a 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -2374,6 +2374,7 @@ where NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), + |_| (), ) .await? .try_into() @@ -2398,6 +2399,7 @@ where NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), + |_| (), ) .await? .try_into() diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 2c97aba5e70..c5f9b133696 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -3,6 +3,7 @@ use std::future::Future; use beacon_chain::blob_verification::{GossipBlobError, GossipVerifiedBlob}; use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; +use beacon_chain::data_availability_checker::MergedData; use beacon_chain::data_column_verification::{ GossipVerifiedDataColumn, GossipVerifiedFullDataColumn, }; @@ -29,6 +30,7 @@ use std::time::Duration; use tokio::sync::mpsc::UnboundedSender; use tracing::{Span, debug, debug_span, error, info, instrument, warn}; use tree_hash::TreeHash; +use types::das_column::DasColumn; use types::{ AbstractExecPayload, BeaconBlockRef, BlobSidecar, BlobsList, BlockImportSource, DataColumnSidecar, DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, @@ -201,6 +203,41 @@ pub async fn publish_block>( Ok(()) }; + let sender_clone = network_tx.clone(); + let spec = chain.spec.clone(); + let data_publish_fn = move |merged_data: MergedData| { + debug!( + partial = merged_data.updated_partials.len(), + full = merged_data.completed_columns.len(), + "Sending merged data after block availability" + ); + let messages: Vec<_> = merged_data + .updated_partials + .into_iter() + .map(|partial| { + PubsubMessage::PartialDataColumnSidecar(Box::new(( + DataColumnSubnetId::from_column_index(partial.index(), &spec), + partial.column.clone().into(), + ))) + }) + .chain(merged_data.completed_columns.into_iter().flat_map(|full| { + let subnet = DataColumnSubnetId::from_column_index(full.index, &spec); + [ + PubsubMessage::PartialDataColumnSidecar(Box::new(( + subnet, + (*full).clone().into_partial().column.clone().into(), + ))), + PubsubMessage::DataColumnSidecar(Box::new((subnet, full))), + ] + })) + .collect(); + if !messages.is_empty() + && let Err(err) = crate::publish_pubsub_messages(&sender_clone, messages) + { + warn!(?err, "Publishing data after block availability failed") + } + }; + // Wait for blobs/columns to get gossip verified before proceeding further as we need them for import. let (gossip_verified_blobs, gossip_verified_columns) = build_sidecar_task_handle.await?; @@ -275,6 +312,7 @@ pub async fn publish_block>( NotifyExecutionLayer::Yes, BlockImportSource::HttpApi, publish_fn, + data_publish_fn, )) .await; post_block_import_logging_and_response( @@ -325,6 +363,7 @@ pub async fn publish_block>( NotifyExecutionLayer::Yes, BlockImportSource::HttpApi, publish_fn, + data_publish_fn, )) .await; post_block_import_logging_and_response( diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index be6fb20f785..500842ff149 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1800,6 +1800,7 @@ impl NetworkBeaconProcessor { NotifyExecutionLayer::Yes, BlockImportSource::Gossip, || Ok(()), + |_| (), ) .await; register_process_result_metrics(&result, metrics::BlockSource::Gossip, "block"); diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index 41b12fa01b3..a7cff747230 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -178,6 +178,7 @@ impl NetworkBeaconProcessor { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await; register_process_result_metrics(&result, metrics::BlockSource::Rpc, "block"); diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index fc641861754..cb8028c5874 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -1079,7 +1079,7 @@ impl TestRig { .harness .chain .data_availability_checker - .put_executed_block(executed_block) + .put_executed_block(executed_block, |_| ()) .unwrap() { Availability::Available(_) => panic!("block removed from da_checker, available"), diff --git a/beacon_node/network/src/sync/tests/range.rs b/beacon_node/network/src/sync/tests/range.rs index cb728a90c1b..8b9c876907c 100644 --- a/beacon_node/network/src/sync/tests/range.rs +++ b/beacon_node/network/src/sync/tests/range.rs @@ -431,6 +431,7 @@ impl TestRig { NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), + |_| (), ) .await .unwrap() diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 0163c6b779e..f394459439d 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -549,6 +549,7 @@ impl Tester { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ))? .map(|avail: AvailabilityProcessingStatus| avail.try_into()); let success = data_column_success && result.as_ref().is_ok_and(|inner| inner.is_ok()); @@ -639,6 +640,7 @@ impl Tester { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ))? .map(|avail: AvailabilityProcessingStatus| avail.try_into()); let success = blob_success && result.as_ref().is_ok_and(|inner| inner.is_ok()); From 02194da47f88890ad7394438941231159c335ae6 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 29 Oct 2025 17:45:02 +0100 Subject: [PATCH 09/30] fix remaining lints and release test compilation --- beacon_node/beacon_chain/tests/blob_verification.rs | 1 + beacon_node/beacon_chain/tests/block_verification.rs | 9 ++++++++- beacon_node/beacon_chain/tests/column_verification.rs | 1 + beacon_node/beacon_chain/tests/payload_invalidation.rs | 4 ++++ beacon_node/beacon_chain/tests/store_tests.rs | 3 +++ beacon_node/beacon_chain/tests/tests.rs | 1 + .../lighthouse_network/src/service/gossip_cache.rs | 4 ++++ .../network/src/network_beacon_processor/tests.rs | 10 +++++++++- 8 files changed, 31 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/tests/blob_verification.rs b/beacon_node/beacon_chain/tests/blob_verification.rs index c42a2828c01..a6a1aeef519 100644 --- a/beacon_node/beacon_chain/tests/blob_verification.rs +++ b/beacon_node/beacon_chain/tests/blob_verification.rs @@ -86,6 +86,7 @@ async fn rpc_blobs_with_invalid_header_signature() { NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), + |_| (), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 3d1fa8f4af4..508a7105102 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -504,6 +504,7 @@ async fn assert_invalid_signature( NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await; assert!( @@ -575,6 +576,7 @@ async fn invalid_signature_gossip_block() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await; assert!( @@ -1018,6 +1020,7 @@ async fn block_gossip_verification() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .expect("should import valid gossip verified block"); @@ -1327,7 +1330,7 @@ async fn verify_and_process_gossip_data_sidecars( harness .chain - .process_gossip_data_columns(gossip_verified, || Ok(())) + .process_gossip_data_columns(gossip_verified, || Ok(()), |_| ()) .await .expect("should import valid gossip verified columns"); } @@ -1377,6 +1380,7 @@ async fn verify_block_for_gossip_slashing_detection() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap(); @@ -1413,6 +1417,7 @@ async fn verify_block_for_gossip_doppelganger_detection() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap(); @@ -1577,6 +1582,7 @@ async fn add_base_block_to_altair_chain() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .expect_err("should error when processing base block"), @@ -1714,6 +1720,7 @@ async fn add_altair_block_to_base_chain() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .expect_err("should error when processing altair block"), diff --git a/beacon_node/beacon_chain/tests/column_verification.rs b/beacon_node/beacon_chain/tests/column_verification.rs index 229ae1e1998..491969fc663 100644 --- a/beacon_node/beacon_chain/tests/column_verification.rs +++ b/beacon_node/beacon_chain/tests/column_verification.rs @@ -90,6 +90,7 @@ async fn rpc_columns_with_invalid_header_signature() { NotifyExecutionLayer::Yes, BlockImportSource::RangeSync, || Ok(()), + |_| (), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index 5bd43835e33..81da193574c 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -695,6 +695,7 @@ async fn invalidates_all_descendants() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap() @@ -797,6 +798,7 @@ async fn switches_heads() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap() @@ -1063,6 +1065,7 @@ async fn invalid_parent() { assert!(matches!( rig.harness.chain.process_block(rpc_block.block_root(), rpc_block, NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ).await, Err(BlockError::ParentExecutionPayloadInvalid { parent_root: invalid_root }) if invalid_root == parent_root @@ -1393,6 +1396,7 @@ async fn recover_from_invalid_head_by_importing_blocks() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 0a261e36cef..04efa34f83c 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -2937,6 +2937,7 @@ async fn weak_subjectivity_sync_test( NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap(); @@ -3502,6 +3503,7 @@ async fn process_blocks_and_attestations_for_unaligned_checkpoint() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap_err(); @@ -3517,6 +3519,7 @@ async fn process_blocks_and_attestations_for_unaligned_checkpoint() { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap(); diff --git a/beacon_node/beacon_chain/tests/tests.rs b/beacon_node/beacon_chain/tests/tests.rs index ec0e607d00a..02790a02eb6 100644 --- a/beacon_node/beacon_chain/tests/tests.rs +++ b/beacon_node/beacon_chain/tests/tests.rs @@ -712,6 +712,7 @@ async fn run_skip_slot_test(skip_slots: u64) { NotifyExecutionLayer::Yes, BlockImportSource::Lookup, || Ok(()), + |_| (), ) .await .unwrap(); diff --git a/beacon_node/lighthouse_network/src/service/gossip_cache.rs b/beacon_node/lighthouse_network/src/service/gossip_cache.rs index 120b9e6c245..f72bf3b8a52 100644 --- a/beacon_node/lighthouse_network/src/service/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/service/gossip_cache.rs @@ -13,6 +13,8 @@ use tokio_util::time::delay_queue::{DelayQueue, Key}; /// messages are ignored. This behaviour can be changed using `GossipCacheBuilder::default_timeout` /// to apply the same delay to every kind. Individual timeouts for specific kinds can be set and /// will overwrite the default_timeout if present. +// TODO(dknopik) +#[allow(dead_code)] pub struct GossipCache { /// Expire timeouts for each topic-msg pair. expirations: DelayQueue<(GossipTopic, Vec)>, @@ -196,6 +198,8 @@ impl GossipCache { } // Insert a message to be sent later. + // TODO(dknopik) + #[allow(dead_code)] pub fn insert(&mut self, topic: GossipTopic, data: Vec) { let expire_timeout = match topic.kind() { GossipKind::BeaconBlock => self.beacon_block, diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index a9794cb5c42..d1f8147e7b0 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -1,6 +1,7 @@ #![cfg(not(debug_assertions))] // Tests are too slow in debug. #![cfg(test)] +use crate::partial_data_column_cache::PartialDataColumnCache; use crate::{ network_beacon_processor::{ ChainSegmentProcessId, DuplicateCache, InvalidBlockStorage, NetworkBeaconProcessor, @@ -32,6 +33,7 @@ use lighthouse_network::{ types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield}, }; use matches::assert_matches; +use parking_lot::Mutex; use slot_clock::SlotClock; use std::collections::HashSet; use std::iter::Iterator; @@ -265,6 +267,7 @@ impl TestRig { let network_beacon_processor = NetworkBeaconProcessor { beacon_processor_send: beacon_processor_tx.clone(), duplicate_cache: duplicate_cache.clone(), + partial_data_column_cache: Mutex::new(PartialDataColumnCache::new()), chain: harness.chain.clone(), network_tx, sync_tx, @@ -1134,7 +1137,12 @@ async fn accept_processed_gossip_data_columns_without_import() { let block_root = rig.next_block.canonical_root(); rig.chain .data_availability_checker - .put_gossip_verified_data_columns(block_root, rig.next_block.slot(), verified_data_columns) + .put_gossip_verified_data_columns( + block_root, + rig.next_block.slot(), + verified_data_columns, + |_| (), + ) .expect("should put data columns into availability cache"); // WHEN an already processed but unobserved data column is received via gossip From bd8000d1d289819f61a107e1cc0a63c2054469c2 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 01:48:07 +0100 Subject: [PATCH 10/30] implement caching of partial messages --- .../src/service/gossip_cache.rs | 80 +++++++++++-------- .../lighthouse_network/src/service/mod.rs | 74 ++++++++++++----- .../lighthouse_network/src/types/mod.rs | 2 +- .../lighthouse_network/src/types/pubsub.rs | 34 +++++--- 4 files changed, 126 insertions(+), 64 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/gossip_cache.rs b/beacon_node/lighthouse_network/src/service/gossip_cache.rs index f72bf3b8a52..5aa655def96 100644 --- a/beacon_node/lighthouse_network/src/service/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/service/gossip_cache.rs @@ -1,25 +1,27 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; +use std::marker::PhantomData; use std::pin::Pin; use std::task::{Context, Poll}; use std::time::Duration; use crate::GossipTopic; -use crate::types::GossipKind; +use crate::types::{EncodedPubsubMessage, GossipKind}; use tokio_util::time::delay_queue::{DelayQueue, Key}; +use types::EthSpec; + +type TopicMsg = (Key, EncodedPubsubMessage); /// Store of gossip messages that we failed to publish and will try again later. By default, all /// messages are ignored. This behaviour can be changed using `GossipCacheBuilder::default_timeout` /// to apply the same delay to every kind. Individual timeouts for specific kinds can be set and /// will overwrite the default_timeout if present. -// TODO(dknopik) -#[allow(dead_code)] -pub struct GossipCache { - /// Expire timeouts for each topic-msg pair. +pub struct GossipCache { + /// Expire timeouts for each topic-msgid triple. expirations: DelayQueue<(GossipTopic, Vec)>, /// Messages cached for each topic. - topic_msgs: HashMap, Key>>, + topic_msgs: HashMap, TopicMsg>>, /// Timeout for blocks. beacon_block: Option, /// Timeout for blobs. @@ -49,7 +51,7 @@ pub struct GossipCache { } #[derive(Default)] -pub struct GossipCacheBuilder { +pub struct GossipCacheBuilder { default_timeout: Option, /// Timeout for blocks. beacon_block: Option, @@ -77,10 +79,11 @@ pub struct GossipCacheBuilder { light_client_finality_update: Option, /// Timeout for light client optimistic updates. light_client_optimistic_update: Option, + _phantom: PhantomData, } #[allow(dead_code)] -impl GossipCacheBuilder { +impl GossipCacheBuilder { /// By default, all timeouts all disabled. Setting a default timeout will enable all timeout /// that are not already set. pub fn default_timeout(mut self, timeout: Duration) -> Self { @@ -153,7 +156,7 @@ impl GossipCacheBuilder { self } - pub fn build(self) -> GossipCache { + pub fn build(self) -> GossipCache { let GossipCacheBuilder { default_timeout, beacon_block, @@ -169,6 +172,7 @@ impl GossipCacheBuilder { bls_to_execution_change, light_client_finality_update, light_client_optimistic_update, + .. } = self; GossipCache { expirations: DelayQueue::default(), @@ -190,17 +194,15 @@ impl GossipCacheBuilder { } } -impl GossipCache { +impl GossipCache { /// Get a builder of a `GossipCache`. Topic kinds for which no timeout is defined will be /// ignored if added in `insert`. - pub fn builder() -> GossipCacheBuilder { + pub fn builder() -> GossipCacheBuilder { GossipCacheBuilder::default() } // Insert a message to be sent later. - // TODO(dknopik) - #[allow(dead_code)] - pub fn insert(&mut self, topic: GossipTopic, data: Vec) { + pub fn insert(&mut self, topic: GossipTopic, id: Vec, data: EncodedPubsubMessage) { let expire_timeout = match topic.kind() { GossipKind::BeaconBlock => self.beacon_block, GossipKind::BlobSidecar(_) => self.blob_sidecar, @@ -223,45 +225,54 @@ impl GossipCache { .topic_msgs .entry(topic.clone()) .or_default() - .entry(data.clone()) + .entry(id.clone()) { - Entry::Occupied(key) => self.expirations.reset(key.get(), expire_timeout), + Entry::Occupied(key) => self.expirations.reset(&key.get().0, expire_timeout), Entry::Vacant(entry) => { - let key = self.expirations.insert((topic, data), expire_timeout); - entry.insert(key); + let key = self.expirations.insert((topic, id), expire_timeout); + entry.insert((key, data)); } } } - // Get the registered messages for this topic. - pub fn retrieve(&mut self, topic: &GossipTopic) -> Option> + '_> { - if let Some(msgs) = self.topic_msgs.remove(topic) { - for (_, key) in msgs.iter() { - self.expirations.remove(key); - } - Some(msgs.into_keys()) - } else { - None - } + // Get the registered messages for this topic, removing them + pub fn retrieve_all( + &mut self, + topic: &GossipTopic, + ) -> Option> + '_> { + self.topic_msgs.remove(topic).map(|msgs| { + msgs.into_values().map(|(key, msg)| { + self.expirations.remove(&key); + msg + }) + }) + } + + // Get a registered message, NOT removing it. + pub fn retrieve_one(&self, topic: &GossipTopic, id: &[u8]) -> Option<&EncodedPubsubMessage> { + self.topic_msgs + .get(topic) + .and_then(|msgs| msgs.get(id)) + .map(|(_, msg)| msg) } } -impl futures::stream::Stream for GossipCache { +impl futures::stream::Stream for GossipCache { type Item = Result; // We don't care to retrieve the expired data. fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.expirations.poll_expired(cx) { Poll::Ready(Some(expired)) => { let expected_key = expired.key(); - let (topic, data) = expired.into_inner(); + let (topic, id) = expired.into_inner(); let topic_msg = self.topic_msgs.get_mut(&topic); debug_assert!( topic_msg.is_some(), "Topic for registered message is not present." ); if let Some(msgs) = topic_msg { - let key = msgs.remove(&data); - debug_assert_eq!(key, Some(expected_key)); + let key_and_data = msgs.remove(&id); + debug_assert_eq!(key_and_data.map(|(key, _)| key), Some(expected_key)); if msgs.is_empty() { // no more messages for this topic. self.topic_msgs.remove(&topic); @@ -279,10 +290,11 @@ impl futures::stream::Stream for GossipCache { mod tests { use super::*; use futures::stream::StreamExt; + use types::MainnetEthSpec; #[tokio::test] async fn test_stream() { - let mut cache = GossipCache::builder() + let mut cache = GossipCache::::builder() .default_timeout(Duration::from_millis(300)) .build(); let test_topic = GossipTopic::new( @@ -290,7 +302,7 @@ mod tests { crate::types::GossipEncoding::SSZSnappy, [0u8; 4], ); - cache.insert(test_topic, vec![]); + cache.insert(test_topic, vec![], EncodedPubsubMessage::Full(vec![])); tokio::time::sleep(Duration::from_millis(300)).await; while cache.next().await.is_some() {} assert!(cache.expirations.is_empty()); diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 64120b38f2a..2dd7ebbe5f3 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -15,15 +15,16 @@ use crate::rpc::{ RequestType, ResponseTermination, RpcResponse, RpcSuccessResponse, }; use crate::types::{ - GossipEncoding, GossipKind, GossipTopic, SnappyTransform, Subnet, SubnetDiscovery, - all_topics_at_fork, core_topics_to_subscribe, is_fork_non_core_topic, subnet_from_topic_hash, + EncodedPubsubMessage, GossipEncoding, GossipKind, GossipTopic, SnappyTransform, Subnet, + SubnetDiscovery, all_topics_at_fork, core_topics_to_subscribe, is_fork_non_core_topic, + subnet_from_topic_hash, }; use crate::{Enr, NetworkGlobals, PubsubMessage, TopicHash, metrics}; use api_types::{AppRequestId, Response}; use futures::stream::StreamExt; use gossipsub::{ - Event, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, PublishError, - TopicScoreParams, + Event, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, Partial, + PublishError, TopicScoreParams, }; use gossipsub_scoring_parameters::{PeerScoreSettings, lighthouse_gossip_thresholds}; use libp2p::multiaddr::{self, Multiaddr, Protocol as MProtocol}; @@ -33,6 +34,7 @@ use libp2p::upnp::tokio::Behaviour as Upnp; use libp2p::{PeerId, SwarmBuilder, identify}; use logging::crit; use network_utils::enr_ext::EnrExt; +use sha2::{Digest, Sha256}; use std::num::{NonZeroU8, NonZeroUsize}; use std::path::PathBuf; use std::pin::Pin; @@ -160,7 +162,7 @@ pub struct Network { score_settings: PeerScoreSettings, /// The interval for updating gossipsub scores update_gossipsub_scores: tokio::time::Interval, - gossip_cache: GossipCache, + gossip_cache: GossipCache, /// This node's PeerId. pub local_peer_id: PeerId, } @@ -839,7 +841,8 @@ impl Network { pub fn publish(&mut self, messages: Vec>) { for message in messages { for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) { - let result = message.publish(self.gossipsub_mut(), topic.clone().into()); + let message_data = message.encode(); + let result = message_data.do_publish(self.gossipsub_mut(), topic.clone().into()); if let Err(e) = result { match e { PublishError::Duplicate => { @@ -877,10 +880,17 @@ impl Network { } } - if let PublishError::NoPeersSubscribedToTopic = e { - // TODO(dknopik): fix gossip cache - //self.gossip_cache.insert(gossip_topic, message); + if let PublishError::NoPeersSubscribedToTopic = e + && let EncodedPubsubMessage::Full(bytes) = &message_data + { + let id = Sha256::digest(bytes)[..].to_vec(); + self.gossip_cache.insert(topic, id, message_data); } + } else if let EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) = + &message_data + { + let id = partial.group_id().as_ref().to_vec(); + self.gossip_cache.insert(topic, id, message_data); } } } @@ -1297,8 +1307,38 @@ impl Network { message, metadata: _, } => { - // TODO(dknopik): take a look at the metadata and publish if we have something for them - // maybe by reusing the gossip cache?! + let topic = GossipTopic::decode(topic_id.as_str()) + .inspect_err(|error| { + debug!( + topic = ?topic_id, + error, + "Could not decode gossipsub partial message topic" + ); + }) + .ok()?; + + // TODO(dknopik): maybe do this after validation? + if let Some(cached_partial) = self.gossip_cache.retrieve_one(&topic, &group_id) { + // We do not bother checking here if that cached msg contains relevant data, + // as gossipsub will do that anyway. + let result = cached_partial + .do_publish(&mut self.swarm.behaviour_mut().gossipsub, topic.into()); + match result { + Ok(_) => { + debug!( + group_id = hex::encode(&group_id), + "Replied with cached partial message" + ) + } + Err(err) => { + warn!( + group_id = hex::encode(&group_id), + ?err, + "Failed to reply with cached partial message" + ); + } + } + } if let Some(message) = message { match PubsubMessage::decode_partial(&topic_id, &group_id, &message) { @@ -1337,15 +1377,13 @@ impl Network { .add_subscription(&peer_id, subnet_id); } // Try to send the cached messages for this topic - if let Some(msgs) = self.gossip_cache.retrieve(&topic) { + if let Some(msgs) = self.gossip_cache.retrieve_all(&topic) { for data in msgs { let topic_str: &str = topic.kind().as_ref(); - match self - .swarm - .behaviour_mut() - .gossipsub - .publish(Topic::from(topic.clone()), data) - { + match data.do_publish( + &mut self.swarm.behaviour_mut().gossipsub, + topic.clone().into(), + ) { Ok(_) => { debug!(topic = topic_str, "Gossip message published on retry"); metrics::inc_counter_vec( diff --git a/beacon_node/lighthouse_network/src/types/mod.rs b/beacon_node/lighthouse_network/src/types/mod.rs index 748df3cb15c..b2abd1d38a2 100644 --- a/beacon_node/lighthouse_network/src/types/mod.rs +++ b/beacon_node/lighthouse_network/src/types/mod.rs @@ -13,7 +13,7 @@ pub type Enr = discv5::enr::Enr; pub use eth2::lighthouse::sync_state::{BackFillState, CustodyBackFillState, SyncState}; pub use globals::NetworkGlobals; -pub use pubsub::{PubsubMessage, SnappyTransform}; +pub use pubsub::{EncodedPubsubMessage, PubsubMessage, SnappyTransform}; pub use subnet::{Subnet, SubnetDiscovery}; pub use topics::{ GossipEncoding, GossipKind, GossipTopic, TopicConfig, all_topics_at_fork, diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index a784eae2e9a..2c2c110a2be 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -1,13 +1,13 @@ //! Handles the encoding and decoding of pubsub messages. +use crate::{Gossipsub, TopicHash}; use crate::types::partial::PartialDataColumnSidecarMessage; use crate::types::{GossipEncoding, GossipKind, GossipTopic}; -use crate::{Gossipsub, TopicHash}; -use gossipsub::{IdentTopic, PublishError}; use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; +use gossipsub::{IdentTopic, PublishError}; use types::partial_data_column_sidecar::{DanglingPartialDataColumn, PartialDataColumnSidecar}; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, @@ -425,13 +425,8 @@ impl PubsubMessage { } } - /// Encodes a `PubsubMessage` based on the topic encodings. The first known encoding is used. If - /// no encoding is known, and error is returned. - pub fn publish( - &self, - gossipsub: &mut Gossipsub, - topic: IdentTopic, - ) -> Result<(), PublishError> { + /// Encodes a `PubsubMessage`. + pub fn encode(&self) -> EncodedPubsubMessage { // Currently do not employ encoding strategies based on the topic. All messages are ssz // encoded. // Also note, that the compression is handled by the `SnappyTransform` struct. Gossipsub will compress the @@ -441,7 +436,7 @@ impl PubsubMessage { PubsubMessage::BlobSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::DataColumnSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::PartialDataColumnSidecar(data) => { - return gossipsub.publish_partial(topic, &data.1); + return EncodedPubsubMessage::PartialDataColumnSidecarMessage(data.1.clone()); } PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(), @@ -454,7 +449,7 @@ impl PubsubMessage { PubsubMessage::LightClientFinalityUpdate(data) => data.as_ssz_bytes(), PubsubMessage::LightClientOptimisticUpdate(data) => data.as_ssz_bytes(), }; - gossipsub.publish(topic, bytes).map(|_| ()) + EncodedPubsubMessage::Full(bytes) } } @@ -529,3 +524,20 @@ impl std::fmt::Display for PubsubMessage { } } } + +pub enum EncodedPubsubMessage { + Full(Vec), + PartialDataColumnSidecarMessage(PartialDataColumnSidecarMessage), +} + +impl EncodedPubsubMessage { + pub fn do_publish(&self, gossipsub: &mut Gossipsub, topic: IdentTopic) -> Result<(), PublishError> { + match self { + EncodedPubsubMessage::Full(bytes) => gossipsub + .publish(topic, bytes.clone()) + .map(|_| ()), + EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) => gossipsub + .publish_partial(topic, partial), + } + } +} From bb575adc3a92a39985a6af97aa929048a40e3165 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 01:50:29 +0100 Subject: [PATCH 11/30] update gossipsub --- Cargo.lock | 46 +++++++++---------- .../lighthouse_network/src/service/mod.rs | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 64430c3cf72..a1e5c52fb16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2168,7 +2168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn 2.0.100", + "syn 1.0.109", ] [[package]] @@ -5165,7 +5165,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "bytes", "either", @@ -5197,7 +5197,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5207,7 +5207,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5217,7 +5217,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "either", "fnv", @@ -5241,7 +5241,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "async-trait", "futures", @@ -5256,7 +5256,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "async-channel 2.3.1", "asynchronous-codec", @@ -5286,7 +5286,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "asynchronous-codec", "either", @@ -5328,7 +5328,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "hickory-proto", @@ -5346,7 +5346,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "libp2p-core", @@ -5361,7 +5361,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "asynchronous-codec", "bytes", @@ -5379,7 +5379,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "asynchronous-codec", "bytes", @@ -5401,7 +5401,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "asynchronous-codec", "bytes", @@ -5416,7 +5416,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "futures-timer", @@ -5437,7 +5437,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "either", "fnv", @@ -5458,7 +5458,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "heck 0.5.0", "quote", @@ -5468,7 +5468,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "futures-timer", @@ -5483,7 +5483,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "futures-rustls", @@ -5501,7 +5501,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "futures-timer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "either", "futures", @@ -6188,7 +6188,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "bytes", "futures", @@ -7437,7 +7437,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "asynchronous-codec", "bytes", @@ -8250,7 +8250,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#954514071c660133a96efa616498d835fda3114d" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" dependencies = [ "futures", "pin-project", diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 2dd7ebbe5f3..5dde1a55b95 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -810,7 +810,7 @@ impl Network { let partial = topic.kind().supports_partial_messages(); let topic: Topic = topic.into(); - match self.gossipsub_mut().subscribe(&topic, partial) { + match self.gossipsub_mut().subscribe(&topic, partial, partial) { Err(e) => { warn!(%topic, error = ?e, "Failed to subscribe to topic"); false From eeaf03b0222fff7cc7baccb0e68e862d7b9de0ce Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 02:05:06 +0100 Subject: [PATCH 12/30] add CLI flags for disable request and disable support --- beacon_node/lighthouse_network/src/config.rs | 8 +++++++ .../lighthouse_network/src/service/mod.rs | 11 +++++++++- .../lighthouse_network/src/types/pubsub.rs | 21 ++++++++++++------- beacon_node/src/cli.rs | 16 ++++++++++++++ beacon_node/src/config.rs | 8 +++++++ 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 89c6c58d4f6..e022fcbed1e 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -143,6 +143,12 @@ pub struct Config { /// Flag for advertising a fake CGC to peers for testing ONLY. pub advertise_false_custody_group_count: Option, + + /// Whether to disable signaling partial message support. Implies disable_partial_messages_request + pub disable_partial_messages_support: bool, + + /// Whether to disable requesting partial messages + pub disable_partial_messages_request: bool, } impl Config { @@ -368,6 +374,8 @@ impl Default for Config { inbound_rate_limiter_config: None, idontwant_message_size_threshold: DEFAULT_IDONTWANT_MESSAGE_SIZE_THRESHOLD, advertise_false_custody_group_count: None, + disable_partial_messages_support: false, + disable_partial_messages_request: false, } } } diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 5dde1a55b95..0f54d524d03 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -807,10 +807,19 @@ impl Network { .write() .insert(topic.clone()); + let config = &self.network_globals.config; + let disable_request = + config.disable_partial_messages_request || config.disable_partial_messages_support; + let disable_support = config.disable_partial_messages_support; + let partial = topic.kind().supports_partial_messages(); let topic: Topic = topic.into(); - match self.gossipsub_mut().subscribe(&topic, partial, partial) { + match self.gossipsub_mut().subscribe( + &topic, + partial && !disable_request, + partial && !disable_support, + ) { Err(e) => { warn!(%topic, error = ?e, "Failed to subscribe to topic"); false diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 2c2c110a2be..b8c4aac6612 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -1,13 +1,13 @@ //! Handles the encoding and decoding of pubsub messages. -use crate::{Gossipsub, TopicHash}; use crate::types::partial::PartialDataColumnSidecarMessage; use crate::types::{GossipEncoding, GossipKind, GossipTopic}; +use crate::{Gossipsub, TopicHash}; +use gossipsub::{IdentTopic, PublishError}; use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; -use gossipsub::{IdentTopic, PublishError}; use types::partial_data_column_sidecar::{DanglingPartialDataColumn, PartialDataColumnSidecar}; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, @@ -531,13 +531,18 @@ pub enum EncodedPubsubMessage { } impl EncodedPubsubMessage { - pub fn do_publish(&self, gossipsub: &mut Gossipsub, topic: IdentTopic) -> Result<(), PublishError> { + pub fn do_publish( + &self, + gossipsub: &mut Gossipsub, + topic: IdentTopic, + ) -> Result<(), PublishError> { match self { - EncodedPubsubMessage::Full(bytes) => gossipsub - .publish(topic, bytes.clone()) - .map(|_| ()), - EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) => gossipsub - .publish_partial(topic, partial), + EncodedPubsubMessage::Full(bytes) => { + gossipsub.publish(topic, bytes.clone()).map(|_| ()) + } + EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) => { + gossipsub.publish_partial(topic, partial) + } } } } diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index e4c7c6ff1fe..1bf6303b3ab 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -670,6 +670,22 @@ pub fn cli_app() -> Command { .hide(true) .display_order(0) ) + .arg( + Arg::new("disable-partial-messages-request") + .long("disable-partial-messages-request") + .help("Do not request partial messages for data columns.") + .action(ArgAction::SetTrue) + .help_heading(FLAG_HEADER) + .display_order(0) + ) + .arg( + Arg::new("disable-partial-messages-support") + .long("disable-partial-messages-support") + .help("Do not support receiving partial messages for data columns.") + .action(ArgAction::SetTrue) + .help_heading(FLAG_HEADER) + .display_order(0) + ) /* * Monitoring metrics */ diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 3b0e80e0b7d..05051fcbe0f 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1492,6 +1492,14 @@ pub fn set_network_config( })?; } + if parse_flag(cli_args, "disable-partial-messages-request") { + config.disable_partial_messages_request = true; + } + + if parse_flag(cli_args, "disable-partial-messages-support") { + config.disable_partial_messages_support = true; + } + Ok(()) } From 47a411962c22eb5a1003ad27f55436930636d196 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 10:46:00 +0100 Subject: [PATCH 13/30] update CLI docs --- book/src/help_bn.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/book/src/help_bn.md b/book/src/help_bn.md index 5f3c43a7e42..9965938006c 100644 --- a/book/src/help_bn.md +++ b/book/src/help_bn.md @@ -477,6 +477,10 @@ Flags: --disable-packet-filter Disables the discovery packet filter. Useful for testing in smaller networks + --disable-partial-messages-request + Do not request partial messages for data columns. + --disable-partial-messages-support + Do not support receiving partial messages for data columns. --disable-proposer-reorgs Do not attempt to reorg late blocks from other validators when proposing. From 65c236a3a64060193146f92ed1b293aef01bcf4e Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 11:19:26 +0100 Subject: [PATCH 14/30] do not error for column on failed observation check --- beacon_node/beacon_chain/src/data_column_verification.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index cdfd6957539..3791c9e2009 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -602,11 +602,10 @@ pub fn validate_data_column_sidecar_for_gossip Date: Fri, 31 Oct 2025 11:55:03 +0100 Subject: [PATCH 15/30] Fix get blobs v2 tests --- .../beacon_chain/src/fetch_blobs/tests.rs | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs index cbe2f78fbda..059f4e07138 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/tests.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/tests.rs @@ -25,7 +25,7 @@ mod get_blobs_v2 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_no_blobs_in_block() { - let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu); + let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu, false); let (publish_fn, _s) = mock_publish_fn(); let block = SignedBeaconBlock::::Fulu(SignedBeaconBlockFulu { message: BeaconBlockFulu::empty(mock_adapter.spec()), @@ -53,7 +53,7 @@ mod get_blobs_v2 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_no_blobs_returned() { - let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu); + let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu, false); let (publish_fn, _) = mock_publish_fn(); let (block, _blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -78,7 +78,7 @@ mod get_blobs_v2 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_partial_blobs_returned() { - let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu); + let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let (block, mut blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -111,7 +111,7 @@ mod get_blobs_v2 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_block_imported_after_el_response() { - let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu); + let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -144,7 +144,7 @@ mod get_blobs_v2 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_no_new_columns_to_import() { - let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu); + let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -184,7 +184,7 @@ mod get_blobs_v2 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v2_success() { - let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu); + let mut mock_adapter = mock_beacon_adapter(ForkName::Fulu, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -259,7 +259,7 @@ mod get_blobs_v1 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v1_no_blobs_in_block() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK); + let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let spec = mock_adapter.spec(); let (publish_fn, _s) = mock_publish_fn(); let block_no_blobs = @@ -287,7 +287,7 @@ mod get_blobs_v1 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v1_no_blobs_returned() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK); + let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let (publish_fn, _) = mock_publish_fn(); let (block, _blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -314,7 +314,7 @@ mod get_blobs_v1 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v1_partial_blobs_returned() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK); + let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let blob_count = 2; let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, blob_count); @@ -372,7 +372,7 @@ mod get_blobs_v1 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v1_block_imported_after_el_response() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK); + let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -405,7 +405,7 @@ mod get_blobs_v1 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v1_no_new_blobs_to_import() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK); + let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, 2); let block_root = block.canonical_root(); @@ -453,7 +453,7 @@ mod get_blobs_v1 { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_fetch_blobs_v1_success() { - let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK); + let mut mock_adapter = mock_beacon_adapter(ELECTRA_FORK, false); let (publish_fn, publish_fn_args) = mock_publish_fn(); let blob_count = 2; let (block, blobs_and_proofs) = create_test_block_and_blobs(&mock_adapter, blob_count); @@ -606,7 +606,7 @@ fn mock_publish_fn() -> ( (publish_fn, captured_args) } -fn mock_beacon_adapter(fork_name: ForkName) -> MockFetchBlobsBeaconAdapter { +fn mock_beacon_adapter(fork_name: ForkName, get_blobs_v3: bool) -> MockFetchBlobsBeaconAdapter { let test_runtime = TestRuntime::default(); let spec = Arc::new(fork_name.make_genesis_spec(E::default_spec())); let kzg = get_kzg(&spec); @@ -618,4 +618,7 @@ fn mock_beacon_adapter(fork_name: ForkName) -> MockFetchBlobsBeaconAdapter { .expect_executor() .return_const(test_runtime.task_executor.clone()); mock_adapter + .expect_supports_get_blobs_v3() + .returning(move || Ok(get_blobs_v3)); + mock_adapter } From 8442bf3db13edd418d64daa94b53628e41e5e843 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 11:55:32 +0100 Subject: [PATCH 16/30] Reintroduce sanity check for get blobs v2 --- beacon_node/beacon_chain/src/fetch_blobs/mod.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index ea77b9eb9d4..048203ff15f 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -32,7 +32,7 @@ use mockall_double::double; use ssz_types::FixedVector; use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::sync::Arc; -use tracing::{Span, debug, instrument}; +use tracing::{Span, debug, instrument, warn}; use types::blob_sidecar::BlobSidecarError; use types::das_column::DasColumn; use types::data_column_sidecar::DataColumnSidecarError; @@ -246,7 +246,8 @@ async fn fetch_and_process_blobs_v2_or_v3( metrics::observe(&metrics::BLOBS_FROM_EL_EXPECTED, num_expected_blobs as f64); - let response = if chain_adapter.supports_get_blobs_v3().await? { + let get_blobs_v3 = chain_adapter.supports_get_blobs_v3().await?; + let response = if get_blobs_v3 { debug!(num_expected_blobs, "Fetching available blobs from the EL"); chain_adapter .get_blobs_v3(versioned_hashes) @@ -274,6 +275,18 @@ async fn fetch_and_process_blobs_v2_or_v3( let num_fetched_blobs = blobs_and_proofs.len(); metrics::observe(&metrics::BLOBS_FROM_EL_RECEIVED, num_fetched_blobs as f64); + if !get_blobs_v3 && num_fetched_blobs != num_expected_blobs { + // This scenario is not supposed to happen if the EL is spec compliant. + // It should either return all requested blobs or none, but NOT partial responses. + // If we attempt to compute columns with partial blobs, we'd end up with invalid columns. + warn!( + num_fetched_blobs, + num_expected_blobs, "The EL did not return all requested blobs" + ); + inc_counter(&metrics::BLOBS_FROM_EL_MISS_TOTAL); + return Ok(None); + } + debug!(num_fetched_blobs, "Blobs received from the EL"); inc_counter(&metrics::BLOBS_FROM_EL_HIT_TOTAL); From 7c68ba449d761e8270fac3070e9838f4d64c5148 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 16:20:13 +0100 Subject: [PATCH 17/30] Only count actually received blobs --- beacon_node/beacon_chain/src/fetch_blobs/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 048203ff15f..7b0295dacca 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -272,7 +272,7 @@ async fn fetch_and_process_blobs_v2_or_v3( return Ok(None); }; - let num_fetched_blobs = blobs_and_proofs.len(); + let num_fetched_blobs = response.iter().filter(|opt| opt.is_some()).count(); metrics::observe(&metrics::BLOBS_FROM_EL_RECEIVED, num_fetched_blobs as f64); if !get_blobs_v3 && num_fetched_blobs != num_expected_blobs { From 09e220b2922f85f8dabcc0fd79d62367d863ff4a Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 16:20:55 +0100 Subject: [PATCH 18/30] Log full vs partial --- beacon_node/lighthouse_network/src/types/pubsub.rs | 4 ++++ beacon_node/network/src/service.rs | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index b8c4aac6612..0d04995ec93 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -451,6 +451,10 @@ impl PubsubMessage { }; EncodedPubsubMessage::Full(bytes) } + + pub fn is_partial(&self) -> bool { + matches!(self, PubsubMessage::PartialDataColumnSidecar(_)) + } } impl std::fmt::Display for PubsubMessage { diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index dd9448983c6..23d662c9366 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -637,8 +637,14 @@ impl NetworkService { NetworkMessage::Publish { messages } => { let mut topic_kinds = Vec::new(); for message in &messages { - if !topic_kinds.contains(&message.kind()) { - topic_kinds.push(message.kind()); + let message_kind = if message.is_partial() { + "partial" + } else { + "full" + }; + let kind = (message.kind(), message_kind); + if !topic_kinds.contains(&kind) { + topic_kinds.push(kind); } } debug!( From 7cbb5c652e80a94ed64f92292b2bcbd1ed0d41e6 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Fri, 31 Oct 2025 16:44:23 +0100 Subject: [PATCH 19/30] fix and improve logging --- .../beacon_chain/src/fetch_blobs/mod.rs | 2 +- .../lighthouse_network/src/service/mod.rs | 28 +++++++++++++++++-- .../lighthouse_network/src/types/pubsub.rs | 2 +- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 7b0295dacca..66e82f548b7 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -272,7 +272,7 @@ async fn fetch_and_process_blobs_v2_or_v3( return Ok(None); }; - let num_fetched_blobs = response.iter().filter(|opt| opt.is_some()).count(); + let num_fetched_blobs = blobs_and_proofs.iter().filter(|opt| opt.is_some()).count(); metrics::observe(&metrics::BLOBS_FROM_EL_RECEIVED, num_fetched_blobs as f64); if !get_blobs_v3 && num_fetched_blobs != num_expected_blobs { diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 0a34bffb1dd..5f2dc7083a7 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -35,12 +35,14 @@ use libp2p::{PeerId, SwarmBuilder, identify}; use logging::crit; use network_utils::enr_ext::EnrExt; use sha2::{Digest, Sha256}; +use ssz::Decode; use std::num::{NonZeroU8, NonZeroUsize}; use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; use tracing::{debug, error, info, trace, warn}; +use types::partial_data_column_sidecar::CellBitmap; use types::{ChainSpec, ForkName}; use types::{ EnrForkId, EthSpec, ForkContext, Slot, SubnetId, consts::altair::SYNC_COMMITTEE_SUBNET_COUNT, @@ -1314,8 +1316,21 @@ impl Network { propagation_source, group_id, message, - metadata: _, + metadata, } => { + // TODO(dknopik): this currently hardcodes the data column metadata format + if let Some(metadata) = metadata { + if let Ok(metadata) = CellBitmap::::from_ssz_bytes(&metadata) { + let metadata = hex::encode(metadata.as_slice()); + debug!(%metadata, "Got metadata") + } else { + warn!(?metadata, "Got weird metadata"); + } + } else { + debug!("Got no metadata") + } + + let hex_group_id = hex::encode(&group_id); let topic = GossipTopic::decode(topic_id.as_str()) .inspect_err(|error| { debug!( @@ -1335,18 +1350,20 @@ impl Network { match result { Ok(_) => { debug!( - group_id = hex::encode(&group_id), + group_id = hex_group_id, "Replied with cached partial message" ) } Err(err) => { warn!( - group_id = hex::encode(&group_id), + group_id = hex_group_id, ?err, "Failed to reply with cached partial message" ); } } + } else { + debug!(group_id = hex_group_id, "Did not reply to partial message"); } if let Some(message) = message { @@ -1366,6 +1383,11 @@ impl Network { //); } Ok(message) => { + debug!( + %message, + %propagation_source, + "Decoded partial message" + ); // Notify the network return Some(NetworkEvent::PubsubMessage { id: MessageId::new(&[]), // TODO(dknopik): waht to send diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index 0d04995ec93..d9f9997d2d0 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -482,7 +482,7 @@ impl std::fmt::Display for PubsubMessage { f, "PartialDataColumnSidecar: group: {}, column index: {}, cells: {}", data.1.partial_column.block_root, - data.1.partial_column.block_root, + data.1.partial_column.index, hex::encode( data.1 .partial_column From b6626f610e436290f6d5d1ae7337458a05621387 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 3 Nov 2025 11:24:51 +0100 Subject: [PATCH 20/30] update gossipsub --- Cargo.lock | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a1e5c52fb16..e1b8f41038d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2168,7 +2168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.100", ] [[package]] @@ -5165,7 +5165,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "bytes", "either", @@ -5197,7 +5197,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5207,7 +5207,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5217,7 +5217,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "either", "fnv", @@ -5241,7 +5241,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "async-trait", "futures", @@ -5256,7 +5256,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "async-channel 2.3.1", "asynchronous-codec", @@ -5286,7 +5286,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "asynchronous-codec", "either", @@ -5328,7 +5328,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "hickory-proto", @@ -5346,7 +5346,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "libp2p-core", @@ -5361,7 +5361,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "asynchronous-codec", "bytes", @@ -5379,7 +5379,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "asynchronous-codec", "bytes", @@ -5401,7 +5401,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "asynchronous-codec", "bytes", @@ -5416,7 +5416,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "futures-timer", @@ -5437,7 +5437,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "either", "fnv", @@ -5458,7 +5458,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "heck 0.5.0", "quote", @@ -5468,7 +5468,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "futures-timer", @@ -5483,7 +5483,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "futures-rustls", @@ -5501,7 +5501,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "futures-timer", @@ -5515,7 +5515,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "either", "futures", @@ -6188,7 +6188,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "bytes", "futures", @@ -7437,7 +7437,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "asynchronous-codec", "bytes", @@ -8250,7 +8250,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#dd21893d23073295dfcea66c5496e92d3a000e73" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" dependencies = [ "futures", "pin-project", From f0b7d8fd4ee6330f0cb0dd5bb7ca9db533b36d34 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 3 Nov 2025 12:10:29 +0100 Subject: [PATCH 21/30] look for block in data availability checker --- .../gossip_methods.rs | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 500842ff149..3aef2f6cae6 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -10,8 +10,9 @@ use beacon_chain::data_availability_checker::MergedData; use beacon_chain::data_column_verification::{GossipDataColumnError, GossipVerifiedDataColumn}; use beacon_chain::store::Error; use beacon_chain::{ - AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, ForkChoiceError, - GossipVerifiedBlock, IntoExecutionPendingBlock, NotifyExecutionLayer, + AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, + BlockProcessStatus, ForkChoiceError, GossipVerifiedBlock, IntoExecutionPendingBlock, + NotifyExecutionLayer, attestation_verification::{self, Error as AttnError, VerifiedAttestation}, data_availability_checker::AvailabilityCheckErrorCategory, light_client_finality_update_verification::Error as LightClientFinalityUpdateError, @@ -791,26 +792,17 @@ impl NetworkBeaconProcessor { ) { let partial_column = match self .chain - .store - .try_get_full_block(&column_sidecar.block_root) + .data_availability_checker + .get_cached_block(&column_sidecar.block_root) { - Ok(Some(DatabaseBlock::Full(block))) => { - Some(VerifiablePartialDataColumn::from_dangling_and_block( - column_sidecar.clone(), - &block, - )) - } - Ok(Some(DatabaseBlock::Blinded(block))) => { - Some(VerifiablePartialDataColumn::from_dangling_and_block( - column_sidecar.clone(), - &block, - )) - } - Ok(None) => None, - Err(err) => { - warn!(?err, "Error getting block for partial data column"); - None - } + Some( + BlockProcessStatus::ExecutionValidated(block) + | BlockProcessStatus::NotValidated(block, _), + ) => Some(VerifiablePartialDataColumn::from_dangling_and_block( + column_sidecar.clone(), + &block, + )), + None | Some(BlockProcessStatus::Unknown) => None, }; match partial_column { From 99d3568161cf1cb29deb26353fc8702d38092151 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 3 Nov 2025 13:46:11 +0100 Subject: [PATCH 22/30] re-enable SSE --- beacon_node/beacon_chain/src/beacon_chain.rs | 38 ++++++++++--------- .../gossip_methods.rs | 1 - 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 11694acb24c..5f2bc2a3005 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3089,11 +3089,10 @@ impl BeaconChain { return Err(BlockError::DuplicateFullyImported(block_root)); } - // TODO(dknopik): impl SSE - //self.emit_sse_data_column_sidecar_events( - // &block_root, - // data_columns.iter().map(|column| column.as_data_column()), - //); + self.emit_sse_data_column_sidecar_events( + &block_root, + data_columns.iter().map(|column| column.as_data_column()), + ); self.check_gossip_data_columns_availability_and_import( slot, @@ -3166,12 +3165,11 @@ impl BeaconChain { EngineGetBlobsOutput::Blobs(blobs) => { self.emit_sse_blob_sidecar_events(&block_root, blobs.iter().map(|b| b.as_blob())); } - EngineGetBlobsOutput::CustodyColumns(_columns) => { - // TODO(dknopik): impl SSE - //self.emit_sse_data_column_sidecar_events( - // &block_root, - // columns.iter().map(|column| column.as_data_column()), - //); + EngineGetBlobsOutput::CustodyColumns(columns) => { + self.emit_sse_data_column_sidecar_events( + &block_root, + columns.iter().map(|column| column.as_data_column()), + ); } } @@ -3200,26 +3198,30 @@ impl BeaconChain { } } - fn emit_sse_data_column_sidecar_events<'a, I>( + fn emit_sse_data_column_sidecar_events<'a, I, C>( self: &Arc, block_root: &Hash256, data_columns_iter: I, ) where - I: Iterator>, + I: Iterator, + C: DasColumn + 'a, { if let Some(event_handler) = self.event_handler.as_ref() && event_handler.has_data_column_sidecar_subscribers() { let imported_data_columns = self .data_availability_checker - .cached_data_column_indexes(block_root) + .get_data_columns(*block_root) .unwrap_or_default(); - let new_data_columns = - data_columns_iter.filter(|b| !imported_data_columns.contains(&b.index)); - for data_column in new_data_columns { + let new_data_columns_indices = data_columns_iter.map(|c| c.index()).collect::>(); + + for data_column in imported_data_columns + .into_iter() + .filter(|c| new_data_columns_indices.contains(&c.index)) + { event_handler.register(EventKind::DataColumnSidecar( - SseDataColumnSidecar::from_data_column_sidecar(data_column), + SseDataColumnSidecar::from_data_column_sidecar(data_column.as_ref()), )); } } diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 3aef2f6cae6..9333a6ef9d6 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -56,7 +56,6 @@ use beacon_processor::{ ReprocessQueueMessage, }, }; -use store::DatabaseBlock; use types::das_column::DasColumn; use types::partial_data_column_sidecar::{DanglingPartialDataColumn, VerifiablePartialDataColumn}; From 646fef59f994a1e24aade34dcbb44d2be3c24e11 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 3 Nov 2025 15:25:54 +0100 Subject: [PATCH 23/30] Properly handly observing in new_for_block_publishing --- .../beacon_chain/src/data_column_verification.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index 3791c9e2009..e5df753a91c 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -292,12 +292,11 @@ impl // In this case, we should accept it for gossip propagation. verify_is_unknown_sidecar(chain, &column_sidecar)?; - // TODO(dknopik): is proper handling of none? if chain .data_availability_checker .determine_missing_cells(&column_sidecar.block_root(), column_sidecar.as_ref()) - .ok_or_else(|| GossipDataColumnError::UnexpectedDataColumn)? - .is_empty() + .map(|cells| cells.is_empty()) + .unwrap_or(false) { // Observe this data column so we don't process it again. if O::observe() { @@ -680,12 +679,6 @@ pub fn validate_partial_data_column_sidecar_for_gossip< let kzg_verified_data_column = verify_kzg_for_data_column(filtered_column, kzg) .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; - // TODO(dknopik): observe?! I do not think so, as publishing works differently anyway... - // Basically we publish if we update our internal view, but this is merged elsewhere - //if O::observe() { - // observe_gossip_data_column(&data_column, chain)?; - //} - Ok(GossipVerifiedDataColumn { block_root: data_column.block_root(), data_column: kzg_verified_data_column, From 6794e5fddc98e4146fe7a5245b1361d24e7802e9 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 5 Nov 2025 16:57:48 +0100 Subject: [PATCH 24/30] Fix cache and better logging --- .../src/service/gossip_cache.rs | 6 +++ .../lighthouse_network/src/service/mod.rs | 52 +++++++++++++------ 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/gossip_cache.rs b/beacon_node/lighthouse_network/src/service/gossip_cache.rs index 5aa655def96..8259fa0701d 100644 --- a/beacon_node/lighthouse_network/src/service/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/service/gossip_cache.rs @@ -96,6 +96,12 @@ impl GossipCacheBuilder { self } + /// Timeout for data column sidecars. + pub fn data_column_sidecar(mut self, timeout: Duration) -> Self { + self.data_column_sidecar = Some(timeout); + self + } + /// Timeout for aggregate attestations. pub fn aggregates_timeout(mut self, timeout: Duration) -> Self { self.aggregates = Some(timeout); diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 5f2dc7083a7..7672782fb81 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -259,6 +259,7 @@ impl Network { // .signed_contribution_and_proof_timeout(timeout) // Do not retry // .sync_committee_message_timeout(timeout) // Do not retry .bls_to_execution_change_timeout(half_epoch * 2) + .data_column_sidecar(slot_duration) .build() }; @@ -901,6 +902,11 @@ impl Network { &message_data { let id = partial.group_id().as_ref().to_vec(); + debug!( + kind = %topic.kind(), + group_id = hex::encode(&id), + "Adding partial message to cache" + ); self.gossip_cache.insert(topic, id, message_data); } } @@ -1318,18 +1324,6 @@ impl Network { message, metadata, } => { - // TODO(dknopik): this currently hardcodes the data column metadata format - if let Some(metadata) = metadata { - if let Ok(metadata) = CellBitmap::::from_ssz_bytes(&metadata) { - let metadata = hex::encode(metadata.as_slice()); - debug!(%metadata, "Got metadata") - } else { - warn!(?metadata, "Got weird metadata"); - } - } else { - debug!("Got no metadata") - } - let hex_group_id = hex::encode(&group_id); let topic = GossipTopic::decode(topic_id.as_str()) .inspect_err(|error| { @@ -1341,29 +1335,55 @@ impl Network { }) .ok()?; + // TODO(dknopik): this currently hardcodes the data column metadata format + if let Some(metadata) = metadata { + if let Ok(metadata) = CellBitmap::::from_ssz_bytes(&metadata) { + let mut metadata_string = String::with_capacity(metadata.len()); + for bit in metadata.iter() { + if bit { + metadata_string.push('1'); + } else { + metadata_string.push('0'); + } + } + debug!(metadata = metadata_string, %topic, %propagation_source, "Got metadata") + } else { + warn!(?metadata, %topic, %propagation_source, "Got weird metadata"); + } + } else { + debug!(%propagation_source, "Got no metadata") + } + // TODO(dknopik): maybe do this after validation? if let Some(cached_partial) = self.gossip_cache.retrieve_one(&topic, &group_id) { // We do not bother checking here if that cached msg contains relevant data, // as gossipsub will do that anyway. - let result = cached_partial - .do_publish(&mut self.swarm.behaviour_mut().gossipsub, topic.into()); + let result = cached_partial.do_publish( + &mut self.swarm.behaviour_mut().gossipsub, + topic.clone().into(), + ); match result { Ok(_) => { debug!( group_id = hex_group_id, - "Replied with cached partial message" + %topic, %propagation_source, "Replied with cached partial message" ) } Err(err) => { warn!( group_id = hex_group_id, + %topic, + %propagation_source, ?err, "Failed to reply with cached partial message" ); } } } else { - debug!(group_id = hex_group_id, "Did not reply to partial message"); + debug!( + group_id = hex_group_id, + %topic, %propagation_source, "Did not reply to partial message" + ); } if let Some(message) = message { From e98dce0f75ace74dfdf80e2e306e185f5ec19f7b Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 5 Nov 2025 16:59:45 +0100 Subject: [PATCH 25/30] Fix invalid `UnexpectedDataColumn` error --- .../src/data_availability_checker.rs | 5 +++- .../src/data_column_verification.rs | 26 +++++++++++-------- consensus/types/src/das_column.rs | 1 + 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index b43e0826f7b..51d9382877f 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -195,7 +195,10 @@ impl DataAvailabilityChecker { match cached.compare(data_column) { ColumnComparison::Equal => Some(vec![]), ColumnComparison::MissingCells { missing_in_lhs, .. } => Some(missing_in_lhs), - _ => None, + comparison => { + debug!(?comparison, "Unexpected columns comparison"); + None + } } } diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index e5df753a91c..0ccdcef6afd 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -655,22 +655,26 @@ pub fn validate_partial_data_column_sidecar_for_gossip< > { // TODO(dknopik): This is kinda underspecified, just slap everything in here that *could* apply let column_slot = data_column.slot(); + let block_root = data_column.block_root(); verify_sidecar_not_from_future_slot(chain, column_slot)?; verify_slot_greater_than_latest_finalized_slot(chain, column_slot)?; - let missing_cells = chain + let filtered_column = if let Some(missing_cells) = chain .data_availability_checker - .determine_missing_cells(&data_column.block_root(), data_column.as_ref()) - .ok_or_else(|| GossipDataColumnError::UnexpectedDataColumn)?; - if missing_cells.is_empty() { - return Err(GossipDataColumnError::PriorKnownUnpublished); - } + .determine_missing_cells(&block_root, data_column.as_ref()) + { + if missing_cells.is_empty() { + return Err(GossipDataColumnError::PriorKnownUnpublished); + } - let filtered_column = Arc::new( + Arc::new( + data_column + .clone_filter(|idx| missing_cells.contains(&idx)) + .ok_or_else(|| GossipDataColumnError::PriorKnownUnpublished)?, + ) + } else { data_column - .clone_filter(|idx| missing_cells.contains(&idx)) - .ok_or_else(|| GossipDataColumnError::UnexpectedDataColumn)?, - ); + }; // We do not have to check block related data here, as we create the verifiable column from // gossip accepted block @@ -680,7 +684,7 @@ pub fn validate_partial_data_column_sidecar_for_gossip< .map_err(|(_, e)| GossipDataColumnError::InvalidKzgProof(e))?; Ok(GossipVerifiedDataColumn { - block_root: data_column.block_root(), + block_root, data_column: kzg_verified_data_column, _phantom: PhantomData, }) diff --git a/consensus/types/src/das_column.rs b/consensus/types/src/das_column.rs index 3e25576fdac..a1597dbd8d2 100644 --- a/consensus/types/src/das_column.rs +++ b/consensus/types/src/das_column.rs @@ -74,6 +74,7 @@ pub trait DasColumn: Clone { } } +#[derive(Debug)] pub enum ColumnComparison { DifferentColumns, DataConflict, From 5e1f05dd2fb231360ad8091c4bb31e1c7d221a84 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 5 Nov 2025 17:01:42 +0100 Subject: [PATCH 26/30] remove custom cache in favour of reprocessing queue --- beacon_node/beacon_processor/src/lib.rs | 7 +- .../src/scheduler/work_reprocessing_queue.rs | 117 ++++++++++++++++- beacon_node/network/src/lib.rs | 1 - .../gossip_methods.rs | 122 +++++++++++------- .../src/network_beacon_processor/mod.rs | 7 +- .../network_beacon_processor/sync_methods.rs | 15 +++ .../network/src/partial_data_column_cache.rs | 76 ----------- beacon_node/network/src/router.rs | 6 +- 8 files changed, 217 insertions(+), 134 deletions(-) delete mode 100644 beacon_node/network/src/partial_data_column_cache.rs diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index c3dfd0a9126..21f98175445 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -39,7 +39,8 @@ //! task. use crate::work_reprocessing_queue::{ - QueuedBackfillBatch, QueuedColumnReconstruction, QueuedGossipBlock, ReprocessQueueMessage, + QueuedBackfillBatch, QueuedColumnReconstruction, QueuedGossipBlock, QueuedPartialColumn, + ReprocessQueueMessage, }; use futures::stream::{Stream, StreamExt}; use futures::task::Poll; @@ -493,6 +494,10 @@ impl From for WorkEvent { work: Work::ColumnReconstruction(process_fn), } } + ReadyWork::PartialColumn(QueuedPartialColumn { process_fn, .. }) => Self { + drop_during_sync: true, + work: Work::GossipPartialDataColumnSidecar(process_fn), + }, } } } diff --git a/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs b/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs index c99388287c0..04286714ec0 100644 --- a/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs +++ b/beacon_node/beacon_processor/src/scheduler/work_reprocessing_queue.rs @@ -30,7 +30,7 @@ use strum::AsRefStr; use task_executor::TaskExecutor; use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio_util::time::delay_queue::{DelayQueue, Key as DelayKey}; -use tracing::{debug, error, trace, warn}; +use tracing::{debug, error, info, trace, warn}; use types::{EthSpec, Hash256, Slot}; const TASK_NAME: &str = "beacon_processor_reprocess_queue"; @@ -60,6 +60,9 @@ pub const QUEUED_SAMPLING_REQUESTS_DELAY: Duration = Duration::from_secs(12); /// For how long to queue delayed column reconstruction. pub const QUEUED_RECONSTRUCTION_DELAY: Duration = Duration::from_millis(150); +/// For how long to queue partial columns. +pub const QUEUED_PARTIAL_COLUMN_DELAY: Duration = Duration::from_secs(4); + /// Set an arbitrary upper-bound on the number of queued blocks to avoid DoS attacks. The fact that /// we signature-verify blocks before putting them in the queue *should* protect against this, but /// it's nice to have extra protection. @@ -71,6 +74,9 @@ const MAXIMUM_QUEUED_ATTESTATIONS: usize = 16_384; /// How many light client updates we keep before new ones get dropped. const MAXIMUM_QUEUED_LIGHT_CLIENT_UPDATES: usize = 128; +/// How many partial column messages we keep before new ones get dropped. +const MAXIMUM_QUEUED_PARTIAL_COLUMNS: usize = 256; + // Process backfill batch 50%, 60%, 80% through each slot. // // Note: use caution to set these fractions in a way that won't cause panic-y @@ -115,6 +121,10 @@ pub enum ReprocessQueueMessage { BackfillSync(QueuedBackfillBatch), /// A delayed column reconstruction that needs checking DelayColumnReconstruction(QueuedColumnReconstruction), + /// A block with pending data availability was added. + BlockPending { block_root: Hash256 }, + /// A partial data column that references an unknown block. + UnknownPartialColumn(QueuedPartialColumn), } /// Events sent by the scheduler once they are ready for re-processing. @@ -127,6 +137,7 @@ pub enum ReadyWork { LightClientUpdate(QueuedLightClientUpdate), BackfillSync(QueuedBackfillBatch), ColumnReconstruction(QueuedColumnReconstruction), + PartialColumn(QueuedPartialColumn), } /// An Attestation for which the corresponding block was not seen while processing, queued for @@ -182,6 +193,11 @@ pub struct QueuedColumnReconstruction { pub process_fn: AsyncFn, } +pub struct QueuedPartialColumn { + pub beacon_block_root: Hash256, + pub process_fn: AsyncFn, +} + impl TryFrom> for QueuedBackfillBatch { type Error = WorkEvent; @@ -220,6 +236,8 @@ enum InboundEvent { ReadyBackfillSync(QueuedBackfillBatch), /// A column reconstruction that was queued is ready for processing. ReadyColumnReconstruction(QueuedColumnReconstruction), + /// A partial column that was queued has timed out. + ReadyPartialColumn((DelayKey, QueuedPartialColumn)), /// A message sent to the `ReprocessQueue` Msg(ReprocessQueueMessage), } @@ -242,6 +260,8 @@ struct ReprocessQueue { lc_updates_delay_queue: DelayQueue, /// Queue to manage scheduled column reconstructions. column_reconstructions_delay_queue: DelayQueue, + /// Queue to manage scheduled partial columns. + partial_columns_delay_queue: DelayQueue, /* Queued items */ /// Queued blocks. @@ -260,6 +280,8 @@ struct ReprocessQueue { queued_column_reconstructions: HashMap, /// Queued backfill batches queued_backfill_batches: Vec, + /// Partial columns per root. + awaiting_partial_columns_per_root: HashMap>, /* Aux */ /// Next attestation id, used for both aggregated and unaggregated attestations @@ -355,6 +377,16 @@ impl Stream for ReprocessQueue { Poll::Ready(None) | Poll::Pending => (), } + match self.partial_columns_delay_queue.poll_expired(cx) { + Poll::Ready(Some(partial_column)) => { + return Poll::Ready(Some(InboundEvent::ReadyPartialColumn(( + partial_column.key(), + partial_column.into_inner(), + )))); + } + Poll::Ready(None) | Poll::Pending => (), + } + if let Some(next_backfill_batch_event) = self.next_backfill_batch_event.as_mut() { match next_backfill_batch_event.as_mut().poll(cx) { Poll::Ready(_) => { @@ -422,6 +454,7 @@ impl ReprocessQueue { attestations_delay_queue: DelayQueue::new(), lc_updates_delay_queue: DelayQueue::new(), column_reconstructions_delay_queue: DelayQueue::new(), + partial_columns_delay_queue: DelayQueue::new(), queued_gossip_block_roots: HashSet::new(), queued_lc_updates: FnvHashMap::default(), queued_aggregates: FnvHashMap::default(), @@ -430,6 +463,7 @@ impl ReprocessQueue { awaiting_lc_updates_per_parent_root: HashMap::new(), queued_backfill_batches: Vec::new(), queued_column_reconstructions: HashMap::new(), + awaiting_partial_columns_per_root: HashMap::new(), next_attestation: 0, next_lc_update: 0, early_block_debounce: TimeLatch::default(), @@ -702,6 +736,15 @@ impl ReprocessQueue { ); } } + + // Remove waiting partial columns - as the block is imported they will not be needed + if let Some(queued_partials) = + self.awaiting_partial_columns_per_root.remove(&block_root) + { + for key in queued_partials { + self.partial_columns_delay_queue.remove(&key); + } + } } InboundEvent::Msg(NewLightClientOptimisticUpdate { parent_root }) => { // Unqueue the light client optimistic updates we have for this root, if any. @@ -784,6 +827,62 @@ impl ReprocessQueue { } } } + InboundEvent::Msg(BlockPending { block_root }) => { + if let Some(queued_partials) = + self.awaiting_partial_columns_per_root.remove(&block_root) + { + let mut sent_count = 0; + let mut failed_count = 0; + for key in queued_partials { + let partial_column = self.partial_columns_delay_queue.remove(&key); + + if self + .ready_work_tx + .try_send(ReadyWork::PartialColumn(partial_column.into_inner())) + .is_ok() + { + sent_count += 1; + } else { + failed_count += 1; + } + } + + if failed_count > 0 { + error!( + hint = "system may be overloaded", + ?block_root, + failed_count, + sent_count, + "Ignored scheduled partial column(s) for block" + ); + } else { + info!(?block_root, sent_count, "Sent partial column(s) for block") + } + } + } + InboundEvent::Msg(UnknownPartialColumn(queued_partial_column)) => { + if self.partial_columns_delay_queue.len() >= MAXIMUM_QUEUED_PARTIAL_COLUMNS { + error!( + queue_size = MAXIMUM_QUEUED_PARTIAL_COLUMNS, + msg = "system resources may be saturated", + "Partial columns delay queue is full" + ); + return; + } + + let block_root = queued_partial_column.beacon_block_root; + + // Register the delay. + let delay_key = self + .partial_columns_delay_queue + .insert(queued_partial_column, QUEUED_ATTESTATION_DELAY); + + // Register this attestation for the corresponding root. + self.awaiting_partial_columns_per_root + .entry(block_root) + .or_default() + .push(delay_key); + } // A block that was queued for later processing is now ready to be processed. InboundEvent::ReadyGossipBlock(ready_block) => { let block_root = ready_block.beacon_block_root; @@ -934,6 +1033,22 @@ impl ReprocessQueue { ); } } + InboundEvent::ReadyPartialColumn((key, partial_column)) => { + if let Entry::Occupied(mut queued_cols) = self + .awaiting_partial_columns_per_root + .entry(partial_column.beacon_block_root) + && let Some(index) = queued_cols.get().iter().position(|&k| k == key) + { + let queued_atts_mut = queued_cols.get_mut(); + queued_atts_mut.swap_remove(index); + + if queued_atts_mut.is_empty() { + queued_cols.remove_entry(); + } + } + + // Do not send partial column, we timed out already. + } } metrics::set_gauge_vec( diff --git a/beacon_node/network/src/lib.rs b/beacon_node/network/src/lib.rs index db2fd876287..2a7fedb53e9 100644 --- a/beacon_node/network/src/lib.rs +++ b/beacon_node/network/src/lib.rs @@ -4,7 +4,6 @@ pub mod service; mod metrics; mod nat; mod network_beacon_processor; -mod partial_data_column_cache; mod persisted_dht; mod router; mod status; diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 9333a6ef9d6..ce2aad392bb 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -11,8 +11,7 @@ use beacon_chain::data_column_verification::{GossipDataColumnError, GossipVerifi use beacon_chain::store::Error; use beacon_chain::{ AvailabilityProcessingStatus, BeaconChainError, BeaconChainTypes, BlockError, - BlockProcessStatus, ForkChoiceError, GossipVerifiedBlock, IntoExecutionPendingBlock, - NotifyExecutionLayer, + BlockProcessStatus, ForkChoiceError, GossipVerifiedBlock, NotifyExecutionLayer, attestation_verification::{self, Error as AttnError, VerifiedAttestation}, data_availability_checker::AvailabilityCheckErrorCategory, light_client_finality_update_verification::Error as LightClientFinalityUpdateError, @@ -48,7 +47,7 @@ use types::{ Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, beacon_block::BlockImportSource, }; -use beacon_processor::work_reprocessing_queue::QueuedColumnReconstruction; +use beacon_processor::work_reprocessing_queue::{QueuedColumnReconstruction, QueuedPartialColumn}; use beacon_processor::{ DuplicateCache, GossipAggregatePackage, GossipAttestationBatch, work_reprocessing_queue::{ @@ -785,9 +784,9 @@ impl NetworkBeaconProcessor { pub async fn process_gossip_dangling_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - _subnet_id: DataColumnSubnetId, column_sidecar: Arc>, seen_duration: Duration, + allow_reprocess: bool, ) { let partial_column = match self .chain @@ -817,17 +816,65 @@ impl NetworkBeaconProcessor { Some(Err(err)) => { warn!(?err, "Error creating verifiable partial data column"); } - None => { - debug!("Received partial while not having block"); - metrics::inc_counter( - &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_CACHED_TOTAL, - ); - self.partial_data_column_cache.lock().insert( - column_sidecar, - peer_id, - seen_duration, - ); - } + None => self.handle_dangling_partial_data_column_sidecar_missing_block( + peer_id, + column_sidecar, + seen_duration, + allow_reprocess, + ), + } + } + + fn handle_dangling_partial_data_column_sidecar_missing_block( + self: &Arc, + peer_id: PeerId, + column_sidecar: Arc>, + seen_duration: Duration, + allow_reprocess: bool, + ) { + // TODO(dknopik): is this method of checking adequate?... + if self + .chain + .store + .block_exists(&column_sidecar.block_root) + .unwrap_or(false) + { + debug!("Received partial for already imported block"); + return; + } + if !allow_reprocess { + debug!("Not reprocessing"); + return; + } + debug!("Received partial while not having block"); + metrics::inc_counter( + &metrics::BEACON_PROCESSOR_GOSSIP_PARTIAL_DATA_COLUMN_SIDECAR_CACHED_TOTAL, + ); + // TODO(dknopik): self.send_sync_message() ? + let cloned_self = self.clone(); + if self + .beacon_processor_send + .try_send(WorkEvent { + drop_during_sync: false, + work: Work::Reprocess(ReprocessQueueMessage::UnknownPartialColumn( + QueuedPartialColumn { + beacon_block_root: column_sidecar.block_root, + process_fn: Box::pin(async move { + cloned_self + .process_gossip_dangling_partial_data_column_sidecar( + peer_id, + column_sidecar, + seen_duration, + false, + ) + .await; + }), + }, + )), + }) + .is_err() + { + error!("Failed to send attestation for re-processing") } } @@ -1421,35 +1468,6 @@ impl NetworkBeaconProcessor { Span::current().record("block_root", block_root.to_string()); if let Some(handle) = duplicate_cache.check_and_insert(block_root) { - // First, get the partial columns that have been waiting. Do this first in order to gossip them quickly (i guess?) TODO(dknopik): check if good - let partials = { - let mut partial_cache = self.partial_data_column_cache.lock(); - let partials = partial_cache.get_for_block(block_root); - // Opportunistically clean the cache as we are holding the lock anyway. TODO(dknopik): do somewhere else - partial_cache.clean(); - partials - }; - - for cached_partial in partials { - // TODO(dknopik): Do this in parallel (queueing it maybe) - match VerifiablePartialDataColumn::from_dangling_and_block( - cached_partial.sidecar, - gossip_verified_block.block(), - ) { - Ok(partial_data_column) => { - self.process_gossip_verifiable_partial_data_column_sidecar( - cached_partial.peer_id, - Arc::new(partial_data_column), - cached_partial.seen_duration, - ) - .await; - } - Err(err) => { - warn!(?err, "Unable to process cached partial column") - } - } - } - self.process_gossip_verified_block( peer_id, gossip_verified_block, @@ -1835,6 +1853,22 @@ impl NetworkBeaconProcessor { %block_root, "Processed block, waiting for other components" ); + if self + .beacon_processor_send + .try_send(WorkEvent { + drop_during_sync: false, + work: Work::Reprocess(ReprocessQueueMessage::BlockPending { + block_root: *block_root, + }), + }) + .is_err() + { + error!( + source = "gossip", + ?block_root, + "Failed to inform block pending" + ) + }; } Err(BlockError::ParentUnknown { .. }) => { // This should not occur. It should be checked by `should_forward_block`. diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 7bb4884c851..ed607406020 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -1,4 +1,3 @@ -use crate::partial_data_column_cache::PartialDataColumnCache; use crate::sync::manager::BlockProcessType; use crate::{service::NetworkMessage, sync::manager::SyncMessage}; use beacon_chain::blob_verification::{GossipBlobError, observe_gossip_blob}; @@ -22,7 +21,6 @@ use lighthouse_network::{ Client, MessageId, NetworkGlobals, PeerId, PubsubMessage, rpc::{BlocksByRangeRequest, BlocksByRootRequest, LightClientBootstrapRequest, StatusMessage}, }; -use parking_lot::Mutex; use rand::prelude::SliceRandom; use std::path::PathBuf; use std::sync::Arc; @@ -58,7 +56,6 @@ pub enum InvalidBlockStorage { pub struct NetworkBeaconProcessor { pub beacon_processor_send: BeaconProcessorSend, pub duplicate_cache: DuplicateCache, - pub partial_data_column_cache: Mutex>, pub chain: Arc>, pub network_tx: mpsc::UnboundedSender>, pub sync_tx: mpsc::UnboundedSender>, @@ -257,7 +254,6 @@ impl NetworkBeaconProcessor { pub fn send_gossip_partial_data_column_sidecar( self: &Arc, peer_id: PeerId, - subnet_id: DataColumnSubnetId, column_sidecar: Arc>, seen_timestamp: Duration, ) -> Result<(), Error> { @@ -266,9 +262,9 @@ impl NetworkBeaconProcessor { processor .process_gossip_dangling_partial_data_column_sidecar( peer_id, - subnet_id, column_sidecar, seen_timestamp, + true, ) .await }; @@ -1123,7 +1119,6 @@ impl NetworkBeaconProcessor> { let network_beacon_processor = Self { beacon_processor_send: beacon_processor_tx, duplicate_cache: DuplicateCache::default(), - partial_data_column_cache: Mutex::new(PartialDataColumnCache::new()), chain, network_tx, sync_tx, diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index a7cff747230..06f6f13f0b7 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -221,6 +221,21 @@ impl NetworkBeaconProcessor { self.chain.recompute_head_at_current_slot().await; } Ok(AvailabilityProcessingStatus::MissingComponents(..)) => { + if self + .beacon_processor_send + .try_send(WorkEvent { + drop_during_sync: false, + work: Work::Reprocess(ReprocessQueueMessage::BlockPending { block_root }), + }) + .is_err() + { + error!( + source = "rpc", + ?block_root, + "Failed to inform block pending" + ) + }; + // Block is valid, we can now attempt fetching blobs from EL using version hashes // derived from kzg commitments from the block, without having to wait for all blobs // to be sent from the peers if we already have them. diff --git a/beacon_node/network/src/partial_data_column_cache.rs b/beacon_node/network/src/partial_data_column_cache.rs deleted file mode 100644 index 07f7cccabcc..00000000000 --- a/beacon_node/network/src/partial_data_column_cache.rs +++ /dev/null @@ -1,76 +0,0 @@ -use lighthouse_network::PeerId; -use std::collections::HashMap; -use std::collections::hash_map::Entry; -use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use types::partial_data_column_sidecar::DanglingPartialDataColumn; -use types::{EthSpec, Hash256}; - -const BLOCK_LIMIT: usize = 16; -const SIDECAR_PER_BLOCK_LIMIT: usize = 16; -const EXPIRATION_TIME: Duration = Duration::from_secs(24); - -/// Really dumb hacky implementation of a cache for partial data column sidecars. -/// -/// The issue is that (with the current spec draft) we have to match up the dangling partial data -/// columns sidecars with the corresponding block. Of course, usually we have to take great care to -/// not be exploitable, but this cache design (for now) assumes no malicious behaviour. -/// -/// Do not take anything in this file as an actual implementation suggestion, it is just a hack -/// while we discuss the spec! -pub struct PartialDataColumnCache { - per_block: HashMap>>, -} - -pub struct CachedPartial { - pub sidecar: Arc>, - pub peer_id: PeerId, - pub seen_duration: Duration, -} - -impl PartialDataColumnCache { - pub fn new() -> Self { - Self { - per_block: HashMap::new(), - } - } - - pub fn insert( - &mut self, - sidecar: Arc>, - peer_id: PeerId, - seen_duration: Duration, - ) { - let len = self.per_block.len(); - let entry = self.per_block.entry(sidecar.block_root); - if matches!(entry, Entry::Vacant(_)) && len >= BLOCK_LIMIT { - return; - } - - let sidecars = entry.or_default(); - if sidecars.len() < SIDECAR_PER_BLOCK_LIMIT { - sidecars.push(CachedPartial { - sidecar, - peer_id, - seen_duration, - }); - } - } - - pub fn get_for_block(&mut self, block_root: Hash256) -> Vec> { - self.per_block.remove(&block_root).unwrap_or_default() - } - - pub fn clean(&mut self) { - if let Ok(now) = SystemTime::now().duration_since(UNIX_EPOCH) { - self.per_block.retain(|_, sidecars| { - sidecars - .iter() - .map(|cached| cached.seen_duration) - .max() - .map(|last_seen| now.saturating_sub(last_seen) < EXPIRATION_TIME) - .unwrap_or(false) - }); - } - } -} diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 30ccb4319ae..a39bfb8c4e3 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -6,7 +6,6 @@ #![allow(clippy::unit_arg)] use crate::network_beacon_processor::{InvalidBlockStorage, NetworkBeaconProcessor}; -use crate::partial_data_column_cache::PartialDataColumnCache; use crate::service::NetworkMessage; use crate::status::status_message; use crate::sync::SyncMessage; @@ -20,7 +19,6 @@ use lighthouse_network::{ }; use logging::TimeLatch; use logging::crit; -use parking_lot::Mutex; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::mpsc; @@ -99,7 +97,6 @@ impl Router { let network_beacon_processor = NetworkBeaconProcessor { beacon_processor_send, duplicate_cache: DuplicateCache::default(), - partial_data_column_cache: Mutex::new(PartialDataColumnCache::new()), chain: beacon_chain.clone(), network_tx: network_send.clone(), sync_tx: sync_send.clone(), @@ -388,12 +385,11 @@ impl Router { ) } PubsubMessage::PartialDataColumnSidecar(data) => { - let (subnet_id, column_sidecar) = *data; + let (_, column_sidecar) = *data; self.handle_beacon_processor_send_result( self.network_beacon_processor .send_gossip_partial_data_column_sidecar( peer_id, - subnet_id, column_sidecar.partial_column, timestamp_now(), ), From 695c648a4bc491a0ae21177e2a826755b5b2b3b9 Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 5 Nov 2025 17:12:02 +0100 Subject: [PATCH 27/30] publish partials on http API block --- beacon_node/http_api/src/publish_blocks.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index c5f9b133696..f8d680cc86b 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -564,9 +564,15 @@ fn publish_column_sidecars( } let pubsub_messages = data_column_sidecars .into_iter() - .map(|data_col| { + .flat_map(|data_col| { let subnet = DataColumnSubnetId::from_column_index(data_col.index, &chain.spec); - PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))) + [ + PubsubMessage::PartialDataColumnSidecar(Box::new(( + subnet, + (*data_col).clone().into_partial().column.into(), + ))), + PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))), + ] }) .collect::>(); crate::publish_pubsub_messages(sender_clone, pubsub_messages) From 6fb3721e8ea2bbe797110c2e646081a5831b20fe Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 5 Nov 2025 20:34:54 +0100 Subject: [PATCH 28/30] experimentally full eager publish for API blocks --- beacon_node/http_api/src/publish_blocks.rs | 15 +++++--- .../lighthouse_network/src/types/partial.rs | 32 ++++++++++------- .../lighthouse_network/src/types/pubsub.rs | 36 ++++++++++++------- .../gossip_methods.rs | 6 ++-- .../src/network_beacon_processor/mod.rs | 3 +- beacon_node/network/src/router.rs | 4 +-- 6 files changed, 60 insertions(+), 36 deletions(-) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index f8d680cc86b..8f6ea9d632e 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -209,7 +209,7 @@ pub async fn publish_block>( debug!( partial = merged_data.updated_partials.len(), full = merged_data.completed_columns.len(), - "Sending merged data after block availability" + "Sending merged data after block publish (should not happen?)" ); let messages: Vec<_> = merged_data .updated_partials @@ -217,7 +217,8 @@ pub async fn publish_block>( .map(|partial| { PubsubMessage::PartialDataColumnSidecar(Box::new(( DataColumnSubnetId::from_column_index(partial.index(), &spec), - partial.column.clone().into(), + partial.column.clone(), + None, ))) }) .chain(merged_data.completed_columns.into_iter().flat_map(|full| { @@ -225,7 +226,8 @@ pub async fn publish_block>( [ PubsubMessage::PartialDataColumnSidecar(Box::new(( subnet, - (*full).clone().into_partial().column.clone().into(), + (*full).clone().into_partial().column.clone(), + None, ))), PubsubMessage::DataColumnSidecar(Box::new((subnet, full))), ] @@ -234,7 +236,7 @@ pub async fn publish_block>( if !messages.is_empty() && let Err(err) = crate::publish_pubsub_messages(&sender_clone, messages) { - warn!(?err, "Publishing data after block availability failed") + warn!(?err, "Publishing data after block publish") } }; @@ -566,10 +568,13 @@ fn publish_column_sidecars( .into_iter() .flat_map(|data_col| { let subnet = DataColumnSubnetId::from_column_index(data_col.index, &chain.spec); + let column = (*data_col).clone().into_partial().column; + let all_cells = column.sidecar.cells_present_bitmap.clone(); [ PubsubMessage::PartialDataColumnSidecar(Box::new(( subnet, - (*data_col).clone().into_partial().column.into(), + column, + Some(all_cells), ))), PubsubMessage::DataColumnSidecar(Box::new((subnet, data_col))), ] diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index 3a165127e1e..da712e4daf8 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -9,29 +9,35 @@ use types::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataColumn}; #[derive(Debug, Clone, PartialEq)] pub struct PartialDataColumnSidecarMessage { pub partial_column: Arc>, - pub send_eager: Option>, + send_eager: Option>, } -impl From> for PartialDataColumnSidecarMessage { - fn from(value: DanglingPartialDataColumn) -> Self { - Self { - partial_column: Arc::new(value), +impl PartialDataColumnSidecarMessage { + pub fn new(partial_column: Arc>) -> Self { + PartialDataColumnSidecarMessage { + partial_column, send_eager: None, } } -} -impl From>> for PartialDataColumnSidecarMessage { - fn from(value: Arc>) -> Self { - Self { - partial_column: value, - send_eager: None, - } + pub fn eagerly_send(&mut self, cells: &CellBitmap) { + let Some(eager) = self + .partial_column + .sidecar + .clone_filter(|idx| cells.get(idx).unwrap_or(false)) + else { + return; + }; + + self.send_eager = Some(SendEager { + data: eager.as_ssz_bytes(), + metadata: eager.cells_present_bitmap.into(), + }) } } #[derive(Debug, Clone, PartialEq)] -pub struct SendEager { +struct SendEager { /// The encoded message to send eagerly, i.e. when we have no metadata for that peer. data: Vec, /// The metadata to associate with a peer after sending it the eager message. diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index d9f9997d2d0..db475213ab6 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -8,7 +8,9 @@ use snap::raw::{Decoder, Encoder, decompress_len}; use ssz::{Decode, Encode}; use std::io::{Error, ErrorKind}; use std::sync::Arc; -use types::partial_data_column_sidecar::{DanglingPartialDataColumn, PartialDataColumnSidecar}; +use types::partial_data_column_sidecar::{ + CellBitmap, DanglingPartialDataColumn, PartialDataColumnSidecar, +}; use types::{ AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BlobSidecar, DataColumnSidecar, DataColumnSubnetId, EthSpec, ForkContext, ForkName, Hash256, @@ -21,6 +23,12 @@ use types::{ SyncCommitteeMessage, SyncSubnetId, }; +type PartialDataColumnSidecarTuple = ( + DataColumnSubnetId, + Arc>, + Option>, +); + #[derive(Debug, Clone, PartialEq)] pub enum PubsubMessage { /// Gossipsub message providing notification of a new block. @@ -30,7 +38,8 @@ pub enum PubsubMessage { /// Gossipsub message providing notification of a [`DataColumnSidecar`] along with the subnet id where it was received. DataColumnSidecar(Box<(DataColumnSubnetId, Arc>)>), /// Gossipsub message providing notification of a [`PartialDataColumnSidecar`] along with the subnet id where it was received. - PartialDataColumnSidecar(Box<(DataColumnSubnetId, PartialDataColumnSidecarMessage)>), // TODO(dknopik): review `Arc` situation + /// TODO(dknopik) - it is time for this to move into its own enum. + PartialDataColumnSidecar(Box>), // TODO(dknopik): review `Arc` situation /// Gossipsub message providing notification of a Aggregate attestation and associated proof. AggregateAndProofAttestation(Box>), /// Gossipsub message providing notification of a `SingleAttestation` with its subnet id. @@ -417,7 +426,8 @@ impl PubsubMessage { }; Ok(Self::PartialDataColumnSidecar(Box::new(( *id, - data_column.into(), + Arc::new(data_column), + None, )))) } other => Err(format!("Partial message unsupported for topic: {other}")), @@ -436,7 +446,13 @@ impl PubsubMessage { PubsubMessage::BlobSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::DataColumnSidecar(data) => data.1.as_ssz_bytes(), PubsubMessage::PartialDataColumnSidecar(data) => { - return EncodedPubsubMessage::PartialDataColumnSidecarMessage(data.1.clone()); + let sidecar = &data.1; + let eager_cells = &data.2; + let mut message = PartialDataColumnSidecarMessage::new(sidecar.clone()); + if let Some(eager_cells) = eager_cells { + message.eagerly_send(eager_cells); + } + return EncodedPubsubMessage::PartialDataColumnSidecarMessage(message); } PubsubMessage::AggregateAndProofAttestation(data) => data.as_ssz_bytes(), PubsubMessage::VoluntaryExit(data) => data.as_ssz_bytes(), @@ -481,15 +497,9 @@ impl std::fmt::Display for PubsubMessage { PubsubMessage::PartialDataColumnSidecar(data) => write!( f, "PartialDataColumnSidecar: group: {}, column index: {}, cells: {}", - data.1.partial_column.block_root, - data.1.partial_column.index, - hex::encode( - data.1 - .partial_column - .sidecar - .cells_present_bitmap - .as_slice() - ), + data.1.block_root, + data.1.index, + hex::encode(data.1.sidecar.cells_present_bitmap.as_slice()), ), PubsubMessage::AggregateAndProofAttestation(att) => write!( f, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 1ddfe8c6580..9e0fd816373 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1307,7 +1307,8 @@ impl NetworkBeaconProcessor { partial.index(), &cloned_self.chain.spec, ), - partial.column.clone().into(), + partial.column.clone(), + None, ))) }) .chain(merged_data.completed_columns.into_iter().flat_map(|full| { @@ -1316,7 +1317,8 @@ impl NetworkBeaconProcessor { [ PubsubMessage::PartialDataColumnSidecar(Box::new(( subnet, - (*full).clone().into_partial().column.clone().into(), + (*full).clone().into_partial().column.clone(), + None, ))), PubsubMessage::DataColumnSidecar(Box::new((subnet, full))), ] diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 535f587514b..5ac99d3e612 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -823,7 +823,8 @@ impl NetworkBeaconProcessor { c.index(), &self_cloned.chain.spec, ), - c.into_partial().into_inner().column.clone().into(), + c.into_partial().into_inner().column.clone(), + None, ))) }) .collect(), diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index a39bfb8c4e3..2659a2e8283 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -385,12 +385,12 @@ impl Router { ) } PubsubMessage::PartialDataColumnSidecar(data) => { - let (_, column_sidecar) = *data; + let (_, column_sidecar, _) = *data; self.handle_beacon_processor_send_result( self.network_beacon_processor .send_gossip_partial_data_column_sidecar( peer_id, - column_sidecar.partial_column, + column_sidecar, timestamp_now(), ), ) From 870fa0d2522fccdab45475128b7b6b8895a4e34b Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Mon, 10 Nov 2025 17:04:41 +0100 Subject: [PATCH 29/30] experimental libp2p changes --- Cargo.lock | 46 ++++---- .../src/service/gossip_cache.rs | 82 +++++--------- .../lighthouse_network/src/service/mod.rs | 78 ++++--------- .../lighthouse_network/src/types/partial.rs | 105 +++++++++--------- .../lighthouse_network/src/types/pubsub.rs | 4 +- 5 files changed, 127 insertions(+), 188 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4101a2c224a..77c26f9054c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2168,7 +2168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn 2.0.100", + "syn 1.0.109", ] [[package]] @@ -5143,7 +5143,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "bytes", "either", @@ -5175,7 +5175,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5185,7 +5185,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5195,7 +5195,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "either", "fnv", @@ -5219,7 +5219,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "async-trait", "futures", @@ -5234,7 +5234,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "async-channel 2.3.1", "asynchronous-codec", @@ -5264,7 +5264,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "asynchronous-codec", "either", @@ -5304,7 +5304,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "hickory-proto", @@ -5322,7 +5322,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "libp2p-core", @@ -5337,7 +5337,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "asynchronous-codec", "bytes", @@ -5355,7 +5355,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "asynchronous-codec", "bytes", @@ -5377,7 +5377,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "asynchronous-codec", "bytes", @@ -5392,7 +5392,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "futures-timer", @@ -5413,7 +5413,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "either", "fnv", @@ -5434,7 +5434,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "heck 0.5.0", "quote", @@ -5444,7 +5444,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "futures-timer", @@ -5459,7 +5459,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "futures-rustls", @@ -5477,7 +5477,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "futures-timer", @@ -5491,7 +5491,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "either", "futures", @@ -6163,7 +6163,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "bytes", "futures", @@ -7382,7 +7382,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "asynchronous-codec", "bytes", @@ -8195,7 +8195,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#d97ae1f50865647d6bb2fda1c59e9f7ec0b8a4f6" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" dependencies = [ "futures", "pin-project", diff --git a/beacon_node/lighthouse_network/src/service/gossip_cache.rs b/beacon_node/lighthouse_network/src/service/gossip_cache.rs index 8259fa0701d..120b9e6c245 100644 --- a/beacon_node/lighthouse_network/src/service/gossip_cache.rs +++ b/beacon_node/lighthouse_network/src/service/gossip_cache.rs @@ -1,27 +1,23 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; -use std::marker::PhantomData; use std::pin::Pin; use std::task::{Context, Poll}; use std::time::Duration; use crate::GossipTopic; -use crate::types::{EncodedPubsubMessage, GossipKind}; +use crate::types::GossipKind; use tokio_util::time::delay_queue::{DelayQueue, Key}; -use types::EthSpec; - -type TopicMsg = (Key, EncodedPubsubMessage); /// Store of gossip messages that we failed to publish and will try again later. By default, all /// messages are ignored. This behaviour can be changed using `GossipCacheBuilder::default_timeout` /// to apply the same delay to every kind. Individual timeouts for specific kinds can be set and /// will overwrite the default_timeout if present. -pub struct GossipCache { - /// Expire timeouts for each topic-msgid triple. +pub struct GossipCache { + /// Expire timeouts for each topic-msg pair. expirations: DelayQueue<(GossipTopic, Vec)>, /// Messages cached for each topic. - topic_msgs: HashMap, TopicMsg>>, + topic_msgs: HashMap, Key>>, /// Timeout for blocks. beacon_block: Option, /// Timeout for blobs. @@ -51,7 +47,7 @@ pub struct GossipCache { } #[derive(Default)] -pub struct GossipCacheBuilder { +pub struct GossipCacheBuilder { default_timeout: Option, /// Timeout for blocks. beacon_block: Option, @@ -79,11 +75,10 @@ pub struct GossipCacheBuilder { light_client_finality_update: Option, /// Timeout for light client optimistic updates. light_client_optimistic_update: Option, - _phantom: PhantomData, } #[allow(dead_code)] -impl GossipCacheBuilder { +impl GossipCacheBuilder { /// By default, all timeouts all disabled. Setting a default timeout will enable all timeout /// that are not already set. pub fn default_timeout(mut self, timeout: Duration) -> Self { @@ -96,12 +91,6 @@ impl GossipCacheBuilder { self } - /// Timeout for data column sidecars. - pub fn data_column_sidecar(mut self, timeout: Duration) -> Self { - self.data_column_sidecar = Some(timeout); - self - } - /// Timeout for aggregate attestations. pub fn aggregates_timeout(mut self, timeout: Duration) -> Self { self.aggregates = Some(timeout); @@ -162,7 +151,7 @@ impl GossipCacheBuilder { self } - pub fn build(self) -> GossipCache { + pub fn build(self) -> GossipCache { let GossipCacheBuilder { default_timeout, beacon_block, @@ -178,7 +167,6 @@ impl GossipCacheBuilder { bls_to_execution_change, light_client_finality_update, light_client_optimistic_update, - .. } = self; GossipCache { expirations: DelayQueue::default(), @@ -200,15 +188,15 @@ impl GossipCacheBuilder { } } -impl GossipCache { +impl GossipCache { /// Get a builder of a `GossipCache`. Topic kinds for which no timeout is defined will be /// ignored if added in `insert`. - pub fn builder() -> GossipCacheBuilder { + pub fn builder() -> GossipCacheBuilder { GossipCacheBuilder::default() } // Insert a message to be sent later. - pub fn insert(&mut self, topic: GossipTopic, id: Vec, data: EncodedPubsubMessage) { + pub fn insert(&mut self, topic: GossipTopic, data: Vec) { let expire_timeout = match topic.kind() { GossipKind::BeaconBlock => self.beacon_block, GossipKind::BlobSidecar(_) => self.blob_sidecar, @@ -231,54 +219,45 @@ impl GossipCache { .topic_msgs .entry(topic.clone()) .or_default() - .entry(id.clone()) + .entry(data.clone()) { - Entry::Occupied(key) => self.expirations.reset(&key.get().0, expire_timeout), + Entry::Occupied(key) => self.expirations.reset(key.get(), expire_timeout), Entry::Vacant(entry) => { - let key = self.expirations.insert((topic, id), expire_timeout); - entry.insert((key, data)); + let key = self.expirations.insert((topic, data), expire_timeout); + entry.insert(key); } } } - // Get the registered messages for this topic, removing them - pub fn retrieve_all( - &mut self, - topic: &GossipTopic, - ) -> Option> + '_> { - self.topic_msgs.remove(topic).map(|msgs| { - msgs.into_values().map(|(key, msg)| { - self.expirations.remove(&key); - msg - }) - }) - } - - // Get a registered message, NOT removing it. - pub fn retrieve_one(&self, topic: &GossipTopic, id: &[u8]) -> Option<&EncodedPubsubMessage> { - self.topic_msgs - .get(topic) - .and_then(|msgs| msgs.get(id)) - .map(|(_, msg)| msg) + // Get the registered messages for this topic. + pub fn retrieve(&mut self, topic: &GossipTopic) -> Option> + '_> { + if let Some(msgs) = self.topic_msgs.remove(topic) { + for (_, key) in msgs.iter() { + self.expirations.remove(key); + } + Some(msgs.into_keys()) + } else { + None + } } } -impl futures::stream::Stream for GossipCache { +impl futures::stream::Stream for GossipCache { type Item = Result; // We don't care to retrieve the expired data. fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { match self.expirations.poll_expired(cx) { Poll::Ready(Some(expired)) => { let expected_key = expired.key(); - let (topic, id) = expired.into_inner(); + let (topic, data) = expired.into_inner(); let topic_msg = self.topic_msgs.get_mut(&topic); debug_assert!( topic_msg.is_some(), "Topic for registered message is not present." ); if let Some(msgs) = topic_msg { - let key_and_data = msgs.remove(&id); - debug_assert_eq!(key_and_data.map(|(key, _)| key), Some(expected_key)); + let key = msgs.remove(&data); + debug_assert_eq!(key, Some(expected_key)); if msgs.is_empty() { // no more messages for this topic. self.topic_msgs.remove(&topic); @@ -296,11 +275,10 @@ impl futures::stream::Stream for GossipCache { mod tests { use super::*; use futures::stream::StreamExt; - use types::MainnetEthSpec; #[tokio::test] async fn test_stream() { - let mut cache = GossipCache::::builder() + let mut cache = GossipCache::builder() .default_timeout(Duration::from_millis(300)) .build(); let test_topic = GossipTopic::new( @@ -308,7 +286,7 @@ mod tests { crate::types::GossipEncoding::SSZSnappy, [0u8; 4], ); - cache.insert(test_topic, vec![], EncodedPubsubMessage::Full(vec![])); + cache.insert(test_topic, vec![]); tokio::time::sleep(Duration::from_millis(300)).await; while cache.next().await.is_some() {} assert!(cache.expirations.is_empty()); diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 7672782fb81..0eee3cfca81 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -23,7 +23,7 @@ use crate::{Enr, NetworkGlobals, PubsubMessage, TopicHash, metrics}; use api_types::{AppRequestId, Response}; use futures::stream::StreamExt; use gossipsub::{ - Event, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, Partial, + Event, IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, PublishError, TopicScoreParams, }; use gossipsub_scoring_parameters::{PeerScoreSettings, lighthouse_gossip_thresholds}; @@ -34,7 +34,6 @@ use libp2p::upnp::tokio::Behaviour as Upnp; use libp2p::{PeerId, SwarmBuilder, identify}; use logging::crit; use network_utils::enr_ext::EnrExt; -use sha2::{Digest, Sha256}; use ssz::Decode; use std::num::{NonZeroU8, NonZeroUsize}; use std::path::PathBuf; @@ -164,7 +163,7 @@ pub struct Network { score_settings: PeerScoreSettings, /// The interval for updating gossipsub scores update_gossipsub_scores: tokio::time::Interval, - gossip_cache: GossipCache, + gossip_cache: GossipCache, /// This node's PeerId. pub local_peer_id: PeerId, } @@ -259,7 +258,6 @@ impl Network { // .signed_contribution_and_proof_timeout(timeout) // Do not retry // .sync_committee_message_timeout(timeout) // Do not retry .bls_to_execution_change_timeout(half_epoch * 2) - .data_column_sidecar(slot_duration) .build() }; @@ -854,7 +852,17 @@ impl Network { for message in messages { for topic in message.topics(GossipEncoding::default(), self.enr_fork_id.fork_digest) { let message_data = message.encode(); - let result = message_data.do_publish(self.gossipsub_mut(), topic.clone().into()); + let gossipsub = self.gossipsub_mut(); + let publish_topic: Topic = topic.clone().into(); + let (cache, result) = match message_data { + EncodedPubsubMessage::Full(bytes) => ( + Some(bytes.clone()), + gossipsub.publish(publish_topic, bytes).map(|_| ()), + ), + EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) => { + (None, gossipsub.publish_partial(publish_topic, partial)) + } + }; if let Err(e) = result { match e { PublishError::Duplicate => { @@ -893,21 +901,10 @@ impl Network { } if let PublishError::NoPeersSubscribedToTopic = e - && let EncodedPubsubMessage::Full(bytes) = &message_data + && let Some(bytes) = cache { - let id = Sha256::digest(bytes)[..].to_vec(); - self.gossip_cache.insert(topic, id, message_data); + self.gossip_cache.insert(topic, bytes); } - } else if let EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) = - &message_data - { - let id = partial.group_id().as_ref().to_vec(); - debug!( - kind = %topic.kind(), - group_id = hex::encode(&id), - "Adding partial message to cache" - ); - self.gossip_cache.insert(topic, id, message_data); } } } @@ -1324,7 +1321,6 @@ impl Network { message, metadata, } => { - let hex_group_id = hex::encode(&group_id); let topic = GossipTopic::decode(topic_id.as_str()) .inspect_err(|error| { debug!( @@ -1354,38 +1350,6 @@ impl Network { debug!(%propagation_source, "Got no metadata") } - // TODO(dknopik): maybe do this after validation? - if let Some(cached_partial) = self.gossip_cache.retrieve_one(&topic, &group_id) { - // We do not bother checking here if that cached msg contains relevant data, - // as gossipsub will do that anyway. - let result = cached_partial.do_publish( - &mut self.swarm.behaviour_mut().gossipsub, - topic.clone().into(), - ); - match result { - Ok(_) => { - debug!( - group_id = hex_group_id, - %topic, %propagation_source, "Replied with cached partial message" - ) - } - Err(err) => { - warn!( - group_id = hex_group_id, - %topic, - %propagation_source, - ?err, - "Failed to reply with cached partial message" - ); - } - } - } else { - debug!( - group_id = hex_group_id, - %topic, %propagation_source, "Did not reply to partial message" - ); - } - if let Some(message) = message { match PubsubMessage::decode_partial(&topic_id, &group_id, &message) { Err(error) => { @@ -1428,13 +1392,15 @@ impl Network { .add_subscription(&peer_id, subnet_id); } // Try to send the cached messages for this topic - if let Some(msgs) = self.gossip_cache.retrieve_all(&topic) { + if let Some(msgs) = self.gossip_cache.retrieve(&topic) { for data in msgs { let topic_str: &str = topic.kind().as_ref(); - match data.do_publish( - &mut self.swarm.behaviour_mut().gossipsub, - topic.clone().into(), - ) { + match self + .swarm + .behaviour_mut() + .gossipsub + .publish(Topic::from(topic.clone()), data) + { Ok(_) => { debug!(topic = topic_str, "Gossip message published on retry"); metrics::inc_counter_vec( diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index da712e4daf8..f61604d7694 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -1,6 +1,7 @@ -use gossipsub::partial::{Metadata, PublishAction}; +use gossipsub::partial::Metadata; use gossipsub::{Partial, PartialMessageError}; use ssz::{Decode, Encode}; +use std::cmp::Ordering; use std::fmt::Debug; use std::sync::Arc; use types::EthSpec; @@ -9,12 +10,14 @@ use types::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataColumn}; #[derive(Debug, Clone, PartialEq)] pub struct PartialDataColumnSidecarMessage { pub partial_column: Arc>, + metadata: CellBitmapMetadata, send_eager: Option>, } impl PartialDataColumnSidecarMessage { pub fn new(partial_column: Arc>) -> Self { PartialDataColumnSidecarMessage { + metadata: partial_column.sidecar.cells_present_bitmap.clone().into(), partial_column, send_eager: None, } @@ -45,85 +48,77 @@ struct SendEager { } #[derive(Debug, Clone, PartialEq, Eq)] -struct CellBitmapMetadata { +pub struct CellBitmapMetadata { bitmap: CellBitmap, - encoded: Vec, } impl Metadata for CellBitmapMetadata { - fn as_slice(&self) -> &[u8] { - &self.encoded + fn decode(bytes: &[u8]) -> Result + where + Self: Sized, + { + Ok(CellBitmapMetadata { + bitmap: CellBitmap::::from_ssz_bytes(bytes) + .map_err(|_| PartialMessageError::InvalidFormat)?, + }) } - fn update(&mut self, data: &[u8]) -> Result { - let data = CellBitmap::::from_ssz_bytes(data) - .map_err(|_| PartialMessageError::InvalidFormat)?; - if data.len() != self.bitmap.len() { - return Err(PartialMessageError::OutOfRange); + fn compare(&self, other: &Self) -> Option { + let self_subset = self.bitmap.is_subset(&other.bitmap); + let other_subset = other.bitmap.is_subset(&self.bitmap); + match (self_subset, other_subset) { + (true, true) => Some(Ordering::Equal), + (true, false) => Some(Ordering::Less), + (false, true) => Some(Ordering::Greater), + (false, false) => None, } - let new_bitmap = self.bitmap.union(&data); + } + + fn encode(&self) -> Vec { + self.bitmap.as_ssz_bytes() + } + + fn update(&mut self, data: &Self) -> Result { + let new_bitmap = self.bitmap.union(&data.bitmap); if self.bitmap == new_bitmap { return Ok(false); } self.bitmap = new_bitmap; - self.encoded = self.bitmap.as_ssz_bytes(); Ok(true) } } impl From> for CellBitmapMetadata { - fn from(value: CellBitmap) -> Self { - Self { - encoded: value.as_ssz_bytes(), - bitmap: value, - } + fn from(bitmap: CellBitmap) -> Self { + Self { bitmap } } } impl Partial for PartialDataColumnSidecarMessage { - fn group_id(&self) -> impl AsRef<[u8]> { - &self.partial_column.block_root + type Metadata = CellBitmapMetadata; + + fn group_id(&self) -> Vec { + self.partial_column.block_root.to_vec() } - fn parts_metadata(&self) -> impl AsRef<[u8]> { - self.partial_column - .sidecar - .cells_present_bitmap - .as_ssz_bytes() + fn metadata(&self) -> &Self::Metadata { + &self.metadata } fn partial_message_bytes_from_metadata( &self, - metadata: Option>, - ) -> Result { - match metadata { - None => { - // Send the eager message if any - match &self.send_eager { - None => Ok(PublishAction::NothingToSend), - Some(send) => Ok(PublishAction::Send { - message: send.data.clone(), - metadata: Box::new(send.metadata.clone()), - }), - } - } - Some(metadata) => { - let peer_has = CellBitmap::::from_ssz_bytes(metadata.as_ref()) - .map_err(|_| PartialMessageError::InvalidFormat)?; - if peer_has == self.partial_column.sidecar.cells_present_bitmap { - return Ok(PublishAction::SameMetadata); - } - - let Some(send) = self.partial_column.sidecar.with_missing_cells(&peer_has) else { - return Ok(PublishAction::NothingToSend); - }; - - let new_metadata = peer_has.union(&send.cells_present_bitmap); - Ok(PublishAction::Send { - message: send.as_ssz_bytes().to_vec(), - metadata: Box::new(CellBitmapMetadata::::from(new_metadata)), - }) - } - } + metadata: &Self::Metadata, + ) -> Result>, PartialMessageError> { + Ok(self + .partial_column + .sidecar + .with_missing_cells(&metadata.bitmap) + .map(|message| message.as_ssz_bytes())) + } + + fn data_for_eager_push( + &self, + ) -> Result, Self::Metadata)>, PartialMessageError> { + Ok(self.send_eager.clone().map(|eager| (eager.data, eager.metadata))) } } diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index db475213ab6..54eb76d4ae5 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -546,13 +546,13 @@ pub enum EncodedPubsubMessage { impl EncodedPubsubMessage { pub fn do_publish( - &self, + self, gossipsub: &mut Gossipsub, topic: IdentTopic, ) -> Result<(), PublishError> { match self { EncodedPubsubMessage::Full(bytes) => { - gossipsub.publish(topic, bytes.clone()).map(|_| ()) + gossipsub.publish(topic, bytes).map(|_| ()) } EncodedPubsubMessage::PartialDataColumnSidecarMessage(partial) => { gossipsub.publish_partial(topic, partial) From 995344eacdfbd48e9f465973fb3238ce78ff02ee Mon Sep 17 00:00:00 2001 From: Daniel Knopik Date: Wed, 12 Nov 2025 16:21:35 +0100 Subject: [PATCH 30/30] joaos libp2p changes --- Cargo.lock | 46 ++++---- .../lighthouse_network/src/types/partial.rs | 105 ++++++++++-------- 2 files changed, 79 insertions(+), 72 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77c26f9054c..c33d7bdec17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2168,7 +2168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.100", ] [[package]] @@ -5143,7 +5143,7 @@ dependencies = [ [[package]] name = "libp2p" version = "0.56.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "bytes", "either", @@ -5175,7 +5175,7 @@ dependencies = [ [[package]] name = "libp2p-allow-block-list" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5185,7 +5185,7 @@ dependencies = [ [[package]] name = "libp2p-connection-limits" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "libp2p-core", "libp2p-identity", @@ -5195,7 +5195,7 @@ dependencies = [ [[package]] name = "libp2p-core" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "either", "fnv", @@ -5219,7 +5219,7 @@ dependencies = [ [[package]] name = "libp2p-dns" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "async-trait", "futures", @@ -5234,7 +5234,7 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" version = "0.50.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "async-channel 2.3.1", "asynchronous-codec", @@ -5264,7 +5264,7 @@ dependencies = [ [[package]] name = "libp2p-identify" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "asynchronous-codec", "either", @@ -5304,7 +5304,7 @@ dependencies = [ [[package]] name = "libp2p-mdns" version = "0.48.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "hickory-proto", @@ -5322,7 +5322,7 @@ dependencies = [ [[package]] name = "libp2p-metrics" version = "0.17.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "libp2p-core", @@ -5337,7 +5337,7 @@ dependencies = [ [[package]] name = "libp2p-mplex" version = "0.43.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "asynchronous-codec", "bytes", @@ -5355,7 +5355,7 @@ dependencies = [ [[package]] name = "libp2p-noise" version = "0.46.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "asynchronous-codec", "bytes", @@ -5377,7 +5377,7 @@ dependencies = [ [[package]] name = "libp2p-plaintext" version = "0.43.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "asynchronous-codec", "bytes", @@ -5392,7 +5392,7 @@ dependencies = [ [[package]] name = "libp2p-quic" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "futures-timer", @@ -5413,7 +5413,7 @@ dependencies = [ [[package]] name = "libp2p-swarm" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "either", "fnv", @@ -5434,7 +5434,7 @@ dependencies = [ [[package]] name = "libp2p-swarm-derive" version = "0.35.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "heck 0.5.0", "quote", @@ -5444,7 +5444,7 @@ dependencies = [ [[package]] name = "libp2p-tcp" version = "0.44.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "futures-timer", @@ -5459,7 +5459,7 @@ dependencies = [ [[package]] name = "libp2p-tls" version = "0.6.2" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "futures-rustls", @@ -5477,7 +5477,7 @@ dependencies = [ [[package]] name = "libp2p-upnp" version = "0.6.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "futures-timer", @@ -5491,7 +5491,7 @@ dependencies = [ [[package]] name = "libp2p-yamux" version = "0.47.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "either", "futures", @@ -6163,7 +6163,7 @@ dependencies = [ [[package]] name = "multistream-select" version = "0.13.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "bytes", "futures", @@ -7382,7 +7382,7 @@ dependencies = [ [[package]] name = "quick-protobuf-codec" version = "0.3.1" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "asynchronous-codec", "bytes", @@ -8195,7 +8195,7 @@ dependencies = [ [[package]] name = "rw-stream-sink" version = "0.4.0" -source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#aa943174438429695c2766dddeadac1af0115b4a" +source = "git+https://github.com/jxs/rust-libp2p.git?branch=gossipsub-partial-messages#8c342f134fdf97dd50ceabc6242be8bcc39134e6" dependencies = [ "futures", "pin-project", diff --git a/beacon_node/lighthouse_network/src/types/partial.rs b/beacon_node/lighthouse_network/src/types/partial.rs index f61604d7694..556dc147e9e 100644 --- a/beacon_node/lighthouse_network/src/types/partial.rs +++ b/beacon_node/lighthouse_network/src/types/partial.rs @@ -1,7 +1,6 @@ -use gossipsub::partial::Metadata; +use gossipsub::partial::{Metadata, PublishAction}; use gossipsub::{Partial, PartialMessageError}; use ssz::{Decode, Encode}; -use std::cmp::Ordering; use std::fmt::Debug; use std::sync::Arc; use types::EthSpec; @@ -10,14 +9,12 @@ use types::partial_data_column_sidecar::{CellBitmap, DanglingPartialDataColumn}; #[derive(Debug, Clone, PartialEq)] pub struct PartialDataColumnSidecarMessage { pub partial_column: Arc>, - metadata: CellBitmapMetadata, send_eager: Option>, } impl PartialDataColumnSidecarMessage { pub fn new(partial_column: Arc>) -> Self { PartialDataColumnSidecarMessage { - metadata: partial_column.sidecar.cells_present_bitmap.clone().into(), partial_column, send_eager: None, } @@ -48,77 +45,87 @@ struct SendEager { } #[derive(Debug, Clone, PartialEq, Eq)] -pub struct CellBitmapMetadata { +struct CellBitmapMetadata { bitmap: CellBitmap, + encoded: Vec, } impl Metadata for CellBitmapMetadata { - fn decode(bytes: &[u8]) -> Result - where - Self: Sized, - { - Ok(CellBitmapMetadata { - bitmap: CellBitmap::::from_ssz_bytes(bytes) - .map_err(|_| PartialMessageError::InvalidFormat)?, - }) + fn as_slice(&self) -> &[u8] { + &self.encoded } - fn compare(&self, other: &Self) -> Option { - let self_subset = self.bitmap.is_subset(&other.bitmap); - let other_subset = other.bitmap.is_subset(&self.bitmap); - match (self_subset, other_subset) { - (true, true) => Some(Ordering::Equal), - (true, false) => Some(Ordering::Less), - (false, true) => Some(Ordering::Greater), - (false, false) => None, + fn update(&mut self, data: &[u8]) -> Result { + let data = CellBitmap::::from_ssz_bytes(data) + .map_err(|_| PartialMessageError::InvalidFormat)?; + if data.len() != self.bitmap.len() { + return Err(PartialMessageError::OutOfRange); } - } - - fn encode(&self) -> Vec { - self.bitmap.as_ssz_bytes() - } - - fn update(&mut self, data: &Self) -> Result { - let new_bitmap = self.bitmap.union(&data.bitmap); + let new_bitmap = self.bitmap.union(&data); if self.bitmap == new_bitmap { return Ok(false); } self.bitmap = new_bitmap; + self.encoded = self.bitmap.as_ssz_bytes(); Ok(true) } } impl From> for CellBitmapMetadata { - fn from(bitmap: CellBitmap) -> Self { - Self { bitmap } + fn from(value: CellBitmap) -> Self { + Self { + encoded: value.as_ssz_bytes(), + bitmap: value, + } } } impl Partial for PartialDataColumnSidecarMessage { - type Metadata = CellBitmapMetadata; - fn group_id(&self) -> Vec { - self.partial_column.block_root.to_vec() + self.partial_column.block_root.as_slice().to_vec() } - fn metadata(&self) -> &Self::Metadata { - &self.metadata - } - - fn partial_message_bytes_from_metadata( - &self, - metadata: &Self::Metadata, - ) -> Result>, PartialMessageError> { - Ok(self - .partial_column + fn metadata(&self) -> Vec { + self.partial_column .sidecar - .with_missing_cells(&metadata.bitmap) - .map(|message| message.as_ssz_bytes())) + .cells_present_bitmap + .as_ssz_bytes() } - fn data_for_eager_push( + fn partial_message_bytes_from_metadata( &self, - ) -> Result, Self::Metadata)>, PartialMessageError> { - Ok(self.send_eager.clone().map(|eager| (eager.data, eager.metadata))) + metadata: Option<&[u8]>, + ) -> Result { + match metadata { + None => Ok(PublishAction { + need: false, + send: self.send_eager.clone().map(|eager| { + ( + eager.data, + Box::new(eager.metadata) as Box, + ) + }), + }), + Some(metadata) => { + let peer_has = CellBitmap::::from_ssz_bytes(metadata) + .map_err(|_| PartialMessageError::InvalidFormat)?; + let need = !peer_has.is_subset(&self.partial_column.sidecar.cells_present_bitmap); + + let send = self + .partial_column + .sidecar + .with_missing_cells(&peer_has) + .map(|sidecar| { + ( + sidecar.as_ssz_bytes(), + Box::new(CellBitmapMetadata::::from( + peer_has.union(&sidecar.cells_present_bitmap), + )) as Box, + ) + }); + + Ok(PublishAction { need, send }) + } + } } }