Bits and bobs to make everything work

Author: durch

common/credential-verification/src/ecash/mod.rs

@@ -253,4 +253,8 @@ impl traits::EcashManager for MockEcashManager {
     }
 
     fn async_verify(&self, _ticket: ClientTicket) {}
+
+    fn is_mock(&self) -> bool {
+        true
+    }
 }

common/credential-verification/src/ecash/traits.rs

@@ -20,4 +20,10 @@ pub trait EcashManager {
         aggregated_verification_key: &VerificationKeyAuth,
     ) -> Result<(), EcashTicketError>;
     fn async_verify(&self, ticket: ClientTicket);
+
+    /// Returns true if this is a mock ecash manager (for local testing).
+    /// Default implementation returns false.
+    fn is_mock(&self) -> bool {
+        false
+    }
 }

common/wireguard/src/lib.rs

@@ -159,6 +159,7 @@ impl WireguardGatewayData {
 pub struct WireguardData {
     pub inner: WireguardGatewayData,
     pub peer_rx: Receiver<PeerControlRequest>,
+    pub use_userspace: bool,
 }
 
 /// Start wireguard device
@@ -170,6 +171,7 @@ pub async fn start_wireguard(
     upgrade_mode_status: nym_credential_verification::upgrade_mode::UpgradeModeStatus,
     shutdown_token: nym_task::ShutdownToken,
     wireguard_data: WireguardData,
+    use_userspace: bool,
 ) -> Result<std::sync::Arc<WgApiWrapper>, Box<dyn std::error::Error + Send + Sync + 'static>> {
     use base64::{Engine, prelude::BASE64_STANDARD};
     use defguard_wireguard_rs::{InterfaceConfiguration, WireguardInterfaceApi};
@@ -181,7 +183,8 @@ pub async fn start_wireguard(
     use tracing::info;
 
     let ifname = String::from(WG_TUN_BASE_NAME);
-    let wg_api = defguard_wireguard_rs::WGApi::new(ifname.clone(), false)?;
+    info!("Initializing WireGuard interface '{}' with use_userspace={}", ifname, use_userspace);
+    let wg_api = defguard_wireguard_rs::WGApi::new(ifname.clone(), use_userspace)?;
     let mut peer_bandwidth_managers = HashMap::with_capacity(peers.len());
 
     for peer in peers.iter() {
@@ -212,7 +215,13 @@ pub async fn start_wireguard(
         interface_config.address, interface_config.port
     );
 
-    wg_api.configure_interface(&interface_config)?;
+    info!("Configuring WireGuard interface...");
+    wg_api.configure_interface(&interface_config).map_err(|e| {
+        log::error!("Failed to configure WireGuard interface: {:?}", e);
+        e
+    })?;
+
+    info!("Adding IPv6 address to interface...");
     std::process::Command::new("ip")
         .args([
             "-6",
@@ -226,7 +235,11 @@ pub async fn start_wireguard(
             "dev",
             (&ifname),
         ])
-        .output()?;
+        .output()
+        .map_err(|e| {
+            log::error!("Failed to add IPv6 address: {:?}", e);
+            e
+        })?;
 
     // Use a dummy peer to create routing rule for the entire network space
     let mut catch_all_peer = Peer::new(Key::new([0; 32]));

docker/localnet/Dockerfile.localnet

@@ -15,15 +15,27 @@ RUN cargo build --release --locked \
     -p nym-network-requester \
     -p nym-socks5-client
 
-# Install runtime dependencies
+# Install runtime dependencies including Go for wireguard-go
 RUN apt update && apt install -y \
     python3 \
     python3-pip \
     netcat-openbsd \
     jq \
     iproute2 \
+    net-tools \
+    wireguard-tools \
+    golang-go \
+    git \
     && rm -rf /var/lib/apt/lists/*
 
+# Install wireguard-go (userspace WireGuard implementation)
+RUN git clone https://git.zx2c4.com/wireguard-go && \
+    cd wireguard-go && \
+    make && \
+    cp wireguard-go /usr/local/bin/ && \
+    cd .. && \
+    rm -rf wireguard-go
+
 # Install Python dependencies for build_topology.py
 RUN pip3 install --break-system-packages base58
 

docker/localnet/localnet.sh

@@ -222,7 +222,7 @@ start_mixnode() {
 start_gateway() {
     log_info "Starting $GATEWAY_CONTAINER..."
 
-    container run \
+        container run \
         --name "$GATEWAY_CONTAINER" \
         -m 2G \
         --network "$NETWORK_NAME" \
@@ -256,14 +256,15 @@ start_gateway() {
                 --lp-use-mock-ecash true \
                 --output=json \
                 --wireguard-enabled true \
+                --wireguard-userspace true \
                 --bonding-information-output="/localnet/gateway.json";
 
             echo "Waiting for network.json...";
             while [ ! -f /localnet/network.json ]; do
                 sleep 2;
             done;
             echo "Starting gateway with LP listener (mock ecash)...";
-            exec nym-node run --id gateway-localnet --unsafe-disable-replay-protection --local
+            exec nym-node run --id gateway-localnet --unsafe-disable-replay-protection --local --wireguard-enabled true --wireguard-userspace true --lp-use-mock-ecash true
         '
 
     log_success "$GATEWAY_CONTAINER started"

gateway/src/node/lp_listener/handler.rs

@@ -367,9 +367,14 @@ impl LpConnectionHandler {
             GatewayError::LpProtocolError(format!("Failed to serialize response: {}", e))
         })?;
 
-        // Create LP packet with response
-        let packet = session.create_data_packet(data).map_err(|e| {
-            GatewayError::LpProtocolError(format!("Failed to create data packet: {}", e))
+        // Encrypt data first (this increments Noise internal counter)
+        let encrypted_message = session.encrypt_data(&data).map_err(|e| {
+            GatewayError::LpProtocolError(format!("Failed to encrypt data: {}", e))
+        })?;
+
+        // Create LP packet with encrypted message (this increments LP protocol counter)
+        let packet = session.next_packet(encrypted_message).map_err(|e| {
+            GatewayError::LpProtocolError(format!("Failed to create packet: {}", e))
         })?;
 
         // Send the packet
@@ -473,28 +478,6 @@ impl LpConnectionHandler {
     }
 }
 
-// Extension trait for LpSession to create packets
-// This would ideally be part of nym-lp
-trait LpSessionExt {
-    fn create_data_packet(&self, data: Vec<u8>) -> Result<LpPacket, nym_lp::LpError>;
-}
-
-impl LpSessionExt for LpSession {
-    fn create_data_packet(&self, data: Vec<u8>) -> Result<LpPacket, nym_lp::LpError> {
-        use nym_lp::packet::LpHeader;
-
-        let header = LpHeader {
-            protocol_version: 1,
-            session_id: self.id(),
-            counter: 0, // TODO: Use actual counter from session
-        };
-
-        let message = LpMessage::EncryptedData(data);
-
-        Ok(LpPacket::new(header, message))
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

gateway/src/node/lp_listener/registration.rs

@@ -59,16 +59,26 @@ async fn credential_storage_preparation(
     ecash_verifier: Arc<dyn EcashManager + Send + Sync>,
     client_id: i64,
 ) -> Result<PersistedBandwidth, GatewayError> {
-    ecash_verifier
+    // Check if bandwidth entry already exists (idempotent)
+    let existing_bandwidth = ecash_verifier
         .storage()
-        .create_bandwidth_entry(client_id)
+        .get_available_bandwidth(client_id)
         .await?;
+
+    // Only create if it doesn't exist
+    if existing_bandwidth.is_none() {
+        ecash_verifier
+            .storage()
+            .create_bandwidth_entry(client_id)
+            .await?;
+    }
+
     let bandwidth = ecash_verifier
         .storage()
         .get_available_bandwidth(client_id)
         .await?
         .ok_or_else(|| {
-            GatewayError::InternalError("bandwidth entry should have just been created".to_string())
+            GatewayError::InternalError("bandwidth entry should exist".to_string())
         })?;
     Ok(bandwidth)
 }
@@ -96,18 +106,30 @@ async fn credential_verification(
     // Track credential verification attempts
     inc!("lp_credential_verification_attempts");
 
-    match verifier.verify().await {
-        Ok(allocated) => {
-            inc!("lp_credential_verification_success");
-            // Track allocated bandwidth
-            inc_by!("lp_bandwidth_allocated_bytes_total", allocated);
-            Ok(allocated)
-        }
-        Err(e) => {
-            inc!("lp_credential_verification_failed");
-            Err(e.into())
+    // For mock ecash mode (local testing), skip cryptographic verification
+    // and just return a dummy bandwidth value since we don't have blockchain access
+    let allocated = if ecash_verifier.is_mock() {
+        // Return a reasonable test bandwidth value (e.g., 1GB in bytes)
+        const MOCK_BANDWIDTH: i64 = 1024 * 1024 * 1024;
+        inc!("lp_credential_verification_success");
+        inc_by!("lp_bandwidth_allocated_bytes_total", MOCK_BANDWIDTH);
+        Ok::<i64, GatewayError>(MOCK_BANDWIDTH)
+    } else {
+        match verifier.verify().await {
+            Ok(allocated) => {
+                inc!("lp_credential_verification_success");
+                // Track allocated bandwidth
+                inc_by!("lp_bandwidth_allocated_bytes_total", allocated);
+                Ok(allocated)
+            }
+            Err(e) => {
+                inc!("lp_credential_verification_failed");
+                Err(e.into())
+            }
         }
-    }
+    }?;
+
+    Ok(allocated)
 }
 
 /// Process an LP registration request
@@ -320,7 +342,22 @@ async fn register_wg_peer(
     ];
     peer.persistent_keepalive_interval = Some(25);
 
-    // Send to WireGuard peer controller and track latency
+    // Store peer in database FIRST (before adding to controller)
+    // This ensures bandwidth storage exists when controller's generate_bandwidth_manager() is called
+    let client_id = state
+        .storage
+        .insert_wireguard_peer(&peer, ticket_type.into())
+        .await
+        .map_err(|e| {
+            error!("Failed to store WireGuard peer in database: {}", e);
+            GatewayError::InternalError(format!("Failed to store peer: {}", e))
+        })?;
+
+    // Create bandwidth entry for the client
+    // This must happen BEFORE AddPeer because generate_bandwidth_manager() expects it to exist
+    credential_storage_preparation(state.ecash_verifier.clone(), client_id).await?;
+
+    // Now send to WireGuard peer controller and track latency
     let controller_start = std::time::Instant::now();
     let (tx, rx) = oneshot::channel();
     wg_controller
@@ -348,16 +385,6 @@ async fn register_wg_peer(
 
     result?;
 
-    // Store bandwidth allocation and get client_id
-    let client_id = state
-        .storage
-        .insert_wireguard_peer(&peer, ticket_type.into())
-        .await
-        .map_err(|e| {
-            error!("Failed to store WireGuard peer in database: {}", e);
-            GatewayError::InternalError(format!("Failed to store peer: {}", e))
-        })?;
-
     // Get gateway's actual WireGuard public key
     let gateway_pubkey = *wg_data.keypair().public_key();
 

gateway/src/node/mod.rs

@@ -624,13 +624,15 @@ impl GatewayTasksBuilder {
             wireguard_data.inner.config().announced_metadata_port,
         );
 
+        let use_userspace = wireguard_data.use_userspace;
         let wg_handle = nym_wireguard::start_wireguard(
             ecash_manager,
             self.metrics.clone(),
             all_peers,
             self.upgrade_mode_state.upgrade_mode_status(),
             self.shutdown_tracker.clone_shutdown_token(),
             wireguard_data,
+            use_userspace,
         )
         .await?;
 

nym-gateway-probe/src/bandwidth_helpers.rs

@@ -11,6 +11,7 @@ use nym_sdk::bandwidth::BandwidthImporter;
 use nym_sdk::mixnet::{DisconnectedMixnetClient, EphemeralCredentialStorage};
 use nym_validator_client::nyxd::error::NyxdError;
 use std::time::Duration;
+use time::OffsetDateTime;
 use tracing::{error, info};
 
 pub(crate) async fn import_bandwidth(
@@ -237,6 +238,11 @@ pub(crate) fn create_dummy_credential(
         0, 0, 0, 0, 0, 1,
     ];
 
-    CredentialSpendingData::try_from_bytes(&CREDENTIAL_BYTES)
-        .expect("Failed to deserialize test credential - this is a bug in the test harness")
+    let mut credential = CredentialSpendingData::try_from_bytes(&CREDENTIAL_BYTES)
+        .expect("Failed to deserialize test credential - this is a bug in the test harness");
+
+    // Update spend_date to today to pass validation
+    credential.spend_date = OffsetDateTime::now_utc().date();
+
+    credential
 }

nym-node/src/cli/helpers.rs

@@ -293,6 +293,14 @@ pub(crate) struct WireguardArgs {
         env = NYMNODE_WG_PRIVATE_NETWORK_PREFIX_ARG
     )]
     pub(crate) wireguard_private_network_prefix: Option<u8>,
+
+    /// Use userspace implementation of WireGuard (wireguard-go) instead of kernel module.
+    /// Useful in containerized environments without kernel WireGuard support.
+    #[clap(
+        long,
+        env = NYMNODE_WG_USERSPACE_ARG
+    )]
+    pub(crate) wireguard_userspace: Option<bool>,
 }
 
 impl WireguardArgs {
@@ -321,6 +329,10 @@ impl WireguardArgs {
             section.private_network_prefix_v4 = private_network_prefix
         }
 
+        if let Some(userspace) = self.wireguard_userspace {
+            section.use_userspace = userspace
+        }
+
         section
     }
 }