Merge branch 'launcher' of https://github.com/AFLplusplus/LibAFL into launcher

Author: s1341

libafl/Cargo.toml

@@ -37,7 +37,7 @@ harness = false
 
 [features]
 default = ["std", "anymap_debug", "derive", "llmp_compression"]
-std = ["derive_builder"] # print, sharedmap, ... support
+std = [] # print, env, launcher ... support
 anymap_debug = ["serde_json"] # uses serde_json to Debug the anymap trait. Disable for smaller footprint.
 derive = ["libafl_derive"] # provide derive(SerdeAny) macro.
 llmp_small_maps = [] # reduces initial map size for llmp
@@ -66,7 +66,7 @@ compression = { version = "0.1.5" }
 core_affinity = { version = "0.5", git = "https://github.com/s1341/core_affinity_rs" }
 num_enum = "0.5.1"
 hostname = "^0.3" # Is there really no gethostname in the stdlib?
-derive_builder = { version="0.10", optional = true }
+typed-builder = "0.9.0"
 
 [target.'cfg(target_os = "android")'.dependencies]
 backtrace = { version = "0.3", optional = true, default-features = false, features = ["std", "libbacktrace"] } # for llmp_debug

libafl/src/bolts/llmp.rs

@@ -1707,7 +1707,7 @@ where
         // TODO: handle broker_ids properly/at all.
         let map_description = Self::b2b_thread_on(
             stream,
-            &self.shmem_provider,
+            &mut self.shmem_provider,
             self.llmp_clients.len() as ClientId,
             &self.llmp_out.out_maps.first().unwrap().shmem.description(),
         )?;
@@ -1858,11 +1858,13 @@ where
     #[allow(clippy::let_and_return)]
     fn b2b_thread_on(
         mut stream: TcpStream,
-        shmem_provider: &SP,
+        shmem_provider: &mut SP,
         b2b_client_id: ClientId,
         broker_map_description: &ShMemDescription,
     ) -> Result<ShMemDescription, Error> {
         let broker_map_description = *broker_map_description;
+
+        shmem_provider.pre_fork()?;
         let mut shmem_provider_clone = shmem_provider.clone();
 
         // A channel to get the new "client's" sharedmap id from
@@ -1963,6 +1965,8 @@ where
             }
         });
 
+        shmem_provider.post_fork(false)?;
+
         let ret = recv.recv().map_err(|_| {
             Error::Unknown("Error launching background thread for b2b communcation".to_string())
         });
@@ -1980,7 +1984,7 @@ where
         request: &TcpRequest,
         current_client_id: &mut u32,
         sender: &mut LlmpSender<SP>,
-        shmem_provider: &SP,
+        shmem_provider: &mut SP,
         broker_map_description: &ShMemDescription,
     ) {
         match request {
@@ -2060,6 +2064,7 @@ where
         let tcp_out_map_description = tcp_out_map.shmem.description();
         self.register_client(tcp_out_map);
 
+        self.shmem_provider.pre_fork()?;
         let mut shmem_provider_clone = self.shmem_provider.clone();
 
         let ret = thread::spawn(move || {
@@ -2116,7 +2121,7 @@ where
                             &req,
                             &mut current_client_id,
                             &mut tcp_incoming_sender,
-                            &shmem_provider_clone,
+                            &mut shmem_provider_clone,
                             &broker_map_description,
                         );
                     }

libafl/src/bolts/shmem.rs

@@ -196,13 +196,15 @@ pub trait ShMemProvider: Send + Clone + Default + Debug {
 
     /// This method should be called before a fork or a thread creation event, allowing the [`ShMemProvider`] to
     /// get ready for a potential reset of thread specific info, and for potential reconnects.
+    /// Make sure to call [`Self::post_fork()`] after threading!
     fn pre_fork(&mut self) -> Result<(), Error> {
         // do nothing
         Ok(())
     }
 
     /// This method should be called after a fork or after cloning/a thread creation event, allowing the [`ShMemProvider`] to
     /// reset thread specific info, and potentially reconnect.
+    /// Make sure to call [`Self::pre_fork()`] before threading!
     fn post_fork(&mut self, _is_child: bool) -> Result<(), Error> {
         // do nothing
         Ok(())
@@ -356,7 +358,7 @@ where
                 Ok(())
             }
             None => Err(Error::IllegalState(
-                "Unexpected `None` Pipe in RcShMemProvider!".to_string(),
+                "Unexpected `None` Pipe in RcShMemProvider! Missing post_fork()?".to_string(),
             )),
         }
     }
@@ -378,7 +380,7 @@ where
                 }
             }
             None => Err(Error::IllegalState(
-                "Unexpected `None` Pipe in RcShMemProvider!".to_string(),
+                "Unexpected `None` Pipe in RcShMemProvider! Missing post_fork()?".to_string(),
             )),
         }
     }

libafl/src/events/llmp.rs

@@ -48,8 +48,7 @@ use crate::utils::{fork, ForkResult};
 #[cfg(all(target_os = "android", feature = "std"))]
 use crate::bolts::os::ashmem_server::AshmemService;
 
-#[cfg(feature = "std")]
-use derive_builder::Builder;
+use typed_builder::TypedBuilder;
 
 /// Forward this to the client
 const _LLMP_TAG_EVENT_TO_CLIENT: llmp::Tag = 0x2C11E471;
@@ -580,22 +579,20 @@ where
     #[cfg(target_os = "android")]
     AshmemService::start().expect("Error starting Ashmem Service");
 
-    RestartingMgrBuilder::default()
+    RestartingMgr::builder()
         .shmem_provider(StdShMemProvider::new()?)
         .stats(stats)
         .broker_port(broker_port)
         .build()
-        .unwrap()
         .launch()
 }
 
-/// Provides a builder which can be used to build a restarting manager, which is a combination of a
-/// restarter and runner, that can be used on systems both with and without `fork` support. The
-/// restarter will start a nre process each time the child crashes or timesout.
+/// Provides a `builder` which can be used to build a [`RestartingMgr`], which is a combination of a
+/// `restarter` and `runner`, that can be used on systems both with and without `fork` support. The
+/// `restarter` will start a new process each time the child crashes or times out.
 #[cfg(feature = "std")]
 #[allow(clippy::default_trait_access)]
-#[derive(Builder, Debug)]
-#[builder(pattern = "owned")]
+#[derive(TypedBuilder, Debug)]
 pub struct RestartingMgr<I, S, SP, ST>
 where
     I: Input,
@@ -609,15 +606,15 @@ where
     /// The stats to use
     stats: ST,
     /// The broker port to use
-    #[builder(default = "1337")]
+    #[builder(default = 1337_u16)]
     broker_port: u16,
     /// The address to connect to
-    #[builder(default = "Option::None")]
+    #[builder(default = None)]
     remote_broker_addr: Option<SocketAddr>,
     /// The type of manager to build
-    #[builder(default = "ManagerKind::Any")]
+    #[builder(default = ManagerKind::Any)]
     kind: ManagerKind,
-    #[builder(setter(skip))]
+    #[builder(setter(skip), default = PhantomData {})]
     _phantom: PhantomData<(I, S)>,
 }
 
@@ -642,11 +639,12 @@ where
         )?;
 
         // We start ourself as child process to actually fuzz
-        let (sender, mut receiver, mut new_shmem_provider, core_id) = if std::env::var(
+        let (sender, mut receiver, new_shmem_provider, core_id) = if std::env::var(
             _ENV_FUZZER_SENDER,
         )
         .is_err()
         {
+            // We get here if we are on Unix, or we are a broker on Windows.
             let core_id = if mgr.is_broker() {
                 match self.kind {
                     ManagerKind::Broker | ManagerKind::Any => {
@@ -709,12 +707,19 @@ where
             loop {
                 dbg!("Spawning next client (id {})", ctr);
 
-                // On Unix, we fork (todo: measure if that is actually faster.)
+                // On Unix, we fork
                 #[cfg(unix)]
-                let child_status = match unsafe { fork() }? {
-                    ForkResult::Parent(handle) => handle.status(),
-                    ForkResult::Child => {
-                        break (sender, receiver, self.shmem_provider.clone(), core_id)
+                let child_status = {
+                    self.shmem_provider.pre_fork()?;
+                    match unsafe { fork() }? {
+                        ForkResult::Parent(handle) => {
+                            self.shmem_provider.post_fork(false)?;
+                            handle.status()
+                        }
+                        ForkResult::Child => {
+                            self.shmem_provider.post_fork(true)?;
+                            break (sender, receiver, self.shmem_provider.clone(), core_id);
+                        }
                     }
                 };
 
@@ -739,9 +744,9 @@ where
                 ctr = ctr.wrapping_add(1);
             }
         } else {
-            // We are the newly started fuzzing instance, first, connect to our own restore map.
+            // We are the newly started fuzzing instance (i.e. on Windows), first, connect to our own restore map.
+            // We get here *only on Windows*, if we were started by a restarting fuzzer.
             // A sender and a receiver for single communication
-            self.shmem_provider.post_fork(true)?;
             (
                 LlmpSender::on_existing_from_env(self.shmem_provider.clone(), _ENV_FUZZER_SENDER)?,
                 LlmpReceiver::on_existing_from_env(
@@ -753,8 +758,6 @@ where
             )
         };
 
-        new_shmem_provider.post_fork(false)?;
-
         if let Some(core_id) = core_id {
             core_affinity::set_for_current(core_id);
         }

libafl/src/utils.rs

@@ -18,7 +18,7 @@ use xxhash_rust::xxh3::xxh3_64_with_seed;
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 
 #[cfg(feature = "std")]
-use crate::events::llmp::RestartingMgrBuilder;
+use crate::events::llmp::RestartingMgr;
 
 #[cfg(unix)]
 use libc::pid_t;
@@ -40,8 +40,7 @@ use core_affinity::CoreId;
 #[cfg(all(windows, feature = "std"))]
 use std::process::Stdio;
 
-#[cfg(feature = "std")]
-use derive_builder::Builder;
+use typed_builder::TypedBuilder;
 
 /// Can be converted to a slice
 pub trait AsSlice<T> {
@@ -601,9 +600,8 @@ mod tests {
 
 /// Provides a Launcher, which can be used to launch a fuzzing run on a specified list of cores
 #[cfg(feature = "std")]
-#[derive(Builder)]
+#[derive(TypedBuilder)]
 #[allow(clippy::type_complexity)]
-#[builder(pattern = "owned")]
 pub struct Launcher<'a, I, S, SP, ST>
 where
     I: Input,
@@ -621,16 +619,16 @@ where
     run_client:
         &'a mut dyn FnMut(Option<S>, LlmpRestartingEventManager<I, S, SP, ST>) -> Result<(), Error>,
     /// The broker port to use
-    #[builder(default = "1337")]
+    #[builder(default = 1337_u16)]
     broker_port: u16,
     /// The list of cores to run on
     cores: &'a [usize],
     /// A file name to write all client output to
-    #[builder(default = "None")]
+    #[builder(default = None)]
     stdout_file: Option<&'a str>,
     /// The `ip:port` address of another broker to connect our new broker to for multi-machine
     /// clusters.
-    #[builder(default = "None")]
+    #[builder(default = None)]
     remote_broker_addr: Option<SocketAddr>,
 }
 
@@ -658,6 +656,7 @@ where
         //spawn clients
         for (id, bind_to) in core_ids.iter().enumerate().take(num_cores) {
             if self.cores.iter().any(|&x| x == id) {
+                self.shmem_provider.pre_fork()?;
                 match unsafe { fork() }? {
                     ForkResult::Parent(child) => {
                         self.shmem_provider.post_fork(false)?;
@@ -678,15 +677,14 @@ where
                         }
                         //fuzzer client. keeps retrying the connection to broker till the broker starts
                         let stats = (self.client_init_stats)()?;
-                        let (state, mgr) = RestartingMgrBuilder::default()
+                        let (state, mgr) = RestartingMgr::builder()
                             .shmem_provider(self.shmem_provider.clone())
                             .stats(stats)
                             .broker_port(self.broker_port)
                             .kind(ManagerKind::Client {
                                 cpu_core: Some(*bind_to),
                             })
                             .build()
-                            .unwrap()
                             .launch()?;
 
                         (self.run_client)(state, mgr)?;
@@ -698,25 +696,14 @@ where
         #[cfg(feature = "std")]
         println!("I am broker!!.");
 
-        if let Some(remote_broker_addr) = self.remote_broker_addr {
-            RestartingMgrBuilder::<I, S, SP, ST>::default()
-                .shmem_provider(self.shmem_provider.clone())
-                .stats(self.stats.clone())
-                .broker_port(self.broker_port)
-                .remote_broker_addr(Some(remote_broker_addr))
-                .build()
-                .unwrap()
-                .launch()?;
-        } else {
-            RestartingMgrBuilder::<I, S, SP, ST>::default()
-                .shmem_provider(self.shmem_provider.clone())
-                .stats(self.stats.clone())
-                .broker_port(self.broker_port)
-                .kind(ManagerKind::Broker)
-                .build()
-                .unwrap()
-                .launch()?;
-        }
+        RestartingMgr::<I, S, SP, ST>::builder()
+            .shmem_provider(self.shmem_provider.clone())
+            .stats(self.stats.clone())
+            .broker_port(self.broker_port)
+            .kind(ManagerKind::Broker)
+            .remote_broker_addr(self.remote_broker_addr)
+            .build()
+            .launch()?;
 
         //broker exited. kill all clients.
         for handle in &handles {
@@ -740,7 +727,7 @@ where
 
                 // the actual client. do the fuzzing
                 let stats = (self.client_init_stats)()?;
-                let (state, mgr) = RestartingMgrBuilder::<I, S, SP, ST>::default()
+                let (state, mgr) = RestartingMgr::<I, S, SP, ST>::builder()
                     .shmem_provider(self.shmem_provider.clone())
                     .stats(stats)
                     .broker_port(self.broker_port)
@@ -798,7 +785,7 @@ where
         #[cfg(feature = "std")]
         println!("I am broker!!.");
 
-        RestartingMgrBuilder::<I, S, SP, ST>::default()
+        RestartingMgr::<I, S, SP, ST>::builder()
             .shmem_provider(self.shmem_provider.clone())
             .stats(self.stats.clone())
             .broker_port(self.broker_port)