Merge pull request #700 from FractalFir/fuzz_support

Add support for automatically reducing found fuzz cases.

Author: antoyo

build_system/src/fuzz.rs

@@ -1,13 +1,19 @@
 use std::ffi::OsStr;
 use std::path::Path;
 
+mod reduce;
+
 use crate::utils::run_command_with_output;
 
 fn show_usage() {
     println!(
         r#"
 `fuzz` command help:
-    --help                 : Show this help"#
+    --reduce               : Reduces a file generated by rustlantis
+    --help                 : Show this help
+    --start                : Start of the fuzzed range
+    --count                : The number of cases to fuzz
+    -j --jobs              : The number of threads to use during fuzzing"#
     );
 }
 
@@ -20,6 +26,16 @@ pub fn run() -> Result<(), String> {
         std::thread::available_parallelism().map(|threads| threads.get()).unwrap_or(1);
     while let Some(arg) = args.next() {
         match arg.as_str() {
+            "--reduce" => {
+                let Some(path) = args.next() else {
+                    return Err("--reduce must be provided with a path".into());
+                };
+                if !std::fs::exists(&path).unwrap_or(false) {
+                    return Err("--reduce must be provided with a valid path".into());
+                }
+                reduce::reduce(&path);
+                return Ok(());
+            }
             "--help" => {
                 show_usage();
                 return Ok(());
@@ -75,16 +91,17 @@ fn fuzz_range(start: u64, end: u64, threads: usize) {
     let start = Arc::new(AtomicU64::new(start));
     // Count time during fuzzing
     let start_time = Instant::now();
+    let mut workers = Vec::with_capacity(threads);
     // Spawn `threads`..
     for _ in 0..threads {
         let start = start.clone();
         // .. which each will ..
-        std::thread::spawn(move || {
+        workers.push(std::thread::spawn(move || {
             // ... grab the next fuzz seed ...
             while start.load(Ordering::Relaxed) < end {
                 let next = start.fetch_add(1, Ordering::Relaxed);
                 // .. test that seed .
-                match test(next) {
+                match test(next, false) {
                     Err(err) => {
                         // If the test failed at compile-time...
                         println!("test({}) failed because {err:?}", next);
@@ -99,29 +116,38 @@ fn fuzz_range(start: u64, end: u64, threads: usize) {
                     Ok(Err(err)) => {
                         // If the test failed at run-time...
                         println!("The LLVM and GCC results don't match for {err:?}");
-                        // ... copy that file to the directory `target/fuzz/runtime_error`...
+                        // ... generate a new file, which prints temporaries(instead of hashing them)...
                         let mut out_path: std::path::PathBuf = "target/fuzz/runtime_error".into();
                         std::fs::create_dir_all(&out_path).unwrap();
-                        // .. into a file named `fuzz{seed}.rs`.
+                        let Ok(Err(tmp_print_err)) = test(next, true) else {
+                            // ... if that file does not reproduce the issue...
+                            // ... save the original sample in a file named `fuzz{seed}.rs`...
+                            out_path.push(&format!("fuzz{next}.rs"));
+                            std::fs::copy(err, &out_path).unwrap();
+                            continue;
+                        };
+                        // ... if that new file still produces the issue, copy it to `fuzz{seed}.rs`..
                         out_path.push(&format!("fuzz{next}.rs"));
-                        std::fs::copy(err, out_path).unwrap();
+                        std::fs::copy(tmp_print_err, &out_path).unwrap();
+                        // ... and start reducing it, using some properties of `rustlantis` to speed up the process.
+                        reduce::reduce(&out_path);
                     }
                     // If the test passed, do nothing
                     Ok(Ok(())) => (),
                 }
             }
-        });
+        }));
     }
     // The "manager" thread loop.
-    while start.load(Ordering::Relaxed) < end {
+    while start.load(Ordering::Relaxed) < end || !workers.iter().all(|t| t.is_finished()) {
         // Every 500 ms...
         let five_hundred_millis = Duration::from_millis(500);
         std::thread::sleep(five_hundred_millis);
         // ... calculate the remaining fuzz iters ...
         let remaining = end - start.load(Ordering::Relaxed);
         // ... fix the count(the start counter counts the cases that
         // begun fuzzing, and not only the ones that are done)...
-        let fuzzed = (total - remaining) - threads as u64;
+        let fuzzed = (total - remaining).saturating_sub(threads as u64);
         // ... and the fuzz speed ...
         let iter_per_sec = fuzzed as f64 / start_time.elapsed().as_secs_f64();
         // .. and use them to display fuzzing stats.
@@ -131,6 +157,7 @@ fn fuzz_range(start: u64, end: u64, threads: usize) {
             (remaining as f64) / iter_per_sec
         )
     }
+    drop(workers);
 }
 
 /// Builds & runs a file with LLVM.
@@ -198,37 +225,61 @@ fn release_gcc(path: &std::path::Path) -> Result<Vec<u8>, String> {
     res.extend(output.stderr);
     Ok(res)
 }
-
+type ResultCache = Option<(Vec<u8>, Vec<u8>)>;
 /// Generates a new rustlantis file, & compares the result of running it with GCC and LLVM.
-fn test(seed: u64) -> Result<Result<(), std::path::PathBuf>, String> {
+fn test(seed: u64, print_tmp_vars: bool) -> Result<Result<(), std::path::PathBuf>, String> {
     // Generate a Rust source...
-    let source_file = generate(seed)?;
-    // ... test it with debug LLVM ...
-    let llvm_res = debug_llvm(&source_file)?;
-    // ... test it with release GCC ...
+    let source_file = generate(seed, print_tmp_vars)?;
+    test_file(&source_file, true)
+}
+/// Tests a file with a cached LLVM result. Used for reduction, when it is known
+/// that a given transformation should not change the execution result.
+fn test_cached(
+    source_file: &Path,
+    remove_tmps: bool,
+    cache: &mut ResultCache,
+) -> Result<Result<(), std::path::PathBuf>, String> {
+    //  Test `source_file` with release GCC ...
     let gcc_res = release_gcc(&source_file)?;
+    if cache.is_none() {
+        // ...test `source_file` with debug LLVM ...
+        *cache = Some((debug_llvm(&source_file)?, gcc_res.clone()));
+    }
+    let (llvm_res, old_gcc) = cache.as_ref().unwrap();
     // ... compare the results ...
-    if llvm_res != gcc_res {
+    if *llvm_res != gcc_res && gcc_res == *old_gcc {
         // .. if they don't match, report an error.
-        Ok(Err(source_file))
+        Ok(Err(source_file.to_path_buf()))
     } else {
-        std::fs::remove_file(source_file).map_err(|err| format!("{err:?}"))?;
+        if remove_tmps {
+            std::fs::remove_file(source_file).map_err(|err| format!("{err:?}"))?;
+        }
         Ok(Ok(()))
     }
 }
+fn test_file(
+    source_file: &Path,
+    remove_tmps: bool,
+) -> Result<Result<(), std::path::PathBuf>, String> {
+    let mut uncached = None;
+    test_cached(source_file, remove_tmps, &mut uncached)
+}
 
 /// Generates a new rustlantis file for us to run tests on.
-fn generate(seed: u64) -> Result<std::path::PathBuf, String> {
+fn generate(seed: u64, print_tmp_vars: bool) -> Result<std::path::PathBuf, String> {
     use std::io::Write;
     let mut out_path = std::env::temp_dir();
     out_path.push(&format!("fuzz{seed}.rs"));
     // We need to get the command output here.
-    let out = std::process::Command::new("cargo")
+    let mut generate = std::process::Command::new("cargo");
+    generate
         .args(["run", "--release", "--bin", "generate"])
         .arg(&format!("{seed}"))
-        .current_dir("clones/rustlantis")
-        .output()
-        .map_err(|err| format!("{err:?}"))?;
+        .current_dir("clones/rustlantis");
+    if print_tmp_vars {
+        generate.arg("--debug");
+    }
+    let out = generate.output().map_err(|err| format!("{err:?}"))?;
     // Stuff the rustlantis output in a source file.
     std::fs::File::create(&out_path)
         .map_err(|err| format!("{err:?}"))?