Specify directory for BOLT profiles

Author: Kobzol

src/tools/opt-dist/src/bolt.rs

@@ -8,7 +8,7 @@ use crate::utils::io::copy_file;
 
 /// Instruments an artifact at the given `path` (in-place) with BOLT and then calls `func`.
 /// After this function finishes, the original file will be restored.
-pub fn with_bolt_instrumented<F: FnOnce() -> anyhow::Result<R>, R>(
+pub fn with_bolt_instrumented<F: FnOnce(&Utf8Path) -> anyhow::Result<R>, R>(
     path: &Utf8Path,
     func: F,
 ) -> anyhow::Result<R> {
@@ -20,10 +20,16 @@ pub fn with_bolt_instrumented<F: FnOnce() -> anyhow::Result<R>, R>(
 
     let instrumented_path = tempfile::NamedTempFile::new()?.into_temp_path();
 
+    let profile_dir =
+        tempfile::TempDir::new().context("Could not create directory for BOLT profiles")?;
+    let profile_prefix = profile_dir.path().join("prof.fdata");
+    let profile_prefix = Utf8Path::from_path(&profile_prefix).unwrap();
+
     // Instrument the original file with BOLT, saving the result into `instrumented_path`
     cmd(&["llvm-bolt"])
         .arg("-instrument")
         .arg(path)
+        .arg(&format!("--instrumentation-file={profile_prefix}"))
         // Make sure that each process will write its profiles into a separate file
         .arg("--instrumentation-file-append-pid")
         .arg("-o")
@@ -36,7 +42,7 @@ pub fn with_bolt_instrumented<F: FnOnce() -> anyhow::Result<R>, R>(
 
     // Run the function that will make use of the instrumented artifact.
     // The original file will be restored when `_backup_file` is dropped.
-    func()
+    func(profile_prefix)
 }
 
 /// Optimizes the file at `path` with BOLT in-place using the given `profile`.

src/tools/opt-dist/src/main.rs

@@ -12,7 +12,9 @@ use crate::environment::{Environment, EnvironmentBuilder};
 use crate::exec::{cmd, Bootstrap};
 use crate::tests::run_tests;
 use crate::timer::Timer;
-use crate::training::{gather_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles};
+use crate::training::{
+    gather_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles, llvm_benchmarks,
+};
 use crate::utils::io::{copy_directory, move_directory, reset_directory};
 use crate::utils::{
     clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space,
@@ -270,9 +272,9 @@ fn execute_pipeline(
             log::info!("Optimizing {llvm_lib} with BOLT");
 
             // Instrument it and gather profiles
-            let profile = with_bolt_instrumented(&llvm_lib, || {
+            let profile = with_bolt_instrumented(&llvm_lib, |llvm_profile_dir| {
                 stage.section("Gather profiles", |_| {
-                    gather_bolt_profiles(env, "LLVM", llvm_benchmarks(env))
+                    gather_bolt_profiles(env, "LLVM", llvm_benchmarks(env), llvm_profile_dir)
                 })
             })?;
             print_free_disk_space()?;
@@ -291,34 +293,34 @@ fn execute_pipeline(
         None
     };
 
-    let rustc_bolt_profile = if env.supports_bolt() {
-        // Stage 4: Build BOLT instrumented rustc
-        timer.section("Stage 4 (Rustc BOLT)", |stage| {
-            // Find the path to the `librustc_driver.so` file
-            let rustc_lib = io::find_file_in_dir(
-                &env.build_artifacts().join("stage2").join("lib"),
-                "librustc_driver",
-                ".so",
-            )?;
-
-            log::info!("Optimizing {rustc_lib} with BOLT");
-
-            // Instrument it and gather profiles
-            let profile = with_bolt_instrumented(&rustc_lib, || {
-                stage.section("Gather profiles", |_| {
-                    gather_bolt_profiles(env, "rustc", rustc_benchmarks(env))
-                })
-            })?;
-            print_free_disk_space()?;
-
-            // Now optimize the library with BOLT.
-            bolt_optimize(&rustc_lib, &profile).context("Could not optimize rustc with BOLT")?;
-
-            Ok(Some(profile))
-        })?
-    } else {
-        None
-    };
+    // let rustc_bolt_profile = if env.use_bolt() {
+    //     // Stage 4: Build BOLT instrumented rustc
+    //     timer.section("Stage 4 (Rustc BOLT)", |stage| {
+    //         // Find the path to the `librustc_driver.so` file
+    //         let rustc_lib = io::find_file_in_dir(
+    //             &env.build_artifacts().join("stage2").join("lib"),
+    //             "librustc_driver",
+    //             ".so",
+    //         )?;
+    //
+    //         log::info!("Optimizing {rustc_lib} with BOLT");
+    //
+    //         // Instrument it and gather profiles
+    //         let profile = with_bolt_instrumented(&rustc_lib, || {
+    //             stage.section("Gather profiles", |_| {
+    //                 gather_bolt_profiles(env, "rustc", rustc_benchmarks(env))
+    //             })
+    //         })?;
+    //         print_free_disk_space()?;
+    //
+    //         // Now optimize the library with BOLT.
+    //         bolt_optimize(&rustc_lib, &profile).context("Could not optimize rustc with BOLT")?;
+    //
+    //         Ok(Some(profile))
+    //     })?
+    // } else {
+    //     None
+    // };
 
     let mut dist = Bootstrap::dist(env, &dist_args)
         .llvm_pgo_optimize(&llvm_pgo_profile)
@@ -328,9 +330,9 @@ fn execute_pipeline(
     if let Some(llvm_bolt_profile) = llvm_bolt_profile {
         dist = dist.with_bolt_profile(llvm_bolt_profile);
     }
-    if let Some(rustc_bolt_profile) = rustc_bolt_profile {
-        dist = dist.with_bolt_profile(rustc_bolt_profile);
-    }
+    // if let Some(rustc_bolt_profile) = rustc_bolt_profile {
+    //     dist = dist.with_bolt_profile(rustc_bolt_profile);
+    // }
 
     // Final stage: Assemble the dist artifacts
     // The previous PGO optimized rustc build and PGO optimized LLVM builds should be reused.

src/tools/opt-dist/src/training.rs

@@ -111,11 +111,11 @@ fn log_profile_stats(
     Ok(())
 }
 
-pub fn llvm_benchmarks(env: &dyn Environment) -> CmdBuilder {
+pub fn llvm_benchmarks(env: &Environment) -> CmdBuilder {
     init_compiler_benchmarks(env, &["Debug", "Opt"], &["Full"], LLVM_PGO_CRATES)
 }
 
-pub fn rustc_benchmarks(env: &dyn Environment) -> CmdBuilder {
+pub fn rustc_benchmarks(env: &Environment) -> CmdBuilder {
     init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["All"], RUSTC_PGO_CRATES)
 }
 
@@ -186,6 +186,7 @@ pub fn gather_bolt_profiles(
     env: &Environment,
     name: &str,
     benchmarks: CmdBuilder,
+    profile_prefix: &Utf8Path,
 ) -> anyhow::Result<BoltProfile> {
     log::info!("Running benchmarks with BOLT instrumented {name}");
 
@@ -194,11 +195,10 @@ pub fn gather_bolt_profiles(
     })?;
 
     let merged_profile = env.artifact_dir().join(format!("{name}-bolt.profdata"));
-    let profile_root = Utf8PathBuf::from("/tmp/prof.fdata");
-    log::info!("Merging {name} BOLT profiles to {merged_profile}");
+    log::info!("Merging {name} BOLT profiles from {profile_prefix} to {merged_profile}");
 
     let profiles: Vec<_> =
-        glob::glob(&format!("{profile_root}*"))?.collect::<Result<Vec<_>, _>>()?;
+        glob::glob(&format!("{profile_prefix}*"))?.collect::<Result<Vec<_>, _>>()?;
 
     let mut merge_args = vec!["merge-fdata"];
     merge_args.extend(profiles.iter().map(|p| p.to_str().unwrap()));
@@ -222,11 +222,11 @@ pub fn gather_bolt_profiles(
         .collect::<Result<Vec<_>, _>>()?
         .into_iter()
         .sum::<u64>();
-    log::info!("{profile_root}: {}", humansize::format_size(size, BINARY));
+    log::info!("{profile_prefix}: {}", humansize::format_size(size, BINARY));
     log::info!("Profile file count: {}", profiles.len());
 
     // Delete the gathered profiles
-    for profile in glob::glob(&format!("{profile_root}*"))?.into_iter() {
+    for profile in glob::glob(&format!("{profile_prefix}*"))?.into_iter() {
         if let Ok(profile) = profile {
             if let Err(error) = std::fs::remove_file(&profile) {
                 log::error!("Cannot delete BOLT profile {}: {error:?}", profile.display());