add transaction & traces to build process

Author: syphar

crates/metadata/lib.rs

@@ -98,7 +98,7 @@ pub enum MetadataError {
 /// ```
 ///
 /// You can define one or more fields in your `Cargo.toml`.
-#[derive(Default, Deserialize)]
+#[derive(Debug, Default, Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub struct Metadata {
     /// Whether the current crate is a proc-macro (used by docs.rs to hack around cargo bugs).

src/bin/cratesfyi.rs

@@ -23,6 +23,7 @@ use docs_rs::{
 use futures_util::StreamExt;
 use humantime::Duration;
 use once_cell::sync::OnceCell;
+use sentry::TransactionContext;
 use tokio::runtime::{Builder, Runtime};
 use tracing_log::LogTracer;
 use tracing_subscriber::{filter::Directive, prelude::*, EnvFilter};
@@ -53,15 +54,33 @@ fn main() {
         ))
         .unwrap();
 
+        let traces_sample_rate = env::var("SENTRY_TRACES_SAMPLE_RATE")
+            .ok()
+            .and_then(|v| v.parse().ok())
+            .unwrap_or(0.0);
+
+        let traces_sampler = move |ctx: &TransactionContext| -> f32 {
+            if let Some(sampled) = ctx.sampled() {
+                // if the transaction was already marked as "to be sampled" by
+                // the JS/frontend SDK, we want to sample it in the backend too.
+                return if sampled { 1.0 } else { 0.0 };
+            }
+
+            let op = ctx.operation();
+            if op == "docbuilder.build_package" {
+                // record all transactions for builds
+                1.
+            } else {
+                traces_sample_rate
+            }
+        };
+
         Some(sentry::init((
             sentry_dsn,
             sentry::ClientOptions {
                 release: Some(docs_rs::BUILD_VERSION.into()),
                 attach_stacktrace: true,
-                traces_sample_rate: env::var("SENTRY_TRACES_SAMPLE_RATE")
-                    .ok()
-                    .and_then(|v| v.parse().ok())
-                    .unwrap_or(0.0),
+                traces_sampler: Some(Arc::new(traces_sampler)),
                 ..Default::default()
             }
             .add_integration(sentry_panic::PanicIntegration::default()),

src/db/add_package.rs

@@ -17,7 +17,7 @@ use std::{
     io::{BufRead, BufReader},
     path::Path,
 };
-use tracing::{debug, info};
+use tracing::{debug, info, instrument};
 
 /// Adds a package into database.
 ///
@@ -26,6 +26,7 @@ use tracing::{debug, info};
 /// NOTE: `source_files` refers to the files originally in the crate,
 /// not the files generated by rustdoc.
 #[allow(clippy::too_many_arguments)]
+#[instrument(skip(conn))]
 pub(crate) async fn add_package_into_database(
     conn: &mut sqlx::PgConnection,
     metadata_pkg: &MetadataPackage,
@@ -195,6 +196,7 @@ async fn crate_id_from_release_id(conn: &mut sqlx::PgConnection, release_id: i32
     .await?)
 }
 
+#[instrument(skip(conn))]
 pub(crate) async fn add_doc_coverage(
     conn: &mut sqlx::PgConnection,
     release_id: i32,
@@ -225,6 +227,7 @@ pub(crate) async fn add_doc_coverage(
 }
 
 /// Adds a build into database
+#[instrument(skip(conn))]
 pub(crate) async fn add_build_into_database(
     conn: &mut sqlx::PgConnection,
     release_id: i32,
@@ -426,6 +429,7 @@ async fn add_keywords_into_database(
     Ok(())
 }
 
+#[instrument(skip(conn))]
 pub async fn update_crate_data_in_database(
     conn: &mut sqlx::PgConnection,
     name: &str,

src/db/file.rs

@@ -9,9 +9,9 @@
 
 use crate::error::Result;
 use crate::storage::{AsyncStorage, CompressionAlgorithm, CompressionAlgorithms};
-
 use serde_json::Value;
 use std::path::{Path, PathBuf};
+use tracing::instrument;
 
 /// Store all files in a directory and return [[mimetype, filename]] as Json
 ///
@@ -34,7 +34,8 @@ pub async fn add_path_into_database<P: AsRef<Path>>(
     ))
 }
 
-pub async fn add_path_into_remote_archive<P: AsRef<Path>>(
+#[instrument(skip(storage))]
+pub async fn add_path_into_remote_archive<P: AsRef<Path> + std::fmt::Debug>(
     storage: &AsyncStorage,
     archive_path: &str,
     path: P,

src/docbuilder/rustwide_builder.rs

@@ -29,7 +29,7 @@ use std::path::Path;
 use std::sync::Arc;
 use std::time::Instant;
 use tokio::runtime::Runtime;
-use tracing::{debug, info, warn};
+use tracing::{debug, info, info_span, instrument, warn};
 
 const USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)";
 const COMPONENTS: &[&str] = &["llvm-tools-preview", "rustc-dev", "rustfmt"];
@@ -75,6 +75,7 @@ fn build_workspace(context: &dyn Context) -> Result<Workspace> {
     Ok(workspace)
 }
 
+#[derive(Debug)]
 pub enum PackageKind<'a> {
     Local(&'a Path),
     CratesIo,
@@ -130,6 +131,7 @@ impl RustwideBuilder {
         Ok(())
     }
 
+    #[instrument(skip(self))]
     fn prepare_sandbox(&self, limits: &Limits) -> SandboxBuilder {
         SandboxBuilder::new()
             .cpu_limit(self.config.build_cpu_limit.map(|limit| limit as f32))
@@ -257,6 +259,7 @@ impl RustwideBuilder {
         }
     }
 
+    #[instrument(skip(self))]
     fn get_limits(&self, krate: &str) -> Result<Limits> {
         self.runtime.block_on({
             let db = self.db.clone();
@@ -358,6 +361,7 @@ impl RustwideBuilder {
         self.build_package(&package.name, &package.version, PackageKind::Local(path))
     }
 
+    #[instrument(name = "docbuilder.build_package", parent = None, skip(self))]
     pub fn build_package(
         &mut self,
         name: &str,
@@ -403,21 +407,29 @@ impl RustwideBuilder {
         // fill up disk space.
         // This also prevents having multiple builders using the same rustwide workspace,
         // which we don't do. Currently our separate builders use a separate rustwide workspace.
-        self.workspace
-            .purge_all_build_dirs()
-            .map_err(FailureError::compat)?;
+        {
+            let _span = info_span!("purge_all_build_dirs").entered();
+            self.workspace
+                .purge_all_build_dirs()
+                .map_err(FailureError::compat)?;
+        }
 
         let mut build_dir = self.workspace.build_dir(&format!("{name}-{version}"));
 
         let is_local = matches!(kind, PackageKind::Local(_));
-        let krate = match kind {
-            PackageKind::Local(path) => Crate::local(path),
-            PackageKind::CratesIo => Crate::crates_io(name, version),
-            PackageKind::Registry(registry) => {
-                Crate::registry(AlternativeRegistry::new(registry), name, version)
-            }
+        let krate = {
+            let _span = info_span!("krate.fetch").entered();
+
+            let krate = match kind {
+                PackageKind::Local(path) => Crate::local(path),
+                PackageKind::CratesIo => Crate::crates_io(name, version),
+                PackageKind::Registry(registry) => {
+                    Crate::registry(AlternativeRegistry::new(registry), name, version)
+                }
+            };
+            krate.fetch(&self.workspace).map_err(FailureError::compat)?;
+            krate
         };
-        krate.fetch(&self.workspace).map_err(FailureError::compat)?;
 
         fs::create_dir_all(&self.config.temp_dir)?;
         let local_storage = tempfile::tempdir_in(&self.config.temp_dir)?;
@@ -432,8 +444,12 @@ impl RustwideBuilder {
                 } = metadata.targets(self.config.include_default_targets);
                 let mut targets = vec![default_target];
                 targets.extend(&other_targets);
-                // Fetch this before we enter the sandbox, so networking isn't blocked.
-                build.fetch_build_std_dependencies(&targets)?;
+
+                {
+                    let _span = info_span!("fetch_build_std_dependencies").entered();
+                    // Fetch this before we enter the sandbox, so networking isn't blocked.
+                    build.fetch_build_std_dependencies(&targets)?;
+                }
 
                 (|| -> Result<bool> {
                     let mut has_docs = false;
@@ -448,14 +464,20 @@ impl RustwideBuilder {
                     if !res.result.successful && cargo_lock.exists() {
                         info!("removing lockfile and reattempting build");
                         std::fs::remove_file(cargo_lock)?;
-                        Command::new(&self.workspace, self.toolchain.cargo())
-                            .cd(build.host_source_dir())
-                            .args(&["generate-lockfile"])
-                            .run()?;
-                        Command::new(&self.workspace, self.toolchain.cargo())
-                            .cd(build.host_source_dir())
-                            .args(&["fetch", "--locked"])
-                            .run()?;
+                        {
+                            let _span = info_span!("cargo_generate_lockfile").entered();
+                            Command::new(&self.workspace, self.toolchain.cargo())
+                                .cd(build.host_source_dir())
+                                .args(&["generate-lockfile"])
+                                .run()?;
+                        }
+                        {
+                            let _span = info_span!("cargo fetch --locked").entered();
+                            Command::new(&self.workspace, self.toolchain.cargo())
+                                .cd(build.host_source_dir())
+                                .args(&["fetch", "--locked"])
+                                .run()?;
+                        }
                         res = self.execute_build(
                             default_target,
                             true,
@@ -595,11 +617,14 @@ impl RustwideBuilder {
                         build_status,
                     ))?;
 
-                    let build_log_path = format!("build-logs/{build_id}/{default_target}.txt");
-                    self.storage.store_one(build_log_path, res.build_log)?;
-                    for (target, log) in target_build_logs {
-                        let build_log_path = format!("build-logs/{build_id}/{target}.txt");
-                        self.storage.store_one(build_log_path, log)?;
+                    {
+                        let _span = info_span!("store_build_logs").entered();
+                        let build_log_path = format!("build-logs/{build_id}/{default_target}.txt");
+                        self.storage.store_one(build_log_path, res.build_log)?;
+                        for (target, log) in target_build_logs {
+                            let build_log_path = format!("build-logs/{build_id}/{target}.txt");
+                            self.storage.store_one(build_log_path, log)?;
+                        }
                     }
 
                     // Some crates.io crate data is mutable, so we proactively update it during a release
@@ -638,13 +663,17 @@ impl RustwideBuilder {
             })
             .map_err(|e| e.compat())?;
 
-        krate
-            .purge_from_cache(&self.workspace)
-            .map_err(FailureError::compat)?;
-        local_storage.close()?;
+        {
+            let _span = info_span!("purge_from_cache").entered();
+            krate
+                .purge_from_cache(&self.workspace)
+                .map_err(FailureError::compat)?;
+            local_storage.close()?;
+        }
         Ok(successful)
     }
 
+    #[instrument(skip(self, build))]
     fn build_target(
         &self,
         target: &str,
@@ -668,6 +697,7 @@ impl RustwideBuilder {
         Ok(target_res)
     }
 
+    #[instrument(skip(self, build))]
     fn get_coverage(
         &self,
         target: &str,
@@ -723,6 +753,7 @@ impl RustwideBuilder {
         )
     }
 
+    #[instrument(skip(self, build))]
     fn execute_build(
         &self,
         target: &str,
@@ -764,11 +795,14 @@ impl RustwideBuilder {
             }
         };
 
-        let successful = logging::capture(&storage, || {
-            self.prepare_command(build, target, metadata, limits, rustdoc_flags)
-                .and_then(|command| command.run().map_err(Error::from))
-                .is_ok()
-        });
+        let successful = {
+            let _span = info_span!("cargo_build", target = %target, is_default_target).entered();
+            logging::capture(&storage, || {
+                self.prepare_command(build, target, metadata, limits, rustdoc_flags)
+                    .and_then(|command| command.run().map_err(Error::from))
+                    .is_ok()
+            })
+        };
 
         // For proc-macros, cargo will put the output in `target/doc`.
         // Move it to the target-specific directory for consistency with other builds.
@@ -879,6 +913,7 @@ impl RustwideBuilder {
         Ok(command.args(&cargo_args))
     }
 
+    #[instrument(skip(self))]
     fn copy_docs(
         &self,
         target_dir: &Path,
@@ -917,7 +952,7 @@ struct FullBuildResult {
     build_log: String,
 }
 
-#[derive(Clone, Copy)]
+#[derive(Debug, Clone, Copy)]
 pub(crate) struct DocCoverage {
     /// The total items that could be documented in the current crate, used to calculate
     /// documentation coverage.
@@ -931,6 +966,7 @@ pub(crate) struct DocCoverage {
     pub(crate) items_with_examples: i32,
 }
 
+#[derive(Debug)]
 pub(crate) struct BuildResult {
     pub(crate) rustc_version: String,
     pub(crate) docsrs_version: String,

src/registry_api.rs

@@ -4,6 +4,7 @@ use chrono::{DateTime, Utc};
 use reqwest::header::{HeaderValue, ACCEPT, USER_AGENT};
 use semver::Version;
 use serde::Deserialize;
+use tracing::instrument;
 use url::Url;
 
 const APP_USER_AGENT: &str = concat!(
@@ -67,6 +68,7 @@ impl RegistryApi {
         })
     }
 
+    #[instrument(skip(self))]
     pub async fn get_crate_data(&self, name: &str) -> Result<CrateData> {
         let owners = self
             .get_owners(name)
@@ -76,6 +78,7 @@ impl RegistryApi {
         Ok(CrateData { owners })
     }
 
+    #[instrument(skip(self))]
     pub(crate) async fn get_release_data(&self, name: &str, version: &str) -> Result<ReleaseData> {
         let (release_time, yanked, downloads) = self
             .get_release_time_yanked_downloads(name, version)

src/storage/archive_index.rs

@@ -23,7 +23,8 @@ impl FileInfo {
 /// create an archive index based on a zipfile.
 ///
 /// Will delete the destination file if it already exists.
-pub(crate) fn create<R: io::Read + io::Seek, P: AsRef<Path>>(
+#[instrument(skip(zipfile))]
+pub(crate) fn create<R: io::Read + io::Seek, P: AsRef<Path> + std::fmt::Debug>(
     zipfile: &mut R,
     destination: P,
 ) -> Result<()> {