[WIP]: generating rust-project file for r-a

.gitignore

@@ -35,6 +35,7 @@ Session.vim
 config.mk
 config.stamp
 no_llvm_build
+rust-project.json
 
 ## Build
 /dl/

src/bootstrap/src/core/build_steps/setup.rs

@@ -2,6 +2,7 @@ use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::t;
 use crate::utils::change_tracker::CONFIG_CHANGE_HISTORY;
 use crate::utils::helpers::hex_encode;
+use crate::utils::ra_project::RustAnalyzerProject;
 use crate::Config;
 use sha2::Digest;
 use std::env::consts::EXE_SUFFIX;
@@ -624,3 +625,54 @@ fn create_vscode_settings_maybe(config: &Config) -> io::Result<bool> {
     }
     Ok(should_create)
 }
+
+/// Sets up `rust-project.json`
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
+pub struct RustProjectJson;
+
+impl Step for RustProjectJson {
+    type Output = ();
+    const DEFAULT: bool = true;
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.alias("rust-project")
+    }
+    fn make_run(run: RunConfig<'_>) {
+        if run.builder.config.dry_run() {
+            return;
+        }
+
+        if let [cmd] = &run.paths[..] {
+            if cmd.assert_single_path().path.as_path().as_os_str() == "rust-project" {
+                run.builder.ensure(RustProjectJson);
+            }
+        }
+    }
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let config = &builder.config;
+        if config.dry_run() {
+            return;
+        }
+
+        while !t!(create_ra_project_json_maybe(&config)) {}
+    }
+}
+
+fn create_ra_project_json_maybe(config: &Config) -> io::Result<bool> {
+    println!("\nx.py can automatically generate `rust-project.json` file for rust-analyzer");
+
+    let should_create = match prompt_user("Would you like to create rust-project.json?: [y/N]")? {
+        Some(PromptResult::Yes) => true,
+        _ => {
+            println!("Ok, skipping rust-project.json!");
+            return Ok(true);
+        }
+    };
+
+    if should_create {
+        let ra_project = RustAnalyzerProject::collect_ra_project_data(&config);
+        ra_project.generate_file(&config.src.join("rust-project.json"))?;
+        println!("Created `rust-project.json`");
+    }
+
+    Ok(should_create)
+}

src/bootstrap/src/core/builder.rs

@@ -880,7 +880,13 @@ impl<'a> Builder<'a> {
                 run::GenerateWindowsSys,
                 run::GenerateCompletions,
             ),
-            Kind::Setup => describe!(setup::Profile, setup::Hook, setup::Link, setup::Vscode),
+            Kind::Setup => describe!(
+                setup::Profile,
+                setup::Hook,
+                setup::Link,
+                setup::Vscode,
+                setup::RustProjectJson
+            ),
             Kind::Clean => describe!(clean::CleanAll, clean::Rustc, clean::Std),
             // special-cased in Build::build()
             Kind::Format | Kind::Suggest => vec![],

src/bootstrap/src/core/metadata.rs

@@ -5,7 +5,7 @@ use serde_derive::Deserialize;
 
 use crate::utils::cache::INTERNER;
 use crate::utils::helpers::output;
-use crate::{t, Build, Crate};
+use crate::{t, Build, Config, Crate};
 
 /// For more information, see the output of
 /// <https://doc.rust-lang.org/nightly/cargo/commands/cargo-metadata.html>
@@ -17,31 +17,35 @@ struct Output {
 /// For more information, see the output of
 /// <https://doc.rust-lang.org/nightly/cargo/commands/cargo-metadata.html>
 #[derive(Debug, Deserialize)]
-struct Package {
-    name: String,
-    source: Option<String>,
-    manifest_path: String,
-    dependencies: Vec<Dependency>,
-    targets: Vec<Target>,
+pub(crate) struct Package {
+    pub(crate) name: String,
+    pub(crate) source: Option<String>,
+    pub(crate) manifest_path: String,
+    pub(crate) dependencies: Vec<Dependency>,
+    pub(crate) targets: Vec<Target>,
 }
 
 /// For more information, see the output of
 /// <https://doc.rust-lang.org/nightly/cargo/commands/cargo-metadata.html>
-#[derive(Debug, Deserialize)]
-struct Dependency {
-    name: String,
-    source: Option<String>,
+#[derive(Debug, Deserialize, PartialEq)]
+pub(crate) struct Dependency {
+    pub(crate) name: String,
+    pub(crate) source: Option<String>,
 }
 
 #[derive(Debug, Deserialize)]
-struct Target {
-    kind: Vec<String>,
+pub(crate) struct Target {
+    pub(crate) name: String,
+    pub(crate) kind: Vec<String>,
+    pub(crate) crate_types: Vec<String>,
+    pub(crate) src_path: String,
+    pub(crate) edition: String,
 }
 
 /// Collects and stores package metadata of each workspace members into `build`,
 /// by executing `cargo metadata` commands.
 pub fn build(build: &mut Build) {
-    for package in workspace_members(build) {
+    for package in workspace_members(&build.config) {
         if package.source.is_none() {
             let name = INTERNER.intern_string(package.name);
             let mut path = PathBuf::from(package.manifest_path);
@@ -70,9 +74,9 @@ pub fn build(build: &mut Build) {
 ///
 /// Note that `src/tools/cargo` is no longer a workspace member but we still
 /// treat it as one here, by invoking an additional `cargo metadata` command.
-fn workspace_members(build: &Build) -> impl Iterator<Item = Package> {
+pub(crate) fn workspace_members(config: &Config) -> impl Iterator<Item = Package> {
     let collect_metadata = |manifest_path| {
-        let mut cargo = Command::new(&build.initial_cargo);
+        let mut cargo = Command::new(&config.initial_cargo);
         cargo
             // Will read the libstd Cargo.toml
             // which uses the unstable `public-dependency` feature.
@@ -82,7 +86,37 @@ fn workspace_members(build: &Build) -> impl Iterator<Item = Package> {
             .arg("1")
             .arg("--no-deps")
             .arg("--manifest-path")
-            .arg(build.src.join(manifest_path));
+            .arg(config.src.join(manifest_path));
+        let metadata_output = output(&mut cargo);
+        let Output { packages, .. } = t!(serde_json::from_str(&metadata_output));
+        packages
+    };
+
+    // Collects `metadata.packages` from all workspaces.
+    let packages = collect_metadata("Cargo.toml");
+    let cargo_packages = collect_metadata("src/tools/cargo/Cargo.toml");
+    let ra_packages = collect_metadata("src/tools/rust-analyzer/Cargo.toml");
+    let bootstrap_packages = collect_metadata("src/bootstrap/Cargo.toml");
+
+    // We only care about the root package from `src/tool/cargo` workspace.
+    let cargo_package = cargo_packages.into_iter().find(|pkg| pkg.name == "cargo").into_iter();
+
+    packages.into_iter().chain(cargo_package).chain(ra_packages).chain(bootstrap_packages)
+}
+
+/// Invokes `cargo metadata` to get package metadata of whole workspace including the dependencies.
+pub(crate) fn project_metadata(config: &Config) -> impl Iterator<Item = Package> {
+    let collect_metadata = |manifest_path| {
+        let mut cargo = Command::new(&config.initial_cargo);
+        cargo
+            // Will read the libstd Cargo.toml
+            // which uses the unstable `public-dependency` feature.
+            .env("RUSTC_BOOTSTRAP", "1")
+            .arg("metadata")
+            .arg("--format-version")
+            .arg("1")
+            .arg("--manifest-path")
+            .arg(config.src.join(manifest_path));
         let metadata_output = output(&mut cargo);
         let Output { packages, .. } = t!(serde_json::from_str(&metadata_output));
         packages

src/bootstrap/src/utils/mod.rs

@@ -12,5 +12,6 @@ pub(crate) mod helpers;
 pub(crate) mod job;
 #[cfg(feature = "build-metrics")]
 pub(crate) mod metrics;
+pub(crate) mod ra_project;
 pub(crate) mod render_tests;
 pub(crate) mod tarball;

src/bootstrap/src/utils/ra_project.rs

@@ -0,0 +1,155 @@
+//! This module contains the implementation for generating rust-project.json data which can be
+//! utilized for LSPs (Language Server Protocols).
+//!
+//! The primary reason for relying on rust-analyzer.json instead of the default rust-analyzer
+//! is because rust-analyzer is not so capable of handling rust-lang/rust workspaces out of the box.
+//! It often encounters new issues while trying to fix current problems with some hacky workarounds.
+//!
+//! For additional context, see the [zulip thread].
+//!
+//! [zulip thread]: https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/r-a.20support.20for.20rust-lang.2Frust.20via.20project-rust.2Ejson/near/412505824
+
+use serde_derive::Serialize;
+use std::collections::{BTreeMap, BTreeSet};
+use std::io;
+use std::path::Path;
+
+use crate::core::metadata::{project_metadata, workspace_members, Dependency};
+use crate::Config;
+
+#[derive(Debug, Serialize)]
+/// FIXME(before-merge): doc-comment
+pub(crate) struct RustAnalyzerProject {
+    crates: Vec<Crate>,
+    sysroot: String,
+    sysroot_src: String,
+}
+
+#[derive(Debug, Default, Serialize, PartialEq)]
+struct Crate {
+    cfg: Vec<String>,
+    deps: BTreeSet<Dep>,
+    display_name: String,
+    edition: String,
+    env: BTreeMap<String, String>,
+    is_proc_macro: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    proc_macro_dylib_path: Option<String>,
+    is_workspace_member: bool,
+    root_module: String,
+}
+
+#[derive(Debug, Default, Serialize, PartialEq, PartialOrd, Ord, Eq)]
+struct Dep {
+    #[serde(rename = "crate")]
+    crate_index: usize,
+    name: String,
+}
+
+impl RustAnalyzerProject {
+    #[allow(dead_code)] // FIXME(before-merge): remove this
+    pub(crate) fn collect_ra_project_data(config: &Config) -> Self {
+        let mut ra_project = RustAnalyzerProject {
+            crates: vec![],
+            sysroot: format!("{}", config.out.join("host").join("stage0").display()),
+            sysroot_src: format!("{}", config.src.join("library").display()),
+        };
+
+        let packages: Vec<_> = project_metadata(config).collect();
+        let workspace_members: Vec<_> = workspace_members(config).collect();
+
+        for package in &packages {
+            let is_not_indirect_dependency = packages
+                .iter()
+                .filter(|t| {
+                    let used_from_other_crates = t.dependencies.contains(&Dependency {
+                        name: package.name.clone(),
+                        source: package.source.clone(),
+                    });
+
+                    let is_local = t.source.is_none();
+
+                    (used_from_other_crates && is_local) || package.source.is_none()
+                })
+                .next()
+                .is_some();
+
+            if !is_not_indirect_dependency {
+                continue;
+            }
+
+            for target in &package.targets {
+                let mut krate = Crate::default();
+                krate.display_name = target.name.clone();
+                krate.root_module = target.src_path.clone();
+                krate.edition = target.edition.clone();
+                krate.is_workspace_member = workspace_members.iter().any(|p| p.name == target.name);
+                krate.is_proc_macro = target.crate_types.contains(&"proc-macro".to_string());
+
+                // FIXME(before-merge): We need to figure out how to find proc-macro dylibs.
+                // if krate.is_proc_macro {
+                //     krate.proc_macro_dylib_path =
+                // }
+
+                krate.env.insert("RUSTC_BOOTSTRAP".into(), "1".into());
+
+                if target
+                    .src_path
+                    .starts_with(&config.src.join("library").to_string_lossy().to_string())
+                {
+                    krate.cfg.push("bootstrap".into());
+                }
+
+                ra_project.crates.push(krate);
+            }
+        }
+
+        ra_project.crates.sort_by_key(|c| c.display_name.clone());
+        ra_project.crates.dedup_by_key(|c| c.display_name.clone());
+
+        // Find and fill dependencies of crates.
+        for package in packages {
+            if package.dependencies.is_empty() {
+                continue;
+            }
+
+            for dependency in package.dependencies {
+                if let Some(index) =
+                    ra_project.crates.iter().position(|c| c.display_name == package.name)
+                {
+                    if let Some(dependency_index) =
+                        ra_project.crates.iter().position(|c| c.display_name == dependency.name)
+                    {
+                        // no need to find indirect dependencies of direct dependencies, just continue
+                        if ra_project.crates[index].root_module.contains(".cargo/registry") {
+                            continue;
+                        }
+
+                        let dependency_name = dependency.name.replace('-', "_").to_lowercase();
+
+                        ra_project.crates[index]
+                            .deps
+                            .insert(Dep { crate_index: dependency_index, name: dependency_name });
+                    }
+                }
+            }
+        }
+
+        ra_project
+    }
+
+    #[allow(dead_code)] // FIXME(before-merge): remove this
+    pub(crate) fn generate_file(&self, path: &Path) -> io::Result<()> {
+        if path.exists() {
+            return Err(io::Error::new(
+                io::ErrorKind::AlreadyExists,
+                format!("File '{}' already exists.", path.display()),
+            ));
+        }
+
+        let mut file = std::fs::File::create(path)?;
+        serde_json::to_writer_pretty(&mut file, self)?;
+
+        Ok(())
+    }
+}