build-manifest: stop receiving release numbers from bootstrap

pietroalbini · pietroalbini · commit 3bddfea7e233 · 2020-09-18T17:42:58.000+02:00
diff --git a/Cargo.lock b/Cargo.lock
@@ -230,6 +230,7 @@ dependencies = [
 name = "build-manifest"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
  "serde",
  "serde_json",
  "toml",
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
@@ -2356,15 +2356,9 @@ impl Step for HashSign {
         cmd.arg(sign);
         cmd.arg(distdir(builder));
         cmd.arg(today.trim());
-        cmd.arg(builder.rust_package_vers());
         cmd.arg(addr);
-        cmd.arg(builder.package_vers(&builder.release_num("cargo")));
-        cmd.arg(builder.package_vers(&builder.release_num("rls")));
-        cmd.arg(builder.package_vers(&builder.release_num("rust-analyzer/crates/rust-analyzer")));
-        cmd.arg(builder.package_vers(&builder.release_num("clippy")));
-        cmd.arg(builder.package_vers(&builder.release_num("miri")));
-        cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
-        cmd.arg(builder.llvm_tools_package_vers());
+        cmd.arg(&builder.config.channel);
+        cmd.arg(&builder.src);
 
         builder.create_dir(&distdir(builder));
 
diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml
@@ -8,3 +8,4 @@ edition = "2018"
 toml = "0.5"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+anyhow = "1.0.32"
diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md
@@ -21,10 +21,9 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to
 
 ```
 $ BUILD_MANIFEST_DISABLE_SIGNING=1 cargo +nightly run \
-    path/to/dist path/to/output 1970-01-01 \
-    nightly nightly nightly nightly nightly nightly nightly nightly \
-    http://example.com
+    path/to/dist path/to/output 1970-01-01 http://example.com \
+    CHANNEL path/to/rust/repo
 ```
 
-In the future, if the tool complains about missing arguments just add more
-`nightly`s in the middle.
+Remember to replace `CHANNEL` with the channel you produced dist artifacts of
+and `path/to/rust/repo` with the path to your checkout of the Rust repository.
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
@@ -6,7 +6,7 @@
 
 mod versions;
 
-use crate::versions::PkgType;
+use crate::versions::{PkgType, Versions};
 use serde::Serialize;
 use std::collections::BTreeMap;
 use std::collections::HashMap;
@@ -227,14 +227,7 @@ macro_rules! t {
 }
 
 struct Builder {
-    rust_release: String,
-    cargo_release: String,
-    rls_release: String,
-    rust_analyzer_release: String,
-    clippy_release: String,
-    rustfmt_release: String,
-    llvm_tools_release: String,
-    miri_release: String,
+    versions: Versions,
 
     input: PathBuf,
     output: PathBuf,
@@ -281,15 +274,9 @@ fn main() {
     let input = PathBuf::from(args.next().unwrap());
     let output = PathBuf::from(args.next().unwrap());
     let date = args.next().unwrap();
-    let rust_release = args.next().unwrap();
     let s3_address = args.next().unwrap();
-    let cargo_release = args.next().unwrap();
-    let rls_release = args.next().unwrap();
-    let rust_analyzer_release = args.next().unwrap();
-    let clippy_release = args.next().unwrap();
-    let miri_release = args.next().unwrap();
-    let rustfmt_release = args.next().unwrap();
-    let llvm_tools_release = args.next().unwrap();
+    let channel = args.next().unwrap();
+    let monorepo_path = args.next().unwrap();
 
     // Do not ask for a passphrase while manually testing
     let mut passphrase = String::new();
@@ -299,14 +286,7 @@ fn main() {
     }
 
     Builder {
-        rust_release,
-        cargo_release,
-        rls_release,
-        rust_analyzer_release,
-        clippy_release,
-        rustfmt_release,
-        llvm_tools_release,
-        miri_release,
+        versions: Versions::new(&channel, Path::new(&monorepo_path)).unwrap(),
 
         input,
         output,
@@ -363,10 +343,11 @@ impl Builder {
         self.check_toolstate();
         self.digest_and_sign();
         let manifest = self.build_manifest();
-        self.write_channel_files(&self.rust_release, &manifest);
 
-        if self.rust_release != "beta" && self.rust_release != "nightly" {
-            self.write_channel_files("stable", &manifest);
+        let rust_version = self.versions.package_version(&PkgType::Rust).unwrap();
+        self.write_channel_files(self.versions.channel(), &manifest);
+        if self.versions.channel() != rust_version {
+            self.write_channel_files(&rust_version, &manifest);
         }
     }
 
@@ -473,7 +454,7 @@ impl Builder {
         // The compiler libraries are not stable for end users, and they're also huge, so we only
         // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
         // for users to install the additional component manually, if needed.
-        if self.rust_release == "nightly" {
+        if self.versions.channel() == "nightly" {
             self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
             self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]);
         }
@@ -511,7 +492,7 @@ impl Builder {
     }
 
     fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
-        let filename = self.filename("rust", host);
+        let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap();
         let digest = self.digests.remove(&filename)?;
         let xz_filename = filename.replace(".tar.gz", ".tar.xz");
         let xz_digest = self.digests.remove(&xz_filename);
@@ -610,7 +591,7 @@ impl Builder {
             .unwrap_or_default(); // `is_present` defaults to `false` here.
 
         // Never ship nightly-only components for other trains.
-        if self.rust_release != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
+        if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
             is_present = false; // Pretend the component is entirely missing.
         }
 
@@ -619,7 +600,10 @@ impl Builder {
             .map(|name| {
                 if is_present {
                     // The component generally exists, but it might still be missing for this target.
-                    let filename = self.filename(pkgname, name);
+                    let filename = self
+                        .versions
+                        .tarball_name(&PkgType::from_component(pkgname), name)
+                        .unwrap();
                     let digest = match self.digests.remove(&filename) {
                         Some(digest) => digest,
                         // This component does not exist for this target -- skip it.
@@ -662,23 +646,6 @@ impl Builder {
         format!("{}/{}/{}", self.s3_address, self.date, filename)
     }
 
-    fn filename(&self, component: &str, target: &str) -> String {
-        use PkgType::*;
-        match PkgType::from_component(component) {
-            RustSrc => format!("rust-src-{}.tar.gz", self.rust_release),
-            Cargo => format!("cargo-{}-{}.tar.gz", self.cargo_release, target),
-            Rls => format!("rls-{}-{}.tar.gz", self.rls_release, target),
-            RustAnalyzer => {
-                format!("rust-analyzer-{}-{}.tar.gz", self.rust_analyzer_release, target)
-            }
-            Clippy => format!("clippy-{}-{}.tar.gz", self.clippy_release, target),
-            Rustfmt => format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target),
-            LlvmTools => format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target),
-            Miri => format!("miri-{}-{}.tar.gz", self.miri_release, target),
-            Other(_) => format!("{}-{}-{}.tar.gz", component, self.rust_release, target),
-        }
-    }
-
     fn cached_version(&self, component: &str) -> &Option<String> {
         use PkgType::*;
         match PkgType::from_component(component) {
@@ -707,20 +674,24 @@ impl Builder {
         }
     }
 
-    fn version(&self, component: &str, target: &str) -> Option<String> {
+    fn version(&mut self, component: &str, target: &str) -> Option<String> {
         self.untar(component, target, |filename| format!("{}/version", filename))
     }
 
-    fn git_commit_hash(&self, component: &str, target: &str) -> Option<String> {
+    fn git_commit_hash(&mut self, component: &str, target: &str) -> Option<String> {
         self.untar(component, target, |filename| format!("{}/git-commit-hash", filename))
     }
 
-    fn untar<F>(&self, component: &str, target: &str, dir: F) -> Option<String>
+    fn untar<F>(&mut self, component: &str, target: &str, dir: F) -> Option<String>
     where
         F: FnOnce(String) -> String,
     {
+        let filename = self
+            .versions
+            .tarball_name(&PkgType::from_component(component), target)
+            .expect("failed to retrieve the tarball path");
+
         let mut cmd = Command::new("tar");
-        let filename = self.filename(component, target);
         cmd.arg("xf")
             .arg(self.input.join(&filename))
             .arg(dir(filename.replace(".tar.gz", "")))
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
@@ -1,4 +1,10 @@
+use anyhow::{Context, Error};
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+#[derive(Debug, Hash, Eq, PartialEq, Clone)]
 pub(crate) enum PkgType {
+    Rust,
     RustSrc,
     Cargo,
     Rls,
@@ -13,6 +19,7 @@ pub(crate) enum PkgType {
 impl PkgType {
     pub(crate) fn from_component(component: &str) -> Self {
         match component {
+            "rust" => PkgType::Rust,
             "rust-src" => PkgType::RustSrc,
             "cargo" => PkgType::Cargo,
             "rls" | "rls-preview" => PkgType::Rls,
@@ -24,4 +31,111 @@ impl PkgType {
             other => PkgType::Other(other.into()),
         }
     }
+
+    fn rust_monorepo_path(&self) -> Option<&'static str> {
+        match self {
+            PkgType::Cargo => Some("src/tools/cargo"),
+            PkgType::Rls => Some("src/tools/rls"),
+            PkgType::RustAnalyzer => Some("src/tools/rust-analyzer/crates/rust-analyzer"),
+            PkgType::Clippy => Some("src/tools/clippy"),
+            PkgType::Rustfmt => Some("src/tools/rustfmt"),
+            PkgType::Miri => Some("src/tools/miri"),
+            PkgType::Rust => None,
+            PkgType::RustSrc => None,
+            PkgType::LlvmTools => None,
+            PkgType::Other(_) => None,
+        }
+    }
+
+    fn tarball_component_name(&self) -> &str {
+        match self {
+            PkgType::Rust => "rust",
+            PkgType::RustSrc => "rust-src",
+            PkgType::Cargo => "cargo",
+            PkgType::Rls => "rls",
+            PkgType::RustAnalyzer => "rust-analyzer",
+            PkgType::Clippy => "clippy",
+            PkgType::Rustfmt => "rustfmt",
+            PkgType::LlvmTools => "llvm-tools",
+            PkgType::Miri => "miri",
+            PkgType::Other(component) => component,
+        }
+    }
+}
+
+pub(crate) struct Versions {
+    channel: String,
+    rustc_version: String,
+    monorepo_root: PathBuf,
+    package_versions: HashMap<PkgType, String>,
+}
+
+impl Versions {
+    pub(crate) fn new(channel: &str, monorepo_root: &Path) -> Result<Self, Error> {
+        Ok(Self {
+            channel: channel.into(),
+            rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version"))
+                .context("failed to read the rustc version from src/version")?
+                .trim()
+                .to_string(),
+            monorepo_root: monorepo_root.into(),
+            package_versions: HashMap::new(),
+        })
+    }
+
+    pub(crate) fn channel(&self) -> &str {
+        &self.channel
+    }
+
+    pub(crate) fn tarball_name(
+        &mut self,
+        package: &PkgType,
+        target: &str,
+    ) -> Result<String, Error> {
+        Ok(format!(
+            "{}-{}-{}.tar.gz",
+            package.tarball_component_name(),
+            self.package_version(package).with_context(|| format!(
+                "failed to get the package version for component {:?}",
+                package,
+            ))?,
+            target
+        ))
+    }
+
+    pub(crate) fn package_version(&mut self, package: &PkgType) -> Result<String, Error> {
+        match self.package_versions.get(package) {
+            Some(release) => Ok(release.clone()),
+            None => {
+                let version = match package.rust_monorepo_path() {
+                    Some(path) => {
+                        let path = self.monorepo_root.join(path).join("Cargo.toml");
+                        let cargo_toml: CargoToml = toml::from_slice(&std::fs::read(path)?)?;
+                        cargo_toml.package.version
+                    }
+                    None => self.rustc_version.clone(),
+                };
+
+                let release = match self.channel.as_str() {
+                    "stable" => version,
+                    "beta" => "beta".into(),
+                    "nightly" => "nightly".into(),
+                    _ => format!("{}-dev", version),
+                };
+
+                self.package_versions.insert(package.clone(), release.clone());
+                Ok(release)
+            }
+        }
+    }
+}
+
+#[derive(serde::Deserialize)]
+struct CargoToml {
+    package: CargoTomlPackage,
+}
+
+#[derive(serde::Deserialize)]
+struct CargoTomlPackage {
+    version: String,
 }
