Skip to content

Commit c3bf2fc

Browse files
KobzolKobzol
committed
Implement downloading GCC from CI
1 parent 624ba30 commit c3bf2fc

File tree

2 files changed

+211
-111
lines changed

2 files changed

+211
-111
lines changed

‎src/bootstrap/src/core/build_steps/gcc.rs‎

Lines changed: 183 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,70 @@ use std::path::{Path, PathBuf};
1313
use std::sync::OnceLock;
1414

1515
use build_helper::ci::CiEnv;
16+
use build_helper::git::get_closest_merge_commit;
1617

17-
use crate::Kind;
18-
use crate::core::builder::{Builder, Cargo, RunConfig, ShouldRun, Step};
19-
use crate::core::config::TargetSelection;
18+
use crate::Config;
19+
use crate::core::builder::{Builder, Cargo, Kind, RunConfig, ShouldRun, Step};
20+
use crate::core::config::{GccCiMode, TargetSelection};
2021
use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash};
2122
use crate::utils::exec::command;
2223
use crate::utils::helpers::{self, t};
2324

25+
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
26+
pub struct Gcc {
27+
pub target: TargetSelection,
28+
}
29+
30+
#[derive(Clone)]
31+
pub struct GccOutput {
32+
pub libgccjit: PathBuf,
33+
}
34+
35+
impl Step for Gcc {
36+
type Output = GccOutput;
37+
38+
const ONLY_HOSTS: bool = true;
39+
40+
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
41+
run.path("src/gcc").alias("gcc")
42+
}
43+
44+
fn make_run(run: RunConfig<'_>) {
45+
run.builder.ensure(Gcc { target: run.target });
46+
}
47+
48+
/// Compile GCC (specifically `libgccjit`) for `target`.
49+
fn run(self, builder: &Builder<'_>) -> Self::Output {
50+
let target = self.target;
51+
52+
// If GCC has already been built, we avoid building it again.
53+
let metadata = match get_gcc_build_status(builder, target) {
54+
GccBuildStatus::AlreadyBuilt(path) => return GccOutput { libgccjit: path },
55+
GccBuildStatus::ShouldBuild(m) => m,
56+
};
57+
58+
let _guard = builder.msg_unstaged(Kind::Build, "GCC", target);
59+
t!(metadata.stamp.remove());
60+
let _time = helpers::timeit(builder);
61+
62+
let libgccjit_path = libgccjit_built_path(&metadata.install_dir);
63+
if builder.config.dry_run() {
64+
return GccOutput { libgccjit: libgccjit_path };
65+
}
66+
67+
build_gcc(&metadata, builder, target);
68+
69+
let lib_alias = metadata.install_dir.join("lib/libgccjit.so.0");
70+
if !lib_alias.exists() {
71+
t!(builder.symlink_file(&libgccjit_path, lib_alias));
72+
}
73+
74+
t!(metadata.stamp.write());
75+
76+
GccOutput { libgccjit: libgccjit_path }
77+
}
78+
}
79+
2480
pub struct Meta {
2581
stamp: BuildStamp,
2682
out_dir: PathBuf,
@@ -34,17 +90,39 @@ pub enum GccBuildStatus {
3490
ShouldBuild(Meta),
3591
}
3692

37-
/// This returns whether we've already previously built GCC.
93+
/// Tries to download GCC from CI if it is enabled and GCC artifacts
94+
/// are available for the given target.
95+
/// Returns a path to the libgccjit.so file.
96+
fn try_download_gcc(builder: &Builder<'_>, target: TargetSelection) -> Option<PathBuf> {
97+
// Try to download GCC from CI if configured and available
98+
if !matches!(builder.config.gcc_ci_mode, GccCiMode::DownloadFromCi) {
99+
return None;
100+
}
101+
if target != "x86_64-unknown-linux-gnu" {
102+
eprintln!("GCC CI download is only available for the `x86_64-unknown-linux-gnu` target");
103+
return None;
104+
}
105+
let sha =
106+
detect_gcc_sha(&builder.config, builder.config.rust_info.is_managed_git_subrepository());
107+
let root = ci_gcc_root(&builder.config);
108+
let gcc_stamp = BuildStamp::new(&root).with_prefix("gcc").add_stamp(&sha);
109+
if !gcc_stamp.is_up_to_date() && !builder.config.dry_run() {
110+
builder.config.download_ci_gcc(&sha, &root);
111+
t!(gcc_stamp.write());
112+
}
113+
// FIXME: put libgccjit.so into a lib directory in dist::Gcc
114+
Some(root.join("libgccjit.so"))
115+
}
116+
117+
/// This returns information about whether GCC should be built or if it's already built.
118+
/// It transparently handles downloading GCC from CI if needed.
38119
///
39120
/// It's used to avoid busting caches during x.py check -- if we've already built
40121
/// GCC, it's fine for us to not try to avoid doing so.
41-
pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> GccBuildStatus {
42-
// Initialize the gcc submodule if not initialized already.
43-
builder.config.update_submodule("src/gcc");
44-
45-
let root = builder.src.join("src/gcc");
46-
let out_dir = builder.gcc_out(target).join("build");
47-
let install_dir = builder.gcc_out(target).join("install");
122+
pub fn get_gcc_build_status(builder: &Builder<'_>, target: TargetSelection) -> GccBuildStatus {
123+
if let Some(path) = try_download_gcc(builder, target) {
124+
return GccBuildStatus::AlreadyBuilt(path);
125+
}
48126

49127
static STAMP_HASH_MEMO: OnceLock<String> = OnceLock::new();
50128
let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| {
@@ -55,6 +133,13 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc
55133
)
56134
});
57135

136+
// Initialize the gcc submodule if not initialized already.
137+
builder.config.update_submodule("src/gcc");
138+
139+
let root = builder.src.join("src/gcc");
140+
let out_dir = builder.gcc_out(target).join("build");
141+
let install_dir = builder.gcc_out(target).join("install");
142+
58143
let stamp = BuildStamp::new(&out_dir).with_prefix("gcc").add_stamp(smart_stamp_hash);
59144

60145
if stamp.is_up_to_date() {
@@ -87,120 +172,107 @@ fn libgccjit_built_path(install_dir: &Path) -> PathBuf {
87172
install_dir.join("lib/libgccjit.so")
88173
}
89174

90-
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
91-
pub struct Gcc {
92-
pub target: TargetSelection,
93-
}
94-
95-
#[derive(Clone)]
96-
pub struct GccOutput {
97-
pub libgccjit: PathBuf,
98-
}
175+
fn build_gcc(metadata: &Meta, builder: &Builder<'_>, target: TargetSelection) {
176+
let Meta { stamp: _, out_dir, install_dir, root } = metadata;
99177

100-
impl Step for Gcc {
101-
type Output = GccOutput;
102-
103-
const ONLY_HOSTS: bool = true;
104-
105-
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
106-
run.path("src/gcc").alias("gcc")
107-
}
108-
109-
fn make_run(run: RunConfig<'_>) {
110-
run.builder.ensure(Gcc { target: run.target });
111-
}
112-
113-
/// Compile GCC (specifically `libgccjit`) for `target`.
114-
fn run(self, builder: &Builder<'_>) -> Self::Output {
115-
let target = self.target;
178+
t!(fs::create_dir_all(out_dir));
179+
t!(fs::create_dir_all(install_dir));
116180

117-
// If GCC has already been built, we avoid building it again.
118-
let Meta { stamp, out_dir, install_dir, root } = match prebuilt_gcc_config(builder, target)
119-
{
120-
GccBuildStatus::AlreadyBuilt(path) => return GccOutput { libgccjit: path },
121-
GccBuildStatus::ShouldBuild(m) => m,
122-
};
123-
124-
let _guard = builder.msg_unstaged(Kind::Build, "GCC", target);
125-
t!(stamp.remove());
126-
let _time = helpers::timeit(builder);
127-
t!(fs::create_dir_all(&out_dir));
128-
129-
let libgccjit_path = libgccjit_built_path(&install_dir);
130-
if builder.config.dry_run() {
131-
return GccOutput { libgccjit: libgccjit_path };
181+
// GCC creates files (e.g. symlinks to the downloaded dependencies)
182+
// in the source directory, which does not work with our CI setup, where we mount
183+
// source directories as read-only on Linux.
184+
// Therefore, as a part of the build in CI, we first copy the whole source directory
185+
// to the build directory, and perform the build from there.
186+
let src_dir = if CiEnv::is_ci() {
187+
let src_dir = builder.gcc_out(target).join("src");
188+
if src_dir.exists() {
189+
builder.remove_dir(&src_dir);
132190
}
191+
builder.create_dir(&src_dir);
192+
builder.cp_link_r(root, &src_dir);
193+
src_dir
194+
} else {
195+
root.clone()
196+
};
133197

134-
// GCC creates files (e.g. symlinks to the downloaded dependencies)
135-
// in the source directory, which does not work with our CI setup, where we mount
136-
// source directories as read-only on Linux.
137-
// Therefore, as a part of the build in CI, we first copy the whole source directory
138-
// to the build directory, and perform the build from there.
139-
let src_dir = if CiEnv::is_ci() {
140-
let src_dir = builder.gcc_out(target).join("src");
141-
if src_dir.exists() {
142-
builder.remove_dir(&src_dir);
143-
}
144-
builder.create_dir(&src_dir);
145-
builder.cp_link_r(&root, &src_dir);
146-
src_dir
147-
} else {
148-
root
149-
};
198+
command(src_dir.join("contrib/download_prerequisites")).current_dir(&src_dir).run(builder);
199+
let mut configure_cmd = command(src_dir.join("configure"));
200+
configure_cmd
201+
.current_dir(out_dir)
202+
// On CI, we compile GCC with Clang.
203+
// The -Wno-everything flag is needed to make GCC compile with Clang 19.
204+
// `-g -O2` are the default flags that are otherwise used by Make.
205+
// FIXME(kobzol): change the flags once we have [gcc] configuration in config.toml.
206+
.env("CXXFLAGS", "-Wno-everything -g -O2")
207+
.env("CFLAGS", "-Wno-everything -g -O2")
208+
.arg("--enable-host-shared")
209+
.arg("--enable-languages=jit")
210+
.arg("--enable-checking=release")
211+
.arg("--disable-bootstrap")
212+
.arg("--disable-multilib")
213+
.arg(format!("--prefix={}", install_dir.display()));
214+
let cc = builder.build.cc(target).display().to_string();
215+
let cc = builder
216+
.build
217+
.config
218+
.ccache
219+
.as_ref()
220+
.map_or_else(|| cc.clone(), |ccache| format!("{ccache} {cc}"));
221+
configure_cmd.env("CC", cc);
150222

151-
command(src_dir.join("contrib/download_prerequisites")).current_dir(&src_dir).run(builder);
152-
let mut configure_cmd = command(src_dir.join("configure"));
153-
configure_cmd
154-
.current_dir(&out_dir)
155-
// On CI, we compile GCC with Clang.
156-
// The -Wno-everything flag is needed to make GCC compile with Clang 19.
157-
// `-g -O2` are the default flags that are otherwise used by Make.
158-
// FIXME(kobzol): change the flags once we have [gcc] configuration in config.toml.
159-
.env("CXXFLAGS", "-Wno-everything -g -O2")
160-
.env("CFLAGS", "-Wno-everything -g -O2")
161-
.arg("--enable-host-shared")
162-
.arg("--enable-languages=jit")
163-
.arg("--enable-checking=release")
164-
.arg("--disable-bootstrap")
165-
.arg("--disable-multilib")
166-
.arg(format!("--prefix={}", install_dir.display()));
167-
let cc = builder.build.cc(target).display().to_string();
168-
let cc = builder
223+
if let Ok(ref cxx) = builder.build.cxx(target) {
224+
let cxx = cxx.display().to_string();
225+
let cxx = builder
169226
.build
170227
.config
171228
.ccache
172229
.as_ref()
173-
.map_or_else(|| cc.clone(), |ccache| format!("{ccache} {cc}"));
174-
configure_cmd.env("CC", cc);
175-
176-
if let Ok(ref cxx) = builder.build.cxx(target) {
177-
let cxx = cxx.display().to_string();
178-
let cxx = builder
179-
.build
180-
.config
181-
.ccache
182-
.as_ref()
183-
.map_or_else(|| cxx.clone(), |ccache| format!("{ccache} {cxx}"));
184-
configure_cmd.env("CXX", cxx);
185-
}
186-
configure_cmd.run(builder);
187-
188-
command("make").current_dir(&out_dir).arg(format!("-j{}", builder.jobs())).run(builder);
189-
command("make").current_dir(&out_dir).arg("install").run(builder);
190-
191-
let lib_alias = install_dir.join("lib/libgccjit.so.0");
192-
if !lib_alias.exists() {
193-
t!(builder.symlink_file(&libgccjit_path, lib_alias));
194-
}
195-
196-
t!(stamp.write());
197-
198-
GccOutput { libgccjit: libgccjit_path }
230+
.map_or_else(|| cxx.clone(), |ccache| format!("{ccache} {cxx}"));
231+
configure_cmd.env("CXX", cxx);
199232
}
233+
configure_cmd.run(builder);
234+
235+
command("make")
236+
.current_dir(out_dir)
237+
.arg("--silent")
238+
.arg(format!("-j{}", builder.jobs()))
239+
.run_capture_stdout(builder);
240+
command("make").current_dir(out_dir).arg("--silent").arg("install").run_capture_stdout(builder);
200241
}
201242

202243
/// Configures a Cargo invocation so that it can build the GCC codegen backend.
203244
pub fn add_cg_gcc_cargo_flags(cargo: &mut Cargo, gcc: &GccOutput) {
204245
// Add the path to libgccjit.so to the linker search paths.
205246
cargo.rustflag(&format!("-L{}", gcc.libgccjit.parent().unwrap().to_str().unwrap()));
206247
}
248+
249+
/// The absolute path to the downloaded GCC artifacts.
250+
fn ci_gcc_root(config: &Config) -> PathBuf {
251+
config.out.join(config.build).join("ci-gcc")
252+
}
253+
254+
/// This retrieves the GCC sha we *want* to use, according to git history.
255+
fn detect_gcc_sha(config: &Config, is_git: bool) -> String {
256+
let gcc_sha = if is_git {
257+
get_closest_merge_commit(
258+
Some(&config.src),
259+
&config.git_config(),
260+
&[config.src.join("src/gcc"), config.src.join("src/bootstrap/download-ci-gcc-stamp")],
261+
)
262+
.unwrap()
263+
} else if let Some(info) = crate::utils::channel::read_commit_info_file(&config.src) {
264+
info.sha.trim().to_owned()
265+
} else {
266+
"".to_owned()
267+
};
268+
269+
if gcc_sha.is_empty() {
270+
eprintln!("error: could not find commit hash for downloading GCC");
271+
eprintln!("HELP: maybe your repository history is too shallow?");
272+
eprintln!("HELP: consider disabling `download-ci-gcc`");
273+
eprintln!("HELP: or fetch enough history to include one upstream commit");
274+
panic!();
275+
}
276+
277+
gcc_sha
278+
}

‎src/bootstrap/src/core/download.rs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -826,6 +826,34 @@ download-rustc = false
826826
let llvm_root = self.ci_llvm_root();
827827
self.unpack(&tarball, &llvm_root, "rust-dev");
828828
}
829+
830+
pub fn download_ci_gcc(&self, gcc_sha: &str, root_dir: &Path) {
831+
let cache_prefix = format!("gcc-{gcc_sha}");
832+
let cache_dst =
833+
self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
834+
835+
let gcc_cache = cache_dst.join(cache_prefix);
836+
if !gcc_cache.exists() {
837+
t!(fs::create_dir_all(&gcc_cache));
838+
}
839+
let base = &self.stage0_metadata.config.artifacts_server;
840+
let filename = format!("gcc-nightly-{}.tar.xz", self.build.triple);
841+
let tarball = gcc_cache.join(&filename);
842+
if !tarball.exists() {
843+
let help_on_error = "ERROR: failed to download gcc from ci
844+
845+
HELP: There could be two reasons behind this:
846+
1) The host triple is not supported for `download-ci-gcc`.
847+
2) Old builds get deleted after a certain time.
848+
HELP: In either case, disable `download-ci-gcc` in your config.toml:
849+
850+
[gcc]
851+
download-ci-gcc = false
852+
";
853+
self.download_file(&format!("{base}/{gcc_sha}/{filename}"), &tarball, help_on_error);
854+
}
855+
self.unpack(&tarball, root_dir, "gcc");
856+
}
829857
}
830858

831859
fn path_is_dylib(path: &Path) -> bool {

0 commit comments

Comments
 (0)