Skip to content

Commit c007d0a

Browse files
matthiaskrgrmatthiaskrgr
authored
Rollup merge of #138051 - Kobzol:download-ci-gcc, r=onur-ozkan
Add support for downloading GCC from CI This PR adds a new bootstrap config section called `gcc` and implements a single config `download-ci-gcc`. Its behavior is similar to `download-ci-llvm`. Since #137667, we distribute a CI component that contains the prebuilt `libgccjit.so` library on x64 Linux. With `download-ci-gcc`, this component is downloaded from CI to avoid building GCC locally. This is an MVP of this functionality, designed for local usage. This PR does not enable this functionality on the LLVM 18 PR CI job which builds `cg_gcc`, and does not implement more complex detection logic. It simply uses `false` (build locally) or `true` (download from CI if you're on the right target, if CI download fails, then bootstrap fails). The original LLVM CI download functionality has a lot of features and complexity, which we don't need for GCC (yet). I don't like how the LLVM CI stuff is threaded through multiple parts of bootstrap, so with GCC I would like to take a more centralized approach, where the `build::Gcc` step handles download from CI internally. This means that: - For the rest of bootstrap, it should be transparent whether GCC was built locally or downloaded from CI. - GCC is not downloaded eagerly unless you actually requested GCC (either you requested `x build gcc` or you asked to build/test the GCC backend). This approach will require some modifications once we extend this feature, but so far I like this approach much more than putting this stuff into `Config[::parse]`, which already does a ton of stuff that it arguably shouldn't (but it's super difficult to extract its logic out). This PR is an alternative to #130749, which did a more 1:1 copy of the `download-ci-llvm` logic. r? ``@onur-ozkan``
2 parents 9746ac5 + 75a69a4 commit c007d0a

File tree

7 files changed

+274
-120
lines changed

7 files changed

+274
-120
lines changed

‎config.example.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,16 @@
163163
# Custom CMake defines to set when building LLVM.
164164
#build-config = {}
165165

166+
# =============================================================================
167+
# Tweaking how GCC is compiled
168+
# =============================================================================
169+
[gcc]
170+
# Download GCC from CI instead of building it locally.
171+
# Note that this will attempt to download GCC even if there are local
172+
# modifications to the `src/gcc` submodule.
173+
# Currently, this is only supported for the `x86_64-unknown-linux-gnu` target.
174+
# download-ci-gcc = false
175+
166176
# =============================================================================
167177
# General build configuration options
168178
# =============================================================================

‎src/bootstrap/download-ci-gcc-stamp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Change this file to make users of the `download-ci-gcc` configuration download
2+
a new version of GCC from CI, even if the GCC submodule hasn’t changed.
3+
4+
Last change is for: https://github.com/rust-lang/rust/pull/138051

‎src/bootstrap/src/core/build_steps/gcc.rs‎

Lines changed: 190 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,67 @@ use std::sync::OnceLock;
1414

1515
use build_helper::ci::CiEnv;
1616

17-
use crate::Kind;
18-
use crate::core::builder::{Builder, Cargo, RunConfig, ShouldRun, Step};
17+
use crate::core::builder::{Builder, Cargo, Kind, RunConfig, ShouldRun, Step};
1918
use crate::core::config::TargetSelection;
2019
use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash};
2120
use crate::utils::exec::command;
2221
use crate::utils::helpers::{self, t};
2322

23+
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
24+
pub struct Gcc {
25+
pub target: TargetSelection,
26+
}
27+
28+
#[derive(Clone)]
29+
pub struct GccOutput {
30+
pub libgccjit: PathBuf,
31+
}
32+
33+
impl Step for Gcc {
34+
type Output = GccOutput;
35+
36+
const ONLY_HOSTS: bool = true;
37+
38+
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
39+
run.path("src/gcc").alias("gcc")
40+
}
41+
42+
fn make_run(run: RunConfig<'_>) {
43+
run.builder.ensure(Gcc { target: run.target });
44+
}
45+
46+
/// Compile GCC (specifically `libgccjit`) for `target`.
47+
fn run(self, builder: &Builder<'_>) -> Self::Output {
48+
let target = self.target;
49+
50+
// If GCC has already been built, we avoid building it again.
51+
let metadata = match get_gcc_build_status(builder, target) {
52+
GccBuildStatus::AlreadyBuilt(path) => return GccOutput { libgccjit: path },
53+
GccBuildStatus::ShouldBuild(m) => m,
54+
};
55+
56+
let _guard = builder.msg_unstaged(Kind::Build, "GCC", target);
57+
t!(metadata.stamp.remove());
58+
let _time = helpers::timeit(builder);
59+
60+
let libgccjit_path = libgccjit_built_path(&metadata.install_dir);
61+
if builder.config.dry_run() {
62+
return GccOutput { libgccjit: libgccjit_path };
63+
}
64+
65+
build_gcc(&metadata, builder, target);
66+
67+
let lib_alias = metadata.install_dir.join("lib/libgccjit.so.0");
68+
if !lib_alias.exists() {
69+
t!(builder.symlink_file(&libgccjit_path, lib_alias));
70+
}
71+
72+
t!(metadata.stamp.write());
73+
74+
GccOutput { libgccjit: libgccjit_path }
75+
}
76+
}
77+
2478
pub struct Meta {
2579
stamp: BuildStamp,
2680
out_dir: PathBuf,
@@ -34,17 +88,45 @@ pub enum GccBuildStatus {
3488
ShouldBuild(Meta),
3589
}
3690

37-
/// This returns whether we've already previously built GCC.
91+
/// Tries to download GCC from CI if it is enabled and GCC artifacts
92+
/// are available for the given target.
93+
/// Returns a path to the libgccjit.so file.
94+
#[cfg(not(test))]
95+
fn try_download_gcc(builder: &Builder<'_>, target: TargetSelection) -> Option<PathBuf> {
96+
// Try to download GCC from CI if configured and available
97+
if !matches!(builder.config.gcc_ci_mode, crate::core::config::GccCiMode::DownloadFromCi) {
98+
return None;
99+
}
100+
if target != "x86_64-unknown-linux-gnu" {
101+
eprintln!("GCC CI download is only available for the `x86_64-unknown-linux-gnu` target");
102+
return None;
103+
}
104+
let sha =
105+
detect_gcc_sha(&builder.config, builder.config.rust_info.is_managed_git_subrepository());
106+
let root = ci_gcc_root(&builder.config);
107+
let gcc_stamp = BuildStamp::new(&root).with_prefix("gcc").add_stamp(&sha);
108+
if !gcc_stamp.is_up_to_date() && !builder.config.dry_run() {
109+
builder.config.download_ci_gcc(&sha, &root);
110+
t!(gcc_stamp.write());
111+
}
112+
// FIXME: put libgccjit.so into a lib directory in dist::Gcc
113+
Some(root.join("libgccjit.so"))
114+
}
115+
116+
#[cfg(test)]
117+
fn try_download_gcc(_builder: &Builder<'_>, _target: TargetSelection) -> Option<PathBuf> {
118+
None
119+
}
120+
121+
/// This returns information about whether GCC should be built or if it's already built.
122+
/// It transparently handles downloading GCC from CI if needed.
38123
///
39124
/// It's used to avoid busting caches during x.py check -- if we've already built
40125
/// GCC, it's fine for us to not try to avoid doing so.
41-
pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> GccBuildStatus {
42-
// Initialize the gcc submodule if not initialized already.
43-
builder.config.update_submodule("src/gcc");
44-
45-
let root = builder.src.join("src/gcc");
46-
let out_dir = builder.gcc_out(target).join("build");
47-
let install_dir = builder.gcc_out(target).join("install");
126+
pub fn get_gcc_build_status(builder: &Builder<'_>, target: TargetSelection) -> GccBuildStatus {
127+
if let Some(path) = try_download_gcc(builder, target) {
128+
return GccBuildStatus::AlreadyBuilt(path);
129+
}
48130

49131
static STAMP_HASH_MEMO: OnceLock<String> = OnceLock::new();
50132
let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| {
@@ -55,6 +137,13 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc
55137
)
56138
});
57139

140+
// Initialize the gcc submodule if not initialized already.
141+
builder.config.update_submodule("src/gcc");
142+
143+
let root = builder.src.join("src/gcc");
144+
let out_dir = builder.gcc_out(target).join("build");
145+
let install_dir = builder.gcc_out(target).join("install");
146+
58147
let stamp = BuildStamp::new(&out_dir).with_prefix("gcc").add_stamp(smart_stamp_hash);
59148

60149
if stamp.is_up_to_date() {
@@ -87,129 +176,111 @@ fn libgccjit_built_path(install_dir: &Path) -> PathBuf {
87176
install_dir.join("lib/libgccjit.so")
88177
}
89178

90-
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
91-
pub struct Gcc {
92-
pub target: TargetSelection,
93-
}
94-
95-
#[derive(Clone)]
96-
pub struct GccOutput {
97-
pub libgccjit: PathBuf,
98-
}
99-
100-
impl Step for Gcc {
101-
type Output = GccOutput;
102-
103-
const ONLY_HOSTS: bool = true;
104-
105-
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
106-
run.path("src/gcc").alias("gcc")
107-
}
108-
109-
fn make_run(run: RunConfig<'_>) {
110-
run.builder.ensure(Gcc { target: run.target });
111-
}
112-
113-
/// Compile GCC (specifically `libgccjit`) for `target`.
114-
fn run(self, builder: &Builder<'_>) -> Self::Output {
115-
let target = self.target;
116-
117-
// If GCC has already been built, we avoid building it again.
118-
let Meta { stamp, out_dir, install_dir, root } = match prebuilt_gcc_config(builder, target)
119-
{
120-
GccBuildStatus::AlreadyBuilt(path) => return GccOutput { libgccjit: path },
121-
GccBuildStatus::ShouldBuild(m) => m,
122-
};
179+
fn build_gcc(metadata: &Meta, builder: &Builder<'_>, target: TargetSelection) {
180+
let Meta { stamp: _, out_dir, install_dir, root } = metadata;
123181

124-
let _guard = builder.msg_unstaged(Kind::Build, "GCC", target);
125-
t!(stamp.remove());
126-
let _time = helpers::timeit(builder);
127-
t!(fs::create_dir_all(&out_dir));
128-
t!(fs::create_dir_all(&install_dir));
182+
t!(fs::create_dir_all(out_dir));
183+
t!(fs::create_dir_all(install_dir));
129184

130-
let libgccjit_path = libgccjit_built_path(&install_dir);
131-
if builder.config.dry_run() {
132-
return GccOutput { libgccjit: libgccjit_path };
185+
// GCC creates files (e.g. symlinks to the downloaded dependencies)
186+
// in the source directory, which does not work with our CI setup, where we mount
187+
// source directories as read-only on Linux.
188+
// Therefore, as a part of the build in CI, we first copy the whole source directory
189+
// to the build directory, and perform the build from there.
190+
let src_dir = if CiEnv::is_ci() {
191+
let src_dir = builder.gcc_out(target).join("src");
192+
if src_dir.exists() {
193+
builder.remove_dir(&src_dir);
133194
}
195+
builder.create_dir(&src_dir);
196+
builder.cp_link_r(root, &src_dir);
197+
src_dir
198+
} else {
199+
root.clone()
200+
};
134201

135-
// GCC creates files (e.g. symlinks to the downloaded dependencies)
136-
// in the source directory, which does not work with our CI setup, where we mount
137-
// source directories as read-only on Linux.
138-
// Therefore, as a part of the build in CI, we first copy the whole source directory
139-
// to the build directory, and perform the build from there.
140-
let src_dir = if CiEnv::is_ci() {
141-
let src_dir = builder.gcc_out(target).join("src");
142-
if src_dir.exists() {
143-
builder.remove_dir(&src_dir);
144-
}
145-
builder.create_dir(&src_dir);
146-
builder.cp_link_r(&root, &src_dir);
147-
src_dir
148-
} else {
149-
root
150-
};
202+
command(src_dir.join("contrib/download_prerequisites")).current_dir(&src_dir).run(builder);
203+
let mut configure_cmd = command(src_dir.join("configure"));
204+
configure_cmd
205+
.current_dir(out_dir)
206+
// On CI, we compile GCC with Clang.
207+
// The -Wno-everything flag is needed to make GCC compile with Clang 19.
208+
// `-g -O2` are the default flags that are otherwise used by Make.
209+
// FIXME(kobzol): change the flags once we have [gcc] configuration in config.toml.
210+
.env("CXXFLAGS", "-Wno-everything -g -O2")
211+
.env("CFLAGS", "-Wno-everything -g -O2")
212+
.arg("--enable-host-shared")
213+
.arg("--enable-languages=jit")
214+
.arg("--enable-checking=release")
215+
.arg("--disable-bootstrap")
216+
.arg("--disable-multilib")
217+
.arg(format!("--prefix={}", install_dir.display()));
218+
let cc = builder.build.cc(target).display().to_string();
219+
let cc = builder
220+
.build
221+
.config
222+
.ccache
223+
.as_ref()
224+
.map_or_else(|| cc.clone(), |ccache| format!("{ccache} {cc}"));
225+
configure_cmd.env("CC", cc);
151226

152-
command(src_dir.join("contrib/download_prerequisites")).current_dir(&src_dir).run(builder);
153-
let mut configure_cmd = command(src_dir.join("configure"));
154-
configure_cmd
155-
.current_dir(&out_dir)
156-
// On CI, we compile GCC with Clang.
157-
// The -Wno-everything flag is needed to make GCC compile with Clang 19.
158-
// `-g -O2` are the default flags that are otherwise used by Make.
159-
// FIXME(kobzol): change the flags once we have [gcc] configuration in config.toml.
160-
.env("CXXFLAGS", "-Wno-everything -g -O2")
161-
.env("CFLAGS", "-Wno-everything -g -O2")
162-
.arg("--enable-host-shared")
163-
.arg("--enable-languages=jit")
164-
.arg("--enable-checking=release")
165-
.arg("--disable-bootstrap")
166-
.arg("--disable-multilib")
167-
.arg(format!("--prefix={}", install_dir.display()));
168-
let cc = builder.build.cc(target).display().to_string();
169-
let cc = builder
227+
if let Ok(ref cxx) = builder.build.cxx(target) {
228+
let cxx = cxx.display().to_string();
229+
let cxx = builder
170230
.build
171231
.config
172232
.ccache
173233
.as_ref()
174-
.map_or_else(|| cc.clone(), |ccache| format!("{ccache} {cc}"));
175-
configure_cmd.env("CC", cc);
176-
177-
if let Ok(ref cxx) = builder.build.cxx(target) {
178-
let cxx = cxx.display().to_string();
179-
let cxx = builder
180-
.build
181-
.config
182-
.ccache
183-
.as_ref()
184-
.map_or_else(|| cxx.clone(), |ccache| format!("{ccache} {cxx}"));
185-
configure_cmd.env("CXX", cxx);
186-
}
187-
configure_cmd.run(builder);
188-
189-
command("make")
190-
.current_dir(&out_dir)
191-
.arg("--silent")
192-
.arg(format!("-j{}", builder.jobs()))
193-
.run_capture_stdout(builder);
194-
command("make")
195-
.current_dir(&out_dir)
196-
.arg("--silent")
197-
.arg("install")
198-
.run_capture_stdout(builder);
199-
200-
let lib_alias = install_dir.join("lib/libgccjit.so.0");
201-
if !lib_alias.exists() {
202-
t!(builder.symlink_file(&libgccjit_path, lib_alias));
203-
}
204-
205-
t!(stamp.write());
206-
207-
GccOutput { libgccjit: libgccjit_path }
234+
.map_or_else(|| cxx.clone(), |ccache| format!("{ccache} {cxx}"));
235+
configure_cmd.env("CXX", cxx);
208236
}
237+
configure_cmd.run(builder);
238+
239+
command("make")
240+
.current_dir(out_dir)
241+
.arg("--silent")
242+
.arg(format!("-j{}", builder.jobs()))
243+
.run_capture_stdout(builder);
244+
command("make").current_dir(out_dir).arg("--silent").arg("install").run_capture_stdout(builder);
209245
}
210246

211247
/// Configures a Cargo invocation so that it can build the GCC codegen backend.
212248
pub fn add_cg_gcc_cargo_flags(cargo: &mut Cargo, gcc: &GccOutput) {
213249
// Add the path to libgccjit.so to the linker search paths.
214250
cargo.rustflag(&format!("-L{}", gcc.libgccjit.parent().unwrap().to_str().unwrap()));
215251
}
252+
253+
/// The absolute path to the downloaded GCC artifacts.
254+
#[cfg(not(test))]
255+
fn ci_gcc_root(config: &crate::Config) -> PathBuf {
256+
config.out.join(config.build).join("ci-gcc")
257+
}
258+
259+
/// This retrieves the GCC sha we *want* to use, according to git history.
260+
#[cfg(not(test))]
261+
fn detect_gcc_sha(config: &crate::Config, is_git: bool) -> String {
262+
use build_helper::git::get_closest_merge_commit;
263+
264+
let gcc_sha = if is_git {
265+
get_closest_merge_commit(
266+
Some(&config.src),
267+
&config.git_config(),
268+
&[config.src.join("src/gcc"), config.src.join("src/bootstrap/download-ci-gcc-stamp")],
269+
)
270+
.unwrap()
271+
} else if let Some(info) = crate::utils::channel::read_commit_info_file(&config.src) {
272+
info.sha.trim().to_owned()
273+
} else {
274+
"".to_owned()
275+
};
276+
277+
if gcc_sha.is_empty() {
278+
eprintln!("error: could not find commit hash for downloading GCC");
279+
eprintln!("HELP: maybe your repository history is too shallow?");
280+
eprintln!("HELP: consider disabling `download-ci-gcc`");
281+
eprintln!("HELP: or fetch enough history to include one upstream commit");
282+
panic!();
283+
}
284+
285+
gcc_sha
286+
}

0 commit comments

Comments
 (0)