Skip to content

Commit cf0367a

Browse files
borsbors
committed
Auto merge of #374 - pietroalbini:unify-source-dir, r=pietroalbini
Refactor how source directories are handled during experiments
2 parents 7a7fe3d + 3c29590 commit cf0367a

File tree

11 files changed

+265
-363
lines changed

11 files changed

+265
-363
lines changed

‎src/crates/mod.rs‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
pub(crate) mod lists;
22
mod sources;
33

4-
use dirs::{CRATES_DIR, LOCAL_CRATES_DIR};
4+
use dirs::LOCAL_CRATES_DIR;
55
use prelude::*;
66
use std::fmt;
7-
use std::path::PathBuf;
7+
use std::path::Path;
88
use std::str::FromStr;
99

1010
pub(crate) use crates::sources::github::GitHubRepo;
@@ -25,17 +25,6 @@ impl Crate {
2525
Crate::Local(_) => false,
2626
}
2727
}
28-
pub(crate) fn dir(&self) -> PathBuf {
29-
match *self {
30-
Crate::Registry(ref details) => CRATES_DIR
31-
.join("reg")
32-
.join(format!("{}-{}", details.name, details.version)),
33-
Crate::GitHub(ref repo) => CRATES_DIR
34-
.join("gh")
35-
.join(format!("{}.{}", repo.org, repo.name)),
36-
Crate::Local(ref name) => CRATES_DIR.join("local").join(name),
37-
}
38-
}
3928

4029
pub(crate) fn id(&self) -> String {
4130
match *self {
@@ -45,13 +34,28 @@ impl Crate {
4534
}
4635
}
4736

48-
pub(crate) fn prepare(&self) -> Fallible<()> {
49-
let dir = self.dir();
37+
pub(crate) fn fetch(&self) -> Fallible<()> {
38+
match *self {
39+
Crate::Registry(ref krate) => krate.fetch(),
40+
Crate::GitHub(ref repo) => repo.fetch(),
41+
Crate::Local(_) => Ok(()),
42+
}
43+
}
44+
45+
pub(crate) fn copy_to(&self, dest: &Path) -> Fallible<()> {
46+
if dest.exists() {
47+
info!(
48+
"crate source directory {} already exists, cleaning it up",
49+
dest.display()
50+
);
51+
::utils::fs::remove_dir_all(dest)?;
52+
}
5053
match *self {
51-
Crate::Registry(ref details) => details.prepare(&dir)?,
52-
Crate::GitHub(ref repo) => repo.prepare(&dir)?,
54+
Crate::Registry(ref details) => details.copy_to(&dest)?,
55+
Crate::GitHub(ref repo) => repo.copy_to(&dest)?,
5356
Crate::Local(ref name) => {
54-
::utils::fs::copy_dir(&LOCAL_CRATES_DIR.join(name), &dir)?;
57+
info!("copying local crate {} to {}", name, dest.display());
58+
::utils::fs::copy_dir(&LOCAL_CRATES_DIR.join(name), &dest)?;
5559
}
5660
}
5761

‎src/crates/sources/github.rs‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
use crates::{lists::List, Crate};
2-
use dirs::GH_MIRRORS_DIR;
2+
use dirs::SOURCE_CACHE_DIR;
33
use prelude::*;
4+
use run::RunCommand;
45
use std::borrow::Cow;
56
use std::path::{Path, PathBuf};
67
use std::str::FromStr;
@@ -72,21 +73,47 @@ pub struct GitHubRepo {
7273
}
7374

7475
impl GitHubRepo {
75-
pub(crate) fn slug(&self) -> String {
76-
format!("{}/{}", self.org, self.name)
76+
pub(crate) fn cached_path(&self) -> PathBuf {
77+
SOURCE_CACHE_DIR.join("gh").join(&self.org).join(&self.name)
7778
}
7879

79-
pub(crate) fn url(&self) -> String {
80-
format!("https://github.com/{}/{}", self.org, self.name)
80+
pub(crate) fn slug(&self) -> String {
81+
format!("{}/{}", self.org, self.name)
8182
}
8283

83-
pub(crate) fn mirror_dir(&self) -> PathBuf {
84-
GH_MIRRORS_DIR.join(format!("{}.{}", self.org, self.name))
84+
pub(in crates) fn fetch(&self) -> Fallible<()> {
85+
let path = self.cached_path();
86+
if path.join("HEAD").is_file() {
87+
info!("updating cached repository {}", self.slug());
88+
RunCommand::new("git")
89+
.args(&["fetch", "--all"])
90+
.cd(&path)
91+
.run()
92+
.with_context(|_| format!("failed to update {}", self.slug()))?;
93+
} else {
94+
info!("cloning repository {}", self.slug());
95+
RunCommand::new("git")
96+
.args(&[
97+
"clone",
98+
"--bare",
99+
&format!("git://github.com/{}/{}.git", self.org, self.name),
100+
])
101+
.args(&[&path])
102+
.run()
103+
.with_context(|_| format!("failed to clone {}", self.slug()))?;
104+
}
105+
Ok(())
85106
}
86107

87-
pub(in crates) fn prepare(&self, dest: &Path) -> Fallible<()> {
88-
::git::shallow_clone_or_pull(&self.url(), &self.mirror_dir())?;
89-
::utils::fs::copy_dir(&self.mirror_dir(), &dest)?;
108+
pub(in crates) fn copy_to(&self, dest: &Path) -> Fallible<()> {
109+
if dest.exists() {
110+
::utils::fs::remove_dir_all(dest)?;
111+
}
112+
RunCommand::new("git")
113+
.args(&["clone"])
114+
.args(&[self.cached_path().as_path(), dest])
115+
.run()
116+
.with_context(|_| format!("failed to checkout {}", self.slug()))?;
90117
Ok(())
91118
}
92119
}

‎src/crates/sources/registry.rs‎

Lines changed: 48 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
use crates::{lists::List, Crate};
22
use crates_index::Index;
3-
use dirs::LOCAL_DIR;
3+
use dirs::{LOCAL_DIR, SOURCE_CACHE_DIR};
44
use flate2::read::GzDecoder;
55
use prelude::*;
66
use std::collections::HashMap;
7-
use std::fs;
8-
use std::io::Read;
9-
use std::path::Path;
7+
use std::fs::{self, File};
8+
use std::io::{BufReader, BufWriter, Read};
9+
use std::path::{Path, PathBuf};
1010
use tar::Archive;
1111

1212
static CRATES_ROOT: &str = "https://crates-io.s3-us-west-1.amazonaws.com/crates";
@@ -67,40 +67,57 @@ pub struct RegistryCrate {
6767
}
6868

6969
impl RegistryCrate {
70-
pub(in crates) fn prepare(&self, dest: &Path) -> Fallible<()> {
71-
dl_registry(&self.name, &self.version, &dest).with_context(|_| {
72-
format!("unable to download {} version {}", self.name, self.version)
73-
})?;
74-
Ok(())
70+
fn cached_path(&self) -> PathBuf {
71+
SOURCE_CACHE_DIR
72+
.join("reg")
73+
.join(&self.name)
74+
.join(format!("{}-{}.crate", self.name, self.version))
7575
}
76-
}
7776

78-
fn dl_registry(name: &str, vers: &str, dir: &Path) -> Fallible<()> {
79-
if dir.exists() {
80-
info!(
81-
"crate {}-{} exists at {}. skipping",
82-
name,
83-
vers,
84-
dir.display()
77+
pub(in crates) fn fetch(&self) -> Fallible<()> {
78+
let local = self.cached_path();
79+
if local.exists() {
80+
info!("crate {} {} is already in cache", self.name, self.version);
81+
return Ok(());
82+
}
83+
84+
info!("fetching crate {} {}...", self.name, self.version);
85+
if let Some(parent) = local.parent() {
86+
fs::create_dir_all(parent)?;
87+
}
88+
let remote = format!(
89+
"{0}/{1}/{1}-{2}.crate",
90+
CRATES_ROOT, self.name, self.version
8591
);
86-
return Ok(());
87-
}
88-
info!("downloading crate {}-{} to {}", name, vers, dir.display());
89-
let url = format!("{0}/{1}/{1}-{2}.crate", CRATES_ROOT, name, vers);
90-
let bin =
91-
::utils::http::get_sync(&url).with_context(|_| format!("unable to download {}", url))?;
92+
let mut resp = ::utils::http::get_sync(&remote)?;
93+
resp.copy_to(&mut BufWriter::new(File::create(&local)?))?;
9294

93-
fs::create_dir_all(&dir)?;
95+
Ok(())
96+
}
9497

95-
let mut tar = Archive::new(GzDecoder::new(bin));
96-
let r =
97-
unpack_without_first_dir(&mut tar, dir).with_context(|_| "unable to unpack crate tarball");
98+
pub(in crates) fn copy_to(&self, dest: &Path) -> Fallible<()> {
99+
let cached = self.cached_path();
100+
let mut file = File::open(cached)?;
101+
let mut tar = Archive::new(GzDecoder::new(BufReader::new(&mut file)));
98102

99-
if r.is_err() {
100-
let _ = ::utils::fs::remove_dir_all(dir);
103+
info!(
104+
"extracting crate {} {} into {}",
105+
self.name,
106+
self.version,
107+
dest.display()
108+
);
109+
if let Err(err) = unpack_without_first_dir(&mut tar, dest) {
110+
let _ = ::utils::fs::remove_dir_all(dest);
111+
Err(err
112+
.context(format!(
113+
"unable to download {} version {}",
114+
self.name, self.version
115+
))
116+
.into())
117+
} else {
118+
Ok(())
119+
}
101120
}
102-
103-
r.map_err(|e| e.into())
104121
}
105122

106123
fn unpack_without_first_dir<R: Read>(archive: &mut Archive<R>, path: &Path) -> Fallible<()> {

‎src/dirs.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
1+
use crates::Crate;
2+
use experiments::Experiment;
13
use std::env;
24
use std::ffi::OsStr;
35
use std::path::PathBuf;
6+
use toolchain::Toolchain;
47

58
lazy_static! {
69
pub static ref WORK_DIR: PathBuf = {
@@ -31,4 +34,14 @@ lazy_static! {
3134
pub static ref LOG_DIR: PathBuf = WORK_DIR.join("logs");
3235

3336
pub static ref LOCAL_CRATES_DIR: PathBuf = "local-crates".into();
37+
38+
pub static ref SOURCE_CACHE_DIR: PathBuf = WORK_DIR.join("cache").join("sources");
39+
}
40+
41+
pub(crate) fn crate_source_dir(ex: &Experiment, tc: &Toolchain, krate: &Crate) -> PathBuf {
42+
EXPERIMENT_DIR
43+
.join(&ex.name)
44+
.join("sources")
45+
.join(tc.to_string())
46+
.join(krate.id())
3447
}

‎src/git.rs‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎src/lib.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,6 @@ pub mod db;
7474
pub mod dirs;
7575
pub mod docker;
7676
pub mod experiments;
77-
mod git;
7877
mod native;
7978
mod prelude;
8079
pub mod report;

‎src/runner/graph.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,15 @@ pub(super) fn build_graph(ex: &Experiment, config: &Config) -> TasksGraph {
270270
builds.push(build_id);
271271
}
272272

273-
graph.add_crate(&builds);
273+
let cleanup_id = graph.add_task(
274+
Task {
275+
krate: krate.clone(),
276+
step: TaskStep::Cleanup,
277+
},
278+
&builds,
279+
);
280+
281+
graph.add_crate(&[cleanup_id]);
274282
}
275283

276284
graph

0 commit comments

Comments
 (0)