Read incremental files on-demand.

Author: cjgillot

compiler/rustc_incremental/src/persist/file_format.rs

@@ -54,14 +54,14 @@ pub fn read_file(
     report_incremental_info: bool,
     path: &Path,
     nightly_build: bool,
-) -> io::Result<Option<(Vec<u8>, usize)>> {
-    let data = match fs::read(path) {
-        Ok(data) => data,
+) -> io::Result<Option<io::BufReader<fs::File>>> {
+    let file = match fs::File::open(path) {
+        Ok(file) => file,
         Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None),
         Err(err) => return Err(err),
     };
 
-    let mut file = io::Cursor::new(data);
+    let mut file = io::BufReader::new(file);
 
     // Check FILE_MAGIC
     {
@@ -102,8 +102,7 @@ pub fn read_file(
         }
     }
 
-    let post_header_start_pos = file.position() as usize;
-    Ok(Some((file.into_inner(), post_header_start_pos)))
+    Ok(Some(file))
 }
 
 fn report_format_mismatch(report_incremental_info: bool, file: &Path, message: &str) {

compiler/rustc_incremental/src/persist/load.rs

@@ -4,9 +4,11 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::definitions::Definitions;
 use rustc_middle::dep_graph::{PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId};
 use rustc_middle::ty::query::OnDiskCache;
-use rustc_serialize::opaque::Decoder;
+use rustc_serialize::opaque::FileDecoder;
 use rustc_serialize::Decodable as RustcDecodable;
 use rustc_session::Session;
+use std::fs;
+use std::io::{self, Read, Seek};
 use std::path::Path;
 
 use super::data::*;
@@ -49,9 +51,9 @@ fn load_data(
     report_incremental_info: bool,
     path: &Path,
     nightly_build: bool,
-) -> LoadResult<(Vec<u8>, usize)> {
+) -> LoadResult<io::BufReader<fs::File>> {
     match file_format::read_file(report_incremental_info, path, nightly_build) {
-        Ok(Some(data_and_pos)) => LoadResult::Ok { data: data_and_pos },
+        Ok(Some(file)) => LoadResult::Ok { data: file },
         Ok(None) => {
             // The file either didn't exist or was produced by an incompatible
             // compiler version. Neither is an error.
@@ -116,9 +118,9 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
         let work_products_path = work_products_path(sess);
         let load_result = load_data(report_incremental_info, &work_products_path, nightly_build);
 
-        if let LoadResult::Ok { data: (work_products_data, start_pos) } = load_result {
+        if let LoadResult::Ok { data: file } = load_result {
             // Decode the list of work_products
-            let mut work_product_decoder = Decoder::new(&work_products_data[..], start_pos);
+            let mut work_product_decoder = FileDecoder::new(file);
             let work_products: Vec<SerializedWorkProduct> =
                 RustcDecodable::decode(&mut work_product_decoder).unwrap_or_else(|e| {
                     let msg = format!(
@@ -163,8 +165,8 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
         match load_data(report_incremental_info, &path, nightly_build) {
             LoadResult::DataOutOfDate => LoadResult::DataOutOfDate,
             LoadResult::Error { message } => LoadResult::Error { message },
-            LoadResult::Ok { data: (bytes, start_pos) } => {
-                let mut decoder = Decoder::new(&bytes, start_pos);
+            LoadResult::Ok { data: file } => {
+                let mut decoder = FileDecoder::new(file);
                 let prev_commandline_args_hash = u64::decode(&mut decoder)
                     .expect("Error reading commandline arg hash from cached dep-graph");
 
@@ -211,7 +213,11 @@ pub fn load_query_result_cache<'a>(
         &query_cache_path(sess),
         sess.is_nightly_build(),
     ) {
-        LoadResult::Ok { data: (bytes, start_pos) } => {
+        LoadResult::Ok { data: mut file } => {
+            let start_pos = file.seek(io::SeekFrom::Current(0)).unwrap() as usize;
+            file.seek(io::SeekFrom::Start(0)).unwrap();
+            let mut bytes = Vec::new();
+            file.read_to_end(&mut bytes).unwrap();
             Some(OnDiskCache::new(sess, bytes, start_pos, definitions))
         }
         _ => Some(OnDiskCache::new_empty(sess.source_map())),