Read incremental files on-demand

compiler/rustc_data_structures/src/fingerprint.rs

@@ -1,8 +1,5 @@
 use crate::stable_hasher;
-use rustc_serialize::{
-    opaque::{self, EncodeResult, FileEncodeResult},
-    Decodable, Encodable,
-};
+use rustc_serialize::{Decodable, Encodable};
 use std::hash::{Hash, Hasher};
 use std::mem::{self, MaybeUninit};
 
@@ -63,16 +60,6 @@ impl Fingerprint {
     pub fn to_hex(&self) -> String {
         format!("{:x}{:x}", self.0, self.1)
     }
-
-    pub fn decode_opaque(decoder: &mut opaque::Decoder<'_>) -> Result<Fingerprint, String> {
-        let mut bytes: [MaybeUninit<u8>; 16] = MaybeUninit::uninit_array();
-
-        decoder.read_raw_bytes(&mut bytes)?;
-
-        let [l, r]: [u64; 2] = unsafe { mem::transmute(bytes) };
-
-        Ok(Fingerprint(u64::from_le(l), u64::from_le(r)))
-    }
 }
 
 impl std::fmt::Display for Fingerprint {
@@ -130,55 +117,22 @@ impl stable_hasher::StableHasherResult for Fingerprint {
 impl_stable_hash_via_hash!(Fingerprint);
 
 impl<E: rustc_serialize::Encoder> Encodable<E> for Fingerprint {
+    #[inline]
     fn encode(&self, s: &mut E) -> Result<(), E::Error> {
-        s.encode_fingerprint(self)
+        let bytes: [u8; 16] = unsafe { mem::transmute([self.0.to_le(), self.1.to_le()]) };
+        s.emit_raw_bytes(&bytes)?;
+        Ok(())
     }
 }
 
 impl<D: rustc_serialize::Decoder> Decodable<D> for Fingerprint {
+    #[inline]
     fn decode(d: &mut D) -> Result<Self, D::Error> {
-        d.decode_fingerprint()
-    }
-}
-
-pub trait FingerprintEncoder: rustc_serialize::Encoder {
-    fn encode_fingerprint(&mut self, f: &Fingerprint) -> Result<(), Self::Error>;
-}
-
-pub trait FingerprintDecoder: rustc_serialize::Decoder {
-    fn decode_fingerprint(&mut self) -> Result<Fingerprint, Self::Error>;
-}
-
-impl<E: rustc_serialize::Encoder> FingerprintEncoder for E {
-    default fn encode_fingerprint(&mut self, _: &Fingerprint) -> Result<(), E::Error> {
-        panic!("Cannot encode `Fingerprint` with `{}`", std::any::type_name::<E>());
-    }
-}
-
-impl FingerprintEncoder for opaque::Encoder {
-    fn encode_fingerprint(&mut self, f: &Fingerprint) -> EncodeResult {
-        let bytes: [u8; 16] = unsafe { mem::transmute([f.0.to_le(), f.1.to_le()]) };
-        self.emit_raw_bytes(&bytes);
-        Ok(())
-    }
-}
-
-impl FingerprintEncoder for opaque::FileEncoder {
-    fn encode_fingerprint(&mut self, f: &Fingerprint) -> FileEncodeResult {
-        let bytes: [u8; 16] = unsafe { mem::transmute([f.0.to_le(), f.1.to_le()]) };
-        self.emit_raw_bytes(&bytes)
-    }
-}
-
-impl<D: rustc_serialize::Decoder> FingerprintDecoder for D {
-    default fn decode_fingerprint(&mut self) -> Result<Fingerprint, D::Error> {
-        panic!("Cannot decode `Fingerprint` with `{}`", std::any::type_name::<D>());
-    }
-}
+        let mut bytes: [MaybeUninit<u8>; 16] = MaybeUninit::uninit_array();
+        d.read_raw_bytes(&mut bytes)?;
 
-impl FingerprintDecoder for opaque::Decoder<'_> {
-    fn decode_fingerprint(&mut self) -> Result<Fingerprint, String> {
-        Fingerprint::decode_opaque(self)
+        let [l, r]: [u64; 2] = unsafe { mem::transmute(bytes) };
+        Ok(Fingerprint(u64::from_le(l), u64::from_le(r)))
     }
 }
 

compiler/rustc_incremental/src/persist/file_format.rs

@@ -11,10 +11,11 @@
 
 use std::env;
 use std::fs;
-use std::io::{self, Read};
+use std::io;
 use std::path::Path;
 
-use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
+use rustc_serialize::opaque::{FileDecoder, FileEncodeResult, FileEncoder};
+use rustc_serialize::Encoder;
 
 /// The first few bytes of files generated by incremental compilation.
 const FILE_MAGIC: &[u8] = b"RSIC";
@@ -53,14 +54,14 @@ pub fn read_file(
     report_incremental_info: bool,
     path: &Path,
     nightly_build: bool,
-) -> io::Result<Option<(Vec<u8>, usize)>> {
-    let data = match fs::read(path) {
-        Ok(data) => data,
+) -> io::Result<Option<FileDecoder>> {
+    let file = match fs::File::open(path) {
+        Ok(file) => file,
         Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(None),
         Err(err) => return Err(err),
     };
 
-    let mut file = io::Cursor::new(data);
+    let mut file = FileDecoder::new(file);
 
     // Check FILE_MAGIC
     {
@@ -101,8 +102,7 @@ pub fn read_file(
         }
     }
 
-    let post_header_start_pos = file.position() as usize;
-    Ok(Some((file.into_inner(), post_header_start_pos)))
+    Ok(Some(file))
 }
 
 fn report_format_mismatch(report_incremental_info: bool, file: &Path, message: &str) {

compiler/rustc_incremental/src/persist/load.rs

@@ -4,7 +4,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::definitions::Definitions;
 use rustc_middle::dep_graph::{PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId};
 use rustc_middle::ty::query::OnDiskCache;
-use rustc_serialize::opaque::Decoder;
+use rustc_serialize::opaque::FileDecoder;
 use rustc_serialize::Decodable as RustcDecodable;
 use rustc_session::Session;
 use std::path::Path;
@@ -49,9 +49,9 @@ fn load_data(
     report_incremental_info: bool,
     path: &Path,
     nightly_build: bool,
-) -> LoadResult<(Vec<u8>, usize)> {
+) -> LoadResult<FileDecoder> {
     match file_format::read_file(report_incremental_info, path, nightly_build) {
-        Ok(Some(data_and_pos)) => LoadResult::Ok { data: data_and_pos },
+        Ok(Some(file)) => LoadResult::Ok { data: file },
         Ok(None) => {
             // The file either didn't exist or was produced by an incompatible
             // compiler version. Neither is an error.
@@ -116,9 +116,8 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
         let work_products_path = work_products_path(sess);
         let load_result = load_data(report_incremental_info, &work_products_path, nightly_build);
 
-        if let LoadResult::Ok { data: (work_products_data, start_pos) } = load_result {
+        if let LoadResult::Ok { data: mut work_product_decoder } = load_result {
             // Decode the list of work_products
-            let mut work_product_decoder = Decoder::new(&work_products_data[..], start_pos);
             let work_products: Vec<SerializedWorkProduct> =
                 RustcDecodable::decode(&mut work_product_decoder).unwrap_or_else(|e| {
                     let msg = format!(
@@ -163,8 +162,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
         match load_data(report_incremental_info, &path, nightly_build) {
             LoadResult::DataOutOfDate => LoadResult::DataOutOfDate,
             LoadResult::Error { message } => LoadResult::Error { message },
-            LoadResult::Ok { data: (bytes, start_pos) } => {
-                let mut decoder = Decoder::new(&bytes, start_pos);
+            LoadResult::Ok { data: mut decoder } => {
                 let prev_commandline_args_hash = u64::decode(&mut decoder)
                     .expect("Error reading commandline arg hash from cached dep-graph");
 
@@ -211,7 +209,8 @@ pub fn load_query_result_cache<'a>(
         &query_cache_path(sess),
         sess.is_nightly_build(),
     ) {
-        LoadResult::Ok { data: (bytes, start_pos) } => {
+        LoadResult::Ok { data: file } => {
+            let (bytes, start_pos) = file.read_all().unwrap();
             Some(OnDiskCache::new(sess, bytes, start_pos, definitions))
         }
         _ => Some(OnDiskCache::new_empty(sess.source_map())),

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -7,7 +7,6 @@ use crate::rmeta::*;
 use rustc_ast as ast;
 use rustc_attr as attr;
 use rustc_data_structures::captures::Captures;
-use rustc_data_structures::fingerprint::{Fingerprint, FingerprintDecoder};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::{Lock, LockGuard, Lrc, OnceCell};
@@ -351,12 +350,6 @@ impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for DefIndex {
     }
 }
 
-impl<'a, 'tcx> FingerprintDecoder for DecodeContext<'a, 'tcx> {
-    fn decode_fingerprint(&mut self) -> Result<Fingerprint, String> {
-        Fingerprint::decode_opaque(&mut self.opaque)
-    }
-}
-
 impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for SyntaxContext {
     fn decode(decoder: &mut DecodeContext<'a, 'tcx>) -> Result<SyntaxContext, String> {
         let cdata = decoder.cdata();

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -1,7 +1,6 @@
 use crate::rmeta::table::{FixedSizeEncoding, TableBuilder};
 use crate::rmeta::*;
 
-use rustc_data_structures::fingerprint::{Fingerprint, FingerprintEncoder};
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_data_structures::sync::{join, par_iter, Lrc, ParallelIterator};
@@ -307,12 +306,6 @@ impl<'a, 'tcx> Encodable<EncodeContext<'a, 'tcx>> for Span {
     }
 }
 
-impl<'a, 'tcx> FingerprintEncoder for EncodeContext<'a, 'tcx> {
-    fn encode_fingerprint(&mut self, f: &Fingerprint) -> Result<(), Self::Error> {
-        self.opaque.encode_fingerprint(f)
-    }
-}
-
 impl<'a, 'tcx> TyEncoder<'tcx> for EncodeContext<'a, 'tcx> {
     const CLEAR_CROSS_CRATE: bool = true;
 
@@ -2073,10 +2066,10 @@ pub(super) fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata {
 
 fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata {
     let mut encoder = opaque::Encoder::new(vec![]);
-    encoder.emit_raw_bytes(METADATA_HEADER);
+    encoder.emit_raw_bytes(METADATA_HEADER).unwrap();
 
     // Will be filled with the root position after encoding everything.
-    encoder.emit_raw_bytes(&[0, 0, 0, 0]);
+    encoder.emit_raw_bytes(&[0, 0, 0, 0]).unwrap();
 
     let source_map_files = tcx.sess.source_map().files();
     let source_file_cache = (source_map_files[0].clone(), 0);

compiler/rustc_metadata/src/rmeta/table.rs

@@ -2,6 +2,7 @@ use crate::rmeta::*;
 
 use rustc_index::vec::Idx;
 use rustc_serialize::opaque::Encoder;
+use rustc_serialize::Encoder as _;
 use std::convert::TryInto;
 use std::marker::PhantomData;
 use std::num::NonZeroUsize;
@@ -172,7 +173,7 @@ where
 
     pub(crate) fn encode(&self, buf: &mut Encoder) -> Lazy<Table<I, T>> {
         let pos = buf.position();
-        buf.emit_raw_bytes(&self.bytes);
+        buf.emit_raw_bytes(&self.bytes).unwrap();
         Lazy::from_position_and_meta(NonZeroUsize::new(pos as usize).unwrap(), self.bytes.len())
     }
 }

compiler/rustc_middle/src/ty/query/on_disk_cache.rs

@@ -4,7 +4,6 @@ use crate::mir::{self, interpret};
 use crate::ty::codec::{RefDecodable, TyDecoder, TyEncoder};
 use crate::ty::context::TyCtxt;
 use crate::ty::{self, Ty};
-use rustc_data_structures::fingerprint::{Fingerprint, FingerprintDecoder, FingerprintEncoder};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
 use rustc_data_structures::sync::{HashMapExt, Lock, Lrc, OnceCell};
 use rustc_data_structures::thin_vec::ThinVec;
@@ -913,12 +912,6 @@ impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for DefId {
     }
 }
 
-impl<'a, 'tcx> FingerprintDecoder for CacheDecoder<'a, 'tcx> {
-    fn decode_fingerprint(&mut self) -> Result<Fingerprint, Self::Error> {
-        Fingerprint::decode_opaque(&mut self.opaque)
-    }
-}
-
 impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx FxHashSet<LocalDefId> {
     fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Result<Self, String> {
         RefDecodable::decode(d)
@@ -1011,12 +1004,6 @@ where
     }
 }
 
-impl<'a, 'tcx, E: OpaqueEncoder> FingerprintEncoder for CacheEncoder<'a, 'tcx, E> {
-    fn encode_fingerprint(&mut self, f: &Fingerprint) -> Result<(), E::Error> {
-        self.encoder.encode_fingerprint(f)
-    }
-}
-
 impl<'a, 'tcx, E> Encodable<CacheEncoder<'a, 'tcx, E>> for SyntaxContext
 where
     E: 'a + OpaqueEncoder,