Add my encoding debugging rig

Author: saethlin

compiler/rustc_query_system/src/dep_graph/edges.rs

@@ -41,7 +41,7 @@ impl Deref for EdgesVec {
 
     #[inline]
     fn deref(&self) -> &Self::Target {
-        self.edges.as_slice()
+        &*self.edges
     }
 }
 

compiler/rustc_query_system/src/dep_graph/serialized.rs

@@ -397,13 +397,30 @@ impl NodeInfo {
         if header.len().is_none() {
             e.emit_usize(self.edges.len());
         }
-
         let bytes_per_index = header.bytes_per_index();
-        for node_index in self.edges.iter() {
-            e.write_with(|dest| {
-                *dest = node_index.as_u32().to_le_bytes();
-                bytes_per_index
+        let mut edges = &*self.edges;
+        loop {
+            e.write_with_spare(|dest| {
+                if dest.len() < 4 {
+                    return 0;
+                }
+                let usable_len = dest.len() - 4;
+                let (edges_to_encode, remaining) =
+                    edges.split_at((usable_len / bytes_per_index).min(edges.len()));
+                for (i, node_index) in edges_to_encode.iter().enumerate() {
+                    let start = i * bytes_per_index;
+                    let d: &mut [u8; 4] =
+                        unsafe { dest.get_unchecked_mut(start..start + 4).try_into().unwrap() };
+                    *d = node_index.as_u32().to_le_bytes();
+                }
+                edges = remaining;
+                edges_to_encode.len() * bytes_per_index
             });
+
+            if edges.is_empty() {
+                break;
+            }
+            e.flush();
         }
     }
 }

compiler/rustc_query_system/src/lib.rs

@@ -3,6 +3,7 @@
 #![feature(hash_raw_entry)]
 #![feature(min_specialization)]
 #![feature(let_chains)]
+#![feature(array_windows)]
 #![allow(rustc::potential_query_instability, internal_features)]
 
 #[macro_use]

compiler/rustc_serialize/src/lib.rs

@@ -15,6 +15,7 @@
 #![feature(min_specialization)]
 #![feature(never_type)]
 #![feature(ptr_sub_ptr)]
+#![feature(let_chains)]
 #![cfg_attr(test, feature(test))]
 #![allow(rustc::internal)]
 

compiler/rustc_serialize/src/opaque.rs

@@ -38,6 +38,8 @@ pub struct FileEncoder {
     path: PathBuf,
     #[cfg(debug_assertions)]
     finished: bool,
+    panic_at_offset: Option<usize>,
+    flush_every_write: bool,
 }
 
 impl FileEncoder {
@@ -48,6 +50,9 @@ impl FileEncoder {
         let file =
             File::options().read(true).write(true).create(true).truncate(true).open(&path)?;
 
+        let panic_at_offset = std::env::var_os("RUSTC_FILE_ENCODER_PANIC_AT_OFFSET")
+            .map(|v| v.to_str().unwrap().parse::<usize>().unwrap());
+
         Ok(FileEncoder {
             buf: vec![0u8; BUF_SIZE].into_boxed_slice().try_into().unwrap(),
             path: path.as_ref().into(),
@@ -57,6 +62,8 @@ impl FileEncoder {
             res: Ok(()),
             #[cfg(debug_assertions)]
             finished: false,
+            panic_at_offset,
+            flush_every_write: panic_at_offset.is_some_and(|v| v <= BUF_SIZE),
         })
     }
 
@@ -75,6 +82,19 @@ impl FileEncoder {
         {
             self.finished = false;
         }
+
+        if let Some(panic_offset) = self.panic_at_offset {
+            // If we are within the buffer size of the offset we're to panic at, we need to start
+            // flushing every write so that we don't panic late.
+            if panic_offset - self.flushed <= BUF_SIZE {
+                self.flush_every_write = true;
+            }
+            // If the offset we want to panic at is in the range we're about to write, panic.
+            if (self.flushed..self.flushed + self.buffered).contains(&panic_offset) {
+                panic!()
+            }
+        }
+
         if self.res.is_ok() {
             self.res = self.file.write_all(&self.buf[..self.buffered]);
         }
@@ -107,6 +127,12 @@ impl FileEncoder {
             if self.res.is_ok() {
                 self.res = self.file.write_all(buf);
             }
+            // This write bypasses the buffer, so we need to duplicate the check logic here
+            if let Some(panic_offset) = self.panic_at_offset {
+                if (self.flushed..self.flushed + buf.len()).contains(&panic_offset) {
+                    panic!()
+                }
+            }
             self.flushed += buf.len();
         }
     }
@@ -117,7 +143,9 @@ impl FileEncoder {
         {
             self.finished = false;
         }
-        if let Some(dest) = self.buffer_empty().get_mut(..buf.len()) {
+        if !self.flush_every_write
+            && let Some(dest) = self.buffer_empty().get_mut(..buf.len())
+        {
             dest.copy_from_slice(buf);
             self.buffered += buf.len();
         } else {
@@ -146,21 +174,39 @@ impl FileEncoder {
             self.finished = false;
         }
         let flush_threshold = const { BUF_SIZE.checked_sub(N).unwrap() };
-        if std::intrinsics::unlikely(self.buffered > flush_threshold) {
+        if std::intrinsics::unlikely(self.flush_every_write || self.buffered > flush_threshold) {
             self.flush();
         }
         // SAFETY: We checked above that that N < self.buffer_empty().len(),
         // and if isn't, flush ensures that our empty buffer is now BUF_SIZE.
         // We produce a post-mono error if N > BUF_SIZE.
         let buf = unsafe { self.buffer_empty().first_chunk_mut::<N>().unwrap_unchecked() };
         let written = visitor(buf);
-        // We have to ensure that an errant visitor cannot cause self.buffered to exeed BUF_SIZE.
+        // We have to ensure that an errant visitor cannot cause self.buffered to exceed BUF_SIZE.
         if written > N {
             Self::panic_invalid_write::<N>(written);
         }
         self.buffered += written;
     }
 
+    #[inline]
+    pub fn write_with_spare(&mut self, visitor: impl FnOnce(&mut [u8]) -> usize) {
+        #[cold]
+        #[inline(never)]
+        fn panic_invalid_write_spare() {
+            panic!(
+                "FileEncoder::write_with_spare cannot be used to write more bytes than the current buffer size!"
+            );
+        }
+
+        let buf = self.buffer_empty();
+        let written = visitor(buf);
+        if written > buf.len() {
+            panic_invalid_write_spare();
+        }
+        self.buffered += written;
+    }
+
     #[cold]
     #[inline(never)]
     fn panic_invalid_write<const N: usize>(written: usize) {