Add set_oom_handler and use it print a message when out of memory

src/liballoc/lib.rs

@@ -92,6 +92,7 @@
 #![feature(unsize)]
 #![feature(drop_in_place)]
 #![feature(fn_traits)]
+#![feature(const_fn)]
 
 #![feature(needs_allocator)]
 
@@ -127,15 +128,6 @@ mod boxed_test;
 pub mod arc;
 pub mod rc;
 pub mod raw_vec;
+pub mod oom;
 
-/// Common out-of-memory routine
-#[cold]
-#[inline(never)]
-#[unstable(feature = "oom", reason = "not a scrutinized interface",
-           issue = "27700")]
-pub fn oom() -> ! {
-    // FIXME(#14674): This really needs to do something other than just abort
-    //                here, but any printing done must be *guaranteed* to not
-    //                allocate.
-    unsafe { core::intrinsics::abort() }
-}
+pub use oom::oom;

src/liballoc/oom.rs

@@ -0,0 +1,42 @@
+// Copyright 2014-2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use core::sync::atomic::{AtomicPtr, Ordering};
+use core::mem;
+use core::intrinsics;
+
+static OOM_HANDLER: AtomicPtr<()> = AtomicPtr::new(default_oom_handler as *mut ());
+
+fn default_oom_handler() -> ! {
+    // The default handler can't do much more since we can't assume the presence
+    // of libc or any way of printing an error message.
+    unsafe { intrinsics::abort() }
+}
+
+/// Common out-of-memory routine
+#[cold]
+#[inline(never)]
+#[unstable(feature = "oom", reason = "not a scrutinized interface",
+           issue = "27700")]
+pub fn oom() -> ! {
+    let value = OOM_HANDLER.load(Ordering::SeqCst);
+    let handler: fn() -> ! = unsafe { mem::transmute(value) };
+    handler();
+}
+
+/// Set a custom handler for out-of-memory conditions
+///
+/// To avoid recursive OOM failures, it is critical that the OOM handler does
+/// not allocate any memory itself.
+#[unstable(feature = "oom", reason = "not a scrutinized interface",
+           issue = "27700")]
+pub fn set_oom_handler(handler: fn() -> !) {
+    OOM_HANDLER.store(handler as *mut (), Ordering::SeqCst);
+}

src/libstd/sys/unix/mod.rs

@@ -15,6 +15,7 @@ use io::{self, ErrorKind};
 use libc;
 use num::One;
 use ops::Neg;
+use alloc::oom;
 
 #[cfg(target_os = "android")]   pub use os::android as platform;
 #[cfg(target_os = "bitrig")]    pub use os::bitrig as platform;
@@ -45,6 +46,22 @@ pub mod thread_local;
 pub mod time;
 pub mod stdio;
 
+// A nicer handler for out-of-memory situations than the default one. This one
+// prints a message to stderr before aborting. It is critical that this code
+// does not allocate any memory since we are in an OOM situation. Any errors are
+// ignored while printing since there's nothing we can do about them and we are
+// about to exit anyways.
+fn oom_handler() -> ! {
+    use intrinsics;
+    let msg = "fatal runtime error: out of memory\n";
+    unsafe {
+        libc::write(libc::STDERR_FILENO,
+                    msg.as_ptr() as *const libc::c_void,
+                    msg.len() as libc::size_t);
+        intrinsics::abort();
+    }
+}
+
 #[cfg(not(any(target_os = "nacl", test)))]
 pub fn init() {
     use libc::signal;
@@ -58,10 +75,14 @@ pub fn init() {
     unsafe {
         assert!(signal(libc::SIGPIPE, libc::SIG_IGN) != !0);
     }
+
+    oom::set_oom_handler(oom_handler);
 }
 
 #[cfg(all(target_os = "nacl", not(test)))]
-pub fn init() { }
+pub fn init() {
+    oom::set_oom_handler(oom_handler);
+}
 
 pub fn decode_error_kind(errno: i32) -> ErrorKind {
     match errno as libc::c_int {

src/libstd/sys/windows/mod.rs

@@ -20,6 +20,7 @@ use num::Zero;
 use os::windows::ffi::{OsStrExt, OsStringExt};
 use path::PathBuf;
 use time::Duration;
+use alloc::oom;
 
 #[macro_use] pub mod compat;
 
@@ -42,7 +43,26 @@ pub mod thread_local;
 pub mod time;
 pub mod stdio;
 
-pub fn init() {}
+// See comment in sys/unix/mod.rs
+fn oom_handler() -> ! {
+    use intrinsics;
+    use ptr;
+    let msg = "fatal runtime error: out of memory\n";
+    unsafe {
+        // WriteFile silently fails if it is passed an invalid handle, so there
+        // is no need to check the result of GetStdHandle.
+        c::WriteFile(c::GetStdHandle(c::STD_ERROR_HANDLE),
+                     msg.as_ptr() as c::LPVOID,
+                     msg.len() as DWORD,
+                     ptr::null_mut(),
+                     ptr::null_mut());
+        intrinsics::abort();
+    }
+}
+
+pub fn init() {
+    oom::set_oom_handler(oom_handler);
+}
 
 pub fn decode_error_kind(errno: i32) -> ErrorKind {
     match errno as c::DWORD {