Bug 1465709 - Hook rust OOM handler on rustc 1.28. r=froydnj

Bug 1458161 added a rust OOM handler based on an unstable API that was
removed in 1.27, replaced with something that didn't allow to get the
failed allocation size.

Latest 1.28 nightly (2018-06-13) has
rust-lang/rust#50880,
rust-lang/rust#51264 and
rust-lang/rust#51241 merged, which allow to
hook the OOM handler and get the failed allocation size again.

Because this is still an unstable API, we explicitly depend on strict
versions of rustc. We also explicitly error out if automation builds
end up using a rustc version that doesn't allow us to get the allocation
size for rust OOM, because we don't want that to happen without knowing.

--HG--
extra : rebase_source : 6c097151046d088cf51f4755dd69bde97bb8bd8b

toolkit/crashreporter/nsExceptionHandler.cpp

@@ -117,6 +117,7 @@ using mozilla::ipc::CrashReporterClient;
 // From toolkit/library/rust/shared/lib.rs
 extern "C" {
   void install_rust_panic_hook();
+  void install_rust_oom_hook();
   bool get_rust_panic_reason(char** reason, size_t* length);
 }
 
@@ -1687,6 +1688,8 @@ nsresult SetExceptionHandler(nsIFile* aXREDirectory,
 
   install_rust_panic_hook();
 
+  install_rust_oom_hook();
+
   InitThreadAnnotation();
 
   return NS_OK;

toolkit/library/gtest/rust/Cargo.toml

@@ -14,6 +14,7 @@ cubeb_pulse_rust = ["gkrust-shared/cubeb_pulse_rust"]
 gecko_debug = ["gkrust-shared/gecko_debug"]
 simd-accel = ["gkrust-shared/simd-accel"]
 oom_with_global_alloc = ["gkrust-shared/oom_with_global_alloc"]
+oom_with_hook = ["gkrust-shared/oom_with_hook"]
 moz_memory = ["gkrust-shared/moz_memory"]
 
 [dependencies]

toolkit/library/rust/Cargo.toml

@@ -14,6 +14,7 @@ cubeb_pulse_rust = ["gkrust-shared/cubeb_pulse_rust"]
 gecko_debug = ["gkrust-shared/gecko_debug"]
 simd-accel = ["gkrust-shared/simd-accel"]
 oom_with_global_alloc = ["gkrust-shared/oom_with_global_alloc"]
+oom_with_hook = ["gkrust-shared/oom_with_hook"]
 moz_memory = ["gkrust-shared/moz_memory"]
 
 [dependencies]

toolkit/library/rust/gkrust-features.mozbuild

@@ -30,3 +30,10 @@ if CONFIG['MOZ_MEMORY']:
 # A string test is not the best thing, but it works well enough here.
 if CONFIG['RUSTC_VERSION'] < "1.27":
     gkrust_features += ['oom_with_global_alloc']
+elif CONFIG['RUSTC_VERSION'] >= "1.28" and CONFIG['RUSTC_VERSION'] < "1.29":
+    gkrust_features += ['oom_with_hook']
+elif not CONFIG['MOZ_AUTOMATION']:
+    # We don't want builds on automation to unwillingly stop annotating OOM
+    # crash reports from rust.
+    error('Builds on automation must use a version of rust that supports OOM '
+          'hooking')

toolkit/library/rust/shared/Cargo.toml

@@ -38,6 +38,7 @@ cubeb_pulse_rust = ["cubeb-sys", "cubeb-pulse"]
 gecko_debug = ["geckoservo/gecko_debug", "nsstring/gecko_debug"]
 simd-accel = ["encoding_c/simd-accel", "encoding_glue/simd-accel"]
 oom_with_global_alloc = []
+oom_with_hook = []
 moz_memory = ["mp4parse_capi/mp4parse_fallible"]
 
 [lib]

toolkit/library/rust/shared/build.rs

@@ -3,6 +3,6 @@ fn main() {
     // versions of rustc, >= 1.24 < 1.27, that are not going to change
     // the unstable APIs we use from under us (1.26 being a beta as of
     // writing, and close to release).
-    #[cfg(feature = "oom_with_global_alloc")]
+    #[cfg(any(feature = "oom_with_global_alloc", feature = "oom_with_hook"))]
     println!("cargo:rustc-env=RUSTC_BOOTSTRAP=1");
 }

toolkit/library/rust/shared/lib.rs

@@ -4,6 +4,7 @@
 
 #![cfg_attr(feature = "oom_with_global_alloc",
             feature(global_allocator, alloc, alloc_system, allocator_api))]
+#![cfg_attr(feature = "oom_with_hook", feature(oom_hook))]
 
 #[cfg(feature="servo")]
 extern crate geckoservo;
@@ -218,3 +219,28 @@ mod global_alloc {
 #[cfg(feature = "oom_with_global_alloc")]
 #[global_allocator]
 static HEAP: global_alloc::GeckoHeap = global_alloc::GeckoHeap;
+
+#[cfg(feature = "oom_with_hook")]
+mod oom_hook {
+    use std::alloc::{Layout, set_oom_hook};
+
+    extern "C" {
+        fn GeckoHandleOOM(size: usize) -> !;
+    }
+
+    pub fn hook(layout: Layout) {
+        unsafe {
+            GeckoHandleOOM(layout.size());
+        }
+    }
+
+    pub fn install() {
+        set_oom_hook(hook);
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn install_rust_oom_hook() {
+    #[cfg(feature = "oom_with_hook")]
+    oom_hook::install();
+}