Add #[inline] attribute to the inner functions (#596)

Author: newpavlov

src/backends/apple_other.rs

@@ -4,6 +4,7 @@ use core::{ffi::c_void, mem::MaybeUninit};
 
 pub use crate::util::{inner_u32, inner_u64};
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     let dst_ptr = dest.as_mut_ptr().cast::<c_void>();
     let ret = unsafe { libc::CCRandomGenerateBytes(dst_ptr, dest.len()) };

src/backends/custom.rs

@@ -4,6 +4,7 @@ use core::mem::MaybeUninit;
 
 pub use crate::util::{inner_u32, inner_u64};
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     extern "Rust" {
         fn __getrandom_v03_custom(dest: *mut u8, len: usize) -> Result<(), Error>;

src/backends/esp_idf.rs

@@ -8,6 +8,7 @@ extern "C" {
     fn esp_fill_random(buf: *mut c_void, len: usize) -> u32;
 }
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     // Not that NOT enabling WiFi, BT, or the voltage noise entropy source (via `bootloader_random_enable`)
     // will cause ESP-IDF to return pseudo-random numbers based on the voltage noise entropy, after the initial boot process:

src/backends/fuchsia.rs

@@ -9,6 +9,7 @@ extern "C" {
     fn zx_cprng_draw(buffer: *mut u8, length: usize);
 }
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     unsafe { zx_cprng_draw(dest.as_mut_ptr().cast::<u8>(), dest.len()) }
     Ok(())

src/backends/getentropy.rs

@@ -15,6 +15,7 @@ pub use crate::util::{inner_u32, inner_u64};
 #[path = "../util_libc.rs"]
 mod util_libc;
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     for chunk in dest.chunks_mut(256) {
         let ret = unsafe { libc::getentropy(chunk.as_mut_ptr().cast::<c_void>(), chunk.len()) };

src/backends/getrandom.rs

@@ -23,6 +23,7 @@ pub use crate::util::{inner_u32, inner_u64};
 #[path = "../util_libc.rs"]
 mod util_libc;
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     util_libc::sys_fill_exact(dest, |buf| unsafe {
         libc::getrandom(buf.as_mut_ptr().cast::<c_void>(), buf.len(), 0)

src/backends/hermit.rs

@@ -12,6 +12,7 @@ extern "C" {
     fn sys_secure_rand64(value: *mut u64) -> i32;
 }
 
+#[inline]
 pub fn inner_u32() -> Result<u32, Error> {
     let mut res = MaybeUninit::uninit();
     let ret = unsafe { sys_secure_rand32(res.as_mut_ptr()) };
@@ -22,6 +23,7 @@ pub fn inner_u32() -> Result<u32, Error> {
     }
 }
 
+#[inline]
 pub fn inner_u64() -> Result<u64, Error> {
     let mut res = MaybeUninit::uninit();
     let ret = unsafe { sys_secure_rand64(res.as_mut_ptr()) };
@@ -32,6 +34,7 @@ pub fn inner_u64() -> Result<u64, Error> {
     }
 }
 
+#[inline]
 pub fn fill_inner(mut dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     while !dest.is_empty() {
         let res = unsafe { sys_read_entropy(dest.as_mut_ptr().cast::<u8>(), dest.len(), 0) };

src/backends/linux_android.rs

@@ -10,6 +10,7 @@ mod util_libc;
 #[cfg(not(any(target_os = "android", target_os = "linux")))]
 compile_error!("`linux_getrandom` backend can be enabled only for Linux/Android targets!");
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     util_libc::sys_fill_exact(dest, |buf| unsafe {
         libc::getrandom(buf.as_mut_ptr().cast(), buf.len(), 0)

src/backends/linux_android_with_fallback.rs

@@ -20,6 +20,7 @@ const NOT_AVAILABLE: NonNull<c_void> = unsafe { NonNull::new_unchecked(usize::MA
 static GETRANDOM_FN: AtomicPtr<c_void> = AtomicPtr::new(ptr::null_mut());
 
 #[cold]
+#[inline(never)]
 fn init() -> NonNull<c_void> {
     static NAME: &[u8] = b"getrandom\0";
     let name_ptr = NAME.as_ptr().cast::<libc::c_char>();
@@ -59,6 +60,7 @@ fn use_file_fallback(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     use_file::fill_inner(dest)
 }
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     // Despite being only a single atomic variable, we still cannot always use
     // Ordering::Relaxed, as we need to make sure a successful call to `init`
@@ -75,7 +77,7 @@ pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     if fptr == NOT_AVAILABLE {
         use_file_fallback(dest)
     } else {
-        // note: `transume` is currently the only way to convert pointer into function reference
+        // note: `transmute` is currently the only way to convert a pointer into a function reference
         let getrandom_fn = unsafe { mem::transmute::<NonNull<c_void>, GetRandomFn>(fptr) };
         util_libc::sys_fill_exact(dest, |buf| unsafe {
             getrandom_fn(buf.as_mut_ptr().cast(), buf.len(), 0)

src/backends/netbsd.rs

@@ -45,6 +45,7 @@ type GetRandomFn = unsafe extern "C" fn(*mut c_void, libc::size_t, libc::c_uint)
 static GETRANDOM: AtomicPtr<c_void> = AtomicPtr::new(ptr::null_mut());
 
 #[cold]
+#[inline(never)]
 fn init() -> *mut c_void {
     static NAME: &[u8] = b"getrandom\0";
     let name_ptr = NAME.as_ptr().cast::<libc::c_char>();
@@ -58,6 +59,7 @@ fn init() -> *mut c_void {
     ptr
 }
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     // Despite being only a single atomic variable, we still cannot always use
     // Ordering::Relaxed, as we need to make sure a successful call to `init`

src/backends/rdrand.rs

@@ -147,6 +147,7 @@ unsafe fn rdrand_u64() -> Option<u64> {
     Some((u64::from(a) << 32) | u64::from(b))
 }
 
+#[inline]
 pub fn inner_u32() -> Result<u32, Error> {
     if !RDRAND_GOOD.unsync_init(is_rdrand_good) {
         return Err(Error::NO_RDRAND);
@@ -155,6 +156,7 @@ pub fn inner_u32() -> Result<u32, Error> {
     unsafe { rdrand_u32() }.ok_or(Error::FAILED_RDRAND)
 }
 
+#[inline]
 pub fn inner_u64() -> Result<u64, Error> {
     if !RDRAND_GOOD.unsync_init(is_rdrand_good) {
         return Err(Error::NO_RDRAND);
@@ -163,6 +165,7 @@ pub fn inner_u64() -> Result<u64, Error> {
     unsafe { rdrand_u64() }.ok_or(Error::FAILED_RDRAND)
 }
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     if !RDRAND_GOOD.unsync_init(is_rdrand_good) {
         return Err(Error::NO_RDRAND);

src/backends/rndr.rs

@@ -108,6 +108,7 @@ fn is_rndr_available() -> bool {
     }
 }
 
+#[inline]
 pub fn inner_u32() -> Result<u32, Error> {
     if !is_rndr_available() {
         return Err(Error::RNDR_NOT_AVAILABLE);
@@ -117,6 +118,7 @@ pub fn inner_u32() -> Result<u32, Error> {
     res.map(truncate).ok_or(Error::RNDR_FAILURE)
 }
 
+#[inline]
 pub fn inner_u64() -> Result<u64, Error> {
     if !is_rndr_available() {
         return Err(Error::RNDR_NOT_AVAILABLE);
@@ -126,6 +128,7 @@ pub fn inner_u64() -> Result<u64, Error> {
     res.ok_or(Error::RNDR_FAILURE)
 }
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     if !is_rndr_available() {
         return Err(Error::RNDR_NOT_AVAILABLE);

src/backends/solaris.rs

@@ -22,6 +22,7 @@ mod util_libc;
 
 const MAX_BYTES: usize = 1024;
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     for chunk in dest.chunks_mut(MAX_BYTES) {
         let ptr = chunk.as_mut_ptr().cast::<c_void>();

src/backends/solid.rs

@@ -8,6 +8,7 @@ extern "C" {
     pub fn SOLID_RNG_SampleRandomBytes(buffer: *mut u8, length: usize) -> i32;
 }
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     let ret = unsafe { SOLID_RNG_SampleRandomBytes(dest.as_mut_ptr().cast::<u8>(), dest.len()) };
     if ret >= 0 {

src/backends/use_file.rs

@@ -40,6 +40,7 @@ const FD_ONGOING_INIT: libc::c_int = -2;
 // `Ordering::Acquire` to synchronize with it.
 static FD: AtomicI32 = AtomicI32::new(FD_UNINIT);
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     let mut fd = FD.load(Ordering::Acquire);
     if fd == FD_UNINIT || fd == FD_ONGOING_INIT {
@@ -77,6 +78,7 @@ fn open_readonly(path: &[u8]) -> Result<libc::c_int, Error> {
 }
 
 #[cold]
+#[inline(never)]
 fn open_or_wait() -> Result<libc::c_int, Error> {
     loop {
         match FD.load(Ordering::Acquire) {

src/backends/vxworks.rs

@@ -11,20 +11,25 @@ mod util_libc;
 
 pub use crate::util::{inner_u32, inner_u64};
 
+static RNG_INIT: AtomicBool = AtomicBool::new(false);
+
+#[cold]
+fn init() -> Result<(), Error> {
+    let ret = unsafe { libc::randSecure() };
+    match ret.cmp(&0) {
+        Greater => RNG_INIT.store(true, Relaxed),
+        Equal => unsafe {
+            libc::usleep(10);
+        },
+        Less => return Err(Error::VXWORKS_RAND_SECURE),
+    }
+    Ok(())
+}
+
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
-    static RNG_INIT: AtomicBool = AtomicBool::new(false);
     while !RNG_INIT.load(Relaxed) {
-        let ret = unsafe { libc::randSecure() };
-        match ret.cmp(&0) {
-            Greater => {
-                RNG_INIT.store(true, Relaxed);
-                break;
-            }
-            Equal => unsafe {
-                libc::usleep(10);
-            },
-            Less => return Err(Error::VXWORKS_RAND_SECURE),
-        }
+        init()?;
     }
 
     // Prevent overflow of i32

src/backends/wasi_p1.rs

@@ -11,6 +11,7 @@ extern "C" {
     fn random_get(arg0: i32, arg1: i32) -> i32;
 }
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     // Based on the wasi code:
     // https://docs.rs/wasi/0.11.0+wasi-snapshot-preview1/src/wasi/lib_generated.rs.html#2046-2062

src/backends/wasi_p2.rs

@@ -3,15 +3,18 @@ use crate::Error;
 use core::mem::MaybeUninit;
 use wasi::random::random::get_random_u64;
 
+#[inline]
 pub fn inner_u32() -> Result<u32, Error> {
     let val = get_random_u64();
     Ok(crate::util::truncate(val))
 }
 
+#[inline]
 pub fn inner_u64() -> Result<u64, Error> {
     Ok(get_random_u64())
 }
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     use core::ptr::copy_nonoverlapping;
     use wasi::random::random::get_random_u64;

src/backends/wasm_js.rs

@@ -14,6 +14,7 @@ use wasm_bindgen::{prelude::wasm_bindgen, JsValue};
 const MAX_BUFFER_SIZE: usize = 65536;
 
 #[cfg(not(target_feature = "atomics"))]
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     for chunk in dest.chunks_mut(MAX_BUFFER_SIZE) {
         if get_random_values(chunk).is_err() {

src/backends/windows.rs

@@ -34,6 +34,7 @@ pub use crate::util::{inner_u32, inner_u64};
 // https://github.com/microsoft/windows-rs/blob/0.60.0/crates/libs/targets/src/lib.rs
 windows_targets::link!("bcryptprimitives.dll" "system" fn ProcessPrng(pbdata: *mut u8, cbdata: usize) -> i32);
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     let result = unsafe { ProcessPrng(dest.as_mut_ptr().cast::<u8>(), dest.len()) };
     // Since Windows 10, calls to the user-mode RNG are guaranteed to never

src/backends/windows7.rs

@@ -25,6 +25,7 @@ extern "system" {
 type BOOLEAN = u8;
 const TRUE: BOOLEAN = 1u8;
 
+#[inline]
 pub fn fill_inner(dest: &mut [MaybeUninit<u8>]) -> Result<(), Error> {
     // Prevent overflow of u32
     let chunk_size = usize::try_from(i32::MAX).expect("Windows does not support 16-bit targets");

src/util.rs

@@ -49,6 +49,7 @@ fn ptr_from_ref<T: ?Sized>(r: &T) -> *const T {
 }
 
 /// Default implementation of `inner_u32` on top of `fill_uninit`
+#[inline]
 pub fn inner_u32() -> Result<u32, Error> {
     let mut res = MaybeUninit::<u32>::uninit();
     // SAFETY: the created slice has the same size as `res`
@@ -63,6 +64,7 @@ pub fn inner_u32() -> Result<u32, Error> {
 }
 
 /// Default implementation of `inner_u64` on top of `fill_uninit`
+#[inline]
 pub fn inner_u64() -> Result<u64, Error> {
     let mut res = MaybeUninit::<u64>::uninit();
     // SAFETY: the created slice has the same size as `res`