Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

micro-optimize dynamic allocation alignment #17095

Closed
wants to merge 2 commits into from
Closed

micro-optimize dynamic allocation alignment #17095

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
92b0926
micro-optimize dynamic allocation alignment
thestinger Sep 8, 2014
72a92b2
implement sized deallocation
thestinger Sep 5, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/jemalloc
Open in desktop
Submodule jemalloc updated 44 files
+3 −0 .autom4te.cfg
+1 −2 .gitignore
+3 −3 INSTALL
+7 −6 Makefile.in
+251 −126 bin/pprof
+27 −165 config.guess
+10 −11 config.sub
+145 −32 configure
+92 −31 configure.ac
+73 −2 doc/jemalloc.xml.in
+106 −90 include/jemalloc/internal/arena.h
+29 −12 include/jemalloc/internal/atomic.h
+1 −0 include/jemalloc/internal/chunk.h
+1 −1 include/jemalloc/internal/extent.h
+2 −2 include/jemalloc/internal/huge.h
+50 −21 include/jemalloc/internal/jemalloc_internal.h.in
+6 −0 include/jemalloc/internal/jemalloc_internal_defs.h.in
+20 −9 include/jemalloc/internal/private_symbols.txt
+1 −1 include/jemalloc/internal/prng.h
+207 −258 include/jemalloc/internal/prof.h
+13 −0 include/jemalloc/internal/rb.h
+4 −1 include/jemalloc/internal/size_classes.sh
+1 −1 include/jemalloc/jemalloc.sh
+7 −6 include/jemalloc/jemalloc_protos.h.in
+2 −0 include/jemalloc/jemalloc_typedefs.h.in
+4 −0 include/msvc_compat/C99/stdbool.h
+260 −465 src/arena.c
+1 −0 src/chunk.c
+123 −26 src/ctl.c
+7 −7 src/huge.c
+252 −165 src/jemalloc.c
+812 −409 src/prof.c
+1 −1 src/stats.c
+6 −9 src/tcache.c
+25 −9 src/zone.c
+2 −0 test/include/test/jemalloc_test.h.in
+15 −0 test/include/test/timer.h
+57 −0 test/integration/sdallocx.c
+10 −10 test/src/SFMT.c
+57 −0 test/src/timer.c
+160 −0 test/stress/microbench.c
+97 −0 test/unit/atomic.c
+13 −0 test/unit/mallctl.c
+3 −0 test/unit/rb.c
109 changes: 75 additions & 34 deletions src/liballoc/heap.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,12 +149,24 @@ unsafe fn closure_exchange_malloc(drop_glue: fn(*mut u8), size: uint,
alloc as *mut u8
}

// The minimum alignment guaranteed by the architecture. This value is used to
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"guaranteed by jemalloc on this architecture", or is it actually part of the architecture's ABI?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's part of the ABI and therefore jemalloc and other allocators have to provide it as the minimum. Since jemalloc has to be compatible with system allocators, I just took the numbers from there.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, cool. I was just wondering if this was an additional guarantee provided by jemalloc or not.

// add fast paths for low alignment values. In practice, the alignment is a
// constant at the call site and the branch will be optimized out.
#[cfg(target_arch = "arm")]
#[cfg(target_arch = "mips")]
#[cfg(target_arch = "mipsel")]
static MIN_ALIGN: uint = 8;
#[cfg(target_arch = "x86")]
#[cfg(target_arch = "x86_64")]
static MIN_ALIGN: uint = 16;

#[cfg(jemalloc)]
mod imp {
use core::option::{None, Option};
use core::ptr::{RawPtr, mut_null, null};
use core::num::Int;
use libc::{c_char, c_int, c_void, size_t};
use super::MIN_ALIGN;

#[link(name = "jemalloc", kind = "static")]
#[cfg(not(test))]
Expand All @@ -166,7 +178,10 @@ mod imp {
flags: c_int) -> *mut c_void;
fn je_xallocx(ptr: *mut c_void, size: size_t, extra: size_t,
flags: c_int) -> size_t;
#[cfg(stage0)]
fn je_dallocx(ptr: *mut c_void, flags: c_int);
#[cfg(not(stage0))]
fn je_sdallocx(ptr: *mut c_void, size: size_t, flags: c_int);
fn je_nallocx(size: size_t, flags: c_int) -> size_t;
fn je_malloc_stats_print(write_cb: Option<extern "C" fn(cbopaque: *mut c_void,
*const c_char)>,
Expand All @@ -183,9 +198,15 @@ mod imp {
#[inline(always)]
fn mallocx_align(a: uint) -> c_int { a.trailing_zeros() as c_int }

#[inline(always)]
fn align_to_flags(align: uint) -> c_int {
if align <= MIN_ALIGN { 0 } else { mallocx_align(align) }
}

#[inline]
pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
let ptr = je_mallocx(size as size_t, mallocx_align(align)) as *mut u8;
let flags = align_to_flags(align);
let ptr = je_mallocx(size as size_t, flags) as *mut u8;
if ptr.is_null() {
::oom()
}
Expand All @@ -195,8 +216,8 @@ mod imp {
#[inline]
pub unsafe fn reallocate(ptr: *mut u8, size: uint, align: uint,
_old_size: uint) -> *mut u8 {
let ptr = je_rallocx(ptr as *mut c_void, size as size_t,
mallocx_align(align)) as *mut u8;
let flags = align_to_flags(align);
let ptr = je_rallocx(ptr as *mut c_void, size as size_t, flags) as *mut u8;
if ptr.is_null() {
::oom()
}
Expand All @@ -206,18 +227,28 @@ mod imp {
#[inline]
pub unsafe fn reallocate_inplace(ptr: *mut u8, size: uint, align: uint,
_old_size: uint) -> bool {
je_xallocx(ptr as *mut c_void, size as size_t, 0,
mallocx_align(align)) == size as size_t
let flags = align_to_flags(align);
je_xallocx(ptr as *mut c_void, size as size_t, 0, flags) == size as size_t
}

#[inline]
#[cfg(stage0)]
pub unsafe fn deallocate(ptr: *mut u8, _size: uint, align: uint) {
je_dallocx(ptr as *mut c_void, mallocx_align(align))
let flags = align_to_flags(align);
je_dallocx(ptr as *mut c_void, flags)
}

#[inline]
#[cfg(not(stage0))]
pub unsafe fn deallocate(ptr: *mut u8, size: uint, align: uint) {
let flags = align_to_flags(align);
je_sdallocx(ptr as *mut c_void, size as size_t, flags)
}

#[inline]
pub fn usable_size(size: uint, align: uint) -> uint {
unsafe { je_nallocx(size as size_t, mallocx_align(align)) as uint }
let flags = align_to_flags(align);
unsafe { je_nallocx(size as size_t, flags) as uint }
}

pub fn stats_print() {
Expand All @@ -234,6 +265,7 @@ mod imp {
use core::ptr;
use libc;
use libc_heap;
use super::MIN_ALIGN;

extern {
fn posix_memalign(memptr: *mut *mut libc::c_void,
Expand All @@ -243,16 +275,7 @@ mod imp {

#[inline]
pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
// The posix_memalign manpage states
//
// alignment [...] must be a power of and a multiple of
// sizeof(void *)
//
// The `align` parameter to this function is the *minimum* alignment for
// a block of memory, so we special case everything under `*uint` to
// just pass it to malloc, which is guaranteed to align to at least the
// size of `*uint`.
if align < mem::size_of::<uint>() {
if align <= MIN_ALIGN {
libc_heap::malloc_raw(size)
} else {
let mut out = 0 as *mut libc::c_void;
Expand All @@ -269,10 +292,14 @@ mod imp {
#[inline]
pub unsafe fn reallocate(ptr: *mut u8, size: uint, align: uint,
old_size: uint) -> *mut u8 {
let new_ptr = allocate(size, align);
ptr::copy_memory(new_ptr, ptr as *const u8, cmp::min(size, old_size));
deallocate(ptr, old_size, align);
return new_ptr;
if align <= MIN_ALIGN {
libc_heap::realloc_raw(ptr, size)
} else {
let new_ptr = allocate(size, align);
ptr::copy_memory(new_ptr, ptr as *const u8, cmp::min(size, old_size));
deallocate(ptr, old_size, align);
new_ptr
}
}

#[inline]
Expand All @@ -291,14 +318,16 @@ mod imp {
size
}

pub fn stats_print() {
}
pub fn stats_print() {}
}

#[cfg(not(jemalloc), windows)]
mod imp {
use libc::{c_void, size_t};
use libc;
use libc_heap;
use core::ptr::RawPtr;
use super::MIN_ALIGN;

extern {
fn _aligned_malloc(size: size_t, align: size_t) -> *mut c_void;
Expand All @@ -309,22 +338,30 @@ mod imp {

#[inline]
pub unsafe fn allocate(size: uint, align: uint) -> *mut u8 {
let ptr = _aligned_malloc(size as size_t, align as size_t);
if ptr.is_null() {
::oom();
if align <= MIN_ALIGN {
libc_heap::malloc_raw(size)
} else {
let ptr = _aligned_malloc(size as size_t, align as size_t);
if ptr.is_null() {
::oom();
}
ptr as *mut u8
}
ptr as *mut u8
}

#[inline]
pub unsafe fn reallocate(ptr: *mut u8, size: uint, align: uint,
_old_size: uint) -> *mut u8 {
let ptr = _aligned_realloc(ptr as *mut c_void, size as size_t,
align as size_t);
if ptr.is_null() {
::oom();
if align <= MIN_ALIGN {
libc_heap::realloc_raw(ptr, size)
} else {
let ptr = _aligned_realloc(ptr as *mut c_void, size as size_t,
align as size_t);
if ptr.is_null() {
::oom();
}
ptr as *mut u8
}
ptr as *mut u8
}

#[inline]
Expand All @@ -334,8 +371,12 @@ mod imp {
}

#[inline]
pub unsafe fn deallocate(ptr: *mut u8, _size: uint, _align: uint) {
_aligned_free(ptr as *mut c_void)
pub unsafe fn deallocate(ptr: *mut u8, _size: uint, align: uint) {
if align <= MIN_ALIGN {
libc::free(ptr as *mut libc::c_void)
} else {
_aligned_free(ptr as *mut c_void)
}
}

#[inline]
Expand Down