librustc: Speed up byte copy operations

Author: pcwalton

src/libcore/libc.rs

@@ -1123,10 +1123,23 @@ pub mod funcs {
             unsafe fn strerror(n: c_int) -> *c_char;
             unsafe fn strtok(s: *c_char, t: *c_char) -> *c_char;
             unsafe fn strxfrm(s: *c_char, ct: *c_char, n: size_t) -> size_t;
+
+            // These are fine to execute on the Rust stack. They must be, in
+            // fact, because LLVM generates calls to them!
+            #[rust_stack]
+            #[inline(always)]
             unsafe fn memcpy(s: *c_void, ct: *c_void, n: size_t) -> *c_void;
+            #[rust_stack]
+            #[inline(always)]
             unsafe fn memmove(s: *c_void, ct: *c_void, n: size_t) -> *c_void;
+            #[rust_stack]
+            #[inline(always)]
             unsafe fn memcmp(cx: *c_void, ct: *c_void, n: size_t) -> c_int;
+            #[rust_stack]
+            #[inline(always)]
             unsafe fn memchr(cx: *c_void, c: c_int, n: size_t) -> *c_void;
+            #[rust_stack]
+            #[inline(always)]
             unsafe fn memset(s: *c_void, c: c_int, n: size_t) -> *c_void;
         }
     }

src/libcore/ptr.rs

@@ -116,9 +116,16 @@ pub pure fn is_not_null<T>(ptr: *const T) -> bool { !is_null(ptr) }
  * and destination may overlap.
  */
 #[inline(always)]
+#[cfg(target_word_size = "32")]
 pub unsafe fn copy_memory<T>(dst: *mut T, src: *const T, count: uint) {
     let n = count * sys::size_of::<T>();
-    libc_::memmove(dst as *mut c_void, src as *c_void, n as size_t);
+    memmove32(dst as *mut u8, src as *u8, n as u32);
+}
+#[inline(always)]
+#[cfg(target_word_size = "64")]
+pub unsafe fn copy_memory<T>(dst: *mut T, src: *const T, count: uint) {
+    let n = count * sys::size_of::<T>();
+    memmove64(dst as *mut u8, src as *u8, n as u64);
 }
 
 #[inline(always)]
@@ -183,6 +190,23 @@ pub trait Ptr<T> {
     pure fn offset(count: uint) -> self;
 }
 
+#[cfg(stage0)]
+unsafe fn memmove32(dst: *mut u8, src: *const u8, count: u32) {
+    libc::memmove(dst as *c_void, src as *c_void, count as size_t);
+}
+#[cfg(stage0)]
+unsafe fn memmove64(dst: *mut u8, src: *const u8, count: u64) {
+    libc::memmove(dst as *c_void, src as *c_void, count as size_t);
+}
+
+#[abi="rust-intrinsic"]
+#[cfg(stage1)]
+#[cfg(stage2)]
+pub extern {
+    fn memmove32(dst: *mut u8, src: *u8, size: u32);
+    fn memmove64(dst: *mut u8, src: *u8, size: u64);
+}
+
 /// Extension methods for immutable pointers
 impl<T> *T: Ptr<T> {
     /// Returns true if the pointer is equal to the null pointer.

src/libcore/vec.rs

@@ -266,6 +266,7 @@ pub pure fn slice<T: Copy>(v: &[const T], start: uint, end: uint) -> ~[T] {
 }
 
 /// Return a slice that points into another slice.
+#[inline(always)]
 pub pure fn view<T>(v: &r/[T], start: uint, end: uint) -> &r/[T] {
     assert (start <= end);
     assert (end <= len(v));
@@ -279,6 +280,7 @@ pub pure fn view<T>(v: &r/[T], start: uint, end: uint) -> &r/[T] {
 }
 
 /// Return a slice that points into another slice.
+#[inline(always)]
 pub pure fn mut_view<T>(v: &r/[mut T], start: uint, end: uint) -> &r/[mut T] {
     assert (start <= end);
     assert (end <= len(v));
@@ -292,6 +294,7 @@ pub pure fn mut_view<T>(v: &r/[mut T], start: uint, end: uint) -> &r/[mut T] {
 }
 
 /// Return a slice that points into another slice.
+#[inline(always)]
 pub pure fn const_view<T>(v: &r/[const T], start: uint,
                       end: uint) -> &r/[const T] {
     assert (start <= end);
@@ -305,6 +308,8 @@ pub pure fn const_view<T>(v: &r/[const T], start: uint,
     }
 }
 
+/// Copies 
+
 /// Split the vector `v` by applying each element against the predicate `f`.
 pub fn split<T: Copy>(v: &[T], f: fn(t: &T) -> bool) -> ~[~[T]] {
     let ln = len(v);
@@ -2127,6 +2132,7 @@ pub mod raw {
       * Copies `count` bytes from `src` to `dst`. The source and destination
       * may overlap.
       */
+    #[inline(always)]
     pub unsafe fn copy_memory<T>(dst: &[mut T], src: &[const T],
                                  count: uint) {
         assert dst.len() >= count;
@@ -2193,6 +2199,7 @@ pub mod bytes {
       * Copies `count` bytes from `src` to `dst`. The source and destination
       * may overlap.
       */
+    #[inline(always)]
     pub fn copy_memory(dst: &[mut u8], src: &[const u8], count: uint) {
         // Bound checks are done at vec::raw::copy_memory.
         unsafe { vec::raw::copy_memory(dst, src, count) }

src/librustc/middle/trans/base.rs

@@ -2593,9 +2593,15 @@ fn declare_intrinsics(llmod: ModuleRef) -> HashMap<~str, ValueRef> {
                            T_void()));
     let memcpy32 =
         decl_cdecl_fn(llmod, ~"llvm.memcpy.p0i8.p0i8.i32",
-                      T_fn(T_memcpy32_args, T_void()));
+                      T_fn(copy T_memcpy32_args, T_void()));
     let memcpy64 =
         decl_cdecl_fn(llmod, ~"llvm.memcpy.p0i8.p0i8.i64",
+                      T_fn(copy T_memcpy64_args, T_void()));
+    let memmove32 =
+        decl_cdecl_fn(llmod, ~"llvm.memmove.p0i8.p0i8.i32",
+                      T_fn(T_memcpy32_args, T_void()));
+    let memmove64 =
+        decl_cdecl_fn(llmod, ~"llvm.memmove.p0i8.p0i8.i64",
                       T_fn(T_memcpy64_args, T_void()));
     let memset32 =
         decl_cdecl_fn(llmod, ~"llvm.memset.p0i8.i32",
@@ -2704,6 +2710,8 @@ fn declare_intrinsics(llmod: ModuleRef) -> HashMap<~str, ValueRef> {
     intrinsics.insert(~"llvm.gcread", gcread);
     intrinsics.insert(~"llvm.memcpy.p0i8.p0i8.i32", memcpy32);
     intrinsics.insert(~"llvm.memcpy.p0i8.p0i8.i64", memcpy64);
+    intrinsics.insert(~"llvm.memmove.p0i8.p0i8.i32", memmove32);
+    intrinsics.insert(~"llvm.memmove.p0i8.p0i8.i64", memmove64);
     intrinsics.insert(~"llvm.memset.p0i8.i32", memset32);
     intrinsics.insert(~"llvm.memset.p0i8.i64", memset64);
     intrinsics.insert(~"llvm.trap", trap);

src/librustc/middle/trans/foreign.rs

@@ -568,6 +568,24 @@ fn trans_intrinsic(ccx: @crate_ctxt, decl: ValueRef, item: @ast::foreign_item,
                                              T_ptr(T_nil()));
             Store(bcx, morestack_addr, fcx.llretptr);
         }
+        ~"memmove32" => {
+            let dst_ptr = get_param(decl, first_real_arg);
+            let src_ptr = get_param(decl, first_real_arg + 1);
+            let size = get_param(decl, first_real_arg + 2);
+            let align = C_i32(1);
+            let volatile = C_bool(false);
+            let llfn = bcx.ccx().intrinsics.get(~"llvm.memmove.p0i8.p0i8.i32");
+            Call(bcx, llfn, ~[dst_ptr, src_ptr, size, align, volatile]);
+        }
+        ~"memmove64" => {
+            let dst_ptr = get_param(decl, first_real_arg);
+            let src_ptr = get_param(decl, first_real_arg + 1);
+            let size = get_param(decl, first_real_arg + 2);
+            let align = C_i32(1);
+            let volatile = C_bool(false);
+            let llfn = bcx.ccx().intrinsics.get(~"llvm.memmove.p0i8.p0i8.i64");
+            Call(bcx, llfn, ~[dst_ptr, src_ptr, size, align, volatile]);
+        }
         ~"sqrtf32" => {
             let x = get_param(decl, first_real_arg);
             let sqrtf = ccx.intrinsics.get(~"llvm.sqrt.f32");

src/librustc/middle/trans/type_use.rs

@@ -129,6 +129,8 @@ fn type_uses_for(ccx: @crate_ctxt, fn_id: def_id, n_tps: uint)
                 ~"visit_tydesc"  | ~"forget" | ~"addr_of" |
                 ~"frame_address" | ~"morestack_addr" => 0,
 
+                ~"memmove32" | ~"memmove64" => 0,
+
                 ~"sqrtf32" | ~"sqrtf64" | ~"powif32" | ~"powif64" |
                 ~"sinf32"  | ~"sinf64"  | ~"cosf32"  | ~"cosf64"  |
                 ~"powf32"  | ~"powf64"  | ~"expf32"  | ~"expf64"  |

src/librustc/middle/typeck/check/mod.rs

@@ -3152,6 +3152,28 @@ fn check_intrinsic_type(ccx: @crate_ctxt, it: @ast::foreign_item) {
       ~"morestack_addr" => {
         (0u, ~[], ty::mk_nil_ptr(tcx))
       }
+      ~"memmove32" => {
+        (0, ~[arg(ast::by_copy,
+                  ty::mk_ptr(tcx,
+                    ty::mt { ty: ty::mk_u8(tcx), mutbl: ast::m_mutbl })),
+              arg(ast::by_copy,
+                  ty::mk_ptr(tcx,
+                    ty::mt { ty: ty::mk_u8(tcx), mutbl: ast::m_imm })),
+              arg(ast::by_copy,
+                  ty::mk_u32(tcx))],
+         ty::mk_nil(tcx))
+      }
+      ~"memmove64" => {
+        (0, ~[arg(ast::by_copy,
+                  ty::mk_ptr(tcx,
+                    ty::mt { ty: ty::mk_u8(tcx), mutbl: ast::m_mutbl })),
+              arg(ast::by_copy,
+                  ty::mk_ptr(tcx,
+                    ty::mt { ty: ty::mk_u8(tcx), mutbl: ast::m_imm })),
+              arg(ast::by_copy,
+                  ty::mk_u64(tcx))],
+         ty::mk_nil(tcx))
+      }
      ~"sqrtf32" => {
         (0u, ~[arg(ast::by_copy, ty::mk_f32(tcx))],
          ty::mk_f32(tcx))