remove headers from exchange allocations

src/librustc/middle/lang_items.rs

@@ -63,33 +63,34 @@ pub enum LangItem {
     FailFnLangItem,                    // 24
     FailBoundsCheckFnLangItem,         // 25
     ExchangeMallocFnLangItem,          // 26
-    ClosureExchangeMallocFnLangItem,   // 27
-    ExchangeFreeFnLangItem,            // 28
-    MallocFnLangItem,                  // 29
-    FreeFnLangItem,                    // 30
-    BorrowAsImmFnLangItem,             // 31
-    BorrowAsMutFnLangItem,             // 32
-    ReturnToMutFnLangItem,             // 33
-    CheckNotBorrowedFnLangItem,        // 34
-    StrDupUniqFnLangItem,              // 35
-    RecordBorrowFnLangItem,            // 36
-    UnrecordBorrowFnLangItem,          // 37
-
-    StartFnLangItem,                   // 38
-
-    TyDescStructLangItem,              // 39
-    TyVisitorTraitLangItem,            // 40
-    OpaqueStructLangItem,              // 41
+    VectorExchangeMallocFnLangItem,    // 27
+    ClosureExchangeMallocFnLangItem,   // 28
+    ExchangeFreeFnLangItem,            // 29
+    MallocFnLangItem,                  // 30
+    FreeFnLangItem,                    // 31
+    BorrowAsImmFnLangItem,             // 32
+    BorrowAsMutFnLangItem,             // 33
+    ReturnToMutFnLangItem,             // 34
+    CheckNotBorrowedFnLangItem,        // 35
+    StrDupUniqFnLangItem,              // 36
+    RecordBorrowFnLangItem,            // 37
+    UnrecordBorrowFnLangItem,          // 38
+
+    StartFnLangItem,                   // 39
+
+    TyDescStructLangItem,              // 40
+    TyVisitorTraitLangItem,            // 41
+    OpaqueStructLangItem,              // 42
 }
 
 pub struct LanguageItems {
-    items: [Option<def_id>, ..42]
+    items: [Option<def_id>, ..43]
 }
 
 impl LanguageItems {
     pub fn new() -> LanguageItems {
         LanguageItems {
-            items: [ None, ..42 ]
+            items: [ None, ..43 ]
         }
     }
 
@@ -129,23 +130,24 @@ impl LanguageItems {
             24 => "fail_",
             25 => "fail_bounds_check",
             26 => "exchange_malloc",
-            27 => "closure_exchange_malloc",
-            28 => "exchange_free",
-            29 => "malloc",
-            30 => "free",
-            31 => "borrow_as_imm",
-            32 => "borrow_as_mut",
-            33 => "return_to_mut",
-            34 => "check_not_borrowed",
-            35 => "strdup_uniq",
-            36 => "record_borrow",
-            37 => "unrecord_borrow",
-
-            38 => "start",
-
-            39 => "ty_desc",
-            40 => "ty_visitor",
-            41 => "opaque",
+            27 => "vector_exchange_malloc",
+            28 => "closure_exchange_malloc",
+            29 => "exchange_free",
+            30 => "malloc",
+            31 => "free",
+            32 => "borrow_as_imm",
+            33 => "borrow_as_mut",
+            34 => "return_to_mut",
+            35 => "check_not_borrowed",
+            36 => "strdup_uniq",
+            37 => "record_borrow",
+            38 => "unrecord_borrow",
+
+            39 => "start",
+
+            40 => "ty_desc",
+            41 => "ty_visitor",
+            42 => "opaque",
 
             _ => "???"
         }
@@ -238,6 +240,9 @@ impl LanguageItems {
     pub fn exchange_malloc_fn(&self) -> def_id {
         self.items[ExchangeMallocFnLangItem as uint].get()
     }
+    pub fn vector_exchange_malloc_fn(&self) -> def_id {
+        self.items[VectorExchangeMallocFnLangItem as uint].get()
+    }
     pub fn closure_exchange_malloc_fn(&self) -> def_id {
         self.items[ClosureExchangeMallocFnLangItem as uint].get()
     }
@@ -331,6 +336,7 @@ impl<'self> LanguageItemCollector<'self> {
         item_refs.insert(@"fail_bounds_check",
                          FailBoundsCheckFnLangItem as uint);
         item_refs.insert(@"exchange_malloc", ExchangeMallocFnLangItem as uint);
+        item_refs.insert(@"vector_exchange_malloc", VectorExchangeMallocFnLangItem as uint);
         item_refs.insert(@"closure_exchange_malloc", ClosureExchangeMallocFnLangItem as uint);
         item_refs.insert(@"exchange_free", ExchangeFreeFnLangItem as uint);
         item_refs.insert(@"malloc", MallocFnLangItem as uint);

src/librustc/middle/trans/_match.rs

@@ -1395,8 +1395,12 @@ pub fn compile_submatch(bcx: block,
     }
 
     if any_uniq_pat(m, col) {
+        let pat_ty = node_id_type(bcx, pat_id);
         let llbox = Load(bcx, val);
-        let unboxed = GEPi(bcx, llbox, [0u, abi::box_field_body]);
+        let unboxed = match ty::get(pat_ty).sty {
+            ty::ty_uniq(*) if !ty::type_contents(bcx.tcx(), pat_ty).contains_managed() => llbox,
+            _ => GEPi(bcx, llbox, [0u, abi::box_field_body])
+        };
         compile_submatch(bcx, enter_uniq(bcx, dm, m, col, val),
                          vec::append(~[unboxed], vals_left), chk);
         return;
@@ -1868,8 +1872,12 @@ pub fn bind_irrefutable_pat(bcx: block,
             }
         }
         ast::pat_box(inner) | ast::pat_uniq(inner) => {
+            let pat_ty = node_id_type(bcx, pat.id);
             let llbox = Load(bcx, val);
-            let unboxed = GEPi(bcx, llbox, [0u, abi::box_field_body]);
+            let unboxed = match ty::get(pat_ty).sty {
+                ty::ty_uniq(*) if !ty::type_contents(bcx.tcx(), pat_ty).contains_managed() => llbox,
+                    _ => GEPi(bcx, llbox, [0u, abi::box_field_body])
+            };
             bcx = bind_irrefutable_pat(bcx,
                                        inner,
                                        unboxed,

src/librustc/middle/trans/base.rs

@@ -289,21 +289,25 @@ pub fn malloc_raw_dyn(bcx: block,
     let _icx = push_ctxt("malloc_raw");
     let ccx = bcx.ccx();
 
-    let (mk_fn, langcall) = match heap {
-        heap_managed | heap_managed_unique => {
-            (ty::mk_imm_box, bcx.tcx().lang_items.malloc_fn())
-        }
-        heap_exchange => {
-            (ty::mk_imm_uniq, bcx.tcx().lang_items.exchange_malloc_fn())
-        }
-        heap_exchange_closure => {
-            (ty::mk_imm_uniq, bcx.tcx().lang_items.closure_exchange_malloc_fn())
-        }
-    };
-
     if heap == heap_exchange {
+        let llty_value = type_of::type_of(ccx, t);
+        let llalign = llalign_of_min(ccx, llty_value);
+
+        // Allocate space:
+        let rval = alloca(bcx, Type::i8p());
+        let bcx = callee::trans_lang_call(
+            bcx,
+            bcx.tcx().lang_items.exchange_malloc_fn(),
+            [C_i32(llalign as i32), size],
+            expr::SaveIn(rval));
+        rslt(bcx, PointerCast(bcx, Load(bcx, rval), llty_value.ptr_to()))
+    } else if heap == heap_exchange_vector {
         // Grab the TypeRef type of box_ptr_ty.
-        let box_ptr_ty = mk_fn(bcx.tcx(), t);
+        let element_type = match ty::get(t).sty {
+            ty::ty_unboxed_vec(e) => e,
+            _ => fail!("not a vector body")
+        };
+        let box_ptr_ty = ty::mk_evec(bcx.tcx(), element_type, ty::vstore_uniq);
         let llty = type_of(ccx, box_ptr_ty);
 
         let llty_value = type_of::type_of(ccx, t);
@@ -313,11 +317,22 @@ pub fn malloc_raw_dyn(bcx: block,
         let rval = alloca(bcx, Type::i8p());
         let bcx = callee::trans_lang_call(
             bcx,
-            langcall,
+            bcx.tcx().lang_items.vector_exchange_malloc_fn(),
             [C_i32(llalign as i32), size],
             expr::SaveIn(rval));
         rslt(bcx, PointerCast(bcx, Load(bcx, rval), llty))
     } else {
+        // we treat ~fn, @fn and @[] as @ here, which isn't ideal
+        let (mk_fn, langcall) = match heap {
+            heap_managed | heap_managed_unique => {
+                (ty::mk_imm_box, bcx.tcx().lang_items.malloc_fn())
+            }
+            heap_exchange_closure => {
+                (ty::mk_imm_box, bcx.tcx().lang_items.closure_exchange_malloc_fn())
+            }
+            _ => fail!("heap_exchange/heap_exchange_vector already handled")
+        };
+
         // Grab the TypeRef type of box_ptr_ty.
         let box_ptr_ty = mk_fn(bcx.tcx(), t);
         let llty = type_of(ccx, box_ptr_ty);
@@ -359,16 +374,17 @@ pub struct MallocResult {
 // and pulls out the body
 pub fn malloc_general_dyn(bcx: block, t: ty::t, heap: heap, size: ValueRef)
     -> MallocResult {
+    assert!(heap != heap_exchange);
     let _icx = push_ctxt("malloc_general");
     let Result {bcx: bcx, val: llbox} = malloc_raw_dyn(bcx, t, heap, size);
     let body = GEPi(bcx, llbox, [0u, abi::box_field_body]);
 
     MallocResult { bcx: bcx, box: llbox, body: body }
 }
 
-pub fn malloc_general(bcx: block, t: ty::t, heap: heap)
-    -> MallocResult {
-        let ty = type_of(bcx.ccx(), t);
+pub fn malloc_general(bcx: block, t: ty::t, heap: heap) -> MallocResult {
+    let ty = type_of(bcx.ccx(), t);
+    assert!(heap != heap_exchange);
     malloc_general_dyn(bcx, t, heap, llsize_of(bcx.ccx(), ty))
 }
 pub fn malloc_boxed(bcx: block, t: ty::t)
@@ -385,6 +401,7 @@ pub fn heap_for_unique(bcx: block, t: ty::t) -> heap {
 }
 
 pub fn maybe_set_managed_unique_rc(bcx: block, bx: ValueRef, heap: heap) {
+    assert!(heap != heap_exchange);
     if heap == heap_managed_unique {
         // In cases where we are looking at a unique-typed allocation in the
         // managed heap (thus have refcount 1 from the managed allocator),
@@ -396,11 +413,6 @@ pub fn maybe_set_managed_unique_rc(bcx: block, bx: ValueRef, heap: heap) {
     }
 }
 
-pub fn malloc_unique(bcx: block, t: ty::t)
-    -> MallocResult {
-    malloc_general(bcx, t, heap_for_unique(bcx, t))
-}
-
 // Type descriptor and type glue stuff
 
 pub fn get_tydesc_simple(ccx: &mut CrateContext, t: ty::t) -> ValueRef {

src/librustc/middle/trans/common.rs

@@ -274,6 +274,7 @@ pub enum heap {
     heap_managed,
     heap_managed_unique,
     heap_exchange,
+    heap_exchange_vector,
     heap_exchange_closure
 }
 
@@ -395,7 +396,7 @@ pub fn add_clean_free(cx: block, ptr: ValueRef, heap: heap) {
         let f: @fn(block) -> block = |a| glue::trans_free(a, ptr);
         f
       }
-      heap_exchange | heap_exchange_closure => {
+      heap_exchange | heap_exchange_vector | heap_exchange_closure => {
         let f: @fn(block) -> block = |a| glue::trans_exchange_free(a, ptr);
         f
       }

src/librustc/middle/trans/datum.rs

@@ -100,6 +100,7 @@ use middle::trans::glue;
 use middle::trans::tvec;
 use middle::trans::type_of;
 use middle::trans::write_guard;
+use middle::trans::type_::Type;
 use middle::ty;
 use util::common::indenter;
 use util::ppaux::ty_to_str;
@@ -567,18 +568,31 @@ impl Datum {
          * This datum must represent an @T or ~T box.  Returns a new
          * by-ref datum of type T, pointing at the contents. */
 
-        let content_ty = match ty::get(self.ty).sty {
-            ty::ty_box(mt) | ty::ty_uniq(mt) => mt.ty,
+        let (content_ty, header) = match ty::get(self.ty).sty {
+            ty::ty_box(mt) => (mt.ty, true),
+            ty::ty_uniq(mt) => (mt.ty, false),
+            ty::ty_evec(_, ty::vstore_uniq) | ty::ty_estr(ty::vstore_uniq) => {
+                let unit_ty = ty::sequence_element_type(bcx.tcx(), self.ty);
+                let unboxed_vec_ty = ty::mk_mut_unboxed_vec(bcx.tcx(), unit_ty);
+                (unboxed_vec_ty, true)
+            }
             _ => {
                 bcx.tcx().sess.bug(fmt!(
                     "box_body() invoked on non-box type %s",
                     ty_to_str(bcx.tcx(), self.ty)));
             }
         };
 
-        let ptr = self.to_value_llval(bcx);
-        let body = opaque_box_body(bcx, content_ty, ptr);
-        Datum {val: body, ty: content_ty, mode: ByRef(ZeroMem)}
+        if !header && !ty::type_contents(bcx.tcx(), content_ty).contains_managed() {
+            let ptr = self.to_value_llval(bcx);
+            let ty = type_of(bcx.ccx(), content_ty);
+            let body = PointerCast(bcx, ptr, ty.ptr_to());
+            Datum {val: body, ty: content_ty, mode: ByRef(ZeroMem)}
+        } else { // has a header
+            let ptr = self.to_value_llval(bcx);
+            let body = opaque_box_body(bcx, content_ty, ptr);
+            Datum {val: body, ty: content_ty, mode: ByRef(ZeroMem)}
+        }
     }
 
     pub fn to_rptr(&self, bcx: block) -> Datum {

src/librustc/middle/trans/expr.rs

@@ -150,6 +150,7 @@ use middle::ty::{AutoPtr, AutoBorrowVec, AutoBorrowVecRef, AutoBorrowFn,
 use middle::ty;
 use util::common::indenter;
 use util::ppaux::Repr;
+use middle::trans::machine::llsize_of;
 
 use middle::trans::type_::Type;
 
@@ -464,7 +465,7 @@ fn trans_rvalue_datum_unadjusted(bcx: block, expr: @ast::expr) -> DatumBlock {
                                                       expr, contents);
         }
         ast::expr_vstore(contents, ast::expr_vstore_uniq) => {
-            let heap = heap_for_unique(bcx, expr_ty(bcx, contents));
+            let heap = tvec::heap_for_unique_vector(bcx, expr_ty(bcx, contents));
             return tvec::trans_uniq_or_managed_vstore(bcx, heap,
                                                       expr, contents);
         }
@@ -1329,12 +1330,23 @@ fn trans_unary_datum(bcx: block,
                         contents_ty: ty::t,
                         heap: heap) -> DatumBlock {
         let _icx = push_ctxt("trans_boxed_expr");
-        let base::MallocResult { bcx, box: bx, body } =
-            base::malloc_general(bcx, contents_ty, heap);
-        add_clean_free(bcx, bx, heap);
-        let bcx = trans_into(bcx, contents, SaveIn(body));
-        revoke_clean(bcx, bx);
-        return immediate_rvalue_bcx(bcx, bx, box_ty);
+        if heap == heap_exchange {
+            let llty = type_of(bcx.ccx(), contents_ty);
+            let size = llsize_of(bcx.ccx(), llty);
+            let Result { bcx: bcx, val: val } = malloc_raw_dyn(bcx, contents_ty,
+                                                               heap_exchange, size);
+            add_clean_free(bcx, val, heap_exchange);
+            let bcx = trans_into(bcx, contents, SaveIn(val));
+            revoke_clean(bcx, val);
+            return immediate_rvalue_bcx(bcx, val, box_ty);
+        } else {
+            let base::MallocResult { bcx, box: bx, body } =
+                base::malloc_general(bcx, contents_ty, heap);
+            add_clean_free(bcx, bx, heap);
+            let bcx = trans_into(bcx, contents, SaveIn(body));
+            revoke_clean(bcx, bx);
+            return immediate_rvalue_bcx(bcx, bx, box_ty);
+        }
     }
 }
 

src/librustc/middle/trans/glue.rs

@@ -386,7 +386,9 @@ pub fn make_free_glue(bcx: block, v: ValueRef, t: ty::t) {
       ty::ty_uniq(*) => {
         uniq::make_free_glue(bcx, v, t)
       }
-      ty::ty_evec(_, ty::vstore_uniq) | ty::ty_estr(ty::vstore_uniq) |
+      ty::ty_evec(_, ty::vstore_uniq) | ty::ty_estr(ty::vstore_uniq) => {
+        tvec::make_uniq_free_glue(bcx, v, t)
+      }
       ty::ty_evec(_, ty::vstore_box) | ty::ty_estr(ty::vstore_box) => {
         make_free_glue(bcx, v,
                        tvec::expand_boxed_vec_ty(bcx.tcx(), t));

src/librustc/middle/trans/meth.rs

@@ -548,6 +548,7 @@ pub fn trans_trait_callee_from_llval(bcx: block,
 
     let _icx = push_ctxt("impl::trans_trait_callee");
     let ccx = bcx.ccx();
+    let mut bcx = bcx;
 
     // Load the vtable from the @Trait pair
     debug!("(translating trait callee) loading vtable from pair %s",
@@ -576,6 +577,10 @@ pub fn trans_trait_callee_from_llval(bcx: block,
         }
         ast::sty_region(*) => {
             match store {
+                ty::UniqTraitStore
+                    if !ty::type_contents(bcx.tcx(), callee_ty).contains_managed() => {
+                    llself = llbox;
+                }
                 ty::BoxTraitStore |
                 ty::UniqTraitStore => {
                     llself = GEPi(bcx, llbox, [0u, abi::box_field_body]);

src/librustc/middle/trans/reflect.rs

@@ -194,7 +194,11 @@ impl Reflector {
           }
           ty::ty_uniq(ref mt) => {
               let extra = self.c_mt(mt);
-              self.visit("uniq", extra)
+              if ty::type_contents(bcx.tcx(), t).contains_managed() {
+                  self.visit("uniq_managed", extra)
+              } else {
+                  self.visit("uniq", extra)
+              }
           }
           ty::ty_ptr(ref mt) => {
               let extra = self.c_mt(mt);