Fix Win64 ABI wrt. passing structs > 64 bit

Author: kinke

gen/abi-win64.cpp

@@ -36,14 +36,29 @@
 static bool canRewriteAsInt(Type* t)
 {
     unsigned size = t->size();
-    return size <= 8 &&
-           (size == 1 || size == 2 || size == 4 || size == 8) &&
-           (t->ty == Tstruct || t->ty == Tsarray);
+    return size == 1 || size == 2 || size == 4 || size == 8;
+}
+
+// Returns true if the D type is a composite (struct or static array).
+static bool isComposite(const Type* t)
+{
+    return t->ty == Tstruct || t->ty == Tsarray;
+}
 
+// Returns true if the D type is passed byval (the callee getting a pointer
+// to a dedicated hidden copy).
+static bool isPassedWithByvalSemantics(Type* t)
+{
+    return
+        // * structs and static arrays which can NOT be rewritten as integers
+        (isComposite(t) && !canRewriteAsInt(t)) ||
+        // * 80-bit real and ireal
+        (t->ty == Tfloat80 || t->ty == Timaginary80) ||
+        // * cdouble and creal
+        (t->ty == Tcomplex64 || t->ty == Tcomplex80);
 }
 
 // FIXME: This should actually be handled by LLVM and the ByVal arg attribute.
-// Implements byval argument passing for scalar non-struct types.
 struct Win64_byval_rewrite : ABIRewrite
 {
     // Get instance from pointer.
@@ -60,8 +75,9 @@ struct Win64_byval_rewrite : ABIRewrite
         DtoStore(DtoLoad(ptr), lval); // *lval = *ptr
     }
 
-    // Turn an instance into a pointer (to a private copy for the callee,
-    // allocated by the caller).
+    // Convert the caller's instance to a pointer for the callee.
+    // The pointer points to a dedicated copy for the callee which
+    // is allocated by the caller.
     LLValue* put(Type* dty, DValue* v)
     {
         /* NOTE: probably not safe
@@ -71,12 +87,25 @@ struct Win64_byval_rewrite : ABIRewrite
         */
 
         LLValue* original = v->getRVal();
-        LLValue* copy = DtoRawAlloca(original->getType(), 16, "copy_for_callee");
-        DtoStore(original, copy); // *copy = *original
+        LLValue* copy;
+
+        llvm::Type* type = original->getType();
+        if (type->isPointerTy())
+        {
+            type = type->getPointerElementType();
+            copy = DtoRawAlloca(type, 16, "copy_for_callee");
+            DtoStore(DtoLoad(original), copy); // *copy = *original
+        }
+        else
+        {
+            copy = DtoRawAlloca(type, 16, "copy_for_callee");
+            DtoStore(original, copy);          // *copy = original
+        }
+
         return copy;
     }
 
-    /// should return the transformed type for this rewrite
+    // T => T*
     LLType* type(Type* dty, LLType* t)
     {
         return getPtrToType(DtoType(dty));
@@ -134,22 +163,17 @@ bool Win64TargetABI::returnInArg(TypeFunction* tf)
     // everything <= 64 bits and of a size that is a power of 2
     // is returned in a register (RAX, or XMM0 for single float/
     // double) - except for cfloat
-    // real/ireal is returned on top of the x87 stack: ST(0)
+    // 80-bit real/ireal is returned on top of the x87 stack: ST(0)
     // complex numbers are returned in XMM0 & XMM1 (cfloat, cdouble)
     // or ST(1) & ST(0) (creal)
     // all other structs and static arrays are returned by struct-return (sret)
-    return (rt->ty == Tstruct
-            || rt->ty == Tsarray
-           ) && !canRewriteAsInt(rt);
+    return isComposite(rt) && !canRewriteAsInt(rt);
 }
 
 bool Win64TargetABI::passByVal(Type* t)
 {
     t = t->toBasetype();
-
-    // structs and static arrays are passed byval unless they can be
-    // rewritten as integers
-    return (t->ty == Tstruct || t->ty == Tsarray) && !canRewriteAsInt(t);
+    return isPassedWithByvalSemantics(t);
 }
 
 void Win64TargetABI::rewriteFunctionType(TypeFunction* tf, IrFuncTy &fty)
@@ -168,7 +192,7 @@ void Win64TargetABI::rewriteFunctionType(TypeFunction* tf, IrFuncTy &fty)
             // the other complex number types are returned via XMM0 = re and XMM1 = im
             fty.ret->rewrite = &swapComplex;
         }
-        else if (canRewriteAsInt(rt))
+        else if (isComposite(rt) && canRewriteAsInt(rt))
         {
             fty.ret->rewrite = &compositeToInt;
             fty.ret->ltype = compositeToInt.type(fty.ret->type, fty.ret->ltype);
@@ -195,12 +219,13 @@ void Win64TargetABI::rewriteFunctionType(TypeFunction* tf, IrFuncTy &fty)
             arg.rewrite = &cfloatToInt;
             arg.ltype = cfloatToInt.type(arg.type, arg.ltype);
         }
-        else if (canRewriteAsInt(ty))
+        else if (isComposite(ty) && canRewriteAsInt(ty))
         {
             arg.rewrite = &compositeToInt;
             arg.ltype = compositeToInt.type(arg.type, arg.ltype);
         }
-        else if (ty->iscomplex() || ty->ty == Tfloat80 || ty->ty == Timaginary80)
+        // FIXME: this should actually be handled by LLVM and the ByVal arg attribute
+        else if (isPassedWithByvalSemantics(ty))
         {
             // these types are passed byval:
             // the caller allocates a copy and then passes a pointer to the copy