Remove struct padding when passing or returning in registers on x86-64 (extern(D) only)

gen/abi-x86-64.cpp

@@ -43,6 +43,7 @@
 #include "gen/llvmhelpers.h"
 #include "gen/abi.h"
 #include "gen/abi-x86-64.h"
+#include "gen/structs.h"
 //#include "gen/llvm-version.h"         // only use is commented out.
 #include "ir/irfunction.h"
 
@@ -498,6 +499,38 @@ struct X86_64_C_struct_rewrite : ABIRewrite {
 };
 
 
+/// Removes padding fields for (non-union-containing!) structs
+struct RemoveStructPadding : ABIRewrite {
+    /// get a rewritten value back to its original form
+    virtual LLValue* get(Type* dty, DValue* v) {
+        LLValue* lval = DtoAlloca(dty, ".rewritetmp");
+        
+        // Make sure the padding is zero, so struct comparisons work.
+        // TODO: Only do this if there's padding, and/or only initialize padding.
+        DtoMemSetZero(lval, DtoConstSize_t(getTypePaddedSize(DtoType(dty))));
+        
+        DtoPaddedStruct(dty, v->getRVal(), lval);
+        return lval;
+    }
+
+    /// get a rewritten value back to its original form and store result in provided lvalue
+    /// this one is optional and defaults to calling the one above
+    virtual void getL(Type* dty, DValue* v, llvm::Value* lval) {
+        DtoPaddedStruct(dty, v->getRVal(), lval);
+    }
+
+    /// put out rewritten value
+    virtual LLValue* put(Type* dty, DValue* v) {
+        return DtoUnpaddedStruct(dty, v->getRVal());
+    }
+
+    /// return the transformed type for this rewrite
+    virtual const LLType* type(Type* dty, const LLType* t) {
+        return DtoUnpaddedStructType(dty);
+    }
+};
+
+
 struct RegCount {
     unsigned char int_regs, sse_regs;
 };
@@ -505,6 +538,7 @@ struct RegCount {
 
 struct X86_64TargetABI : TargetABI {
     X86_64_C_struct_rewrite struct_rewrite;
+    RemoveStructPadding remove_padding;
     
     void newFunctionType(TypeFunction* tf) {
         funcTypeStack.push_back(FuncTypeData(tf->linkage));
@@ -544,6 +578,7 @@ private:
         return funcTypeStack.back().state;
     }
     
+    void fixup_D(IrFuncTyArg& arg);
     void fixup(IrFuncTyArg& arg);
 };
 
@@ -643,6 +678,19 @@ bool X86_64TargetABI::passByVal(Type* t) {
     }
 }
 
+// Helper function for rewriteFunctionType.
+// Structs passed or returned in registers are passed here
+// to get their padding removed (if necessary).
+void X86_64TargetABI::fixup_D(IrFuncTyArg& arg) {
+    assert(arg.type->ty == Tstruct);
+    LLType* abiTy = DtoUnpaddedStructType(arg.type);
+    
+    if (abiTy && abiTy != arg.ltype) {
+        arg.ltype = abiTy;
+        arg.rewrite = &remove_padding;
+    }
+}
+
 // Helper function for rewriteFunctionType.
 // Return type and parameters are passed here (unless they're already in memory)
 // to get the rewrite applied (if necessary).
@@ -657,12 +705,40 @@ void X86_64TargetABI::fixup(IrFuncTyArg& arg) {
 }
 
 void X86_64TargetABI::rewriteFunctionType(TypeFunction* tf) {
-    // extern(D) is handled entirely by passByVal and returnInArg
+    IrFuncTy& fty = tf->fty;
     
-    if (tf->linkage != LINKd) {
-        // TODO: See if this is correct for more than just extern(C).
+    if (tf->linkage == LINKd) {
+        if (!fty.arg_sret) {
+            Type* rt = fty.ret->type->toBasetype();
+            if (rt->ty == Tstruct)  {
+                Logger::println("x86-64 D ABI: Transforming return type");
+                fixup_D(*fty.ret);
+            }
+        }
         
-        IrFuncTy& fty = tf->fty;
+        Logger::println("x86-64 D ABI: Transforming arguments");
+        LOG_SCOPE;
+        
+        for (IrFuncTy::ArgIter I = fty.args.begin(), E = fty.args.end(); I != E; ++I) {
+            IrFuncTyArg& arg = **I;
+            
+            if (Logger::enabled())
+                Logger::cout() << "Arg: " << arg.type->toChars() << '\n';
+            
+            // Arguments that are in memory are of no interest to us.
+            if (arg.byref)
+                continue;
+            
+            Type* ty = arg.type->toBasetype();
+            if (ty->ty == Tstruct)
+                fixup_D(arg);
+            
+            if (Logger::enabled())
+                Logger::cout() << "New arg type: " << *arg.ltype << '\n';
+        }
+        
+    } else {
+        // TODO: See if this is correct for more than just extern(C).
         
         if (!fty.arg_sret) {
             Logger::println("x86-64 ABI: Transforming return type");

gen/structs.cpp

@@ -1,6 +1,8 @@
 #include <algorithm>
 
 #include "gen/llvm.h"
+#include "llvm/AbstractTypeUser.h"
+#include "llvm/ADT/DenseMap.h"
 
 #include "mtype.h"
 #include "aggregate.h"
@@ -332,3 +334,80 @@ std::vector<llvm::Value*> DtoStructLiteralValues(const StructDeclaration* sd, co
     return values;
 }
 
+/// Return the type returned by DtoUnpaddedStruct called on a value of the
+/// specified type.
+/// Union types will get expanded into a struct, with a type for each member.
+LLType* DtoUnpaddedStructType(Type* dty) {
+    assert(dty->ty == Tstruct);
+    
+    typedef llvm::DenseMap<Type*, llvm::PATypeHolder> CacheT;
+    static CacheT cache;
+    CacheT::iterator it = cache.find(dty);
+    if (it != cache.end())
+        return it->second;
+    
+    TypeStruct* sty = (TypeStruct*) dty;
+    Array& fields = sty->sym->fields;
+
+    std::vector<const LLType*> types;
+
+    for (unsigned i = 0; i < fields.dim; i++) {
+        VarDeclaration* vd = (VarDeclaration*) fields.data[i];
+        const LLType* fty;
+        if (vd->type->ty == Tstruct) {
+            // Nested structs are the only members that can contain padding
+            fty = DtoUnpaddedStructType(vd->type);
+        } else {
+            fty = DtoType(vd->type);
+        }
+        types.push_back(fty);
+    }
+    LLType* Ty = LLStructType::get(types);
+    cache.insert(std::make_pair(dty, Ty));
+    return Ty;
+}
+
+/// Return the struct value represented by v without the padding fields.
+/// Unions will be expanded, with a value for each member.
+/// Note: v must be a pointer to a struct, but the return value will be a
+///       first-class struct value.
+LLValue* DtoUnpaddedStruct(Type* dty, LLValue* v) {
+    assert(dty->ty == Tstruct);
+    TypeStruct* sty = (TypeStruct*) dty;
+    Array& fields = sty->sym->fields;
+    
+    LLValue* newval = llvm::UndefValue::get(DtoUnpaddedStructType(dty));
+    
+    for (unsigned i = 0; i < fields.dim; i++) {
+        VarDeclaration* vd = (VarDeclaration*) fields.data[i];
+        LLValue* fieldptr = DtoIndexStruct(v, sty->sym, vd);
+        LLValue* fieldval;
+        if (vd->type->ty == Tstruct) {
+            // Nested structs are the only members that can contain padding
+            fieldval = DtoUnpaddedStruct(vd->type, fieldptr);
+        } else {
+            fieldval = DtoLoad(fieldptr);
+        }
+        newval = DtoInsertValue(newval, fieldval, i);
+    }
+    return newval;
+}
+
+/// Undo the transformation performed by DtoUnpaddedStruct, writing to lval.
+void DtoPaddedStruct(Type* dty, LLValue* v, LLValue* lval) {
+    assert(dty->ty == Tstruct);
+    TypeStruct* sty = (TypeStruct*) dty;
+    Array& fields = sty->sym->fields;
+    
+    for (unsigned i = 0; i < fields.dim; i++) {
+        VarDeclaration* vd = (VarDeclaration*) fields.data[i];
+        LLValue* fieldptr = DtoIndexStruct(lval, sty->sym, vd);
+        LLValue* fieldval = DtoExtractValue(v, i);
+        if (vd->type->ty == Tstruct) {
+            // Nested structs are the only members that can contain padding
+            DtoPaddedStruct(vd->type, fieldval, fieldptr);
+        } else {
+            DtoStore(fieldval, fieldptr);
+        }
+    }
+}

gen/structs.h

@@ -18,4 +18,18 @@ LLValue* DtoStructEquals(TOK op, DValue* lhs, DValue* rhs);
 /// index a struct one level
 LLValue* DtoIndexStruct(LLValue* src, StructDeclaration* sd, VarDeclaration* vd);
 
+/// Return the type returned by DtoUnpaddedStruct called on a value of the
+/// specified type.
+/// Union types will get expanded into a struct, with a type for each member.
+LLType* DtoUnpaddedStructType(Type* dty);
+
+/// Return the struct value represented by v without the padding fields.
+/// Unions will be expanded, with a value for each member.
+/// Note: v must be a pointer to a struct, but the return value will be a
+///       first-class struct value.
+LLValue* DtoUnpaddedStruct(Type* dty, LLValue* v);
+
+/// Undo the transformation performed by DtoUnpaddedStruct, writing to lval.
+void DtoPaddedStruct(Type* dty, LLValue* v, LLValue* lval);
+
 #endif

gen/tollvm.cpp

@@ -630,6 +630,18 @@ LLConstant* DtoBitCast(LLConstant* v, const LLType* t)
 
 //////////////////////////////////////////////////////////////////////////////////////////
 
+LLValue* DtoInsertValue(LLValue* aggr, LLValue* v, unsigned idx)
+{
+    return gIR->ir->CreateInsertValue(aggr, v, idx);
+}
+
+LLValue* DtoExtractValue(LLValue* aggr, unsigned idx)
+{
+    return gIR->ir->CreateExtractValue(aggr, idx);
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////
+
 const LLPointerType* isaPointer(LLValue* v)
 {
     return llvm::dyn_cast<LLPointerType>(v->getType());

gen/tollvm.h

@@ -68,6 +68,8 @@ void DtoStore(LLValue* src, LLValue* dst);
 void DtoAlignedStore(LLValue* src, LLValue* dst);
 LLValue* DtoBitCast(LLValue* v, const LLType* t, const char* name=0);
 LLConstant* DtoBitCast(LLConstant* v, const LLType* t);
+LLValue* DtoInsertValue(LLValue* aggr, LLValue* v, unsigned idx);
+LLValue* DtoExtractValue(LLValue* aggr, unsigned idx);
 
 // llvm::dyn_cast wrappers
 const LLPointerType* isaPointer(LLValue* v);