Fixed passing of long double (and other big types) in var_args on x86_64. Removed XFAIL tag from the Feature/VarArgLongDouble.c test

Fixed Executor to (more) correctly handle the alignment of types larger than 64bit (such as long double) when those are passed in var_args on x86_64.

Specifically:
From http://www.x86-64.org/documentation/abi.pdf
AMD64-ABI 3.5.7p5: Step 7.
Align l->overflow_arg_area upwards to a 16 byte boundary if alignment needed by type exceeds 8 byte boundary.

2 files changed, 23 insertions, 7 deletions

diff --git a/lib/Core/Executor.cpp b/lib/Core/Executor.cpp
index 314e5b82..f6f73845 100644
--- a/lib/Core/Executor.cpp
+++ b/lib/Core/Executor.cpp
@@ -1260,6 +1260,8 @@ void Executor::executeCall(ExecutionState &state,
         return;
       }
     } else {
+      Expr::Width WordSize = Context::get().getPointerWidth();
+
       if (callingArgs < funcArgs) {
         terminateStateOnError(state, "calling function with too few arguments", 
                               "user.err");
@@ -1271,12 +1273,18 @@ void Executor::executeCall(ExecutionState &state,
       for (unsigned i = funcArgs; i < callingArgs; i++) {
         // FIXME: This is really specific to the architecture, not the pointer
         // size. This happens to work fir x86-32 and x86-64, however.
-        Expr::Width WordSize = Context::get().getPointerWidth();
         if (WordSize == Expr::Int32) {
           size += Expr::getMinBytesForWidth(arguments[i]->getWidth());
         } else {
-          size += llvm::RoundUpToAlignment(arguments[i]->getWidth(), 
-                                           WordSize) / 8;
+          Expr::Width argWidth = arguments[i]->getWidth();
+          // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
+          // byte boundary if alignment needed by type exceeds 8 byte boundary.
+          //
+          // Alignment requirements for scalar types is the same as their size
+          if (argWidth > Expr::Int64) {
+             size = llvm::RoundUpToAlignment(size, 16);
+          }
+          size += llvm::RoundUpToAlignment(argWidth, WordSize) / 8;
         }
       }
 
@@ -1286,20 +1294,29 @@ void Executor::executeCall(ExecutionState &state,
         terminateStateOnExecError(state, "out of memory (varargs)");
         return;
       }
+
+      if ((WordSize == Expr::Int64) && (mo->address & 15)) {
+        // Both 64bit Linux/Glibc and 64bit MacOSX should align to 16 bytes.
+        klee_warning_once(0, "While allocating varargs: malloc did not align to 16 bytes.");
+      }
+
       ObjectState *os = bindObjectInState(state, mo, true);
       unsigned offset = 0;
       for (unsigned i = funcArgs; i < callingArgs; i++) {
         // FIXME: This is really specific to the architecture, not the pointer
         // size. This happens to work fir x86-32 and x86-64, however.
-        Expr::Width WordSize = Context::get().getPointerWidth();
         if (WordSize == Expr::Int32) {
           os->write(offset, arguments[i]);
           offset += Expr::getMinBytesForWidth(arguments[i]->getWidth());
         } else {
           assert(WordSize == Expr::Int64 && "Unknown word size!");
+
+          Expr::Width argWidth = arguments[i]->getWidth();
+          if (argWidth > Expr::Int64) {
+             offset = llvm::RoundUpToAlignment(offset, 16);
+          }
           os->write(offset, arguments[i]);
-          offset += llvm::RoundUpToAlignment(arguments[i]->getWidth(), 
-                                             WordSize) / 8;
+          offset += llvm::RoundUpToAlignment(argWidth, WordSize) / 8;
         }
       }
     }
diff --git a/test/Feature/VarArgLongDouble.c b/test/Feature/VarArgLongDouble.c
index 189d1dcc..ae553131 100644
--- a/test/Feature/VarArgLongDouble.c
+++ b/test/Feature/VarArgLongDouble.c
@@ -1,7 +1,6 @@
 // RUN: %llvmgcc %s -emit-llvm -O0 -c -o %t.bc
 // RUN: rm -rf %t.klee-out
 // RUN: %klee --output-dir=%t.klee-out %t.bc | FileCheck %s
-// XFAIL:
 
 #include <stdarg.h>
 #include <assert.h>