Merge pull request #1142 from AFLplusplus/dev

Dev

1 files changed, 156 insertions, 57 deletions

diff --git a/instrumentation/cmplog-instructions-pass.cc b/instrumentation/cmplog-instructions-pass.cc
index 0562c5b2..01a8a637 100644
--- a/instrumentation/cmplog-instructions-pass.cc
+++ b/instrumentation/cmplog-instructions-pass.cc
@@ -274,14 +274,15 @@ bool CmpLogInstructions::hookInstrs(Module &M) {
 
       Value *op0 = selectcmpInst->getOperand(0);
       Value *op1 = selectcmpInst->getOperand(1);
+      Value *op0_saved = op0, *op1_saved = op1;
+      auto   ty0 = op0->getType();
+      auto   ty1 = op1->getType();
 
-      IntegerType *        intTyOp0 = NULL;
-      IntegerType *        intTyOp1 = NULL;
-      unsigned             max_size = 0, cast_size = 0;
-      unsigned char        attr = 0;
-      std::vector<Value *> args;
-
-      CmpInst *cmpInst = dyn_cast<CmpInst>(selectcmpInst);
+      IntegerType *intTyOp0 = NULL;
+      IntegerType *intTyOp1 = NULL;
+      unsigned     max_size = 0, cast_size = 0;
+      unsigned     attr = 0, vector_cnt = 0;
+      CmpInst *    cmpInst = dyn_cast<CmpInst>(selectcmpInst);
 
       if (!cmpInst) { continue; }
 
@@ -327,7 +328,23 @@ bool CmpLogInstructions::hookInstrs(Module &M) {
 
       if (selectcmpInst->getOpcode() == Instruction::FCmp) {
 
-        auto ty0 = op0->getType();
+        if (ty0->isVectorTy()) {
+
+          VectorType *tt = dyn_cast<VectorType>(ty0);
+          if (!tt) {
+
+            fprintf(stderr, "Warning: cmplog cmp vector is not a vector!\n");
+            continue;
+
+          }
+
+#if LLVM_MAJOR > 11
+          vector_cnt = tt->getElementCount().getFixedValue();
+          ty0 = tt->getElementType();
+#endif
+
+        }
+
         if (ty0->isHalfTy()
 #if LLVM_VERSION_MAJOR >= 11
             || ty0->isBFloatTy()
@@ -342,13 +359,35 @@ bool CmpLogInstructions::hookInstrs(Module &M) {
           max_size = 80;
         else if (ty0->isFP128Ty() || ty0->isPPC_FP128Ty())
           max_size = 128;
+#if LLVM_MAJOR > 11
+        else if (ty0->getTypeID() != llvm::Type::PointerTyID && !be_quiet)
+          fprintf(stderr, "Warning: unsupported cmp type for cmplog: %u!\n",
+                  ty0->getTypeID());
+#endif
 
         attr += 8;
 
       } else {
 
-        intTyOp0 = dyn_cast<IntegerType>(op0->getType());
-        intTyOp1 = dyn_cast<IntegerType>(op1->getType());
+        if (ty0->isVectorTy()) {
+
+#if LLVM_MAJOR > 11
+          VectorType *tt = dyn_cast<VectorType>(ty0);
+          if (!tt) {
+
+            fprintf(stderr, "Warning: cmplog cmp vector is not a vector!\n");
+            continue;
+
+          }
+
+          vector_cnt = tt->getElementCount().getFixedValue();
+          ty1 = ty0 = tt->getElementType();
+#endif
+
+        }
+
+        intTyOp0 = dyn_cast<IntegerType>(ty0);
+        intTyOp1 = dyn_cast<IntegerType>(ty1);
 
         if (intTyOp0 && intTyOp1) {
 
@@ -356,11 +395,28 @@ bool CmpLogInstructions::hookInstrs(Module &M) {
                          ? intTyOp0->getBitWidth()
                          : intTyOp1->getBitWidth();
 
+        } else {
+
+#if LLVM_MAJOR > 11
+          if (ty0->getTypeID() != llvm::Type::PointerTyID && !be_quiet) {
+
+            fprintf(stderr, "Warning: unsupported cmp type for cmplog: %u\n",
+                    ty0->getTypeID());
+
+          }
+
+#endif
+
         }
 
       }
 
-      if (!max_size || max_size < 16) { continue; }
+      if (!max_size || max_size < 16) {
+
+        // fprintf(stderr, "too small\n");
+        continue;
+
+      }
 
       if (max_size % 8) { max_size = (((max_size / 8) + 1) * 8); }
 
@@ -393,67 +449,110 @@ bool CmpLogInstructions::hookInstrs(Module &M) {
 
       }
 
-      // errs() << "[CMPLOG] cmp  " << *cmpInst << "(in function " <<
-      // cmpInst->getFunction()->getName() << ")\n";
+      uint64_t cur = 0, last_val0 = 0, last_val1 = 0, cur_val;
 
-      // first bitcast to integer type of the same bitsize as the original
-      // type (this is a nop, if already integer)
-      Value *op0_i = IRB.CreateBitCast(
-          op0, IntegerType::get(C, op0->getType()->getPrimitiveSizeInBits()));
-      // then create a int cast, which does zext, trunc or bitcast. In our case
-      // usually zext to the next larger supported type (this is a nop if
-      // already the right type)
-      Value *V0 =
-          IRB.CreateIntCast(op0_i, IntegerType::get(C, cast_size), false);
-      args.push_back(V0);
-      Value *op1_i = IRB.CreateBitCast(
-          op1, IntegerType::get(C, op1->getType()->getPrimitiveSizeInBits()));
-      Value *V1 =
-          IRB.CreateIntCast(op1_i, IntegerType::get(C, cast_size), false);
-      args.push_back(V1);
+      while (1) {
 
-      // errs() << "[CMPLOG] casted parameters:\n0: " << *V0 << "\n1: " << *V1
-      // << "\n";
+        std::vector<Value *> args;
+        uint32_t             skip = 0;
 
-      ConstantInt *attribute = ConstantInt::get(Int8Ty, attr);
-      args.push_back(attribute);
+        if (vector_cnt) {
 
-      if (cast_size != max_size) {
+          op0 = IRB.CreateExtractElement(op0_saved, cur);
+          op1 = IRB.CreateExtractElement(op1_saved, cur);
+          ConstantInt *i0 = dyn_cast<ConstantInt>(op0);
+          ConstantInt *i1 = dyn_cast<ConstantInt>(op1);
+          if (i0 && i0->uge(0xffffffffffffffff) == false) {
 
-        ConstantInt *bitsize = ConstantInt::get(Int8Ty, (max_size / 8) - 1);
-        args.push_back(bitsize);
+            cur_val = i0->getZExtValue();
+            if (last_val0 && last_val0 == cur_val) { skip = 1; }
+            last_val0 = cur_val;
 
-      }
+          }
 
-      // fprintf(stderr, "_ExtInt(%u) castTo %u with attr %u didcast %u\n",
-      //         max_size, cast_size, attr);
+          if (i1 && i1->uge(0xffffffffffffffff) == false) {
 
-      switch (cast_size) {
+            cur_val = i1->getZExtValue();
+            if (last_val1 && last_val1 == cur_val) { skip = 1; }
+            last_val1 = cur_val;
 
-        case 8:
-          IRB.CreateCall(cmplogHookIns1, args);
-          break;
-        case 16:
-          IRB.CreateCall(cmplogHookIns2, args);
-          break;
-        case 32:
-          IRB.CreateCall(cmplogHookIns4, args);
-          break;
-        case 64:
-          IRB.CreateCall(cmplogHookIns8, args);
-          break;
-        case 128:
-          if (max_size == 128) {
+          }
+
+        }
+
+        if (!skip) {
+
+          // errs() << "[CMPLOG] cmp  " << *cmpInst << "(in function " <<
+          // cmpInst->getFunction()->getName() << ")\n";
+
+          // first bitcast to integer type of the same bitsize as the original
+          // type (this is a nop, if already integer)
+          Value *op0_i = IRB.CreateBitCast(
+              op0, IntegerType::get(C, ty0->getPrimitiveSizeInBits()));
+          // then create a int cast, which does zext, trunc or bitcast. In our
+          // case usually zext to the next larger supported type (this is a nop
+          // if already the right type)
+          Value *V0 =
+              IRB.CreateIntCast(op0_i, IntegerType::get(C, cast_size), false);
+          args.push_back(V0);
+          Value *op1_i = IRB.CreateBitCast(
+              op1, IntegerType::get(C, ty1->getPrimitiveSizeInBits()));
+          Value *V1 =
+              IRB.CreateIntCast(op1_i, IntegerType::get(C, cast_size), false);
+          args.push_back(V1);
 
-            IRB.CreateCall(cmplogHookIns16, args);
+          // errs() << "[CMPLOG] casted parameters:\n0: " << *V0 << "\n1: " <<
+          // *V1
+          // << "\n";
 
-          } else {
+          ConstantInt *attribute = ConstantInt::get(Int8Ty, attr);
+          args.push_back(attribute);
 
-            IRB.CreateCall(cmplogHookInsN, args);
+          if (cast_size != max_size) {
+
+            ConstantInt *bitsize = ConstantInt::get(Int8Ty, (max_size / 8) - 1);
+            args.push_back(bitsize);
 
           }
 
-          break;
+          // fprintf(stderr, "_ExtInt(%u) castTo %u with attr %u didcast %u\n",
+          //         max_size, cast_size, attr);
+
+          switch (cast_size) {
+
+            case 8:
+              IRB.CreateCall(cmplogHookIns1, args);
+              break;
+            case 16:
+              IRB.CreateCall(cmplogHookIns2, args);
+              break;
+            case 32:
+              IRB.CreateCall(cmplogHookIns4, args);
+              break;
+            case 64:
+              IRB.CreateCall(cmplogHookIns8, args);
+              break;
+            case 128:
+              if (max_size == 128) {
+
+                IRB.CreateCall(cmplogHookIns16, args);
+
+              } else {
+
+                IRB.CreateCall(cmplogHookInsN, args);
+
+              }
+
+              break;
+
+          }
+
+        }
+
+        /* else fprintf(stderr, "skipped\n"); */
+
+        ++cur;
+        if (cur >= vector_cnt) { break; }
 
       }