added llvm_mode ngram coverage

4 files changed, 150 insertions, 26 deletions

diff --git a/llvm_mode/afl-clang-fast.c b/llvm_mode/afl-clang-fast.c
index 313a2533..77cb1c0f 100644
--- a/llvm_mode/afl-clang-fast.c
+++ b/llvm_mode/afl-clang-fast.c
@@ -158,14 +158,20 @@ static void edit_params(u32 argc, char **argv) {
 #endif
     if (lto_flag[0] != '-')
       FATAL(
-          "afl-clang-lto not possible because Makefile magic did not identify "
-          "the correct -flto flag");
+          "Using afl-clang-lto is not possible because Makefile magic did not "
+          "identify the correct -flto flag");
     if (getenv("AFL_LLVM_INSTRIM") != NULL)
       FATAL("afl-clang-lto does not work with InsTrim mode");
+    if (getenv("AFL_LLVM_NGRAM_SIZE") != NULL)
+      FATAL("afl-clang-lto does not work with ngram coverage mode");
     lto_mode = 1;
 
   }
 
+  if (getenv("AFL_LLVM_NGRAM_SIZE") != NULL &&
+      getenv("AFL_LLVM_INSTRIM") != NULL)
+    FATAL("AFL_LLVM_NGRAM_SIZE and AFL_LLVM_INSTRIM can not be used together");
+
   if (!strcmp(name, "afl-clang-fast++") || !strcmp(name, "afl-clang-lto++")) {
 
     u8 *alt_cxx = getenv("AFL_CXX");
@@ -605,6 +611,7 @@ int main(int argc, char **argv, char **envp) {
             "AFL_LLVM_LAF_SPLIT_COMPARES_BITW: size limit (default 8)\n"
             "AFL_LLVM_INSTRIM: use light weight instrumentation InsTrim\n"
             "AFL_LLVM_INSTRIM_LOOPHEAD: optimize loop tracing for speed\n"
+            "AFL_LLVM_NGRAM_SIZE: use ngram prev_loc coverage\n"
             "AFL_LLVM_CMPLOG: log operands of comparisons (RedQueen mutator)\n"
             "\nafl-clang-fast was built for llvm %s with the llvm binary path "
             "of "
diff --git a/llvm_mode/afl-llvm-pass.so.cc b/llvm_mode/afl-llvm-pass.so.cc
index 133c64b4..fefd9edd 100644
--- a/llvm_mode/afl-llvm-pass.so.cc
+++ b/llvm_mode/afl-llvm-pass.so.cc
@@ -2,12 +2,15 @@
    american fuzzy lop++ - LLVM-mode instrumentation pass
    ---------------------------------------------------
 
-   Written by Laszlo Szekeres <lszekeres@google.com> and
+   Written by Laszlo Szekeres <lszekeres@google.com>,
+              Adrian Herrera <adrian.herrera@anu.edu.au>,
               Michal Zalewski
 
    LLVM integration design comes from Laszlo Szekeres. C bits copied-and-pasted
    from afl-as.c are Michal's fault.
 
+   NGRAM previous location coverage comes from Adrian Herrera.
+
    Copyright 2015, 2016 Google Inc. All rights reserved.
    Copyright 2019-2020 AFLplusplus Project. All rights reserved.
 
@@ -27,7 +30,6 @@
 
 #include "config.h"
 #include "debug.h"
-
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -47,6 +49,7 @@ typedef long double max_align_t;
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
 
 #if LLVM_VERSION_MAJOR > 3 || \
@@ -58,6 +61,8 @@ typedef long double max_align_t;
 #include "llvm/Support/CFG.h"
 #endif
 
+#include "llvm-ngram-coverage.h"
+
 using namespace llvm;
 
 namespace {
@@ -118,6 +123,7 @@ class AFLCoverage : public ModulePass {
 
  protected:
   std::list<std::string> myWhitelist;
+  uint32_t               ngram_size = 0;
 
 };
 
@@ -129,8 +135,10 @@ bool AFLCoverage::runOnModule(Module &M) {
 
   LLVMContext &C = M.getContext();
 
-  IntegerType *   Int8Ty = IntegerType::getInt8Ty(C);
-  IntegerType *   Int32Ty = IntegerType::getInt32Ty(C);
+  IntegerType *Int8Ty = IntegerType::getInt8Ty(C);
+  IntegerType *Int32Ty = IntegerType::getInt32Ty(C);
+  IntegerType *IntLocTy =
+      IntegerType::getIntNTy(C, sizeof(PREV_LOC_T) * CHAR_BIT);
   struct timeval  tv;
   struct timezone tz;
   u32             rand_seed;
@@ -147,7 +155,8 @@ bool AFLCoverage::runOnModule(Module &M) {
 
   if ((isatty(2) && !getenv("AFL_QUIET")) || getenv("AFL_DEBUG") != NULL) {
 
-    SAYF(cCYA "afl-llvm-pass" VERSION cRST " by <lszekeres@google.com>\n");
+    SAYF(cCYA "afl-llvm-pass" VERSION cRST
+              " by <lszekeres@google.com> and <adrian.herrera@anu.edu.au>\n");
 
   } else
 
@@ -170,21 +179,73 @@ bool AFLCoverage::runOnModule(Module &M) {
   char *neverZero_counters_str = getenv("AFL_LLVM_NOT_ZERO");
 #endif
 
+  /* Decide previous location vector size (must be a power of two) */
+
+  char *ngram_size_str = getenv("AFL_LLVM_NGRAM_SIZE");
+  if (!ngram_size_str) ngram_size_str = getenv("AFL_NGRAM_SIZE");
+
+  if (ngram_size_str)
+    if (sscanf(ngram_size_str, "%u", &ngram_size) != 1 || ngram_size < 2 ||
+        ngram_size > MAX_NGRAM_SIZE)
+      FATAL(
+          "Bad value of AFL_NGRAM_SIZE (must be between 2 and MAX_NGRAM_SIZE)");
+
+  unsigned PrevLocSize;
+  if (ngram_size == 1) ngram_size = 0;
+  if (ngram_size)
+    PrevLocSize = ngram_size - 1;
+  else
+    PrevLocSize = 1;
+  uint64_t    PrevLocVecSize = PowerOf2Ceil(PrevLocSize);
+  VectorType *PrevLocTy;
+
+  if (ngram_size) PrevLocTy = VectorType::get(IntLocTy, PrevLocVecSize);
+
   /* Get globals for the SHM region and the previous location. Note that
      __afl_prev_loc is thread-local. */
 
   GlobalVariable *AFLMapPtr =
       new GlobalVariable(M, PointerType::get(Int8Ty, 0), false,
                          GlobalValue::ExternalLinkage, 0, "__afl_area_ptr");
+  GlobalVariable *AFLPrevLoc;
 
+  if (ngram_size)
 #ifdef __ANDROID__
-  GlobalVariable *AFLPrevLoc = new GlobalVariable(
-      M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_loc");
+    AFLPrevLoc = new GlobalVariable(
+        M, PrevLocTy, /* isConstant */ false, GlobalValue::ExternalLinkage,
+        /* Initializer */ nullptr, "__afl_prev_loc");
 #else
-  GlobalVariable *AFLPrevLoc = new GlobalVariable(
-      M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_loc", 0,
-      GlobalVariable::GeneralDynamicTLSModel, 0, false);
+    AFLPrevLoc = new GlobalVariable(
+        M, PrevLocTy, /* isConstant */ false, GlobalValue::ExternalLinkage,
+        /* Initializer */ nullptr, "__afl_prev_loc",
+        /* InsertBefore */ nullptr, GlobalVariable::GeneralDynamicTLSModel,
+        /* AddressSpace */ 0, /* IsExternallyInitialized */ false);
 #endif
+  else
+#ifdef __ANDROID__
+    AFLPrevLoc = new GlobalVariable(
+        M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_loc");
+#else
+    AFLPrevLoc = new GlobalVariable(
+        M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_loc", 0,
+        GlobalVariable::GeneralDynamicTLSModel, 0, false);
+#endif
+
+  /* Create the vector shuffle mask for updating the previous block history.
+     Note that the first element of the vector will store cur_loc, so just set
+     it to undef to allow the optimizer to do its thing. */
+
+  SmallVector<Constant *, 32> PrevLocShuffle = {UndefValue::get(Int32Ty)};
+
+  for (unsigned I = 0; I < PrevLocSize - 1; ++I)
+    PrevLocShuffle.push_back(ConstantInt::get(Int32Ty, I));
+
+  for (unsigned I = PrevLocSize; I < PrevLocVecSize; ++I)
+    PrevLocShuffle.push_back(ConstantInt::get(Int32Ty, PrevLocSize));
+
+  Constant *PrevLocShuffleMask = ConstantVector::get(PrevLocShuffle);
+
+  // other constants we need
   ConstantInt *Zero = ConstantInt::get(Int8Ty, 0);
   ConstantInt *One = ConstantInt::get(Int8Ty, 1);
 
@@ -356,20 +417,41 @@ bool AFLCoverage::runOnModule(Module &M) {
       // fprintf(stderr, " == %d\n", more_than_one);
       if (more_than_one != 1) continue;
 #endif
-      ConstantInt *CurLoc = ConstantInt::get(Int32Ty, cur_loc);
+
+      ConstantInt *CurLoc;
+
+      if (ngram_size)
+        CurLoc = ConstantInt::get(IntLocTy, cur_loc);
+      else
+        CurLoc = ConstantInt::get(Int32Ty, cur_loc);
 
       /* Load prev_loc */
 
       LoadInst *PrevLoc = IRB.CreateLoad(AFLPrevLoc);
       PrevLoc->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None));
-      Value *PrevLocCasted = IRB.CreateZExt(PrevLoc, IRB.getInt32Ty());
+      Value *PrevLocTrans;
+
+      /* "For efficiency, we propose to hash the tuple as a key into the
+         hit_count map as (prev_block_trans << 1) ^ curr_block_trans, where
+         prev_block_trans = (block_trans_1 ^ ... ^ block_trans_(n-1)" */
+
+      if (ngram_size)
+        PrevLocTrans = IRB.CreateXorReduce(PrevLoc);
+      else
+        PrevLocTrans = IRB.CreateZExt(PrevLoc, IRB.getInt32Ty());
 
       /* Load SHM pointer */
 
       LoadInst *MapPtr = IRB.CreateLoad(AFLMapPtr);
       MapPtr->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None));
-      Value *MapPtrIdx =
-          IRB.CreateGEP(MapPtr, IRB.CreateXor(PrevLocCasted, CurLoc));
+
+      Value *MapPtrIdx;
+      if (ngram_size)
+        MapPtrIdx = IRB.CreateGEP(
+            MapPtr,
+            IRB.CreateZExt(IRB.CreateXor(PrevLocTrans, CurLoc), Int32Ty));
+      else
+        MapPtrIdx = IRB.CreateGEP(MapPtr, IRB.CreateXor(PrevLocTrans, CurLoc));
 
       /* Update bitmap */
 
@@ -449,11 +531,27 @@ bool AFLCoverage::runOnModule(Module &M) {
       IRB.CreateStore(Incr, MapPtrIdx)
           ->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None));
 
-      /* Set prev_loc to cur_loc >> 1 */
+      /* Update prev_loc history vector (by placing cur_loc at the head of the
+         vector and shuffle the other elements back by one) */
 
-      StoreInst *Store =
-          IRB.CreateStore(ConstantInt::get(Int32Ty, cur_loc >> 1), AFLPrevLoc);
-      Store->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None));
+      StoreInst *Store;
+
+      if (ngram_size) {
+
+        Value *ShuffledPrevLoc = IRB.CreateShuffleVector(
+            PrevLoc, UndefValue::get(PrevLocTy), PrevLocShuffleMask);
+        Value *UpdatedPrevLoc = IRB.CreateInsertElement(
+            ShuffledPrevLoc, IRB.CreateLShr(CurLoc, (uint64_t)1), (uint64_t)0);
+
+        Store = IRB.CreateStore(UpdatedPrevLoc, AFLPrevLoc);
+        Store->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None));
+
+      } else {
+
+        Store = IRB.CreateStore(ConstantInt::get(Int32Ty, cur_loc >> 1),
+                                AFLPrevLoc);
+
+      }
 
       inst_blocks++;
 
diff --git a/llvm_mode/afl-llvm-rt.o.c b/llvm_mode/afl-llvm-rt.o.c
index 5f9a5534..8fad0fbb 100644
--- a/llvm_mode/afl-llvm-rt.o.c
+++ b/llvm_mode/afl-llvm-rt.o.c
@@ -26,6 +26,7 @@
 #include "config.h"
 #include "types.h"
 #include "cmplog.h"
+#include "llvm-ngram-coverage.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -62,11 +63,11 @@ u8  __afl_area_initial[MAP_SIZE];
 u8 *__afl_area_ptr = __afl_area_initial;
 
 #ifdef __ANDROID__
-u32 __afl_prev_loc;
-u32 __afl_final_loc;
+PREV_LOC_T __afl_prev_loc[MAX_NGRAM_SIZE];
+u32        __afl_final_loc;
 #else
-__thread u32 __afl_prev_loc;
-__thread u32 __afl_final_loc;
+__thread PREV_LOC_T __afl_prev_loc[MAX_NGRAM_SIZE];
+__thread u32        __afl_final_loc;
 #endif
 
 struct cmp_map *__afl_cmp_map;
@@ -281,7 +282,7 @@ int __afl_persistent_loop(unsigned int max_cnt) {
 
       memset(__afl_area_ptr, 0, MAP_SIZE);
       __afl_area_ptr[0] = 1;
-      __afl_prev_loc = 0;
+      memset(__afl_prev_loc, 0, MAX_NGRAM_SIZE * sizeof(PREV_LOC_T));
 
     }
 
@@ -298,7 +299,7 @@ int __afl_persistent_loop(unsigned int max_cnt) {
       raise(SIGSTOP);
 
       __afl_area_ptr[0] = 1;
-      __afl_prev_loc = 0;
+      memset(__afl_prev_loc, 0, MAX_NGRAM_SIZE * sizeof(PREV_LOC_T));
 
       return 1;
 
diff --git a/llvm_mode/llvm-ngram-coverage.h b/llvm_mode/llvm-ngram-coverage.h
new file mode 100644
index 00000000..4459bcd7
--- /dev/null
+++ b/llvm_mode/llvm-ngram-coverage.h
@@ -0,0 +1,18 @@
+#ifndef AFL_NGRAM_CONFIG_H
+#define AFL_NGRAM_CONFIG_H
+
+#include "../config.h"
+
+#if (MAP_SIZE_POW2 <= 16)
+typedef u16 PREV_LOC_T;
+#elif (MAP_SIZE_POW2 <= 32)
+typedef u32 PREV_LOC_T;
+#else
+typedef u64 PREV_LOC_T;
+#endif
+
+/* Maximum ngram size */
+#define MAX_NGRAM_SIZE 128
+
+#endif
+