Merge pull request #778 from AFLplusplus/dev

This fixes 3 different crash issues

9 files changed, 234 insertions, 93 deletions

diff --git a/src/afl-analyze.c b/src/afl-analyze.c
index 20aef2da..d46ecb8d 100644
--- a/src/afl-analyze.c
+++ b/src/afl-analyze.c
@@ -785,6 +785,7 @@ static void set_up_environment(void) {
          "abort_on_error=1:"
          "detect_leaks=0:"
          "allocator_may_return_null=1:"
+         "detect_odr_violation=0:"
          "symbolize=0:"
          "handle_segv=0:"
          "handle_sigbus=0:"
diff --git a/src/afl-cc.c b/src/afl-cc.c
index c3910e6d..ab794877 100644
--- a/src/afl-cc.c
+++ b/src/afl-cc.c
@@ -73,7 +73,8 @@ enum {
   INSTRUMENT_GCC = 6,
   INSTRUMENT_CLANG = 7,
   INSTRUMENT_OPT_CTX = 8,
-  INSTRUMENT_OPT_NGRAM = 16
+  INSTRUMENT_OPT_NGRAM = 16,
+  INSTRUMENT_OPT_CALLER = 32,
 
 };
 
@@ -88,7 +89,7 @@ char instrument_mode_string[18][18] = {
     "GCC",
     "CLANG",
     "CTX",
-    "",
+    "CALLER",
     "",
     "",
     "",
@@ -1273,6 +1274,7 @@ int main(int argc, char **argv, char **envp) {
   }
 
   if (getenv("AFL_LLVM_CTX")) instrument_opt_mode |= INSTRUMENT_OPT_CTX;
+  if (getenv("AFL_LLVM_CALLER")) instrument_opt_mode |= INSTRUMENT_OPT_CALLER;
 
   if (getenv("AFL_LLVM_NGRAM_SIZE")) {
 
@@ -1388,6 +1390,13 @@ int main(int argc, char **argv, char **envp) {
 
       }
 
+      if (strncasecmp(ptr2, "caller", strlen("caller")) == 0) {
+
+        instrument_opt_mode |= INSTRUMENT_OPT_CALLER;
+        setenv("AFL_LLVM_CALLER", "1", 1);
+
+      }
+
       if (strncasecmp(ptr2, "ngram", strlen("ngram")) == 0) {
 
         u8 *ptr3 = ptr2 + strlen("ngram");
@@ -1421,6 +1430,13 @@ int main(int argc, char **argv, char **envp) {
 
   }
 
+  if ((instrument_opt_mode & INSTRUMENT_OPT_CTX) &&
+      (instrument_opt_mode & INSTRUMENT_OPT_CALLER)) {
+
+    FATAL("you cannot set CTX and CALLER together");
+
+  }
+
   if (instrument_opt_mode && instrument_mode == INSTRUMENT_DEFAULT &&
       (compiler_mode == LLVM || compiler_mode == UNSET)) {
 
@@ -1498,12 +1514,13 @@ int main(int argc, char **argv, char **envp) {
         "      CLASSIC              %s      no  yes     module yes yes    "
         "yes\n"
         "        - NORMAL\n"
+        "        - CALLER\n"
         "        - CTX\n"
         "        - NGRAM-{2-16}\n"
         "      INSTRIM                           no  yes     module yes yes "
         "   yes\n"
         "        - NORMAL\n"
-        "        - CTX\n"
+        "        - CALLER\n"
         "        - NGRAM-{2-16}\n"
         "  [GCC_PLUGIN] gcc plugin: %s%s\n"
         "      CLASSIC              DEFAULT      no  yes     no     no  no     "
@@ -1550,7 +1567,10 @@ int main(int argc, char **argv, char **envp) {
         NATIVE_MSG
 
         "  CLASSIC: decision target instrumentation (README.llvm.md)\n"
-        "  CTX:     CLASSIC + callee context (instrumentation/README.ctx.md)\n"
+        "  CALLER:  CLASSIC + single callee context "
+        "(instrumentation/README.ctx.md)\n"
+        "  CTX:     CLASSIC + full callee context "
+        "(instrumentation/README.ctx.md)\n"
         "  NGRAM-x: CLASSIC + previous path "
         "((instrumentation/README.ngram.md)\n"
         "  INSTRIM: Dominator tree (for LLVM <= 6.0) "
@@ -1644,15 +1664,17 @@ int main(int argc, char **argv, char **envp) {
             "  AFL_LLVM_CMPLOG: log operands of comparisons (RedQueen "
             "mutator)\n"
             "  AFL_LLVM_INSTRUMENT: set instrumentation mode:\n"
-            "    CLASSIC, INSTRIM, PCGUARD, LTO, GCC, CLANG, CTX, NGRAM-2 ... "
-            "NGRAM-16\n"
+            "    CLASSIC, INSTRIM, PCGUARD, LTO, GCC, CLANG, CALLER, CTX, "
+            "NGRAM-2 ..-16\n"
             " You can also use the old environment variables instead:\n"
             "  AFL_LLVM_USE_TRACE_PC: use LLVM trace-pc-guard instrumentation\n"
             "  AFL_LLVM_INSTRIM: use light weight instrumentation InsTrim\n"
             "  AFL_LLVM_INSTRIM_LOOPHEAD: optimize loop tracing for speed "
             "(option to INSTRIM)\n"
-            "  AFL_LLVM_CTX: use context sensitive coverage (for CLASSIC and "
-            "INSTRIM)\n"
+            "  AFL_LLVM_CALLER: use single context sensitive coverage (for "
+            "CLASSIC)\n"
+            "  AFL_LLVM_CTX: use full context sensitive coverage (for "
+            "CLASSIC)\n"
             "  AFL_LLVM_NGRAM_SIZE: use ngram prev_loc count coverage (for "
             "CLASSIC & INSTRIM)\n");
 
@@ -1770,7 +1792,7 @@ int main(int argc, char **argv, char **envp) {
   }
 
   if (instrument_opt_mode && compiler_mode != LLVM)
-    FATAL("CTX and NGRAM can only be used in LLVM mode");
+    FATAL("CTX, CALLER and NGRAM can only be used in LLVM mode");
 
   if (!instrument_opt_mode) {
 
@@ -1780,15 +1802,14 @@ int main(int argc, char **argv, char **envp) {
 
   } else {
 
-    if (instrument_opt_mode == INSTRUMENT_OPT_CTX)
+    char *ptr2 = alloc_printf(" + NGRAM-%u", ngram_size);
+    ptr = alloc_printf(
+        "%s%s%s%s", instrument_mode_string[instrument_mode],
+        (instrument_opt_mode & INSTRUMENT_OPT_CTX) ? " + CTX" : "",
+        (instrument_opt_mode & INSTRUMENT_OPT_CALLER) ? " + CALLER" : "",
+        (instrument_opt_mode & INSTRUMENT_OPT_NGRAM) ? ptr2 : "");
 
-      ptr = alloc_printf("%s + CTX", instrument_mode_string[instrument_mode]);
-    else if (instrument_opt_mode == INSTRUMENT_OPT_NGRAM)
-      ptr = alloc_printf("%s + NGRAM-%u",
-                         instrument_mode_string[instrument_mode], ngram_size);
-    else
-      ptr = alloc_printf("%s + CTX + NGRAM-%u",
-                         instrument_mode_string[instrument_mode], ngram_size);
+    ck_free(ptr2);
 
   }
 
@@ -1799,11 +1820,14 @@ int main(int argc, char **argv, char **envp) {
         "(requires LLVM 11 or higher)");
 #endif
 
-  if (instrument_opt_mode && instrument_mode != INSTRUMENT_CLASSIC &&
-      instrument_mode != INSTRUMENT_CFG)
+  if (instrument_opt_mode && instrument_mode == INSTRUMENT_CFG &&
+      instrument_opt_mode & INSTRUMENT_OPT_CTX)
+    FATAL("CFG instrumentation mode supports NGRAM and CALLER, but not CTX.");
+  else if (instrument_opt_mode && instrument_mode != INSTRUMENT_CLASSIC)
+    // we will drop CFG/INSTRIM in the future so do not advertise
     FATAL(
-        "CTX and NGRAM instrumentation options can only be used with LLVM and "
-        "CFG or CLASSIC instrumentation modes!");
+        "CALLER, CTX and NGRAM instrumentation options can only be used with "
+        "the LLVM CLASSIC instrumentation mode.");
 
   if (getenv("AFL_LLVM_SKIP_NEVERZERO") && getenv("AFL_LLVM_NOT_ZERO"))
     FATAL(
diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c
index fd5edc98..6f08f9f4 100644
--- a/src/afl-forkserver.c
+++ b/src/afl-forkserver.c
@@ -492,6 +492,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv,
              "malloc_context_size=0:"
              "symbolize=0:"
              "allocator_may_return_null=1:"
+             "detect_odr_violation=0:"
              "handle_segv=0:"
              "handle_sigbus=0:"
              "handle_abort=0:"
@@ -908,10 +909,12 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv,
   } else if (!fsrv->mem_limit) {
 
     SAYF("\n" cLRD "[-] " cRST
-         "Hmm, looks like the target binary terminated before we could"
-         " complete a handshake with the injected code.\n"
-         "If the target was compiled with afl-clang-lto and AFL_LLVM_MAP_ADDR"
-         " then recompiling without this parameter.\n"
+         "Hmm, looks like the target binary terminated before we could complete"
+         " a\n"
+         "handshake with the injected code.\n"
+         "Most likely the target has a huge coverage map, retry with setting"
+         " the\n"
+         "environment variable AFL_MAP_SIZE=4194304\n"
          "Otherwise there is a horrible bug in the fuzzer.\n"
          "Poke <afl-users@googlegroups.com> for troubleshooting tips.\n");
 
@@ -927,6 +930,10 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv,
         "explanations:\n\n"
 
         "%s"
+
+        "    - Most likely the target has a huge coverage map, retry with setting the\n"
+        "      environment variable AFL_MAP_SIZE=4194304\n\n"
+
         "    - The current memory limit (%s) is too restrictive, causing an "
         "OOM\n"
         "      fault in the dynamic linker. This can be fixed with the -m "
diff --git a/src/afl-fuzz-extras.c b/src/afl-fuzz-extras.c
index 7ecad233..52100fa1 100644
--- a/src/afl-fuzz-extras.c
+++ b/src/afl-fuzz-extras.c
@@ -387,6 +387,130 @@ static inline u8 memcmp_nocase(u8 *m1, u8 *m2, u32 len) {
 
 }
 
+/* add an extra/dict/token - no checks performed, no sorting */
+
+static void add_extra_nocheck(afl_state_t *afl, u8 *mem, u32 len) {
+
+  afl->extras = afl_realloc((void **)&afl->extras,
+                            (afl->extras_cnt + 1) * sizeof(struct extra_data));
+
+  if (unlikely(!afl->extras)) { PFATAL("alloc"); }
+
+  afl->extras[afl->extras_cnt].data = ck_alloc(len);
+  afl->extras[afl->extras_cnt].len = len;
+  memcpy(afl->extras[afl->extras_cnt].data, mem, len);
+  afl->extras_cnt++;
+
+  /* We only want to print this once */
+
+  if (afl->extras_cnt == afl->max_det_extras + 1) {
+
+    WARNF("More than %u tokens - will use them probabilistically.",
+          afl->max_det_extras);
+
+  }
+
+}
+
+/* Sometimes strings in input is transformed to unicode internally, so for
+   fuzzing we should attempt to de-unicode if it looks like simple unicode */
+
+void deunicode_extras(afl_state_t *afl) {
+
+  if (!afl->extras_cnt) return;
+
+  u32 i, j, orig_cnt = afl->extras_cnt;
+  u8  buf[64];
+
+  for (i = 0; i < orig_cnt; ++i) {
+
+    if (afl->extras[i].len < 6 || afl->extras[i].len > 64 ||
+        afl->extras[i].len % 2) {
+
+      continue;
+
+    }
+
+    u32 k = 0, z1 = 0, z2 = 0, z3 = 0, z4 = 0, half = afl->extras[i].len >> 1;
+    u32 quarter = half >> 1;
+
+    for (j = 0; j < afl->extras[i].len; ++j) {
+
+      switch (j % 4) {
+
+        case 2:
+          if (!afl->extras[i].data[j]) { ++z3; }
+          // fall through
+        case 0:
+          if (!afl->extras[i].data[j]) { ++z1; }
+          break;
+        case 3:
+          if (!afl->extras[i].data[j]) { ++z4; }
+          // fall through
+        case 1:
+          if (!afl->extras[i].data[j]) { ++z2; }
+          break;
+
+      }
+
+    }
+
+    if ((z1 < half && z2 < half) || z1 + z2 == afl->extras[i].len) { continue; }
+
+    // also maybe 32 bit unicode?
+    if (afl->extras[i].len % 4 == 0 && afl->extras[i].len >= 12 &&
+        (z3 == quarter || z4 == quarter) && z1 + z2 == quarter * 3) {
+
+      for (j = 0; j < afl->extras[i].len; ++j) {
+
+        if (z4 < quarter) {
+
+          if (j % 4 == 3) { buf[k++] = afl->extras[i].data[j]; }
+
+        } else if (z3 < quarter) {
+
+          if (j % 4 == 2) { buf[k++] = afl->extras[i].data[j]; }
+
+        } else if (z2 < half) {
+
+          if (j % 4 == 1) { buf[k++] = afl->extras[i].data[j]; }
+
+        } else {
+
+          if (j % 4 == 0) { buf[k++] = afl->extras[i].data[j]; }
+
+        }
+
+      }
+
+      add_extra_nocheck(afl, buf, k);
+      k = 0;
+
+    }
+
+    for (j = 0; j < afl->extras[i].len; ++j) {
+
+      if (z1 < half) {
+
+        if (j % 2 == 0) { buf[k++] = afl->extras[i].data[j]; }
+
+      } else {
+
+        if (j % 2 == 1) { buf[k++] = afl->extras[i].data[j]; }
+
+      }
+
+    }
+
+    add_extra_nocheck(afl, buf, k);
+
+  }
+
+  qsort(afl->extras, afl->extras_cnt, sizeof(struct extra_data),
+        compare_extras_len);
+
+}
+
 /* Removes duplicates from the loaded extras. This can happen if multiple files
    are loaded */
 
@@ -396,9 +520,9 @@ void dedup_extras(afl_state_t *afl) {
 
   u32 i, j, orig_cnt = afl->extras_cnt;
 
-  for (i = 0; i < afl->extras_cnt - 1; i++) {
+  for (i = 0; i < afl->extras_cnt - 1; ++i) {
 
-    for (j = i + 1; j < afl->extras_cnt; j++) {
+    for (j = i + 1; j < afl->extras_cnt; ++j) {
 
     restart_dedup:
 
@@ -462,30 +586,11 @@ void add_extra(afl_state_t *afl, u8 *mem, u32 len) {
 
   }
 
-  afl->extras = afl_realloc((void **)&afl->extras,
-                            (afl->extras_cnt + 1) * sizeof(struct extra_data));
-
-  if (unlikely(!afl->extras)) { PFATAL("alloc"); }
-
-  afl->extras[afl->extras_cnt].data = ck_alloc(len);
-  afl->extras[afl->extras_cnt].len = len;
-
-  memcpy(afl->extras[afl->extras_cnt].data, mem, len);
-
-  afl->extras_cnt++;
+  add_extra_nocheck(afl, mem, len);
 
   qsort(afl->extras, afl->extras_cnt, sizeof(struct extra_data),
         compare_extras_len);
 
-  /* We only want to print this once */
-
-  if (afl->extras_cnt == afl->max_det_extras + 1) {
-
-    WARNF("More than %u tokens - will use them probabilistically.",
-          afl->max_det_extras);
-
-  }
-
 }
 
 /* Maybe add automatic extra. */
diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index ad3e3b8e..835aba40 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -198,34 +198,35 @@ void create_alias_table(afl_state_t *afl) {
   while (nS)
     afl->alias_probability[S[--nS]] = 1;
 
-#ifdef INTROSPECTION
-  u8 fn[PATH_MAX];
-  snprintf(fn, PATH_MAX, "%s/introspection_corpus.txt", afl->out_dir);
-  FILE *f = fopen(fn, "a");
-  if (f) {
+  /*
+  #ifdef INTROSPECTION
+    u8 fn[PATH_MAX];
+    snprintf(fn, PATH_MAX, "%s/introspection_corpus.txt", afl->out_dir);
+    FILE *f = fopen(fn, "a");
+    if (f) {
+
+      for (i = 0; i < n; i++) {
+
+        struct queue_entry *q = afl->queue_buf[i];
+        fprintf(
+            f,
+            "entry=%u name=%s favored=%s variable=%s disabled=%s len=%u "
+            "exec_us=%u "
+            "bitmap_size=%u bitsmap_size=%u tops=%u weight=%f perf_score=%f\n",
+            i, q->fname, q->favored ? "true" : "false",
+            q->var_behavior ? "true" : "false", q->disabled ? "true" : "false",
+            q->len, (u32)q->exec_us, q->bitmap_size, q->bitsmap_size, q->tc_ref,
+            q->weight, q->perf_score);
 
-    for (i = 0; i < n; i++) {
+      }
 
-      struct queue_entry *q = afl->queue_buf[i];
-      fprintf(
-          f,
-          "entry=%u name=%s favored=%s variable=%s disabled=%s len=%u "
-          "exec_us=%u "
-          "bitmap_size=%u bitsmap_size=%u tops=%u weight=%f perf_score=%f\n",
-          i, q->fname, q->favored ? "true" : "false",
-          q->var_behavior ? "true" : "false", q->disabled ? "true" : "false",
-          q->len, (u32)q->exec_us, q->bitmap_size, q->bitsmap_size, q->tc_ref,
-          q->weight, q->perf_score);
+      fprintf(f, "\n");
+      fclose(f);
 
     }
 
-    fprintf(f, "\n");
-    fclose(f);
-
-  }
-
-#endif
-
+  #endif
+  */
   /*
   fprintf(stderr, "  entry  alias  probability  perf_score   weight
   filename\n"); for (u32 i = 0; i < n; ++i) fprintf(stderr, "  %5u  %5u  %11u
diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c
index 1ab5f996..9bfbf95b 100644
--- a/src/afl-fuzz-redqueen.c
+++ b/src/afl-fuzz-redqueen.c
@@ -1853,7 +1853,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
 }
 
 static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl,
-                              u8 *o_pattern, u8 *changed_val, u32 idx,
+                              u8 *o_pattern, u8 *changed_val, u8 plen, u32 idx,
                               u32 taint_len, u8 *orig_buf, u8 *buf, u8 *cbuf,
                               u32 len, u8 lvl, u8 *status) {
 
@@ -1866,7 +1866,7 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl,
 
   u8  save[40];
   u32 saved_idx = idx, pre, from = 0, to = 0, i, j;
-  u32 its_len = MIN((u32)32, len - idx);
+  u32 its_len = MIN((u32)plen, len - idx);
   its_len = MIN(its_len, taint_len);
   u32 saved_its_len = its_len;
 
@@ -2365,9 +2365,9 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
 
       status = 0;
 
-      if (unlikely(rtn_extend_encoding(afl, o->v0, o->v1, orig_o->v0,
-                                       orig_o->v1, idx, taint_len, orig_buf,
-                                       buf, cbuf, len, lvl, &status))) {
+      if (unlikely(rtn_extend_encoding(
+              afl, o->v0, o->v1, orig_o->v0, orig_o->v1, SHAPE_BYTES(h->shape),
+              idx, taint_len, orig_buf, buf, cbuf, len, lvl, &status))) {
 
         return 1;
 
@@ -2382,9 +2382,9 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
 
       status = 0;
 
-      if (unlikely(rtn_extend_encoding(afl, o->v1, o->v0, orig_o->v1,
-                                       orig_o->v0, idx, taint_len, orig_buf,
-                                       buf, cbuf, len, lvl, &status))) {
+      if (unlikely(rtn_extend_encoding(
+              afl, o->v1, o->v0, orig_o->v1, orig_o->v0, SHAPE_BYTES(h->shape),
+              idx, taint_len, orig_buf, buf, cbuf, len, lvl, &status))) {
 
         return 1;
 
diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c
index a02eadb2..09aff4fb 100644
--- a/src/afl-fuzz.c
+++ b/src/afl-fuzz.c
@@ -1437,23 +1437,8 @@ int main(int argc, char **argv_orig, char **envp) {
   // read_foreign_testcases(afl, 1); for the moment dont do this
   OKF("Loaded a total of %u seeds.", afl->queued_paths);
 
-  load_auto(afl);
-
   pivot_inputs(afl);
 
-  if (extras_dir_cnt) {
-
-    for (i = 0; i < extras_dir_cnt; i++) {
-
-      load_extras(afl, extras_dir[i]);
-
-    }
-
-    dedup_extras(afl);
-    OKF("Loaded a total of %u extras.", afl->extras_cnt);
-
-  }
-
   if (!afl->timeout_given) { find_timeout(afl); }  // only for resumes!
 
   if ((afl->tmp_dir = afl->afl_env.afl_tmpdir) != NULL &&
@@ -1681,6 +1666,22 @@ int main(int argc, char **argv_orig, char **envp) {
 
   }
 
+  load_auto(afl);
+
+  if (extras_dir_cnt) {
+
+    for (i = 0; i < extras_dir_cnt; i++) {
+
+      load_extras(afl, extras_dir[i]);
+
+    }
+
+  }
+
+  deunicode_extras(afl);
+  dedup_extras(afl);
+  if (afl->extras_cnt) { OKF("Loaded a total of %u extras.", afl->extras_cnt); }
+
   // after we have the correct bitmap size we can read the bitmap -B option
   // and set the virgin maps
   if (afl->in_bitmap) {
diff --git a/src/afl-showmap.c b/src/afl-showmap.c
index b40527d3..0fc76193 100644
--- a/src/afl-showmap.c
+++ b/src/afl-showmap.c
@@ -563,6 +563,7 @@ static void set_up_environment(afl_forkserver_t *fsrv) {
          "detect_leaks=0:"
          "allocator_may_return_null=1:"
          "symbolize=0:"
+         "detect_odr_violation=0:"
          "handle_segv=0:"
          "handle_sigbus=0:"
          "handle_abort=0:"
diff --git a/src/afl-tmin.c b/src/afl-tmin.c
index 15336959..6d04c652 100644
--- a/src/afl-tmin.c
+++ b/src/afl-tmin.c
@@ -717,6 +717,7 @@ static void set_up_environment(afl_forkserver_t *fsrv) {
          "detect_leaks=0:"
          "allocator_may_return_null=1:"
          "symbolize=0:"
+         "detect_odr_violation=0:"
          "handle_segv=0:"
          "handle_sigbus=0:"
          "handle_abort=0:"