Merge pull request #1141 from AFLplusplus/afl4

cmplog enhancement variant

7 files changed, 398 insertions, 133 deletions

diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c
index 44b6c6f9..6320a26b 100644
--- a/src/afl-forkserver.c
+++ b/src/afl-forkserver.c
@@ -342,6 +342,16 @@ static void report_error_and_exit(int error) {
           "the fuzzing target reports that the mmap() call to the shared "
           "memory failed.");
       break;
+    case FS_ERROR_OLD_CMPLOG:
+      FATAL(
+          "the -c cmplog target was instrumented with an too old afl++ "
+          "version, you need to recompile it.");
+      break;
+    case FS_ERROR_OLD_CMPLOG_QEMU:
+      FATAL(
+          "The AFL++ QEMU/FRIDA loaders are from an older version, for -c you "
+          "need to recompile it.\n");
+      break;
     default:
       FATAL("unknown error code %d from fuzzing target!", error);
 
@@ -663,6 +673,20 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv,
       if ((status & FS_OPT_OLD_AFLPP_WORKAROUND) == FS_OPT_OLD_AFLPP_WORKAROUND)
         status = (status & 0xf0ffffff);
 
+      if ((status & FS_OPT_NEWCMPLOG) == 0 && fsrv->cmplog_binary) {
+
+        if (fsrv->qemu_mode || fsrv->frida_mode) {
+
+          report_error_and_exit(FS_ERROR_OLD_CMPLOG_QEMU);
+
+        } else {
+
+          report_error_and_exit(FS_ERROR_OLD_CMPLOG);
+
+        }
+
+      }
+
       if ((status & FS_OPT_SNAPSHOT) == FS_OPT_SNAPSHOT) {
 
         fsrv->snapshot = 1;
diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c
index a1134a22..f4d3b77f 100644
--- a/src/afl-fuzz-one.c
+++ b/src/afl-fuzz-one.c
@@ -448,11 +448,11 @@ u8 fuzz_one_original(afl_state_t *afl) {
 
     ACTF(
         "Fuzzing test case #%u (%u total, %llu uniq crashes found, "
-        "perf_score=%0.0f, exec_us=%llu, hits=%u, map=%u)...",
+        "perf_score=%0.0f, exec_us=%llu, hits=%u, map=%u, ascii=%u)...",
         afl->current_entry, afl->queued_paths, afl->unique_crashes,
         afl->queue_cur->perf_score, afl->queue_cur->exec_us,
         likely(afl->n_fuzz) ? afl->n_fuzz[afl->queue_cur->n_fuzz_entry] : 0,
-        afl->queue_cur->bitmap_size);
+        afl->queue_cur->bitmap_size, afl->queue_cur->is_ascii);
     fflush(stdout);
 
   }
@@ -2003,11 +2003,16 @@ havoc_stage:
      where we take the input file and make random stacked tweaks. */
 
 #define MAX_HAVOC_ENTRY 59                                      /* 55 to 60 */
+#define MUTATE_ASCII_DICT 64
 
   u32 r_max, r;
 
   r_max = (MAX_HAVOC_ENTRY + 1) + (afl->extras_cnt ? 4 : 0) +
-          (afl->a_extras_cnt ? 4 : 0);
+          (afl->a_extras_cnt
+               ? (unlikely(afl->cmplog_binary && afl->queue_cur->is_ascii)
+                      ? MUTATE_ASCII_DICT
+                      : 4)
+               : 0);
 
   if (unlikely(afl->expand_havoc && afl->ready_for_splicing_count > 1)) {
 
@@ -2592,7 +2597,15 @@ havoc_stage:
 
           if (afl->a_extras_cnt) {
 
-            if (r < 2) {
+            u32 r_cmp = 2;
+
+            if (unlikely(afl->cmplog_binary && afl->queue_cur->is_ascii)) {
+
+              r_cmp = MUTATE_ASCII_DICT >> 1;
+
+            }
+
+            if (r < r_cmp) {
 
               /* Use the dictionary. */
 
@@ -2612,7 +2625,7 @@ havoc_stage:
 
               break;
 
-            } else if (r < 4) {
+            } else if (r < (r_cmp << 1)) {
 
               u32 use_extra = rand_below(afl, afl->a_extras_cnt);
               u32 extra_len = afl->a_extras[use_extra].len;
@@ -2641,7 +2654,7 @@ havoc_stage:
 
             } else {
 
-              r -= 4;
+              r -= (r_cmp << 1);
 
             }
 
diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index 33c2b561..1523d556 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -315,7 +315,96 @@ void mark_as_redundant(afl_state_t *afl, struct queue_entry *q, u8 state) {
 
 }
 
-/* check if ascii or UTF-8 */
+/* check if pointer is ascii or UTF-8 */
+
+u8 check_if_text_buf(u8 *buf, u32 len) {
+
+  u32 offset = 0, ascii = 0, utf8 = 0;
+
+  while (offset < len) {
+
+    // ASCII: <= 0x7F to allow ASCII control characters
+    if ((buf[offset + 0] == 0x09 || buf[offset + 0] == 0x0A ||
+         buf[offset + 0] == 0x0D ||
+         (0x20 <= buf[offset + 0] && buf[offset + 0] <= 0x7E))) {
+
+      offset++;
+      utf8++;
+      ascii++;
+      continue;
+
+    }
+
+    if (isascii((int)buf[offset]) || isprint((int)buf[offset])) {
+
+      ascii++;
+      // we continue though as it can also be a valid utf8
+
+    }
+
+    // non-overlong 2-byte
+    if (len - offset > 1 &&
+        ((0xC2 <= buf[offset + 0] && buf[offset + 0] <= 0xDF) &&
+         (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF))) {
+
+      offset += 2;
+      utf8++;
+      continue;
+
+    }
+
+    // excluding overlongs
+    if ((len - offset > 2) &&
+        ((buf[offset + 0] == 0xE0 &&
+          (0xA0 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) &&
+          (0x80 <= buf[offset + 2] &&
+           buf[offset + 2] <= 0xBF)) ||  // straight 3-byte
+         (((0xE1 <= buf[offset + 0] && buf[offset + 0] <= 0xEC) ||
+           buf[offset + 0] == 0xEE || buf[offset + 0] == 0xEF) &&
+          (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) &&
+          (0x80 <= buf[offset + 2] &&
+           buf[offset + 2] <= 0xBF)) ||  // excluding surrogates
+         (buf[offset + 0] == 0xED &&
+          (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0x9F) &&
+          (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF)))) {
+
+      offset += 3;
+      utf8++;
+      continue;
+
+    }
+
+    // planes 1-3
+    if ((len - offset > 3) &&
+        ((buf[offset + 0] == 0xF0 &&
+          (0x90 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) &&
+          (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF) &&
+          (0x80 <= buf[offset + 3] &&
+           buf[offset + 3] <= 0xBF)) ||  // planes 4-15
+         ((0xF1 <= buf[offset + 0] && buf[offset + 0] <= 0xF3) &&
+          (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) &&
+          (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF) &&
+          (0x80 <= buf[offset + 3] && buf[offset + 3] <= 0xBF)) ||  // plane 16
+         (buf[offset + 0] == 0xF4 &&
+          (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0x8F) &&
+          (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF) &&
+          (0x80 <= buf[offset + 3] && buf[offset + 3] <= 0xBF)))) {
+
+      offset += 4;
+      utf8++;
+      continue;
+
+    }
+
+    offset++;
+
+  }
+
+  return (utf8 > ascii ? utf8 : ascii);
+
+}
+
+/* check if queue entry is ascii or UTF-8 */
 
 static u8 check_if_text(afl_state_t *afl, struct queue_entry *q) {
 
diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c
index a1d6e021..0a6e5eee 100644
--- a/src/afl-fuzz-redqueen.c
+++ b/src/afl-fuzz-redqueen.c
@@ -45,6 +45,23 @@ enum {
 
 };
 
+// add to dictionary enum
+// DEFAULT = 1, notTXT = 2, FOUND = 4, notSAME = 8
+enum {
+
+  DICT_ADD_NEVER = 0,
+  DICT_ADD_NOTFOUND_SAME_TXT = 1,
+  DICT_ADD_NOTFOUND_SAME = 3,
+  DICT_ADD_FOUND_SAME_TXT = 5,
+  DICT_ADD_FOUND_SAME = 7,
+  DICT_ADD_NOTFOUND_TXT = 9,
+  DICT_ADD_NOTFOUND = 11,
+  DICT_ADD_FOUND_TXT = 13,
+  DICT_ADD_FOUND = 15,
+  DICT_ADD_ANY = DICT_ADD_FOUND
+
+};
+
 // CMPLOG LVL
 enum {
 
@@ -54,6 +71,8 @@ enum {
 
 };
 
+#define DICT_ADD_STRATEGY DICT_ADD_FOUND_SAME
+
 struct range {
 
   u32           start;
@@ -64,6 +83,10 @@ struct range {
 
 };
 
+static u32 hshape;
+static u64 screen_update;
+static u64 last_update;
+
 static struct range *add_range(struct range *ranges, u32 start, u32 end) {
 
   struct range *r = ck_alloc_nozero(sizeof(struct range));
@@ -252,7 +275,6 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len,
   u64 start_time = get_cur_time();
 #endif
 
-  u32 screen_update;
   u64 orig_hit_cnt, new_hit_cnt, exec_cksum;
   orig_hit_cnt = afl->queued_paths + afl->unique_crashes;
 
@@ -261,24 +283,6 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len,
   afl->stage_max = (len << 1);
   afl->stage_cur = 0;
 
-  if (likely(afl->queue_cur->exec_us)) {
-
-    if (likely((100000 / 2) >= afl->queue_cur->exec_us)) {
-
-      screen_update = 100000 / afl->queue_cur->exec_us;
-
-    } else {
-
-      screen_update = 1;
-
-    }
-
-  } else {
-
-    screen_update = 100000;
-
-  }
-
   // in colorization we do not classify counts, hence we have to calculate
   // the original checksum.
   if (unlikely(get_exec_checksum(afl, buf, len, &exec_cksum))) {
@@ -348,7 +352,7 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len,
 
     }
 
-    if (++afl->stage_cur % screen_update) { show_stats(afl); };
+    if (++afl->stage_cur % screen_update == 0) { show_stats(afl); };
 
   }
 
@@ -440,10 +444,10 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len,
     fprintf(
         f,
         "Colorization: fname=%s len=%u ms=%llu result=%u execs=%u found=%llu "
-        "taint=%u\n",
+        "taint=%u ascii=%u auto_extra_before=%u\n",
         afl->queue_cur->fname, len, get_cur_time() - start_time,
         afl->queue_cur->colorized, afl->stage_cur, new_hit_cnt - orig_hit_cnt,
-        positions);
+        positions, afl->queue_cur->is_ascii ? 1 : 0, afl->a_extras_cnt);
 
   #ifndef _DEBUG
     if (afl->not_on_tty) { fclose(f); }
@@ -759,11 +763,18 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
   u32 its_len = MIN(len - idx, taint_len);
 
+  if (afl->fsrv.total_execs - last_update > screen_update) {
+
+    show_stats(afl);
+    last_update = afl->fsrv.total_execs;
+
+  }
+
   // fprintf(stderr,
   //         "Encode: %llx->%llx into %llx(<-%llx) at idx=%u "
   //         "taint_len=%u shape=%u attr=%u\n",
   //         o_pattern, pattern, repl, changed_val, idx, taint_len,
-  //         h->shape + 1, attr);
+  //         hshape, attr);
 
   //#ifdef CMPLOG_SOLVE_TRANSFORM
   // reverse atoi()/strnu?toll() is expensive, so we only to it in lvl 3
@@ -845,7 +856,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
       u64 b_val, o_b_val, mask;
       u8  bytes;
 
-      switch (SHAPE_BYTES(h->shape)) {
+      switch (hshape) {
 
         case 0:
         case 1:
@@ -924,7 +935,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
       s64 diff = pattern - b_val;
       s64 o_diff = o_pattern - o_b_val;
       /* fprintf(stderr, "DIFF1 idx=%03u shape=%02u %llx-%llx=%lx\n", idx,
-                 h->shape + 1, o_pattern, o_b_val, o_diff);
+                 hshape, o_pattern, o_b_val, o_diff);
          fprintf(stderr, "DIFF1 %016llx %llx-%llx=%lx\n", repl, pattern,
                  b_val, diff); */
       if (diff == o_diff && diff) {
@@ -953,7 +964,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
         s64 o_diff = o_pattern ^ o_b_val;
 
         /* fprintf(stderr, "DIFF2 idx=%03u shape=%02u %llx-%llx=%lx\n",
-                   idx, h->shape + 1, o_pattern, o_b_val, o_diff);
+                   idx, hshape, o_pattern, o_b_val, o_diff);
            fprintf(stderr,
                    "DIFF2 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff);
         */
@@ -1002,7 +1013,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
         }
 
         /* fprintf(stderr, "DIFF3 idx=%03u shape=%02u %llx-%llx=%lx\n",
-                   idx, h->shape + 1, o_pattern, o_b_val, o_diff);
+                   idx, hshape, o_pattern, o_b_val, o_diff);
            fprintf(stderr,
                    "DIFF3 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff);
         */
@@ -1051,7 +1062,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
         }
 
         /* fprintf(stderr, "DIFF4 idx=%03u shape=%02u %llx-%llx=%lx\n",
-                   idx, h->shape + 1, o_pattern, o_b_val, o_diff);
+                   idx, hshape, o_pattern, o_b_val, o_diff);
            fprintf(stderr,
                    "DIFF4 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff);
         */
@@ -1089,7 +1100,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
   if ((lvl & LVL1) || attr >= IS_FP_MOD) {
 
-    if (SHAPE_BYTES(h->shape) >= 8 && *status != 1) {
+    if (hshape >= 8 && *status != 1) {
 
       // if (its_len >= 8)
       //   fprintf(stderr,
@@ -1132,7 +1143,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
     }
 
-    if (SHAPE_BYTES(h->shape) >= 4 && *status != 1) {
+    if (hshape >= 4 && *status != 1) {
 
       // if (its_len >= 4 && (attr <= 1 || attr >= 8))
       //   fprintf(stderr,
@@ -1173,7 +1184,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
     }
 
-    if (SHAPE_BYTES(h->shape) >= 2 && *status != 1) {
+    if (hshape >= 2 && *status != 1) {
 
       if (its_len >= 2 &&
           ((*buf_16 == (u16)pattern && *o_buf_16 == (u16)o_pattern) ||
@@ -1244,11 +1255,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
   }
 
-  if (!(attr & (IS_GREATER | IS_LESSER)) || SHAPE_BYTES(h->shape) < 4) {
-
-    return 0;
-
-  }
+  if (!(attr & (IS_GREATER | IS_LESSER)) || hshape < 4) { return 0; }
 
   // transform >= to < and <= to >
   if ((attr & IS_EQUAL) && (attr & (IS_GREATER | IS_LESSER))) {
@@ -1272,7 +1279,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
     if (attr & IS_GREATER) {
 
-      if (SHAPE_BYTES(h->shape) == 4 && its_len >= 4) {
+      if (hshape == 4 && its_len >= 4) {
 
         float *f = (float *)&repl;
         float  g = *f;
@@ -1280,7 +1287,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
         u32 *r = (u32 *)&g;
         repl_new = (u32)*r;
 
-      } else if (SHAPE_BYTES(h->shape) == 8 && its_len >= 8) {
+      } else if (hshape == 8 && its_len >= 8) {
 
         double *f = (double *)&repl;
         double  g = *f;
@@ -1307,7 +1314,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
     } else {
 
-      if (SHAPE_BYTES(h->shape) == 4) {
+      if (hshape == 4) {
 
         float *f = (float *)&repl;
         float  g = *f;
@@ -1315,7 +1322,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
         u32 *r = (u32 *)&g;
         repl_new = (u32)*r;
 
-      } else if (SHAPE_BYTES(h->shape) == 8) {
+      } else if (hshape == 8) {
 
         double *f = (double *)&repl;
         double  g = *f;
@@ -1342,7 +1349,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
     }
 
     // transform double to float, llvm likes to do that internally ...
-    if (SHAPE_BYTES(h->shape) == 8 && its_len >= 4) {
+    if (hshape == 8 && its_len >= 4) {
 
       double *f = (double *)&repl;
       float   g = (float)*f;
@@ -1353,7 +1360,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
       memcpy(((char *)&repl_new) + 4, (char *)&g, 4);
 #endif
       changed_val = repl_new;
-      h->shape = 3;  // modify shape
+      hshape = 4;  // modify shape
 
       // fprintf(stderr, "DOUBLE2FLOAT %llx\n", repl_new);
 
@@ -1361,12 +1368,12 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
               afl, h, pattern, repl_new, o_pattern, changed_val, 16, idx,
               taint_len, orig_buf, buf, cbuf, len, 1, lvl, status))) {
 
-        h->shape = 7;  // recover shape
+        hshape = 8;  // recover shape
         return 1;
 
       }
 
-      h->shape = 7;  // recover shape
+      hshape = 8;  // recover shape
 
     }
 
@@ -1421,6 +1428,13 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h,
                                u32 taint_len, u8 *orig_buf, u8 *buf, u8 *cbuf,
                                u32 len, u8 do_reverse, u8 lvl, u8 *status) {
 
+  if (afl->fsrv.total_execs - last_update > screen_update) {
+
+    show_stats(afl);
+    last_update = afl->fsrv.total_execs;
+
+  }
+
   u8 *ptr = (u8 *)&buf[idx];
   u8 *o_ptr = (u8 *)&orig_buf[idx];
   u8 *p = (u8 *)&pattern;
@@ -1428,52 +1442,51 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h,
   u8 *r = (u8 *)&repl;
   u8  backup[16];
   u32 its_len = MIN(len - idx, taint_len);
-  u32 shape = h->shape + 1;
   #if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
   size_t off = 0;
   #else
-  size_t off = 16 - shape;
+  size_t off = 16 - hshape;
   #endif
 
-  if (its_len >= shape) {
+  if (its_len >= hshape) {
 
   #ifdef _DEBUG
     fprintf(stderr, "TestUN: %u>=%u (len=%u idx=%u attr=%u off=%lu) (%u) ",
-            its_len, shape, len, idx, attr, off, do_reverse);
+            its_len, hshape, len, idx, attr, off, do_reverse);
     u32 i;
     u8 *o_r = (u8 *)&changed_val;
-    for (i = 0; i < shape; i++)
+    for (i = 0; i < hshape; i++)
       fprintf(stderr, "%02x", ptr[i]);
     fprintf(stderr, "==");
-    for (i = 0; i < shape; i++)
+    for (i = 0; i < hshape; i++)
       fprintf(stderr, "%02x", p[off + i]);
     fprintf(stderr, " ");
-    for (i = 0; i < shape; i++)
+    for (i = 0; i < hshape; i++)
       fprintf(stderr, "%02x", o_ptr[i]);
     fprintf(stderr, "==");
-    for (i = 0; i < shape; i++)
+    for (i = 0; i < hshape; i++)
       fprintf(stderr, "%02x", o_p[off + i]);
     fprintf(stderr, " <= ");
-    for (i = 0; i < shape; i++)
+    for (i = 0; i < hshape; i++)
       fprintf(stderr, "%02x", r[off + i]);
     fprintf(stderr, "<-");
-    for (i = 0; i < shape; i++)
+    for (i = 0; i < hshape; i++)
       fprintf(stderr, "%02x", o_r[off + i]);
     fprintf(stderr, "\n");
   #endif
 
-    if (!memcmp(ptr, p + off, shape) && !memcmp(o_ptr, o_p + off, shape)) {
+    if (!memcmp(ptr, p + off, hshape) && !memcmp(o_ptr, o_p + off, hshape)) {
 
-      memcpy(backup, ptr, shape);
-      memcpy(ptr, r + off, shape);
+      memcpy(backup, ptr, hshape);
+      memcpy(ptr, r + off, hshape);
 
       if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; }
 
   #ifdef CMPLOG_COMBINE
-      if (*status == 1) { memcpy(cbuf + idx, r, shape); }
+      if (*status == 1) { memcpy(cbuf + idx, r, hshape); }
   #endif
 
-      memcpy(ptr, backup, shape);
+      memcpy(ptr, backup, hshape);
 
   #ifdef _DEBUG
       fprintf(stderr, "Status=%u\n", *status);
@@ -1485,10 +1498,10 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h,
     if (do_reverse && *status != 1) {
 
       if (unlikely(cmp_extend_encodingN(
-              afl, h, SWAPN(pattern, (shape << 3)), SWAPN(repl, (shape << 3)),
-              SWAPN(o_pattern, (shape << 3)), SWAPN(changed_val, (shape << 3)),
-              attr, idx, taint_len, orig_buf, buf, cbuf, len, 0, lvl,
-              status))) {
+              afl, h, SWAPN(pattern, (hshape << 3)), SWAPN(repl, (hshape << 3)),
+              SWAPN(o_pattern, (hshape << 3)),
+              SWAPN(changed_val, (hshape << 3)), attr, idx, taint_len, orig_buf,
+              buf, cbuf, len, 0, lvl, status))) {
 
         return 1;
 
@@ -1615,6 +1628,8 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
   u8  s_v0_inc = 1, s_v1_inc = 1;
   u8  s_v0_dec = 1, s_v1_dec = 1;
 
+  hshape = SHAPE_BYTES(h->shape);
+
   if (h->hits > CMP_MAP_H) {
 
     loggeds = CMP_MAP_H;
@@ -1626,7 +1641,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
   }
 
 #ifdef WORD_SIZE_64
-  switch (SHAPE_BYTES(h->shape)) {
+  switch (hshape) {
 
     case 1:
     case 2:
@@ -1679,8 +1694,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
 
 #ifdef _DEBUG
     fprintf(stderr, "Handling: %llx->%llx vs %llx->%llx attr=%u shape=%u\n",
-            orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute,
-            SHAPE_BYTES(h->shape));
+            orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute, hshape);
 #endif
 
     t = taint;
@@ -1830,27 +1844,41 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
             "END: %llx->%llx vs %llx->%llx attr=%u i=%u found=%u "
             "isN=%u size=%u\n",
             orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute, i, found_one,
-            is_n, SHAPE_BYTES(h->shape));
+            is_n, hshape);
 #endif
 
-    // If failed, add to dictionary
-    if (!found_one) {
+    // we only learn 16 bit +
+    if (hshape > 1) {
 
-      if (afl->pass_stats[key].total == 0) {
+      if (!found_one || afl->queue_cur->is_ascii) {
 
 #ifdef WORD_SIZE_64
         if (unlikely(is_n)) {
 
-          try_to_add_to_dictN(afl, s128_v0, SHAPE_BYTES(h->shape));
-          try_to_add_to_dictN(afl, s128_v1, SHAPE_BYTES(h->shape));
+          if (!found_one ||
+              check_if_text_buf((u8 *)&s128_v0, SHAPE_BYTES(h->shape)) ==
+                  SHAPE_BYTES(h->shape))
+            try_to_add_to_dictN(afl, s128_v0, SHAPE_BYTES(h->shape));
+          if (!found_one ||
+              check_if_text_buf((u8 *)&s128_v1, SHAPE_BYTES(h->shape)) ==
+                  SHAPE_BYTES(h->shape))
+            try_to_add_to_dictN(afl, s128_v1, SHAPE_BYTES(h->shape));
 
         } else
 
 #endif
         {
 
-          try_to_add_to_dict(afl, o->v0, SHAPE_BYTES(h->shape));
-          try_to_add_to_dict(afl, o->v1, SHAPE_BYTES(h->shape));
+          if (!memcmp((u8 *)&o->v0, (u8 *)&orig_o->v0, SHAPE_BYTES(h->shape)) &&
+              (!found_one ||
+               check_if_text_buf((u8 *)&o->v0, SHAPE_BYTES(h->shape)) ==
+                   SHAPE_BYTES(h->shape)))
+            try_to_add_to_dict(afl, o->v0, SHAPE_BYTES(h->shape));
+          if (!memcmp((u8 *)&o->v1, (u8 *)&orig_o->v1, SHAPE_BYTES(h->shape)) &&
+              (!found_one ||
+               check_if_text_buf((u8 *)&o->v1, SHAPE_BYTES(h->shape)) ==
+                   SHAPE_BYTES(h->shape)))
+            try_to_add_to_dict(afl, o->v1, SHAPE_BYTES(h->shape));
 
         }
 
@@ -1882,8 +1910,9 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
 
 }
 
-static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl,
-                              u8 *o_pattern, u8 *changed_val, u8 plen, u32 idx,
+static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry,
+                              struct cmpfn_operands *o,
+                              struct cmpfn_operands *orig_o, u32 idx,
                               u32 taint_len, u8 *orig_buf, u8 *buf, u8 *cbuf,
                               u32 len, u8 lvl, u8 *status) {
 
@@ -1894,9 +1923,60 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl,
   //  (void)(changed_val);
   //#endif
 
+  if (afl->fsrv.total_execs - last_update > screen_update) {
+
+    show_stats(afl);
+    last_update = afl->fsrv.total_execs;
+
+  }
+
+  u8 *pattern, *repl, *o_pattern, *changed_val;
+  u8  l0, l1, ol0, ol1;
+
+  if (entry == 0) {
+
+    pattern = o->v0;
+    repl = o->v1;
+    o_pattern = orig_o->v0;
+    changed_val = orig_o->v1;
+    l0 = o->v0_len;
+    ol0 = orig_o->v0_len;
+    l1 = o->v1_len;
+    ol1 = orig_o->v1_len;
+
+  } else {
+
+    pattern = o->v1;
+    repl = o->v0;
+    o_pattern = orig_o->v1;
+    changed_val = orig_o->v0;
+    l0 = o->v1_len;
+    ol0 = orig_o->v1_len;
+    l1 = o->v0_len;
+    ol1 = orig_o->v0_len;
+
+  }
+
+  if (l0 >= 0x80 || ol0 >= 0x80) {
+
+    l0 -= 0x80;
+    l1 -= 0x80;
+    ol0 -= 0x80;
+    ol1 -= 0x80;
+
+  }
+
+  if (l0 == 0 || l1 == 0 || ol0 == 0 || ol1 == 0 || l0 > 31 || l1 > 31 ||
+      ol0 > 31 || ol1 > 31) {
+
+    l0 = l1 = ol0 = ol1 = hshape;
+
+  }
+
+  u8  lmax = MAX(l0, ol0);
   u8  save[40];
   u32 saved_idx = idx, pre, from = 0, to = 0, i, j;
-  u32 its_len = MIN((u32)plen, len - idx);
+  u32 its_len = MIN(MIN(lmax, hshape), len - idx);
   its_len = MIN(its_len, taint_len);
   u32 saved_its_len = its_len;
 
@@ -1912,7 +1992,8 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl,
   (void)(j);
 
 #ifdef _DEBUG
-  fprintf(stderr, "RTN T idx=%u lvl=%02x ", idx, lvl);
+  fprintf(stderr, "RTN T idx=%u lvl=%02x is_txt=%u shape=%u/%u ", idx, lvl,
+          o->v0_len >= 0x80 ? 1 : 0, hshape, l0);
   for (j = 0; j < 8; j++)
     fprintf(stderr, "%02x", orig_buf[idx + j]);
   fprintf(stderr, " -> ");
@@ -1972,10 +2053,10 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl,
 
   }
 
-  //#ifdef CMPLOG_SOLVE_TRANSFORM
-
   if (*status == 1) return 0;
 
+  // transform solving
+
   if (afl->cmplog_enable_transform && (lvl & LVL3)) {
 
     u32 toupper = 0, tolower = 0, xor = 0, arith = 0, tohex = 0, fromhex = 0;
@@ -2322,6 +2403,8 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
   u32                i, j, idx, have_taint = 1, taint_len, loggeds;
   u8                 status = 0, found_one = 0;
 
+  hshape = SHAPE_BYTES(h->shape);
+
   if (h->hits > CMP_MAP_RTN_H) {
 
     loggeds = CMP_MAP_RTN_H;
@@ -2353,18 +2436,22 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
     }
 
     /*
-      struct cmp_header *hh = &afl->orig_cmp_map->headers[key];
-      fprintf(stderr, "RTN N hits=%u id=%u shape=%u attr=%u v0=", h->hits,
-              h->id, h->shape, h->attribute);
-      for (j = 0; j < 8; j++) fprintf(stderr, "%02x", o->v0[j]);
-      fprintf(stderr, " v1=");
-      for (j = 0; j < 8; j++) fprintf(stderr, "%02x", o->v1[j]);
-      fprintf(stderr, "\nRTN O hits=%u id=%u shape=%u attr=%u o0=",
-              hh->hits, hh->id, hh->shape, hh->attribute);
-      for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_o->v0[j]);
-      fprintf(stderr, " o1=");
-      for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_o->v1[j]);
-      fprintf(stderr, "\n");
+    struct cmp_header *hh = &afl->orig_cmp_map->headers[key];
+    fprintf(stderr, "RTN N hits=%u id=%u shape=%u attr=%u v0=", h->hits, h->id,
+            hshape, h->attribute);
+    for (j = 0; j < 8; j++)
+      fprintf(stderr, "%02x", o->v0[j]);
+    fprintf(stderr, " v1=");
+    for (j = 0; j < 8; j++)
+      fprintf(stderr, "%02x", o->v1[j]);
+    fprintf(stderr, "\nRTN O hits=%u id=%u shape=%u attr=%u o0=", hh->hits,
+            hh->id, hshape, hh->attribute);
+    for (j = 0; j < 8; j++)
+      fprintf(stderr, "%02x", orig_o->v0[j]);
+    fprintf(stderr, " o1=");
+    for (j = 0; j < 8; j++)
+      fprintf(stderr, "%02x", orig_o->v1[j]);
+    fprintf(stderr, "\n");
     */
 
     t = taint;
@@ -2400,25 +2487,24 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
 
 #ifdef _DEBUG
       int w;
-      fprintf(stderr, "key=%u idx=%u len=%u o0=", key, idx,
-              SHAPE_BYTES(h->shape));
-      for (w = 0; w < SHAPE_BYTES(h->shape); ++w)
+      fprintf(stderr, "key=%u idx=%u len=%u o0=", key, idx, hshape);
+      for (w = 0; w < hshape; ++w)
         fprintf(stderr, "%02x", orig_o->v0[w]);
       fprintf(stderr, " v0=");
-      for (w = 0; w < SHAPE_BYTES(h->shape); ++w)
+      for (w = 0; w < hshape; ++w)
         fprintf(stderr, "%02x", o->v0[w]);
       fprintf(stderr, " o1=");
-      for (w = 0; w < SHAPE_BYTES(h->shape); ++w)
+      for (w = 0; w < hshape; ++w)
         fprintf(stderr, "%02x", orig_o->v1[w]);
       fprintf(stderr, " v1=");
-      for (w = 0; w < SHAPE_BYTES(h->shape); ++w)
+      for (w = 0; w < hshape; ++w)
         fprintf(stderr, "%02x", o->v1[w]);
       fprintf(stderr, "\n");
 #endif
 
-      if (unlikely(rtn_extend_encoding(
-              afl, o->v0, o->v1, orig_o->v0, orig_o->v1, SHAPE_BYTES(h->shape),
-              idx, taint_len, orig_buf, buf, cbuf, len, lvl, &status))) {
+      if (unlikely(rtn_extend_encoding(afl, 0, o, orig_o, idx, taint_len,
+                                       orig_buf, buf, cbuf, len, lvl,
+                                       &status))) {
 
         return 1;
 
@@ -2433,9 +2519,9 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
 
       status = 0;
 
-      if (unlikely(rtn_extend_encoding(
-              afl, o->v1, o->v0, orig_o->v1, orig_o->v0, SHAPE_BYTES(h->shape),
-              idx, taint_len, orig_buf, buf, cbuf, len, lvl, &status))) {
+      if (unlikely(rtn_extend_encoding(afl, 1, o, orig_o, idx, taint_len,
+                                       orig_buf, buf, cbuf, len, lvl,
+                                       &status))) {
 
         return 1;
 
@@ -2450,16 +2536,42 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
 
     }
 
-    // If failed, add to dictionary
-    if (!found_one && (lvl & LVL1)) {
+    //  if (unlikely(!afl->pass_stats[key].total)) {
+
+    if ((!found_one && (lvl & LVL1)) || afl->queue_cur->is_ascii) {
+
+      // if (unlikely(!afl->pass_stats[key].total)) {
+
+      u32 shape_len = SHAPE_BYTES(h->shape);
+      u32 v0_len = shape_len, v1_len = shape_len;
+      if (afl->queue_cur->is_ascii ||
+          check_if_text_buf((u8 *)&o->v0, shape_len) == shape_len) {
+
+        if (strlen(o->v0)) v0_len = strlen(o->v0);
+
+      }
 
-      if (unlikely(!afl->pass_stats[key].total)) {
+      if (afl->queue_cur->is_ascii ||
+          check_if_text_buf((u8 *)&o->v1, shape_len) == shape_len) {
 
-        maybe_add_auto(afl, o->v0, SHAPE_BYTES(h->shape));
-        maybe_add_auto(afl, o->v1, SHAPE_BYTES(h->shape));
+        if (strlen(o->v1)) v1_len = strlen(o->v1);
 
       }
 
+      // fprintf(stderr, "SHOULD: found:%u ascii:%u text?%u:%u %u:%s %u:%s \n",
+      // found_one, afl->queue_cur->is_ascii, check_if_text_buf((u8 *)&o->v0,
+      // shape_len), check_if_text_buf((u8 *)&o->v1, shape_len), v0_len,
+      // o->v0, v1_len, o->v1);
+
+      if (!memcmp(o->v0, orig_o->v0, v0_len) ||
+          (!found_one || check_if_text_buf((u8 *)&o->v0, v0_len) == v0_len))
+        maybe_add_auto(afl, o->v0, v0_len);
+      if (!memcmp(o->v1, orig_o->v1, v1_len) ||
+          (!found_one || check_if_text_buf((u8 *)&o->v1, v1_len) == v1_len))
+        maybe_add_auto(afl, o->v1, v1_len);
+
+      //}
+
     }
 
   rtn_fuzz_next_iter:
@@ -2492,6 +2604,23 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) {
   }
 
   struct tainted *taint = NULL;
+  if (likely(afl->queue_cur->exec_us)) {
+
+    if (likely((100000 / 2) >= afl->queue_cur->exec_us)) {
+
+      screen_update = 100000 / afl->queue_cur->exec_us;
+
+    } else {
+
+      screen_update = 1;
+
+    }
+
+  } else {
+
+    screen_update = 100000;
+
+  }
 
   if (!afl->queue_cur->taint || !afl->queue_cur->cmplog_colorinput) {
 
@@ -2592,8 +2721,6 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) {
   u64 orig_hit_cnt, new_hit_cnt;
   u64 orig_execs = afl->fsrv.total_execs;
   orig_hit_cnt = afl->queued_paths + afl->unique_crashes;
-  u64 screen_update = 100000 / afl->queue_cur->exec_us,
-      execs = afl->fsrv.total_execs;
 
   afl->stage_name = "input-to-state";
   afl->stage_short = "its";
@@ -2630,11 +2757,13 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) {
 
     if (afl->shm.cmp_map->headers[k].type == CMP_TYPE_INS) {
 
+      // fprintf(stderr, "INS %u\n", k);
       afl->stage_max +=
           MIN((u32)(afl->shm.cmp_map->headers[k].hits), (u32)CMP_MAP_H);
 
     } else {
 
+      // fprintf(stderr, "RTN %u\n", k);
       afl->stage_max +=
           MIN((u32)(afl->shm.cmp_map->headers[k].hits), (u32)CMP_MAP_RTN_H);
 
@@ -2673,13 +2802,6 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) {
 
     }
 
-    if (afl->fsrv.total_execs - execs > screen_update) {
-
-      execs = afl->fsrv.total_execs;
-      show_stats(afl);
-
-    }
-
   }
 
   r = 0;
@@ -2795,9 +2917,10 @@ exit_its:
   if (f) {
 
     fprintf(f,
-            "Cmplog: fname=%s len=%u ms=%llu result=%u finds=%llu entries=%u\n",
+            "Cmplog: fname=%s len=%u ms=%llu result=%u finds=%llu entries=%u "
+            "auto_extra_after=%u\n",
             afl->queue_cur->fname, len, get_cur_time() - start_time, r,
-            new_hit_cnt - orig_hit_cnt, cmp_locations);
+            new_hit_cnt - orig_hit_cnt, cmp_locations, afl->a_extras_cnt);
 
   #ifndef _DEBUG
     if (afl->not_on_tty) { fclose(f); }
diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c
index 1d48a76c..808bf258 100644
--- a/src/afl-fuzz-stats.c
+++ b/src/afl-fuzz-stats.c
@@ -278,6 +278,7 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg,
           "total_edges       : %u\n"
           "var_byte_count    : %u\n"
           "havoc_expansion   : %u\n"
+          "auto_dict_entries : %u\n"
           "testcache_size    : %llu\n"
           "testcache_count   : %u\n"
           "testcache_evict   : %u\n"
@@ -316,7 +317,7 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg,
           -1,
 #endif
           t_bytes, afl->fsrv.real_map_size, afl->var_byte_count,
-          afl->expand_havoc, afl->q_testcase_cache_size,
+          afl->expand_havoc, afl->a_extras_cnt, afl->q_testcase_cache_size,
           afl->q_testcase_cache_count, afl->q_testcase_evictions,
           afl->use_banner, afl->unicorn_mode ? "unicorn" : "",
           afl->fsrv.qemu_mode ? "qemu " : "",
diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c
index a4093693..c08b8fbb 100644
--- a/src/afl-fuzz.c
+++ b/src/afl-fuzz.c
@@ -2235,13 +2235,12 @@ int main(int argc, char **argv_orig, char **envp) {
 
   }
 
-  write_bitmap(afl);
-  save_auto(afl);
-
 stop_fuzzing:
 
   afl->force_ui_update = 1;  // ensure the screen is reprinted
   show_stats(afl);           // print the screen one last time
+  write_bitmap(afl);
+  save_auto(afl);
 
   SAYF(CURSOR_SHOW cLRD "\n\n+++ Testing aborted %s +++\n" cRST,
        afl->stop_soon == 2 ? "programmatically" : "by user");
@@ -2270,6 +2269,20 @@ stop_fuzzing:
 
   }
 
+  if (afl->not_on_tty) {
+
+    u32 t_bytes = count_non_255_bytes(afl, afl->virgin_bits);
+    u8  time_tmp[64];
+    u_stringify_time_diff(time_tmp, get_cur_time(), afl->start_time);
+    ACTF(
+        "Statistics: %u new paths found, %.02f%% coverage achieved, %llu "
+        "crashes found, %llu timeouts found, total runtime %s",
+        afl->queued_discovered,
+        ((double)t_bytes * 100) / afl->fsrv.real_map_size, afl->unique_crashes,
+        afl->unique_hangs, time_tmp);
+
+  }
+
   #ifdef PROFILING
   SAYF(cYEL "[!] " cRST
             "Profiling information: %llu ms total work, %llu ns/run\n",
diff --git a/src/afl-performance.c b/src/afl-performance.c
index c6fa554b..04507410 100644
--- a/src/afl-performance.c
+++ b/src/afl-performance.c
@@ -90,7 +90,8 @@ inline u32 hash32(u8 *key, u32 len, u32 seed) {
 
 #endif
 
-  return (u32)XXH64(key, len, seed);
+  (void)seed;
+  return (u32)XXH3_64bits(key, len);
 
 }
 
@@ -102,7 +103,8 @@ inline u64 hash64(u8 *key, u32 len, u64 seed) {
 
 #endif
 
-  return XXH64(key, len, seed);
+  (void)seed;
+  return XXH3_64bits(key, len);
 
 }