improve cmplog level 3

5 files changed, 100 insertions, 90 deletions

diff --git a/docs/Changelog.md b/docs/Changelog.md
index d61ce8ec..75167172 100644
--- a/docs/Changelog.md
+++ b/docs/Changelog.md
@@ -15,6 +15,7 @@
       command line tool! See custom_mutators/aflpp/standalone/
     - display the state of the fuzzing run in the UI :-)
     - fix timeout setting if '+' is used or a session is restarted
+    - -c X option to enable base64 transformation solving
   - afl-cmin/afl-cmin.bash:
     - fixed a bug inherited from vanilla AFL where a coverage of
       map[123] = 11 would be the same as map[1123] = 1
diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h
index 27668da0..e114b0fc 100644
--- a/include/afl-fuzz.h
+++ b/include/afl-fuzz.h
@@ -674,7 +674,8 @@ typedef struct afl_state {
   u32 cmplog_max_filesize;
   u32 cmplog_lvl;
   u32 colorize_success;
-  u8  cmplog_enable_arith, cmplog_enable_transform, cmplog_random_colorization;
+  u8  cmplog_enable_arith, cmplog_enable_transform,
+      cmplog_enable_xtreme_transform, cmplog_random_colorization;
 
   struct afl_pass_stat *pass_stats;
   struct cmp_map       *orig_cmp_map;
diff --git a/include/config.h b/include/config.h
index 7c29a674..df545583 100644
--- a/include/config.h
+++ b/include/config.h
@@ -60,10 +60,6 @@
  *
  */
 
-/* if TRANSFORM is enabled with '-l T', this additionally enables base64
-   encoding/decoding */
-// #define CMPLOG_SOLVE_TRANSFORM_BASE64
-
 /* If a redqueen pass finds more than one solution, try to combine them? */
 #define CMPLOG_COMBINE
 
@@ -71,10 +67,10 @@
 #define CMPLOG_CORPUS_PERCENT 5U
 
 /* Number of potential positions from which we decide if cmplog becomes
-   useless, default 8096 */
+   useless, default 12288 */
 #define CMPLOG_POSITIONS_MAX (12 * 1024)
 
-/* Maximum allowed fails per CMP value. Default: 128 */
+/* Maximum allowed fails per CMP value. Default: 96 */
 #define CMPLOG_FAIL_MAX 96
 
 /* -------------------------------------*/
diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c
index 73e188e7..5a1f512d 100644
--- a/src/afl-fuzz-redqueen.c
+++ b/src/afl-fuzz-redqueen.c
@@ -571,7 +571,6 @@ static u8 its_fuzz(afl_state_t *afl, u8 *buf, u32 len, u8 *status) {
 
 }
 
-// #ifdef CMPLOG_SOLVE_TRANSFORM
 static int strntoll(const char *str, size_t sz, char **end, int base,
                     long long *out) {
 
@@ -656,7 +655,6 @@ static int is_hex(const char *str) {
 
 }
 
-#ifdef CMPLOG_SOLVE_TRANSFORM_BASE64
 // tests 4 bytes at location
 static int is_base64(const char *str) {
 
@@ -769,10 +767,6 @@ static void to_base64(u8 *src, u8 *dst, u32 dst_len) {
 
 }
 
-#endif
-
-// #endif
-
 static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
                               u64 pattern, u64 repl, u64 o_pattern,
                               u64 changed_val, u8 attr, u32 idx, u32 taint_len,
@@ -797,42 +791,54 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
   }
 
-  // fprintf(stderr,
-  //         "Encode: %llx->%llx into %llx(<-%llx) at idx=%u "
-  //         "taint_len=%u shape=%u attr=%u\n",
-  //         o_pattern, pattern, repl, changed_val, idx, taint_len,
-  //         hshape, attr);
+  /*
+  fprintf(stderr,
+          "Encode: %llx->%llx into %llx(<-%llx) at idx=%u "
+          "taint_len=%u shape=%u attr=%u\n",
+          o_pattern, pattern, repl, changed_val, idx, taint_len,
+          hshape, attr);
+  */
 
-  // #ifdef CMPLOG_SOLVE_TRANSFORM
   //  reverse atoi()/strnu?toll() is expensive, so we only to it in lvl 3
   if (afl->cmplog_enable_transform && (lvl & LVL3)) {
 
     u8                *endptr;
     u8                 use_num = 0, use_unum = 0;
-    unsigned long long unum;
-    long long          num;
+    unsigned long long unum = 0;
+    long long          num = 0;
+
+    // if (afl->queue_cur->is_ascii) {
+
+    // we first check if our input are ascii numbers that are transformed to
+    // an integer and used for comparison:
 
-    if (afl->queue_cur->is_ascii) {
+    endptr = buf_8;
+    if (strntoll(buf_8, len - idx, (char **)&endptr, 0, &num)) {
 
-      endptr = buf_8;
-      if (strntoll(buf_8, len - idx, (char **)&endptr, 0, &num)) {
+      if (!strntoull(buf_8, len - idx, (char **)&endptr, 0, &unum)) {
 
-        if (!strntoull(buf_8, len - idx, (char **)&endptr, 0, &unum))
-          use_unum = 1;
+        use_unum = 1;
 
-      } else
+      }
+
+    } else {
 
-        use_num = 1;
+      use_num = 1;
 
     }
 
+    //}
+
 #ifdef _DEBUG
     if (idx == 0)
-      fprintf(stderr, "ASCII is=%u use_num=%u use_unum=%u idx=%u %llx==%llx\n",
-              afl->queue_cur->is_ascii, use_num, use_unum, idx, num, pattern);
+      fprintf(stderr,
+              "ASCII is=%u use_num=%u>%lld use_unum=%u>%llu idx=%u "
+              "pattern=0x%llx\n",
+              afl->queue_cur->is_ascii, use_num, num, use_unum, unum, idx,
+              pattern);
 #endif
 
-    // num is likely not pattern as atoi("AAA") will be zero...
+    // atoi("AAA") == 0 so !num means we have to investigate
     if (use_num && ((u64)num == pattern || !num)) {
 
       u8     tmp_buf[32];
@@ -961,10 +967,12 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
       // test for arithmetic, eg. "if ((user_val - 0x1111) == 0x1234) ..."
       s64 diff = pattern - b_val;
       s64 o_diff = o_pattern - o_b_val;
-      /* fprintf(stderr, "DIFF1 idx=%03u shape=%02u %llx-%llx=%lx\n", idx,
-                 hshape, o_pattern, o_b_val, o_diff);
-         fprintf(stderr, "DIFF1 %016llx %llx-%llx=%lx\n", repl, pattern,
-                 b_val, diff); */
+      /*
+             fprintf(stderr, "DIFF1 idx=%03u shape=%02u %llx-%llx=%lx\n", idx,
+                       hshape, o_pattern, o_b_val, o_diff);
+               fprintf(stderr, "DIFF1 %016llx %llx-%llx=%lx\n", repl, pattern,
+                       b_val, diff);
+      */
       if (diff == o_diff && diff) {
 
         // this could be an arithmetic transformation
@@ -1275,7 +1283,6 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
   //       16 = modified float, 32 = modified integer (modified = wont match
   //                                                   in original buffer)
 
-  // #ifdef CMPLOG_SOLVE_ARITHMETIC
   if (!afl->cmplog_enable_arith || lvl < LVL3 || attr == IS_TRANSFORM) {
 
     return 0;
@@ -2009,8 +2016,12 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry,
   its_len = MIN(its_len, taint_len);
   u32 saved_its_len = its_len;
 
+  if (its_len <= 1) { return 0; }
+
   if (lvl & LVL3) {
 
+    if (memcmp(changed_val, repl, its_len) != 0) { return 0; }
+
     u32 max_to = MIN(4U, idx);
     if (!(lvl & LVL1) && max_to) { from = 1; }
     to = max_to;
@@ -2089,9 +2100,7 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry,
   if (afl->cmplog_enable_transform && (lvl & LVL3)) {
 
     u32 toupper = 0, tolower = 0, xor = 0, arith = 0, tohex = 0, fromhex = 0;
-#ifdef CMPLOG_SOLVE_TRANSFORM_BASE64
     u32 tob64 = 0, fromb64 = 0;
-#endif
     u32 from_0 = 0, from_x = 0, from_X = 0, from_slash = 0, from_up = 0;
     u32 to_0 = 0, to_x = 0, to_slash = 0, to_up = 0;
     u8  xor_val[32], arith_val[32], tmp[48];
@@ -2144,7 +2153,8 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry,
 
       }
 
-      if (i < 16 && is_hex(repl + (i << 1))) {
+      if (afl->cmplog_enable_xtreme_transform && i < 16 &&
+          is_hex(repl + (i << 1))) {
 
         ++tohex;
 
@@ -2163,7 +2173,7 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry,
 
       }
 
-      if ((i % 2)) {
+      if (afl->cmplog_enable_xtreme_transform && (i % 2)) {
 
         if (len > idx + i + 1 && is_hex(orig_buf + idx + i)) {
 
@@ -2187,20 +2197,21 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry,
 
       }
 
-#ifdef CMPLOG_SOLVE_TRANSFORM_BASE64
-      if (i % 3 == 2 && i < 24) {
+      if (afl->cmplog_enable_xtreme_transform) {
 
-        if (is_base64(repl + ((i / 3) << 2))) tob64 += 3;
+        if (i % 3 == 2 && i < 24) {
 
-      }
+          if (is_base64(repl + ((i / 3) << 2))) tob64 += 3;
+
+        }
 
-      if (i % 4 == 3 && i < 24) {
+        if (i % 4 == 3 && i < 24) {
 
-        if (is_base64(orig_buf + idx + i - 3)) fromb64 += 4;
+          if (is_base64(orig_buf + idx + i - 3)) fromb64 += 4;
 
-      }
+        }
 
-#endif
+      }
 
       if ((o_pattern[i] ^ orig_buf[idx + i]) == xor_val[i] && xor_val[i]) {
 
@@ -2229,45 +2240,50 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry,
       }
 
 #ifdef _DEBUG
+      fprintf(stderr, "RTN %s %s %s %s\n", buf, pattern, orig_buf, o_pattern);
       fprintf(stderr,
-              "RTN idx=%u loop=%u xor=%u arith=%u tolower=%u toupper=%u "
+              "RTN idx=%u len=%u loop=%u xor=%u arith=%u tolower=%u toupper=%u "
               "tohex=%u fromhex=%u to_0=%u to_slash=%u to_x=%u "
               "from_0=%u from_slash=%u from_x=%u\n",
-              idx, i, xor, arith, tolower, toupper, tohex, fromhex, to_0,
-              to_slash, to_x, from_0, from_slash, from_x);
-  #ifdef CMPLOG_SOLVE_TRANSFORM_BASE64
-      fprintf(stderr, "RTN idx=%u loop=%u tob64=%u from64=%u\n", tob64,
-              fromb64);
-  #endif
+              idx, its_len, i, xor, arith, tolower, toupper, tohex, fromhex,
+              to_0, to_slash, to_x, from_0, from_slash, from_x);
+      if (afl->cmplog_enable_xtreme_transform) {
+
+        fprintf(stderr, "RTN idx=%u loop=%u tob64=%u from64=%u\n", idx, i,
+                tob64, fromb64);
+
+      }
+
 #endif
 
-#ifdef CMPLOG_SOLVE_TRANSFORM_BASE64
-      // input is base64 and converted to binary? convert repl to base64!
-      if ((i % 4) == 3 && i < 24 && fromb64 > i) {
+      if (afl->cmplog_enable_xtreme_transform) {
 
-        to_base64(repl, tmp, i + 1);
-        memcpy(buf + idx, tmp, i + 1);
-        if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; }
-        // fprintf(stderr, "RTN ATTEMPT fromb64 %u result %u\n", fromb64,
-        // *status);
+        // input is base64 and converted to binary? convert repl to base64!
+        if ((i % 4) == 3 && i < 24 && fromb64 > i) {
 
-      }
+          to_base64(repl, tmp, i + 1);
+          memcpy(buf + idx, tmp, i + 1);
+          if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; }
+          // fprintf(stderr, "RTN ATTEMPT fromb64 %u result %u\n", fromb64,
+          // *status);
+
+        }
 
-      // input is converted to base64? decode repl with base64!
-      if ((i % 3) == 2 && i < 24 && tob64 > i) {
+        // input is converted to base64? decode repl with base64!
+        if ((i % 3) == 2 && i < 24 && tob64 > i) {
 
-        u32 olen = from_base64(repl, tmp, i + 1);
-        memcpy(buf + idx, tmp, olen);
-        if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; }
-        // fprintf(stderr, "RTN ATTEMPT tob64 %u idx=%u result %u\n", tob64,
-        // idx, *status);
+          u32 olen = from_base64(repl, tmp, i + 1);
+          memcpy(buf + idx, tmp, olen);
+          if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; }
+          // fprintf(stderr, "RTN ATTEMPT tob64 %u idx=%u result %u\n", tob64,
+          // idx, *status);
 
-      }
+        }
 
-#endif
+      }
 
       // input is converted to hex? convert repl to binary!
-      if (i < 16 && tohex > i) {
+      if (afl->cmplog_enable_xtreme_transform && i < 16 && tohex > i) {
 
         u32 off;
         if (to_slash + to_x + to_0 == 2) {
@@ -2292,8 +2308,8 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry,
       }
 
       // input is hex and converted to binary? convert repl to hex!
-      if (i && (i % 2) && i < 16 && fromhex &&
-          fromhex + from_slash + from_x + from_0 > i) {
+      if (afl->cmplog_enable_xtreme_transform && i && (i % 2) && i < 16 &&
+          fromhex && fromhex + from_slash + from_x + from_0 > i) {
 
         u8 off = 0;
         if (from_slash && from_x) {
@@ -2401,11 +2417,9 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry,
 
       if ((i >= 7 &&
            (i >= xor&&i >= arith &&i >= tolower &&i >= toupper &&i > tohex &&i >
-                (fromhex + from_0 + from_x + from_slash + 1)
-#ifdef CMPLOG_SOLVE_TRANSFORM_BASE64
-            && i > tob64 + 3 && i > fromb64 + 4
-#endif
-            )) ||
+                (fromhex + from_0 + from_x + from_slash + 1) &&
+            (afl->cmplog_enable_xtreme_transform && i > tob64 + 3 &&
+             i > fromb64 + 4))) ||
           repl[i] != changed_val[i] || *status == 1) {
 
         break;
@@ -2418,8 +2432,6 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry,
 
   }
 
-  // #endif
-
   return 0;
 
 }
@@ -2818,12 +2830,7 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) {
 
       }
 
-    } else if ((lvl & LVL1)
-
-               // #ifdef CMPLOG_SOLVE_TRANSFORM
-               || ((lvl & LVL3) && afl->cmplog_enable_transform)
-               // #endif
-    ) {
+    } else if ((lvl & LVL1) || ((lvl & LVL3) && afl->cmplog_enable_transform)) {
 
       if (unlikely(rtn_fuzz(afl, k, orig_buf, buf, cbuf, len, lvl, taint))) {
 
diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c
index d8a88f00..21a8915c 100644
--- a/src/afl-fuzz.c
+++ b/src/afl-fuzz.c
@@ -185,7 +185,8 @@ static void usage(u8 *argv0, int more_help) {
       "                  1=small files, 2=larger files (default), 3=all "
       "files,\n"
       "                  A=arithmetic solving, T=transformational solving,\n"
-      "                  R=random colorization bytes.\n\n"
+      "                  X=extreme transform solving, R=random colorization "
+      "bytes.\n\n"
       "Fuzzing behavior settings:\n"
       "  -Z            - sequential queue selection instead of weighted "
       "random\n"
@@ -1120,6 +1121,10 @@ int main(int argc, char **argv_orig, char **envp) {
             case 'T':
               afl->cmplog_enable_transform = 1;
               break;
+            case 'x':
+            case 'X':
+              afl->cmplog_enable_xtreme_transform = 1;
+              break;
             case 'r':
             case 'R':
               afl->cmplog_random_colorization = 1;