simplify SHAPE_BYTES(h->shape)

1 files changed, 51 insertions, 46 deletions

diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c
index 268f726c..2d610132 100644
--- a/src/afl-fuzz-redqueen.c
+++ b/src/afl-fuzz-redqueen.c
@@ -64,6 +64,8 @@ struct range {
 
 };
 
+static u32 hshape;
+
 static struct range *add_range(struct range *ranges, u32 start, u32 end) {
 
   struct range *r = ck_alloc_nozero(sizeof(struct range));
@@ -763,7 +765,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
   //         "Encode: %llx->%llx into %llx(<-%llx) at idx=%u "
   //         "taint_len=%u shape=%u attr=%u\n",
   //         o_pattern, pattern, repl, changed_val, idx, taint_len,
-  //         h->shape + 1, attr);
+  //         hshape, attr);
 
   //#ifdef CMPLOG_SOLVE_TRANSFORM
   // reverse atoi()/strnu?toll() is expensive, so we only to it in lvl 3
@@ -845,7 +847,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
       u64 b_val, o_b_val, mask;
       u8  bytes;
 
-      switch (SHAPE_BYTES(h->shape)) {
+      switch (hshape) {
 
         case 0:
         case 1:
@@ -924,7 +926,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
       s64 diff = pattern - b_val;
       s64 o_diff = o_pattern - o_b_val;
       /* fprintf(stderr, "DIFF1 idx=%03u shape=%02u %llx-%llx=%lx\n", idx,
-                 h->shape + 1, o_pattern, o_b_val, o_diff);
+                 hshape, o_pattern, o_b_val, o_diff);
          fprintf(stderr, "DIFF1 %016llx %llx-%llx=%lx\n", repl, pattern,
                  b_val, diff); */
       if (diff == o_diff && diff) {
@@ -953,7 +955,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
         s64 o_diff = o_pattern ^ o_b_val;
 
         /* fprintf(stderr, "DIFF2 idx=%03u shape=%02u %llx-%llx=%lx\n",
-                   idx, h->shape + 1, o_pattern, o_b_val, o_diff);
+                   idx, hshape, o_pattern, o_b_val, o_diff);
            fprintf(stderr,
                    "DIFF2 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff);
         */
@@ -1002,7 +1004,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
         }
 
         /* fprintf(stderr, "DIFF3 idx=%03u shape=%02u %llx-%llx=%lx\n",
-                   idx, h->shape + 1, o_pattern, o_b_val, o_diff);
+                   idx, hshape, o_pattern, o_b_val, o_diff);
            fprintf(stderr,
                    "DIFF3 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff);
         */
@@ -1051,7 +1053,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
         }
 
         /* fprintf(stderr, "DIFF4 idx=%03u shape=%02u %llx-%llx=%lx\n",
-                   idx, h->shape + 1, o_pattern, o_b_val, o_diff);
+                   idx, hshape, o_pattern, o_b_val, o_diff);
            fprintf(stderr,
                    "DIFF4 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff);
         */
@@ -1089,7 +1091,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
   if ((lvl & LVL1) || attr >= IS_FP_MOD) {
 
-    if (SHAPE_BYTES(h->shape) >= 8 && *status != 1) {
+    if (hshape >= 8 && *status != 1) {
 
       // if (its_len >= 8)
       //   fprintf(stderr,
@@ -1132,7 +1134,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
     }
 
-    if (SHAPE_BYTES(h->shape) >= 4 && *status != 1) {
+    if (hshape >= 4 && *status != 1) {
 
       // if (its_len >= 4 && (attr <= 1 || attr >= 8))
       //   fprintf(stderr,
@@ -1173,7 +1175,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
     }
 
-    if (SHAPE_BYTES(h->shape) >= 2 && *status != 1) {
+    if (hshape >= 2 && *status != 1) {
 
       if (its_len >= 2 &&
           ((*buf_16 == (u16)pattern && *o_buf_16 == (u16)o_pattern) ||
@@ -1244,7 +1246,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
   }
 
-  if (!(attr & (IS_GREATER | IS_LESSER)) || SHAPE_BYTES(h->shape) < 4) {
+  if (!(attr & (IS_GREATER | IS_LESSER)) || hshape < 4) {
 
     return 0;
 
@@ -1272,7 +1274,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
     if (attr & IS_GREATER) {
 
-      if (SHAPE_BYTES(h->shape) == 4 && its_len >= 4) {
+      if (hshape == 4 && its_len >= 4) {
 
         float *f = (float *)&repl;
         float  g = *f;
@@ -1280,7 +1282,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
         u32 *r = (u32 *)&g;
         repl_new = (u32)*r;
 
-      } else if (SHAPE_BYTES(h->shape) == 8 && its_len >= 8) {
+      } else if (hshape == 8 && its_len >= 8) {
 
         double *f = (double *)&repl;
         double  g = *f;
@@ -1307,7 +1309,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
 
     } else {
 
-      if (SHAPE_BYTES(h->shape) == 4) {
+      if (hshape == 4) {
 
         float *f = (float *)&repl;
         float  g = *f;
@@ -1315,7 +1317,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
         u32 *r = (u32 *)&g;
         repl_new = (u32)*r;
 
-      } else if (SHAPE_BYTES(h->shape) == 8) {
+      } else if (hshape == 8) {
 
         double *f = (double *)&repl;
         double  g = *f;
@@ -1342,7 +1344,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
     }
 
     // transform double to float, llvm likes to do that internally ...
-    if (SHAPE_BYTES(h->shape) == 8 && its_len >= 4) {
+    if (hshape == 8 && its_len >= 4) {
 
       double *f = (double *)&repl;
       float   g = (float)*f;
@@ -1353,7 +1355,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
       memcpy(((char *)&repl_new) + 4, (char *)&g, 4);
 #endif
       changed_val = repl_new;
-      h->shape = 3;  // modify shape
+      hshape = 4;  // modify shape
 
       // fprintf(stderr, "DOUBLE2FLOAT %llx\n", repl_new);
 
@@ -1361,12 +1363,12 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h,
               afl, h, pattern, repl_new, o_pattern, changed_val, 16, idx,
               taint_len, orig_buf, buf, cbuf, len, 1, lvl, status))) {
 
-        h->shape = 7;  // recover shape
+        hshape = 8;  // recover shape
         return 1;
 
       }
 
-      h->shape = 7;  // recover shape
+      hshape = 7;  // recover shape
 
     }
 
@@ -1428,14 +1430,13 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h,
   u8 *r = (u8 *)&repl;
   u8  backup[16];
   u32 its_len = MIN(len - idx, taint_len);
-  u32 shape = h->shape + 1;
   #if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
   size_t off = 0;
   #else
-  size_t off = 16 - shape;
+  size_t off = 16 - hshape;
   #endif
 
-  if (its_len >= shape) {
+  if (its_len >= hshape) {
 
   #ifdef _DEBUG
     fprintf(stderr, "TestUN: %u>=%u (len=%u idx=%u attr=%u off=%lu) (%u) ",
@@ -1462,18 +1463,18 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h,
     fprintf(stderr, "\n");
   #endif
 
-    if (!memcmp(ptr, p + off, shape) && !memcmp(o_ptr, o_p + off, shape)) {
+    if (!memcmp(ptr, p + off, hshape) && !memcmp(o_ptr, o_p + off, hshape)) {
 
-      memcpy(backup, ptr, shape);
-      memcpy(ptr, r + off, shape);
+      memcpy(backup, ptr, hshape);
+      memcpy(ptr, r + off, hshape);
 
       if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; }
 
   #ifdef CMPLOG_COMBINE
-      if (*status == 1) { memcpy(cbuf + idx, r, shape); }
+      if (*status == 1) { memcpy(cbuf + idx, r, hshape); }
   #endif
 
-      memcpy(ptr, backup, shape);
+      memcpy(ptr, backup, hshape);
 
   #ifdef _DEBUG
       fprintf(stderr, "Status=%u\n", *status);
@@ -1485,8 +1486,8 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h,
     if (do_reverse && *status != 1) {
 
       if (unlikely(cmp_extend_encodingN(
-              afl, h, SWAPN(pattern, (shape << 3)), SWAPN(repl, (shape << 3)),
-              SWAPN(o_pattern, (shape << 3)), SWAPN(changed_val, (shape << 3)),
+              afl, h, SWAPN(pattern, (hshape << 3)), SWAPN(repl, (hshape << 3)),
+              SWAPN(o_pattern, (hshape << 3)), SWAPN(changed_val, (hshape << 3)),
               attr, idx, taint_len, orig_buf, buf, cbuf, len, 0, lvl,
               status))) {
 
@@ -1615,6 +1616,8 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
   u8  s_v0_inc = 1, s_v1_inc = 1;
   u8  s_v0_dec = 1, s_v1_dec = 1;
 
+  hshape = SHAPE_BYTES(h->shape);
+
   if (h->hits > CMP_MAP_H) {
 
     loggeds = CMP_MAP_H;
@@ -1626,7 +1629,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
   }
 
 #ifdef WORD_SIZE_64
-  switch (SHAPE_BYTES(h->shape)) {
+  switch (hshape) {
 
     case 1:
     case 2:
@@ -1680,7 +1683,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
 #ifdef _DEBUG
     fprintf(stderr, "Handling: %llx->%llx vs %llx->%llx attr=%u shape=%u\n",
             orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute,
-            SHAPE_BYTES(h->shape));
+            hshape);
 #endif
 
     t = taint;
@@ -1830,7 +1833,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
             "END: %llx->%llx vs %llx->%llx attr=%u i=%u found=%u "
             "isN=%u size=%u\n",
             orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute, i, found_one,
-            is_n, SHAPE_BYTES(h->shape));
+            is_n, hshape);
 #endif
 
     // If failed, add to dictionary
@@ -1841,16 +1844,16 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
 #ifdef WORD_SIZE_64
         if (unlikely(is_n)) {
 
-          try_to_add_to_dictN(afl, s128_v0, SHAPE_BYTES(h->shape));
-          try_to_add_to_dictN(afl, s128_v1, SHAPE_BYTES(h->shape));
+          try_to_add_to_dictN(afl, s128_v0, hshape);
+          try_to_add_to_dictN(afl, s128_v1, hshape);
 
         } else
 
 #endif
         {
 
-          try_to_add_to_dict(afl, o->v0, SHAPE_BYTES(h->shape));
-          try_to_add_to_dict(afl, o->v1, SHAPE_BYTES(h->shape));
+          try_to_add_to_dict(afl, o->v0, hshape);
+          try_to_add_to_dict(afl, o->v1, hshape);
 
         }
 
@@ -2322,6 +2325,8 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
   u32                i, j, idx, have_taint = 1, taint_len, loggeds;
   u8                 status = 0, found_one = 0;
 
+  hshape = SHAPE_BYTES(h->shape);
+
   if (h->hits > CMP_MAP_RTN_H) {
 
     loggeds = CMP_MAP_RTN_H;
@@ -2355,12 +2360,12 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
     /*
       struct cmp_header *hh = &afl->orig_cmp_map->headers[key];
       fprintf(stderr, "RTN N hits=%u id=%u shape=%u attr=%u v0=", h->hits,
-              h->id, h->shape, h->attribute);
+              h->id, hshape, h->attribute);
       for (j = 0; j < 8; j++) fprintf(stderr, "%02x", o->v0[j]);
       fprintf(stderr, " v1=");
       for (j = 0; j < 8; j++) fprintf(stderr, "%02x", o->v1[j]);
       fprintf(stderr, "\nRTN O hits=%u id=%u shape=%u attr=%u o0=",
-              hh->hits, hh->id, hh->shape, hh->attribute);
+              hh->hits, hh->id, hshape, hh->attribute);
       for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_o->v0[j]);
       fprintf(stderr, " o1=");
       for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_o->v1[j]);
@@ -2401,23 +2406,23 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
 #ifdef _DEBUG
       int w;
       fprintf(stderr, "key=%u idx=%u len=%u o0=", key, idx,
-              SHAPE_BYTES(h->shape));
-      for (w = 0; w < SHAPE_BYTES(h->shape); ++w)
+              hshape);
+      for (w = 0; w < hshape; ++w)
         fprintf(stderr, "%02x", orig_o->v0[w]);
       fprintf(stderr, " v0=");
-      for (w = 0; w < SHAPE_BYTES(h->shape); ++w)
+      for (w = 0; w < hshape; ++w)
         fprintf(stderr, "%02x", o->v0[w]);
       fprintf(stderr, " o1=");
-      for (w = 0; w < SHAPE_BYTES(h->shape); ++w)
+      for (w = 0; w < hshape; ++w)
         fprintf(stderr, "%02x", orig_o->v1[w]);
       fprintf(stderr, " v1=");
-      for (w = 0; w < SHAPE_BYTES(h->shape); ++w)
+      for (w = 0; w < hshape; ++w)
         fprintf(stderr, "%02x", o->v1[w]);
       fprintf(stderr, "\n");
 #endif
 
       if (unlikely(rtn_extend_encoding(
-              afl, o->v0, o->v1, orig_o->v0, orig_o->v1, SHAPE_BYTES(h->shape),
+              afl, o->v0, o->v1, orig_o->v0, orig_o->v1, hshape,
               idx, taint_len, orig_buf, buf, cbuf, len, lvl, &status))) {
 
         return 1;
@@ -2434,7 +2439,7 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
       status = 0;
 
       if (unlikely(rtn_extend_encoding(
-              afl, o->v1, o->v0, orig_o->v1, orig_o->v0, SHAPE_BYTES(h->shape),
+              afl, o->v1, o->v0, orig_o->v1, orig_o->v0, hshape,
               idx, taint_len, orig_buf, buf, cbuf, len, lvl, &status))) {
 
         return 1;
@@ -2455,8 +2460,8 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf,
 
       if (unlikely(!afl->pass_stats[key].total)) {
 
-        maybe_add_auto(afl, o->v0, SHAPE_BYTES(h->shape));
-        maybe_add_auto(afl, o->v1, SHAPE_BYTES(h->shape));
+        maybe_add_auto(afl, o->v0, hshape);
+        maybe_add_auto(afl, o->v1, hshape);
 
       }