Merge pull request #1141 from AFLplusplus/afl4

cmplog enhancement variant

3 files changed, 117 insertions, 35 deletions

diff --git a/frida_mode/src/cmplog/cmplog_arm64.c b/frida_mode/src/cmplog/cmplog_arm64.c
index dd97f38d..ccc8e89e 100644
--- a/frida_mode/src/cmplog/cmplog_arm64.c
+++ b/frida_mode/src/cmplog/cmplog_arm64.c
@@ -104,9 +104,9 @@ static void cmplog_call_callout(GumCpuContext *context, gpointer user_data) {
   gsize x0 = ctx_read_reg(context, ARM64_REG_X0);
   gsize x1 = ctx_read_reg(context, ARM64_REG_X1);
 
-  if (((G_MAXULONG - x0) < 32) || ((G_MAXULONG - x1) < 32)) return;
+  if (((G_MAXULONG - x0) < 31) || ((G_MAXULONG - x1) < 31)) return;
 
-  if (!cmplog_is_readable(x0, 32) || !cmplog_is_readable(x1, 32)) return;
+  if (!cmplog_is_readable(x0, 31) || !cmplog_is_readable(x1, 31)) return;
 
   void *ptr1 = GSIZE_TO_POINTER(x0);
   void *ptr2 = GSIZE_TO_POINTER(x1);
@@ -116,18 +116,36 @@ static void cmplog_call_callout(GumCpuContext *context, gpointer user_data) {
   k = (k >> 4) ^ (k << 8);
   k &= CMP_MAP_W - 1;
 
-  __afl_cmp_map->headers[k].type = CMP_TYPE_RTN;
+  if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) {
+
+    __afl_cmp_map->headers[k].type = CMP_TYPE_RTN;
+    __afl_cmp_map->headers[k].hits = 0;
+
+  }
+
+  u32 hits = 0;
+
+  if (__afl_cmp_map->headers[k].hits == 0) {
+
+    __afl_cmp_map->headers[k].shape = 30;
+
+  } else {
+
+    hits = __afl_cmp_map->headers[k].hits;
+
+  }
 
-  u32 hits = __afl_cmp_map->headers[k].hits;
   __afl_cmp_map->headers[k].hits = hits + 1;
 
-  __afl_cmp_map->headers[k].shape = 31;
+  __afl_cmp_map->headers[k].shape = 30;
 
   hits &= CMP_MAP_RTN_H - 1;
+  ((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0_len = 31;
+  ((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1_len = 31;
   gum_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0, ptr1,
-             32);
+             31);
   gum_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1, ptr2,
-             32);
+             31);
 
 }
 
@@ -193,12 +211,23 @@ static void cmplog_handle_cmp_sub(GumCpuContext *context, gsize operand1,
   k = (k >> 4) ^ (k << 8);
   k &= CMP_MAP_W - 1;
 
-  __afl_cmp_map->headers[k].type = CMP_TYPE_INS;
+  if (__afl_cmp_map->headers[k].type != CMP_TYPE_INS)
+    __afl_cmp_map->headers[k].hits = 0;
 
-  u32 hits = __afl_cmp_map->headers[k].hits;
-  __afl_cmp_map->headers[k].hits = hits + 1;
+  u32 hits = 0;
+
+  if (__afl_cmp_map->headers[k].hits == 0) {
+
+    __afl_cmp_map->headers[k].type = CMP_TYPE_INS;
+    __afl_cmp_map->headers[k].shape = (size - 1);
+
+  } else {
 
-  __afl_cmp_map->headers[k].shape = (size - 1);
+    hits = __afl_cmp_map->headers[k].hits;
+
+  }
+
+  __afl_cmp_map->headers[k].hits = hits + 1;
 
   hits &= CMP_MAP_H - 1;
   __afl_cmp_map->log[k][hits].v0 = operand1;
diff --git a/frida_mode/src/cmplog/cmplog_x64.c b/frida_mode/src/cmplog/cmplog_x64.c
index 0d18767a..5319f727 100644
--- a/frida_mode/src/cmplog/cmplog_x64.c
+++ b/frida_mode/src/cmplog/cmplog_x64.c
@@ -99,9 +99,9 @@ static void cmplog_call_callout(GumCpuContext *context, gpointer user_data) {
   gsize rdi = ctx_read_reg(context, X86_REG_RDI);
   gsize rsi = ctx_read_reg(context, X86_REG_RSI);
 
-  if (((G_MAXULONG - rdi) < 32) || ((G_MAXULONG - rsi) < 32)) return;
+  if (((G_MAXULONG - rdi) < 31) || ((G_MAXULONG - rsi) < 31)) return;
 
-  if (!cmplog_is_readable(rdi, 32) || !cmplog_is_readable(rsi, 32)) return;
+  if (!cmplog_is_readable(rdi, 31) || !cmplog_is_readable(rsi, 31)) return;
 
   void *ptr1 = GSIZE_TO_POINTER(rdi);
   void *ptr2 = GSIZE_TO_POINTER(rsi);
@@ -111,18 +111,34 @@ static void cmplog_call_callout(GumCpuContext *context, gpointer user_data) {
   k = (k >> 4) ^ (k << 8);
   k &= CMP_MAP_W - 1;
 
-  __afl_cmp_map->headers[k].type = CMP_TYPE_RTN;
+  if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) {
 
-  u32 hits = __afl_cmp_map->headers[k].hits;
-  __afl_cmp_map->headers[k].hits = hits + 1;
+    __afl_cmp_map->headers[k].type = CMP_TYPE_RTN;
+    __afl_cmp_map->headers[k].hits = 0;
+
+  }
+
+  u32 hits = 0;
+
+  if (__afl_cmp_map->headers[k].hits == 0) {
+
+    __afl_cmp_map->headers[k].shape = 30;
+
+  } else {
+
+    hits = __afl_cmp_map->headers[k].hits;
+
+  }
 
-  __afl_cmp_map->headers[k].shape = 31;
+  __afl_cmp_map->headers[k].hits = hits + 1;
 
   hits &= CMP_MAP_RTN_H - 1;
+  ((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0_len = 31;
+  ((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1_len = 31;
   gum_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0, ptr1,
-             32);
+             31);
   gum_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1, ptr2,
-             32);
+             31);
 
 }
 
@@ -179,13 +195,23 @@ static void cmplog_handle_cmp_sub(GumCpuContext *context, gsize operand1,
   k = (k >> 4) ^ (k << 8);
   k &= CMP_MAP_W - 7;
 
-  __afl_cmp_map->headers[k].type = CMP_TYPE_INS;
+  if (__afl_cmp_map->headers[k].type != CMP_TYPE_INS)
+    __afl_cmp_map->headers[k].hits = 0;
 
-  u32 hits = __afl_cmp_map->headers[k].hits;
-  __afl_cmp_map->headers[k].hits = hits + 1;
+  u32 hits = 0;
+
+  if (__afl_cmp_map->headers[k].hits == 0) {
+
+    __afl_cmp_map->headers[k].type = CMP_TYPE_INS;
+    __afl_cmp_map->headers[k].shape = (size - 1);
 
-  __afl_cmp_map->headers[k].shape = (size - 1);
+  } else {
 
+    hits = __afl_cmp_map->headers[k].hits;
+
+  }
+
+  __afl_cmp_map->headers[k].hits = hits + 1;
   hits &= CMP_MAP_H - 1;
   __afl_cmp_map->log[k][hits].v0 = operand1;
   __afl_cmp_map->log[k][hits].v1 = operand2;
diff --git a/frida_mode/src/cmplog/cmplog_x86.c b/frida_mode/src/cmplog/cmplog_x86.c
index dd666c34..27d06720 100644
--- a/frida_mode/src/cmplog/cmplog_x86.c
+++ b/frida_mode/src/cmplog/cmplog_x86.c
@@ -104,9 +104,9 @@ static void cmplog_call_callout(GumCpuContext *context, gpointer user_data) {
   gsize arg1 = esp[0];
   gsize arg2 = esp[1];
 
-  if (((G_MAXULONG - arg1) < 32) || ((G_MAXULONG - arg2) < 32)) return;
+  if (((G_MAXULONG - arg1) < 31) || ((G_MAXULONG - arg2) < 31)) return;
 
-  if (!cmplog_is_readable(arg1, 32) || !cmplog_is_readable(arg2, 32)) return;
+  if (!cmplog_is_readable(arg1, 31) || !cmplog_is_readable(arg2, 31)) return;
 
   void *ptr1 = GSIZE_TO_POINTER(arg1);
   void *ptr2 = GSIZE_TO_POINTER(arg2);
@@ -116,18 +116,34 @@ static void cmplog_call_callout(GumCpuContext *context, gpointer user_data) {
   k = (k >> 4) ^ (k << 8);
   k &= CMP_MAP_W - 1;
 
-  __afl_cmp_map->headers[k].type = CMP_TYPE_RTN;
+  if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) {
 
-  u32 hits = __afl_cmp_map->headers[k].hits;
-  __afl_cmp_map->headers[k].hits = hits + 1;
+    __afl_cmp_map->headers[k].type = CMP_TYPE_RTN;
+    __afl_cmp_map->headers[k].hits = 0;
+
+  }
+
+  u32 hits = 0;
+
+  if (__afl_cmp_map->headers[k].hits == 0) {
 
-  __afl_cmp_map->headers[k].shape = 31;
+    __afl_cmp_map->headers[k].shape = 30;
+
+  } else {
+
+    hits = __afl_cmp_map->headers[k].hits;
+
+  }
+
+  __afl_cmp_map->headers[k].hits = hits + 1;
 
   hits &= CMP_MAP_RTN_H - 1;
+  ((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0_len = 31;
+  ((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1_len = 31;
   gum_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0, ptr1,
-             32);
+             31);
   gum_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1, ptr2,
-             32);
+             31);
 
 }
 
@@ -184,12 +200,23 @@ static void cmplog_handle_cmp_sub(GumCpuContext *context, gsize operand1,
   k = (k >> 4) ^ (k << 8);
   k &= CMP_MAP_W - 1;
 
-  __afl_cmp_map->headers[k].type = CMP_TYPE_INS;
+  if (__afl_cmp_map->headers[k].type != CMP_TYPE_INS)
+    __afl_cmp_map->headers[k].hits = 0;
 
-  u32 hits = __afl_cmp_map->headers[k].hits;
-  __afl_cmp_map->headers[k].hits = hits + 1;
+  u32 hits = 0;
+
+  if (__afl_cmp_map->headers[k].hits == 0) {
+
+    __afl_cmp_map->headers[k].type = CMP_TYPE_INS;
+    __afl_cmp_map->headers[k].shape = (size - 1);
+
+  } else {
 
-  __afl_cmp_map->headers[k].shape = (size - 1);
+    hits = __afl_cmp_map->headers[k].hits;
+
+  }
+
+  __afl_cmp_map->headers[k].hits = hits + 1;
 
   hits &= CMP_MAP_H - 1;
   __afl_cmp_map->log[k][hits].v0 = operand1;