Misc (#986)

* Changes to fix accidental ranges deletion and add support for SCAS/CMPS

* Fix syscall issues on OSX

* Changes to more closely match QEMU mode

* Changes to use double hashing on cmplog

* Changes to use msync

* Review changes

Co-authored-by: Your Name <you@example.com>

4 files changed, 64 insertions, 56 deletions

diff --git a/frida_mode/GNUmakefile b/frida_mode/GNUmakefile
index c736006a..2f637412 100644
--- a/frida_mode/GNUmakefile
+++ b/frida_mode/GNUmakefile
@@ -59,6 +59,7 @@ else
 ifdef DEBUG
  RT_CFLAGS:=$(RT_CFLAGS) -Wno-prio-ctor-dtor
 endif
+LDFLAGS+=-z noexecstack
 endif
 
 ifeq "$(shell uname)" "Linux"
diff --git a/frida_mode/src/cmplog/cmplog.c b/frida_mode/src/cmplog/cmplog.c
index c65b98d0..8814f7f3 100644
--- a/frida_mode/src/cmplog/cmplog.c
+++ b/frida_mode/src/cmplog/cmplog.c
@@ -1,7 +1,9 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <limits.h>
-#include <syscall.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <unistd.h>
 
 #include "frida-gum.h"
 
@@ -13,12 +15,13 @@
 #define MAX_MEMFD_SIZE (64UL << 10)
 
 extern struct cmp_map *__afl_cmp_map;
-static GArray *cmplog_ranges = NULL;
-static GHashTable * hash = NULL;
+static GArray *        cmplog_ranges = NULL;
+static GHashTable *    hash_yes = NULL;
+static GHashTable *    hash_no = NULL;
 
-static int     memfd = -1;
-static size_t  memfd_size = 0;
-static u8 scratch[MAX_MEMFD_SIZE] = {0};
+static long page_size = 0;
+static long page_offset_mask = 0;
+static long page_mask = 0;
 
 static gboolean cmplog_range(const GumRangeDetails *details,
                              gpointer               user_data) {
@@ -41,19 +44,10 @@ static void cmplog_get_ranges(void) {
 
   OKF("CMPLOG - Collecting ranges");
 
-  cmplog_ranges =
-      g_array_sized_new(false, false, sizeof(GumMemoryRange), 100);
+  cmplog_ranges = g_array_sized_new(false, false, sizeof(GumMemoryRange), 100);
   gum_process_enumerate_ranges(GUM_PAGE_READ, cmplog_range, cmplog_ranges);
   g_array_sort(cmplog_ranges, cmplog_sort);
 
-  for (guint i = 0; i < cmplog_ranges->len; i++) {
-
-    GumMemoryRange *range = &g_array_index(cmplog_ranges, GumMemoryRange, i);
-
-  }
-
-  g_array_free(cmplog_ranges, TRUE);
-
 }
 
 void cmplog_init(void) {
@@ -71,16 +65,22 @@ void cmplog_init(void) {
 
   }
 
-  memfd = syscall(__NR_memfd_create, "cmplog_memfd", 0);
-  if (memfd < 0) {
+  page_size = sysconf(_SC_PAGE_SIZE);
+  page_offset_mask = page_size - 1;
+  page_mask = ~(page_offset_mask);
+
+  hash_yes = g_hash_table_new(g_direct_hash, g_direct_equal);
+  if (hash_yes == NULL) {
 
-    FATAL("Failed to create_memfd, errno: %d", errno);
+    FATAL("Failed to g_hash_table_new, errno: %d", errno);
 
   }
 
-  hash = g_hash_table_new (g_direct_hash, g_direct_equal);
-  if (hash == NULL) {
+  hash_no = g_hash_table_new(g_direct_hash, g_direct_equal);
+  if (hash_no == NULL) {
+
     FATAL("Failed to g_hash_table_new, errno: %d", errno);
+
   }
 
 }
@@ -94,38 +94,41 @@ static gboolean cmplog_contains(GumAddress inner_base, GumAddress inner_limit,
 
 gboolean cmplog_test_addr(guint64 addr, size_t size) {
 
-  if (g_hash_table_contains(hash, (gpointer)addr)) { return true; }
+  if (g_hash_table_contains(hash_yes, (gpointer)addr)) { return true; }
+  if (g_hash_table_contains(hash_no, (gpointer)addr)) { return false; }
 
-  if (memfd_size > MAX_MEMFD_SIZE) {
-    if (lseek(memfd, 0, SEEK_SET) < 0) {
-      FATAL("CMPLOG - Failed lseek, errno: %d", errno);
-    }
-  }
+  void * page_addr = (void *)(addr & page_mask);
+  size_t page_offset = addr & page_offset_mask;
+
+  /* If it spans a page, then bail */
+  if (page_size - page_offset < size) { return false; }
 
   /*
-   * Our address map can change (e.g. stack growth), use write as a fallback to
+   * Our address map can change (e.g. stack growth), use msync as a fallback to
    * validate our address.
    */
-  ssize_t written = syscall(SYS_write, memfd, (void *)addr, size);
-  if (written < 0 && errno != EFAULT && errno != 0) {
-    FATAL("CMPLOG - Failed __NR_write, errno: %d", errno);
-  }
-  /*
-   * If the write succeeds, then the buffer must be valid otherwise it would
-   * return EFAULT
-   */
-  if (written > 0) { memfd_size += written; }
+  if (msync(page_addr, page_offset + size, MS_ASYNC) < 0) {
+
+    if (!g_hash_table_add(hash_no, (gpointer)addr)) {
 
-  if ((size_t)written == size) {
-    if (!g_hash_table_add (hash, (gpointer)addr)) {
       FATAL("Failed - g_hash_table_add");
+
     }
-    return true;
-  }
 
+    return false;
 
+  } else {
+
+    if (!g_hash_table_add(hash_yes, (gpointer)addr)) {
+
+      FATAL("Failed - g_hash_table_add");
+
+    }
+
+    return true;
+
+  }
 
-  return false;
 }
 
 gboolean cmplog_is_readable(guint64 addr, size_t size) {
@@ -152,8 +155,8 @@ gboolean cmplog_is_readable(guint64 addr, size_t size) {
 
     GumMemoryRange *range = &g_array_index(cmplog_ranges, GumMemoryRange, i);
 
-    GumAddress      outer_base = range->base_address;
-    GumAddress      outer_limit = outer_base + range->size;
+    GumAddress outer_base = range->base_address;
+    GumAddress outer_limit = outer_base + range->size;
 
     if (cmplog_contains(inner_base, inner_limit, outer_base, outer_limit))
       return true;
diff --git a/frida_mode/src/cmplog/cmplog_x64.c b/frida_mode/src/cmplog/cmplog_x64.c
index 9f56c32a..ba16445d 100644
--- a/frida_mode/src/cmplog/cmplog_x64.c
+++ b/frida_mode/src/cmplog/cmplog_x64.c
@@ -177,7 +177,7 @@ static void cmplog_handle_cmp_sub(GumCpuContext *context, gsize operand1,
   register uintptr_t k = (uintptr_t)address;
 
   k = (k >> 4) ^ (k << 8);
-  k &= CMP_MAP_W - 1;
+  k &= CMP_MAP_W - 7;
 
   __afl_cmp_map->headers[k].type = CMP_TYPE_INS;
 
@@ -198,8 +198,6 @@ static void cmplog_cmp_sub_callout(GumCpuContext *context, gpointer user_data) {
   gsize              operand1;
   gsize              operand2;
 
-  if (ctx->operand1.size != ctx->operand2.size) FATAL("Operand size mismatch");
-
   if (!cmplog_get_operand_value(context, &ctx->operand1, &operand1)) { return; }
   if (!cmplog_get_operand_value(context, &ctx->operand2, &operand2)) { return; }
 
@@ -233,6 +231,15 @@ static void cmplog_instrument_cmp_sub(const cs_insn *     instr,
 
     case X86_INS_CMP:
     case X86_INS_SUB:
+    case X86_INS_SCASB:
+    case X86_INS_SCASD:
+    case X86_INS_SCASQ:
+    case X86_INS_SCASW:
+    case X86_INS_CMPSB:
+    case X86_INS_CMPSD:
+    case X86_INS_CMPSQ:
+    case X86_INS_CMPSS:
+    case X86_INS_CMPSW:
       break;
     default:
       return;
@@ -247,13 +254,8 @@ static void cmplog_instrument_cmp_sub(const cs_insn *     instr,
   if (operand1->type == X86_OP_INVALID) return;
   if (operand2->type == X86_OP_INVALID) return;
 
-  if ((operand1->type == X86_OP_MEM) &&
-      (operand1->mem.segment != X86_REG_INVALID))
-    return;
-
-  if ((operand2->type == X86_OP_MEM) &&
-      (operand2->mem.segment != X86_REG_INVALID))
-    return;
+  /* Both operands are the same size */
+  if (operand1->size == 1) { return; }
 
   cmplog_instrument_cmp_sub_put_callout(iterator, operand1, operand2);
 
diff --git a/frida_mode/src/main.c b/frida_mode/src/main.c
index 7ff23755..b17d9f49 100644
--- a/frida_mode/src/main.c
+++ b/frida_mode/src/main.c
@@ -101,7 +101,8 @@ static void afl_print_cmdline(void) {
 
   if (fd < 0) {
 
-    FATAL("Failed to open /proc/self/cmdline, errno: (%d)", errno);
+    WARNF("Failed to open /proc/self/cmdline, errno: (%d)", errno);
+    return;
 
   }
 
@@ -138,7 +139,8 @@ static void afl_print_env(void) {
 
   if (fd < 0) {
 
-    FATAL("Failed to open /proc/self/cmdline, errno: (%d)", errno);
+    WARNF("Failed to open /proc/self/cmdline, errno: (%d)", errno);
+    return;
 
   }