autodisable and better performance

5 files changed, 179 insertions, 90 deletions

diff --git a/custom_mutators/autotokens/Makefile b/custom_mutators/autotokens/Makefile
index ab1da4b6..6ee7d324 100644
--- a/custom_mutators/autotokens/Makefile
+++ b/custom_mutators/autotokens/Makefile
@@ -1,16 +1,22 @@
 ifdef debug
-	CFLAGS += -fsanitize=address -Wall
+	CPPLAGS += -fsanitize=address
+	CXXFLAGS += -Wall
+	CC := clang
 	CXX := clang++
 endif
 ifdef DEBUG
-	CFLAGS += -fsanitize=address -Wall
+	CPPFLAGS += -fsanitize=address
+	CXXFLAGS += -Wall
+	CC := clang
 	CXX := clang++
 endif
 
 all:	autotokens.so
 
 autotokens.so:	autotokens.cpp
-	$(CXX) -g -O3 $(CFLAGS) -shared -fPIC -o autotokens.so -I../../include autotokens.cpp ../../src/afl-performance.o
+	$(CC) -D_STANDALONE_MODULE=1 -I../../include -g -O3 $(CPPFLAGS) -fPIC -c -o ./afl-fuzz-queue.o ../../src/afl-fuzz-queue.c
+	$(CC) -I../../include -g -O3 $(CPPFLAGS) -DBIN_PATH=\"dummy\" -Wno-pointer-sign -fPIC -c -o ./afl-common.o ../../src/afl-common.c
+	$(CXX) -Wno-deprecated -g -O3 $(CXXFLAGS) $(CPPFLAGS) -shared -fPIC -o autotokens.so -I../../include autotokens.cpp  ./afl-fuzz-queue.o ../../src/afl-performance.o ./afl-common.o
 
 clean:
 	rm -f autotokens.so *~ core
diff --git a/custom_mutators/autotokens/TODO b/custom_mutators/autotokens/TODO
index 496bfd45..2e99e147 100644
--- a/custom_mutators/autotokens/TODO
+++ b/custom_mutators/autotokens/TODO
@@ -1,24 +1,3 @@
-create from thin air if no good seed after a cycle and dict large enough?
-(static u32 no_of_struct_inputs;) 
-
-splicing -> check if whitespace/token is needed
-
-whitespace/token check only AFTER mutation
-
-analyse welche einen DICT haben, und welche davon rein ascii
-
-corpus analyse:
-	+ libxml
-	- sqlite
-	- libpcap
-min len, max len, % wenn 95/98/99/100 ascii
-
 env für menge an per mutation run
 
-AFL_TXT_MAX_LEN 65535
-AFL_TXT_MIN_LEN 16
-AFL_TXT_MIN_PERCENT=99
-
--> KEIN FAV!
-
 change_min/_max werte
diff --git a/custom_mutators/autotokens/autotokens.cpp b/custom_mutators/autotokens/autotokens.cpp
index 10afa2c2..cda90a38 100644
--- a/custom_mutators/autotokens/autotokens.cpp
+++ b/custom_mutators/autotokens/autotokens.cpp
@@ -25,10 +25,12 @@ extern "C" {
 #define AUTOTOKENS_CHANGE_MIN 8
 #define AUTOTOKENS_CHANGE_MAX 64
 #define AUTOTOKENS_SIZE_MIN 8
+#define AUTOTOKENS_SIZE_MAX 65535
 #define AUTOTOKENS_SPLICE_MIN 4
 #define AUTOTOKENS_SPLICE_MAX 64
 #define AUTOTOKENS_CREATE_FROM_THIN_AIR 0
 #define AUTOTOKENS_FUZZ_COUNT_SHIFT 0
+#define AUTOTOKENS_AUTO_DISABLE 0
 // 0 = no learning, 1 only from -x dict/autodict, 2 also from cmplog
 #define AUTOTOKENS_LEARN_DICT 1
 #ifndef AUTOTOKENS_SPLICE_DISABLE
@@ -56,6 +58,8 @@ typedef struct my_mutator {
 #define IFDEBUG if (unlikely(debug))
 
 static afl_state *afl_ptr;
+static int        module_disabled = 0;
+static int        auto_disable = AUTOTOKENS_AUTO_DISABLE;
 static int        debug = AUTOTOKENS_DEBUG;
 static int        only_fav = AUTOTOKENS_ONLY_FAV;
 static int        learn_dictionary_tokens = AUTOTOKENS_LEARN_DICT;
@@ -93,6 +97,99 @@ static void first_run(void *data) {
 
   (void)(data);
 
+  /* For auto-loading this module we check here if we can analyze from the
+     input if the inputs look like text inputs and disable the module if
+     not. */
+
+  if (afl_ptr->custom_only || !auto_disable) { return; }
+
+  if (unlikely(afl_ptr->active_items == 1 &&
+               afl_ptr->queue_cur->len < AFL_TXT_MIN_LEN)) {
+
+    if (afl_ptr->extras_cnt > 8) {
+
+      u32 valid = 0;
+
+      while (extras_cnt < afl_ptr->extras_cnt) {
+
+        u32 ok = 1, l = afl_ptr->extras[extras_cnt].len;
+        u8 *buf, *ptr = afl_ptr->extras[extras_cnt].data;
+
+        for (u32 i = 0; i < l; ++i) {
+
+          if (!isascii((int)ptr[i]) && !isprint((int)ptr[i])) {
+
+            ok = 0;
+            break;
+
+          }
+
+        }
+
+        if (ok) {
+
+          buf = (u8 *)malloc(afl_ptr->extras[extras_cnt].len + 1);
+          memcpy(buf, afl_ptr->extras[extras_cnt].data,
+                 afl_ptr->extras[extras_cnt].len);
+          buf[afl_ptr->extras[extras_cnt].len] = 0;
+          token_to_id[(char *)buf] = current_id;
+          id_to_token[current_id] = (char *)buf;
+          ++current_id;
+          ++valid;
+
+        }
+
+        ++extras_cnt;
+
+      }
+
+      if ((valid * 100) / afl_ptr->extras_cnt < 95) { module_disabled = 1; }
+
+    } else {
+
+      module_disabled = 1;
+
+    }
+
+    return;
+
+  }
+
+  u32 is_ascii = 0, valid = 0;
+
+  for (u32 i = 0; i < afl_ptr->queued_items; ++i) {
+
+    struct queue_entry *q;
+
+    q = afl_ptr->queue_buf[i];
+
+    if (!q->disabled && q->len >= AUTOTOKENS_SIZE_MIN &&
+        q->len <= AFL_TXT_MAX_LEN) {
+
+      ++valid;
+      u8 *input = queue_testcase_get(afl_ptr, q);
+
+      u32 valid_chars = 0;
+      for (u32 i = 0; i < q->len; ++i) {
+
+        if (isascii((int)input[i]) || isprint((int)input[i])) { ++valid_chars; }
+
+      }
+
+      // we want at least 99% of text characters ...
+      if (((q->len * AFL_TXT_MIN_PERCENT) / 100) <= valid_chars) {
+
+        ++is_ascii;
+        q->is_ascii = 1;
+
+      }
+
+    }
+
+  }
+
+  if ((is_ascii * 100) / valid < 70) { module_disabled = 1; }
+
 }
 
 static u32 good_whitespace_or_singleval() {
@@ -441,21 +538,25 @@ extern "C" unsigned char afl_custom_queue_get(void                *data,
     is_first_run = 0;
     first_run(data);
 
+    if (module_disabled) { WARNF("Autotokens custom module is disabled."); }
+
   }
 
-  if (unlikely(!afl_ptr->custom_only) && !create_from_thin_air &&
-      ((afl_ptr->shm.cmplog_mode && !afl_ptr->queue_cur->is_ascii) ||
-       (only_fav && !afl_ptr->queue_cur->favored))) {
+  if (likely(module_disabled) ||
+      (unlikely(!afl_ptr->custom_only) && !create_from_thin_air &&
+       ((afl_ptr->shm.cmplog_mode && !afl_ptr->queue_cur->is_ascii) ||
+        (only_fav && !afl_ptr->queue_cur->favored)))) {
 
     s = NULL;
-    DEBUGF(stderr, "cmplog not ascii or only_fav and not favorite\n");
+    DEBUGF(stderr,
+           "cmplog not ascii or only_fav and not favorite or disabled\n");
     return 1;
 
   }
 
   // check if there are new dictionary entries and add them to the tokens
-  if (likely(valid_structures || create_from_thin_air) &&
-      learn_state < learn_dictionary_tokens) {
+  if (unlikely(learn_state < learn_dictionary_tokens) &&
+      likely(valid_structures || create_from_thin_air)) {
 
     if (unlikely(!learn_state)) { learn_state = 1; }
 
@@ -569,21 +670,10 @@ extern "C" unsigned char afl_custom_queue_get(void                *data,
   if (entry == file_mapping.end()) {
 
     // this input file was not analyzed for tokens yet, so let's do it!
-
-    FILE *fp = fopen((char *)filename, "rb");
-    if (!fp) {
-
-      s = NULL;
-      return 1;
-
-    }  // should not happen
-
-    fseek(fp, 0, SEEK_END);
-    size_t len = (size_t)ftell(fp);
+    size_t len = afl_ptr->queue_cur->len;
 
     if (len < AFL_TXT_MIN_LEN) {
 
-      fclose(fp);
       file_mapping[fn] = structure;  // NULL ptr so we don't read the file again
       s = NULL;
       DEBUGF(stderr, "Too short (%lu) %s\n", len, filename);
@@ -591,7 +681,6 @@ extern "C" unsigned char afl_custom_queue_get(void                *data,
 
     } else if (len > AFL_TXT_MAX_LEN) {
 
-      fclose(fp);
       file_mapping[fn] = structure;  // NULL ptr so we don't read the file again
       s = NULL;
       DEBUGF(stderr, "Too long (%lu) %s\n", len, filename);
@@ -599,19 +688,8 @@ extern "C" unsigned char afl_custom_queue_get(void                *data,
 
     }
 
-    string input;
-    input.resize(len);
-    rewind(fp);
-
-    if (fread((void *)input.data(), 1, len, fp) != len) {
-
-      s = NULL;
-      DEBUGF(stderr, "Too short read %s\n", filename);
-      return 1;
-
-    }
-
-    fclose(fp);
+    u8    *input_buf = queue_testcase_get(afl_ptr, afl_ptr->queue_cur);
+    string input((char *)input_buf, afl_ptr->queue_cur->len);
 
     if (!afl_ptr->shm.cmplog_mode) {
 
@@ -866,6 +944,7 @@ extern "C" my_mutator_t *afl_custom_init(afl_state *afl, unsigned int seed) {
   }
 
   if (getenv("AUTOTOKENS_DEBUG")) { debug = 1; }
+  if (getenv("AUTOTOKENS_AUTO_DISABLE")) { auto_disable = 1; }
   if (getenv("AUTOTOKENS_ONLY_FAV")) { only_fav = 1; }
   if (getenv("AUTOTOKENS_CREATE_FROM_THIN_AIR")) { create_from_thin_air = 1; }
 
diff --git a/include/config.h b/include/config.h
index 49d09174..ad8b76a8 100644
--- a/include/config.h
+++ b/include/config.h
@@ -491,6 +491,10 @@
 
 #define AFL_TXT_MIN_LEN 16
 
+/* Maximum length of a queue input to be evaluated for "is_ascii"? */
+
+#define AFL_TXT_MAX_LEN 65535
+
 /* What is the minimum percentage of ascii characters present to be classifed
    as "is_ascii"? */
 
diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c
index e3faa392..3c8a3e46 100644
--- a/src/afl-fuzz-queue.c
+++ b/src/afl-fuzz-queue.c
@@ -27,6 +27,22 @@
 #include <ctype.h>
 #include <math.h>
 
+#ifdef _STANDALONE_MODULE
+void minimize_bits(afl_state_t *afl, u8 *dst, u8 *src) {
+
+  return;
+
+}
+
+void run_afl_custom_queue_new_entry(afl_state_t *afl, struct queue_entry *q,
+                                    u8 *a, u8 *b) {
+
+  return;
+
+}
+
+#endif
+
 /* select next queue entry based on alias algo - fast! */
 
 inline u32 select_next_queue_entry(afl_state_t *afl) {
@@ -78,8 +94,8 @@ void create_alias_table(afl_state_t *afl) {
   afl->alias_probability = (double *)afl_realloc(
       (void **)&afl->alias_probability, n * sizeof(double));
   double *P = (double *)afl_realloc(AFL_BUF_PARAM(out), n * sizeof(double));
-  int    *S = (u32 *)afl_realloc(AFL_BUF_PARAM(out_scratch), n * sizeof(u32));
-  int    *L = (u32 *)afl_realloc(AFL_BUF_PARAM(in_scratch), n * sizeof(u32));
+  int    *S = (int *)afl_realloc(AFL_BUF_PARAM(out_scratch), n * sizeof(u32));
+  int    *L = (int *)afl_realloc(AFL_BUF_PARAM(in_scratch), n * sizeof(u32));
 
   if (!P || !S || !L || !afl->alias_table || !afl->alias_probability) {
 
@@ -247,11 +263,11 @@ void create_alias_table(afl_state_t *afl) {
 
 void mark_as_det_done(afl_state_t *afl, struct queue_entry *q) {
 
-  u8  fn[PATH_MAX];
-  s32 fd;
+  char fn[PATH_MAX];
+  s32  fd;
 
   snprintf(fn, PATH_MAX, "%s/queue/.state/deterministic_done/%s", afl->out_dir,
-           strrchr(q->fname, '/') + 1);
+           strrchr((char *)q->fname, '/') + 1);
 
   fd = open(fn, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION);
   if (fd < 0) { PFATAL("Unable to create '%s'", fn); }
@@ -266,10 +282,10 @@ void mark_as_det_done(afl_state_t *afl, struct queue_entry *q) {
 
 void mark_as_variable(afl_state_t *afl, struct queue_entry *q) {
 
-  u8 fn[PATH_MAX];
-  u8 ldest[PATH_MAX];
+  char fn[PATH_MAX];
+  char ldest[PATH_MAX];
 
-  u8 *fn_name = strrchr(q->fname, '/') + 1;
+  char *fn_name = strrchr((char *)q->fname, '/') + 1;
 
   sprintf(ldest, "../../%s", fn_name);
   sprintf(fn, "%s/queue/.state/variable_behavior/%s", afl->out_dir, fn_name);
@@ -293,12 +309,12 @@ void mark_as_redundant(afl_state_t *afl, struct queue_entry *q, u8 state) {
 
   if (likely(state == q->fs_redundant)) { return; }
 
-  u8 fn[PATH_MAX];
+  char fn[PATH_MAX];
 
   q->fs_redundant = state;
 
   sprintf(fn, "%s/queue/.state/redundant_edges/%s", afl->out_dir,
-          strrchr(q->fname, '/') + 1);
+          strrchr((char *)q->fname, '/') + 1);
 
   if (state) {
 
@@ -409,7 +425,7 @@ u8 check_if_text_buf(u8 *buf, u32 len) {
 
 static u8 check_if_text(afl_state_t *afl, struct queue_entry *q) {
 
-  if (q->len < AFL_TXT_MIN_LEN) return 0;
+  if (q->len < AFL_TXT_MIN_LEN || q->len < AFL_TXT_MAX_LEN) return 0;
 
   u8     *buf;
   int     fd;
@@ -417,8 +433,8 @@ static u8 check_if_text(afl_state_t *afl, struct queue_entry *q) {
   ssize_t comp;
 
   if (len >= MAX_FILE) len = MAX_FILE - 1;
-  if ((fd = open(q->fname, O_RDONLY)) < 0) return 0;
-  buf = afl_realloc(AFL_BUF_PARAM(in_scratch), len + 1);
+  if ((fd = open((char *)q->fname, O_RDONLY)) < 0) return 0;
+  buf = (u8 *)afl_realloc(AFL_BUF_PARAM(in_scratch), len + 1);
   comp = read(fd, buf, len);
   close(fd);
   if (comp != (ssize_t)len) return 0;
@@ -520,7 +536,8 @@ static u8 check_if_text(afl_state_t *afl, struct queue_entry *q) {
 
 void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
 
-  struct queue_entry *q = ck_alloc(sizeof(struct queue_entry));
+  struct queue_entry *q =
+      (struct queue_entry *)ck_alloc(sizeof(struct queue_entry));
 
   q->fname = fname;
   q->len = len;
@@ -554,7 +571,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
 
   afl->cycles_wo_finds = 0;
 
-  struct queue_entry **queue_buf = afl_realloc(
+  struct queue_entry **queue_buf = (struct queue_entry **)afl_realloc(
       AFL_BUF_PARAM(queue), afl->queued_items * sizeof(struct queue_entry *));
   if (unlikely(!queue_buf)) { PFATAL("alloc"); }
   queue_buf[afl->queued_items - 1] = q;
@@ -574,7 +591,11 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
   }
 
   /* only redqueen currently uses is_ascii */
-  if (afl->shm.cmplog_mode) q->is_ascii = check_if_text(afl, q);
+  if (unlikely(afl->shm.cmplog_mode && !q->is_ascii)) {
+
+    q->is_ascii = check_if_text(afl, q);
+
+  }
 
 }
 
@@ -704,7 +725,7 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) {
       if (!q->trace_mini) {
 
         u32 len = (afl->fsrv.map_size >> 3);
-        q->trace_mini = ck_alloc(len);
+        q->trace_mini = (u8 *)ck_alloc(len);
         minimize_bits(afl, q->trace_mini, afl->fsrv.trace_bits);
 
       }
@@ -1090,19 +1111,19 @@ inline void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q,
     if (len != old_len) {
 
       afl->q_testcase_cache_size = afl->q_testcase_cache_size + len - old_len;
-      q->testcase_buf = realloc(q->testcase_buf, len);
+      q->testcase_buf = (u8 *)realloc(q->testcase_buf, len);
 
       if (unlikely(!q->testcase_buf)) {
 
-        PFATAL("Unable to malloc '%s' with len %u", q->fname, len);
+        PFATAL("Unable to malloc '%s' with len %u", (char *)q->fname, len);
 
       }
 
     }
 
-    int fd = open(q->fname, O_RDONLY);
+    int fd = open((char *)q->fname, O_RDONLY);
 
-    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); }
+    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", (char *)q->fname); }
 
     ck_read(fd, q->testcase_buf, len, q->fname);
     close(fd);
@@ -1122,7 +1143,7 @@ inline void queue_testcase_retake_mem(afl_state_t *afl, struct queue_entry *q,
 
     if (likely(len != old_len)) {
 
-      u8 *ptr = realloc(q->testcase_buf, len);
+      u8 *ptr = (u8 *)realloc(q->testcase_buf, len);
 
       if (likely(ptr)) {
 
@@ -1154,23 +1175,23 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
 
     if (unlikely(q == afl->queue_cur)) {
 
-      buf = afl_realloc((void **)&afl->testcase_buf, len);
+      buf = (u8 *)afl_realloc((void **)&afl->testcase_buf, len);
 
     } else {
 
-      buf = afl_realloc((void **)&afl->splicecase_buf, len);
+      buf = (u8 *)afl_realloc((void **)&afl->splicecase_buf, len);
 
     }
 
     if (unlikely(!buf)) {
 
-      PFATAL("Unable to malloc '%s' with len %u", q->fname, len);
+      PFATAL("Unable to malloc '%s' with len %u", (char *)q->fname, len);
 
     }
 
-    int fd = open(q->fname, O_RDONLY);
+    int fd = open((char *)q->fname, O_RDONLY);
 
-    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); }
+    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", (char *)q->fname); }
 
     ck_read(fd, buf, len, q->fname);
     close(fd);
@@ -1214,7 +1235,7 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
 
         do_once = 1;
         // release unneeded memory
-        afl->q_testcase_cache = ck_realloc(
+        afl->q_testcase_cache = (struct queue_entry **)ck_realloc(
             afl->q_testcase_cache,
             (afl->q_testcase_max_cache_entries + 1) * sizeof(size_t));
 
@@ -1261,15 +1282,15 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) {
 
     /* Map the test case into memory. */
 
-    int fd = open(q->fname, O_RDONLY);
+    int fd = open((char *)q->fname, O_RDONLY);
 
-    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); }
+    if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", (char *)q->fname); }
 
-    q->testcase_buf = malloc(len);
+    q->testcase_buf = (u8 *)malloc(len);
 
     if (unlikely(!q->testcase_buf)) {
 
-      PFATAL("Unable to malloc '%s' with len %u", q->fname, len);
+      PFATAL("Unable to malloc '%s' with len %u", (char *)q->fname, len);
 
     }
 
@@ -1332,11 +1353,11 @@ inline void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q,
 
   /* Map the test case into memory. */
 
-  q->testcase_buf = malloc(len);
+  q->testcase_buf = (u8 *)malloc(len);
 
   if (unlikely(!q->testcase_buf)) {
 
-    PFATAL("Unable to malloc '%s' with len %u", q->fname, len);
+    PFATAL("Unable to malloc '%s' with len %u", (char *)q->fname, len);
 
   }