LTO optimization, variable map size, autodictionary (#307)

* lto module clean-up

* step 1/3

* step 1/3 completed

* if tmp is ever made non-static

* parts 2 and 3 - autodictionary is complete

* variable map_size support

* variable map size: changed overlooked functions

* remove debug for autodict

* 64 bit alignment of map size

* fix review comments

* force 64 bit alignment on both sides

* typo

1 files changed, 14 insertions, 13 deletions

diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c
index 9bbdd23a..850a18bc 100644
--- a/src/afl-fuzz-run.c
+++ b/src/afl-fuzz-run.c
@@ -44,7 +44,7 @@ u8 run_target(afl_state_t *afl, u32 timeout) {
      must prevent any earlier operations from venturing into that
      territory. */
 
-  memset(afl->fsrv.trace_bits, 0, MAP_SIZE);
+  memset(afl->fsrv.trace_bits, 0, afl->fsrv.map_size);
 
   MEM_BARRIER();
 
@@ -122,9 +122,9 @@ u8 run_target(afl_state_t *afl, u32 timeout) {
   tb4 = *(u32 *)afl->fsrv.trace_bits;
 
 #ifdef WORD_SIZE_64
-  classify_counts((u64 *)afl->fsrv.trace_bits);
+  classify_counts(afl, (u64 *)afl->fsrv.trace_bits);
 #else
-  classify_counts((u32 *)afl->fsrv.trace_bits);
+  classify_counts(afl, (u32 *)afl->fsrv.trace_bits);
 #endif                                                     /* ^WORD_SIZE_64 */
 
   afl->fsrv.prev_timed_out = afl->fsrv.child_timed_out;
@@ -315,7 +315,8 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem,
       afl->shm.cmplog_mode)
     init_cmplog_forkserver(afl);
 
-  if (q->exec_cksum) memcpy(afl->first_trace, afl->fsrv.trace_bits, MAP_SIZE);
+  if (q->exec_cksum)
+    memcpy(afl->first_trace, afl->fsrv.trace_bits, afl->fsrv.map_size);
 
   start_us = get_cur_time_us();
 
@@ -336,14 +337,14 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem,
     if (afl->stop_soon || fault != afl->crash_mode) goto abort_calibration;
 
     if (!afl->dumb_mode && !afl->stage_cur &&
-        !count_bytes(afl->fsrv.trace_bits)) {
+        !count_bytes(afl, afl->fsrv.trace_bits)) {
 
       fault = FAULT_NOINST;
       goto abort_calibration;
 
     }
 
-    cksum = hash32(afl->fsrv.trace_bits, MAP_SIZE, HASH_CONST);
+    cksum = hash32(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST);
 
     if (q->exec_cksum != cksum) {
 
@@ -354,7 +355,7 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem,
 
         u32 i;
 
-        for (i = 0; i < MAP_SIZE; ++i) {
+        for (i = 0; i < afl->fsrv.map_size; ++i) {
 
           if (unlikely(!afl->var_bytes[i]) &&
               unlikely(afl->first_trace[i] != afl->fsrv.trace_bits[i]))
@@ -368,7 +369,7 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem,
       } else {
 
         q->exec_cksum = cksum;
-        memcpy(afl->first_trace, afl->fsrv.trace_bits, MAP_SIZE);
+        memcpy(afl->first_trace, afl->fsrv.trace_bits, afl->fsrv.map_size);
 
       }
 
@@ -385,7 +386,7 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem,
      This is used for fuzzing air time calculations in calculate_score(). */
 
   q->exec_us = (stop_us - start_us) / afl->stage_max;
-  q->bitmap_size = count_bytes(afl->fsrv.trace_bits);
+  q->bitmap_size = count_bytes(afl, afl->fsrv.trace_bits);
   q->handicap = handicap;
   q->cal_failed = 0;
 
@@ -413,7 +414,7 @@ abort_calibration:
 
   if (var_detected) {
 
-    afl->var_byte_count = count_bytes(afl->var_bytes);
+    afl->var_byte_count = count_bytes(afl, afl->var_bytes);
 
     if (!q->var_behavior) {
 
@@ -640,7 +641,7 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) {
       /* Note that we don't keep track of crashes or hangs here; maybe TODO?
        */
 
-      cksum = hash32(afl->fsrv.trace_bits, MAP_SIZE, HASH_CONST);
+      cksum = hash32(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST);
 
       /* If the deletion had no impact on the trace, make it permanent. This
          isn't perfect for variable-path inputs, but we're just making a
@@ -663,7 +664,7 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) {
         if (!needs_write) {
 
           needs_write = 1;
-          memcpy(afl->clean_trace, afl->fsrv.trace_bits, MAP_SIZE);
+          memcpy(afl->clean_trace, afl->fsrv.trace_bits, afl->fsrv.map_size);
 
         }
 
@@ -705,7 +706,7 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) {
     ck_write(fd, in_buf, q->len, q->fname);
     close(fd);
 
-    memcpy(afl->fsrv.trace_bits, afl->clean_trace, MAP_SIZE);
+    memcpy(afl->fsrv.trace_bits, afl->clean_trace, afl->fsrv.map_size);
     update_bitmap_score(afl, q);
 
   }