LTO optimization, variable map size, autodictionary (#307)

* lto module clean-up

* step 1/3

* step 1/3 completed

* if tmp is ever made non-static

* parts 2 and 3 - autodictionary is complete

* variable map_size support

* variable map size: changed overlooked functions

* remove debug for autodict

* 64 bit alignment of map size

* fix review comments

* force 64 bit alignment on both sides

* typo

3 files changed, 30 insertions, 12 deletions

diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h
index 56135d0e..edda81e1 100644
--- a/include/afl-fuzz.h
+++ b/include/afl-fuzz.h
@@ -577,7 +577,9 @@ typedef struct afl_state {
   u32 document_counter;
 #endif
 
-  /* statis file */
+  void *maybe_add_auto;
+
+  /* statistics file */
   double last_bitmap_cvg, last_stability, last_eps;
 
   /* plot file saves from last run */
@@ -840,18 +842,18 @@ u32  calculate_score(afl_state_t *, struct queue_entry *);
 
 void read_bitmap(afl_state_t *, u8 *);
 void write_bitmap(afl_state_t *);
-u32  count_bits(u8 *);
-u32  count_bytes(u8 *);
-u32  count_non_255_bytes(u8 *);
+u32  count_bits(afl_state_t *, u8 *);
+u32  count_bytes(afl_state_t *, u8 *);
+u32  count_non_255_bytes(afl_state_t *, u8 *);
 #ifdef WORD_SIZE_64
-void simplify_trace(u64 *);
-void classify_counts(u64 *);
+void simplify_trace(afl_state_t *, u64 *);
+void classify_counts(afl_state_t *, u64 *);
 #else
-void simplify_trace(u32 *);
-void classify_counts(u32 *);
+void simplify_trace(afl_state_t *, u32 *);
+void classify_counts(afl_state_t *, u32 *);
 #endif
 void init_count_class16(void);
-void minimize_bits(u8 *, u8 *);
+void minimize_bits(afl_state_t *, u8 *, u8 *);
 #ifndef SIMPLE_FILES
 u8 *describe_op(afl_state_t *, u8);
 #endif
@@ -862,7 +864,7 @@ u8 has_new_bits(afl_state_t *, u8 *);
 
 void load_extras_file(afl_state_t *, u8 *, u32 *, u32 *, u32);
 void load_extras(afl_state_t *, u8 *);
-void maybe_add_auto(afl_state_t *, u8 *, u32);
+void maybe_add_auto(void *, u8 *, u32);
 void save_auto(afl_state_t *);
 void load_auto(afl_state_t *);
 void destroy_extras(afl_state_t *);
diff --git a/include/config.h b/include/config.h
index cf73772f..f0274fd3 100644
--- a/include/config.h
+++ b/include/config.h
@@ -201,8 +201,8 @@
    (first value), and to keep in memory as candidates. The latter should be much
    higher than the former. */
 
-#define USE_AUTO_EXTRAS 50
-#define MAX_AUTO_EXTRAS (USE_AUTO_EXTRAS * 10)
+#define USE_AUTO_EXTRAS 128
+#define MAX_AUTO_EXTRAS (USE_AUTO_EXTRAS * 64)
 
 /* Scaling factor for the effector map used to skip some of the more
    expensive deterministic steps. The actual divisor is set to
@@ -400,5 +400,15 @@
 #endif
 #endif                           /* __APPLE__ || __FreeBSD__ || __OpenBSD__ */
 
+/* Extended forkserver option values */
+
+#define FS_OPT_ENABLED 0x8f000001
+#define FS_OPT_MAPSIZE 0x40000000
+#define FS_OPT_SNAPSHOT 0x20000000
+#define FS_OPT_AUTODICT 0x10000000
+#define FS_OPT_GET_MAPSIZE(x) (((x & 0x00fffffe) >> 1) + 1)
+#define FS_OPT_SET_MAPSIZE(x) \
+  (x <= 1 || x > MAP_SIZE || x > 0x1000000 ? 0 : ((x - 1) << 1))
+
 #endif                                                  /* ! _HAVE_CONFIG_H */
 
diff --git a/include/forkserver.h b/include/forkserver.h
index 4110df7d..7470dbbc 100644
--- a/include/forkserver.h
+++ b/include/forkserver.h
@@ -51,6 +51,8 @@ typedef struct afl_forkserver {
       fsrv_st_fd;                       /* Fork server status pipe (read)   */
 
   u32 exec_tmout;                       /* Configurable exec timeout (ms)   */
+  u32 map_size;                         /* map size used by the target      */
+  u32 snapshot;                         /* is snapshot feature used         */
   u64 mem_limit;                        /* Memory cap for child (MB)        */
 
   u8 *out_file,                         /* File to fuzz, if any             */
@@ -64,6 +66,10 @@ typedef struct afl_forkserver {
 
   u32 prev_timed_out;                   /* if prev forkserver run timed out */
 
+  u8 *function_opt;                     /* for autodictionary: afl ptr      */
+
+  void (*function_ptr)(void *afl_tmp, u8 *mem, u32 len);
+
 } afl_forkserver_t;
 
 void afl_fsrv_init(afl_forkserver_t *fsrv);