Merge branch 'dev' of https://github.com/AFLplusplus/AFLplusplus into dev

12 files changed, 165 insertions, 99 deletions

diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h
index 988a907d..1d5ec1f0 100644
--- a/include/afl-fuzz.h
+++ b/include/afl-fuzz.h
@@ -37,10 +37,6 @@
   #define _FILE_OFFSET_BITS 64
 #endif
 
-#ifdef __ANDROID__
-  #include "android-ashmem.h"
-#endif
-
 #include "config.h"
 #include "types.h"
 #include "debug.h"
@@ -145,12 +141,23 @@ extern s16 interesting_16[INTERESTING_8_LEN + INTERESTING_16_LEN];
 extern s32
     interesting_32[INTERESTING_8_LEN + INTERESTING_16_LEN + INTERESTING_32_LEN];
 
+struct tainted {
+
+  u32             pos;
+  u32             len;
+  struct tainted *next;
+  struct tainted *prev;
+
+};
+
 struct queue_entry {
 
   u8 *fname;                            /* File name for the test case      */
   u32 len;                              /* Input length                     */
+  u32 id;                               /* entry number in queue_buf        */
 
-  u8   cal_failed;                      /* Calibration failed?              */
+  u8 colorized,                         /* Do not run redqueen stage again  */
+      cal_failed;                       /* Calibration failed?              */
   bool trim_done,                       /* Trimmed?                         */
       was_fuzzed,                       /* historical, but needed for MOpt  */
       passed_det,                       /* Deterministic stages passed?     */
@@ -158,7 +165,6 @@ struct queue_entry {
       var_behavior,                     /* Variable behavior?               */
       favored,                          /* Currently favored?               */
       fs_redundant,                     /* Marked as redundant in the fs?   */
-      fully_colorized,                  /* Do not run redqueen stage again  */
       is_ascii,                         /* Is the input just ascii text?    */
       disabled;                         /* Is disabled from fuzz selection  */
 
@@ -183,7 +189,10 @@ struct queue_entry {
 
   u8 *testcase_buf;                     /* The testcase buffer, if loaded.  */
 
-  struct queue_entry *next;             /* Next element, if any             */
+  u8 *            cmplog_colorinput;    /* the result buf of colorization   */
+  struct tainted *taint;                /* Taint information from CmpLog    */
+
+  struct queue_entry *mother;           /* queue entry this based on        */
 
 };
 
@@ -375,7 +384,7 @@ typedef struct afl_env_vars {
       afl_dumb_forksrv, afl_import_first, afl_custom_mutator_only, afl_no_ui,
       afl_force_ui, afl_i_dont_care_about_missing_crashes, afl_bench_just_one,
       afl_bench_until_crash, afl_debug_child, afl_autoresume, afl_cal_fast,
-      afl_cycle_schedules, afl_expand_havoc, afl_statsd;
+      afl_cycle_schedules, afl_expand_havoc, afl_statsd, afl_cmplog_only_new;
 
   u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path,
       *afl_hang_tmout, *afl_forksrv_init_tmout, *afl_skip_crashes, *afl_preload,
@@ -395,7 +404,7 @@ struct afl_pass_stat {
 struct foreign_sync {
 
   u8 *   dir;
-  time_t ctime;
+  time_t mtime;
 
 };
 
@@ -416,7 +425,8 @@ typedef struct afl_state {
     really makes no sense to haul them around as function parameters. */
   u64 orig_hit_cnt_puppet, last_limit_time_start, tmp_pilot_time,
       total_pacemaker_time, total_puppet_find, temp_puppet_find, most_time_key,
-      most_time, most_execs_key, most_execs, old_hit_count, force_ui_update;
+      most_time, most_execs_key, most_execs, old_hit_count, force_ui_update,
+      prev_run_time;
 
   MOpt_globals_t mopt_globals_core, mopt_globals_pilot;
 
@@ -636,6 +646,9 @@ typedef struct afl_state {
   /* cmplog forkserver ids */
   s32 cmplog_fsrv_ctl_fd, cmplog_fsrv_st_fd;
   u32 cmplog_prev_timed_out;
+  u32 cmplog_max_filesize;
+  u32 cmplog_lvl;
+  u32 colorize_success;
 
   struct afl_pass_stat *pass_stats;
   struct cmp_map *      orig_cmp_map;
@@ -1055,6 +1068,7 @@ void destroy_extras(afl_state_t *);
 
 /* Stats */
 
+void load_stats_file(afl_state_t *);
 void write_setup_file(afl_state_t *, u32, char **);
 void write_stats_file(afl_state_t *, double, double, double);
 void maybe_update_plot_file(afl_state_t *, double, double);
@@ -1121,9 +1135,9 @@ void   read_foreign_testcases(afl_state_t *, int);
 u8 common_fuzz_cmplog_stuff(afl_state_t *afl, u8 *out_buf, u32 len);
 
 /* RedQueen */
-u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len,
-                        u64 exec_cksum);
+u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len);
 
+/* our RNG wrapper */
 AFL_RAND_RETURN rand_next(afl_state_t *afl);
 
 /* probability between 0.0 and 1.0 */
diff --git a/include/android-ashmem.h b/include/android-ashmem.h
index 41d4d2da..91699b27 100644
--- a/include/android-ashmem.h
+++ b/include/android-ashmem.h
@@ -1,62 +1,34 @@
-/*
-   american fuzzy lop++ - android shared memory compatibility layer
-   ----------------------------------------------------------------
-
-   Originally written by Michal Zalewski
-
-   Now maintained by Marc Heuse <mh@mh-sec.de>,
-                     Heiko Eißfeldt <heiko.eissfeldt@hexco.de>,
-                     Andrea Fioraldi <andreafioraldi@gmail.com>,
-                     Dominik Maier <mail@dmnk.co>
-
-   Copyright 2016, 2017 Google Inc. All rights reserved.
-   Copyright 2019-2020 AFLplusplus Project. All rights reserved.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at:
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-   This header re-defines the shared memory routines used by AFL++
-   using the Andoid API.
-
- */
-
-#ifndef _ANDROID_ASHMEM_H
-#define _ANDROID_ASHMEM_H
-
 #ifdef __ANDROID__
-
-  #include <fcntl.h>
-  #include <linux/shm.h>
-  #include <linux/ashmem.h>
-  #include <sys/ioctl.h>
-  #include <sys/mman.h>
-
-  #if __ANDROID_API__ >= 26
-    #define shmat bionic_shmat
-    #define shmctl bionic_shmctl
-    #define shmdt bionic_shmdt
-    #define shmget bionic_shmget
-  #endif
-
-  #include <sys/shm.h>
-  #undef shmat
-  #undef shmctl
-  #undef shmdt
-  #undef shmget
-  #include <stdio.h>
-
-  #define ASHMEM_DEVICE "/dev/ashmem"
-
-static inline int shmctl(int __shmid, int __cmd, struct shmid_ds *__buf) {
+  #ifndef _ANDROID_ASHMEM_H
+    #define _ANDROID_ASHMEM_H
+
+    #include <fcntl.h>
+    #include <linux/ashmem.h>
+    #include <sys/ioctl.h>
+    #include <sys/mman.h>
+
+    #if __ANDROID_API__ >= 26
+      #define shmat bionic_shmat
+      #define shmctl bionic_shmctl
+      #define shmdt bionic_shmdt
+      #define shmget bionic_shmget
+    #endif
+    #include <sys/shm.h>
+    #undef shmat
+    #undef shmctl
+    #undef shmdt
+    #undef shmget
+    #include <stdio.h>
+
+    #define ASHMEM_DEVICE "/dev/ashmem"
+
+int shmctl(int __shmid, int __cmd, struct shmid_ds *__buf) {
 
   int ret = 0;
   if (__cmd == IPC_RMID) {
 
     int               length = ioctl(__shmid, ASHMEM_GET_SIZE, NULL);
-    struct ashmem_pin pin = {0, (unsigned int)length};
+    struct ashmem_pin pin = {0, length};
     ret = ioctl(__shmid, ASHMEM_UNPIN, &pin);
     close(__shmid);
 
@@ -66,7 +38,7 @@ static inline int shmctl(int __shmid, int __cmd, struct shmid_ds *__buf) {
 
 }
 
-static inline int shmget(key_t __key, size_t __size, int __shmflg) {
+int shmget(key_t __key, size_t __size, int __shmflg) {
 
   (void)__shmflg;
   int  fd, ret;
@@ -90,7 +62,7 @@ error:
 
 }
 
-static inline void *shmat(int __shmid, const void *__shmaddr, int __shmflg) {
+void *shmat(int __shmid, const void *__shmaddr, int __shmflg) {
 
   (void)__shmflg;
   int   size;
@@ -106,7 +78,6 @@ static inline void *shmat(int __shmid, const void *__shmaddr, int __shmflg) {
 
 }
 
-#endif                                                       /* __ANDROID__ */
-
-#endif
+  #endif                                              /* !_ANDROID_ASHMEM_H */
+#endif                                                      /* !__ANDROID__ */
 
diff --git a/include/cmplog.h b/include/cmplog.h
index bf557785..878ed60c 100644
--- a/include/cmplog.h
+++ b/include/cmplog.h
@@ -30,24 +30,25 @@
 
 #include "config.h"
 
+#define CMPLOG_LVL_MAX 3
+
 #define CMP_MAP_W 65536
-#define CMP_MAP_H 256
+#define CMP_MAP_H 32
 #define CMP_MAP_RTN_H (CMP_MAP_H / 4)
 
 #define SHAPE_BYTES(x) (x + 1)
 
-#define CMP_TYPE_INS 0
-#define CMP_TYPE_RTN 1
+#define CMP_TYPE_INS 1
+#define CMP_TYPE_RTN 2
 
 struct cmp_header {
 
-  unsigned hits : 20;
-
-  unsigned cnt : 20;
-  unsigned id : 16;
-
-  unsigned shape : 5;  // from 0 to 31
-  unsigned type : 1;
+  unsigned hits : 24;
+  unsigned id : 24;
+  unsigned shape : 5;
+  unsigned type : 2;
+  unsigned attribute : 4;
+  unsigned reserved : 5;
 
 } __attribute__((packed));
 
@@ -55,6 +56,8 @@ struct cmp_operands {
 
   u64 v0;
   u64 v1;
+  u64 v0_128;
+  u64 v1_128;
 
 };
 
diff --git a/include/common.h b/include/common.h
index 9490ec5f..bb8831f2 100644
--- a/include/common.h
+++ b/include/common.h
@@ -47,6 +47,7 @@ void   argv_cpy_free(char **argv);
 char **get_qemu_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv);
 char **get_wine_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv);
 char * get_afl_env(char *env);
+u8 *   get_libqasan_path(u8 *own_loc);
 
 extern u8  be_quiet;
 extern u8 *doc_path;                    /* path to documentation dir        */
diff --git a/include/config.h b/include/config.h
index 973bbcbb..181285cd 100644
--- a/include/config.h
+++ b/include/config.h
@@ -10,7 +10,7 @@
                      Dominik Maier <mail@dmnk.co>
 
    Copyright 2016, 2017 Google Inc. All rights reserved.
-   Copyright 2019-2020 AFLplusplus Project. All rights reserved.
+   Copyright 2019-2021 AFLplusplus Project. All rights reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -23,8 +23,6 @@
 #ifndef _HAVE_CONFIG_H
 #define _HAVE_CONFIG_H
 
-#include "types.h"
-
 /* Version string: */
 
 // c = release, d = volatile github dev, e = experimental branch
@@ -36,6 +34,38 @@
  *                                                    *
  ******************************************************/
 
+/* CMPLOG/REDQUEEN TUNING
+ *
+ * Here you can modify tuning and solving options for CMPLOG.
+ * Note that these are run-time options for afl-fuzz, no target
+ * recompilation required.
+ *
+ */
+
+/* Enable arithmetic compare solving for both path */
+#define CMPLOG_SOLVE_ARITHMETIC
+
+/* Enable transform following (XOR/ADD/SUB manipulations, hex en/decoding) */
+#define CMPLOG_SOLVE_TRANSFORM
+
+/* if TRANSFORM is enabled, this additionally enables base64 en/decoding */
+// #define CMPLOG_SOLVE_TRANSFORM_BASE64
+
+/* If a redqueen pass finds more than one solve, try to combine them? */
+#define CMPLOG_COMBINE
+
+/* Minimum % of the corpus to perform cmplog on. Default: 20% */
+#define CMPLOG_CORPUS_PERCENT 20U
+
+/* Number of potential positions from which we decide if cmplog becomes
+   useless, default 16384 */
+#define CMPLOG_POSITIONS_MAX 16384U
+
+/* Maximum allowed fails per CMP value. Default: 32 * 3 */
+#define CMPLOG_FAIL_MAX 96
+
+/* Now non-cmplog configuration options */
+
 /* console output colors: There are three ways to configure its behavior
  * 1. default: colored outputs fixed on: defined USE_COLOR && defined
  * ALWAYS_COLORED The env var. AFL_NO_COLOR will have no effect
@@ -69,7 +99,7 @@
 /* If you want to have the original afl internal memory corruption checks.
    Disabled by default for speed. it is better to use "make ASAN_BUILD=1". */
 
-//#define _WANT_ORIGINAL_AFL_ALLOC
+// #define _WANT_ORIGINAL_AFL_ALLOC
 
 /* Comment out to disable fancy ANSI boxes and use poor man's 7-bit UI: */
 
diff --git a/include/coverage-32.h b/include/coverage-32.h
index a5cc498c..ca36c29f 100644
--- a/include/coverage-32.h
+++ b/include/coverage-32.h
@@ -97,7 +97,7 @@ inline void discover_word(u8 *ret, u32 *current, u32 *virgin) {
 #define PACK_SIZE 16
 inline u32 skim(const u32 *virgin, const u32 *current, const u32 *current_end) {
 
-  for (; current != current_end; virgin += 4, current += 4) {
+  for (; current < current_end; virgin += 4, current += 4) {
 
     if (current[0] && classify_word(current[0]) & virgin[0]) return 1;
     if (current[1] && classify_word(current[1]) & virgin[1]) return 1;
diff --git a/include/coverage-64.h b/include/coverage-64.h
index 0ede5fa5..54fe9d33 100644
--- a/include/coverage-64.h
+++ b/include/coverage-64.h
@@ -145,7 +145,7 @@ inline u32 skim(const u64 *virgin, const u64 *current, const u64 *current_end) {
 
   __m256i zeroes = _mm256_setzero_si256();
 
-  for (; current != current_end; virgin += 4, current += 4) {
+  for (; current < current_end; virgin += 4, current += 4) {
 
     __m256i value = *(__m256i *)current;
     __m256i cmp = _mm256_cmpeq_epi64(value, zeroes);
@@ -172,7 +172,7 @@ inline u32 skim(const u64 *virgin, const u64 *current, const u64 *current_end) {
   #define PACK_SIZE 32
 inline u32 skim(const u64 *virgin, const u64 *current, const u64 *current_end) {
 
-  for (; current != current_end; virgin += 4, current += 4) {
+  for (; current < current_end; virgin += 4, current += 4) {
 
     if (current[0] && classify_word(current[0]) & virgin[0]) return 1;
     if (current[1] && classify_word(current[1]) & virgin[1]) return 1;
diff --git a/include/debug.h b/include/debug.h
index ef5b195b..fc1f39cb 100644
--- a/include/debug.h
+++ b/include/debug.h
@@ -295,8 +295,8 @@ static inline const char *colorfilter(const char *x) {
                                                                          \
     SAYF(bSTOP RESET_G1 CURSOR_SHOW cRST cLRD                            \
          "\n[-] PROGRAM ABORT : " cRST   x);                               \
-    SAYF(cLRD "\n         Location : " cRST "%s(), %s:%d\n\n", __func__, \
-         __FILE__, __LINE__);                                            \
+    SAYF(cLRD "\n         Location : " cRST "%s(), %s:%u\n\n", __func__, \
+         __FILE__, (u32)__LINE__);                                       \
     exit(1);                                                             \
                                                                          \
   } while (0)
@@ -308,8 +308,8 @@ static inline const char *colorfilter(const char *x) {
                                                                          \
     SAYF(bSTOP RESET_G1 CURSOR_SHOW cRST cLRD                            \
          "\n[-] PROGRAM ABORT : " cRST   x);                               \
-    SAYF(cLRD "\n    Stop location : " cRST "%s(), %s:%d\n\n", __func__, \
-         __FILE__, __LINE__);                                            \
+    SAYF(cLRD "\n    Stop location : " cRST "%s(), %s:%u\n\n", __func__, \
+         __FILE__, (u32)__LINE__);                                       \
     abort();                                                             \
                                                                          \
   } while (0)
@@ -322,8 +322,8 @@ static inline const char *colorfilter(const char *x) {
     fflush(stdout);                                                    \
     SAYF(bSTOP RESET_G1 CURSOR_SHOW cRST cLRD                          \
          "\n[-]  SYSTEM ERROR : " cRST   x);                             \
-    SAYF(cLRD "\n    Stop location : " cRST "%s(), %s:%d\n", __func__, \
-         __FILE__, __LINE__);                                          \
+    SAYF(cLRD "\n    Stop location : " cRST "%s(), %s:%u\n", __func__, \
+         __FILE__, (u32)__LINE__);                                     \
     SAYF(cLRD "       OS message : " cRST "%s\n", strerror(errno));    \
     exit(1);                                                           \
                                                                        \
@@ -344,12 +344,12 @@ static inline const char *colorfilter(const char *x) {
 
 /* Show a prefixed debug output. */
 
-#define DEBUGF(x...)                         \
-  do {                                       \
-                                             \
-    SAYF(cMGN "[D] " cBRI "DEBUG: " cRST x); \
-    SAYF(cRST "");                           \
-                                             \
+#define DEBUGF(x...)                                    \
+  do {                                                  \
+                                                        \
+    fprintf(stderr, cMGN "[D] " cBRI "DEBUG: " cRST x); \
+    fprintf(stderr, cRST "");                           \
+                                                        \
   } while (0)
 
 /* Error-checking versions of read() and write() that call RPFATAL() as
diff --git a/include/envs.h b/include/envs.h
index 97367fae..4313e053 100644
--- a/include/envs.h
+++ b/include/envs.h
@@ -28,6 +28,7 @@ static char *afl_environment_variables[] = {
     "AFL_CC",
     "AFL_CMIN_ALLOW_ANY",
     "AFL_CMIN_CRASHES_ONLY",
+    "AFL_CMPLOG_ONLY_NEW",
     "AFL_CODE_END",
     "AFL_CODE_START",
     "AFL_COMPCOV_BINNAME",
@@ -42,11 +43,13 @@ static char *afl_environment_variables[] = {
     "AFL_DEBUG_GDB",
     "AFL_DISABLE_TRIM",
     "AFL_DONT_OPTIMIZE",
+    "AFL_DRIVER_STDERR_DUPLICATE_FILENAME",
     "AFL_DUMB_FORKSRV",
     "AFL_ENTRYPOINT",
     "AFL_EXIT_WHEN_DONE",
     "AFL_FAST_CAL",
     "AFL_FORCE_UI",
+    "AFL_FUZZER_ARGS",  // oss-fuzz
     "AFL_GCC_ALLOWLIST",
     "AFL_GCC_DENYLIST",
     "AFL_GCC_BLOCKLIST",
@@ -58,6 +61,7 @@ static char *afl_environment_variables[] = {
     "AFL_FORKSRV_INIT_TMOUT",
     "AFL_HARDEN",
     "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES",
+    "AFL_IGNORE_UNKNOWN_ENVS",
     "AFL_IMPORT_FIRST",
     "AFL_INST_LIBS",
     "AFL_INST_RATIO",
@@ -162,6 +166,7 @@ static char *afl_environment_variables[] = {
     "AFL_WINE_PATH",
     "AFL_NO_SNAPSHOT",
     "AFL_EXPAND_HAVOC_NOW",
+    "AFL_USE_QASAN",
     NULL
 
 };
diff --git a/include/forkserver.h b/include/forkserver.h
index 3019e289..ac027f81 100644
--- a/include/forkserver.h
+++ b/include/forkserver.h
@@ -83,6 +83,8 @@ typedef struct afl_forkserver {
 
   bool uses_asan;                       /* Target uses ASAN?                */
 
+  bool debug;                           /* debug mode?                      */
+
   bool uses_crash_exitcode;             /* Custom crash exitcode specified? */
   u8   crash_exitcode;                  /* The crash exitcode specified     */
 
@@ -118,11 +120,14 @@ void afl_fsrv_init(afl_forkserver_t *fsrv);
 void afl_fsrv_init_dup(afl_forkserver_t *fsrv_to, afl_forkserver_t *from);
 void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv,
                     volatile u8 *stop_soon_p, u8 debug_child_output);
+u32  afl_fsrv_get_mapsize(afl_forkserver_t *fsrv, char **argv,
+                          volatile u8 *stop_soon_p, u8 debug_child_output);
 void afl_fsrv_write_to_testcase(afl_forkserver_t *fsrv, u8 *buf, size_t len);
 fsrv_run_result_t afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout,
                                       volatile u8 *stop_soon_p);
 void              afl_fsrv_killall(void);
 void              afl_fsrv_deinit(afl_forkserver_t *fsrv);
+void              afl_fsrv_kill(afl_forkserver_t *fsrv);
 
 #ifdef __APPLE__
   #define MSG_FORK_ON_APPLE                                                    \
diff --git a/include/sharedmem.h b/include/sharedmem.h
index b15d0535..fdc947f9 100644
--- a/include/sharedmem.h
+++ b/include/sharedmem.h
@@ -51,6 +51,7 @@ typedef struct sharedmem {
   size_t map_size;                                 /* actual allocated size */
 
   int             cmplog_mode;
+  int             shmemfuzz_mode;
   struct cmp_map *cmp_map;
 
 } sharedmem_t;
diff --git a/include/types.h b/include/types.h
index 3e3bc953..7b94fb83 100644
--- a/include/types.h
+++ b/include/types.h
@@ -25,10 +25,15 @@
 
 #include <stdint.h>
 #include <stdlib.h>
+#include "config.h"
 
 typedef uint8_t  u8;
 typedef uint16_t u16;
 typedef uint32_t u32;
+#ifdef WORD_SIZE_64
+typedef unsigned __int128 uint128_t;
+typedef uint128_t         u128;
+#endif
 
 /* Extended forkserver option values */
 
@@ -61,6 +66,10 @@ typedef int8_t  s8;
 typedef int16_t s16;
 typedef int32_t s32;
 typedef int64_t s64;
+#ifdef WORD_SIZE_64
+typedef __int128 int128_t;
+typedef int128_t s128;
+#endif
 
 #ifndef MIN
   #define MIN(a, b)           \
@@ -114,6 +123,33 @@ typedef int64_t s64;
                                                                                \
   })
 
+// It is impossible to define 128 bit constants, so ...
+#ifdef WORD_SIZE_64
+  #define SWAPN(_x, _l)                            \
+    ({                                             \
+                                                   \
+      u128  _res = (_x), _ret;                     \
+      char *d = (char *)&_ret, *s = (char *)&_res; \
+      int   i;                                     \
+      for (i = 0; i < 16; i++)                     \
+        d[15 - i] = s[i];                          \
+      u32 sr = 128U - ((_l) << 3U);                \
+      (_ret >>= sr);                               \
+      (u128) _ret;                                 \
+                                                   \
+    })
+#endif
+
+#define SWAPNN(_x, _y, _l)                     \
+  ({                                           \
+                                               \
+    char *d = (char *)(_x), *s = (char *)(_y); \
+    u32   i, l = (_l)-1;                       \
+    for (i = 0; i <= l; i++)                   \
+      d[l - i] = s[i];                         \
+                                               \
+  })
+
 #ifdef AFL_LLVM_PASS
   #if defined(__linux__) || !defined(__ANDROID__)
     #define AFL_SR(s) (srandom(s))