Merge branch 'master' of github.com:vanhauser-thc/AFLplusplus

14 files changed, 85 insertions, 43 deletions

diff --git a/Makefile b/Makefile
index 5af8444a..ea991cd3 100644
--- a/Makefile
+++ b/Makefile
@@ -15,6 +15,9 @@
 
 # For Heiko:
 #TEST_MMAP=1
+# the hash character is treated differently in different make versions
+# so use a variable for '#'
+HASH=\#
 
 PREFIX     ?= /usr/local
 BIN_PATH    = $(PREFIX)/bin
@@ -24,7 +27,7 @@ MISC_PATH   = $(PREFIX)/share/afl
 MAN_PATH    = $(PREFIX)/man/man8
 
 PROGNAME    = afl
-VERSION     = $(shell grep '^\#define VERSION ' ../config.h | cut -d '"' -f2)
+VERSION     = $(shell grep '^$(HASH)define VERSION ' ../config.h | cut -d '"' -f2)
 
 # PROGS intentionally omit afl-as, which gets installed elsewhere.
 
@@ -127,7 +130,7 @@ endif
 
 COMM_HDR    = include/alloc-inl.h include/config.h include/debug.h include/types.h
 
-ifeq "$(shell echo '\#include <Python.h>@int main() {return 0; }' | tr @ '\n' | $(CC) -x c - -o .test $(PYTHON_INCLUDE) $(LDFLAGS) $(PYTHON_LIB) 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1"
+ifeq "$(shell echo '$(HASH)include <Python.h>@int main() {return 0; }' | tr @ '\n' | $(CC) -x c - -o .test $(PYTHON_INCLUDE) $(LDFLAGS) $(PYTHON_LIB) 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1"
 	PYTHON_OK=1
 	PYFLAGS=-DUSE_PYTHON $(PYTHON_INCLUDE) $(LDFLAGS) $(PYTHON_LIB) -DPYTHON_VERSION="\"$(PYTHON_VERSION)\""
 else
@@ -145,7 +148,7 @@ ifdef STATIC
   LDFLAGS += -lm -lrt -lpthread -lz -lutil
 endif
 
-ifeq "$(shell echo '\#include <sys/ipc.h>@\#include <sys/shm.h>@int main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, 0); return 0;}' | tr @ '\n' | $(CC) -x c - -o .test2 2>/dev/null && echo 1 || echo 0 ; rm -f .test2 )" "1"
+ifeq "$(shell echo '$(HASH)include <sys/ipc.h>@$(HASH)include <sys/shm.h>@int main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, 0); return 0;}' | tr @ '\n' | $(CC) -x c - -o .test2 2>/dev/null && echo 1 || echo 0 ; rm -f .test2 )" "1"
 	SHMAT_OK=1
 else
 	SHMAT_OK=0
diff --git a/README.md b/README.md
index 939cfb92..8c759779 100644
--- a/README.md
+++ b/README.md
@@ -56,6 +56,7 @@
 
   * qbdi_mode: fuzz android native libraries via QBDI framework
 
+  * The new CmpLog instrumentation for LLVM and QEMU inspired by [Redqueen](https://www.syssec.ruhr-uni-bochum.de/media/emma/veroeffentlichungen/2018/12/17/NDSS19-Redqueen.pdf)
 
   A more thorough list is available in the PATCHES file.
 
diff --git a/TODO.md b/TODO.md
index 02850276..5e46e7d1 100644
--- a/TODO.md
+++ b/TODO.md
@@ -1,4 +1,3 @@
-
 ## Roadmap 2.61
 
 Makefile:
diff --git a/docs/Changelog.md b/docs/Changelog.md
index 23b6ec6a..e0bb500a 100644
--- a/docs/Changelog.md
+++ b/docs/Changelog.md
@@ -35,6 +35,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
      - AFL_PERSISTENT_HOOK callback module for persistent QEMU
        (see examples/qemu_persistent_hook)
      - added qemu_mode/README.persistent.md documentation
+     - AFL_ENTRYPOINT noew has instruction granularity
   - afl-cmin is now a sh script (invoking awk) instead of bash for portability
     the original script is still present as afl-cmin.bash
   - afl-showmap: -i dir option now allows processing multiple inputs using the
diff --git a/gcc_plugin/Makefile b/gcc_plugin/Makefile
index 9ad5c382..9d312a94 100644
--- a/gcc_plugin/Makefile
+++ b/gcc_plugin/Makefile
@@ -35,8 +35,9 @@ CC          ?= gcc
 CXX         ?= g++
 
 PLUGIN_FLAGS = -fPIC -fno-rtti -I"$(shell $(CC) -print-file-name=plugin)/include"
+HASH=\#
 
-ifeq "$(shell echo '\#include <sys/ipc.h>@\#include <sys/shm.h>@int main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, 0); return 0;}' | tr @ '\n' | $(CC) -x c - -o .test2 2>/dev/null && echo 1 || echo 0 ; rm -f .test2 )" "1"
+ifeq "$(shell echo '$(HASH)include <sys/ipc.h>@$(HASH)include <sys/shm.h>@int main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, 0); return 0;}' | tr @ '\n' | $(CC) -x c - -o .test2 2>/dev/null && echo 1 || echo 0 ; rm -f .test2 )" "1"
 	SHMAT_OK=1
 else
 	SHMAT_OK=0
diff --git a/llvm_mode/Makefile b/llvm_mode/Makefile
index 6e0a27af..6fa04e2c 100644
--- a/llvm_mode/Makefile
+++ b/llvm_mode/Makefile
@@ -18,12 +18,13 @@
 
 # For Heiko:
 #TEST_MMAP=1
+HASH=\#
 
 PREFIX      ?= /usr/local
 HELPER_PATH  = $(PREFIX)/lib/afl
 BIN_PATH     = $(PREFIX)/bin
 
-VERSION     = $(shell grep '^\#define VERSION ' ../config.h | cut -d '"' -f2)
+VERSION     = $(shell grep '^$(HASH)define VERSION ' ../config.h | cut -d '"' -f2)
 
 ifeq "$(shell uname)" "OpenBSD"
   LLVM_CONFIG ?= $(BIN_PATH)/llvm-config
@@ -116,7 +117,7 @@ endif
 CLANGVER = $(shell $(CC) --version | sed -E -ne '/^.*version\ ([0-9]\.[0-9]\.[0-9]).*/s//\1/p')
 
 
-ifeq "$(shell echo '\#include <sys/ipc.h>@\#include <sys/shm.h>@int main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, 0); return 0;}' | tr @ '\n' | $(CC) -x c - -o .test2 2>/dev/null && echo 1 || echo 0 ; rm -f .test2 )" "1"
+ifeq "$(shell echo '$(HASH)include <sys/ipc.h>@$(HASH)include <sys/shm.h>@int main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, 0); return 0;}' | tr @ '\n' | $(CC) -x c - -o .test2 2>/dev/null && echo 1 || echo 0 ; rm -f .test2 )" "1"
         SHMAT_OK=1
 else
         SHMAT_OK=0
diff --git a/qemu_mode/README.md b/qemu_mode/README.md
index 4198af14..0759f4fb 100644
--- a/qemu_mode/README.md
+++ b/qemu_mode/README.md
@@ -66,8 +66,7 @@ the deferred initialization.
 This can be enabled setting the environment variable AFL_ENTRYPOINT which allows
 to move the forkserver to a different part, e.g. just before the file is
 opened (e.g. way after command line parsing and config file loading, etc.)
-which can be a huge speed improvement. Note that the specified address
-must be an address of a basic block.
+which can be a huge speed improvement.
 
 ## 4) Bonus feature #2: persistent mode
 
diff --git a/qemu_mode/build_qemu_support.sh b/qemu_mode/build_qemu_support.sh
index 79993ce2..0671a66c 100755
--- a/qemu_mode/build_qemu_support.sh
+++ b/qemu_mode/build_qemu_support.sh
@@ -158,6 +158,9 @@ patch -p1 <../patches/i386-ops_sse.diff || exit 1
 patch -p1 <../patches/i386-fpu_helper.diff || exit 1
 patch -p1 <../patches/softfloat.diff || exit 1
 patch -p1 <../patches/configure.diff || exit 1
+patch -p1 <../patches/tcg-runtime.diff || exit 1
+patch -p1 <../patches/tcg-runtime-head.diff || exit 1
+patch -p1 <../patches/translator.diff || exit 1
 
 echo "[+] Patching done."
 
diff --git a/qemu_mode/patches/afl-qemu-common.h b/qemu_mode/patches/afl-qemu-common.h
index 4303a5e6..f2a44ba3 100644
--- a/qemu_mode/patches/afl-qemu-common.h
+++ b/qemu_mode/patches/afl-qemu-common.h
@@ -69,7 +69,7 @@ typedef void (*afl_persistent_hook_fn)(uint64_t *regs, uint64_t guest_base);
 
 extern unsigned char *afl_area_ptr;
 extern unsigned int   afl_inst_rms;
-extern abi_ulong      afl_start_code, afl_end_code;
+extern abi_ulong      afl_entry_point, afl_start_code, afl_end_code;
 extern abi_ulong      afl_persistent_addr;
 extern abi_ulong      afl_persistent_ret_addr;
 extern u8             afl_compcov_level;
@@ -88,6 +88,9 @@ extern __thread abi_ulong afl_prev_loc;
 extern struct cmp_map *__afl_cmp_map;
 extern __thread u32    __afl_cmp_counter;
 
+void afl_setup(void);
+void afl_forkserver(CPUState *cpu);
+
 void afl_debug_dump_saved_regs();
 
 void afl_persistent_loop();
diff --git a/qemu_mode/patches/afl-qemu-cpu-inl.h b/qemu_mode/patches/afl-qemu-cpu-inl.h
index 5e155c74..28fa1ab6 100644
--- a/qemu_mode/patches/afl-qemu-cpu-inl.h
+++ b/qemu_mode/patches/afl-qemu-cpu-inl.h
@@ -42,22 +42,6 @@
  * VARIOUS AUXILIARY STUFF *
  ***************************/
 
-/* This snippet kicks in when the instruction pointer is positioned at
-   _start and does the usual forkserver stuff, not very different from
-   regular instrumentation injected via afl-as.h. */
-
-#define AFL_QEMU_CPU_SNIPPET2         \
-  do {                                \
-                                      \
-    if (itb->pc == afl_entry_point) { \
-                                      \
-      afl_setup();                    \
-      afl_forkserver(cpu);            \
-                                      \
-    }                                 \
-                                      \
-  } while (0)
-
 /* We use one additional file descriptor to relay "needs translation"
    messages between the child and the fork server. */
 
@@ -107,9 +91,6 @@ unsigned int afl_inst_rms = MAP_SIZE;         /* Exported for afl_gen_trace */
 
 /* Function declarations. */
 
-static void afl_setup(void);
-static void afl_forkserver(CPUState *);
-
 static void afl_wait_tsl(CPUState *, int);
 static void afl_request_tsl(target_ulong, target_ulong, uint32_t, uint32_t,
                             TranslationBlock *, int);
@@ -155,7 +136,7 @@ static inline void              tb_add_jump(TranslationBlock *tb, int n,
 
 /* Set up SHM region and initialize other stuff. */
 
-static void afl_setup(void) {
+void afl_setup(void) {
 
   char *id_str = getenv(SHM_ENV_VAR), *inst_r = getenv("AFL_INST_RATIO");
 
@@ -310,7 +291,7 @@ static void print_mappings(void) {
 
 /* Fork server logic, invoked once we hit _start. */
 
-static void afl_forkserver(CPUState *cpu) {
+void afl_forkserver(CPUState *cpu) {
 
   static unsigned char tmp[4];
 
diff --git a/qemu_mode/patches/cpu-exec.diff b/qemu_mode/patches/cpu-exec.diff
index cd35eef6..844be58c 100644
--- a/qemu_mode/patches/cpu-exec.diff
+++ b/qemu_mode/patches/cpu-exec.diff
@@ -1,5 +1,5 @@
 diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
-index 870027d4..841ba557 100644
+index 870027d4..0bc87dfc 100644
 --- a/accel/tcg/cpu-exec.c
 +++ b/accel/tcg/cpu-exec.c
 @@ -36,6 +36,8 @@
@@ -11,16 +11,7 @@ index 870027d4..841ba557 100644
  /* -icount align implementation. */
  
  typedef struct SyncClocks {
-@@ -144,6 +146,8 @@ static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb)
-     int tb_exit;
-     uint8_t *tb_ptr = itb->tc.ptr;
- 
-+    AFL_QEMU_CPU_SNIPPET2;
-+
-     qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc,
-                            "Trace %d: %p ["
-                            TARGET_FMT_lx "/" TARGET_FMT_lx "/%#x] %s\n",
-@@ -397,11 +401,13 @@ static inline TranslationBlock *tb_find(CPUState *cpu,
+@@ -397,11 +399,13 @@ static inline TranslationBlock *tb_find(CPUState *cpu,
      TranslationBlock *tb;
      target_ulong cs_base, pc;
      uint32_t flags;
@@ -34,7 +25,7 @@ index 870027d4..841ba557 100644
          mmap_unlock();
          /* We add the TB in the virtual pc hash table for the fast lookup */
          atomic_set(&cpu->tb_jmp_cache[tb_jmp_cache_hash_func(pc)], tb);
-@@ -418,6 +424,10 @@ static inline TranslationBlock *tb_find(CPUState *cpu,
+@@ -418,6 +422,10 @@ static inline TranslationBlock *tb_find(CPUState *cpu,
      /* See if we can patch the calling TB. */
      if (last_tb) {
          tb_add_jump(last_tb, tb_exit, tb);
diff --git a/qemu_mode/patches/tcg-runtime-head.diff b/qemu_mode/patches/tcg-runtime-head.diff
new file mode 100644
index 00000000..d2deafaa
--- /dev/null
+++ b/qemu_mode/patches/tcg-runtime-head.diff
@@ -0,0 +1,10 @@
+diff --git a/accel/tcg/tcg-runtime.h b/accel/tcg/tcg-runtime.h
+index 1bd39d13..944997ee 100644
+--- a/accel/tcg/tcg-runtime.h
++++ b/accel/tcg/tcg-runtime.h
+@@ -260,3 +260,5 @@ DEF_HELPER_FLAGS_4(gvec_leu8, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+ DEF_HELPER_FLAGS_4(gvec_leu16, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+ DEF_HELPER_FLAGS_4(gvec_leu32, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+ DEF_HELPER_FLAGS_4(gvec_leu64, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
++
++DEF_HELPER_FLAGS_1(afl_entry_routine, TCG_CALL_NO_RWG, void, env)
diff --git a/qemu_mode/patches/tcg-runtime.diff b/qemu_mode/patches/tcg-runtime.diff
new file mode 100644
index 00000000..54a62ba8
--- /dev/null
+++ b/qemu_mode/patches/tcg-runtime.diff
@@ -0,0 +1,24 @@
+diff --git a/accel/tcg/tcg-runtime.c b/accel/tcg/tcg-runtime.c
+index d0d44844..46154af1 100644
+--- a/accel/tcg/tcg-runtime.c
++++ b/accel/tcg/tcg-runtime.c
+@@ -31,6 +31,8 @@
+ #include "disas/disas.h"
+ #include "exec/log.h"
+ 
++#include "../../../patches/afl-qemu-common.h"
++
+ /* 32-bit helpers */
+ 
+ int32_t HELPER(div_i32)(int32_t arg1, int32_t arg2)
+@@ -167,3 +169,10 @@ void HELPER(exit_atomic)(CPUArchState *env)
+ {
+     cpu_loop_exit_atomic(ENV_GET_CPU(env), GETPC());
+ }
++
++
++void HELPER(afl_entry_routine)(CPUArchState *env) {
++  
++  afl_forkserver(ENV_GET_CPU(env));
++  
++}
diff --git a/qemu_mode/patches/translator.diff b/qemu_mode/patches/translator.diff
new file mode 100644
index 00000000..842e861d
--- /dev/null
+++ b/qemu_mode/patches/translator.diff
@@ -0,0 +1,25 @@
+diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c
+index afd0a49e..773ea712 100644
+--- a/accel/tcg/translator.c
++++ b/accel/tcg/translator.c
+@@ -18,6 +18,8 @@
+ #include "exec/log.h"
+ #include "exec/translator.h"
+ 
++#include "../../../patches/afl-qemu-common.h"
++
+ /* Pairs with tcg_clear_temp_count.
+    To be called by #TranslatorOps.{translate_insn,tb_stop} if
+    (1) the target is sufficiently clean to support reporting,
+@@ -92,6 +94,11 @@ void translator_loop(const TranslatorOps *ops, DisasContextBase *db,
+                 break;
+             }
+         }
++        
++        if (db->pc_next == afl_entry_point) {
++          afl_setup();
++          gen_helper_afl_entry_routine(cpu_env);
++        }
+ 
+         /* Disassemble one instruction.  The translate_insn hook should
+            update db->pc_next and db->is_jmp to indicate what should be