From 2971b5b31527be94037dfc4f60231ee2a0a1ea25 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 8 Aug 2019 10:36:43 +0200 Subject: documentation update --- docs/ChangeLog | 3 ++- docs/env_variables.txt | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) (limited to 'docs') diff --git a/docs/ChangeLog b/docs/ChangeLog index 5e78610e..dfebb68a 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -17,6 +17,8 @@ sending a mail to . Version ++2.53d (dev): ---------------------- + - llvm 9 is now supported (still needs testing) + - fix building qemu on some Ubuntus (thanks to floyd!) - custom mutator by a loaded library is now supported (thanks to kyakdan!) - fix for a few features to support different map sized than 2^16 - afl-showmap: new option -r now shows the real values in the buckets (stock @@ -26,7 +28,6 @@ Version ++2.53d (dev): - ... your patch? :) - -------------------------- Version ++2.53c (release): -------------------------- diff --git a/docs/env_variables.txt b/docs/env_variables.txt index 36fdc369..93066dbc 100644 --- a/docs/env_variables.txt +++ b/docs/env_variables.txt @@ -257,10 +257,14 @@ The QEMU wrapper used to instrument binary-only code supports several settings: Use this if you are unsure if the entrypoint might be wrong - but use it directly, e.g. afl-qemu-trace ./program - - If you want to specify a specific entrypoint into the binary (this can - be very good for the performance!), use AFL_ENTRYPOINT for this. + - AFL_ENTRYPOINT allows you to specify a specific entrypoint into the + binary (this can be very good for the performance!). The entrypoint is specified as hex address, e.g. 0x4004110 + - AFL_QEMU_COMPCOV is for a sub-project in qemu_mode called ./libcompcov + which implements laf-intel for qemu. It also needs AFL_PRELOAD and + you can find more information in qemu_mode/libcompcov/README.compcov + 5) Settings for afl-cmin ------------------------ -- cgit 1.4.1 From d3d0682310b840b027083133837bcd9be0638281 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 19 Aug 2019 12:54:55 +0200 Subject: seperated the forkserver from afl-fuzz and afl-tmin --- Makefile | 23 ++-- TODO | 23 +++- afl-analyze.c | 2 +- afl-common.c | 2 + afl-forkserver.c | 401 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ afl-forkserver.h | 25 ++++ afl-fuzz.c | 344 +++-------------------------------------------- afl-sharedmem.c | 137 +++++++++++++++++++ afl-sharedmem.h | 7 + afl-showmap.c | 2 +- afl-tmin.c | 124 +++++++++-------- docs/ChangeLog | 5 +- sharedmem.c | 137 ------------------- sharedmem.h | 6 - 14 files changed, 693 insertions(+), 545 deletions(-) create mode 100644 afl-forkserver.c create mode 100644 afl-forkserver.h create mode 100644 afl-sharedmem.c create mode 100644 afl-sharedmem.h delete mode 100644 sharedmem.c delete mode 100644 sharedmem.h (limited to 'docs') diff --git a/Makefile b/Makefile index e6e3af85..3d5059f7 100644 --- a/Makefile +++ b/Makefile @@ -134,20 +134,23 @@ afl-as: afl-as.c afl-as.h $(COMM_HDR) | test_x86 afl-common.o : afl-common.c $(CC) $(CFLAGS) -c afl-common.c -sharedmem.o : sharedmem.c - $(CC) $(CFLAGS) -c sharedmem.c +afl-forkserver.o : afl-forkserver.c + $(CC) $(CFLAGS) -c afl-forkserver.c -afl-fuzz: afl-fuzz.c afl-common.o sharedmem.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $@.c afl-common.o sharedmem.o -o $@ $(LDFLAGS) $(PYFLAGS) +afl-sharedmem.o : afl-sharedmem.c + $(CC) $(CFLAGS) -c afl-sharedmem.c -afl-showmap: afl-showmap.c afl-common.o sharedmem.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $@.c afl-common.o sharedmem.o -o $@ $(LDFLAGS) +afl-fuzz: afl-fuzz.c afl-common.o afl-sharedmem.o afl-forkserver.o $(COMM_HDR) | test_x86 + $(CC) $(CFLAGS) $@.c afl-common.o afl-sharedmem.o afl-forkserver.o -o $@ $(LDFLAGS) $(PYFLAGS) -afl-tmin: afl-tmin.c afl-common.o sharedmem.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $@.c afl-common.o sharedmem.o -o $@ $(LDFLAGS) +afl-showmap: afl-showmap.c afl-common.o afl-sharedmem.o $(COMM_HDR) | test_x86 + $(CC) $(CFLAGS) $@.c afl-common.o afl-sharedmem.o -o $@ $(LDFLAGS) -afl-analyze: afl-analyze.c afl-common.o sharedmem.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $@.c afl-common.o sharedmem.o -o $@ $(LDFLAGS) +afl-tmin: afl-tmin.c afl-common.o afl-sharedmem.o afl-forkserver.o $(COMM_HDR) | test_x86 + $(CC) $(CFLAGS) $@.c afl-common.o afl-sharedmem.o afl-forkserver.o -o $@ $(LDFLAGS) + +afl-analyze: afl-analyze.c afl-common.o afl-sharedmem.o $(COMM_HDR) | test_x86 + $(CC) $(CFLAGS) $@.c afl-common.o afl-sharedmem.o -o $@ $(LDFLAGS) afl-gotcpu: afl-gotcpu.c $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) $@.c -o $@ $(LDFLAGS) diff --git a/TODO b/TODO index 42987cb9..cc075abd 100644 --- a/TODO +++ b/TODO @@ -1,12 +1,24 @@ Roadmap 2.53d: ============== - - indent all the code: .clang-format - - update docs/sister_projects.txt +all: + - indent all the code: .clang-format? + (vh: tried, the variable definion look very ugly then, what to do?) afl-fuzz: - - put mutator, scheduler, forkserver and input channels in individual files - - reuse forkserver for showmap, afl-cmin, etc. + - modularize: forkserver is in a module + others: + mutator - is deeply integrated and would loose performance if split + scheduler - is within this and as the values it operates on are afl + specific it does not make sense to seperate this + input - if we get different input vectors then this would make sense, + e.g. network (which we have seen is super non-performant and using + desock is much faster) + so for the moment we are done? (vh) + +docs/: + - update docs/sister_projects.txt + - doc + example for AFL_CUSTOM_MUTATOR_LIBRARY gcc_plugin: - needs to be rewritten @@ -17,8 +29,9 @@ gcc_plugin: - neverZero qemu_mode: + - update to 4.x - deferred mode with AFL_DEFERRED_QEMU=0xaddress - @andrea - dont we have that already with AFL_ENTRYPOINT? + (vh: @andrea - dont we have that already with AFL_ENTRYPOINT?) unit testing / or large testcase campaign diff --git a/afl-analyze.c b/afl-analyze.c index af93150e..18b7456d 100644 --- a/afl-analyze.c +++ b/afl-analyze.c @@ -26,7 +26,7 @@ #include "debug.h" #include "alloc-inl.h" #include "hash.h" -#include "sharedmem.h" +#include "afl-sharedmem.h" #include "afl-common.h" #include diff --git a/afl-common.c b/afl-common.c index 5e2d0628..f3bbdfb4 100644 --- a/afl-common.c +++ b/afl-common.c @@ -15,6 +15,8 @@ #ifndef __glibc__ #include #endif + + void detect_file_args(char** argv, u8* prog_in) { u32 i = 0; diff --git a/afl-forkserver.c b/afl-forkserver.c new file mode 100644 index 00000000..226175e1 --- /dev/null +++ b/afl-forkserver.c @@ -0,0 +1,401 @@ +#include "config.h" +#include "types.h" +#include "debug.h" +#include "afl-forkserver.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* a program that includes afl-forkserver needs to define these */ +extern u8 uses_asan; +extern u8 *trace_bits; +extern s32 forksrv_pid, child_pid, fsrv_ctl_fd, fsrv_st_fd; +extern s32 out_fd, out_dir_fd, dev_urandom_fd, dev_null_fd; /* initialize these with -1 */ +extern u32 exec_tmout; +extern u64 mem_limit; +extern u8 *out_file, *target_path, *doc_path; +extern FILE *plot_file; + +/* we need this internally but can be defined and read extern in the main source */ +u8 child_timed_out; + + +/* Describe integer as memory size. */ + +u8* forkserver_DMS(u64 val) { + + static u8 tmp[12][16]; + static u8 cur; + +#define CHK_FORMAT(_divisor, _limit_mult, _fmt, _cast) do { \ + if (val < (_divisor) * (_limit_mult)) { \ + sprintf(tmp[cur], _fmt, ((_cast)val) / (_divisor)); \ + return tmp[cur]; \ + } \ + } while (0) + + + cur = (cur + 1) % 12; + + /* 0-9999 */ + CHK_FORMAT(1, 10000, "%llu B", u64); + + /* 10.0k - 99.9k */ + CHK_FORMAT(1024, 99.95, "%0.01f kB", double); + + /* 100k - 999k */ + CHK_FORMAT(1024, 1000, "%llu kB", u64); + + /* 1.00M - 9.99M */ + CHK_FORMAT(1024 * 1024, 9.995, "%0.02f MB", double); + + /* 10.0M - 99.9M */ + CHK_FORMAT(1024 * 1024, 99.95, "%0.01f MB", double); + + /* 100M - 999M */ + CHK_FORMAT(1024 * 1024, 1000, "%llu MB", u64); + + /* 1.00G - 9.99G */ + CHK_FORMAT(1024LL * 1024 * 1024, 9.995, "%0.02f GB", double); + + /* 10.0G - 99.9G */ + CHK_FORMAT(1024LL * 1024 * 1024, 99.95, "%0.01f GB", double); + + /* 100G - 999G */ + CHK_FORMAT(1024LL * 1024 * 1024, 1000, "%llu GB", u64); + + /* 1.00T - 9.99G */ + CHK_FORMAT(1024LL * 1024 * 1024 * 1024, 9.995, "%0.02f TB", double); + + /* 10.0T - 99.9T */ + CHK_FORMAT(1024LL * 1024 * 1024 * 1024, 99.95, "%0.01f TB", double); + +#undef CHK_FORMAT + + /* 100T+ */ + strcpy(tmp[cur], "infty"); + return tmp[cur]; + +} + + + +/* the timeout handler */ + +void handle_timeout(int sig) { + if (child_pid > 0) { + child_timed_out = 1; + kill(child_pid, SIGKILL); + } else if (child_pid == -1 && forksrv_pid > 0) { + child_timed_out = 1; + kill(forksrv_pid, SIGKILL); + } +} + + +/* Spin up fork server (instrumented mode only). The idea is explained here: + + http://lcamtuf.blogspot.com/2014/10/fuzzing-binaries-without-execve.html + + In essence, the instrumentation allows us to skip execve(), and just keep + cloning a stopped child. So, we just execute once, and then send commands + through a pipe. The other part of this logic is in afl-as.h / llvm_mode */ + +void init_forkserver(char **argv) { + + static struct itimerval it; + int st_pipe[2], ctl_pipe[2]; + int status; + s32 rlen; + + ACTF("Spinning up the fork server..."); + + if (pipe(st_pipe) || pipe(ctl_pipe)) + PFATAL("pipe() failed"); + + child_timed_out = 0; + forksrv_pid = fork(); + + if (forksrv_pid < 0) + PFATAL("fork() failed"); + + if (!forksrv_pid) { + + /* CHILD PROCESS */ + + struct rlimit r; + + /* Umpf. On OpenBSD, the default fd limit for root users is set to + soft 128. Let's try to fix that... */ + + if (!getrlimit(RLIMIT_NOFILE, &r) && r.rlim_cur < FORKSRV_FD + 2) { + r.rlim_cur = FORKSRV_FD + 2; + setrlimit(RLIMIT_NOFILE, &r); /* Ignore errors */ + } + + if (mem_limit) { + r.rlim_max = r.rlim_cur = ((rlim_t)mem_limit) << 20; + +#ifdef RLIMIT_AS + setrlimit(RLIMIT_AS, &r); /* Ignore errors */ +#else + /* This takes care of OpenBSD, which doesn't have RLIMIT_AS, but + according to reliable sources, RLIMIT_DATA covers anonymous + maps - so we should be getting good protection against OOM bugs. */ + + setrlimit(RLIMIT_DATA, &r); /* Ignore errors */ +#endif /* ^RLIMIT_AS */ + } + + /* Dumping cores is slow and can lead to anomalies if SIGKILL is delivered + before the dump is complete. */ + +// r.rlim_max = r.rlim_cur = 0; +// setrlimit(RLIMIT_CORE, &r); /* Ignore errors */ + + /* Isolate the process and configure standard descriptors. If out_file is + specified, stdin is /dev/null; otherwise, out_fd is cloned instead. */ + + setsid(); + + if (!getenv("AFL_DEBUG_CHILD_OUTPUT")) { + dup2(dev_null_fd, 1); + dup2(dev_null_fd, 2); + } + + if (out_file) { + dup2(dev_null_fd, 0); + } else { + dup2(out_fd, 0); + close(out_fd); + } + + /* Set up control and status pipes, close the unneeded original fds. */ + + if (dup2(ctl_pipe[0], FORKSRV_FD) < 0) + PFATAL("dup2() failed"); + if (dup2(st_pipe[1], FORKSRV_FD + 1) < 0) + PFATAL("dup2() failed"); + + close(ctl_pipe[0]); + close(ctl_pipe[1]); + close(st_pipe[0]); + close(st_pipe[1]); + + close(out_dir_fd); + close(dev_null_fd); + close(dev_urandom_fd); + close(plot_file == NULL ? -1 : fileno(plot_file)); + + /* This should improve performance a bit, since it stops the linker from + doing extra work post-fork(). */ + + if (!getenv("LD_BIND_LAZY")) + setenv("LD_BIND_NOW", "1", 0); + + /* Set sane defaults for ASAN if nothing else specified. */ + + setenv("ASAN_OPTIONS", + "abort_on_error=1:" + "detect_leaks=0:" + "symbolize=0:" + "allocator_may_return_null=1", + 0); + + /* MSAN is tricky, because it doesn't support abort_on_error=1 at this + point. So, we do this in a very hacky way. */ + + setenv("MSAN_OPTIONS", + "exit_code=" STRINGIFY(MSAN_ERROR) ":" + "symbolize=0:" + "abort_on_error=1:" + "allocator_may_return_null=1:" + "msan_track_origins=0", + 0); + + execv(target_path, argv); + + /* Use a distinctive bitmap signature to tell the parent about execv() + falling through. */ + + *(u32 *)trace_bits = EXEC_FAIL_SIG; + exit(0); + } + + /* PARENT PROCESS */ + + /* Close the unneeded endpoints. */ + + close(ctl_pipe[0]); + close(st_pipe[1]); + + fsrv_ctl_fd = ctl_pipe[1]; + fsrv_st_fd = st_pipe[0]; + + /* Wait for the fork server to come up, but don't wait too long. */ + + if (exec_tmout) { + it.it_value.tv_sec = ((exec_tmout * FORK_WAIT_MULT) / 1000); + it.it_value.tv_usec = ((exec_tmout * FORK_WAIT_MULT) % 1000) * 1000; + } + + setitimer(ITIMER_REAL, &it, NULL); + + rlen = read(fsrv_st_fd, &status, 4); + + it.it_value.tv_sec = 0; + it.it_value.tv_usec = 0; + + setitimer(ITIMER_REAL, &it, NULL); + + /* If we have a four-byte "hello" message from the server, we're all set. + Otherwise, try to figure out what went wrong. */ + + if (rlen == 4) { + OKF("All right - fork server is up."); + return; + } + + if (child_timed_out) + FATAL("Timeout while initializing fork server (adjusting -t may help)"); + + if (waitpid(forksrv_pid, &status, 0) <= 0) + PFATAL("waitpid() failed"); + + if (WIFSIGNALED(status)) { + + if (mem_limit && mem_limit < 500 && uses_asan) { + + SAYF("\n" cLRD "[-] " cRST "Whoops, the target binary crashed suddenly, " + "before receiving any input\n" + " from the fuzzer! Since it seems to be built with ASAN and you " + "have a\n" + " restrictive memory limit configured, this is expected; please " + "read\n" + " %s/notes_for_asan.txt for help.\n", + doc_path); + + } else if (!mem_limit) { + + SAYF("\n" cLRD "[-] " cRST "Whoops, the target binary crashed suddenly, " + "before receiving any input\n" + " from the fuzzer! There are several probable explanations:\n\n" + + " - The binary is just buggy and explodes entirely on its own. " + "If so, you\n" + " need to fix the underlying problem or find a better " + "replacement.\n\n" + + MSG_FORK_ON_APPLE + + " - Less likely, there is a horrible bug in the fuzzer. If other " + "options\n" + " fail, poke for troubleshooting " + "tips.\n"); + + } else { + + SAYF("\n" cLRD "[-] " cRST "Whoops, the target binary crashed suddenly, " + "before receiving any input\n" + " from the fuzzer! There are several probable explanations:\n\n" + + " - The current memory limit (%s) is too restrictive, causing " + "the\n" + " target to hit an OOM condition in the dynamic linker. Try " + "bumping up\n" + " the limit with the -m setting in the command line. A simple " + "way confirm\n" + " this diagnosis would be:\n\n" + + MSG_ULIMIT_USAGE " /path/to/fuzzed_app )\n\n" + + " Tip: you can use http://jwilk.net/software/recidivm to " + "quickly\n" + " estimate the required amount of virtual memory for the " + "binary.\n\n" + + " - The binary is just buggy and explodes entirely on its own. " + "If so, you\n" + " need to fix the underlying problem or find a better " + "replacement.\n\n" + + MSG_FORK_ON_APPLE + + " - Less likely, there is a horrible bug in the fuzzer. If other " + "options\n" + " fail, poke for troubleshooting " + "tips.\n", + forkserver_DMS(mem_limit << 20), mem_limit - 1); + } + + FATAL("Fork server crashed with signal %d", WTERMSIG(status)); + } + + if (*(u32 *)trace_bits == EXEC_FAIL_SIG) + FATAL("Unable to execute target application ('%s')", argv[0]); + + if (mem_limit && mem_limit < 500 && uses_asan) { + + SAYF("\n" cLRD "[-] " cRST "Hmm, looks like the target binary terminated " + "before we could complete a\n" + " handshake with the injected code. Since it seems to be built " + "with ASAN and\n" + " you have a restrictive memory limit configured, this is " + "expected; please\n" + " read %s/notes_for_asan.txt for help.\n", + doc_path); + + } else if (!mem_limit) { + + SAYF("\n" cLRD "[-] " cRST "Hmm, looks like the target binary terminated " + "before we could complete a\n" + " handshake with the injected code. Perhaps there is a horrible " + "bug in the\n" + " fuzzer. Poke for troubleshooting " + "tips.\n"); + + } else { + + SAYF( + "\n" cLRD "[-] " cRST "Hmm, looks like the target binary terminated " + "before we could complete a\n" + " handshake with the injected code. There are %s probable " + "explanations:\n\n" + + "%s" + " - The current memory limit (%s) is too restrictive, causing an " + "OOM\n" + " fault in the dynamic linker. This can be fixed with the -m " + "option. A\n" + " simple way to confirm the diagnosis may be:\n\n" + + MSG_ULIMIT_USAGE " /path/to/fuzzed_app )\n\n" + + " Tip: you can use http://jwilk.net/software/recidivm to quickly\n" + " estimate the required amount of virtual memory for the " + "binary.\n\n" + + " - Less likely, there is a horrible bug in the fuzzer. If other " + "options\n" + " fail, poke for troubleshooting " + "tips.\n", + getenv(DEFER_ENV_VAR) ? "three" : "two", + getenv(DEFER_ENV_VAR) + ? " - You are using deferred forkserver, but __AFL_INIT() is " + "never\n" + " reached before the program terminates.\n\n" + : "", + forkserver_DMS(mem_limit << 20), mem_limit - 1); + } + + FATAL("Fork server handshake failed"); +} + diff --git a/afl-forkserver.h b/afl-forkserver.h new file mode 100644 index 00000000..fa40d9c6 --- /dev/null +++ b/afl-forkserver.h @@ -0,0 +1,25 @@ +#ifndef __AFL_FORKSERVER_H +#define __AFL_FORKSERVER_H + +void handle_timeout(int sig); +void init_forkserver(char **argv); + +#ifdef __APPLE__ +#define MSG_FORK_ON_APPLE \ + " - On MacOS X, the semantics of fork() syscalls are non-standard and " \ + "may\n" \ + " break afl-fuzz performance optimizations when running " \ + "platform-specific\n" \ + " targets. To fix this, set AFL_NO_FORKSRV=1 in the environment.\n\n" +#else +#define MSG_FORK_ON_APPLE "" +#endif + +#ifdef RLIMIT_AS + #define MSG_ULIMIT_USAGE " ( ulimit -Sv $[%llu << 10];" +#else + #define MSG_ULIMIT_USAGE " ( ulimit -Sd $[%llu << 10];" +#endif /* ^RLIMIT_AS */ + + +#endif diff --git a/afl-fuzz.c b/afl-fuzz.c index 2accde86..ec54cc85 100644 --- a/afl-fuzz.c +++ b/afl-fuzz.c @@ -33,7 +33,8 @@ #include "debug.h" #include "alloc-inl.h" #include "hash.h" -#include "sharedmem.h" +#include "afl-sharedmem.h" +#include "afl-forkserver.h" #include "afl-common.h" #include @@ -144,7 +145,6 @@ double period_pilot_tmp = 5000.0; int key_lv = 0; EXP_ST u8 *in_dir, /* Input directory with test cases */ - *out_file, /* File to fuzz, if any */ *out_dir, /* Working & output directory */ *tmp_dir , /* Temporary directory for input */ *sync_dir, /* Synchronization directory */ @@ -152,15 +152,16 @@ EXP_ST u8 *in_dir, /* Input directory with test cases */ *power_name, /* Power schedule name */ *use_banner, /* Display banner */ *in_bitmap, /* Input bitmap */ - *doc_path, /* Path to documentation dir */ - *target_path, /* Path to target binary */ *file_extension, /* File extension */ *orig_cmdline; /* Original command line */ + u8 *doc_path, /* Path to documentation dir */ + *target_path, /* Path to target binary */ + *out_file; /* File to fuzz, if any */ -EXP_ST u32 exec_tmout = EXEC_TIMEOUT; /* Configurable exec timeout (ms) */ + u32 exec_tmout = EXEC_TIMEOUT; /* Configurable exec timeout (ms) */ static u32 hang_tmout = EXEC_TIMEOUT; /* Timeout used for hang det (ms) */ -EXP_ST u64 mem_limit = MEM_LIMIT; /* Memory cap for child (MB) */ + u64 mem_limit = MEM_LIMIT; /* Memory cap for child (MB) */ EXP_ST u8 cal_cycles = CAL_CYCLES; /* Calibration cycles defaults */ EXP_ST u8 cal_cycles_long = CAL_CYCLES_LONG; @@ -200,7 +201,6 @@ EXP_ST u8 skip_deterministic, /* Skip deterministic stages? */ timeout_given, /* Specific timeout given? */ not_on_tty, /* stdout is not a tty */ term_too_small, /* terminal dimensions too small */ - uses_asan, /* Target uses ASAN? */ no_forkserver, /* Disable forkserver? */ crash_mode, /* Crash mode! Yeah! */ in_place_resume, /* Attempt in-place resume? */ @@ -217,14 +217,15 @@ EXP_ST u8 skip_deterministic, /* Skip deterministic stages? */ deferred_mode, /* Deferred forkserver mode? */ fixed_seed, /* do not reseed */ fast_cal; /* Try to calibrate faster? */ + u8 uses_asan; /* Target uses ASAN? */ -static s32 out_fd, /* Persistent fd for out_file */ + s32 out_fd, /* Persistent fd for out_file */ dev_urandom_fd = -1, /* Persistent fd for /dev/urandom */ dev_null_fd = -1, /* Persistent fd for /dev/null */ fsrv_ctl_fd, /* Fork server control pipe (write) */ fsrv_st_fd; /* Fork server status pipe (read) */ -static s32 forksrv_pid, /* PID of the fork server */ + s32 forksrv_pid, /* PID of the fork server */ child_pid = -1, /* PID of the fuzzed program */ out_dir_fd = -1; /* FD of the lock file */ @@ -313,7 +314,7 @@ static s32 cpu_aff = -1; /* Selected CPU core */ #endif /* HAVE_AFFINITY */ -static FILE* plot_file; /* Gnuplot output file */ +FILE* plot_file; /* Gnuplot output file */ struct queue_entry { @@ -2308,299 +2309,6 @@ static void destroy_extras(void) { } -/* Spin up fork server (instrumented mode only). The idea is explained here: - - http://lcamtuf.blogspot.com/2014/10/fuzzing-binaries-without-execve.html - - In essence, the instrumentation allows us to skip execve(), and just keep - cloning a stopped child. So, we just execute once, and then send commands - through a pipe. The other part of this logic is in afl-as.h. */ - -EXP_ST void init_forkserver(char** argv) { - - static struct itimerval it; - int st_pipe[2], ctl_pipe[2]; - int status; - s32 rlen; - - ACTF("Spinning up the fork server..."); - - if (pipe(st_pipe) || pipe(ctl_pipe)) PFATAL("pipe() failed"); - - forksrv_pid = fork(); - - if (forksrv_pid < 0) PFATAL("fork() failed"); - - if (!forksrv_pid) { - - /* CHILD PROCESS */ - - struct rlimit r; - - /* Umpf. On OpenBSD, the default fd limit for root users is set to - soft 128. Let's try to fix that... */ - - if (!getrlimit(RLIMIT_NOFILE, &r) && r.rlim_cur < FORKSRV_FD + 2) { - - r.rlim_cur = FORKSRV_FD + 2; - setrlimit(RLIMIT_NOFILE, &r); /* Ignore errors */ - - } - - if (mem_limit) { - - r.rlim_max = r.rlim_cur = ((rlim_t)mem_limit) << 20; - -#ifdef RLIMIT_AS - - setrlimit(RLIMIT_AS, &r); /* Ignore errors */ - -#else - - /* This takes care of OpenBSD, which doesn't have RLIMIT_AS, but - according to reliable sources, RLIMIT_DATA covers anonymous - maps - so we should be getting good protection against OOM bugs. */ - - setrlimit(RLIMIT_DATA, &r); /* Ignore errors */ - -#endif /* ^RLIMIT_AS */ - - - } - - /* Dumping cores is slow and can lead to anomalies if SIGKILL is delivered - before the dump is complete. */ - - r.rlim_max = r.rlim_cur = 0; - - setrlimit(RLIMIT_CORE, &r); /* Ignore errors */ - - /* Isolate the process and configure standard descriptors. If out_file is - specified, stdin is /dev/null; otherwise, out_fd is cloned instead. */ - - setsid(); - - if (!getenv("AFL_DEBUG_CHILD_OUTPUT")) { - dup2(dev_null_fd, 1); - dup2(dev_null_fd, 2); - } - - if (out_file) { - - dup2(dev_null_fd, 0); - - } else { - - dup2(out_fd, 0); - close(out_fd); - - } - - /* Set up control and status pipes, close the unneeded original fds. */ - - if (dup2(ctl_pipe[0], FORKSRV_FD) < 0) PFATAL("dup2() failed"); - if (dup2(st_pipe[1], FORKSRV_FD + 1) < 0) PFATAL("dup2() failed"); - - close(ctl_pipe[0]); - close(ctl_pipe[1]); - close(st_pipe[0]); - close(st_pipe[1]); - - close(out_dir_fd); - close(dev_null_fd); - close(dev_urandom_fd); - close(fileno(plot_file)); - - /* This should improve performance a bit, since it stops the linker from - doing extra work post-fork(). */ - - if (!getenv("LD_BIND_LAZY")) setenv("LD_BIND_NOW", "1", 0); - - /* Set sane defaults for ASAN if nothing else specified. */ - - setenv("ASAN_OPTIONS", "abort_on_error=1:" - "detect_leaks=0:" - "symbolize=0:" - "allocator_may_return_null=1", 0); - - /* MSAN is tricky, because it doesn't support abort_on_error=1 at this - point. So, we do this in a very hacky way. */ - - setenv("MSAN_OPTIONS", "exit_code=" STRINGIFY(MSAN_ERROR) ":" - "symbolize=0:" - "abort_on_error=1:" - "allocator_may_return_null=1:" - "msan_track_origins=0", 0); - - execv(target_path, argv); - - /* Use a distinctive bitmap signature to tell the parent about execv() - falling through. */ - - *(u32*)trace_bits = EXEC_FAIL_SIG; - exit(0); - - } - - /* PARENT PROCESS */ - - /* Close the unneeded endpoints. */ - - close(ctl_pipe[0]); - close(st_pipe[1]); - - fsrv_ctl_fd = ctl_pipe[1]; - fsrv_st_fd = st_pipe[0]; - - /* Wait for the fork server to come up, but don't wait too long. */ - - it.it_value.tv_sec = ((exec_tmout * FORK_WAIT_MULT) / 1000); - it.it_value.tv_usec = ((exec_tmout * FORK_WAIT_MULT) % 1000) * 1000; - - setitimer(ITIMER_REAL, &it, NULL); - - rlen = read(fsrv_st_fd, &status, 4); - - it.it_value.tv_sec = 0; - it.it_value.tv_usec = 0; - - setitimer(ITIMER_REAL, &it, NULL); - - /* If we have a four-byte "hello" message from the server, we're all set. - Otherwise, try to figure out what went wrong. */ - - if (rlen == 4) { - OKF("All right - fork server is up."); - return; - } - - if (child_timed_out) - FATAL("Timeout while initializing fork server (adjusting -t may help)"); - - if (waitpid(forksrv_pid, &status, 0) <= 0) - PFATAL("waitpid() failed"); - - if (WIFSIGNALED(status)) { - - if (mem_limit && mem_limit < 500 && uses_asan) { - - SAYF("\n" cLRD "[-] " cRST - "Whoops, the target binary crashed suddenly, before receiving any input\n" - " from the fuzzer! Since it seems to be built with ASAN and you have a\n" - " restrictive memory limit configured, this is expected; please read\n" - " %s/notes_for_asan.txt for help.\n", doc_path); - - } else if (!mem_limit) { - -#ifdef __APPLE__ -#define MSG_FORK_ON_APPLE \ - " - On MacOS X, the semantics of fork() syscalls are non-standard and may\n" \ - " break afl-fuzz performance optimizations when running platform-specific\n" \ - " targets. To fix this, set AFL_NO_FORKSRV=1 in the environment.\n\n" -#else -#define MSG_FORK_ON_APPLE "" -#endif - - SAYF("\n" cLRD "[-] " cRST - "Whoops, the target binary crashed suddenly, before receiving any input\n" - " from the fuzzer! There are several probable explanations:\n\n" - - " - The binary is just buggy and explodes entirely on its own. If so, you\n" - " need to fix the underlying problem or find a better replacement.\n\n" - - MSG_FORK_ON_APPLE - - " - Less likely, there is a horrible bug in the fuzzer. If other options\n" - " fail, poke for troubleshooting tips.\n"); - - } else { - -#ifdef RLIMIT_AS -#define MSG_ULIMIT_USAGE \ - " ( ulimit -Sv $[%llu << 10];" -#else -#define MSG_ULIMIT_USAGE \ - " ( ulimit -Sd $[%llu << 10];" -#endif /* ^RLIMIT_AS */ - - SAYF("\n" cLRD "[-] " cRST - "Whoops, the target binary crashed suddenly, before receiving any input\n" - " from the fuzzer! There are several probable explanations:\n\n" - - " - The current memory limit (%s) is too restrictive, causing the\n" - " target to hit an OOM condition in the dynamic linker. Try bumping up\n" - " the limit with the -m setting in the command line. A simple way confirm\n" - " this diagnosis would be:\n\n" - - MSG_ULIMIT_USAGE " /path/to/fuzzed_app )\n\n" - - " Tip: you can use http://jwilk.net/software/recidivm to quickly\n" - " estimate the required amount of virtual memory for the binary.\n\n" - - " - The binary is just buggy and explodes entirely on its own. If so, you\n" - " need to fix the underlying problem or find a better replacement.\n\n" - - MSG_FORK_ON_APPLE - - " - Less likely, there is a horrible bug in the fuzzer. If other options\n" - " fail, poke for troubleshooting tips.\n", - DMS(mem_limit << 20), mem_limit - 1); - - } - - - FATAL("Fork server crashed with signal %d", WTERMSIG(status)); - - } - - if (*(u32*)trace_bits == EXEC_FAIL_SIG) - FATAL("Unable to execute target application ('%s')", argv[0]); - - if (mem_limit && mem_limit < 500 && uses_asan) { - - SAYF("\n" cLRD "[-] " cRST - "Hmm, looks like the target binary terminated before we could complete a\n" - " handshake with the injected code. Since it seems to be built with ASAN and\n" - " you have a restrictive memory limit configured, this is expected; please\n" - " read %s/notes_for_asan.txt for help.\n", doc_path); - - } else if (!mem_limit) { - - SAYF("\n" cLRD "[-] " cRST - "Hmm, looks like the target binary terminated before we could complete a\n" - " handshake with the injected code. Perhaps there is a horrible bug in the\n" - " fuzzer. Poke for troubleshooting tips.\n"); - - } else { - - SAYF("\n" cLRD "[-] " cRST - "Hmm, looks like the target binary terminated before we could complete a\n" - " handshake with the injected code. There are %s probable explanations:\n\n" - - "%s" - " - The current memory limit (%s) is too restrictive, causing an OOM\n" - " fault in the dynamic linker. This can be fixed with the -m option. A\n" - " simple way to confirm the diagnosis may be:\n\n" - - MSG_ULIMIT_USAGE " /path/to/fuzzed_app )\n\n" - - " Tip: you can use http://jwilk.net/software/recidivm to quickly\n" - " estimate the required amount of virtual memory for the binary.\n\n" - - " - Less likely, there is a horrible bug in the fuzzer. If other options\n" - " fail, poke for troubleshooting tips.\n", - getenv(DEFER_ENV_VAR) ? "three" : "two", - getenv(DEFER_ENV_VAR) ? - " - You are using deferred forkserver, but __AFL_INIT() is never\n" - " reached before the program terminates.\n\n" : "", - DMS(mem_limit << 20), mem_limit - 1); - - } - - FATAL("Fork server handshake failed"); - -} - /* Execute target application, monitoring for timeouts. Return status information. The called program will update trace_bits[]. */ @@ -5165,6 +4873,12 @@ static u32 calculate_score(struct queue_entry* q) { global average. Multiplier ranges from 0.1x to 3x. Fast inputs are less expensive to fuzz, so we're giving them more air time. */ + // TODO BUG FIXME: is this really a good idea? + // This sounds like looking for lost keys under a street light just because + // the light is better there. + // Longer execution time means longer work on the input, the deeper in + // coverage, the better the fuzzing, right? -mh + if (q->exec_us * 0.1 > avg_exec_us) perf_score = 10; else if (q->exec_us * 0.25 > avg_exec_us) perf_score = 25; else if (q->exec_us * 0.5 > avg_exec_us) perf_score = 50; @@ -5188,15 +4902,11 @@ static u32 calculate_score(struct queue_entry* q) { for a bit longer until they catch up with the rest. */ if (q->handicap >= 4) { - perf_score *= 4; q->handicap -= 4; - } else if (q->handicap) { - perf_score *= 2; --q->handicap; - } /* Final adjustment based on input depth, under the assumption that fuzzing @@ -11041,24 +10751,6 @@ static void handle_skipreq(int sig) { } -/* Handle timeout (SIGALRM). */ - -static void handle_timeout(int sig) { - - if (child_pid > 0) { - - child_timed_out = 1; - kill(child_pid, SIGKILL); - - } else if (child_pid == -1 && forksrv_pid > 0) { - - child_timed_out = 1; - kill(forksrv_pid, SIGKILL); - - } - -} - /* Do a PATH search and find target binary to see that it exists and isn't a shell script - a common and painful mistake. We also check for @@ -12443,9 +12135,7 @@ int main(int argc, char** argv) { #ifdef USE_PYTHON if (init_py()) FATAL("Failed to initialize Python module"); - u8 with_python_support = 1; #else - if (getenv("AFL_PYTHON_MODULE")) FATAL("Your AFL binary was built without Python support"); #endif diff --git a/afl-sharedmem.c b/afl-sharedmem.c new file mode 100644 index 00000000..400a0a46 --- /dev/null +++ b/afl-sharedmem.c @@ -0,0 +1,137 @@ +/* + + */ + +#define AFL_MAIN + +#include "config.h" +#include "types.h" +#include "debug.h" +#include "alloc-inl.h" +#include "hash.h" +#include "afl-sharedmem.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifndef USEMMAP + #include + #include +#endif + +extern unsigned char*trace_bits; + +#ifdef USEMMAP +/* ================ Proteas ================ */ +int g_shm_fd = -1; +unsigned char *g_shm_base = NULL; +char g_shm_file_path[L_tmpnam]; +/* ========================================= */ +#else +static s32 shm_id; /* ID of the SHM region */ +#endif + +/* Get rid of shared memory (atexit handler). */ + +void remove_shm(void) { +#ifdef USEMMAP + if (g_shm_base != NULL) { + munmap(g_shm_base, MAP_SIZE); + g_shm_base = NULL; + } + + if (g_shm_fd != -1) { + close(g_shm_fd); + g_shm_fd = -1; + } +#else + shmctl(shm_id, IPC_RMID, NULL); +#endif +} + + +/* Configure shared memory. */ + +void setup_shm(unsigned char dumb_mode) { +#ifdef USEMMAP + /* generate random file name for multi instance */ + + /* thanks to f*cking glibc we can not use tmpnam securely, it generates a security warning that cannot be suppressed */ + /* so we do this worse workaround */ + snprintf(g_shm_file_path, L_tmpnam, "/afl_%d_%ld", getpid(), random()); + + /* create the shared memory segment as if it was a file */ + g_shm_fd = shm_open(g_shm_file_path, O_CREAT | O_RDWR | O_EXCL, 0600); + if (g_shm_fd == -1) { + PFATAL("shm_open() failed"); + } + + /* configure the size of the shared memory segment */ + if (ftruncate(g_shm_fd, MAP_SIZE)) { + PFATAL("setup_shm(): ftruncate() failed"); + } + + /* map the shared memory segment to the address space of the process */ + g_shm_base = mmap(0, MAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, g_shm_fd, 0); + if (g_shm_base == MAP_FAILED) { + close(g_shm_fd); + g_shm_fd = -1; + PFATAL("mmap() failed"); + } + + atexit(remove_shm); + + /* If somebody is asking us to fuzz instrumented binaries in dumb mode, + we don't want them to detect instrumentation, since we won't be sending + fork server commands. This should be replaced with better auto-detection + later on, perhaps? */ + + if (!dumb_mode) setenv(SHM_ENV_VAR, g_shm_file_path, 1); + + trace_bits = g_shm_base; + + if (!trace_bits) PFATAL("mmap() failed"); + +#else + u8* shm_str; + + shm_id = shmget(IPC_PRIVATE, MAP_SIZE, IPC_CREAT | IPC_EXCL | 0600); + + if (shm_id < 0) PFATAL("shmget() failed"); + + atexit(remove_shm); + + shm_str = alloc_printf("%d", shm_id); + + setenv(SHM_ENV_VAR, shm_str, 1); + + /* If somebody is asking us to fuzz instrumented binaries in dumb mode, + we don't want them to detect instrumentation, since we won't be sending + fork server commands. This should be replaced with better auto-detection + later on, perhaps? */ + + if (!dumb_mode) setenv(SHM_ENV_VAR, shm_str, 1); + + ck_free(shm_str); + + trace_bits = shmat(shm_id, NULL, 0); + + if (!trace_bits) PFATAL("shmat() failed"); + +#endif +} + diff --git a/afl-sharedmem.h b/afl-sharedmem.h new file mode 100644 index 00000000..9aa44d0e --- /dev/null +++ b/afl-sharedmem.h @@ -0,0 +1,7 @@ +#ifndef __AFL_SHAREDMEM_H +#define __AFL_SHAREDMEM_H + +void setup_shm(unsigned char dumb_mode); +void remove_shm(void); + +#endif diff --git a/afl-showmap.c b/afl-showmap.c index af3b36ee..96b7b5e0 100644 --- a/afl-showmap.c +++ b/afl-showmap.c @@ -28,7 +28,7 @@ #include "debug.h" #include "alloc-inl.h" #include "hash.h" -#include "sharedmem.h" +#include "afl-sharedmem.h" #include "afl-common.h" #include diff --git a/afl-tmin.c b/afl-tmin.c index 09ce8c62..e83b217d 100644 --- a/afl-tmin.c +++ b/afl-tmin.c @@ -21,12 +21,14 @@ #define AFL_MAIN + #include "config.h" #include "types.h" #include "debug.h" #include "alloc-inl.h" #include "hash.h" -#include "sharedmem.h" +#include "afl-forkserver.h" +#include "afl-sharedmem.h" #include "afl-common.h" #include @@ -46,22 +48,22 @@ #include #include -static s32 forksrv_pid, /* PID of the fork server */ - child_pid; /* PID of the tested program */ +s32 forksrv_pid, /* PID of the fork server */ + child_pid; /* PID of the tested program */ -static s32 fsrv_ctl_fd, /* Fork server control pipe (write) */ - fsrv_st_fd; /* Fork server status pipe (read) */ +s32 fsrv_ctl_fd, /* Fork server control pipe (write) */ + fsrv_st_fd; /* Fork server status pipe (read) */ u8 *trace_bits; /* SHM with instrumentation bitmap */ static u8 *mask_bitmap; /* Mask for trace bits (-B) */ -static u8 *in_file, /* Minimizer input test case */ - *out_file, /* Minimizer output file */ - *prog_in, /* Targeted program input file */ + u8 *in_file, /* Minimizer input test case */ + *output_file, /* Minimizer output file */ + *out_file, /* Targeted program input file */ *target_path, /* Path to target binary */ *doc_path; /* Path to docs */ -static s32 prog_in_fd; /* Persistent fd for prog_in */ + s32 out_fd; /* Persistent fd for out_file */ static u8* in_data; /* Input data for trimming */ @@ -70,12 +72,12 @@ static u32 in_len, /* Input data length */ total_execs, /* Total number of execs */ missed_hangs, /* Misses due to hangs */ missed_crashes, /* Misses due to crashes */ - missed_paths, /* Misses due to exec path diffs */ - exec_tmout = EXEC_TIMEOUT; /* Exec timeout (ms) */ + missed_paths; /* Misses due to exec path diffs */ + u32 exec_tmout = EXEC_TIMEOUT; /* Exec timeout (ms) */ -static u64 mem_limit = MEM_LIMIT; /* Memory limit (MB) */ + u64 mem_limit = MEM_LIMIT; /* Memory limit (MB) */ -static s32 dev_null_fd = -1; /* FD to /dev/null */ + s32 dev_null_fd = -1; /* FD to /dev/null */ static u8 crash_mode, /* Crash-centric mode? */ exit_crash, /* Treat non-zero exit as crash? */ @@ -84,8 +86,19 @@ static u8 crash_mode, /* Crash-centric mode? */ use_stdin = 1; /* Use stdin for program input? */ static volatile u8 - stop_soon, /* Ctrl-C pressed? */ - child_timed_out; /* Child timed out? */ + stop_soon; /* Ctrl-C pressed? */ + +/* + * forkserver section + */ + +/* we only need this to use afl-forkserver */ +FILE *plot_file; +u8 uses_asan; +s32 out_fd = -1, out_dir_fd = -1, dev_urandom_fd = -1; + +/* we import this as we need this information */ +extern u8 child_timed_out; /* Classify tuple counts. This is a slow & naive version, but good enough here. */ @@ -163,7 +176,7 @@ static inline u8 anything_set(void) { /* Get rid of temp files (atexit handler). */ static void at_exit_handler(void) { - if (prog_in) unlink(prog_in); /* Ignore errors */ + if (out_file) unlink(out_file); /* Ignore errors */ } /* Read initial file. */ @@ -214,24 +227,24 @@ static s32 write_to_file(u8* path, u8* mem, u32 len) { } /* Write modified data to file for testing. If use_stdin is clear, the old file - is unlinked and a new one is created. Otherwise, prog_in_fd is rewound and + is unlinked and a new one is created. Otherwise, out_fd is rewound and truncated. */ static void write_to_testcase(void* mem, u32 len) { - s32 fd = prog_in_fd; + s32 fd = out_fd; if (!use_stdin) { - unlink(prog_in); /* Ignore errors. */ + unlink(out_file); /* Ignore errors. */ - fd = open(prog_in, O_WRONLY | O_CREAT | O_EXCL, 0600); + fd = open(out_file, O_WRONLY | O_CREAT | O_EXCL, 0600); - if (fd < 0) PFATAL("Unable to create '%s'", prog_in); + if (fd < 0) PFATAL("Unable to create '%s'", out_file); } else lseek(fd, 0, SEEK_SET); - ck_write(fd, mem, len, prog_in); + ck_write(fd, mem, len, out_file); if (use_stdin) { @@ -245,7 +258,7 @@ static void write_to_testcase(void* mem, u32 len) { /* Handle timeout signal. */ - +/* static void handle_timeout(int sig) { if (child_pid > 0) { @@ -261,8 +274,10 @@ static void handle_timeout(int sig) { } } +*/ /* start the app and it's forkserver */ +/* static void init_forkserver(char **argv) { static struct itimerval it; int st_pipe[2], ctl_pipe[2]; @@ -280,7 +295,7 @@ static void init_forkserver(char **argv) { struct rlimit r; - if (dup2(use_stdin ? prog_in_fd : dev_null_fd, 0) < 0 || + if (dup2(use_stdin ? out_fd : dev_null_fd, 0) < 0 || dup2(dev_null_fd, 1) < 0 || dup2(dev_null_fd, 2) < 0) { @@ -290,7 +305,7 @@ static void init_forkserver(char **argv) { } close(dev_null_fd); - close(prog_in_fd); + close(out_fd); setsid(); @@ -300,20 +315,20 @@ static void init_forkserver(char **argv) { #ifdef RLIMIT_AS - setrlimit(RLIMIT_AS, &r); /* Ignore errors */ + setrlimit(RLIMIT_AS, &r); // Ignore errors #else - setrlimit(RLIMIT_DATA, &r); /* Ignore errors */ + setrlimit(RLIMIT_DATA, &r); // Ignore errors -#endif /* ^RLIMIT_AS */ +#endif // ^RLIMIT_AS } r.rlim_max = r.rlim_cur = 0; - setrlimit(RLIMIT_CORE, &r); /* Ignore errors */ + setrlimit(RLIMIT_CORE, &r); // Ignore errors - /* Set up control and status pipes, close the unneeded original fds. */ + // Set up control and status pipes, close the unneeded original fds. if (dup2(ctl_pipe[0], FORKSRV_FD) < 0) PFATAL("dup2() failed"); if (dup2(st_pipe[1], FORKSRV_FD + 1) < 0) PFATAL("dup2() failed"); @@ -330,7 +345,7 @@ static void init_forkserver(char **argv) { } - /* Close the unneeded endpoints. */ + // Close the unneeded endpoints. close(ctl_pipe[0]); close(st_pipe[1]); @@ -338,7 +353,7 @@ static void init_forkserver(char **argv) { fsrv_ctl_fd = ctl_pipe[1]; fsrv_st_fd = st_pipe[0]; - /* Configure timeout, wait for child, cancel timeout. */ + // Configure timeout, wait for child, cancel timeout. if (exec_tmout) { @@ -356,8 +371,8 @@ static void init_forkserver(char **argv) { it.it_value.tv_usec = 0; setitimer(ITIMER_REAL, &it, NULL); - /* If we have a four-byte "hello" message from the server, we're all set. - Otherwise, try to figure out what went wrong. */ + // If we have a four-byte "hello" message from the server, we're all set. + // Otherwise, try to figure out what went wrong. if (rlen == 4) { ACTF("All right - fork server is up."); @@ -380,7 +395,7 @@ static void init_forkserver(char **argv) { SAYF(cLRD "\n+++ Program killed by signal %u +++\n" cRST, WTERMSIG(status)); } - +*/ /* Execute target application. Returns 0 if the changes are a dud, or 1 if they should be kept. */ @@ -422,11 +437,8 @@ static u8 run_target(char** argv, u8* mem, u32 len, u8 first_run) { /* Configure timeout, wait for child, cancel timeout. */ if (exec_tmout) { - - child_timed_out = 0; - it.it_value.tv_sec = (exec_tmout / 1000); - it.it_value.tv_usec = (exec_tmout % 1000) * 1000; - + it.it_value.tv_sec = (exec_tmout / 1000); + it.it_value.tv_usec = (exec_tmout % 1000) * 1000; } setitimer(ITIMER_REAL, &it, NULL); @@ -458,7 +470,7 @@ static u8 run_target(char** argv, u8* mem, u32 len, u8 first_run) { if (stop_soon) { SAYF(cRST cLRD "\n+++ Minimization aborted by user +++\n" cRST); - close(write_to_file(out_file, in_data, in_len)); + close(write_to_file(output_file, in_data, in_len)); exit(1); } @@ -787,7 +799,7 @@ static void set_up_environment(void) { dev_null_fd = open("/dev/null", O_RDWR); if (dev_null_fd < 0) PFATAL("Unable to open /dev/null"); - if (!prog_in) { + if (!out_file) { u8* use_dir = "."; @@ -798,15 +810,15 @@ static void set_up_environment(void) { } - prog_in = alloc_printf("%s/.afl-tmin-temp-%u", use_dir, getpid()); + out_file = alloc_printf("%s/.afl-tmin-temp-%u", use_dir, getpid()); } - unlink(prog_in); + unlink(out_file); - prog_in_fd = open(prog_in, O_RDWR | O_CREAT | O_EXCL, 0600); + out_fd = open(out_file, O_RDWR | O_CREAT | O_EXCL, 0600); - if (prog_in_fd < 0) PFATAL("Unable to create '%s'", prog_in); + if (out_fd < 0) PFATAL("Unable to create '%s'", out_file); /* Set sane defaults... */ @@ -1067,15 +1079,15 @@ int main(int argc, char** argv) { case 'o': - if (out_file) FATAL("Multiple -o options not supported"); - out_file = optarg; + if (output_file) FATAL("Multiple -o options not supported"); + output_file = optarg; break; case 'f': - if (prog_in) FATAL("Multiple -f options not supported"); + if (out_file) FATAL("Multiple -f options not supported"); use_stdin = 0; - prog_in = optarg; + out_file = optarg; break; case 'e': @@ -1181,7 +1193,7 @@ int main(int argc, char** argv) { } - if (optind == argc || !in_file || !out_file) usage(argv[0]); + if (optind == argc || !in_file || !output_file) usage(argv[0]); setup_shm(0); atexit(at_exit_handler); @@ -1190,7 +1202,7 @@ int main(int argc, char** argv) { set_up_environment(); find_binary(argv[optind]); - detect_file_args(argv + optind, prog_in); + detect_file_args(argv + optind, out_file); if (qemu_mode) use_argv = get_qemu_argv(argv[0], argv + optind, argc - optind); @@ -1229,12 +1241,12 @@ int main(int argc, char** argv) { minimize(use_argv); - ACTF("Writing output to '%s'...", out_file); + ACTF("Writing output to '%s'...", output_file); - unlink(prog_in); - prog_in = NULL; + unlink(out_file); + out_file = NULL; - close(write_to_file(out_file, in_data, in_len)); + close(write_to_file(output_file, in_data, in_len)); OKF("We're done here. Have a nice day!\n"); diff --git a/docs/ChangeLog b/docs/ChangeLog index dfebb68a..ed8e0022 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -20,12 +20,13 @@ Version ++2.53d (dev): - llvm 9 is now supported (still needs testing) - fix building qemu on some Ubuntus (thanks to floyd!) - custom mutator by a loaded library is now supported (thanks to kyakdan!) + - more support for *BSD (thanks to devnexen!) + - fix building on *BSD (thanks to tobias.kortkamp for the patch) - fix for a few features to support different map sized than 2^16 - afl-showmap: new option -r now shows the real values in the buckets (stock afl never did), plus shows tuple content summary information now - - fix building on *BSD (thanks to tobias.kortkamp for the patch) + - the forkserver is now in its own C file to be easily integratable - small docu updates - - ... your patch? :) -------------------------- diff --git a/sharedmem.c b/sharedmem.c deleted file mode 100644 index 3fd38444..00000000 --- a/sharedmem.c +++ /dev/null @@ -1,137 +0,0 @@ -/* - - */ - -#define AFL_MAIN - -#include "config.h" -#include "types.h" -#include "debug.h" -#include "alloc-inl.h" -#include "hash.h" -#include "sharedmem.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#ifndef USEMMAP - #include - #include -#endif - -extern unsigned char*trace_bits; - -#ifdef USEMMAP -/* ================ Proteas ================ */ -int g_shm_fd = -1; -unsigned char *g_shm_base = NULL; -char g_shm_file_path[L_tmpnam]; -/* ========================================= */ -#else -static s32 shm_id; /* ID of the SHM region */ -#endif - -/* Get rid of shared memory (atexit handler). */ - -void remove_shm(void) { -#ifdef USEMMAP - if (g_shm_base != NULL) { - munmap(g_shm_base, MAP_SIZE); - g_shm_base = NULL; - } - - if (g_shm_fd != -1) { - close(g_shm_fd); - g_shm_fd = -1; - } -#else - shmctl(shm_id, IPC_RMID, NULL); -#endif -} - - -/* Configure shared memory. */ - -void setup_shm(unsigned char dumb_mode) { -#ifdef USEMMAP - /* generate random file name for multi instance */ - - /* thanks to f*cking glibc we can not use tmpnam securely, it generates a security warning that cannot be suppressed */ - /* so we do this worse workaround */ - snprintf(g_shm_file_path, L_tmpnam, "/afl_%d_%ld", getpid(), random()); - - /* create the shared memory segment as if it was a file */ - g_shm_fd = shm_open(g_shm_file_path, O_CREAT | O_RDWR | O_EXCL, 0600); - if (g_shm_fd == -1) { - PFATAL("shm_open() failed"); - } - - /* configure the size of the shared memory segment */ - if (ftruncate(g_shm_fd, MAP_SIZE)) { - PFATAL("setup_shm(): ftruncate() failed"); - } - - /* map the shared memory segment to the address space of the process */ - g_shm_base = mmap(0, MAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, g_shm_fd, 0); - if (g_shm_base == MAP_FAILED) { - close(g_shm_fd); - g_shm_fd = -1; - PFATAL("mmap() failed"); - } - - atexit(remove_shm); - - /* If somebody is asking us to fuzz instrumented binaries in dumb mode, - we don't want them to detect instrumentation, since we won't be sending - fork server commands. This should be replaced with better auto-detection - later on, perhaps? */ - - if (!dumb_mode) setenv(SHM_ENV_VAR, g_shm_file_path, 1); - - trace_bits = g_shm_base; - - if (!trace_bits) PFATAL("mmap() failed"); - -#else - u8* shm_str; - - shm_id = shmget(IPC_PRIVATE, MAP_SIZE, IPC_CREAT | IPC_EXCL | 0600); - - if (shm_id < 0) PFATAL("shmget() failed"); - - atexit(remove_shm); - - shm_str = alloc_printf("%d", shm_id); - - setenv(SHM_ENV_VAR, shm_str, 1); - - /* If somebody is asking us to fuzz instrumented binaries in dumb mode, - we don't want them to detect instrumentation, since we won't be sending - fork server commands. This should be replaced with better auto-detection - later on, perhaps? */ - - if (!dumb_mode) setenv(SHM_ENV_VAR, shm_str, 1); - - ck_free(shm_str); - - trace_bits = shmat(shm_id, NULL, 0); - - if (!trace_bits) PFATAL("shmat() failed"); - -#endif -} - diff --git a/sharedmem.h b/sharedmem.h deleted file mode 100644 index 53a85fcb..00000000 --- a/sharedmem.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef __SHAREDMEM_H -#define __SHAREDMEM_H - -void setup_shm(unsigned char dumb_mode); -void remove_shm(void); -#endif -- cgit 1.4.1 From b1ebd62c78e81bcd0731782f102276e4af459cea Mon Sep 17 00:00:00 2001 From: Andrea Fioraldi Date: Wed, 21 Aug 2019 09:57:26 +0200 Subject: update env_variables.txt with compcov levels --- docs/env_variables.txt | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'docs') diff --git a/docs/env_variables.txt b/docs/env_variables.txt index 93066dbc..821463ae 100644 --- a/docs/env_variables.txt +++ b/docs/env_variables.txt @@ -245,9 +245,19 @@ The QEMU wrapper used to instrument binary-only code supports several settings: - Setting AFL_INST_LIBS causes the translator to also instrument the code inside any dynamically linked libraries (notably including glibc). + - Setting AFL_COMPCOV_LEVEL enables the CompareCoverage tracing of all cmp + and sub in x86 and x86_64 and memory comparions functions (e.g. strcmp, + memcmp, ...) when libcompcov is preloaded using AFL_PRELOAD. + More info at qemu_mode/libcompcov/README.compcov. + There are two levels at the moment, AFL_COMPCOV_LEVEL=1 that instruments + only comparisons with immediate values / read-only memory and + AFL_COMPCOV_LEVEL=2 that instruments all the comparions. Level 2 is more + accurate but may need a larger shared memory. + - Setting AFL_QEMU_COMPCOV enables the CompareCoverage tracing of all - cmp and sub in x86 and x86_64. Support for other architectures and - comparison functions (mem/strcmp et al.) is planned. + cmp and sub in x86 and x86_64. + This is an alias of AFL_COMPCOV_LEVEL=1 when AFL_COMPCOV_LEVEL is + not specified. - The underlying QEMU binary will recognize any standard "user space emulation" variables (e.g., QEMU_STACK_SIZE), but there should be no @@ -260,10 +270,7 @@ The QEMU wrapper used to instrument binary-only code supports several settings: - AFL_ENTRYPOINT allows you to specify a specific entrypoint into the binary (this can be very good for the performance!). The entrypoint is specified as hex address, e.g. 0x4004110 - - - AFL_QEMU_COMPCOV is for a sub-project in qemu_mode called ./libcompcov - which implements laf-intel for qemu. It also needs AFL_PRELOAD and - you can find more information in qemu_mode/libcompcov/README.compcov + Note that the address must be the address of a basic block. 5) Settings for afl-cmin ------------------------ -- cgit 1.4.1 From 10df5ad0ac3dcff705f6932487fecbdaf690e1f0 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Tue, 27 Aug 2019 16:22:25 +0200 Subject: docu update --- README.md | 3 ++- TODO | 2 ++ docs/ChangeLog | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) (limited to 'docs') diff --git a/README.md b/README.md index 2bd31a54..14e1ae59 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,8 @@ Among others afl++ has, e.g. more performant llvm_mode, supporting llvm up to version 9, Qemu 3.1, more speed and crashfixes for Qemu, - laf-intel feature for Qemu (with libcompcov) and more. + laf-intel feature for Qemu (with libcompcov), better *BSD and Android + support and more. Additionally the following patches have been integrated: diff --git a/TODO b/TODO index 89e307cf..cb95f899 100644 --- a/TODO +++ b/TODO @@ -7,6 +7,8 @@ Roadmap 2.53d: afl-fuzz: - put mutator, scheduler, forkserver and input channels in individual files - reuse forkserver for showmap, afl-cmin, etc. + - custom mutator lib: example and readme + - env var to exclusively run the custom lib/py mutator gcc_plugin: - needs to be rewritten diff --git a/docs/ChangeLog b/docs/ChangeLog index dfebb68a..6d56d314 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -18,6 +18,7 @@ Version ++2.53d (dev): ---------------------- - llvm 9 is now supported (still needs testing) + - Android is now supported (thank to JoeyJiao!) - still need to modify the Makefile though - fix building qemu on some Ubuntus (thanks to floyd!) - custom mutator by a loaded library is now supported (thanks to kyakdan!) - fix for a few features to support different map sized than 2^16 -- cgit 1.4.1 From 7b36afd5f16894257c92695d200e59eb51d08e1c Mon Sep 17 00:00:00 2001 From: Andrea Fioraldi Date: Fri, 30 Aug 2019 11:38:33 +0200 Subject: modernize docs and readme for qemu and unicorn --- docs/unicorn_mode.txt | 109 --------------- qemu_mode/README.md | 137 +++++++++++++++++++ qemu_mode/README.qemu | 146 --------------------- qemu_mode/build_qemu_support.sh | 11 +- qemu_mode/libcompcov/libcompcov.so.c | 2 +- qemu_mode/patches/afl-qemu-common.h | 15 +-- qemu_mode/patches/afl-qemu-cpu-inl.h | 15 +-- qemu_mode/patches/afl-qemu-cpu-translate-inl.h | 17 ++- qemu_mode/patches/afl-qemu-tcg-inl.h | 15 +-- qemu_mode/patches/afl-qemu-translate-inl.h | 15 +-- unicorn_mode/README.md | 130 +++++++++++++++--- unicorn_mode/build_unicorn_support.sh | 12 +- unicorn_mode/patches/afl-unicorn-common.h | 18 +-- unicorn_mode/patches/afl-unicorn-cpu-inl.h | 18 +-- .../patches/afl-unicorn-cpu-translate-inl.h | 18 +-- unicorn_mode/patches/afl-unicorn-tcg-op-inl.h | 18 +-- unicorn_mode/patches/afl-unicorn-tcg-runtime-inl.h | 18 +-- unicorn_mode/samples/compcov_x64/COMPILE.md | 3 +- unicorn_mode/samples/compcov_x64/compcov_target.c | 2 +- unicorn_mode/samples/simple/COMPILE.md | 5 +- 20 files changed, 353 insertions(+), 371 deletions(-) delete mode 100644 docs/unicorn_mode.txt create mode 100644 qemu_mode/README.md delete mode 100644 qemu_mode/README.qemu (limited to 'docs') diff --git a/docs/unicorn_mode.txt b/docs/unicorn_mode.txt deleted file mode 100644 index b691fff8..00000000 --- a/docs/unicorn_mode.txt +++ /dev/null @@ -1,109 +0,0 @@ -========================================================= -Unicorn-based binary-only instrumentation for afl-fuzz -========================================================= - -1) Introduction ---------------- - -The code in ./unicorn_mode allows you to build a standalone feature that -leverages the Unicorn Engine and allows callers to obtain instrumentation -output for black-box, closed-source binary code snippets. This mechanism -can be then used by afl-fuzz to stress-test targets that couldn't be built -with afl-gcc or used in QEMU mode, or with other extensions such as -TriforceAFL. - -There is a significant performance penalty compared to native AFL, -but at least we're able to use AFL on these binaries, right? - -The idea and much of the implementation comes from Nathan Voss . - -2) How to use -------------- - -Requirements: you need an installed python2 environment. - -*** Building AFL's Unicorn Mode *** - -First, make afl as usual. -Once that completes successfully you need to build and add in the Unicorn Mode -features: - - $ cd unicorn_mode - $ ./build_unicorn_support.sh - -NOTE: This script downloads a recent Unicorn Engine commit that has been tested -and is stable-ish from the Unicorn github page. If you are offline, you'll need -to hack up this script a little bit and supply your own copy of Unicorn's latest -stable release. It's not very hard, just check out the beginning of the -build_unicorn_support.sh script and adjust as necessary. - -Building Unicorn will take a little bit (~5-10 minutes). Once it completes -it automatically compiles a sample application and verify that it works. - -*** Fuzzing with Unicorn Mode *** - -To really use unicorn-mode effectively you need to prepare the following: - - * Relevant binary code to be fuzzed - * Knowledge of the memory map and good starting state - * Folder containing sample inputs to start fuzzing with - - Same ideas as any other AFL inputs - - Quality/speed of results will depend greatly on quality of starting - samples - - See AFL's guidance on how to create a sample corpus - * Unicorn-based test harness which: - - Adds memory map regions - - Loads binary code into memory - - Emulates at least one instruction* - - Yeah, this is lame. See 'Gotchas' section below for more info - - Loads and verifies data to fuzz from a command-line specified file - - AFL will provide mutated inputs by changing the file passed to - the test harness - - Presumably the data to be fuzzed is at a fixed buffer address - - If input constraints (size, invalid bytes, etc.) are known they - should be checked after the file is loaded. If a constraint - fails, just exit the test harness. AFL will treat the input as - 'uninteresting' and move on. - - Sets up registers and memory state for beginning of test - - Emulates the interested code from beginning to end - - If a crash is detected, the test harness must 'crash' by - throwing a signal (SIGSEGV, SIGKILL, SIGABORT, etc.) - -Once you have all those things ready to go you just need to run afl-fuzz in -'unicorn-mode' by passing in the '-U' flag: - - $ afl-fuzz -U -m none -i /path/to/inputs -o /path/to/results -- ./test_harness @@ - -The normal afl-fuzz command line format applies to everything here. Refer to -AFL's main documentation for more info about how to use afl-fuzz effectively. - -For a much clearer vision of what all of this looks like, please refer to the -sample provided in the 'unicorn_mode/samples' directory. There is also a blog -post that goes over the basics at: - -https://medium.com/@njvoss299/afl-unicorn-fuzzing-arbitrary-binary-code-563ca28936bf - -The 'helper_scripts' directory also contains several helper scripts that allow you -to dump context from a running process, load it, and hook heap allocations. For details -on how to use this check out the follow-up blog post to the one linked above. - -A example use of AFL-Unicorn mode is discussed in the Paper Unicorefuzz: -https://www.usenix.org/conference/woot19/presentation/maier - -3) Gotchas, feedback, bugs --------------------------- - -To make sure that AFL's fork server starts up correctly the Unicorn test -harness script must emulate at least one instruction before loading the -data that will be fuzzed from the input file. It doesn't matter what the -instruction is, nor if it is valid. This is an artifact of how the fork-server -is started and could likely be fixed with some clever re-arranging of the -patches applied to Unicorn. - -Running the build script builds Unicorn and its python bindings and installs -them on your system. This installation will supersede any existing Unicorn -installation with the patched afl-unicorn version. - -Refer to the unicorn_mode/samples/arm_example/arm_tester.c for an example -of how to do this properly! If you don't get this right, AFL will not -load any mutated inputs and your fuzzing will be useless! diff --git a/qemu_mode/README.md b/qemu_mode/README.md new file mode 100644 index 00000000..610f6860 --- /dev/null +++ b/qemu_mode/README.md @@ -0,0 +1,137 @@ +# High-performance binary-only instrumentation for afl-fuzz + + (See ../docs/README for the general instruction manual.) + +## 1) Introduction + +The code in this directory allows you to build a standalone feature that +leverages the QEMU "user emulation" mode and allows callers to obtain +instrumentation output for black-box, closed-source binaries. This mechanism +can be then used by afl-fuzz to stress-test targets that couldn't be built +with afl-gcc. + +The usual performance cost is 2-5x, which is considerably better than +seen so far in experiments with tools such as DynamoRIO and PIN. + +The idea and much of the initial implementation comes from Andrew Griffiths. +The actual implementation on QEMU 3 (shipped with afl++) is from +Andrea Fioraldi. Special thanks to abiondo that re-enabled TCG chaining. + +## 2) How to use + +The feature is implemented with a patch to QEMU 3.1.0. The simplest way +to build it is to run ./build_qemu_support.sh. The script will download, +configure, and compile the QEMU binary for you. + +QEMU is a big project, so this will take a while, and you may have to +resolve a couple of dependencies (most notably, you will definitely need +libtool and glib2-devel). + +Once the binaries are compiled, you can leverage the QEMU tool by calling +afl-fuzz and all the related utilities with -Q in the command line. + +Note that QEMU requires a generous memory limit to run; somewhere around +200 MB is a good starting point, but considerably more may be needed for +more complex programs. The default -m limit will be automatically bumped up +to 200 MB when specifying -Q to afl-fuzz; be careful when overriding this. + +In principle, if you set CPU_TARGET before calling ./build_qemu_support.sh, +you should get a build capable of running non-native binaries (say, you +can try CPU_TARGET=arm). This is also necessary for running 32-bit binaries +on a 64-bit system (CPU_TARGET=i386). + +Note: if you want the QEMU helper to be installed on your system for all +users, you need to build it before issuing 'make install' in the parent +directory. + +## 3) Options + +There is ./libcompcov/ which implements laf-intel (splitting memcmp, +strncmp, etc. to make these conditions easier solvable by afl-fuzz). +Highly recommended. + +The option that enables QEMU CompareCoverage is AFL_COMPCOV_LEVEL. +AFL_COMPCOV_LEVEL=1 is to instrument comparisons with only immediate +values / read-only memory. AFL_COMPCOV_LEVEL=2 instruments all +comparison instructions and memory comparison functions when libcompcov +is preloaded. Comparison instructions are currently instrumented only +on the x86 and x86_64 targets. + +Another option is the environment variable AFL_ENTRYPOINT which allows +move the forkserver to a different part, e.g. just before the file is +opened (e.g. way after command line parsing and config file loading, etc) +which can be a huge speed improvement. Note that the specified address +must be an address of a basic block. + +## 4) Notes on linking + +The feature is supported only on Linux. Supporting BSD may amount to porting +the changes made to linux-user/elfload.c and applying them to +bsd-user/elfload.c, but I have not looked into this yet. + +The instrumentation follows only the .text section of the first ELF binary +encountered in the linking process. It does not trace shared libraries. In +practice, this means two things: + + - Any libraries you want to analyze *must* be linked statically into the + executed ELF file (this will usually be the case for closed-source + apps). + + - Standard C libraries and other stuff that is wasteful to instrument + should be linked dynamically - otherwise, AFL will have no way to avoid + peeking into them. + +Setting AFL_INST_LIBS=1 can be used to circumvent the .text detection logic +and instrument every basic block encountered. + +## 5) Benchmarking + +If you want to compare the performance of the QEMU instrumentation with that of +afl-gcc compiled code against the same target, you need to build the +non-instrumented binary with the same optimization flags that are normally +injected by afl-gcc, and make sure that the bits to be tested are statically +linked into the binary. A common way to do this would be: + +$ CFLAGS="-O3 -funroll-loops" ./configure --disable-shared +$ make clean all + +Comparative measurements of execution speed or instrumentation coverage will be +fairly meaningless if the optimization levels or instrumentation scopes don't +match. + +## 6) Gotchas, feedback, bugs + +If you need to fix up checksums or do other cleanup on mutated test cases, see +experimental/post_library/ for a viable solution. + +Do not mix QEMU mode with ASAN, MSAN, or the likes; QEMU doesn't appreciate +the "shadow VM" trick employed by the sanitizers and will probably just +run out of memory. + +Compared to fully-fledged virtualization, the user emulation mode is *NOT* a +security boundary. The binaries can freely interact with the host OS. If you +somehow need to fuzz an untrusted binary, put everything in a sandbox first. + +QEMU does not necessarily support all CPU or hardware features that your +target program may be utilizing. In particular, it does not appear to have +full support for AVX2 / FMA3. Using binaries for older CPUs, or recompiling them +with -march=core2, can help. + +Beyond that, this is an early-stage mechanism, so fields reports are welcome. +You can send them to . + +## 7) Alternatives: static rewriting + +Statically rewriting binaries just once, instead of attempting to translate +them at run time, can be a faster alternative. That said, static rewriting is +fraught with peril, because it depends on being able to properly and fully model +program control flow without actually executing each and every code path. + +The best implementation is this one: + + https://github.com/vanhauser-thc/afl-dyninst + +The issue however is Dyninst which is not rewriting the binaries so that +they run stable. a lot of crashes happen, especially in C++ programs that +use throw/catch. Try it first, and if it works for you be happy as it is +2-3x as fast as qemu_mode. diff --git a/qemu_mode/README.qemu b/qemu_mode/README.qemu deleted file mode 100644 index cd8559ad..00000000 --- a/qemu_mode/README.qemu +++ /dev/null @@ -1,146 +0,0 @@ -========================================================= -High-performance binary-only instrumentation for afl-fuzz -========================================================= - - (See ../docs/README for the general instruction manual.) - -1) Introduction ---------------- - -The code in this directory allows you to build a standalone feature that -leverages the QEMU "user emulation" mode and allows callers to obtain -instrumentation output for black-box, closed-source binaries. This mechanism -can be then used by afl-fuzz to stress-test targets that couldn't be built -with afl-gcc. - -The usual performance cost is 2-5x, which is considerably better than -seen so far in experiments with tools such as DynamoRIO and PIN. - -The idea and much of the initial implementation comes from Andrew Griffiths. -The actual implementation on QEMU 3 (shipped with afl++) is from -Andrea Fioraldi. Special thanks to abiondo that re-enabled TCG chaining. - -2) How to use -------------- - -The feature is implemented with a patch to QEMU 3.1.0. The simplest way -to build it is to run ./build_qemu_support.sh. The script will download, -configure, and compile the QEMU binary for you. - -QEMU is a big project, so this will take a while, and you may have to -resolve a couple of dependencies (most notably, you will definitely need -libtool and glib2-devel). - -Once the binaries are compiled, you can leverage the QEMU tool by calling -afl-fuzz and all the related utilities with -Q in the command line. - -Note that QEMU requires a generous memory limit to run; somewhere around -200 MB is a good starting point, but considerably more may be needed for -more complex programs. The default -m limit will be automatically bumped up -to 200 MB when specifying -Q to afl-fuzz; be careful when overriding this. - -In principle, if you set CPU_TARGET before calling ./build_qemu_support.sh, -you should get a build capable of running non-native binaries (say, you -can try CPU_TARGET=arm). This is also necessary for running 32-bit binaries -on a 64-bit system (CPU_TARGET=i386). - -Note: if you want the QEMU helper to be installed on your system for all -users, you need to build it before issuing 'make install' in the parent -directory. - -3) Options ----------- - -There is ./libcompcov/ which implements laf-intel (splitting memcmp, -strncmp, etc. to make these conditions easier solvable by afl-fuzz). -Highly recommended. - -The option that enables QEMU CompareCoverage is QEMU_COMPCOV_LEVEL. -QEMU_COMPCOV_LEVEL=1 is to instrument comparisons with only immediate -values / read-only memory. QEMU_COMPCOV_LEVEL=2 instruments all -comparison instructions and memory comparison functions when libcompcov -is preloaded. Comparison instructions are currently instrumented only -on the x86 and x86_64 targets. - -Another option is the environment variable AFL_ENTRYPOINT which allows -move the forkserver to a different part, e.g. just before the file is -opened (e.g. way after command line parsing and config file loading, etc) -which can be a huge speed improvement. Note that the specified address -must be an address of a basic block. - -4) Notes on linking -------------------- - -The feature is supported only on Linux. Supporting BSD may amount to porting -the changes made to linux-user/elfload.c and applying them to -bsd-user/elfload.c, but I have not looked into this yet. - -The instrumentation follows only the .text section of the first ELF binary -encountered in the linking process. It does not trace shared libraries. In -practice, this means two things: - - - Any libraries you want to analyze *must* be linked statically into the - executed ELF file (this will usually be the case for closed-source - apps). - - - Standard C libraries and other stuff that is wasteful to instrument - should be linked dynamically - otherwise, AFL will have no way to avoid - peeking into them. - -Setting AFL_INST_LIBS=1 can be used to circumvent the .text detection logic -and instrument every basic block encountered. - -5) Benchmarking ---------------- - -If you want to compare the performance of the QEMU instrumentation with that of -afl-gcc compiled code against the same target, you need to build the -non-instrumented binary with the same optimization flags that are normally -injected by afl-gcc, and make sure that the bits to be tested are statically -linked into the binary. A common way to do this would be: - -$ CFLAGS="-O3 -funroll-loops" ./configure --disable-shared -$ make clean all - -Comparative measurements of execution speed or instrumentation coverage will be -fairly meaningless if the optimization levels or instrumentation scopes don't -match. - -6) Gotchas, feedback, bugs --------------------------- - -If you need to fix up checksums or do other cleanup on mutated test cases, see -experimental/post_library/ for a viable solution. - -Do not mix QEMU mode with ASAN, MSAN, or the likes; QEMU doesn't appreciate -the "shadow VM" trick employed by the sanitizers and will probably just -run out of memory. - -Compared to fully-fledged virtualization, the user emulation mode is *NOT* a -security boundary. The binaries can freely interact with the host OS. If you -somehow need to fuzz an untrusted binary, put everything in a sandbox first. - -QEMU does not necessarily support all CPU or hardware features that your -target program may be utilizing. In particular, it does not appear to have -full support for AVX2 / FMA3. Using binaries for older CPUs, or recompiling them -with -march=core2, can help. - -Beyond that, this is an early-stage mechanism, so fields reports are welcome. -You can send them to . - -7) Alternatives: static rewriting ---------------------------------- - -Statically rewriting binaries just once, instead of attempting to translate -them at run time, can be a faster alternative. That said, static rewriting is -fraught with peril, because it depends on being able to properly and fully model -program control flow without actually executing each and every code path. - -The best implementation is this one: - - https://github.com/vanhauser-thc/afl-dyninst - -The issue however is Dyninst which is not rewriting the binaries so that -they run stable. a lot of crashes happen, especially in C++ programs that -use throw/catch. Try it first, and if it works for you be happy as it is -2-3x as fast as qemu_mode. diff --git a/qemu_mode/build_qemu_support.sh b/qemu_mode/build_qemu_support.sh index 78ad5680..35f5b8ca 100755 --- a/qemu_mode/build_qemu_support.sh +++ b/qemu_mode/build_qemu_support.sh @@ -3,10 +3,17 @@ # american fuzzy lop - QEMU build script # -------------------------------------- # -# Written by Andrew Griffiths and -# Michal Zalewski +# Originally written by Andrew Griffiths and +# Michal Zalewski +# +# TCG instrumentation and block chaining support by Andrea Biondo +# +# +# QEMU 3.1.0 port, TCG thread-safety, CompareCoverage and NeverZero +# counters by Andrea Fioraldi # # Copyright 2015, 2016, 2017 Google Inc. All rights reserved. +# Copyright 2019 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/qemu_mode/libcompcov/libcompcov.so.c b/qemu_mode/libcompcov/libcompcov.so.c index 92e4dbaa..0ccda927 100644 --- a/qemu_mode/libcompcov/libcompcov.so.c +++ b/qemu_mode/libcompcov/libcompcov.so.c @@ -5,7 +5,7 @@ Written and maintained by Andrea Fioraldi - Copyright 2019 Andrea Fioraldi. All rights reserved. + Copyright 2019 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/qemu_mode/patches/afl-qemu-common.h b/qemu_mode/patches/afl-qemu-common.h index 8013800d..c475cb58 100644 --- a/qemu_mode/patches/afl-qemu-common.h +++ b/qemu_mode/patches/afl-qemu-common.h @@ -1,19 +1,18 @@ /* - american fuzzy lop - high-performance binary-only instrumentation - ----------------------------------------------------------------- + american fuzzy lop++ - high-performance binary-only instrumentation + ------------------------------------------------------------------- - Written by Andrew Griffiths and - Michal Zalewski - - Idea & design very much by Andrew Griffiths. + Originally written by Andrew Griffiths and + Michal Zalewski TCG instrumentation and block chaining support by Andrea Biondo - QEMU 3.1.0 port, TCG thread-safety and CompareCoverage by Andrea Fioraldi - + QEMU 3.1.0 port, TCG thread-safety, CompareCoverage and NeverZero + counters by Andrea Fioraldi Copyright 2015, 2016, 2017 Google Inc. All rights reserved. + Copyright 2019 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/qemu_mode/patches/afl-qemu-cpu-inl.h b/qemu_mode/patches/afl-qemu-cpu-inl.h index 04d9007d..4ad31b60 100644 --- a/qemu_mode/patches/afl-qemu-cpu-inl.h +++ b/qemu_mode/patches/afl-qemu-cpu-inl.h @@ -1,19 +1,18 @@ /* - american fuzzy lop - high-performance binary-only instrumentation - ----------------------------------------------------------------- + american fuzzy lop++ - high-performance binary-only instrumentation + ------------------------------------------------------------------- - Written by Andrew Griffiths and - Michal Zalewski - - Idea & design very much by Andrew Griffiths. + Originally written by Andrew Griffiths and + Michal Zalewski TCG instrumentation and block chaining support by Andrea Biondo - QEMU 3.1.0 port, TCG thread-safety and CompareCoverage by Andrea Fioraldi - + QEMU 3.1.0 port, TCG thread-safety, CompareCoverage and NeverZero + counters by Andrea Fioraldi Copyright 2015, 2016, 2017 Google Inc. All rights reserved. + Copyright 2019 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/qemu_mode/patches/afl-qemu-cpu-translate-inl.h b/qemu_mode/patches/afl-qemu-cpu-translate-inl.h index fc78e652..09ecb9d2 100644 --- a/qemu_mode/patches/afl-qemu-cpu-translate-inl.h +++ b/qemu_mode/patches/afl-qemu-cpu-translate-inl.h @@ -1,19 +1,18 @@ /* - american fuzzy lop - high-performance binary-only instrumentation - ----------------------------------------------------------------- + american fuzzy lop++ - high-performance binary-only instrumentation + ------------------------------------------------------------------- - Written by Andrew Griffiths and - Michal Zalewski - - Idea & design very much by Andrew Griffiths. + Originally written by Andrew Griffiths and + Michal Zalewski TCG instrumentation and block chaining support by Andrea Biondo - - QEMU 3.1.0 port, TCG thread-safety and CompareCoverage by Andrea Fioraldi - + + QEMU 3.1.0 port, TCG thread-safety, CompareCoverage and NeverZero + counters by Andrea Fioraldi Copyright 2015, 2016, 2017 Google Inc. All rights reserved. + Copyright 2019 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/qemu_mode/patches/afl-qemu-tcg-inl.h b/qemu_mode/patches/afl-qemu-tcg-inl.h index ff90d1b9..a9c53b8c 100644 --- a/qemu_mode/patches/afl-qemu-tcg-inl.h +++ b/qemu_mode/patches/afl-qemu-tcg-inl.h @@ -1,19 +1,18 @@ /* - american fuzzy lop - high-performance binary-only instrumentation - ----------------------------------------------------------------- + american fuzzy lop++ - high-performance binary-only instrumentation + ------------------------------------------------------------------- - Written by Andrew Griffiths and - Michal Zalewski - - Idea & design very much by Andrew Griffiths. + Originally written by Andrew Griffiths and + Michal Zalewski TCG instrumentation and block chaining support by Andrea Biondo - QEMU 3.1.0 port, TCG thread-safety and CompareCoverage by Andrea Fioraldi - + QEMU 3.1.0 port, TCG thread-safety, CompareCoverage and NeverZero + counters by Andrea Fioraldi Copyright 2015, 2016, 2017 Google Inc. All rights reserved. + Copyright 2019 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/qemu_mode/patches/afl-qemu-translate-inl.h b/qemu_mode/patches/afl-qemu-translate-inl.h index d63c5167..ffe43dba 100644 --- a/qemu_mode/patches/afl-qemu-translate-inl.h +++ b/qemu_mode/patches/afl-qemu-translate-inl.h @@ -1,19 +1,18 @@ /* - american fuzzy lop - high-performance binary-only instrumentation - ----------------------------------------------------------------- + american fuzzy lop++ - high-performance binary-only instrumentation + ------------------------------------------------------------------- - Written by Andrew Griffiths and - Michal Zalewski - - Idea & design very much by Andrew Griffiths. + Originally written by Andrew Griffiths and + Michal Zalewski TCG instrumentation and block chaining support by Andrea Biondo - QEMU 3.1.0 port, TCG thread-safety and CompareCoverage by Andrea Fioraldi - + QEMU 3.1.0 port, TCG thread-safety, CompareCoverage and NeverZero + counters by Andrea Fioraldi Copyright 2015, 2016, 2017 Google Inc. All rights reserved. + Copyright 2019 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/unicorn_mode/README.md b/unicorn_mode/README.md index 9ee975ef..ea3e3c9b 100644 --- a/unicorn_mode/README.md +++ b/unicorn_mode/README.md @@ -1,23 +1,119 @@ -``` - __ _ _ - __ _ / _| | _ _ _ __ (_) ___ ___ _ __ _ __ - / _` | |_| |___| | | | '_ \| |/ __/ _ \| '__| '_ \ -| (_| | _| |___| |_| | | | | | (_| (_) | | | | | | - \__,_|_| |_| \__,_|_| |_|_|\___\___/|_| |_| |_| - -``` +# Unicorn-based binary-only instrumentation for afl-fuzz -afl-unicorn lets you fuzz any piece of binary that can be emulated by -[Unicorn Engine](http://www.unicorn-engine.org/). +The idea and much of the original implementation comes from Nathan Voss . -Requirements: Python2 +The port to afl++ if by Dominik Maier . -For the full readme please see docs/unicorn_mode.txt +The CompareCoverage and NeverZero counters features by Andrea Fioraldi . -For an in-depth description of what this is, how to install it, and how to use -it check out this [blog post](https://medium.com/@njvoss299/afl-unicorn-fuzzing-arbitrary-binary-code-563ca28936bf). +## 1) Introduction -For general help with AFL, please refer to the documents in the ./docs/ directory. +The code in ./unicorn_mode allows you to build a standalone feature that +leverages the Unicorn Engine and allows callers to obtain instrumentation +output for black-box, closed-source binary code snippets. This mechanism +can be then used by afl-fuzz to stress-test targets that couldn't be built +with afl-gcc or used in QEMU mode, or with other extensions such as +TriforceAFL. -Created by Nathan Voss, originally funded by -[Battelle](https://www.battelle.org/cyber). +There is a significant performance penalty compared to native AFL, +but at least we're able to use AFL on these binaries, right? + +## 2) How to use + +Requirements: you need an installed python2 environment. + +### Building AFL's Unicorn Mode + +First, make afl++ as usual. +Once that completes successfully you need to build and add in the Unicorn Mode +features: + + $ cd unicorn_mode + $ ./build_unicorn_support.sh + +NOTE: This script downloads a Unicorn Engine commit that has been tested +and is stable-ish from the Unicorn github page. If you are offline, you'll need +to hack up this script a little bit and supply your own copy of Unicorn's latest +stable release. It's not very hard, just check out the beginning of the +build_unicorn_support.sh script and adjust as necessary. + +Building Unicorn will take a little bit (~5-10 minutes). Once it completes +it automatically compiles a sample application and verify that it works. + +### Fuzzing with Unicorn Mode + +To really use unicorn-mode effectively you need to prepare the following: + + * Relevant binary code to be fuzzed + * Knowledge of the memory map and good starting state + * Folder containing sample inputs to start fuzzing with + + Same ideas as any other AFL inputs + + Quality/speed of results will depend greatly on quality of starting + samples + + See AFL's guidance on how to create a sample corpus + * Unicorn-based test harness which: + + Adds memory map regions + + Loads binary code into memory + + Emulates at least one instruction* + + Yeah, this is lame. See 'Gotchas' section below for more info + + Loads and verifies data to fuzz from a command-line specified file + + AFL will provide mutated inputs by changing the file passed to + the test harness + + Presumably the data to be fuzzed is at a fixed buffer address + + If input constraints (size, invalid bytes, etc.) are known they + should be checked after the file is loaded. If a constraint + fails, just exit the test harness. AFL will treat the input as + 'uninteresting' and move on. + + Sets up registers and memory state for beginning of test + + Emulates the interested code from beginning to end + + If a crash is detected, the test harness must 'crash' by + throwing a signal (SIGSEGV, SIGKILL, SIGABORT, etc.) + +Once you have all those things ready to go you just need to run afl-fuzz in +'unicorn-mode' by passing in the '-U' flag: + + $ afl-fuzz -U -m none -i /path/to/inputs -o /path/to/results -- ./test_harness @@ + +The normal afl-fuzz command line format applies to everything here. Refer to +AFL's main documentation for more info about how to use afl-fuzz effectively. + +For a much clearer vision of what all of this looks like, please refer to the +sample provided in the 'unicorn_mode/samples' directory. There is also a blog +post that goes over the basics at: + +https://medium.com/@njvoss299/afl-unicorn-fuzzing-arbitrary-binary-code-563ca28936bf + +The 'helper_scripts' directory also contains several helper scripts that allow you +to dump context from a running process, load it, and hook heap allocations. For details +on how to use this check out the follow-up blog post to the one linked above. + +A example use of AFL-Unicorn mode is discussed in the Paper Unicorefuzz: +https://www.usenix.org/conference/woot19/presentation/maier + +## 3) Options + +As for the QEMU-based instrumentation, the afl-unicorn twist of afl++ +comes with a sub-instruction based instrumentation similar in purpose to laf-intel. + +The options that enables Unicorn CompareCoverage are the same used for QEMU. +AFL_COMPCOV_LEVEL=1 is to instrument comparisons with only immediate +values. QEMU_COMPCOV_LEVEL=2 instruments all +comparison instructions. Comparison instructions are currently instrumented only +on the x86 and x86_64 targets. + +## 4) Gotchas, feedback, bugs + +To make sure that AFL's fork server starts up correctly the Unicorn test +harness script must emulate at least one instruction before loading the +data that will be fuzzed from the input file. It doesn't matter what the +instruction is, nor if it is valid. This is an artifact of how the fork-server +is started and could likely be fixed with some clever re-arranging of the +patches applied to Unicorn. + +Running the build script builds Unicorn and its python bindings and installs +them on your system. This installation will supersede any existing Unicorn +installation with the patched afl-unicorn version. + +Refer to the unicorn_mode/samples/arm_example/arm_tester.c for an example +of how to do this properly! If you don't get this right, AFL will not +load any mutated inputs and your fuzzing will be useless! diff --git a/unicorn_mode/build_unicorn_support.sh b/unicorn_mode/build_unicorn_support.sh index 2c0fe4b1..1575f66c 100755 --- a/unicorn_mode/build_unicorn_support.sh +++ b/unicorn_mode/build_unicorn_support.sh @@ -1,16 +1,20 @@ #!/bin/sh # -# american fuzzy lop - Unicorn-Mode build script -# -------------------------------------- +# american fuzzy lop++ - unicorn mode build script +# ------------------------------------------------ # -# Written by Nathan Voss +# Originally written by Nathan Voss # # Adapted from code by Andrew Griffiths and # Michal Zalewski # -# Adapted for Afl++ by Dominik Maier +# Adapted for AFLplusplus by Dominik Maier +# +# CompareCoverage and NeverZero counters by Andrea Fioraldi +# # # Copyright 2017 Battelle Memorial Institute. All rights reserved. +# Copyright 2019 AFLplusplus Project. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/unicorn_mode/patches/afl-unicorn-common.h b/unicorn_mode/patches/afl-unicorn-common.h index 9a1b2a6c..6798832c 100644 --- a/unicorn_mode/patches/afl-unicorn-common.h +++ b/unicorn_mode/patches/afl-unicorn-common.h @@ -1,17 +1,17 @@ /* - american fuzzy lop - high-performance binary-only instrumentation - ----------------------------------------------------------------- + american fuzzy lop++ - unicorn instrumentation + ---------------------------------------------- - Written by Andrew Griffiths and - Michal Zalewski + Originally written by Andrew Griffiths and + Michal Zalewski - TCG instrumentation and block chaining support by Andrea Biondo - Adapted for afl-unicorn by Dominik Maier - Idea & design very much by Andrew Griffiths. + CompareCoverage and NeverZero counters by Andrea Fioraldi + - Copyright 2015, 2016 Google Inc. All rights reserved. + Copyright 2015, 2016, 2017 Google Inc. All rights reserved. + Copyright 2019 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ to implement AFL-style instrumentation and to take care of the remaining parts of the AFL fork server logic. - The resulting QEMU binary is essentially a standalone instrumentation + The resulting libunicorn binary is essentially a standalone instrumentation tool; for an example of how to leverage it for other purposes, you can have a look at afl-showmap.c. diff --git a/unicorn_mode/patches/afl-unicorn-cpu-inl.h b/unicorn_mode/patches/afl-unicorn-cpu-inl.h index 90937a17..a713e4ca 100644 --- a/unicorn_mode/patches/afl-unicorn-cpu-inl.h +++ b/unicorn_mode/patches/afl-unicorn-cpu-inl.h @@ -1,17 +1,17 @@ /* - american fuzzy lop - high-performance binary-only instrumentation - ----------------------------------------------------------------- + american fuzzy lop++ - unicorn instrumentation + ---------------------------------------------- - Written by Andrew Griffiths and - Michal Zalewski + Originally written by Andrew Griffiths and + Michal Zalewski - TCG instrumentation and block chaining support by Andrea Biondo - Adapted for afl-unicorn by Dominik Maier - Idea & design very much by Andrew Griffiths. + CompareCoverage and NeverZero counters by Andrea Fioraldi + - Copyright 2015, 2016 Google Inc. All rights reserved. + Copyright 2015, 2016, 2017 Google Inc. All rights reserved. + Copyright 2019 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ to implement AFL-style instrumentation and to take care of the remaining parts of the AFL fork server logic. - The resulting QEMU binary is essentially a standalone instrumentation + The resulting libunicorn binary is essentially a standalone instrumentation tool; for an example of how to leverage it for other purposes, you can have a look at afl-showmap.c. diff --git a/unicorn_mode/patches/afl-unicorn-cpu-translate-inl.h b/unicorn_mode/patches/afl-unicorn-cpu-translate-inl.h index 7e8f47c9..69877c6b 100644 --- a/unicorn_mode/patches/afl-unicorn-cpu-translate-inl.h +++ b/unicorn_mode/patches/afl-unicorn-cpu-translate-inl.h @@ -1,17 +1,17 @@ /* - american fuzzy lop - high-performance binary-only instrumentation - ----------------------------------------------------------------- + american fuzzy lop++ - unicorn instrumentation + ---------------------------------------------- - Written by Andrew Griffiths and - Michal Zalewski + Originally written by Andrew Griffiths and + Michal Zalewski - TCG instrumentation and block chaining support by Andrea Biondo - Adapted for afl-unicorn by Dominik Maier - Idea & design very much by Andrew Griffiths. + CompareCoverage and NeverZero counters by Andrea Fioraldi + - Copyright 2015, 2016 Google Inc. All rights reserved. + Copyright 2015, 2016, 2017 Google Inc. All rights reserved. + Copyright 2019 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ to implement AFL-style instrumentation and to take care of the remaining parts of the AFL fork server logic. - The resulting QEMU binary is essentially a standalone instrumentation + The resulting libunicorn binary is essentially a standalone instrumentation tool; for an example of how to leverage it for other purposes, you can have a look at afl-showmap.c. diff --git a/unicorn_mode/patches/afl-unicorn-tcg-op-inl.h b/unicorn_mode/patches/afl-unicorn-tcg-op-inl.h index d5a29cce..fa4974d6 100644 --- a/unicorn_mode/patches/afl-unicorn-tcg-op-inl.h +++ b/unicorn_mode/patches/afl-unicorn-tcg-op-inl.h @@ -1,17 +1,17 @@ /* - american fuzzy lop - high-performance binary-only instrumentation - ----------------------------------------------------------------- + american fuzzy lop++ - unicorn instrumentation + ---------------------------------------------- - Written by Andrew Griffiths and - Michal Zalewski + Originally written by Andrew Griffiths and + Michal Zalewski - TCG instrumentation and block chaining support by Andrea Biondo - Adapted for afl-unicorn by Dominik Maier - Idea & design very much by Andrew Griffiths. + CompareCoverage and NeverZero counters by Andrea Fioraldi + - Copyright 2015, 2016 Google Inc. All rights reserved. + Copyright 2015, 2016, 2017 Google Inc. All rights reserved. + Copyright 2019 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ to implement AFL-style instrumentation and to take care of the remaining parts of the AFL fork server logic. - The resulting QEMU binary is essentially a standalone instrumentation + The resulting libunicorn binary is essentially a standalone instrumentation tool; for an example of how to leverage it for other purposes, you can have a look at afl-showmap.c. diff --git a/unicorn_mode/patches/afl-unicorn-tcg-runtime-inl.h b/unicorn_mode/patches/afl-unicorn-tcg-runtime-inl.h index 0019bbfa..1f0667ce 100644 --- a/unicorn_mode/patches/afl-unicorn-tcg-runtime-inl.h +++ b/unicorn_mode/patches/afl-unicorn-tcg-runtime-inl.h @@ -1,17 +1,17 @@ /* - american fuzzy lop - high-performance binary-only instrumentation - ----------------------------------------------------------------- + american fuzzy lop++ - unicorn instrumentation + ---------------------------------------------- - Written by Andrew Griffiths and - Michal Zalewski + Originally written by Andrew Griffiths and + Michal Zalewski - TCG instrumentation and block chaining support by Andrea Biondo - Adapted for afl-unicorn by Dominik Maier - Idea & design very much by Andrew Griffiths. + CompareCoverage and NeverZero counters by Andrea Fioraldi + - Copyright 2015, 2016 Google Inc. All rights reserved. + Copyright 2015, 2016, 2017 Google Inc. All rights reserved. + Copyright 2019 AFLplusplus Project. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ to implement AFL-style instrumentation and to take care of the remaining parts of the AFL fork server logic. - The resulting QEMU binary is essentially a standalone instrumentation + The resulting libunicorn binary is essentially a standalone instrumentation tool; for an example of how to leverage it for other purposes, you can have a look at afl-showmap.c. diff --git a/unicorn_mode/samples/compcov_x64/COMPILE.md b/unicorn_mode/samples/compcov_x64/COMPILE.md index db488d30..35de7ad8 100644 --- a/unicorn_mode/samples/compcov_x64/COMPILE.md +++ b/unicorn_mode/samples/compcov_x64/COMPILE.md @@ -1,5 +1,4 @@ -Compiling compcov_target.c -========================== +# Compiling compcov_target.c compcov_target.c was compiled without optimization, position-independent, and without standard libraries using the following command line: diff --git a/unicorn_mode/samples/compcov_x64/compcov_target.c b/unicorn_mode/samples/compcov_x64/compcov_target.c index 71b4cb0e..eb1205b1 100644 --- a/unicorn_mode/samples/compcov_x64/compcov_target.c +++ b/unicorn_mode/samples/compcov_x64/compcov_target.c @@ -7,7 +7,7 @@ * (0x00300000), so make sure that your Unicorn emulation of this * puts user data there. * - * Written by Nathan Voss + * Written by Andrea Fioraldi */ // Magic address where mutated data will be placed diff --git a/unicorn_mode/samples/simple/COMPILE.md b/unicorn_mode/samples/simple/COMPILE.md index bd4a66c6..f7bf5b50 100644 --- a/unicorn_mode/samples/simple/COMPILE.md +++ b/unicorn_mode/samples/simple/COMPILE.md @@ -1,5 +1,4 @@ -Compiling simple_target.c -========================== +# Compiling simple_target.c You shouldn't need to compile simple_target.c since a MIPS binary version is pre-built and shipped with afl-unicorn. This file documents how the binary @@ -38,4 +37,4 @@ mips-linux-gnu-gcc -o simple_target.elf simple_target.c -fPIC -O0 -nostdlib Note that the output of this is padded with nulls for 16-byte alignment. This is important when emulating it, as NOPs will be added after the return of main() -as necessary. \ No newline at end of file +as necessary. -- cgit 1.4.1 From eadd378f6c54a7e021985bca041d9642fff41034 Mon Sep 17 00:00:00 2001 From: Andrea Fioraldi Date: Fri, 30 Aug 2019 11:42:30 +0200 Subject: update changelog --- docs/ChangeLog | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'docs') diff --git a/docs/ChangeLog b/docs/ChangeLog index 6d56d314..6d4c4792 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -26,6 +26,10 @@ Version ++2.53d (dev): afl never did), plus shows tuple content summary information now - fix building on *BSD (thanks to tobias.kortkamp for the patch) - small docu updates + - NeverZero counters for QEMU + - NeverZero counters for Unicorn + - CompareCoverage Unicorn + - Immediates-only instrumentation for CompareCoverage - ... your patch? :) -- cgit 1.4.1 From e76ad2980faff269a9ac523cb71237462985b3e6 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 2 Sep 2019 09:41:52 +0200 Subject: added force-ui env --- config.h | 363 -------------------------------------------- docs/ChangeLog | 8 + docs/env_variables.txt | 3 + include/config.h | 5 + src/afl-fuzz-src/afl-fuzz.c | 5 + 5 files changed, 21 insertions(+), 363 deletions(-) delete mode 100644 config.h (limited to 'docs') diff --git a/config.h b/config.h deleted file mode 100644 index 29c33d46..00000000 --- a/config.h +++ /dev/null @@ -1,363 +0,0 @@ -/* - american fuzzy lop plus plus - vaguely configurable bits - ---------------------------------------------- - - Written and maintained by Michal Zalewski - - Copyright 2013, 2014, 2015, 2016 Google Inc. All rights reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - - */ - -#ifndef _HAVE_CONFIG_H -#define _HAVE_CONFIG_H - -#include "types.h" - -/* Version string: */ - -#define VERSION "++2.53d" // c = release, d = volatile github dev - -/****************************************************** - * * - * Settings that may be of interest to power users: * - * * - ******************************************************/ - -/* Comment out to disable terminal colors (note that this makes afl-analyze - a lot less nice): */ - -#define USE_COLOR - -/* Comment out to disable fancy ANSI boxes and use poor man's 7-bit UI: */ - -#define FANCY_BOXES - -/* Default timeout for fuzzed code (milliseconds). This is the upper bound, - also used for detecting hangs; the actual value is auto-scaled: */ - -#define EXEC_TIMEOUT 1000 - -/* Timeout rounding factor when auto-scaling (milliseconds): */ - -#define EXEC_TM_ROUND 20 - -/* Default memory limit for child process (MB): */ - -#ifndef __x86_64__ -# define MEM_LIMIT 25 -#else -# define MEM_LIMIT 50 -#endif /* ^!__x86_64__ */ - -/* Default memory limit when running in QEMU mode (MB): */ - -#define MEM_LIMIT_QEMU 200 - -/* Default memory limit when running in Unicorn mode (MB): */ - -#define MEM_LIMIT_UNICORN 200 - -/* Number of calibration cycles per every new test case (and for test - cases that show variable behavior): */ - -#define CAL_CYCLES 8 -#define CAL_CYCLES_LONG 40 - -/* Number of subsequent timeouts before abandoning an input file: */ - -#define TMOUT_LIMIT 250 - -/* Maximum number of unique hangs or crashes to record: */ - -#define KEEP_UNIQUE_HANG 500 -#define KEEP_UNIQUE_CRASH 5000 - -/* Baseline number of random tweaks during a single 'havoc' stage: */ - -#define HAVOC_CYCLES 256 -#define HAVOC_CYCLES_INIT 1024 - -/* Maximum multiplier for the above (should be a power of two, beware - of 32-bit int overflows): */ - -#define HAVOC_MAX_MULT 16 -#define HAVOC_MAX_MULT_MOPT 32 - -/* Absolute minimum number of havoc cycles (after all adjustments): */ - -#define HAVOC_MIN 16 - -/* Power Schedule Divisor */ -#define POWER_BETA 1 -#define MAX_FACTOR (POWER_BETA * 32) - -/* Maximum stacking for havoc-stage tweaks. The actual value is calculated - like this: - - n = random between 1 and HAVOC_STACK_POW2 - stacking = 2^n - - In other words, the default (n = 7) produces 2, 4, 8, 16, 32, 64, or - 128 stacked tweaks: */ - -#define HAVOC_STACK_POW2 7 - -/* Caps on block sizes for cloning and deletion operations. Each of these - ranges has a 33% probability of getting picked, except for the first - two cycles where smaller blocks are favored: */ - -#define HAVOC_BLK_SMALL 32 -#define HAVOC_BLK_MEDIUM 128 -#define HAVOC_BLK_LARGE 1500 - -/* Extra-large blocks, selected very rarely (<5% of the time): */ - -#define HAVOC_BLK_XL 32768 - -/* Probabilities of skipping non-favored entries in the queue, expressed as - percentages: */ - -#define SKIP_TO_NEW_PROB 99 /* ...when there are new, pending favorites */ -#define SKIP_NFAV_OLD_PROB 95 /* ...no new favs, cur entry already fuzzed */ -#define SKIP_NFAV_NEW_PROB 75 /* ...no new favs, cur entry not fuzzed yet */ - -/* Splicing cycle count: */ - -#define SPLICE_CYCLES 15 - -/* Nominal per-splice havoc cycle length: */ - -#define SPLICE_HAVOC 32 - -/* Maximum offset for integer addition / subtraction stages: */ - -#define ARITH_MAX 35 - -/* Limits for the test case trimmer. The absolute minimum chunk size; and - the starting and ending divisors for chopping up the input file: */ - -#define TRIM_MIN_BYTES 4 -#define TRIM_START_STEPS 16 -#define TRIM_END_STEPS 1024 - -/* Maximum size of input file, in bytes (keep under 100MB): */ - -#define MAX_FILE (1 * 1024 * 1024) - -/* The same, for the test case minimizer: */ - -#define TMIN_MAX_FILE (10 * 1024 * 1024) - -/* Block normalization steps for afl-tmin: */ - -#define TMIN_SET_MIN_SIZE 4 -#define TMIN_SET_STEPS 128 - -/* Maximum dictionary token size (-x), in bytes: */ - -#define MAX_DICT_FILE 128 - -/* Length limits for auto-detected dictionary tokens: */ - -#define MIN_AUTO_EXTRA 3 -#define MAX_AUTO_EXTRA 32 - -/* Maximum number of user-specified dictionary tokens to use in deterministic - steps; past this point, the "extras/user" step will be still carried out, - but with proportionally lower odds: */ - -#define MAX_DET_EXTRAS 200 - -/* Maximum number of auto-extracted dictionary tokens to actually use in fuzzing - (first value), and to keep in memory as candidates. The latter should be much - higher than the former. */ - -#define USE_AUTO_EXTRAS 50 -#define MAX_AUTO_EXTRAS (USE_AUTO_EXTRAS * 10) - -/* Scaling factor for the effector map used to skip some of the more - expensive deterministic steps. The actual divisor is set to - 2^EFF_MAP_SCALE2 bytes: */ - -#define EFF_MAP_SCALE2 3 - -/* Minimum input file length at which the effector logic kicks in: */ - -#define EFF_MIN_LEN 128 - -/* Maximum effector density past which everything is just fuzzed - unconditionally (%): */ - -#define EFF_MAX_PERC 90 - -/* UI refresh frequency (Hz): */ - -#define UI_TARGET_HZ 5 - -/* Fuzzer stats file and plot update intervals (sec): */ - -#define STATS_UPDATE_SEC 60 -#define PLOT_UPDATE_SEC 5 - -/* Smoothing divisor for CPU load and exec speed stats (1 - no smoothing). */ - -#define AVG_SMOOTHING 16 - -/* Sync interval (every n havoc cycles): */ - -#define SYNC_INTERVAL 5 - -/* Output directory reuse grace period (minutes): */ - -#define OUTPUT_GRACE 25 - -/* Uncomment to use simple file names (id_NNNNNN): */ - -// #define SIMPLE_FILES - -/* List of interesting values to use in fuzzing. */ - -#define INTERESTING_8 \ - -128, /* Overflow signed 8-bit when decremented */ \ - -1, /* */ \ - 0, /* */ \ - 1, /* */ \ - 16, /* One-off with common buffer size */ \ - 32, /* One-off with common buffer size */ \ - 64, /* One-off with common buffer size */ \ - 100, /* One-off with common buffer size */ \ - 127 /* Overflow signed 8-bit when incremented */ - -#define INTERESTING_16 \ - -32768, /* Overflow signed 16-bit when decremented */ \ - -129, /* Overflow signed 8-bit */ \ - 128, /* Overflow signed 8-bit */ \ - 255, /* Overflow unsig 8-bit when incremented */ \ - 256, /* Overflow unsig 8-bit */ \ - 512, /* One-off with common buffer size */ \ - 1000, /* One-off with common buffer size */ \ - 1024, /* One-off with common buffer size */ \ - 4096, /* One-off with common buffer size */ \ - 32767 /* Overflow signed 16-bit when incremented */ - -#define INTERESTING_32 \ - -2147483648LL, /* Overflow signed 32-bit when decremented */ \ - -100663046, /* Large negative number (endian-agnostic) */ \ - -32769, /* Overflow signed 16-bit */ \ - 32768, /* Overflow signed 16-bit */ \ - 65535, /* Overflow unsig 16-bit when incremented */ \ - 65536, /* Overflow unsig 16 bit */ \ - 100663045, /* Large positive number (endian-agnostic) */ \ - 2147483647 /* Overflow signed 32-bit when incremented */ - -/*********************************************************** - * * - * Really exotic stuff you probably don't want to touch: * - * * - ***********************************************************/ - -/* Call count interval between reseeding the libc PRNG from /dev/urandom: */ - -#define RESEED_RNG 10000 - -/* Maximum line length passed from GCC to 'as' and used for parsing - configuration files: */ - -#define MAX_LINE 8192 - -/* Environment variable used to pass SHM ID to the called program. */ - -#define SHM_ENV_VAR "__AFL_SHM_ID" - -/* Other less interesting, internal-only variables. */ - -#define CLANG_ENV_VAR "__AFL_CLANG_MODE" -#define AS_LOOP_ENV_VAR "__AFL_AS_LOOPCHECK" -#define PERSIST_ENV_VAR "__AFL_PERSISTENT" -#define DEFER_ENV_VAR "__AFL_DEFER_FORKSRV" - -/* In-code signatures for deferred and persistent mode. */ - -#define PERSIST_SIG "##SIG_AFL_PERSISTENT##" -#define DEFER_SIG "##SIG_AFL_DEFER_FORKSRV##" - -/* Distinctive bitmap signature used to indicate failed execution: */ - -#define EXEC_FAIL_SIG 0xfee1dead - -/* Distinctive exit code used to indicate MSAN trip condition: */ - -#define MSAN_ERROR 86 - -/* Designated file descriptors for forkserver commands (the application will - use FORKSRV_FD and FORKSRV_FD + 1): */ - -#define FORKSRV_FD 198 - -/* Fork server init timeout multiplier: we'll wait the user-selected - timeout plus this much for the fork server to spin up. */ - -#define FORK_WAIT_MULT 10 - -/* Calibration timeout adjustments, to be a bit more generous when resuming - fuzzing sessions or trying to calibrate already-added internal finds. - The first value is a percentage, the other is in milliseconds: */ - -#define CAL_TMOUT_PERC 125 -#define CAL_TMOUT_ADD 50 - -/* Number of chances to calibrate a case before giving up: */ - -#define CAL_CHANCES 3 - -/* Map size for the traced binary (2^MAP_SIZE_POW2). Must be greater than - 2; you probably want to keep it under 18 or so for performance reasons - (adjusting AFL_INST_RATIO when compiling is probably a better way to solve - problems with complex programs). You need to recompile the target binary - after changing this - otherwise, SEGVs may ensue. */ - -#define MAP_SIZE_POW2 16 -#define MAP_SIZE (1 << MAP_SIZE_POW2) - -/* Maximum allocator request size (keep well under INT_MAX): */ - -#define MAX_ALLOC 0x40000000 - -/* A made-up hashing seed: */ - -#define HASH_CONST 0xa5b35705 - -/* Constants for afl-gotcpu to control busy loop timing: */ - -#define CTEST_TARGET_MS 5000 -#define CTEST_CORE_TRG_MS 1000 -#define CTEST_BUSY_CYCLES (10 * 1000 * 1000) - -/* Enable NeverZero counters in QEMU mode */ - -#define AFL_QEMU_NOT_ZERO - -/* Uncomment this to use inferior block-coverage-based instrumentation. Note - that you need to recompile the target binary for this to have any effect: */ - -// #define COVERAGE_ONLY - -/* Uncomment this to ignore hit counts and output just one bit per tuple. - As with the previous setting, you will need to recompile the target - binary: */ - -// #define SKIP_COUNTS - -/* Uncomment this to use instrumentation data to record newly discovered paths, - but do not use them as seeds for fuzzing. This is useful for conveniently - measuring coverage that could be attained by a "dumb" fuzzing algorithm: */ - -// #define IGNORE_FINDS - -#endif /* ! _HAVE_CONFIG_H */ diff --git a/docs/ChangeLog b/docs/ChangeLog index 782320d6..dd5b597c 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -17,6 +17,14 @@ sending a mail to . Version ++2.53d (dev): ---------------------- + - big code refactoring: + * all includes are now in include/ + * all afl sources are now in src/ - see src/README + * afl-fuzz was splitted up in various individual files for including + functionality in other programs (e.g. forkserver, memory map, etc.) + or better readability. + * new code indention everywhere + - added AFL_FORCE_UI to show the UI even if the terminal is not detected - llvm 9 is now supported (still needs testing) - Android is now supported (thank to JoeyJiao!) - still need to modify the Makefile though - fix building qemu on some Ubuntus (thanks to floyd!) diff --git a/docs/env_variables.txt b/docs/env_variables.txt index 821463ae..cea3597b 100644 --- a/docs/env_variables.txt +++ b/docs/env_variables.txt @@ -223,6 +223,9 @@ checks or alter some of the more exotic semantics of the tool: some basic stats. This behavior is also automatically triggered when the output from afl-fuzz is redirected to a file or to a pipe. + - Setting AFL_FORCE_UI will force painting the UI on the screen even if + no valid terminal was detected (for virtual consoles) + - If you are Jakub, you may need AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES. Others need not apply. diff --git a/include/config.h b/include/config.h index 324435b3..17836151 100644 --- a/include/config.h +++ b/include/config.h @@ -345,6 +345,11 @@ #define CTEST_CORE_TRG_MS 1000 #define CTEST_BUSY_CYCLES (10 * 1000 * 1000) +/* Enable NeverZero counters in QEMU mode */ + +#define AFL_QEMU_NOT_ZERO + + /* Uncomment this to use inferior block-coverage-based instrumentation. Note that you need to recompile the target binary for this to have any effect: */ diff --git a/src/afl-fuzz-src/afl-fuzz.c b/src/afl-fuzz-src/afl-fuzz.c index 7cc05a39..0e12f493 100644 --- a/src/afl-fuzz-src/afl-fuzz.c +++ b/src/afl-fuzz-src/afl-fuzz.c @@ -10104,6 +10104,9 @@ int main(int argc, char** argv) { } + if (getenv("AFL_NO_UI") && getenv("AFL_FORCE_UI")) + FATAL("AFL_NO_UI and AFL_FORCE_UI are mutually exclusive"); + if (strchr(argv[optind], '/') == NULL) WARNF(cLRD "Target binary called without a prefixed path, make sure you are fuzzing the right binary: " cRST "%s", argv[optind]); OKF("afl++ is maintained by Marc \"van Hauser\" Heuse, Heiko \"hexcoder\" Eissfeldt and Andrea Fioraldi"); @@ -10151,6 +10154,8 @@ int main(int argc, char** argv) { fix_up_banner(argv[optind]); check_if_tty(); + if (getenv("AFL_FORCE_UI")) + not_on_tty = 0; if (getenv("AFL_CAL_FAST")) { /* Use less calibration cycles, for slow applications */ -- cgit 1.4.1 From 39c4bb7a49d22c66b3cb613fb13329ec7a6dc16d Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 2 Sep 2019 10:29:54 +0200 Subject: added peak_rss_mb and slowest_exec_ms in fuzzer_stats report --- docs/ChangeLog | 3 +-- include/afl-fuzz.h | 1 + src/afl-fuzz-src/afl-fuzz.c | 29 ++++++++++++++++++++++++++++- src/afl-fuzz-src/globals.c | 1 + 4 files changed, 31 insertions(+), 3 deletions(-) (limited to 'docs') diff --git a/docs/ChangeLog b/docs/ChangeLog index dd5b597c..1cd95650 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -29,18 +29,17 @@ Version ++2.53d (dev): - Android is now supported (thank to JoeyJiao!) - still need to modify the Makefile though - fix building qemu on some Ubuntus (thanks to floyd!) - custom mutator by a loaded library is now supported (thanks to kyakdan!) + - added PR that includes peak_rss_mb and slowest_exec_ms in the fuzzer_stats report - more support for *BSD (thanks to devnexen!) - fix building on *BSD (thanks to tobias.kortkamp for the patch) - fix for a few features to support different map sized than 2^16 - afl-showmap: new option -r now shows the real values in the buckets (stock afl never did), plus shows tuple content summary information now - - the forkserver is now in its own C file to be easily integratable - small docu updates - NeverZero counters for QEMU - NeverZero counters for Unicorn - CompareCoverage Unicorn - Immediates-only instrumentation for CompareCoverage - - ... your patch? :) -------------------------- diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index c50a21a7..7b380dce 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -350,6 +350,7 @@ extern u64 total_crashes, /* Total number of crashes */ unique_tmouts, /* Timeouts with unique signatures */ unique_hangs, /* Hangs with unique signatures */ total_execs, /* Total execve() calls */ + slowest_exec_ms, /* Slowest testcase non hang in ms */ start_time, /* Unix start time (ms) */ last_path_time, /* Time for most recent path (ms) */ last_crash_time, /* Time for most recent crash (ms) */ diff --git a/src/afl-fuzz-src/afl-fuzz.c b/src/afl-fuzz-src/afl-fuzz.c index 0e12f493..dc21de17 100644 --- a/src/afl-fuzz-src/afl-fuzz.c +++ b/src/afl-fuzz-src/afl-fuzz.c @@ -370,6 +370,7 @@ static u8 run_target(char** argv, u32 timeout) { static struct itimerval it; static u32 prev_timed_out = 0; + static u64 exec_ms = 0; int status = 0; u32 tb4; @@ -519,6 +520,10 @@ static u8 run_target(char** argv, u32 timeout) { } if (!WIFSTOPPED(status)) child_pid = 0; + + getitimer(ITIMER_REAL, &it); + exec_ms = (u64) timeout - (it.it_value.tv_sec * 1000 + it.it_value.tv_usec / 1000); + if (slowest_exec_ms < exec_ms) slowest_exec_ms = exec_ms; it.it_value.tv_sec = 0; it.it_value.tv_usec = 0; @@ -1491,6 +1496,7 @@ static void find_timeout(void) { static void write_stats_file(double bitmap_cvg, double stability, double eps) { static double last_bcvg, last_stab, last_eps; + static struct rusage usage; u8* fn = alloc_printf("%s/fuzzer_stats", out_dir); s32 fd; @@ -1543,6 +1549,8 @@ static void write_stats_file(double bitmap_cvg, double stability, double eps) { "last_hang : %llu\n" "execs_since_crash : %llu\n" "exec_timeout : %u\n" + "slowest_exec_ms : %llu\n" + "peak_rss_mb : %lu\n" "afl_banner : %s\n" "afl_version : " VERSION "\n" "target_mode : %s%s%s%s%s%s%s%s\n" @@ -1554,7 +1562,7 @@ static void write_stats_file(double bitmap_cvg, double stability, double eps) { queued_variable, stability, bitmap_cvg, unique_crashes, unique_hangs, last_path_time / 1000, last_crash_time / 1000, last_hang_time / 1000, total_execs - last_crash_execs, - exec_tmout, use_banner, + exec_tmout, slowest_exec_ms, (unsigned long int)usage.ru_maxrss, use_banner, unicorn_mode ? "unicorn" : "", qemu_mode ? "qemu " : "", dumb_mode ? " dumb " : "", no_forkserver ? "no_forksrv " : "", crash_mode ? "crash " : "", persistent_mode ? "persistent " : "", deferred_mode ? "deferred " : "", @@ -10347,6 +10355,25 @@ int main(int argc, char** argv) { if (queue_cur) show_stats(); + /* + * ATTENTION - the following 10 lines were copied from a PR to Google's afl + * repository - and slightly fixed. + * These lines have nothing to do with the purpose of original PR though. + * Looks like when an exit condition was completed (AFL_BENCH_JUST_ONE, + * AFL_EXIT_WHEN_DONE or AFL_BENCH_UNTIL_CRASH) the child and forkserver + * where not killed? + */ + /* if we stopped programmatically, we kill the forkserver and the current runner. + if we stopped manually, this is done by the signal handler */ + if (stop_soon == 2){ + if (child_pid > 0) kill(child_pid, SIGKILL); + if (forksrv_pid > 0) kill(forksrv_pid, SIGKILL); + /* Now that we've killed the forkserver, we wait for it to be able to get rusage stats. */ + if (waitpid(forksrv_pid, NULL, 0) <= 0) { + WARNF("error waitpid\n"); + } + } + write_bitmap(); write_stats_file(0, 0, 0); save_auto(); diff --git a/src/afl-fuzz-src/globals.c b/src/afl-fuzz-src/globals.c index 127d7609..e28c3099 100644 --- a/src/afl-fuzz-src/globals.c +++ b/src/afl-fuzz-src/globals.c @@ -189,6 +189,7 @@ u64 total_crashes, /* Total number of crashes */ unique_tmouts, /* Timeouts with unique signatures */ unique_hangs, /* Hangs with unique signatures */ total_execs, /* Total execve() calls */ + slowest_exec_ms, /* Slowest testcase non hang in ms */ start_time, /* Unix start time (ms) */ last_path_time, /* Time for most recent path (ms) */ last_crash_time, /* Time for most recent crash (ms) */ -- cgit 1.4.1 From e969afc627ee625472b6e5d8b96c06c81d722aa4 Mon Sep 17 00:00:00 2001 From: Andrea Fioraldi Date: Tue, 3 Sep 2019 11:24:45 +0200 Subject: update todo --- TODO | 16 ++++++---------- docs/ChangeLog | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) (limited to 'docs') diff --git a/TODO b/TODO index c2cf10a5..06e9a634 100644 --- a/TODO +++ b/TODO @@ -1,13 +1,7 @@ Roadmap 2.53d: ============== -all: - - indent all the code: .clang-format? - (vh: tried, the variable definion look very ugly then, what to do?) - afl-fuzz: - - put mutator, scheduler, forkserver and input channels in individual files - - reuse forkserver for showmap, afl-cmin, etc. - custom mutator lib: example and readme - env var to exclusively run the custom lib/py mutator @@ -19,13 +13,10 @@ gcc_plugin: - neverZero qemu_mode: - - update to 4.x - - deferred mode with AFL_DEFERRED_QEMU=0xaddress - (vh: @andrea - dont we have that already with AFL_ENTRYPOINT?) + - update to 4.x (probably this will be skipped :( ) unit testing / or large testcase campaign - Roadmap 2.54d: ============== Problem: Average targets (tiff, jpeg, unrar) go through 1500 edges. @@ -59,3 +50,8 @@ Problem: Average targets (tiff, jpeg, unrar) go through 1500 edges. qemu_mode: - persistent mode patching the return address (WinAFL style) + - deferred mode with AFL_DEFERRED_QEMU=0xaddress + (AFL_ENTRYPOINT let you to specify only a basic block address as starting + point. This will be implemented togheter with the logic for persistent + mode.) + diff --git a/docs/ChangeLog b/docs/ChangeLog index 1cd95650..2fc4efbc 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -39,7 +39,7 @@ Version ++2.53d (dev): - NeverZero counters for QEMU - NeverZero counters for Unicorn - CompareCoverage Unicorn - - Immediates-only instrumentation for CompareCoverage + - immediates-only instrumentation for CompareCoverage -------------------------- -- cgit 1.4.1 From 52bfd1fc3d6c1e6610469dbddad19aacb4e7f848 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 4 Sep 2019 12:14:35 +0200 Subject: added man pages --- Makefile | 34 ++++++++++++++++++++++++++++++---- TODO | 3 +++ afl-cmin | 5 ++++- afl-plot | 4 ++-- afl-system-config | 13 +++++++++++++ afl-whatsup | 7 +++++++ docs/ChangeLog | 5 +++-- qemu_mode/build_qemu_support.sh | 10 +++++++++- qemu_mode/libcompcov/Makefile | 6 +++--- src/README.src | 22 ++++++++++++++++++++++ src/afl-analyze.c | 7 ++++++- src/afl-fuzz.c | 7 ++++++- src/afl-gcc.c | 8 ++++++++ src/afl-gotcpu.c | 8 ++++++++ src/afl-showmap.c | 7 ++++++- src/afl-tmin.c | 7 ++++++- 16 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 src/README.src (limited to 'docs') diff --git a/Makefile b/Makefile index edf3d99b..455facf2 100644 --- a/Makefile +++ b/Makefile @@ -24,11 +24,13 @@ BIN_PATH = $(PREFIX)/bin HELPER_PATH = $(PREFIX)/lib/afl DOC_PATH = $(PREFIX)/share/doc/afl MISC_PATH = $(PREFIX)/share/afl +MAN_PATH = $(PREFIX)/man/man8 # PROGS intentionally omit afl-as, which gets installed elsewhere. PROGS = afl-gcc afl-fuzz afl-showmap afl-tmin afl-gotcpu afl-analyze SH_PROGS = afl-plot afl-cmin afl-whatsup afl-system-config +MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8) CFLAGS ?= -O3 -funroll-loops CFLAGS += -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign -I include/ \ @@ -203,13 +205,33 @@ all_done: test_build .NOTPARALLEL: clean clean: - rm -f $(PROGS) afl-as as afl-g++ afl-clang afl-clang++ *.o *~ a.out core core.[1-9][0-9]* *.stackdump test .test .test1 .test2 test-instr .test-instr0 .test-instr1 qemu_mode/qemu-3.1.0.tar.xz afl-qemu-trace afl-gcc-fast afl-gcc-pass.so afl-gcc-rt.o afl-g++-fast - rm -rf out_dir qemu_mode/qemu-3.1.0 + rm -f $(PROGS) afl-as as afl-g++ afl-clang afl-clang++ *.o *~ a.out core core.[1-9][0-9]* *.stackdump test .test .test1 .test2 test-instr .test-instr0 .test-instr1 qemu_mode/qemu-3.1.0.tar.xz afl-qemu-trace afl-gcc-fast afl-gcc-pass.so afl-gcc-rt.o afl-g++-fast *.so unicorn_mode/24f55a7973278f20f0de21b904851d99d4716263.tar.gz *.8 + rm -rf out_dir qemu_mode/qemu-3.1.0 unicorn_mode/unicorn $(MAKE) -C llvm_mode clean $(MAKE) -C libdislocator clean $(MAKE) -C libtokencap clean - -install: all + $(MAKE) -C qemu_mode/libcompcov clean + +%.8: % + @echo .TH $* 8 `date --iso-8601` "afl++" > $@ + @echo .SH NAME >> $@ + @echo .B $* >> $@ + @echo >> $@ + @echo .SH SYNOPSIS >> $@ + @./$* -h 2>&1 | head -n 3 | tail -n 1 | sed 's/^\.\///' >> $@ + @echo >> $@ + @echo .SH OPTIONS >> $@ + @echo .nf >> $@ + @./$* -h 2>&1 | tail -n +4 >> $@ + @echo >> $@ + @echo .SH AUTHOR >> $@ + @echo "afl++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Heiko \"hexc0der\" Eissfeldt and Andrea Fioraldi " >> $@ + @echo The homepage of afl++ is: https://github.com/vanhauser-thc/AFLplusplus >> $@ + @echo >> $@ + @echo .SH LICENSE >> $@ + @echo Apache License Version 2.0, January 2004 >> $@ + +install: all $(MANPAGES) mkdir -p -m 755 $${DESTDIR}$(BIN_PATH) $${DESTDIR}$(HELPER_PATH) $${DESTDIR}$(DOC_PATH) $${DESTDIR}$(MISC_PATH) rm -f $${DESTDIR}$(BIN_PATH)/afl-plot.sh install -m 755 $(PROGS) $(SH_PROGS) $${DESTDIR}$(BIN_PATH) @@ -226,10 +248,14 @@ endif if [ -f compare-transform-pass.so ]; then set -e; install -m 755 compare-transform-pass.so $${DESTDIR}$(HELPER_PATH); fi if [ -f split-compares-pass.so ]; then set -e; install -m 755 split-compares-pass.so $${DESTDIR}$(HELPER_PATH); fi if [ -f split-switches-pass.so ]; then set -e; install -m 755 split-switches-pass.so $${DESTDIR}$(HELPER_PATH); fi + if [ -f libcompcov.so ]; then set -e; install -m 755 libcompcov.so $${DESTDIR}$(HELPER_PATH); fi set -e; ln -sf afl-gcc $${DESTDIR}$(BIN_PATH)/afl-g++ set -e; if [ -f afl-clang-fast ] ; then ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang ; ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang++ ; else ln -sf afl-gcc $${DESTDIR}$(BIN_PATH)/afl-clang ; ln -sf afl-gcc $${DESTDIR}$(BIN_PATH)/afl-clang++; fi + mkdir -m 0755 -p $(MAN_PATH) + install -m0644 -D *.8 $(MAN_PATH) + install -m 755 afl-as $${DESTDIR}$(HELPER_PATH) ln -sf afl-as $${DESTDIR}$(HELPER_PATH)/as install -m 644 docs/README.md docs/ChangeLog docs/*.txt $${DESTDIR}$(DOC_PATH) diff --git a/TODO b/TODO index df32db84..26311713 100644 --- a/TODO +++ b/TODO @@ -4,6 +4,9 @@ Roadmap 2.53d: afl-fuzz: - custom mutator lib: example and readme +man: + - man page for afl-clang-fast + Roadmap 2.54d: ============== diff --git a/afl-cmin b/afl-cmin index a9ec4082..88635550 100755 --- a/afl-cmin +++ b/afl-cmin @@ -51,10 +51,13 @@ TIMEOUT=none unset IN_DIR OUT_DIR STDIN_FILE EXTRA_PAR MEM_LIMIT_GIVEN \ AFL_CMIN_CRASHES_ONLY AFL_CMIN_ALLOW_ANY QEMU_MODE UNICORN_MODE -while getopts "+i:o:f:m:t:eQUC" opt; do +while getopts "+i:o:f:m:t:eQUCh" opt; do case "$opt" in + "h") + ;; + "i") IN_DIR="$OPTARG" ;; diff --git a/afl-plot b/afl-plot index 25ffde64..bc86fb85 100755 --- a/afl-plot +++ b/afl-plot @@ -21,10 +21,10 @@ echo if [ ! "$#" = "2" ]; then cat 1>&2 <<_EOF_ -This program generates gnuplot images from afl-fuzz output data. Usage: - $0 afl_state_dir graph_output_dir +This program generates gnuplot images from afl-fuzz output data. Usage: + The afl_state_dir parameter should point to an existing state directory for any active or stopped instance of afl-fuzz; while graph_output_dir should point to an empty directory where this tool can write the resulting plots to. diff --git a/afl-system-config b/afl-system-config index 28793c5b..6a495f0a 100755 --- a/afl-system-config +++ b/afl-system-config @@ -1,4 +1,17 @@ #!/bin/sh +test "$1" = "-h" && { + echo afl-system-config by Marc Heuse + echo + echo $0 + echo + echo afl-system-config has no command line options + echo + echo afl-system reconfigures the system to a high performance fuzzing state + echo WARNING: this reduces the security of the system + echo + exit 1 +} + PLATFORM=`uname -s` echo This reconfigures the system to have a better fuzzing performance if [ '!' "$EUID" = 0 ] && [ '!' `id -u` = 0 ] ; then diff --git a/afl-whatsup b/afl-whatsup index c1e41529..505f7eba 100755 --- a/afl-whatsup +++ b/afl-whatsup @@ -19,6 +19,13 @@ echo "status check tool for afl-fuzz by " echo +test "$1" = "-h" && { + echo $0 + echo + echo afl-whatsup has no command line options + echo + exit 1 +} if [ "$1" = "-s" ]; then diff --git a/docs/ChangeLog b/docs/ChangeLog index 2fc4efbc..66f71a42 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -19,11 +19,12 @@ Version ++2.53d (dev): - big code refactoring: * all includes are now in include/ - * all afl sources are now in src/ - see src/README + * all afl sources are now in src/ - see src/README.src * afl-fuzz was splitted up in various individual files for including functionality in other programs (e.g. forkserver, memory map, etc.) - or better readability. + for better readability. * new code indention everywhere + - auto-generating man pages for all (main) tools - added AFL_FORCE_UI to show the UI even if the terminal is not detected - llvm 9 is now supported (still needs testing) - Android is now supported (thank to JoeyJiao!) - still need to modify the Makefile though diff --git a/qemu_mode/build_qemu_support.sh b/qemu_mode/build_qemu_support.sh index 35f5b8ca..88726be4 100755 --- a/qemu_mode/build_qemu_support.sh +++ b/qemu_mode/build_qemu_support.sh @@ -112,7 +112,8 @@ if [ "$CKSUM" = "$QEMU_SHA384" ]; then else - echo "[-] Error: signature mismatch on $ARCHIVE (perhaps download error?)." + echo "[-] Error: signature mismatch on $ARCHIVE (perhaps download error?), removing archive ..." + rm -f "$ARCHIVE" exit 1 fi @@ -200,6 +201,8 @@ if [ "$ORIG_CPU_TARGET" = "" ]; then echo "[+] Instrumentation tests passed. " echo "[+] All set, you can now use the -Q mode in afl-fuzz!" + cd qemu_mode || exit 1 + else echo "[!] Note: can't test instrumentation when CPU_TARGET set." @@ -207,4 +210,9 @@ else fi +echo "[+] Building libcompcov ..." +make -C libcompcov +echo "[+] libcompcov ready" +echo "[+] All done for qemu_mode, enjoy!" + exit 0 diff --git a/qemu_mode/libcompcov/Makefile b/qemu_mode/libcompcov/Makefile index a1f4e31f..d078ae06 100644 --- a/qemu_mode/libcompcov/Makefile +++ b/qemu_mode/libcompcov/Makefile @@ -25,18 +25,18 @@ LDFLAGS += -ldl all: libcompcov.so compcovtest libcompcov.so: libcompcov.so.c ../../config.h - $(CC) $(CFLAGS) -shared -fPIC $< -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) -shared -fPIC $< -o ../../$@ $(LDFLAGS) .NOTPARALLEL: clean clean: rm -f *.o *.so *~ a.out core core.[1-9][0-9]* - rm -f libcompcov.so compcovtest + rm -f ../../libcompcov.so compcovtest compcovtest: compcovtest.cc $(CXX) $< -o $@ install: all - install -m 755 libcompcov.so $${DESTDIR}$(HELPER_PATH) + install -m 755 ../../libcompcov.so $${DESTDIR}$(HELPER_PATH) install -m 644 README.compcov $${DESTDIR}$(HELPER_PATH) diff --git a/src/README.src b/src/README.src new file mode 100644 index 00000000..244f5ddd --- /dev/null +++ b/src/README.src @@ -0,0 +1,22 @@ +Quick explanation about the files here: + +afl-analyze.c - afl-analyze binary tool +afl-as.c - afl-as binary tool +afl-gotcpu.c - afl-gotcpu binary tool +afl-showmap.c - afl-showmap binary tool +afl-tmin.c - afl-tmin binary tool +afl-fuzz.c - afl-fuzz binary tool (just main() and usage()) +afl-fuzz-bitmap.c - afl-fuzz bitmap handling +afl-fuzz-extras.c - afl-fuzz the *extra* function calls +afl-fuzz-globals.c - afl-fuzz global variables +afl-fuzz-init.c - afl-fuzz initialization +afl-fuzz-misc.c - afl-fuzz misc functions +afl-fuzz-one.c - afl-fuzz fuzzer_one big loop, this is where the mutation is happening +afl-fuzz-python.c - afl-fuzz the python mutator extension +afl-fuzz-queue.c - afl-fuzz handling the queue +afl-fuzz-run.c - afl-fuzz running the target +afl-fuzz-stats.c - afl-fuzz writing the statistics file +afl-gcc.c - afl-gcc binary tool (deprecated) +afl-common.c - common functions, used by afl-analyze, afl-fuzz, afl-showmap and afl-tmin +afl-forkserver.c - forkserver implementation, used by afl-fuzz and afl-tmin +afl-sharedmem.c - sharedmem implementation, used by afl-fuzz and afl-tmin diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 357672b1..e30f53b8 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -900,7 +900,7 @@ int main(int argc, char** argv) { SAYF(cCYA "afl-analyze" VERSION cRST " by \n"); - while ((opt = getopt(argc, argv, "+i:f:m:t:eQU")) > 0) + while ((opt = getopt(argc, argv, "+i:f:m:t:eQUh")) > 0) switch (opt) { @@ -988,6 +988,11 @@ int main(int argc, char** argv) { unicorn_mode = 1; break; + + case 'h': + usage(argv[0]); + return -1; + break; default: usage(argv[0]); diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index e94116f5..eb0060a4 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -131,7 +131,7 @@ int main(int argc, char** argv) { gettimeofday(&tv, &tz); init_seed = tv.tv_sec ^ tv.tv_usec ^ getpid(); - while ((opt = getopt(argc, argv, "+i:o:f:m:t:T:dnCB:S:M:x:QUe:p:s:V:E:L:")) > + while ((opt = getopt(argc, argv, "+i:o:f:m:t:T:dnCB:S:M:x:QUe:p:s:V:E:L:h")) > 0) switch (opt) { @@ -490,6 +490,11 @@ int main(int argc, char** argv) { } break; + case 'h': + usage(argv[0]); + return -1; + break; // not needed + default: usage(argv[0]); } diff --git a/src/afl-gcc.c b/src/afl-gcc.c index 2dc17baf..2f72ef34 100644 --- a/src/afl-gcc.c +++ b/src/afl-gcc.c @@ -333,6 +333,14 @@ static void edit_params(u32 argc, char** argv) { int main(int argc, char** argv) { + if (argc == 2 && strcmp(argv[1], "-h") == 0) { + printf("afl-cc" VERSION" by \n\n"); + printf("%s \n\n", argv[0]); + printf("afl-gcc has no command line options\n"); + printf("NOTE: afl-gcc is deprecated, llvm_mode is much faster and has more options\n"); + return -1; + } + if (isatty(2) && !getenv("AFL_QUIET")) { SAYF(cCYA "afl-cc" VERSION cRST " by \n"); diff --git a/src/afl-gotcpu.c b/src/afl-gotcpu.c index a39659bb..85864c6f 100644 --- a/src/afl-gotcpu.c +++ b/src/afl-gotcpu.c @@ -127,6 +127,14 @@ repeat_loop: int main(int argc, char** argv) { + if (argc > 1) { + printf("afl-gotcpu" VERSION " by \n"); + printf("\n%s \n\n", argv[0]); + printf("afl-gotcpu does not have command line options\n"); + printf("afl-gotcpu prints out which CPUs are available\n"); + return -1; + } + #ifdef HAVE_AFFINITY u32 cpu_cnt = sysconf(_SC_NPROCESSORS_ONLN), idle_cpus = 0, maybe_cpus = 0, i; diff --git a/src/afl-showmap.c b/src/afl-showmap.c index f3b6c561..6aa72746 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -563,7 +563,7 @@ int main(int argc, char** argv) { doc_path = access(DOC_PATH, F_OK) ? "docs" : DOC_PATH; - while ((opt = getopt(argc, argv, "+o:m:t:A:eqZQUbcr")) > 0) + while ((opt = getopt(argc, argv, "+o:m:t:A:eqZQUbcrh")) > 0) switch (opt) { @@ -691,6 +691,11 @@ int main(int argc, char** argv) { if (edges_only) FATAL("-e and -r are mutually exclusive"); raw_instr_output = 1; break; + + case 'h': + usage(argv[0]); + return -1; + break; default: usage(argv[0]); diff --git a/src/afl-tmin.c b/src/afl-tmin.c index a501b068..baf22557 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -1091,7 +1091,7 @@ int main(int argc, char** argv) { SAYF(cCYA "afl-tmin" VERSION cRST " by \n"); - while ((opt = getopt(argc, argv, "+i:o:f:m:t:B:xeQU")) > 0) + while ((opt = getopt(argc, argv, "+i:o:f:m:t:B:xeQUh")) > 0) switch (opt) { @@ -1211,6 +1211,11 @@ int main(int argc, char** argv) { mask_bitmap = ck_alloc(MAP_SIZE); read_bitmap(optarg); break; + + case 'h': + usage(argv[0]); + return -1; + break; default: usage(argv[0]); -- cgit 1.4.1 From abf61ecc8f1b4ea3de59f818d859139637b29f32 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 4 Sep 2019 16:15:42 +0200 Subject: add to docs --- docs/status_screen.txt | 3 +++ 1 file changed, 3 insertions(+) (limited to 'docs') diff --git a/docs/status_screen.txt b/docs/status_screen.txt index 37a39a99..c6f9f791 100644 --- a/docs/status_screen.txt +++ b/docs/status_screen.txt @@ -407,6 +407,9 @@ directory. This includes: - variable_paths - number of test cases showing variable behavior - unique_crashes - number of unique crashes recorded - unique_hangs - number of unique hangs encountered + - command_line - full command line used for the fuzzing session + - slowest_exec_ms- real time of the slowest execution in seconds + - peak_rss_mb - max rss usage reached during fuzzing in MB Most of these map directly to the UI elements discussed earlier on. -- cgit 1.4.1