From 45be91ff48554569be7f1ba6e4fc1de57e7286c3 Mon Sep 17 00:00:00 2001 From: Hexcoder Date: Tue, 25 Jun 2019 22:03:59 +0200 Subject: experimental implementation of counters that skip zero on overflow. Enable with AFL_NZERO_COUNTS=1 during compilation of target. --- llvm_mode/afl-llvm-pass.so.cc | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) (limited to 'llvm_mode') diff --git a/llvm_mode/afl-llvm-pass.so.cc b/llvm_mode/afl-llvm-pass.so.cc index d46db7c0..2d283f1f 100644 --- a/llvm_mode/afl-llvm-pass.so.cc +++ b/llvm_mode/afl-llvm-pass.so.cc @@ -118,6 +118,9 @@ bool AFLCoverage::runOnModule(Module &M) { } + char* neverZero_counters_str = getenv("AFL_NZERO_COUNTS"); + bool enable_neverZero_counters = neverZero_counters_str && '1' == *neverZero_counters_str; + /* Get globals for the SHM region and the previous location. Note that __afl_prev_loc is thread-local. */ @@ -234,7 +237,32 @@ bool AFLCoverage::runOnModule(Module &M) { LoadInst *Counter = IRB.CreateLoad(MapPtrIdx); Counter->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - Value *Incr = IRB.CreateAdd(Counter, ConstantInt::get(Int8Ty, 1)); + + Value *Incr; + if (enable_neverZero_counters) { + /* hexcoder: Realize a counter that skips zero during overflow. + * Once this counter reaches its maximum value, it next increments to 1 + * + * Instead of + * Counter + 1 -> Counter + * we inject now this + * Counter + 1 -> {Counter, OverflowFlag} + * Counter + OverflowFlag -> Counter + */ + CallInst *AddOv = IRB.CreateBinaryIntrinsic(Intrinsic::uadd_with_overflow, + Counter, ConstantInt::get(Int8Ty, 1)); + AddOv->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + Value *SumWithOverflowBit = AddOv; + Incr = IRB.CreateAdd( + IRB.CreateExtractValue(SumWithOverflowBit, 0), /* sum */ + IRB.CreateZExt( /* convert from one bit type to 8 bits type */ + IRB.CreateExtractValue(SumWithOverflowBit, 1) /* overflow */ + , Int8Ty)); + } else { + /* standard AFL behavior: wrapping counters */ + Incr = IRB.CreateAdd(Counter, ConstantInt::get(Int8Ty, 1)); + } + IRB.CreateStore(Incr, MapPtrIdx) ->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); -- cgit 1.4.1 From aa4fc44a8018326c97e681fe4663b49de173b5a5 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 27 Jun 2019 15:43:51 +0200 Subject: 2 different implementations --- llvm_mode/afl-llvm-pass.so.cc | 53 +++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 19 deletions(-) (limited to 'llvm_mode') diff --git a/llvm_mode/afl-llvm-pass.so.cc b/llvm_mode/afl-llvm-pass.so.cc index 2d283f1f..d20ca8dd 100644 --- a/llvm_mode/afl-llvm-pass.so.cc +++ b/llvm_mode/afl-llvm-pass.so.cc @@ -119,7 +119,6 @@ bool AFLCoverage::runOnModule(Module &M) { } char* neverZero_counters_str = getenv("AFL_NZERO_COUNTS"); - bool enable_neverZero_counters = neverZero_counters_str && '1' == *neverZero_counters_str; /* Get globals for the SHM region and the previous location. Note that __afl_prev_loc is thread-local. */ @@ -230,16 +229,16 @@ bool AFLCoverage::runOnModule(Module &M) { LoadInst *MapPtr = IRB.CreateLoad(AFLMapPtr); MapPtr->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - Value *MapPtrIdx = - IRB.CreateGEP(MapPtr, IRB.CreateXor(PrevLocCasted, CurLoc)); + Value *MapPtrIdx = IRB.CreateGEP(MapPtr, IRB.CreateXor(PrevLocCasted, CurLoc)); /* Update bitmap */ LoadInst *Counter = IRB.CreateLoad(MapPtrIdx); Counter->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - Value *Incr; - if (enable_neverZero_counters) { + Value *Incr = IRB.CreateAdd(Counter, ConstantInt::get(Int8Ty, 1)); + + if (neverZero_counters_str != NULL) { /* hexcoder: Realize a counter that skips zero during overflow. * Once this counter reaches its maximum value, it next increments to 1 * @@ -249,27 +248,43 @@ bool AFLCoverage::runOnModule(Module &M) { * Counter + 1 -> {Counter, OverflowFlag} * Counter + OverflowFlag -> Counter */ - CallInst *AddOv = IRB.CreateBinaryIntrinsic(Intrinsic::uadd_with_overflow, - Counter, ConstantInt::get(Int8Ty, 1)); + + // Solution #1 - creates + //mov dl,BYTE PTR [rsi+rdi*1] + //mov ecx,edx + //add cl,0x1 + //adc dl,0x1 + /* + CallInst *AddOv = IRB.CreateBinaryIntrinsic(Intrinsic::uadd_with_overflow, Counter, ConstantInt::get(Int8Ty, 1)); AddOv->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); Value *SumWithOverflowBit = AddOv; - Incr = IRB.CreateAdd( - IRB.CreateExtractValue(SumWithOverflowBit, 0), /* sum */ - IRB.CreateZExt( /* convert from one bit type to 8 bits type */ - IRB.CreateExtractValue(SumWithOverflowBit, 1) /* overflow */ - , Int8Ty)); - } else { - /* standard AFL behavior: wrapping counters */ - Incr = IRB.CreateAdd(Counter, ConstantInt::get(Int8Ty, 1)); + Incr = IRB.CreateAdd(IRB.CreateExtractValue(SumWithOverflowBit, 0), // sum + IRB.CreateZExt( // convert from one bit type to 8 bits type + IRB.CreateExtractValue(SumWithOverflowBit, 1), // overflow + Int8Ty)); + */ + // Solution #2 - creates the same code as #1 + ///* + auto cf = IRB.CreateICmpULT(Incr, ConstantInt::get(Int8Ty, 1)); + Incr = IRB.CreateAdd(Incr, cf); + //*/ + + // Solution #3 - creates + //mov cl,BYTE PTR [rsi+rdx*1] + //add cl,0x1 + //cmp cl,0x1 + //adc cl,0x0 + /* + auto cf = IRB.CreateICmpEQ(Incr, ConstantInt::get(Int8Ty, 0)); + Incr = IRB.CreateAdd(Incr, cf); + */ } - IRB.CreateStore(Incr, MapPtrIdx) - ->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + IRB.CreateStore(Incr, MapPtrIdx)->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); /* Set prev_loc to cur_loc >> 1 */ - StoreInst *Store = - IRB.CreateStore(ConstantInt::get(Int32Ty, cur_loc >> 1), AFLPrevLoc); + StoreInst *Store = IRB.CreateStore(ConstantInt::get(Int32Ty, cur_loc >> 1), AFLPrevLoc); Store->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); inst_blocks++; -- cgit 1.4.1 From d9ff84e39ecad47deec8808ea127fd90d9f5e8ef Mon Sep 17 00:00:00 2001 From: Heiko Eißfeldt Date: Sun, 30 Jun 2019 10:06:20 +0200 Subject: Refactor to use an alternative method for shared memory. If USEMMAP is defined, the shared memory segment is created/attached etc. now by shm_open() and mmap(). This API is hopefully more often available (at least for iOS). In order to reduce code duplication I have added new files sharedmem.[ch] which now encapsulate the shared memory method. This is based on the work of Proteas to support iOS fuzzing (thanks). https://github.com/Proteas/afl-ios/commit/866af8ad1cb230d5d753b546380a4af1e55d6946 Currently this is in an experimental status yet. Please report whether this variant works on 32 and 64 bit and on the supported platforms. This branch enables USEMMAP and has been tested on Linux. There is no auto detection for the mmap API yet. --- Makefile | 23 ++++---- afl-analyze.c | 40 +++---------- afl-as.h | 46 +++++++++++++++ afl-fuzz.c | 57 ++++--------------- afl-showmap.c | 41 ++------------ afl-tmin.c | 45 +++------------ llvm_mode/Makefile | 2 +- llvm_mode/afl-llvm-rt.o.c | 27 +++++++++ sharedmem.c | 139 ++++++++++++++++++++++++++++++++++++++++++++++ sharedmem.h | 6 ++ 10 files changed, 262 insertions(+), 164 deletions(-) create mode 100644 sharedmem.c create mode 100644 sharedmem.h (limited to 'llvm_mode') diff --git a/Makefile b/Makefile index 1d8d819f..4f85b53a 100644 --- a/Makefile +++ b/Makefile @@ -30,12 +30,12 @@ SH_PROGS = afl-plot afl-cmin afl-whatsup afl-system-config CFLAGS ?= -O3 -funroll-loops CFLAGS += -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign \ -DAFL_PATH=\"$(HELPER_PATH)\" -DDOC_PATH=\"$(DOC_PATH)\" \ - -DBIN_PATH=\"$(BIN_PATH)\" + -DBIN_PATH=\"$(BIN_PATH)\" -DUSEMMAP=1 PYTHON_INCLUDE ?= /usr/include/python2.7 ifneq "$(filter Linux GNU%,$(shell uname))" "" - LDFLAGS += -ldl + LDFLAGS += -ldl -lrt endif ifeq "$(findstring clang, $(shell $(CC) --version 2>/dev/null))" "" @@ -92,17 +92,20 @@ afl-as: afl-as.c afl-as.h $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) $@.c -o $@ $(LDFLAGS) ln -sf afl-as as -afl-fuzz: afl-fuzz.c $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $@.c -o $@ $(LDFLAGS) $(PYFLAGS) +sharedmem.o : sharedmem.c + $(CC) $(CFLAGS) -c sharedmem.c -afl-showmap: afl-showmap.c $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $@.c -o $@ $(LDFLAGS) +afl-fuzz: afl-fuzz.c sharedmem.o $(COMM_HDR) | test_x86 + $(CC) $(CFLAGS) $@.c sharedmem.o -o $@ $(LDFLAGS) $(PYFLAGS) -afl-tmin: afl-tmin.c $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $@.c -o $@ $(LDFLAGS) +afl-showmap: afl-showmap.c sharedmem.o $(COMM_HDR) | test_x86 + $(CC) $(CFLAGS) $@.c sharedmem.o -o $@ $(LDFLAGS) -afl-analyze: afl-analyze.c $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $@.c -o $@ $(LDFLAGS) +afl-tmin: afl-tmin.c sharedmem.o $(COMM_HDR) | test_x86 + $(CC) $(CFLAGS) $@.c sharedmem.o -o $@ $(LDFLAGS) + +afl-analyze: afl-analyze.c sharedmem.o $(COMM_HDR) | test_x86 + $(CC) $(CFLAGS) $@.c sharedmem.o -o $@ $(LDFLAGS) afl-gotcpu: afl-gotcpu.c $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) $@.c -o $@ $(LDFLAGS) diff --git a/afl-analyze.c b/afl-analyze.c index 44be73f9..834a0357 100644 --- a/afl-analyze.c +++ b/afl-analyze.c @@ -26,6 +26,7 @@ #include "debug.h" #include "alloc-inl.h" #include "hash.h" +#include "sharedmem.h" #include #include @@ -47,7 +48,7 @@ static s32 child_pid; /* PID of the tested program */ -static u8* trace_bits; /* SHM with instrumentation bitmap */ + u8* trace_bits; /* SHM with instrumentation bitmap */ static u8 *in_file, /* Analyzer input test case */ *prog_in, /* Targeted program input file */ @@ -64,8 +65,7 @@ static u32 in_len, /* Input data length */ static u64 mem_limit = MEM_LIMIT; /* Memory limit (MB) */ -static s32 shm_id, /* ID of the SHM region */ - dev_null_fd = -1; /* FD to /dev/null */ +static s32 dev_null_fd = -1; /* FD to /dev/null */ static u8 edges_only, /* Ignore hit counts? */ use_hex_offsets, /* Show hex offsets? */ @@ -76,6 +76,7 @@ static volatile u8 child_timed_out; /* Child timed out? */ + /* Constants used for describing byte behavior. */ #define RESP_NONE 0x00 /* Changing byte is a no-op. */ @@ -141,37 +142,11 @@ static inline u8 anything_set(void) { } -/* Get rid of shared memory and temp files (atexit handler). */ +/* Get rid of temp files (atexit handler). */ -static void remove_shm(void) { +static void at_exit_handler(void) { unlink(prog_in); /* Ignore errors */ - shmctl(shm_id, IPC_RMID, NULL); - -} - - -/* Configure shared memory. */ - -static void setup_shm(void) { - - u8* shm_str; - - shm_id = shmget(IPC_PRIVATE, MAP_SIZE, IPC_CREAT | IPC_EXCL | 0600); - - if (shm_id < 0) PFATAL("shmget() failed"); - - atexit(remove_shm); - - shm_str = alloc_printf("%d", shm_id); - - setenv(SHM_ENV_VAR, shm_str, 1); - - ck_free(shm_str); - - trace_bits = shmat(shm_id, NULL, 0); - - if (!trace_bits) PFATAL("shmat() failed"); } @@ -1036,7 +1011,8 @@ int main(int argc, char** argv) { use_hex_offsets = !!getenv("AFL_ANALYZE_HEX"); - setup_shm(); + setup_shm(0); + atexit(at_exit_handler); setup_signal_handlers(); set_up_environment(); diff --git a/afl-as.h b/afl-as.h index ebd57109..90944614 100644 --- a/afl-as.h +++ b/afl-as.h @@ -220,6 +220,29 @@ static const u8* main_payload_32 = " testl %eax, %eax\n" " je __afl_setup_abort\n" "\n" +#ifdef USEMMAP + " pushl $384 /* shm_open mode 0600 */\n" + " pushl $2 /* flags O_RDWR */\n" + " pushl %eax /* SHM file path */\n" + " call shm_open\n" + " addl $12, %esp\n" + "\n" + " cmpl $-1, %eax\n" + " je __afl_setup_abort\n" + "\n" + " pushl $0 /* mmap off */\n" + " pushl %eax /* shm fd */\n" + " pushl $1 /* mmap flags */\n" + " pushl $3 /* mmap prot */\n" + " pushl $"STRINGIFY(MAP_SIZE)" /* mmap len */\n" + " pushl $0 /* mmap addr */\n" + " call mmap\n" + " addl $12, %esp\n" + "\n" + " cmpl $-1, %eax\n" + " je __afl_setup_abort\n" + "\n" +#else " pushl %eax\n" " call atoi\n" " addl $4, %esp\n" @@ -233,6 +256,7 @@ static const u8* main_payload_32 = " cmpl $-1, %eax\n" " je __afl_setup_abort\n" "\n" +#endif " /* Store the address of the SHM region. */\n" "\n" " movl %eax, __afl_area_ptr\n" @@ -501,6 +525,27 @@ static const u8* main_payload_64 = " testq %rax, %rax\n" " je __afl_setup_abort\n" "\n" +#ifdef USEMMAP + " movl $384, %edx /* shm_open mode 0600 */\n" + " movl $2, %esi /* flags O_RDWR */\n" + " movq %rax, %rdi /* SHM file path */\n" + CALL_L64("shm_open") + "\n" + " cmpq $-1, %rax\n" + " je __afl_setup_abort\n" + "\n" + " movl $0, %r9d\n" + " movl %eax, %r8d\n" + " movl $1, %ecx\n" + " movl $3, %edx\n" + " movl $"STRINGIFY(MAP_SIZE)", %esi\n" + " movl $0, %edi\n" + CALL_L64("mmap") + "\n" + " cmpq $-1, %rax\n" + " je __afl_setup_abort\n" + "\n" +#else " movq %rax, %rdi\n" CALL_L64("atoi") "\n" @@ -512,6 +557,7 @@ static const u8* main_payload_64 = " cmpq $-1, %rax\n" " je __afl_setup_abort\n" "\n" +#endif " /* Store the address of the SHM region. */\n" "\n" " movq %rax, %rdx\n" diff --git a/afl-fuzz.c b/afl-fuzz.c index 6db99acf..af507c88 100644 --- a/afl-fuzz.c +++ b/afl-fuzz.c @@ -31,6 +31,7 @@ #include "debug.h" #include "alloc-inl.h" #include "hash.h" +#include "sharedmem.h" #include #include @@ -163,7 +164,7 @@ static s32 forksrv_pid, /* PID of the fork server */ child_pid = -1, /* PID of the fuzzed program */ out_dir_fd = -1; /* FD of the lock file */ -EXP_ST u8* trace_bits; /* SHM with instrumentation bitmap */ + u8* trace_bits; /* SHM with instrumentation bitmap */ EXP_ST u8 virgin_bits[MAP_SIZE], /* Regions yet untouched by fuzzing */ virgin_tmout[MAP_SIZE], /* Bits we haven't seen in tmouts */ @@ -171,8 +172,6 @@ EXP_ST u8 virgin_bits[MAP_SIZE], /* Regions yet untouched by fuzzing */ static u8 var_bytes[MAP_SIZE]; /* Bytes that appear to be variable */ -static s32 shm_id; /* ID of the SHM region */ - static volatile u8 stop_soon, /* Ctrl-C pressed? */ clear_screen = 1, /* Window resized? */ child_timed_out; /* Traced process timed out? */ @@ -1444,15 +1443,6 @@ static inline void classify_counts(u32* mem) { #endif /* ^__x86_64__ */ -/* Get rid of shared memory (atexit handler). */ - -static void remove_shm(void) { - - shmctl(shm_id, IPC_RMID, NULL); - -} - - /* Compact trace bytes into a smaller bitmap. We effectively just drop the count information here. This is called only sporadically, for some new paths. */ @@ -1606,40 +1596,6 @@ static void cull_queue(void) { } -/* Configure shared memory and virgin_bits. This is called at startup. */ - -EXP_ST void setup_shm(void) { - - u8* shm_str; - - if (!in_bitmap) memset(virgin_bits, 255, MAP_SIZE); - - memset(virgin_tmout, 255, MAP_SIZE); - memset(virgin_crash, 255, MAP_SIZE); - - shm_id = shmget(IPC_PRIVATE, MAP_SIZE, IPC_CREAT | IPC_EXCL | 0600); - - if (shm_id < 0) PFATAL("shmget() failed"); - - atexit(remove_shm); - - shm_str = alloc_printf("%d", shm_id); - - /* If somebody is asking us to fuzz instrumented binaries in dumb mode, - we don't want them to detect instrumentation, since we won't be sending - fork server commands. This should be replaced with better auto-detection - later on, perhaps? */ - - if (!dumb_mode) setenv(SHM_ENV_VAR, shm_str, 1); - - ck_free(shm_str); - - trace_bits = shmat(shm_id, NULL, 0); - - if (!trace_bits) PFATAL("shmat() failed"); - -} - /* Load postprocessor, if available. */ @@ -7466,8 +7422,10 @@ EXP_ST void check_binary(u8* fname) { #else +#if !defined(__arm__) && !defined(__arm64__) if (f_data[0] != 0xCF || f_data[1] != 0xFA || f_data[2] != 0xED) FATAL("Program '%s' is not a 64-bit Mach-O binary", target_path); +#endif #endif /* ^!__APPLE__ */ @@ -8639,7 +8597,12 @@ int main(int argc, char** argv) { check_cpu_governor(); setup_post(); - setup_shm(); + setup_shm(dumb_mode); + + if (!in_bitmap) memset(virgin_bits, 255, MAP_SIZE); + memset(virgin_tmout, 255, MAP_SIZE); + memset(virgin_crash, 255, MAP_SIZE); + init_count_class16(); setup_dirs_fds(); diff --git a/afl-showmap.c b/afl-showmap.c index 316490d8..a5d7568a 100644 --- a/afl-showmap.c +++ b/afl-showmap.c @@ -28,6 +28,7 @@ #include "debug.h" #include "alloc-inl.h" #include "hash.h" +#include "sharedmem.h" #include #include @@ -48,7 +49,7 @@ static s32 child_pid; /* PID of the tested program */ -static u8* trace_bits; /* SHM with instrumentation bitmap */ + u8* trace_bits; /* SHM with instrumentation bitmap */ static u8 *out_file, /* Trace output file */ *doc_path, /* Path to docs */ @@ -59,8 +60,6 @@ static u32 exec_tmout; /* Exec timeout (ms) */ static u64 mem_limit = MEM_LIMIT; /* Memory limit (MB) */ -static s32 shm_id; /* ID of the SHM region */ - static u8 quiet_mode, /* Hide non-essential messages? */ edges_only, /* Ignore hit counts? */ cmin_mode, /* Generate output in afl-cmin mode? */ @@ -72,6 +71,7 @@ static volatile u8 child_timed_out, /* Child timed out? */ child_crashed; /* Child crashed? */ + /* Classify tuple counts. Instead of mapping to individual bits, as in afl-fuzz.c, we map to more user-friendly numbers between 1 and 8. */ @@ -126,39 +126,6 @@ static void classify_counts(u8* mem, const u8* map) { } -/* Get rid of shared memory (atexit handler). */ - -static void remove_shm(void) { - - shmctl(shm_id, IPC_RMID, NULL); - -} - - -/* Configure shared memory. */ - -static void setup_shm(void) { - - u8* shm_str; - - shm_id = shmget(IPC_PRIVATE, MAP_SIZE, IPC_CREAT | IPC_EXCL | 0600); - - if (shm_id < 0) PFATAL("shmget() failed"); - - atexit(remove_shm); - - shm_str = alloc_printf("%d", shm_id); - - setenv(SHM_ENV_VAR, shm_str, 1); - - ck_free(shm_str); - - trace_bits = shmat(shm_id, NULL, 0); - - if (!trace_bits) PFATAL("shmat() failed"); - -} - /* Write results. */ static u32 write_results(void) { @@ -741,7 +708,7 @@ int main(int argc, char** argv) { if (optind == argc || !out_file) usage(argv[0]); - setup_shm(); + setup_shm(0); setup_signal_handlers(); set_up_environment(); diff --git a/afl-tmin.c b/afl-tmin.c index a42be6e9..b9dbb2b5 100644 --- a/afl-tmin.c +++ b/afl-tmin.c @@ -26,6 +26,7 @@ #include "debug.h" #include "alloc-inl.h" #include "hash.h" +#include "sharedmem.h" #include #include @@ -50,8 +51,8 @@ static s32 forksrv_pid, /* PID of the fork server */ static s32 fsrv_ctl_fd, /* Fork server control pipe (write) */ fsrv_st_fd; /* Fork server status pipe (read) */ -static u8 *trace_bits, /* SHM with instrumentation bitmap */ - *mask_bitmap; /* Mask for trace bits (-B) */ + u8 *trace_bits; /* SHM with instrumentation bitmap */ +static u8 *mask_bitmap; /* Mask for trace bits (-B) */ static u8 *in_file, /* Minimizer input test case */ *out_file, /* Minimizer output file */ @@ -73,8 +74,7 @@ static u32 in_len, /* Input data length */ static u64 mem_limit = MEM_LIMIT; /* Memory limit (MB) */ -static s32 shm_id, /* ID of the SHM region */ - dev_null_fd = -1; /* FD to /dev/null */ +static s32 dev_null_fd = -1; /* FD to /dev/null */ static u8 crash_mode, /* Crash-centric mode? */ exit_crash, /* Treat non-zero exit as crash? */ @@ -159,42 +159,12 @@ static inline u8 anything_set(void) { } +/* Get rid of temp files (atexit handler). */ -/* Get rid of shared memory and temp files (atexit handler). */ - -static void remove_shm(void) { - +static void at_exit_handler(void) { if (prog_in) unlink(prog_in); /* Ignore errors */ - shmctl(shm_id, IPC_RMID, NULL); - -} - - -/* Configure shared memory. */ - -static void setup_shm(void) { - - u8* shm_str; - - shm_id = shmget(IPC_PRIVATE, MAP_SIZE, IPC_CREAT | IPC_EXCL | 0600); - - if (shm_id < 0) PFATAL("shmget() failed"); - - atexit(remove_shm); - - shm_str = alloc_printf("%d", shm_id); - - setenv(SHM_ENV_VAR, shm_str, 1); - - ck_free(shm_str); - - trace_bits = shmat(shm_id, NULL, 0); - - if (!trace_bits) PFATAL("shmat() failed"); - } - /* Read initial file. */ static void read_initial_file(void) { @@ -1245,7 +1215,8 @@ int main(int argc, char** argv) { if (optind == argc || !in_file || !out_file) usage(argv[0]); - setup_shm(); + setup_shm(0); + atexit(at_exit_handler); setup_signal_handlers(); set_up_environment(); diff --git a/llvm_mode/Makefile b/llvm_mode/Makefile index 6b277536..0cb2e1c5 100644 --- a/llvm_mode/Makefile +++ b/llvm_mode/Makefile @@ -33,7 +33,7 @@ endif CFLAGS ?= -O3 -funroll-loops CFLAGS += -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ - -DVERSION=\"$(VERSION)\" + -DVERSION=\"$(VERSION)\" -DUSEMMAP=1 -lrt ifdef AFL_TRACE_PC CFLAGS += -DUSE_TRACE_PC=1 endif diff --git a/llvm_mode/afl-llvm-rt.o.c b/llvm_mode/afl-llvm-rt.o.c index 342dcc90..debde204 100644 --- a/llvm_mode/afl-llvm-rt.o.c +++ b/llvm_mode/afl-llvm-rt.o.c @@ -44,6 +44,9 @@ # define CONST_PRIO 0 #endif /* ^USE_TRACE_PC */ +#include +#include + /* Globals needed by the injected instrumentation. The __afl_area_initial region is used for instrumentation output before __afl_map_shm() has a chance to run. @@ -71,10 +74,34 @@ static void __afl_map_shm(void) { hacky .init code to work correctly in projects such as OpenSSL. */ if (id_str) { +#ifdef USEMMAP + const char *shm_file_path = id_str; + int shm_fd = -1; + unsigned char *shm_base = NULL; + + /* create the shared memory segment as if it was a file */ + shm_fd = shm_open(shm_file_path, O_RDWR, 0600); + if (shm_fd == -1) { + printf("shm_open() failed\n"); + exit(1); + } + + /* map the shared memory segment to the address space of the process */ + shm_base = mmap(0, MAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd, 0); + if (shm_base == MAP_FAILED) { + close(shm_fd); + shm_fd = -1; + printf("mmap() failed\n"); + exit(2); + } + + __afl_area_ptr = shm_base; +#else u32 shm_id = atoi(id_str); __afl_area_ptr = shmat(shm_id, NULL, 0); +#endif /* Whooooops. */ diff --git a/sharedmem.c b/sharedmem.c new file mode 100644 index 00000000..23cc8984 --- /dev/null +++ b/sharedmem.c @@ -0,0 +1,139 @@ +/* + + */ + +#define AFL_MAIN + +#include "config.h" +#include "types.h" +#include "debug.h" +#include "alloc-inl.h" +#include "hash.h" +#include "sharedmem.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +extern unsigned char*trace_bits; + +#ifdef USEMMAP +/* ================ Proteas ================ */ +int g_shm_fd = -1; +unsigned char *g_shm_base = NULL; +char g_shm_file_path[L_tmpnam]; +/* ========================================= */ +#else +static s32 shm_id; /* ID of the SHM region */ +#endif + +/* Get rid of shared memory (atexit handler). */ + +void remove_shm(void) { +#ifdef USEMMAP + if (g_shm_base != NULL) { + munmap(g_shm_base, MAP_SIZE); + g_shm_base = NULL; + } + + if (g_shm_fd != -1) { + close(g_shm_fd); + g_shm_fd = -1; + } +#else + shmctl(shm_id, IPC_RMID, NULL); +#endif +} + + +/* Configure shared memory. */ + +void setup_shm(unsigned char dumb_mode) { +#ifdef USEMMAP + /* generate random file name for multi instance */ + memset(g_shm_file_path, 0x0, L_tmpnam); + + char *result = tmpnam(g_shm_file_path); + if (result == 0) + PFATAL("cannot generate filename for shared memory"); + + /* get rid of second slash in /tmp/blabla */ + g_shm_file_path[4] = '_'; + + /* create the shared memory segment as if it was a file */ + g_shm_fd = shm_open(g_shm_file_path, O_CREAT | O_RDWR | O_EXCL, 0600); + if (g_shm_fd == -1) { + PFATAL("shm_open() failed"); + } + + /* configure the size of the shared memory segment */ + if (ftruncate(g_shm_fd, MAP_SIZE)) { + PFATAL("setup_shm(): ftruncate() failed"); + } + + /* map the shared memory segment to the address space of the process */ + g_shm_base = mmap(0, MAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, g_shm_fd, 0); + if (g_shm_base == MAP_FAILED) { + close(g_shm_fd); + g_shm_fd = -1; + PFATAL("mmap() failed"); + } + + atexit(remove_shm); + + /* If somebody is asking us to fuzz instrumented binaries in dumb mode, + we don't want them to detect instrumentation, since we won't be sending + fork server commands. This should be replaced with better auto-detection + later on, perhaps? */ + + if (!dumb_mode) setenv(SHM_ENV_VAR, g_shm_file_path, 1); + + trace_bits = g_shm_base; + + if (!trace_bits) PFATAL("mmap() failed"); + +#else + u8* shm_str; + + shm_id = shmget(IPC_PRIVATE, MAP_SIZE, IPC_CREAT | IPC_EXCL | 0600); + + if (shm_id < 0) PFATAL("shmget() failed"); + + atexit(remove_shm); + + shm_str = alloc_printf("%d", shm_id); + + setenv(SHM_ENV_VAR, shm_str, 1); + + /* If somebody is asking us to fuzz instrumented binaries in dumb mode, + we don't want them to detect instrumentation, since we won't be sending + fork server commands. This should be replaced with better auto-detection + later on, perhaps? */ + + if (!dumb_mode) setenv(SHM_ENV_VAR, shm_str, 1); + + ck_free(shm_str); + + trace_bits = shmat(shm_id, NULL, 0); + + if (!trace_bits) PFATAL("shmat() failed"); + +#endif +} + diff --git a/sharedmem.h b/sharedmem.h new file mode 100644 index 00000000..9300ccf1 --- /dev/null +++ b/sharedmem.h @@ -0,0 +1,6 @@ +#ifndef SHAREDMEM +#define SHAREDMEM + +void setup_shm(unsigned char dumb_mode); +void remove_shm(void); +#endif -- cgit 1.4.1 From 37a379f959aecc465e79197c7adafa42ff5702d9 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Tue, 2 Jul 2019 00:26:27 +0200 Subject: Makefile magic for llvm_mode --- Makefile | 31 ++++++++++++++++++++++--------- llvm_mode/Makefile | 44 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 62 insertions(+), 13 deletions(-) (limited to 'llvm_mode') diff --git a/Makefile b/Makefile index b051b497..2075b90f 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,9 @@ # http://www.apache.org/licenses/LICENSE-2.0 # +# For Heiko: +#TEST_MMAP=1 + PROGNAME = afl VERSION = $(shell grep '^\#define VERSION ' config.h | cut -d '"' -f2) @@ -46,6 +49,7 @@ endif COMM_HDR = alloc-inl.h config.h debug.h types.h + ifeq "$(shell echo '\#include XXXvoid main() {}' | sed 's/XXX/\n/g' | $(CC) -x c - -o .test -I$(PYTHON_INCLUDE) -lpython2.7 && echo 1 || echo 0 )" "1" PYTHON_OK=1 PYFLAGS=-DUSE_PYTHON -I$(PYTHON_INCLUDE) -lpython2.7 @@ -54,12 +58,19 @@ else PYFLAGS= endif + ifeq "$(shell echo '\#include XXX\#include XXX\#include XXXvoid main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, NULL);}' | sed 's/XXX/\n/g' | $(CC) -x c - -o .test2 && echo 1 || echo 0 )" "1" - SHM_OK=1 + SHMAT_OK=1 +else + SHMAT_OK=0 + CFLAGS+=-DUSEMMAP=1 + LDFLAGS+=-Wno-deprecated-declarations +endif + +ifeq "$(TEST_MMAP)" "1" + SHMAT_OK=0 CFLAGS+=-DUSEMMAP=1 LDFLAGS+=-Wno-deprecated-declarations -else - SHM_OK=0 endif @@ -81,16 +92,16 @@ test_x86: endif -ifeq "$(SHM_OK)" "1" +ifeq "$(SHMAT_OK)" "1" test_shm: - @rm -f .test2 2> /dev/null - @echo "[+] shmem seems to be working." + @echo "[+] shmat seems to be working." + @rm -f .test2 else test_shm: - @echo "[-] shmem seems not to be working, switchig to mmap implementation" + @echo "[-] shmat seems not to be working, switchig to mmap implementation" endif @@ -138,6 +149,7 @@ afl-analyze: afl-analyze.c sharedmem.o $(COMM_HDR) | test_x86 afl-gotcpu: afl-gotcpu.c $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) $@.c -o $@ $(LDFLAGS) + ifndef AFL_NO_X86 test_build: afl-gcc afl-as afl-showmap @@ -156,11 +168,12 @@ test_build: afl-gcc afl-as afl-showmap endif + all_done: test_build @if [ ! "`which clang 2>/dev/null`" = "" ]; then echo "[+] LLVM users: see llvm_mode/README.llvm for a faster alternative to afl-gcc."; fi @echo "[+] All done! Be sure to review the README - it's pretty short and useful." -ifeq "$(SHM_OK)" "0" - @echo "[!] shmem isn't working on your platform - compile every target with -lrt:" +ifeq "$(SHMAT_OK)" "0" + @echo "[!] shmat isn't working on your platform - compile every target with -lrt:" @echo "[!] CFLAGS=-lrt LDFLAGS=-lrt CC=afl-gcc CXX=afl-g++ ./configure" endif @if [ "`uname`" = "Darwin" ]; then printf "\nWARNING: Fuzzing on MacOS X is slow because of the unusually high overhead of\nfork() on this OS. Consider using Linux or *BSD. You can also use VirtualBox\n(virtualbox.org) to put AFL inside a Linux or *BSD VM.\n\n"; fi diff --git a/llvm_mode/Makefile b/llvm_mode/Makefile index 0cb2e1c5..9f3cd010 100644 --- a/llvm_mode/Makefile +++ b/llvm_mode/Makefile @@ -16,6 +16,9 @@ # http://www.apache.org/licenses/LICENSE-2.0 # +# For Heiko: +#TEST_MMAP=1 + PREFIX ?= /usr/local HELPER_PATH = $(PREFIX)/lib/afl BIN_PATH = $(PREFIX)/bin @@ -33,7 +36,7 @@ endif CFLAGS ?= -O3 -funroll-loops CFLAGS += -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ - -DVERSION=\"$(VERSION)\" -DUSEMMAP=1 -lrt + -DVERSION=\"$(VERSION)\" ifdef AFL_TRACE_PC CFLAGS += -DUSE_TRACE_PC=1 endif @@ -45,12 +48,26 @@ CXXFLAGS += -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign \ CLANG_CFL = `$(LLVM_CONFIG) --cxxflags` -Wl,-znodelete -fno-rtti -fpic $(CXXFLAGS) CLANG_LFL = `$(LLVM_CONFIG) --ldflags` $(LDFLAGS) -# User teor2345 reports that this is required to make things work on MacOS X. +# User teor2345 reports that this is required to make things work on MacOS X. ifeq "$(shell uname)" "Darwin" CLANG_LFL += -Wl,-flat_namespace -Wl,-undefined,suppress endif + +ifeq "$(shell echo '\#include XXX\#include XXX\#include XXXvoid main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, NULL);}' | sed 's/XXX/\n/g' | $(CC) -x c - -o .test2 && echo 1 || echo 0 )" "1" + SHMAT_OK=1 +else + SHMAT_OK=0 + CFLAGS+=-DUSEMMAP=1 +endif + +ifeq "$(TEST_MMAP)" "1" + SHMAT_OK=0 + CFLAGS+=-DUSEMMAP=1 +endif + + # We were using llvm-config --bindir to get the location of clang, but # this seems to be busted on some distros, so using the one in $PATH is # probably better. @@ -66,7 +83,22 @@ else PROGS = ../afl-clang-fast ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so endif -all: test_deps $(PROGS) test_build all_done +all: test_deps test_shm $(PROGS) test_build all_done + + +ifeq "$(SHMAT_OK)" "1" + +test_shm: + @echo "[+] shmat seems to be working." + @rm -f .test2 + +else + +test_shm: + @echo "[-] shmat seems not to be working, switchig to mmap implementation" + +endif + test_deps: ifndef AFL_TRACE_PC @@ -119,9 +151,13 @@ test_build: $(PROGS) all_done: test_build @echo "[+] All done! You can now use '../afl-clang-fast' to compile programs." +ifeq "$(SHMAT_OK)" "0" + @echo "[!] shmat isn't working on your platform - compile every target with -lrt:" + @echo "[!] CFLAGS=-lrt LDFLAGS=-lrt CC=afl-clang-fast CXX=afl-clang-fast++ ./configure" +endif .NOTPARALLEL: clean clean: - rm -f *.o *.so *~ a.out core core.[1-9][0-9]* test-instr .test-instr0 .test-instr1 + rm -f *.o *.so *~ a.out core core.[1-9][0-9]* .test2 test-instr .test-instr0 .test-instr1 rm -f $(PROGS) ../afl-clang-fast++ -- cgit 1.4.1 From cc48f4499a6b4d646c19b99c0905324161eb03ed Mon Sep 17 00:00:00 2001 From: Heiko Eissfeldt Date: Tue, 2 Jul 2019 20:20:07 +0200 Subject: add librt under NetBSD --- Makefile | 4 ++++ llvm_mode/Makefile | 3 +++ 2 files changed, 7 insertions(+) (limited to 'llvm_mode') diff --git a/Makefile b/Makefile index 2075b90f..7acdd049 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,10 @@ ifneq "$(filter Linux GNU%,$(shell uname))" "" LDFLAGS += -ldl -lrt endif +ifeq "$(shell uname)" "NetBSD" + LDFLAGS += -lrt +endif + ifeq "$(findstring clang, $(shell $(CC) --version 2>/dev/null))" "" TEST_CC = afl-gcc else diff --git a/llvm_mode/Makefile b/llvm_mode/Makefile index 9f3cd010..e2ed07d6 100644 --- a/llvm_mode/Makefile +++ b/llvm_mode/Makefile @@ -54,6 +54,9 @@ ifeq "$(shell uname)" "Darwin" CLANG_LFL += -Wl,-flat_namespace -Wl,-undefined,suppress endif +ifeq "$(shell uname)" "NetBSD" + LDFLAGS += -lrt +endif ifeq "$(shell echo '\#include XXX\#include XXX\#include XXXvoid main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, NULL);}' | sed 's/XXX/\n/g' | $(CC) -x c - -o .test2 && echo 1 || echo 0 )" "1" SHMAT_OK=1 -- cgit 1.4.1 From b57b2073acf85e985f513a12d8aae725f8942689 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 3 Jul 2019 12:05:58 +0200 Subject: LAF_... -> AFL_LLVM_LAF_... --- docs/ChangeLog | 2 ++ docs/env_variables.txt | 6 +++--- llvm_mode/Makefile | 2 +- llvm_mode/README.laf-intel | 8 ++++---- llvm_mode/afl-clang-fast.c | 6 +++--- llvm_mode/split-compares-pass.so.cc | 2 ++ 6 files changed, 15 insertions(+), 11 deletions(-) (limited to 'llvm_mode') diff --git a/docs/ChangeLog b/docs/ChangeLog index 0d730118..b758b211 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -30,6 +30,8 @@ Version ++2.52d (tbd): - added a -s seed switch to allow afl run with a fixed initial seed that is not updated. this is good for performance and path discovery tests as the random numbers are deterministic then + - llvm_mode LAF_... env variables can now be specified as AFL_LLVM_LAF_... + that is longer but in line with other llvm specific env vars - ... your idea or patch? diff --git a/docs/env_variables.txt b/docs/env_variables.txt index f5db3b4f..2a824766 100644 --- a/docs/env_variables.txt +++ b/docs/env_variables.txt @@ -89,11 +89,11 @@ Then there are a few specific features that are only available in llvm_mode: This great feature will split compares to series of single byte comparisons to allow afl-fuzz to find otherwise rather impossible paths. - - Setting LAF_SPLIT_SWITCHES will split switch()es + - Setting AFL_LLVM_LAF_SPLIT_SWITCHES will split switch()es - - Setting LAF_TRANSFORM_COMPARES will split string compare functions + - Setting AFL_LLVM_LAF_TRANSFORM_COMPARES will split string compare functions - - Setting LAF_SPLIT_COMPARES will split > 8 bit CMP instructions + - Setting AFL_LLVM_LAF_SPLIT_COMPARES will split > 8 bit CMP instructions See llvm_mode/README.laf-intel for more information. diff --git a/llvm_mode/Makefile b/llvm_mode/Makefile index 6b277536..3304e62d 100644 --- a/llvm_mode/Makefile +++ b/llvm_mode/Makefile @@ -110,7 +110,7 @@ endif test_build: $(PROGS) @echo "[*] Testing the CC wrapper and instrumentation output..." - unset AFL_USE_ASAN AFL_USE_MSAN AFL_INST_RATIO; AFL_QUIET=1 AFL_PATH=. AFL_CC=$(CC) LAF_SPLIT_SWITCHES=1 LAF_TRANSFORM_COMPARES=1 LAF_SPLIT_COMPARES=1 ../afl-clang-fast $(CFLAGS) ../test-instr.c -o test-instr $(LDFLAGS) + unset AFL_USE_ASAN AFL_USE_MSAN AFL_INST_RATIO; AFL_QUIET=1 AFL_PATH=. AFL_CC=$(CC) AFL_LLVM_LAF_SPLIT_SWITCHES=1 AFL_LLVM_LAF_TRANSFORM_COMPARES=1 AFL_LLVM_LAF_SPLIT_COMPARES=1 ../afl-clang-fast $(CFLAGS) ../test-instr.c -o test-instr $(LDFLAGS) echo 0 | ../afl-showmap -m none -q -o .test-instr0 ./test-instr echo 1 | ../afl-showmap -m none -q -o .test-instr1 ./test-instr @rm -f test-instr diff --git a/llvm_mode/README.laf-intel b/llvm_mode/README.laf-intel index 891ab5fd..340216c3 100644 --- a/llvm_mode/README.laf-intel +++ b/llvm_mode/README.laf-intel @@ -8,13 +8,13 @@ compile the target project. The following options exist: -export LAF_SPLIT_SWITCHES=1 Enables the split-switches pass. +export AFL_LLVM_LAF_SPLIT_SWITCHES=1 Enables the split-switches pass. -export LAF_TRANSFORM_COMPARES=1 Enables the transform-compares pass +export AFL_LLVM_LAF_TRANSFORM_COMPARES=1 Enables the transform-compares pass (strcmp, memcmp, strncmp, strcasecmp, strncasecmp). -export LAF_SPLIT_COMPARES=1 Enables the split-compares pass. +export AFL_LLVM_LAF_SPLIT_COMPARES=1 Enables the split-compares pass. By default it will split all compares with a bit width <= 64 bits. You can change this behaviour by setting - export LAF_SPLIT_COMPARES_BITW=. + export AFL_LLVM_LAF_SPLIT_COMPARES_BITW=. diff --git a/llvm_mode/afl-clang-fast.c b/llvm_mode/afl-clang-fast.c index 1e2e04ea..5bc4ae8c 100644 --- a/llvm_mode/afl-clang-fast.c +++ b/llvm_mode/afl-clang-fast.c @@ -120,21 +120,21 @@ static void edit_params(u32 argc, char** argv) { http://clang.llvm.org/docs/SanitizerCoverage.html#tracing-pcs-with-guards */ // laf - if (getenv("LAF_SPLIT_SWITCHES")) { + if (getenv("LAF_SPLIT_SWITCHES")||getenv("AFL_LLVM_LAF_SPLIT_SWITCHES")) { cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = alloc_printf("%s/split-switches-pass.so", obj_path); } - if (getenv("LAF_TRANSFORM_COMPARES")) { + if (getenv("LAF_TRANSFORM_COMPARES")||getenv("AFL_LLVM_LAF_TRANSFORM_COMPARES")) { cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = alloc_printf("%s/compare-transform-pass.so", obj_path); } - if (getenv("LAF_SPLIT_COMPARES")) { + if (getenv("LAF_SPLIT_COMPARES")||getenv("AFL_LLVM_LAF_SPLIT_COMPARES")) { cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; diff --git a/llvm_mode/split-compares-pass.so.cc b/llvm_mode/split-compares-pass.so.cc index 25ccb3b4..2ea73aaa 100644 --- a/llvm_mode/split-compares-pass.so.cc +++ b/llvm_mode/split-compares-pass.so.cc @@ -477,6 +477,8 @@ bool SplitComparesTransform::runOnModule(Module &M) { int bitw = 64; char* bitw_env = getenv("LAF_SPLIT_COMPARES_BITW"); + if (!bitw_env) + bitw_env = getenv("AFL_LLVM_LAF_SPLIT_COMPARES_BITW"); if (bitw_env) { bitw = atoi(bitw_env); } -- cgit 1.4.1 From aaa810c64a7d54d21df6c74c3f7b77fb06d273fe Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 3 Jul 2019 12:11:02 +0200 Subject: add -lrt with afl-gcc/clang automatically in mmap mode --- Makefile | 8 ++++---- afl-gcc.c | 4 ++++ llvm_mode/Makefile | 8 ++++---- llvm_mode/afl-clang-fast.c | 4 ++++ 4 files changed, 16 insertions(+), 8 deletions(-) (limited to 'llvm_mode') diff --git a/Makefile b/Makefile index 7acdd049..14b5ce0e 100644 --- a/Makefile +++ b/Makefile @@ -176,10 +176,10 @@ endif all_done: test_build @if [ ! "`which clang 2>/dev/null`" = "" ]; then echo "[+] LLVM users: see llvm_mode/README.llvm for a faster alternative to afl-gcc."; fi @echo "[+] All done! Be sure to review the README - it's pretty short and useful." -ifeq "$(SHMAT_OK)" "0" - @echo "[!] shmat isn't working on your platform - compile every target with -lrt:" - @echo "[!] CFLAGS=-lrt LDFLAGS=-lrt CC=afl-gcc CXX=afl-g++ ./configure" -endif +#ifeq "$(SHMAT_OK)" "0" +# @echo "[!] shmat isn't working on your platform - compile every target with -lrt:" +# @echo "[!] CFLAGS=-lrt LDFLAGS=-lrt CC=afl-gcc CXX=afl-g++ ./configure" +#endif @if [ "`uname`" = "Darwin" ]; then printf "\nWARNING: Fuzzing on MacOS X is slow because of the unusually high overhead of\nfork() on this OS. Consider using Linux or *BSD. You can also use VirtualBox\n(virtualbox.org) to put AFL inside a Linux or *BSD VM.\n\n"; fi @! tty <&1 >/dev/null || printf "\033[0;30mNOTE: If you can read this, your terminal probably uses white background.\nThis will make the UI hard to read. See docs/status_screen.txt for advice.\033[0m\n" 2>/dev/null diff --git a/afl-gcc.c b/afl-gcc.c index 8d3988c7..467a9bc1 100644 --- a/afl-gcc.c +++ b/afl-gcc.c @@ -252,6 +252,10 @@ static void edit_params(u32 argc, char** argv) { } +#ifdef USEMMAP + cc_params[cc_par_cnt++] = "-lrt"; +#endif + if (!getenv("AFL_DONT_OPTIMIZE")) { #if defined(__FreeBSD__) && defined(__x86_64__) diff --git a/llvm_mode/Makefile b/llvm_mode/Makefile index e2ed07d6..d3e0c739 100644 --- a/llvm_mode/Makefile +++ b/llvm_mode/Makefile @@ -154,10 +154,10 @@ test_build: $(PROGS) all_done: test_build @echo "[+] All done! You can now use '../afl-clang-fast' to compile programs." -ifeq "$(SHMAT_OK)" "0" - @echo "[!] shmat isn't working on your platform - compile every target with -lrt:" - @echo "[!] CFLAGS=-lrt LDFLAGS=-lrt CC=afl-clang-fast CXX=afl-clang-fast++ ./configure" -endif +#ifeq "$(SHMAT_OK)" "0" +# @echo "[!] shmat isn't working on your platform - compile every target with -lrt:" +# @echo "[!] CFLAGS=-lrt LDFLAGS=-lrt CC=afl-clang-fast CXX=afl-clang-fast++ ./configure" +#endif .NOTPARALLEL: clean diff --git a/llvm_mode/afl-clang-fast.c b/llvm_mode/afl-clang-fast.c index 1e2e04ea..366389b4 100644 --- a/llvm_mode/afl-clang-fast.c +++ b/llvm_mode/afl-clang-fast.c @@ -246,6 +246,10 @@ static void edit_params(u32 argc, char** argv) { } +#ifdef USEMMAP + cc_params[cc_par_cnt++] = "-lrt"; +#endif + cc_params[cc_par_cnt++] = "-D__AFL_HAVE_MANUAL_CONTROL=1"; cc_params[cc_par_cnt++] = "-D__AFL_COMPILER=1"; cc_params[cc_par_cnt++] = "-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"; -- cgit 1.4.1 From 00b22e37df4a4dff32cfe0037de1550c1917387f Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 3 Jul 2019 16:36:31 +0200 Subject: select implementations --- llvm_mode/afl-llvm-pass.so.cc | 54 +++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 20 deletions(-) (limited to 'llvm_mode') diff --git a/llvm_mode/afl-llvm-pass.so.cc b/llvm_mode/afl-llvm-pass.so.cc index d20ca8dd..b77835c5 100644 --- a/llvm_mode/afl-llvm-pass.so.cc +++ b/llvm_mode/afl-llvm-pass.so.cc @@ -236,7 +236,9 @@ bool AFLCoverage::runOnModule(Module &M) { LoadInst *Counter = IRB.CreateLoad(MapPtrIdx); Counter->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - Value *Incr = IRB.CreateAdd(Counter, ConstantInt::get(Int8Ty, 1)); + Value *Incr; + if (neverZero_counters_str == NULL || neverZero_counters_str[0] != '4') + Incr = IRB.CreateAdd(Counter, ConstantInt::get(Int8Ty, 1)); if (neverZero_counters_str != NULL) { /* hexcoder: Realize a counter that skips zero during overflow. @@ -250,34 +252,46 @@ bool AFLCoverage::runOnModule(Module &M) { */ // Solution #1 - creates - //mov dl,BYTE PTR [rsi+rdi*1] //mov ecx,edx //add cl,0x1 //adc dl,0x1 - /* - CallInst *AddOv = IRB.CreateBinaryIntrinsic(Intrinsic::uadd_with_overflow, Counter, ConstantInt::get(Int8Ty, 1)); - AddOv->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - Value *SumWithOverflowBit = AddOv; - Incr = IRB.CreateAdd(IRB.CreateExtractValue(SumWithOverflowBit, 0), // sum - IRB.CreateZExt( // convert from one bit type to 8 bits type - IRB.CreateExtractValue(SumWithOverflowBit, 1), // overflow - Int8Ty)); - */ + if (neverZero_counters_str[0] == '1') { + CallInst *AddOv = IRB.CreateBinaryIntrinsic(Intrinsic::uadd_with_overflow, Counter, ConstantInt::get(Int8Ty, 1)); + AddOv->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + Value *SumWithOverflowBit = AddOv; + Incr = IRB.CreateAdd(IRB.CreateExtractValue(SumWithOverflowBit, 0), // sum + IRB.CreateZExt( // convert from one bit type to 8 bits type + IRB.CreateExtractValue(SumWithOverflowBit, 1), // overflow + Int8Ty)); + // Solution #2 - creates the same code as #1 - ///* - auto cf = IRB.CreateICmpULT(Incr, ConstantInt::get(Int8Ty, 1)); - Incr = IRB.CreateAdd(Incr, cf); - //*/ + } else if (neverZero_counters_str[0] == '2') { + auto cf = IRB.CreateICmpULT(Incr, ConstantInt::get(Int8Ty, 1)); + Incr = IRB.CreateAdd(Incr, cf); // Solution #3 - creates - //mov cl,BYTE PTR [rsi+rdx*1] //add cl,0x1 //cmp cl,0x1 //adc cl,0x0 - /* - auto cf = IRB.CreateICmpEQ(Incr, ConstantInt::get(Int8Ty, 0)); - Incr = IRB.CreateAdd(Incr, cf); - */ + } else if (neverZero_counters_str[0] == '3') { + auto cf = IRB.CreateICmpEQ(Incr, ConstantInt::get(Int8Ty, 0)); + Incr = IRB.CreateAdd(Incr, cf); + + // Solution #4 - creates + // cmp dl, $0xff + // sete cl + // add dl,cl + // add dl,0x01 + } else if (neverZero_counters_str[0] == '4') { + auto cf = IRB.CreateICmpEQ(Counter, ConstantInt::get(Int8Ty, 255)); + Value *HowMuch = IRB.CreateAdd(ConstantInt::get(Int8Ty, 1), cf); + Incr = IRB.CreateAdd(Counter, HowMuch); + + // no other implementations yet + } else { + fprintf(stderr, "Error: unknown value for AFL_NZERO_COUNTS: %s (valid is 1-4)\n", neverZero_counters_str); + exit(-1); + } } IRB.CreateStore(Incr, MapPtrIdx)->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); -- cgit 1.4.1 From 04c92c84705af4c602f134ed9a63b82be5ef75c9 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 3 Jul 2019 19:10:48 +0200 Subject: notzero for afl-gcc --- afl-as.h | 2 ++ llvm_mode/afl-llvm-pass.so.cc | 10 ++++++++++ 2 files changed, 12 insertions(+) (limited to 'llvm_mode') diff --git a/afl-as.h b/afl-as.h index ebd57109..2c84f9f3 100644 --- a/afl-as.h +++ b/afl-as.h @@ -189,6 +189,7 @@ static const u8* main_payload_32 = " orb $1, (%edx, %edi, 1)\n" #else " incb (%edx, %edi, 1)\n" + " adcb $0, (%edx, %edi, 1)\n" #endif /* ^SKIP_COUNTS */ "\n" "__afl_return:\n" @@ -417,6 +418,7 @@ static const u8* main_payload_64 = " orb $1, (%rdx, %rcx, 1)\n" #else " incb (%rdx, %rcx, 1)\n" + " adcb $0, (%rdx, %rcx, 1)\n" #endif /* ^SKIP_COUNTS */ "\n" "__afl_return:\n" diff --git a/llvm_mode/afl-llvm-pass.so.cc b/llvm_mode/afl-llvm-pass.so.cc index b77835c5..6b2232f2 100644 --- a/llvm_mode/afl-llvm-pass.so.cc +++ b/llvm_mode/afl-llvm-pass.so.cc @@ -287,6 +287,16 @@ bool AFLCoverage::runOnModule(Module &M) { Value *HowMuch = IRB.CreateAdd(ConstantInt::get(Int8Ty, 1), cf); Incr = IRB.CreateAdd(Counter, HowMuch); + } else if (neverZero_counters_str[0] == '5') { + auto cf = IRB.CreateICmpEQ(Incr, ConstantInt::get(Int8Ty, 0)); + auto carry = IRB.CreateZExt(cf, Int8Ty); + Incr = IRB.CreateAdd(Incr, carry); + + } else if (neverZero_counters_str[0] == '6') { + auto cf = IRB.CreateICmpULT(Incr, ConstantInt::get(Int8Ty, 1)); + auto carry = IRB.CreateZExt(cf, Int8Ty); + Incr = IRB.CreateAdd(Incr, carry); + // no other implementations yet } else { fprintf(stderr, "Error: unknown value for AFL_NZERO_COUNTS: %s (valid is 1-4)\n", neverZero_counters_str); -- cgit 1.4.1 From 9199967022284da0ee4d78459d8d34513540cf32 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 4 Jul 2019 11:19:18 +0200 Subject: this is the best solution IMHO --- llvm_mode/afl-llvm-pass.so.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'llvm_mode') diff --git a/llvm_mode/afl-llvm-pass.so.cc b/llvm_mode/afl-llvm-pass.so.cc index 6b2232f2..176692e3 100644 --- a/llvm_mode/afl-llvm-pass.so.cc +++ b/llvm_mode/afl-llvm-pass.so.cc @@ -237,7 +237,7 @@ bool AFLCoverage::runOnModule(Module &M) { Counter->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); Value *Incr; - if (neverZero_counters_str == NULL || neverZero_counters_str[0] != '4') + // if (neverZero_counters_str == NULL || neverZero_counters_str[0] != '4') Incr = IRB.CreateAdd(Counter, ConstantInt::get(Int8Ty, 1)); if (neverZero_counters_str != NULL) { @@ -255,6 +255,7 @@ bool AFLCoverage::runOnModule(Module &M) { //mov ecx,edx //add cl,0x1 //adc dl,0x1 +/* if (neverZero_counters_str[0] == '1') { CallInst *AddOv = IRB.CreateBinaryIntrinsic(Intrinsic::uadd_with_overflow, Counter, ConstantInt::get(Int8Ty, 1)); AddOv->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); @@ -276,7 +277,6 @@ bool AFLCoverage::runOnModule(Module &M) { } else if (neverZero_counters_str[0] == '3') { auto cf = IRB.CreateICmpEQ(Incr, ConstantInt::get(Int8Ty, 0)); Incr = IRB.CreateAdd(Incr, cf); - // Solution #4 - creates // cmp dl, $0xff // sete cl @@ -288,10 +288,11 @@ bool AFLCoverage::runOnModule(Module &M) { Incr = IRB.CreateAdd(Counter, HowMuch); } else if (neverZero_counters_str[0] == '5') { +*/ auto cf = IRB.CreateICmpEQ(Incr, ConstantInt::get(Int8Ty, 0)); auto carry = IRB.CreateZExt(cf, Int8Ty); Incr = IRB.CreateAdd(Incr, carry); - +/* } else if (neverZero_counters_str[0] == '6') { auto cf = IRB.CreateICmpULT(Incr, ConstantInt::get(Int8Ty, 1)); auto carry = IRB.CreateZExt(cf, Int8Ty); @@ -302,6 +303,7 @@ bool AFLCoverage::runOnModule(Module &M) { fprintf(stderr, "Error: unknown value for AFL_NZERO_COUNTS: %s (valid is 1-4)\n", neverZero_counters_str); exit(-1); } +*/ } IRB.CreateStore(Incr, MapPtrIdx)->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); -- cgit 1.4.1 From 7f6aaa53147afd4feb549214f49d0f5f69e4af6c Mon Sep 17 00:00:00 2001 From: van Hauser Date: Fri, 5 Jul 2019 11:28:08 +0200 Subject: final touches --- docs/ChangeLog | 2 + docs/env_variables.txt | 7 +++- llvm_mode/Makefile | 6 +++ llvm_mode/README.neverzero | 22 +++++++++++ llvm_mode/afl-llvm-pass.so.cc | 91 ++++++++++++++++++------------------------- 5 files changed, 72 insertions(+), 56 deletions(-) create mode 100644 llvm_mode/README.neverzero (limited to 'llvm_mode') diff --git a/docs/ChangeLog b/docs/ChangeLog index 73c69196..a533de05 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -17,6 +17,8 @@ sending a mail to . Version ++2.52d (tbd): ----------------------------- + - added never zero counters for afl-gcc and optional (because of an + optimization issue in llvm < 9) for llvm_mode (AFL_LLVM_NEVER_ZERO=1) - added whitelist support for llvm_mode via AFL_LLVM_WHITELIST to allow only to instrument what is actually interesting. Gives more speed and less map pollution (originally by choller@mozilla) diff --git a/docs/env_variables.txt b/docs/env_variables.txt index 725dc82e..f8c6c86a 100644 --- a/docs/env_variables.txt +++ b/docs/env_variables.txt @@ -108,8 +108,11 @@ Then there are a few specific features that are only available in llvm_mode: OTHER ===== - - Setting AFL_NZERO_COUNTS=1 during compilation will use counters - that skip zero on overflow. + - Setting export AFL_LLVM_NOT_ZERO=1 during compilation will use counters + that skip zero on overflow. This is the default for llvm >= 9, + however for llvm versions below that this will increase an unnecessary + slowdown due a performance issue that is only fixed in llvm 9+. + This feature increases path discovery by a little bit. Note that AFL_INST_RATIO will behave a bit differently than for afl-gcc, because functions are *not* instrumented unconditionally - so low values diff --git a/llvm_mode/Makefile b/llvm_mode/Makefile index 6b277536..76de10c0 100644 --- a/llvm_mode/Makefile +++ b/llvm_mode/Makefile @@ -25,11 +25,17 @@ VERSION = $(shell grep '^\#define VERSION ' ../config.h | cut -d '"' -f2) LLVM_CONFIG ?= llvm-config #LLVM_OK = $(shell $(LLVM_CONFIG) --version | egrep -q '^[5-6]' && echo 0 || echo 1 ) LLVM_UNSUPPORTED = $(shell $(LLVM_CONFIG) --version | egrep -q '^9|3.0' && echo 1 || echo 0 ) +LLVM_MAJOR = ($shell $(LLVM_CONFIG) --version | sed 's/\..*//') ifeq "$(LLVM_UNSUPPORTED)" "1" $(warn llvm_mode only supports versions 3.8.0 up to 8.x ) endif +# this is not visible yet: +ifeq "$(LLVM_MAJOR)" "9" + $(info llvm_mode deteted llvm 9, enabling neverZero implementation) +endif + CFLAGS ?= -O3 -funroll-loops CFLAGS += -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ diff --git a/llvm_mode/README.neverzero b/llvm_mode/README.neverzero new file mode 100644 index 00000000..ef873acb --- /dev/null +++ b/llvm_mode/README.neverzero @@ -0,0 +1,22 @@ +Usage +===== + +In larger, complex or reiterative programs the map that collects the edge pairs +can easily fill up and wrap. +This is not that much of an issue - unless by chance it wraps just to a 0 +when the program execution ends. +In this case afl-fuzz is not able to see that the pair has been accessed and +will ignore it. + +NeverZero prevents this behaviour. If a counter wraps, it jumps over the 0 +directly to a 1. This improves path discovery (by a very little amount) +at a very little cost (one instruction per edge). + +This is implemented in afl-gcc, however for llvm_mode this is optional if +the llvm version is below 9 - as there is a perfomance bug that is only fixed +in version 9 and onwards. + +If you want to enable this for llvm < 9 then set + +export AFL_LLVM_NOT_ZERO=1 + diff --git a/llvm_mode/afl-llvm-pass.so.cc b/llvm_mode/afl-llvm-pass.so.cc index 176692e3..cfeff968 100644 --- a/llvm_mode/afl-llvm-pass.so.cc +++ b/llvm_mode/afl-llvm-pass.so.cc @@ -118,7 +118,9 @@ bool AFLCoverage::runOnModule(Module &M) { } - char* neverZero_counters_str = getenv("AFL_NZERO_COUNTS"); +#if LLVM_VERSION_MAJOR < 9 + char* neverZero_counters_str = getenv("AFL_LLVM_NOT_ZERO"); +#endif /* Get globals for the SHM region and the previous location. Note that __afl_prev_loc is thread-local. */ @@ -236,75 +238,56 @@ bool AFLCoverage::runOnModule(Module &M) { LoadInst *Counter = IRB.CreateLoad(MapPtrIdx); Counter->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - Value *Incr; - // if (neverZero_counters_str == NULL || neverZero_counters_str[0] != '4') - Incr = IRB.CreateAdd(Counter, ConstantInt::get(Int8Ty, 1)); - - if (neverZero_counters_str != NULL) { - /* hexcoder: Realize a counter that skips zero during overflow. - * Once this counter reaches its maximum value, it next increments to 1 - * - * Instead of - * Counter + 1 -> Counter - * we inject now this - * Counter + 1 -> {Counter, OverflowFlag} - * Counter + OverflowFlag -> Counter - */ - - // Solution #1 - creates - //mov ecx,edx - //add cl,0x1 - //adc dl,0x1 -/* - if (neverZero_counters_str[0] == '1') { - CallInst *AddOv = IRB.CreateBinaryIntrinsic(Intrinsic::uadd_with_overflow, Counter, ConstantInt::get(Int8Ty, 1)); - AddOv->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - Value *SumWithOverflowBit = AddOv; - Incr = IRB.CreateAdd(IRB.CreateExtractValue(SumWithOverflowBit, 0), // sum - IRB.CreateZExt( // convert from one bit type to 8 bits type - IRB.CreateExtractValue(SumWithOverflowBit, 1), // overflow - Int8Ty)); - - // Solution #2 - creates the same code as #1 + Value *Incr = IRB.CreateAdd(Counter, ConstantInt::get(Int8Ty, 1)); + +#if LLVM_VERSION_MAJOR < 9 + if (neverZero_counters_str != NULL) { // with llvm 9 we make this the default as the bug in llvm is then fixed +#endif + /* hexcoder: Realize a counter that skips zero during overflow. + * Once this counter reaches its maximum value, it next increments to 1 + * + * Instead of + * Counter + 1 -> Counter + * we inject now this + * Counter + 1 -> {Counter, OverflowFlag} + * Counter + OverflowFlag -> Counter + */ +/* // we keep the old solutions just in case + // Solution #1 + if (neverZero_counters_str[0] == '1') { + CallInst *AddOv = IRB.CreateBinaryIntrinsic(Intrinsic::uadd_with_overflow, Counter, ConstantInt::get(Int8Ty, 1)); + AddOv->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + Value *SumWithOverflowBit = AddOv; + Incr = IRB.CreateAdd(IRB.CreateExtractValue(SumWithOverflowBit, 0), // sum + IRB.CreateZExt( // convert from one bit type to 8 bits type + IRB.CreateExtractValue(SumWithOverflowBit, 1), // overflow + Int8Ty)); + // Solution #2 } else if (neverZero_counters_str[0] == '2') { - auto cf = IRB.CreateICmpULT(Incr, ConstantInt::get(Int8Ty, 1)); - Incr = IRB.CreateAdd(Incr, cf); - - // Solution #3 - creates - //add cl,0x1 - //cmp cl,0x1 - //adc cl,0x0 - } else if (neverZero_counters_str[0] == '3') { - auto cf = IRB.CreateICmpEQ(Incr, ConstantInt::get(Int8Ty, 0)); - Incr = IRB.CreateAdd(Incr, cf); - // Solution #4 - creates - // cmp dl, $0xff - // sete cl - // add dl,cl - // add dl,0x01 - } else if (neverZero_counters_str[0] == '4') { auto cf = IRB.CreateICmpEQ(Counter, ConstantInt::get(Int8Ty, 255)); Value *HowMuch = IRB.CreateAdd(ConstantInt::get(Int8Ty, 1), cf); Incr = IRB.CreateAdd(Counter, HowMuch); - - } else if (neverZero_counters_str[0] == '5') { + // Solution #3 + } else if (neverZero_counters_str[0] == '3') { */ + // this is the solution we choose because llvm9 should do the right thing here auto cf = IRB.CreateICmpEQ(Incr, ConstantInt::get(Int8Ty, 0)); auto carry = IRB.CreateZExt(cf, Int8Ty); Incr = IRB.CreateAdd(Incr, carry); /* - } else if (neverZero_counters_str[0] == '6') { + // Solution #4 + } else if (neverZero_counters_str[0] == '4') { auto cf = IRB.CreateICmpULT(Incr, ConstantInt::get(Int8Ty, 1)); auto carry = IRB.CreateZExt(cf, Int8Ty); Incr = IRB.CreateAdd(Incr, carry); - - // no other implementations yet - } else { + } else { fprintf(stderr, "Error: unknown value for AFL_NZERO_COUNTS: %s (valid is 1-4)\n", neverZero_counters_str); exit(-1); - } + } */ +#if LLVM_VERSION_MAJOR < 9 } +#endif IRB.CreateStore(Incr, MapPtrIdx)->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); -- cgit 1.4.1 From 984ae35948688cac13ce6908aa02fb96d9daf45a Mon Sep 17 00:00:00 2001 From: heiko Date: Fri, 5 Jul 2019 20:02:40 +0200 Subject: increased portability, replace sed with tr (*BSD) sanity check versions from clang and llvm, adjust clang path if needed. --- Makefile | 24 ++++++++++-------------- llvm_mode/Makefile | 48 ++++++++++++++++++++++++++++++------------------ 2 files changed, 40 insertions(+), 32 deletions(-) (limited to 'llvm_mode') diff --git a/Makefile b/Makefile index 14b5ce0e..a9ae5518 100644 --- a/Makefile +++ b/Makefile @@ -38,11 +38,7 @@ CFLAGS += -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign \ PYTHON_INCLUDE ?= /usr/include/python2.7 ifneq "$(filter Linux GNU%,$(shell uname))" "" - LDFLAGS += -ldl -lrt -endif - -ifeq "$(shell uname)" "NetBSD" - LDFLAGS += -lrt + LDFLAGS += -ldl endif ifeq "$(findstring clang, $(shell $(CC) --version 2>/dev/null))" "" @@ -54,7 +50,7 @@ endif COMM_HDR = alloc-inl.h config.h debug.h types.h -ifeq "$(shell echo '\#include XXXvoid main() {}' | sed 's/XXX/\n/g' | $(CC) -x c - -o .test -I$(PYTHON_INCLUDE) -lpython2.7 && echo 1 || echo 0 )" "1" +ifeq "$(shell echo '\#include @int main() {return 0; }' | tr @ '\n' | $(CC) -x c - -o .test -I$(PYTHON_INCLUDE) -lpython2.7 2>/dev/null && echo 1 || echo 0 )" "1" PYTHON_OK=1 PYFLAGS=-DUSE_PYTHON -I$(PYTHON_INCLUDE) -lpython2.7 else @@ -63,18 +59,18 @@ else endif -ifeq "$(shell echo '\#include XXX\#include XXX\#include XXXvoid main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, NULL);}' | sed 's/XXX/\n/g' | $(CC) -x c - -o .test2 && echo 1 || echo 0 )" "1" +ifeq "$(shell echo '\#include @\#include @int main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, 0); return 0;}' | tr @ '\n' | $(CC) -x c - -o .test2 2>/dev/null && echo 1 || echo 0 )" "1" SHMAT_OK=1 else SHMAT_OK=0 CFLAGS+=-DUSEMMAP=1 - LDFLAGS+=-Wno-deprecated-declarations + LDFLAGS+=-Wno-deprecated-declarations -lrt endif ifeq "$(TEST_MMAP)" "1" SHMAT_OK=0 CFLAGS+=-DUSEMMAP=1 - LDFLAGS+=-Wno-deprecated-declarations + LDFLAGS+=-Wno-deprecated-declarations -lrt endif @@ -105,7 +101,7 @@ test_shm: else test_shm: - @echo "[-] shmat seems not to be working, switchig to mmap implementation" + @echo "[-] shmat seems not to be working, switching to mmap implementation" endif @@ -176,10 +172,10 @@ endif all_done: test_build @if [ ! "`which clang 2>/dev/null`" = "" ]; then echo "[+] LLVM users: see llvm_mode/README.llvm for a faster alternative to afl-gcc."; fi @echo "[+] All done! Be sure to review the README - it's pretty short and useful." -#ifeq "$(SHMAT_OK)" "0" -# @echo "[!] shmat isn't working on your platform - compile every target with -lrt:" -# @echo "[!] CFLAGS=-lrt LDFLAGS=-lrt CC=afl-gcc CXX=afl-g++ ./configure" -#endif +ifeq "$(SHMAT_OK)" "0" + @echo "[!] shmat isn't working on your platform - compile every target with -lrt:" + @echo "[!] CFLAGS=-lrt LDFLAGS=-lrt CC=afl-gcc CXX=afl-g++ ./configure" +endif @if [ "`uname`" = "Darwin" ]; then printf "\nWARNING: Fuzzing on MacOS X is slow because of the unusually high overhead of\nfork() on this OS. Consider using Linux or *BSD. You can also use VirtualBox\n(virtualbox.org) to put AFL inside a Linux or *BSD VM.\n\n"; fi @! tty <&1 >/dev/null || printf "\033[0;30mNOTE: If you can read this, your terminal probably uses white background.\nThis will make the UI hard to read. See docs/status_screen.txt for advice.\033[0m\n" 2>/dev/null diff --git a/llvm_mode/Makefile b/llvm_mode/Makefile index d3e0c739..efabd6b4 100644 --- a/llvm_mode/Makefile +++ b/llvm_mode/Makefile @@ -26,8 +26,9 @@ BIN_PATH = $(PREFIX)/bin VERSION = $(shell grep '^\#define VERSION ' ../config.h | cut -d '"' -f2) LLVM_CONFIG ?= llvm-config +LLVMVER = $(shell $(LLVM_CONFIG) --version) #LLVM_OK = $(shell $(LLVM_CONFIG) --version | egrep -q '^[5-6]' && echo 0 || echo 1 ) -LLVM_UNSUPPORTED = $(shell $(LLVM_CONFIG) --version | egrep -q '^9|3.0' && echo 1 || echo 0 ) +LLVM_UNSUPPORTED = $(shell echo $(LLVMVER) | egrep -q '^9|3.0' && echo 1 || echo 1 ) ifeq "$(LLVM_UNSUPPORTED)" "1" $(warn llvm_mode only supports versions 3.8.0 up to 8.x ) @@ -54,38 +55,46 @@ ifeq "$(shell uname)" "Darwin" CLANG_LFL += -Wl,-flat_namespace -Wl,-undefined,suppress endif -ifeq "$(shell uname)" "NetBSD" - LDFLAGS += -lrt +# We were using llvm-config --bindir to get the location of clang, but +# this seems to be busted on some distros, so using the one in $PATH is +# probably better. + +ifeq "$(origin CC)" "default" + CC = clang + CXX = clang++ endif -ifeq "$(shell echo '\#include XXX\#include XXX\#include XXXvoid main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, NULL);}' | sed 's/XXX/\n/g' | $(CC) -x c - -o .test2 && echo 1 || echo 0 )" "1" +# sanity check. +# Are versions of clang --version and llvm-config --version equal? +CLANGVER = $(shell $(CC) --version | sed -E -ne '/^.*([0-9]\.[0-9]\.[0-9]).*/s//\1/p') + + +ifeq "$(shell echo '\#include @\#include @int main() { int _id = shmget(IPC_PRIVATE, 65536, IPC_CREAT | IPC_EXCL | 0600); shmctl(_id, IPC_RMID, 0); return 0;}' | tr @ '\n' | $(CC) -x c - -o .test2 2>/dev/null && echo 1 || echo 0 )" "1" SHMAT_OK=1 else SHMAT_OK=0 CFLAGS+=-DUSEMMAP=1 + LDFLAGS += -lrt endif ifeq "$(TEST_MMAP)" "1" SHMAT_OK=0 CFLAGS+=-DUSEMMAP=1 + LDFLAGS += -lrt endif -# We were using llvm-config --bindir to get the location of clang, but -# this seems to be busted on some distros, so using the one in $PATH is -# probably better. - -ifeq "$(origin CC)" "default" - CC = clang - CXX = clang++ -endif - ifndef AFL_TRACE_PC PROGS = ../afl-clang-fast ../afl-llvm-pass.so ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so else PROGS = ../afl-clang-fast ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so endif +ifneq "$(CLANGVER)" "$(LLVMVER)" + CC = $(shell llvm-config --bindir)/clang + CXX = $(shell llvm-config --bindir)/clang++ +endif + all: test_deps test_shm $(PROGS) test_build all_done @@ -98,7 +107,7 @@ test_shm: else test_shm: - @echo "[-] shmat seems not to be working, switchig to mmap implementation" + @echo "[-] shmat seems not to be working, switching to mmap implementation" endif @@ -112,6 +121,13 @@ else endif @echo "[*] Checking for working '$(CC)'..." @which $(CC) >/dev/null 2>&1 || ( echo "[-] Oops, can't find '$(CC)'. Make sure that it's in your \$$PATH (or set \$$CC and \$$CXX)."; exit 1 ) + @echo "[*] Checking for matching versions of '$(CC)' and '$(LLVM_CONFIG)'" +ifneq "$(CLANGVER)" "$(LLVMVER)" + @echo "WARNING: we have llvm-config version $(LLVMVER) and a clang version $(CLANGVER)" + @echo "Retrying with the clang compiler from llvm: CC=`llvm-config --bindir`/clang" +else + @echo "we have llvm-config version $(LLVMVER) with a clang version $(CLANGVER), good." +endif @echo "[*] Checking for '../afl-showmap'..." @test -f ../afl-showmap || ( echo "[-] Oops, can't find '../afl-showmap'. Be sure to compile AFL first."; exit 1 ) @echo "[+] All set and ready to build." @@ -154,10 +170,6 @@ test_build: $(PROGS) all_done: test_build @echo "[+] All done! You can now use '../afl-clang-fast' to compile programs." -#ifeq "$(SHMAT_OK)" "0" -# @echo "[!] shmat isn't working on your platform - compile every target with -lrt:" -# @echo "[!] CFLAGS=-lrt LDFLAGS=-lrt CC=afl-clang-fast CXX=afl-clang-fast++ ./configure" -#endif .NOTPARALLEL: clean -- cgit 1.4.1 From d9c70c7b8cdcde0827e167c61f60f862bc9a3ba8 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Fri, 5 Jul 2019 20:33:36 +0200 Subject: add explicit llvm library for OpenBSD --- llvm_mode/Makefile | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'llvm_mode') diff --git a/llvm_mode/Makefile b/llvm_mode/Makefile index efabd6b4..815ac59d 100644 --- a/llvm_mode/Makefile +++ b/llvm_mode/Makefile @@ -55,6 +55,10 @@ ifeq "$(shell uname)" "Darwin" CLANG_LFL += -Wl,-flat_namespace -Wl,-undefined,suppress endif +ifeq "$(shell uname)" "OpenBSD" + CLANG_LFL += `$(LLVM_CONFIG) --libdir`/libLLVM.so.0.0 +endif + # We were using llvm-config --bindir to get the location of clang, but # this seems to be busted on some distros, so using the one in $PATH is # probably better. -- cgit 1.4.1 From 864056fcaaeea0e156e650b7a0f6182e81db566a Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sat, 13 Jul 2019 11:08:13 +0200 Subject: initial commit --- Makefile | 2 +- llvm_mode/LLVMInsTrim.so.cc | 197 ++++++++++++++++++++++++ llvm_mode/Makefile | 6 +- llvm_mode/MarkNodes.cc | 355 ++++++++++++++++++++++++++++++++++++++++++++ llvm_mode/MarkNodes.h | 11 ++ llvm_mode/afl-clang-fast.c | 8 +- 6 files changed, 572 insertions(+), 7 deletions(-) create mode 100644 llvm_mode/LLVMInsTrim.so.cc create mode 100644 llvm_mode/MarkNodes.cc create mode 100644 llvm_mode/MarkNodes.h (limited to 'llvm_mode') diff --git a/Makefile b/Makefile index 601f29a7..0d0d6b79 100644 --- a/Makefile +++ b/Makefile @@ -147,7 +147,7 @@ install: all rm -f $${DESTDIR}$(BIN_PATH)/afl-as if [ -f afl-qemu-trace ]; then install -m 755 afl-qemu-trace $${DESTDIR}$(BIN_PATH); fi ifndef AFL_TRACE_PC - if [ -f afl-clang-fast -a -f afl-llvm-pass.so -a -f afl-llvm-rt.o ]; then set -e; install -m 755 afl-clang-fast $${DESTDIR}$(BIN_PATH); ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-fast++; install -m 755 afl-llvm-pass.so afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH); fi + if [ -f afl-clang-fast -a -f libLLVMInsTrim.so -a -f afl-llvm-rt.o ]; then set -e; install -m 755 afl-clang-fast $${DESTDIR}$(BIN_PATH); ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-fast++; install -m 755 libLLVMInsTrim.so afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH); fi else if [ -f afl-clang-fast -a -f afl-llvm-rt.o ]; then set -e; install -m 755 afl-clang-fast $${DESTDIR}$(BIN_PATH); ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-fast++; install -m 755 afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH); fi endif diff --git a/llvm_mode/LLVMInsTrim.so.cc b/llvm_mode/LLVMInsTrim.so.cc new file mode 100644 index 00000000..2a13981d --- /dev/null +++ b/llvm_mode/LLVMInsTrim.so.cc @@ -0,0 +1,197 @@ +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/IR/CFG.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include +#include + +#include "MarkNodes.h" + +using namespace llvm; + +static cl::opt MarkSetOpt("markset", cl::desc("MarkSet"), + cl::init(false)); +static cl::opt LoopHeadOpt("loophead", cl::desc("LoopHead"), + cl::init(false)); + +namespace { + struct InsTrim : public ModulePass { + private: + std::mt19937 generator; + int total_instr = 0; + + unsigned genLabel() { + return generator() % 65536; + } + + public: + static char ID; + InsTrim() : ModulePass(ID), generator(0) {} + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + } + + StringRef getPassName() const override { + return "InstTrim Instrumentation"; + } + + bool runOnModule(Module &M) override { + if (getenv("LOOPHEAD")) { + LoopHeadOpt = true; + MarkSetOpt = true; + } else if (getenv("MARKSET")) { + MarkSetOpt = true; + } + + LLVMContext &C = M.getContext(); + IntegerType *Int8Ty = IntegerType::getInt8Ty(C); + IntegerType *Int32Ty = IntegerType::getInt32Ty(C); + + GlobalVariable *CovMapPtr = new GlobalVariable( + M, PointerType::getUnqual(Int8Ty), false, GlobalValue::ExternalLinkage, + nullptr, "__afl_area_ptr"); + + GlobalVariable *OldPrev = new GlobalVariable( + M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_loc", + 0, GlobalVariable::GeneralDynamicTLSModel, 0, false); + + unsigned total_rs = 0; + unsigned total_hs = 0; + + for (Function &F : M) { + if (!F.size()) { + continue; + } + + std::unordered_set MS; + if (!MarkSetOpt) { + for (auto &BB : F) { + MS.insert(&BB); + } + total_rs += F.size(); + } else { + auto Result = markNodes(&F); + auto RS = Result.first; + auto HS = Result.second; + + MS.insert(RS.begin(), RS.end()); + if (!LoopHeadOpt) { + MS.insert(HS.begin(), HS.end()); + total_rs += MS.size(); + } else { + DenseSet> EdgeSet; + DominatorTreeWrapperPass *DTWP = + &getAnalysis(F); + auto DT = &DTWP->getDomTree(); + + total_rs += RS.size(); + total_hs += HS.size(); + + for (BasicBlock *BB : HS) { + bool Inserted = false; + for (auto BI = pred_begin(BB), BE = pred_end(BB); + BI != BE; ++BI + ) { + auto Edge = BasicBlockEdge(*BI, BB); + if (Edge.isSingleEdge() && DT->dominates(Edge, BB)) { + EdgeSet.insert({*BI, BB}); + Inserted = true; + break; + } + } + if (!Inserted) { + MS.insert(BB); + total_rs += 1; + total_hs -= 1; + } + } + for (auto I = EdgeSet.begin(), E = EdgeSet.end(); I != E; ++I) { + auto PredBB = I->first; + auto SuccBB = I->second; + auto NewBB = SplitBlockPredecessors(SuccBB, {PredBB}, ".split", + DT, nullptr, false); + MS.insert(NewBB); + } + } + + auto *EBB = &F.getEntryBlock(); + if (succ_begin(EBB) == succ_end(EBB)) { + MS.insert(EBB); + total_rs += 1; + } + + for (BasicBlock &BB : F) { + if (MS.find(&BB) == MS.end()) { + continue; + } + IRBuilder<> IRB(&*BB.getFirstInsertionPt()); + IRB.CreateStore(ConstantInt::get(Int32Ty, genLabel()), OldPrev); + } + } + + for (BasicBlock &BB : F) { + auto PI = pred_begin(&BB); + auto PE = pred_end(&BB); + if (MarkSetOpt && MS.find(&BB) == MS.end()) { + continue; + } + + IRBuilder<> IRB(&*BB.getFirstInsertionPt()); + Value *L = NULL; + if (PI == PE) { + L = ConstantInt::get(Int32Ty, genLabel()); + } else { + auto *PN = PHINode::Create(Int32Ty, 0, "", &*BB.begin()); + DenseMap PredMap; + for (auto PI = pred_begin(&BB), PE = pred_end(&BB); + PI != PE; ++PI + ) { + BasicBlock *PBB = *PI; + auto It = PredMap.insert({PBB, genLabel()}); + unsigned Label = It.first->second; + PN->addIncoming(ConstantInt::get(Int32Ty, Label), PBB); + } + L = PN; + } + + LoadInst *PrevLoc = IRB.CreateLoad(OldPrev); + Value *PrevLocCasted = IRB.CreateZExt(PrevLoc, IRB.getInt32Ty()); + + LoadInst *MapPtr = IRB.CreateLoad(CovMapPtr); + Value *MapPtrIdx = IRB.CreateGEP(MapPtr, + IRB.CreateXor(PrevLocCasted, L)); + + LoadInst *Counter = IRB.CreateLoad(MapPtrIdx); + Value *Incr = IRB.CreateAdd(Counter, ConstantInt::get(Int8Ty, 1)); + IRB.CreateStore(Incr, MapPtrIdx); + total_instr++; + } + } + + errs() << total_instr << " locations instrumented ("<< total_rs << "," << total_hs << ")\n"; + return false; + } + }; // end of struct InsTrim +} // end of anonymous namespace + +char InsTrim::ID = 0; + +static void registerAFLPass(const PassManagerBuilder &, + legacy::PassManagerBase &PM) { + PM.add(new InsTrim()); +} + +static RegisterStandardPasses RegisterAFLPass( + PassManagerBuilder::EP_OptimizerLast, registerAFLPass); + +static RegisterStandardPasses RegisterAFLPass0( + PassManagerBuilder::EP_EnabledOnOptLevel0, registerAFLPass); diff --git a/llvm_mode/Makefile b/llvm_mode/Makefile index b6ab0c61..a66f18ab 100644 --- a/llvm_mode/Makefile +++ b/llvm_mode/Makefile @@ -67,7 +67,7 @@ ifeq "$(origin CC)" "default" endif ifndef AFL_TRACE_PC - PROGS = ../afl-clang-fast ../afl-llvm-pass.so ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so + PROGS = ../afl-clang-fast ../libLLVMInsTrim.so ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so else PROGS = ../afl-clang-fast ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so endif @@ -91,8 +91,8 @@ endif $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) ln -sf afl-clang-fast ../afl-clang-fast++ -../afl-llvm-pass.so: afl-llvm-pass.so.cc | test_deps - $(CXX) $(CLANG_CFL) -shared $< -o $@ $(CLANG_LFL) +../libLLVMInsTrim.so: LLVMInsTrim.so.cc MarkNodes.cc | test_deps + $(CXX) $(CLANG_CFL) -DLLVMInsTrim_EXPORTS -fno-rtti -fPIC -std=gnu++11 -shared $< -o $@ $(CLANG_LFL) # laf ../split-switches-pass.so: split-switches-pass.so.cc | test_deps diff --git a/llvm_mode/MarkNodes.cc b/llvm_mode/MarkNodes.cc new file mode 100644 index 00000000..3c2129ef --- /dev/null +++ b/llvm_mode/MarkNodes.cc @@ -0,0 +1,355 @@ +#include +#include +#include +#include +#include +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CFG.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +DenseMap LMap; +std::vector Blocks; +std::set Marked , Markabove; +std::vector< std::vector > Succs , Preds; + +void reset(){ + LMap.clear(); + Blocks.clear(); + Marked.clear(); + Markabove.clear(); +} + +uint32_t start_point; + +void labelEachBlock(Function *F) { + // Fake single endpoint; + LMap[NULL] = Blocks.size(); + Blocks.push_back(NULL); + + // Assign the unique LabelID to each block; + for (auto I = F->begin(), E = F->end(); I != E; ++I) { + BasicBlock *BB = &*I; + LMap[BB] = Blocks.size(); + Blocks.push_back(BB); + } + + start_point = LMap[&F->getEntryBlock()]; +} + +void buildCFG(Function *F) { + Succs.resize( Blocks.size() ); + Preds.resize( Blocks.size() ); + for( size_t i = 0 ; i < Succs.size() ; i ++ ){ + Succs[ i ].clear(); + Preds[ i ].clear(); + } + + uint32_t FakeID = 0; + for (auto S = F->begin(), E = F->end(); S != E; ++S) { + BasicBlock *BB = &*S; + uint32_t MyID = LMap[BB]; + //if (succ_begin(BB) == succ_end(BB)) { + //Succs[MyID].push_back(FakeID); + //Marked.insert(MyID); + //} + for (auto I = succ_begin(BB), E = succ_end(BB); I != E; ++I) { + Succs[MyID].push_back(LMap[*I]); + } + } +} + +std::vector< std::vector > tSuccs; +std::vector tag , indfs; + +void DFStree(size_t now_id) { + if(tag[now_id]) return; + tag[now_id]=true; + indfs[now_id]=true; + for (auto succ: tSuccs[now_id]) { + if(tag[succ] and indfs[succ]) { + Marked.insert(succ); + Markabove.insert(succ); + continue; + } + Succs[now_id].push_back(succ); + Preds[succ].push_back(now_id); + DFStree(succ); + } + indfs[now_id]=false; +} +void turnCFGintoDAG(Function *F) { + tSuccs = Succs; + tag.resize(Blocks.size()); + indfs.resize(Blocks.size()); + for (size_t i = 0; i < Blocks.size(); ++ i) { + Succs[i].clear(); + tag[i]=false; + indfs[i]=false; + } + DFStree(start_point); + for (size_t i = 0; i < Blocks.size(); ++ i) + if( Succs[i].empty() ){ + Succs[i].push_back(0); + Preds[0].push_back(i); + } +} + +uint32_t timeStamp; +namespace DominatorTree{ + std::vector< std::vector > cov; + std::vector dfn, nfd, par, sdom, idom, mom, mn; + + bool Compare(uint32_t u, uint32_t v) { + return dfn[u] < dfn[v]; + } + uint32_t eval(uint32_t u) { + if( mom[u] == u ) return u; + uint32_t res = eval( mom[u] ); + if(Compare(sdom[mn[mom[u]]] , sdom[mn[u]])) { + mn[u] = mn[mom[u]]; + } + return mom[u] = res; + } + + void DFS(uint32_t now) { + timeStamp += 1; + dfn[now] = timeStamp; + nfd[timeStamp - 1] = now; + for( auto succ : Succs[now] ) { + if( dfn[succ] == 0 ) { + par[succ] = now; + DFS(succ); + } + } + } + + void DominatorTree(Function *F) { + if( Blocks.empty() ) return; + uint32_t s = start_point; + + // Initialization + mn.resize(Blocks.size()); + cov.resize(Blocks.size()); + dfn.resize(Blocks.size()); + nfd.resize(Blocks.size()); + par.resize(Blocks.size()); + mom.resize(Blocks.size()); + sdom.resize(Blocks.size()); + idom.resize(Blocks.size()); + + for( uint32_t i = 0 ; i < Blocks.size() ; i ++ ) { + dfn[i] = 0; + nfd[i] = Blocks.size(); + cov[i].clear(); + idom[i] = mom[i] = mn[i] = sdom[i] = i; + } + + timeStamp = 0; + DFS(s); + + for( uint32_t i = Blocks.size() - 1 ; i >= 1u ; i -- ) { + uint32_t now = nfd[i]; + if( now == Blocks.size() ) { + continue; + } + for( uint32_t pre : Preds[ now ] ) { + if( dfn[ pre ] ) { + eval(pre); + if( Compare(sdom[mn[pre]], sdom[now]) ) { + sdom[now] = sdom[mn[pre]]; + } + } + } + cov[sdom[now]].push_back(now); + mom[now] = par[now]; + for( uint32_t x : cov[par[now]] ) { + eval(x); + if( Compare(sdom[mn[x]], par[now]) ) { + idom[x] = mn[x]; + } else { + idom[x] = par[now]; + } + } + } + + for( uint32_t i = 1 ; i < Blocks.size() ; i += 1 ) { + uint32_t now = nfd[i]; + if( now == Blocks.size() ) { + continue; + } + if(idom[now] != sdom[now]) + idom[now] = idom[idom[now]]; + } + } +}; // End of DominatorTree + +std::vector Visited, InStack; +std::vector TopoOrder, InDeg; +std::vector< std::vector > t_Succ , t_Pred; + +void Go(uint32_t now, uint32_t tt) { + if( now == tt ) return; + Visited[now] = InStack[now] = timeStamp; + + for(uint32_t nxt : Succs[now]) { + if(Visited[nxt] == timeStamp and InStack[nxt] == timeStamp) { + Marked.insert(nxt); + } + t_Succ[now].push_back(nxt); + t_Pred[nxt].push_back(now); + InDeg[nxt] += 1; + if(Visited[nxt] == timeStamp) { + continue; + } + Go(nxt, tt); + } + + InStack[now] = 0; +} + +void TopologicalSort(uint32_t ss, uint32_t tt) { + timeStamp += 1; + + Go(ss, tt); + + TopoOrder.clear(); + std::queue wait; + wait.push(ss); + while( not wait.empty() ) { + uint32_t now = wait.front(); wait.pop(); + TopoOrder.push_back(now); + for(uint32_t nxt : t_Succ[now]) { + InDeg[nxt] -= 1; + if(InDeg[nxt] == 0u) { + wait.push(nxt); + } + } + } +} + +std::vector< std::set > NextMarked; +bool Indistinguish(uint32_t node1, uint32_t node2) { + if(NextMarked[node1].size() > NextMarked[node2].size()){ + uint32_t _swap = node1; + node1 = node2; + node2 = _swap; + } + for(uint32_t x : NextMarked[node1]) { + if( NextMarked[node2].find(x) != NextMarked[node2].end() ) { + return true; + } + } + return false; +} + +void MakeUniq(uint32_t now) { + bool StopFlag = false; + if (Marked.find(now) == Marked.end()) { + for(uint32_t pred1 : t_Pred[now]) { + for(uint32_t pred2 : t_Pred[now]) { + if(pred1 == pred2) continue; + if(Indistinguish(pred1, pred2)) { + Marked.insert(now); + StopFlag = true; + break; + } + } + if (StopFlag) { + break; + } + } + } + if(Marked.find(now) != Marked.end()) { + NextMarked[now].insert(now); + } else { + for(uint32_t pred : t_Pred[now]) { + for(uint32_t x : NextMarked[pred]) { + NextMarked[now].insert(x); + } + } + } +} + +void MarkSubGraph(uint32_t ss, uint32_t tt) { + TopologicalSort(ss, tt); + if(TopoOrder.empty()) return; + + for(uint32_t i : TopoOrder) { + NextMarked[i].clear(); + } + + NextMarked[TopoOrder[0]].insert(TopoOrder[0]); + for(uint32_t i = 1 ; i < TopoOrder.size() ; i += 1) { + MakeUniq(TopoOrder[i]); + } +} + +void MarkVertice(Function *F) { + uint32_t s = start_point; + + InDeg.resize(Blocks.size()); + Visited.resize(Blocks.size()); + InStack.resize(Blocks.size()); + t_Succ.resize(Blocks.size()); + t_Pred.resize(Blocks.size()); + NextMarked.resize(Blocks.size()); + + for( uint32_t i = 0 ; i < Blocks.size() ; i += 1 ) { + Visited[i] = InStack[i] = InDeg[i] = 0; + t_Succ[i].clear(); + t_Pred[i].clear(); + } + timeStamp = 0; + uint32_t t = 0; + //MarkSubGraph(s, t); + //return; + + while( s != t ) { + MarkSubGraph(DominatorTree::idom[t], t); + t = DominatorTree::idom[t]; + } + +} + +// return {marked nodes} +std::pair, + std::vector >markNodes(Function *F) { + assert(F->size() > 0 && "Function can not be empty"); + + reset(); + labelEachBlock(F); + buildCFG(F); + turnCFGintoDAG(F); + DominatorTree::DominatorTree(F); + MarkVertice(F); + + std::vector Result , ResultAbove; + for( uint32_t x : Markabove ) { + auto it = Marked.find( x ); + if( it != Marked.end() ) + Marked.erase( it ); + if( x ) + ResultAbove.push_back(Blocks[x]); + } + for( uint32_t x : Marked ) { + if (x == 0) { + continue; + } else { + Result.push_back(Blocks[x]); + } + } + + return { Result , ResultAbove }; +} diff --git a/llvm_mode/MarkNodes.h b/llvm_mode/MarkNodes.h new file mode 100644 index 00000000..e3bf3ce5 --- /dev/null +++ b/llvm_mode/MarkNodes.h @@ -0,0 +1,11 @@ +#ifndef __MARK_NODES__ +#define __MARK_NODES__ + +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Function.h" +#include + +std::pair, + std::vector> markNodes(llvm::Function *F); + +#endif diff --git a/llvm_mode/afl-clang-fast.c b/llvm_mode/afl-clang-fast.c index 5bc4ae8c..2034f10a 100644 --- a/llvm_mode/afl-clang-fast.c +++ b/llvm_mode/afl-clang-fast.c @@ -32,6 +32,7 @@ #include #include #include +#include static u8* obj_path; /* Path to runtime libraries */ static u8** cc_params; /* Parameters passed to the real CC */ @@ -87,7 +88,7 @@ static void find_obj(u8* argv0) { return; } - FATAL("Unable to find 'afl-llvm-rt.o' or 'afl-llvm-pass.so'. Please set AFL_PATH"); + FATAL("Unable to find 'afl-llvm-rt.o' or 'libLLVMInsTrim.so'. Please set AFL_PATH"); } @@ -113,7 +114,7 @@ static void edit_params(u32 argc, char** argv) { } /* There are two ways to compile afl-clang-fast. In the traditional mode, we - use afl-llvm-pass.so to inject instrumentation. In the experimental + use libLLVMInsTrim.so to inject instrumentation. In the experimental 'trace-pc-guard' mode, we use native LLVM instrumentation callbacks instead. The latter is a very recent addition - see: @@ -150,7 +151,8 @@ static void edit_params(u32 argc, char** argv) { cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); + cc_params[cc_par_cnt++] = alloc_printf("%s/libLLVMInsTrim.so", obj_path); +// cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); #endif /* ^USE_TRACE_PC */ cc_params[cc_par_cnt++] = "-Qunused-arguments"; -- cgit 1.4.1 From 0f1313761686a61983a4deb2646a8265f8acc214 Mon Sep 17 00:00:00 2001 From: Heiko Eißfeldt Date: Sat, 13 Jul 2019 23:40:34 +0200 Subject: compiles now with LLVM 8.0 --- llvm_mode/LLVMInsTrim.so.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'llvm_mode') diff --git a/llvm_mode/LLVMInsTrim.so.cc b/llvm_mode/LLVMInsTrim.so.cc index 2a13981d..bba0b86c 100644 --- a/llvm_mode/LLVMInsTrim.so.cc +++ b/llvm_mode/LLVMInsTrim.so.cc @@ -118,7 +118,7 @@ namespace { auto PredBB = I->first; auto SuccBB = I->second; auto NewBB = SplitBlockPredecessors(SuccBB, {PredBB}, ".split", - DT, nullptr, false); + DT); MS.insert(NewBB); } } -- cgit 1.4.1 From c204efaaab59551bb325af64620626be40aae993 Mon Sep 17 00:00:00 2001 From: Heiko Eißfeldt Date: Sat, 13 Jul 2019 23:12:36 +0200 Subject: Compile fix for LLVM 3.8.0 --- llvm_mode/LLVMInsTrim.so.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'llvm_mode') diff --git a/llvm_mode/LLVMInsTrim.so.cc b/llvm_mode/LLVMInsTrim.so.cc index bba0b86c..0c75eb4e 100644 --- a/llvm_mode/LLVMInsTrim.so.cc +++ b/llvm_mode/LLVMInsTrim.so.cc @@ -40,7 +40,12 @@ namespace { AU.addRequired(); } - StringRef getPassName() const override { +#if LLVM_VERSION_MAJOR < 4 + const char * +#else + StringRef +#endif + getPassName() const override { return "InstTrim Instrumentation"; } -- cgit 1.4.1 From 98a696391183274fd683383d015646ac28e64182 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sun, 14 Jul 2019 10:05:46 +0200 Subject: make fix --- llvm_mode/Makefile | 2 +- llvm_mode/MarkNodes.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'llvm_mode') diff --git a/llvm_mode/Makefile b/llvm_mode/Makefile index a66f18ab..2947dc9b 100644 --- a/llvm_mode/Makefile +++ b/llvm_mode/Makefile @@ -92,7 +92,7 @@ endif ln -sf afl-clang-fast ../afl-clang-fast++ ../libLLVMInsTrim.so: LLVMInsTrim.so.cc MarkNodes.cc | test_deps - $(CXX) $(CLANG_CFL) -DLLVMInsTrim_EXPORTS -fno-rtti -fPIC -std=gnu++11 -shared $< -o $@ $(CLANG_LFL) + $(CXX) $(CLANG_CFL) -DLLVMInsTrim_EXPORTS -fno-rtti -fPIC -std=gnu++11 -shared $< MarkNodes.cc -o $@ $(CLANG_LFL) # laf ../split-switches-pass.so: split-switches-pass.so.cc | test_deps diff --git a/llvm_mode/MarkNodes.cc b/llvm_mode/MarkNodes.cc index 3c2129ef..a156fccb 100644 --- a/llvm_mode/MarkNodes.cc +++ b/llvm_mode/MarkNodes.cc @@ -56,7 +56,7 @@ void buildCFG(Function *F) { Preds[ i ].clear(); } - uint32_t FakeID = 0; + //uint32_t FakeID = 0; for (auto S = F->begin(), E = F->end(); S != E; ++S) { BasicBlock *BB = &*S; uint32_t MyID = LMap[BB]; -- cgit 1.4.1 From 495f3b9a681af527018a92d3d0e3240568ac7997 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sun, 14 Jul 2019 10:23:54 +0200 Subject: notZero added and first attempt at whitelist --- llvm_mode/LLVMInsTrim.so.cc | 159 ++++++++++++++++++++++++++++++--- llvm_mode/compare-transform-pass.so.cc | 3 +- llvm_mode/split-compares-pass.so.cc | 3 +- llvm_mode/split-switches-pass.so.cc | 6 +- 4 files changed, 157 insertions(+), 14 deletions(-) (limited to 'llvm_mode') diff --git a/llvm_mode/LLVMInsTrim.so.cc b/llvm_mode/LLVMInsTrim.so.cc index 0c75eb4e..62977e97 100644 --- a/llvm_mode/LLVMInsTrim.so.cc +++ b/llvm_mode/LLVMInsTrim.so.cc @@ -1,3 +1,5 @@ +#include + #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/IR/CFG.h" @@ -10,8 +12,17 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/IR/DebugInfo.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CFG.h" #include #include +#include +#include +#include + +#include "../config.h" +#include "../debug.h" #include "MarkNodes.h" @@ -24,6 +35,10 @@ static cl::opt LoopHeadOpt("loophead", cl::desc("LoopHead"), namespace { struct InsTrim : public ModulePass { + + protected: + std::list myWhitelist; + private: std::mt19937 generator; int total_instr = 0; @@ -34,7 +49,23 @@ namespace { public: static char ID; - InsTrim() : ModulePass(ID), generator(0) {} + InsTrim() : ModulePass(ID), generator(0) {//} + +// AFLCoverage() : ModulePass(ID) { + char* instWhiteListFilename = getenv("AFL_LLVM_WHITELIST"); + if (instWhiteListFilename) { + std::string line; + std::ifstream fileStream; + fileStream.open(instWhiteListFilename); + if (!fileStream) + report_fatal_error("Unable to open AFL_LLVM_WHITELIST"); + getline(fileStream, line); + while (fileStream) { + myWhitelist.push_back(line); + getline(fileStream, line); + } + } + } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); @@ -50,12 +81,33 @@ namespace { } bool runOnModule(Module &M) override { + char be_quiet = 0; + + if (isatty(2) && !getenv("AFL_QUIET")) { + SAYF(cCYA "LLVMInsTrim" VERSION cRST " by csienslab\n"); + } else be_quiet = 1; + +#if LLVM_VERSION_MAJOR < 9 + char* neverZero_counters_str; + if ((neverZero_counters_str = getenv("AFL_LLVM_NOT_ZERO")) != NULL) + OKF("LLVM neverZero activated (by hexcoder)\n"); +#endif + if (getenv("LOOPHEAD")) { LoopHeadOpt = true; MarkSetOpt = true; } else if (getenv("MARKSET")) { MarkSetOpt = true; } + +/* // I dont think this makes sense to port into LLVMInsTrim + char* inst_ratio_str = getenv("AFL_INST_RATIO"); + unsigned int inst_ratio = 100; + if (inst_ratio_str) { + if (sscanf(inst_ratio_str, "%u", &inst_ratio) != 1 || !inst_ratio || inst_ratio > 100) + FATAL("Bad value of AFL_INST_RATIO (must be between 1 and 100)"); + } +*/ LLVMContext &C = M.getContext(); IntegerType *Int8Ty = IntegerType::getInt8Ty(C); @@ -69,14 +121,59 @@ namespace { M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_loc", 0, GlobalVariable::GeneralDynamicTLSModel, 0, false); - unsigned total_rs = 0; - unsigned total_hs = 0; + u64 total_rs = 0; + u64 total_hs = 0; for (Function &F : M) { if (!F.size()) { continue; } + if (!myWhitelist.empty()) { + bool instrumentBlock = false; + BasicBlock &BB = F.getEntryBlock(); + BasicBlock::iterator IP = BB.getFirstInsertionPt(); + IRBuilder<> IRB(&(*IP)); + DebugLoc Loc = IP->getDebugLoc(); + StringRef instFilename; + + if ( Loc ) { + DILocation *cDILoc = dyn_cast(Loc.getAsMDNode()); + + unsigned int instLine = cDILoc->getLine(); + instFilename = cDILoc->getFilename(); + + if (instFilename.str().empty()) { + /* If the original location is empty, try using the inlined location */ + DILocation *oDILoc = cDILoc->getInlinedAt(); + if (oDILoc) { + instFilename = oDILoc->getFilename(); + instLine = oDILoc->getLine(); + } + } + + /* Continue only if we know where we actually are */ + if (!instFilename.str().empty()) { + for (std::list::iterator it = myWhitelist.begin(); it != myWhitelist.end(); ++it) { + if (instFilename.str().length() >= it->length()) { + if (instFilename.str().compare(instFilename.str().length() - it->length(), it->length(), *it) == 0) { + instrumentBlock = true; + break; + } + } + } + } + } + + /* Either we couldn't figure out our location or the location is + * not whitelisted, so we skip instrumentation. */ + if (!instrumentBlock) { + if (!instFilename.str().empty()) + SAYF( "Not in whitelist, skipping %s ...\n", instFilename.str().c_str()); + continue; + } + } + std::unordered_set MS; if (!MarkSetOpt) { for (auto &BB : F) { @@ -94,8 +191,7 @@ namespace { total_rs += MS.size(); } else { DenseSet> EdgeSet; - DominatorTreeWrapperPass *DTWP = - &getAnalysis(F); + DominatorTreeWrapperPass *DTWP = &getAnalysis(F); auto DT = &DTWP->getDomTree(); total_rs += RS.size(); @@ -123,7 +219,11 @@ namespace { auto PredBB = I->first; auto SuccBB = I->second; auto NewBB = SplitBlockPredecessors(SuccBB, {PredBB}, ".split", - DT); + DT, nullptr, +#if LLVM_VERSION_MAJOR >= 8 + nullptr, +#endif + false); MS.insert(NewBB); } } @@ -168,21 +268,60 @@ namespace { L = PN; } + /* Load prev_loc */ LoadInst *PrevLoc = IRB.CreateLoad(OldPrev); + PrevLoc->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); Value *PrevLocCasted = IRB.CreateZExt(PrevLoc, IRB.getInt32Ty()); + /* Load SHM pointer */ LoadInst *MapPtr = IRB.CreateLoad(CovMapPtr); - Value *MapPtrIdx = IRB.CreateGEP(MapPtr, - IRB.CreateXor(PrevLocCasted, L)); + MapPtr->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + Value *MapPtrIdx = IRB.CreateGEP(MapPtr, IRB.CreateXor(PrevLocCasted, L)); + /* Update bitmap */ LoadInst *Counter = IRB.CreateLoad(MapPtrIdx); + Counter->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + Value *Incr = IRB.CreateAdd(Counter, ConstantInt::get(Int8Ty, 1)); - IRB.CreateStore(Incr, MapPtrIdx); + +#if LLVM_VERSION_MAJOR < 9 + if (neverZero_counters_str != NULL) { // with llvm 9 we make this the default as the bug in llvm is then fixed +#else + #warning "neverZero implementation needs to be reviewed!" +#endif + /* hexcoder: Realize a counter that skips zero during overflow. + * Once this counter reaches its maximum value, it next increments to 1 + * + * Instead of + * Counter + 1 -> Counter + * we inject now this + * Counter + 1 -> {Counter, OverflowFlag} + * Counter + OverflowFlag -> Counter + */ + auto cf = IRB.CreateICmpEQ(Incr, ConstantInt::get(Int8Ty, 0)); + auto carry = IRB.CreateZExt(cf, Int8Ty); + Incr = IRB.CreateAdd(Incr, carry); +#if LLVM_VERSION_MAJOR < 9 + } +#endif + + IRB.CreateStore(Incr, MapPtrIdx)->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + + /* Set prev_loc to cur_loc >> 1 */ + /* + StoreInst *Store = IRB.CreateStore(ConstantInt::get(Int32Ty, cur_loc >> 1), AFLPrevLoc); + Store->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + */ + total_instr++; } } - errs() << total_instr << " locations instrumented ("<< total_rs << "," << total_hs << ")\n"; + OKF("Instrumented %u locations (%llu, %llu) (%s mode)\n"/*", ratio %u%%)."*/, + total_instr, total_rs, total_hs, + getenv("AFL_HARDEN") ? "hardened" : + ((getenv("AFL_USE_ASAN") || getenv("AFL_USE_MSAN")) ? + "ASAN/MSAN" : "non-hardened")/*, inst_ratio*/); return false; } }; // end of struct InsTrim diff --git a/llvm_mode/compare-transform-pass.so.cc b/llvm_mode/compare-transform-pass.so.cc index d9a1f945..95435db7 100644 --- a/llvm_mode/compare-transform-pass.so.cc +++ b/llvm_mode/compare-transform-pass.so.cc @@ -304,7 +304,8 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, const bool CompareTransform::runOnModule(Module &M) { - llvm::errs() << "Running compare-transform-pass by laf.intel@gmail.com, extended by heiko@hexco.de\n"; + if (getenv("AFL_QUIET") == NULL) + llvm::errs() << "Running compare-transform-pass by laf.intel@gmail.com, extended by heiko@hexco.de\n"; transformCmps(M, true, true, true, true, true); verifyModule(M); diff --git a/llvm_mode/split-compares-pass.so.cc b/llvm_mode/split-compares-pass.so.cc index 2ea73aaa..c025628f 100644 --- a/llvm_mode/split-compares-pass.so.cc +++ b/llvm_mode/split-compares-pass.so.cc @@ -487,7 +487,8 @@ bool SplitComparesTransform::runOnModule(Module &M) { simplifySignedness(M); - errs() << "Split-compare-pass by laf.intel@gmail.com\n"; + if (getenv("AFL_QUIET") == NULL) + errs() << "Split-compare-pass by laf.intel@gmail.com\n"; switch (bitw) { case 64: diff --git a/llvm_mode/split-switches-pass.so.cc b/llvm_mode/split-switches-pass.so.cc index 4c28f34c..1ace3185 100644 --- a/llvm_mode/split-switches-pass.so.cc +++ b/llvm_mode/split-switches-pass.so.cc @@ -244,7 +244,8 @@ bool SplitSwitchesTransform::splitSwitches(Module &M) { /* If there is only the default destination or the condition checks 8 bit or less, don't bother with the code below. */ if (!SI->getNumCases() || bitw <= 8) { - errs() << "skip trivial switch..\n"; + if (getenv("AFL_QUIET") == NULL) + errs() << "skip trivial switch..\n"; continue; } @@ -302,7 +303,8 @@ bool SplitSwitchesTransform::splitSwitches(Module &M) { bool SplitSwitchesTransform::runOnModule(Module &M) { - llvm::errs() << "Running split-switches-pass by laf.intel@gmail.com\n"; + if (getenv("AFL_QUIET") == NULL) + llvm::errs() << "Running split-switches-pass by laf.intel@gmail.com\n"; splitSwitches(M); verifyModule(M); -- cgit 1.4.1 From e66402485342088e6fcaecfe2abbba291a48bda5 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sun, 14 Jul 2019 10:50:13 +0200 Subject: whitelist features works now --- docs/PATCHES | 1 + docs/README | 6 +++++- llvm_mode/LLVMInsTrim.so.cc | 19 ++++++++++++++----- llvm_mode/README.llvm | 10 ++++++---- 4 files changed, 26 insertions(+), 10 deletions(-) (limited to 'llvm_mode') diff --git a/docs/PATCHES b/docs/PATCHES index 06da053e..f6ca9284 100644 --- a/docs/PATCHES +++ b/docs/PATCHES @@ -17,6 +17,7 @@ afl-qemu-optimize-entrypoint.diff by mh(at)mh-sec(dot)de afl-qemu-speed.diff by abiondo on github afl-qemu-optimize-map.diff by mh(at)mh-sec(dot)de ++ instrim (https://github.com/csienslab/instrim) was integrated + MOpt (github.com/puppet-meteor/MOpt-AFL) was imported + AFLfast additions (github.com/mboehme/aflfast) were incorporated. + Qemu 3.1 upgrade with enhancement patches (github.com/andreafioraldi/afl) diff --git a/docs/README b/docs/README index 3f19d328..54e3e4a4 100644 --- a/docs/README +++ b/docs/README @@ -19,9 +19,13 @@ american fuzzy lop plus plus C. Hollers afl-fuzz Python mutator module and llvm_mode whitelist support was added too (https://github.com/choller/afl) - The newest additions is the excellent MOpt mutator from + New is the excellent MOpt mutator from https://github.com/puppet-meteor/MOpt-AFL + Also newly integrated is instrim, a very effective CFG llvm_mode + instrumentation implementation which replaced the original afl one and is + from https://github.com/csienslab/instrim + A more thorough list is available in the PATCHES file. So all in all this is the best-of AFL that is currently out there :-) diff --git a/llvm_mode/LLVMInsTrim.so.cc b/llvm_mode/LLVMInsTrim.so.cc index 62977e97..51640870 100644 --- a/llvm_mode/LLVMInsTrim.so.cc +++ b/llvm_mode/LLVMInsTrim.so.cc @@ -1,3 +1,6 @@ +#include +#include +#include #include #include "llvm/ADT/DenseMap.h" @@ -131,12 +134,16 @@ namespace { if (!myWhitelist.empty()) { bool instrumentBlock = false; - BasicBlock &BB = F.getEntryBlock(); - BasicBlock::iterator IP = BB.getFirstInsertionPt(); - IRBuilder<> IRB(&(*IP)); - DebugLoc Loc = IP->getDebugLoc(); + DebugLoc Loc; StringRef instFilename; + for (auto &BB : F) { + BasicBlock::iterator IP = BB.getFirstInsertionPt(); + IRBuilder<> IRB(&(*IP)); + if (!Loc) + Loc = IP->getDebugLoc(); + } + if ( Loc ) { DILocation *cDILoc = dyn_cast(Loc.getAsMDNode()); @@ -169,7 +176,9 @@ namespace { * not whitelisted, so we skip instrumentation. */ if (!instrumentBlock) { if (!instFilename.str().empty()) - SAYF( "Not in whitelist, skipping %s ...\n", instFilename.str().c_str()); + SAYF(cYEL "[!] " cBRI "Not in whitelist, skipping %s ...\n", instFilename.str().c_str()); + else + SAYF(cYEL "[!] " cBRI "No filename information found, skipping it"); continue; } } diff --git a/llvm_mode/README.llvm b/llvm_mode/README.llvm index dc860e97..b4e05a7a 100644 --- a/llvm_mode/README.llvm +++ b/llvm_mode/README.llvm @@ -38,8 +38,8 @@ co-exists with the original code. The idea and much of the implementation comes from Laszlo Szekeres. -2) How to use -------------- +2) How to use this +------------------ In order to leverage this mechanism, you need to have clang installed on your system. You should also make sure that the llvm-config tool is in your path @@ -69,8 +69,10 @@ operating mode of AFL, e.g.: Be sure to also include CXX set to afl-clang-fast++ for C++ code. The tool honors roughly the same environmental variables as afl-gcc (see -../docs/env_variables.txt). This includes AFL_INST_RATIO, AFL_USE_ASAN, -AFL_HARDEN, and AFL_DONT_OPTIMIZE. +../docs/env_variables.txt). This includes AFL_USE_ASAN, +AFL_HARDEN, and AFL_DONT_OPTIMIZE. However AFL_INST_RATIO is not honored +as it does not serve a good purpose with the more effective instrim CFG +analysis. Note: if you want the LLVM helper to be installed on your system for all users, you need to build it before issuing 'make install' in the parent -- cgit 1.4.1 From 013a1731d590eaa1f3e4c58c69985f89b7a3d2f9 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sun, 14 Jul 2019 19:48:28 +0200 Subject: set instrim as default and updated documentation --- docs/env_variables.txt | 13 ++++++++----- llvm_mode/LLVMInsTrim.so.cc | 6 +++--- llvm_mode/README.llvm | 27 +++++++++++++++++++++++---- 3 files changed, 34 insertions(+), 12 deletions(-) (limited to 'llvm_mode') diff --git a/docs/env_variables.txt b/docs/env_variables.txt index d854ea8d..8e2723d7 100644 --- a/docs/env_variables.txt +++ b/docs/env_variables.txt @@ -82,6 +82,9 @@ discussed in section #1, with the exception of: - TMPDIR and AFL_KEEP_ASSEMBLY, since no temporary assembly files are created. + - AFL_INST_RATIO, as we switched for instrim instrumentation which + is more effective but makes not much sense together with this option. + Then there are a few specific features that are only available in llvm_mode: LAF-INTEL @@ -108,16 +111,16 @@ Then there are a few specific features that are only available in llvm_mode: OTHER ===== - - Setting export AFL_LLVM_NOT_ZERO=1 during compilation will use counters + - Setting LOOPHEAD=1 optimized loops. afl-fuzz will only be able to + see the path the loop took, but not how many times it was called + (unless its a complex loop). + + - Setting AFL_LLVM_NOT_ZERO=1 during compilation will use counters that skip zero on overflow. This is the default for llvm >= 9, however for llvm versions below that this will increase an unnecessary slowdown due a performance issue that is only fixed in llvm 9+. This feature increases path discovery by a little bit. -Note that AFL_INST_RATIO will behave a bit differently than for afl-gcc, -because functions are *not* instrumented unconditionally - so low values -will have a more striking effect. For this tool, 0 is not a valid choice. - 3) Settings for afl-fuzz ------------------------ diff --git a/llvm_mode/LLVMInsTrim.so.cc b/llvm_mode/LLVMInsTrim.so.cc index 51640870..8e9f7667 100644 --- a/llvm_mode/LLVMInsTrim.so.cc +++ b/llvm_mode/LLVMInsTrim.so.cc @@ -98,10 +98,10 @@ namespace { if (getenv("LOOPHEAD")) { LoopHeadOpt = true; - MarkSetOpt = true; - } else if (getenv("MARKSET")) { - MarkSetOpt = true; } + + // this is our default + MarkSetOpt = true; /* // I dont think this makes sense to port into LLVMInsTrim char* inst_ratio_str = getenv("AFL_INST_RATIO"); diff --git a/llvm_mode/README.llvm b/llvm_mode/README.llvm index b4e05a7a..77c406f8 100644 --- a/llvm_mode/README.llvm +++ b/llvm_mode/README.llvm @@ -78,13 +78,32 @@ Note: if you want the LLVM helper to be installed on your system for all users, you need to build it before issuing 'make install' in the parent directory. -3) Gotchas, feedback, bugs +3) Options + +Several options are present to make llvm_mode faster or help it rearrange +the code to make afl-fuzz path discovery easier. + +If you need just to instrument specific parts of the code, you can whitelist +which C/C++ files to actually intrument. See README.whitelist + +For splitting memcmp, strncmp, etc. please see README.laf-intel + +As the original afl llvm_mode implementation has been replaced with +then much more effective instrim (https://github.com/csienslab/instrim/) +there is an option for optimizing loops. This optimization shows which +part of the loop has been selected, but not how many time a loop has been +called in a row (unless its a complex loop and a block inside was +instrumented). If you want to enable this set the environment variable +LOOPHEAD=1 + + +4) Gotchas, feedback, bugs -------------------------- This is an early-stage mechanism, so field reports are welcome. You can send bug reports to . -4) Bonus feature #1: deferred instrumentation +5) Bonus feature #1: deferred instrumentation --------------------------------------------- AFL tries to optimize performance by executing the targeted binary just once, @@ -131,7 +150,7 @@ will keep working normally when compiled with a tool other than afl-clang-fast. Finally, recompile the program with afl-clang-fast (afl-gcc or afl-clang will *not* generate a deferred-initialization binary) - and you should be all set! -5) Bonus feature #2: persistent mode +6) Bonus feature #2: persistent mode ------------------------------------ Some libraries provide APIs that are stateless, or whose state can be reset in @@ -171,7 +190,7 @@ PS. Because there are task switches still involved, the mode isn't as fast as faster than the normal fork() model, and compared to in-process fuzzing, should be a lot more robust. -6) Bonus feature #3: new 'trace-pc-guard' mode +8) Bonus feature #3: new 'trace-pc-guard' mode ---------------------------------------------- Recent versions of LLVM are shipping with a built-in execution tracing feature -- cgit 1.4.1 From 2628f9f61b4c2c0dda8b830ceae504d78f8acf69 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sun, 14 Jul 2019 23:32:35 +0200 Subject: fix crash with case insensitive compare functions (str(n)casecmp()) --- llvm_mode/compare-transform-pass.so.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'llvm_mode') diff --git a/llvm_mode/compare-transform-pass.so.cc b/llvm_mode/compare-transform-pass.so.cc index 95435db7..d0dbe8ec 100644 --- a/llvm_mode/compare-transform-pass.so.cc +++ b/llvm_mode/compare-transform-pass.so.cc @@ -257,6 +257,7 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, const std::vector args; args.push_back(load); load = IRB.CreateCall(tolowerFn, args, "tmp"); + load = IRB.CreateTrunc(load, Int8Ty); } Value *isub; if (HasStr1) @@ -272,14 +273,9 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, const next_bb = BasicBlock::Create(C, "cmp_added", end_bb->getParent(), end_bb); BranchInst::Create(end_bb, next_bb); -#if LLVM_VERSION_MAJOR < 8 - TerminatorInst *term = cur_bb->getTerminator(); -#else - Instruction *term = cur_bb->getTerminator(); -#endif Value *icmp = IRB.CreateICmpEQ(isub, ConstantInt::get(Int8Ty, 0)); IRB.CreateCondBr(icmp, next_bb, end_bb); - term->eraseFromParent(); + cur_bb->getTerminator()->eraseFromParent(); } else { //IRB.CreateBr(end_bb); } -- cgit 1.4.1 From 32525238238e96ec0ce64a36f70558f76bc90ff5 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 15 Jul 2019 11:22:54 +0200 Subject: fixing commit fuckup --- Makefile | 2 +- docs/ChangeLog | 8 +++----- docs/README | 3 +-- docs/env_variables.txt | 22 +++++++++++++++++----- llvm_mode/LLVMInsTrim.so.cc | 2 +- llvm_mode/Makefile | 13 ++++++++----- llvm_mode/README.llvm | 19 ++++++++++++------- llvm_mode/afl-clang-fast.c | 16 +++++++++------- 8 files changed, 52 insertions(+), 33 deletions(-) (limited to 'llvm_mode') diff --git a/Makefile b/Makefile index 60dfde18..6b580381 100644 --- a/Makefile +++ b/Makefile @@ -194,7 +194,7 @@ install: all rm -f $${DESTDIR}$(BIN_PATH)/afl-as if [ -f afl-qemu-trace ]; then install -m 755 afl-qemu-trace $${DESTDIR}$(BIN_PATH); fi ifndef AFL_TRACE_PC - if [ -f afl-clang-fast -a -f libLLVMInsTrim.so -a -f afl-llvm-rt.o ]; then set -e; install -m 755 afl-clang-fast $${DESTDIR}$(BIN_PATH); ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-fast++; install -m 755 libLLVMInsTrim.so afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH); fi + if [ -f afl-clang-fast -a -f libLLVMInsTrim.so -a -f afl-llvm-rt.o ]; then set -e; install -m 755 afl-clang-fast $${DESTDIR}$(BIN_PATH); ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-fast++; install -m 755 libLLVMInsTrim.so afl-llvm-pass.so afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH); fi else if [ -f afl-clang-fast -a -f afl-llvm-rt.o ]; then set -e; install -m 755 afl-clang-fast $${DESTDIR}$(BIN_PATH); ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-fast++; install -m 755 afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH); fi endif diff --git a/docs/ChangeLog b/docs/ChangeLog index 9cdca49b..116029ea 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -17,9 +17,9 @@ sending a mail to . Version ++2.52d (tbd): ----------------------------- - - added instrim a much better llvm_mode instrumentation - (https://github.com/csienslab/instrim) - - added MOpt (github.com/puppet-meteor/MOpt-AFL) mode + - added instrim, a much faster llvm_mode instrumentation at the cost of + path discovery. See llvm_mode/README.instrim (https://github.com/csienslab/instrim) + - added MOpt (github.com/puppet-meteor/MOpt-AFL) mode, see docs/README.MOpt - added code to make it more portable to other platforms than Intel Linux - added never zero counters for afl-gcc and optional (because of an optimization issue in llvm < 9) for llvm_mode (AFL_LLVM_NEVER_ZERO=1) @@ -41,8 +41,6 @@ Version ++2.52d (tbd): tests as the random numbers are deterministic then - llvm_mode LAF_... env variables can now be specified as AFL_LLVM_LAF_... that is longer but in line with other llvm specific env vars - - ... your idea or patch? - ----------------------------- diff --git a/docs/README b/docs/README index 54e3e4a4..3a6c2921 100644 --- a/docs/README +++ b/docs/README @@ -23,8 +23,7 @@ american fuzzy lop plus plus https://github.com/puppet-meteor/MOpt-AFL Also newly integrated is instrim, a very effective CFG llvm_mode - instrumentation implementation which replaced the original afl one and is - from https://github.com/csienslab/instrim + instrumentation implementation from https://github.com/csienslab/instrim A more thorough list is available in the PATCHES file. diff --git a/docs/env_variables.txt b/docs/env_variables.txt index 8e2723d7..e58327b4 100644 --- a/docs/env_variables.txt +++ b/docs/env_variables.txt @@ -109,11 +109,21 @@ Then there are a few specific features that are only available in llvm_mode: See llvm_mode/README.whitelist for more information. - OTHER - ===== - - Setting LOOPHEAD=1 optimized loops. afl-fuzz will only be able to - see the path the loop took, but not how many times it was called - (unless its a complex loop). + INSTRIM + ======= + This feature increases the speed by whopping 20% but at the cost of a + lower path discovery and thefore coverage. + + - Setting AFL_LLVM_INSTRIM activates this mode + + - Setting AFL_LLVM_INSTRIM LOOPHEAD=1 expands on INSTRIM to optimize loops. + afl-fuzz will only be able to see the path the loop took, but not how + many times it was called (unless its a complex loop). + + See llvm_mode/README.instrim + + NOT_ZERO + ======== - Setting AFL_LLVM_NOT_ZERO=1 during compilation will use counters that skip zero on overflow. This is the default for llvm >= 9, @@ -121,6 +131,8 @@ Then there are a few specific features that are only available in llvm_mode: slowdown due a performance issue that is only fixed in llvm 9+. This feature increases path discovery by a little bit. + See llvm_mode/README.neverzero + 3) Settings for afl-fuzz ------------------------ diff --git a/llvm_mode/LLVMInsTrim.so.cc b/llvm_mode/LLVMInsTrim.so.cc index 8e9f7667..81cf98c4 100644 --- a/llvm_mode/LLVMInsTrim.so.cc +++ b/llvm_mode/LLVMInsTrim.so.cc @@ -96,7 +96,7 @@ namespace { OKF("LLVM neverZero activated (by hexcoder)\n"); #endif - if (getenv("LOOPHEAD")) { + if (getenv("AFL_LLVM_INSTRIM_LOOPHEAD") != NULL || getenv("LOOPHEAD") != NULL) { LoopHeadOpt = true; } diff --git a/llvm_mode/Makefile b/llvm_mode/Makefile index d0d4b690..2b685ddc 100644 --- a/llvm_mode/Makefile +++ b/llvm_mode/Makefile @@ -94,7 +94,7 @@ endif ifndef AFL_TRACE_PC - PROGS = ../afl-clang-fast ../libLLVMInsTrim.so ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so + PROGS = ../afl-clang-fast ../afl-llvm-pass.so ../libLLVMInsTrim.so ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so else PROGS = ../afl-clang-fast ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so endif @@ -104,7 +104,7 @@ ifneq "$(CLANGVER)" "$(LLVMVER)" CXX = $(shell llvm-config --bindir)/clang++ endif -all: test_deps test_shm $(PROGS) test_build all_done +all: test_shm test_deps $(PROGS) test_build all_done ifeq "$(SHMAT_OK)" "1" @@ -132,10 +132,10 @@ endif @which $(CC) >/dev/null 2>&1 || ( echo "[-] Oops, can't find '$(CC)'. Make sure that it's in your \$$PATH (or set \$$CC and \$$CXX)."; exit 1 ) @echo "[*] Checking for matching versions of '$(CC)' and '$(LLVM_CONFIG)'" ifneq "$(CLANGVER)" "$(LLVMVER)" - @echo "WARNING: we have llvm-config version $(LLVMVER) and a clang version $(CLANGVER)" - @echo "Retrying with the clang compiler from llvm: CC=`llvm-config --bindir`/clang" + @echo "[!] WARNING: we have llvm-config version $(LLVMVER) and a clang version $(CLANGVER)" + @echo "[!] Retrying with the clang compiler from llvm: CC=`llvm-config --bindir`/clang" else - @echo "we have llvm-config version $(LLVMVER) with a clang version $(CLANGVER), good." + @echo "[*] We have llvm-config version $(LLVMVER) with a clang version $(CLANGVER), good." endif @echo "[*] Checking for '../afl-showmap'..." @test -f ../afl-showmap || ( echo "[-] Oops, can't find '../afl-showmap'. Be sure to compile AFL first."; exit 1 ) @@ -148,6 +148,9 @@ endif ../libLLVMInsTrim.so: LLVMInsTrim.so.cc MarkNodes.cc | test_deps $(CXX) $(CLANG_CFL) -DLLVMInsTrim_EXPORTS -fno-rtti -fPIC -std=gnu++11 -shared $< MarkNodes.cc -o $@ $(CLANG_LFL) +../afl-llvm-pass.so: afl-llvm-pass.so.cc | test_deps + $(CXX) $(CLANG_CFL) -DLLVMInsTrim_EXPORTS -fno-rtti -fPIC -std=gnu++11 -shared $< -o $@ $(CLANG_LFL) + # laf ../split-switches-pass.so: split-switches-pass.so.cc | test_deps $(CXX) $(CLANG_CFL) -shared $< -o $@ $(CLANG_LFL) diff --git a/llvm_mode/README.llvm b/llvm_mode/README.llvm index 77c406f8..779ff47c 100644 --- a/llvm_mode/README.llvm +++ b/llvm_mode/README.llvm @@ -88,13 +88,18 @@ which C/C++ files to actually intrument. See README.whitelist For splitting memcmp, strncmp, etc. please see README.laf-intel -As the original afl llvm_mode implementation has been replaced with -then much more effective instrim (https://github.com/csienslab/instrim/) -there is an option for optimizing loops. This optimization shows which -part of the loop has been selected, but not how many time a loop has been -called in a row (unless its a complex loop and a block inside was -instrumented). If you want to enable this set the environment variable -LOOPHEAD=1 +Then there is an optimized instrumentation strategy that uses CFGs and +markers to just instrument what is needed. This increases speed by 20-25% +however has a lower path discovery. +If you want to use this, set AFL_LLVM_INSTRIM=1 +See README.instrim + +Finally if your llvm version is 8 or lower, you can activate a mode that +prevents that a counter overflow result in a 0 value. This is good for +path discovery, but the llvm implementation for intel for this functionality +is not optimal and was only fixed in llvm 9. +You can set this with AFL_LLVM_NOT_ZERO=1 +See README.neverzero 4) Gotchas, feedback, bugs diff --git a/llvm_mode/afl-clang-fast.c b/llvm_mode/afl-clang-fast.c index 249eea7d..19bad86c 100644 --- a/llvm_mode/afl-clang-fast.c +++ b/llvm_mode/afl-clang-fast.c @@ -88,7 +88,7 @@ static void find_obj(u8* argv0) { return; } - FATAL("Unable to find 'afl-llvm-rt.o' or 'libLLVMInsTrim.so'. Please set AFL_PATH"); + FATAL("Unable to find 'afl-llvm-rt.o' or 'afl-llvm-pass.so.cc'. Please set AFL_PATH"); } @@ -113,11 +113,11 @@ static void edit_params(u32 argc, char** argv) { cc_params[0] = alt_cc ? alt_cc : (u8*)"clang"; } - /* There are two ways to compile afl-clang-fast. In the traditional mode, we - use libLLVMInsTrim.so to inject instrumentation. In the experimental + /* There are three ways to compile with afl-clang-fast. In the traditional + mode, we use afl-llvm-pass.so, then there is libLLVMInsTrim.so which is + much faster but has less coverage. Finally tere is the experimental 'trace-pc-guard' mode, we use native LLVM instrumentation callbacks - instead. The latter is a very recent addition - see: - + instead. For trace-pc-guard see: http://clang.llvm.org/docs/SanitizerCoverage.html#tracing-pcs-with-guards */ // laf @@ -151,8 +151,10 @@ static void edit_params(u32 argc, char** argv) { cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = alloc_printf("%s/libLLVMInsTrim.so", obj_path); -// cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); + if (getenv("AFL_LLVM_INSTRIM") != NULL || getenv("INSTRIM_LIB") != NULL) + cc_params[cc_par_cnt++] = alloc_printf("%s/libLLVMInsTrim.so", obj_path); + else + cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); #endif /* ^USE_TRACE_PC */ cc_params[cc_par_cnt++] = "-Qunused-arguments"; -- cgit 1.4.1 From 995eb0cd7972e2179ea9fe727d3c89d0b552c111 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Tue, 16 Jul 2019 08:51:00 +0200 Subject: deprecate afl-gcc --- afl-gcc.c | 2 ++ docs/ChangeLog | 4 +++- llvm_mode/README.llvm | 3 ++- types.h | 9 +++++++-- 4 files changed, 14 insertions(+), 4 deletions(-) (limited to 'llvm_mode') diff --git a/afl-gcc.c b/afl-gcc.c index 467a9bc1..2e3c4f76 100644 --- a/afl-gcc.c +++ b/afl-gcc.c @@ -311,6 +311,8 @@ int main(int argc, char** argv) { } else be_quiet = 1; + SAYF(cYEL "[!] " cBRI "WARNING: " cRST "afl-gcc is deprecated, gcc_plugin is faster, llvm_mode even faster\n"); + if (argc < 2) { SAYF("\n" diff --git a/docs/ChangeLog b/docs/ChangeLog index 8c1aa994..735653c0 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -17,10 +17,12 @@ sending a mail to . Version ++2.52d (tbd): ----------------------------- + - Using the old ineffective afl-gcc will now show a deprecation warning - if llvm_mode was compiled, afl-clang/afl-clang++ will point to these instead of afl-gcc - added gcc_plugin which is like llvm_mode but for gcc. This version - supports gcc version 5 to 8. See gcc_plugin/README (https://github.com/T12z/afl) + supports gcc version 5 to 8. See gcc_plugin/README.gcc + (https://github.com/T12z/afl) - added instrim, a much faster llvm_mode instrumentation at the cost of path discovery. See llvm_mode/README.instrim (https://github.com/csienslab/instrim) - added MOpt (github.com/puppet-meteor/MOpt-AFL) mode, see docs/README.MOpt diff --git a/llvm_mode/README.llvm b/llvm_mode/README.llvm index 779ff47c..aaa7b81f 100644 --- a/llvm_mode/README.llvm +++ b/llvm_mode/README.llvm @@ -3,6 +3,7 @@ Fast LLVM-based instrumentation for afl-fuzz ============================================ (See ../docs/README for the general instruction manual.) + (See ../gcc_plugin/README.gcc for the GCC-based instrumentation.) 1) Introduction --------------- @@ -30,7 +31,7 @@ several interesting properties: - The instrumentation can cope a bit better with multi-threaded targets. - Because the feature relies on the internals of LLVM, it is clang-specific - and will *not* work with GCC. + and will *not* work with GCC (see ../gcc_plugin/ for an alternative). Once this implementation is shown to be sufficiently robust and portable, it will probably replace afl-clang. For now, it can be built separately and diff --git a/types.h b/types.h index 784d3a7a..3497bb2b 100644 --- a/types.h +++ b/types.h @@ -80,7 +80,12 @@ typedef int64_t s64; #define MEM_BARRIER() \ asm volatile("" ::: "memory") -#define likely(_x) __builtin_expect(!!(_x), 1) -#define unlikely(_x) __builtin_expect(!!(_x), 0) +#if __GNUC__ < 6 + #define likely(_x) (_x) + #define unlikely(_x) (_x) +#else + #define likely(_x) __builtin_expect(!!(_x), 1) + #define unlikely(_x) __builtin_expect(!!(_x), 0) +#endif #endif /* ! _HAVE_TYPES_H */ -- cgit 1.4.1 From 80c98f4d0c31d857ff1112369d0a5ff0854e0923 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Tue, 16 Jul 2019 21:05:50 +0200 Subject: added readme --- llvm_mode/README.instrim | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 llvm_mode/README.instrim (limited to 'llvm_mode') diff --git a/llvm_mode/README.instrim b/llvm_mode/README.instrim new file mode 100644 index 00000000..956a9856 --- /dev/null +++ b/llvm_mode/README.instrim @@ -0,0 +1,26 @@ +# InsTrim +InsTrim: Lightweight Instrumentation for Coverage-guided Fuzzing + + +## Introduction + +InsTrim uses CFG and markers to instrument just what is necessary in the +binary in llvm_mode. It is about 20-25% faster but as a cost has a lower +path discovery. + + +## Usage + +Set the environment variable AFL_LLVM_INSTRIM=1 + +There is also an advanced mode which instruments loops in a way so that +afl-fuzz can see which loop path has been selected but not being able to +see how often the loop has been rerun. +This again is a tradeoff for speed for less path information. +To enable this mode set AFL_LLVM_INSTRIM_LOOPHEAD=1 + + +## Background + +The paper: [InsTrim: Lightweight Instrumentation for Coverage-guided Fuzzing] +(https://www.ndss-symposium.org/wp-content/uploads/2018/07/bar2018_14_Hsu_paper.pdf) -- cgit 1.4.1 From fe084b9866c5cb01408e3155078f092f64650edf Mon Sep 17 00:00:00 2001 From: Heiko Eißfeldt Date: Fri, 19 Jul 2019 11:17:30 +0200 Subject: several documentation fixes --- docs/ChangeLog | 8 ++++---- docs/README.MOpt | 30 +++++++++++++++++++----------- docs/binaryonly_fuzzing.txt | 27 ++++++++++++++------------- docs/env_variables.txt | 9 +++++---- gcc_plugin/README.gcc | 4 ++-- llvm_mode/README.llvm | 4 ++-- 6 files changed, 46 insertions(+), 36 deletions(-) (limited to 'llvm_mode') diff --git a/docs/ChangeLog b/docs/ChangeLog index f2709877..be50215c 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -30,7 +30,7 @@ Version ++2.52d (tbd): path discovery. See llvm_mode/README.instrim (https://github.com/csienslab/instrim) - added MOpt (github.com/puppet-meteor/MOpt-AFL) mode, see docs/README.MOpt - added code to make it more portable to other platforms than Intel Linux - - added never zero counters for afl-gcc and optional (because of an + - added never zero counters for afl-gcc and optionally (because of an optimization issue in llvm < 9) for llvm_mode (AFL_LLVM_NEVER_ZERO=1) - added a new doc about binary only fuzzing: docs/binaryonly_fuzzing.txt - more cpu power for afl-system-config @@ -45,8 +45,8 @@ Version ++2.52d (tbd): debugging - added -V time and -E execs option to better comparison runs, runs afl-fuzz for a specific time/executions. - - added a -s seed switch to allow afl run with a fixed initial - seed that is not updated. this is good for performance and path discovery + - added a -s seed switch to allow afl run with a fixed initial + seed that is not updated. This is good for performance and path discovery tests as the random numbers are deterministic then - llvm_mode LAF_... env variables can now be specified as AFL_LLVM_LAF_... that is longer but in line with other llvm specific env vars @@ -59,7 +59,7 @@ Version ++2.52c (2019-06-05): - Applied community patches. See docs/PATCHES for the full list. LLVM and Qemu modes are now faster. Important changes: - afl-fuzz: -e EXTENSION commandline option + afl-fuzz: -e EXTENSION commandline option llvm_mode: LAF-intel performance (needs activation, see llvm/README.laf-intel) a few new environment variables for afl-fuzz, llvm and qemu, see docs/env_variables.txt - Added the power schedules of AFLfast by Marcel Boehme, but set the default diff --git a/docs/README.MOpt b/docs/README.MOpt index 5575189c..94e63959 100644 --- a/docs/README.MOpt +++ b/docs/README.MOpt @@ -17,7 +17,8 @@ We open source all the seed sets used in the paper ### 4. Experiment Results The experiment results can be found in -https://drive.google.com/drive/folders/184GOzkZGls1H2NuLuUfSp9gfqp1E2-lL?usp=sharing. We only open source the crash files since the space is limited. +https://drive.google.com/drive/folders/184GOzkZGls1H2NuLuUfSp9gfqp1E2-lL?usp=sharing. +We only open source the crash files since the space is limited. ### 5. Technical Report MOpt_TechReport.pdf is the technical report of the paper @@ -26,18 +27,25 @@ MOpt_TechReport.pdf is the technical report of the paper ### 6. Parameter Introduction Most important, you must add the parameter `-L` (e.g., `-L 0`) to launch the MOpt scheme. -
`-L` controls the time to move on to the pacemaker fuzzing mode. -
`-L t:` when MOpt-AFL finishes the mutation of one input, if it has not -discovered any new unique crash or path for more than t min, MOpt-AFL will + +Option '-L' controls the time to move on to the pacemaker fuzzing mode. +'-L t': when MOpt-AFL finishes the mutation of one input, if it has not +discovered any new unique crash or path for more than t minutes, MOpt-AFL will enter the pacemaker fuzzing mode. -
Setting 0 will enter the pacemaker fuzzing mode at first, which is + +Setting 0 will enter the pacemaker fuzzing mode at first, which is recommended in a short time-scale evaluation. Other important parameters can be found in afl-fuzz.c, for instance, -
`swarm_num:` the number of the PSO swarms used in the fuzzing process. -
`period_pilot:` how many times MOpt-AFL will execute the target program in the pilot fuzzing module, then it will enter the core fuzzing module. -
`period_core:` how many times MOpt-AFL will execute the target program in the core fuzzing module, then it will enter the PSO updating module. -
`limit_time_bound:` control how many interesting test cases need to be found before MOpt-AFL quits the pacemaker fuzzing mode and reuses the deterministic stage. -0 < `limit_time_bound` < 1, MOpt-AFL-tmp. `limit_time_bound` >= 1, MOpt-AFL-ever. -Having fun with MOpt in AFL! +'swarm_num': the number of the PSO swarms used in the fuzzing process. +'period_pilot': how many times MOpt-AFL will execute the target program + in the pilot fuzzing module, then it will enter the core fuzzing module. +'period_core': how many times MOpt-AFL will execute the target program in the + core fuzzing module, then it will enter the PSO updating module. +'limit_time_bound': control how many interesting test cases need to be found + before MOpt-AFL quits the pacemaker fuzzing mode and reuses the deterministic stage. + 0 < 'limit_time_bound' < 1, MOpt-AFL-tmp. + 'limit_time_bound' >= 1, MOpt-AFL-ever. + +Have fun with MOpt in AFL! diff --git a/docs/binaryonly_fuzzing.txt b/docs/binaryonly_fuzzing.txt index f370ec74..ae5269f0 100644 --- a/docs/binaryonly_fuzzing.txt +++ b/docs/binaryonly_fuzzing.txt @@ -11,7 +11,7 @@ then standard afl++ (dumb mode) is not effective. The following is a description of how these can be fuzzed with afl++ !!!!! -DTLR: try DYNINST with afl-dyninst. If it produces too many crashes then +TL;DR: try DYNINST with afl-dyninst. If it produces too many crashes then use afl -Q qemu_mode. !!!!! @@ -22,7 +22,7 @@ Qemu is the "native" solution to the program. It is available in the ./qemu_mode/ directory and once compiled it can be accessed by the afl-fuzz -Q command line option. The speed decrease is at about 50% -It the easiest to use alternative and even works for cross-platform binaries. +It is the easiest to use alternative and even works for cross-platform binaries. As it is included in afl++ this needs no URL. @@ -30,7 +30,7 @@ As it is included in afl++ this needs no URL. DYNINST ------- Dyninst is a binary instrumentation framework similar to Pintool and Dynamorio -(see far below). Howver whereas Pintool and Dynamorio work at runtime, dyninst +(see far below). However whereas Pintool and Dynamorio work at runtime, dyninst instruments the target at load time, and then let it run. This is great for some things, e.g. fuzzing, and not so effective for others, e.g. malware analysis. @@ -38,15 +38,15 @@ e.g. malware analysis. So what we can do with dyninst is taking every basic block, and put afl's instrumention code in there - and then save the binary. Afterwards we can just fuzz the newly saved target binary with afl-fuzz. -Sounds great? It is. The issue though - this is a non-trivial problem to -insert instructions, which changes addresses in the process space and that -everything still works afterwards. Hence more often than not binaries -crash when they are run. +Sounds great? It is. The issue though - it is a non-trivial problem to +insert instructions, which change addresses in the process space, so +everything is still working afterwards. Hence more often than not binaries +crash when they are run (because of instrumentation). The speed decrease is about 15-35%, depending on the optimization options used with afl-dyninst. -So if dyninst works, its the best option available. Otherwise it just doesn't +So if dyninst works, it is the best option available. Otherwise it just doesn't work well. https://github.com/vanhauser-thc/afl-dyninst @@ -54,13 +54,14 @@ https://github.com/vanhauser-thc/afl-dyninst INTEL-PT -------- +If you have a newer Intel CPU, you can make use of Intels processor trace. The big issue with Intel's PT is the small buffer size and the complex encoding of the debug information collected through PT. This makes the decoding very CPU intensive and hence slow. As a result, the overall speed decrease is about 70-90% (depending on -the implementation and other factors) +the implementation and other factors). -there are two afl intel-pt implementations: +There are two afl intel-pt implementations: 1. https://github.com/junxzm1990/afl-pt => this needs Ubuntu 14.04.05 without any updates and the 4.4 kernel. @@ -73,13 +74,13 @@ there are two afl intel-pt implementations: CORESIGHT --------- -Coresight is the ARM answer to Intel's PT. +Coresight is ARM's answer to Intel's PT. There is no implementation so far which handle coresight and getting -it working on an ARM Linux is very difficult due custom kernel building +it working on an ARM Linux is very difficult due to custom kernel building on embedded systems is difficult. And finding one that has coresight in the ARM chip is difficult too. My guess is that it is slower than Qemu, but faster than Intel PT. -If anyone finds any coresight implemention for afl please ping me: +If anyone finds any coresight implementation for afl please ping me: vh@thc.org diff --git a/docs/env_variables.txt b/docs/env_variables.txt index 338df36f..1703a947 100644 --- a/docs/env_variables.txt +++ b/docs/env_variables.txt @@ -90,7 +90,8 @@ Then there are a few specific features that are only available in llvm_mode: LAF-INTEL ========= This great feature will split compares to series of single byte comparisons - to allow afl-fuzz to find otherwise rather impossible paths. + to allow afl-fuzz to find otherwise rather impossible paths. It is not + restricted to Intel CPUs ;-) - Setting AFL_LLVM_LAF_SPLIT_SWITCHES will split switch()es @@ -105,20 +106,20 @@ Then there are a few specific features that are only available in llvm_mode: This feature allows selectively instrumentation of the source - Setting AFL_LLVM_WHITELIST with a filename will only instrument those - files that match these names. + files that match the names listed in this file. See llvm_mode/README.whitelist for more information. INSTRIM ======= This feature increases the speed by whopping 20% but at the cost of a - lower path discovery and thefore coverage. + lower path discovery and therefore coverage. - Setting AFL_LLVM_INSTRIM activates this mode - Setting AFL_LLVM_INSTRIM_LOOPHEAD=1 expands on INSTRIM to optimize loops. afl-fuzz will only be able to see the path the loop took, but not how - many times it was called (unless its a complex loop). + many times it was called (unless it is a complex loop). See llvm_mode/README.instrim diff --git a/gcc_plugin/README.gcc b/gcc_plugin/README.gcc index b3e9c853..fe62020b 100644 --- a/gcc_plugin/README.gcc +++ b/gcc_plugin/README.gcc @@ -65,8 +65,8 @@ directory. This is an early-stage mechanism, so field reports are welcome. You can send bug reports to . -4) Bonus feature #1: deferred instrumentation ---------------------------------------------- +4) Bonus feature #1: deferred initialization +-------------------------------------------- AFL tries to optimize performance by executing the targeted binary just once, stopping it just before main(), and then cloning this "master" process to get diff --git a/llvm_mode/README.llvm b/llvm_mode/README.llvm index aaa7b81f..00528a46 100644 --- a/llvm_mode/README.llvm +++ b/llvm_mode/README.llvm @@ -109,8 +109,8 @@ See README.neverzero This is an early-stage mechanism, so field reports are welcome. You can send bug reports to . -5) Bonus feature #1: deferred instrumentation ---------------------------------------------- +5) Bonus feature #1: deferred initialization +-------------------------------------------- AFL tries to optimize performance by executing the targeted binary just once, stopping it just before main(), and then cloning this "master" process to get -- cgit 1.4.1 From ce842648ae0462f339625ab395a9d71a9ecd319d Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 25 Jul 2019 11:18:14 +0200 Subject: afl_trace_pc fix --- docs/ChangeLog | 1 + llvm_mode/afl-clang-fast.c | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'llvm_mode') diff --git a/docs/ChangeLog b/docs/ChangeLog index 3b43366d..09ad77ec 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -18,6 +18,7 @@ Version ++2.52d (dev): ----------------------------- - unicorn_mode got added - thanks to domenukk for the patch! + - fix llvm_mode AFL_TRACE_PC with modern llvm - fix a crash in qemu_mode which also exists in stock afl - added libcompcov, a laf-intel implementation for qemu! :) see qemu_mode/libcompcov/README.libcompcov diff --git a/llvm_mode/afl-clang-fast.c b/llvm_mode/afl-clang-fast.c index 19bad86c..a4bb7539 100644 --- a/llvm_mode/afl-clang-fast.c +++ b/llvm_mode/afl-clang-fast.c @@ -144,9 +144,10 @@ static void edit_params(u32 argc, char** argv) { // /laf #ifdef USE_TRACE_PC - cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; - cc_params[cc_par_cnt++] = "-mllvm"; - cc_params[cc_par_cnt++] = "-sanitizer-coverage-block-threshold=0"; + cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; // edge coverage by default + //cc_params[cc_par_cnt++] = "-mllvm"; + //cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-cmp,trace-div,trace-gep"; + //cc_params[cc_par_cnt++] = "-sanitizer-coverage-block-threshold=0"; #else cc_params[cc_par_cnt++] = "-Xclang"; cc_params[cc_par_cnt++] = "-load"; -- cgit 1.4.1 From 8f4f45c524d217236a2e64be0d95d0a6de11df9c Mon Sep 17 00:00:00 2001 From: van Hauser Date: Fri, 26 Jul 2019 10:35:58 +0200 Subject: incorporated most of the 2.53b changes --- README | 1 - README.md | 607 ++++++++++++++++++++++++++++++++++++++++++++++++++ afl-fuzz.c | 2 + docs/README | 592 ------------------------------------------------ llvm_mode/README.llvm | 6 +- types.h | 2 +- 6 files changed, 612 insertions(+), 598 deletions(-) delete mode 120000 README create mode 100644 README.md delete mode 100644 docs/README (limited to 'llvm_mode') diff --git a/README b/README deleted file mode 120000 index a90f4af9..00000000 --- a/README +++ /dev/null @@ -1 +0,0 @@ -docs/README \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..e1371175 --- /dev/null +++ b/README.md @@ -0,0 +1,607 @@ +# american fuzzy lop plus plus (afl++) + + Originally developed by Michal "lcamtuf" Zalewski. + + Repository: [https://github.com/vanhauser-thc/AFLplusplus](https://github.com/vanhauser-thc/AFLplusplus) + + afl++ is maintained by Marc Heuse , Heiko Eissfeldt + and Andrea Fioraldi . + +## The enhancements compared to the original stock afl + + Many improvements were made over the official afl release - which did not + get any improvements since November 2017. + + Among others afl++ has, e.g. more performant llvm_mode, supporting + llvm up to version 8, Qemu 3.1, more speed and crashfixes for Qemu, + laf-intel feature for Qemu (with libcompcov) and more. + + Additionally the following patches have been integrated: + + * AFLfast's power schedules by Marcel Boehme: [https://github.com/mboehme/aflfast](https://github.com/mboehme/aflfast) + + * C. Hollers afl-fuzz Python mutator module and llvm_mode whitelist support: [https://github.com/choller/afl](https://github.com/choller/afl) + + * the new excellent MOpt mutator: [https://github.com/puppet-meteor/MOpt-AFL](https://github.com/puppet-meteor/MOpt-AFL) + + * instrim, a very effective CFG llvm_mode instrumentation implementation for large targets: [https://github.com/csienslab/instrim](https://github.com/csienslab/instrim) + + * unicorn_mode which allows fuzzing of binaries from completely different platforms (integration provided by domenukk) + + A more thorough list is available in the PATCHES file. + + So all in all this is the best-of AFL that is currently out there :-) + + For new versions and additional information, check out: + [https://github.com/vanhauser-thc/AFLplusplus](https://github.com/vanhauser-thc/AFLplusplus) + + To compare notes with other users or get notified about major new features, + send a mail to . + + See [docs/QuickStartGuide.txt](docs/QuickStartGuide.txt) if you don't have time to + read this file. + + +## 1) Challenges of guided fuzzing +------------------------------- + +Fuzzing is one of the most powerful and proven strategies for identifying +security issues in real-world software; it is responsible for the vast +majority of remote code execution and privilege escalation bugs found to date +in security-critical software. + +Unfortunately, fuzzing is also relatively shallow; blind, random mutations +make it very unlikely to reach certain code paths in the tested code, leaving +some vulnerabilities firmly outside the reach of this technique. + +There have been numerous attempts to solve this problem. One of the early +approaches - pioneered by Tavis Ormandy - is corpus distillation. The method +relies on coverage signals to select a subset of interesting seeds from a +massive, high-quality corpus of candidate files, and then fuzz them by +traditional means. The approach works exceptionally well, but requires such +a corpus to be readily available. In addition, block coverage measurements +provide only a very simplistic understanding of program state, and are less +useful for guiding the fuzzing effort in the long haul. + +Other, more sophisticated research has focused on techniques such as program +flow analysis ("concolic execution"), symbolic execution, or static analysis. +All these methods are extremely promising in experimental settings, but tend +to suffer from reliability and performance problems in practical uses - and +currently do not offer a viable alternative to "dumb" fuzzing techniques. + + +## 2) The afl-fuzz approach + +American Fuzzy Lop is a brute-force fuzzer coupled with an exceedingly simple +but rock-solid instrumentation-guided genetic algorithm. It uses a modified +form of edge coverage to effortlessly pick up subtle, local-scale changes to +program control flow. + +Simplifying a bit, the overall algorithm can be summed up as: + + 1) Load user-supplied initial test cases into the queue, + + 2) Take next input file from the queue, + + 3) Attempt to trim the test case to the smallest size that doesn't alter + the measured behavior of the program, + + 4) Repeatedly mutate the file using a balanced and well-researched variety + of traditional fuzzing strategies, + + 5) If any of the generated mutations resulted in a new state transition + recorded by the instrumentation, add mutated output as a new entry in the + queue. + + 6) Go to 2. + +The discovered test cases are also periodically culled to eliminate ones that +have been obsoleted by newer, higher-coverage finds; and undergo several other +instrumentation-driven effort minimization steps. + +As a side result of the fuzzing process, the tool creates a small, +self-contained corpus of interesting test cases. These are extremely useful +for seeding other, labor- or resource-intensive testing regimes - for example, +for stress-testing browsers, office applications, graphics suites, or +closed-source tools. + +The fuzzer is thoroughly tested to deliver out-of-the-box performance far +superior to blind fuzzing or coverage-only tools. + + +## 3) Instrumenting programs for use with AFL + +PLEASE NOTE: llvm_mode compilation with afl-clang-fast/afl-clang-fast++ +instead of afl-gcc/afl-g++ is much faster and has a few cool features. +See llvm_mode/ - however few code does not compile with llvm. +We support llvm versions 4.0 to 8. + +When source code is available, instrumentation can be injected by a companion +tool that works as a drop-in replacement for gcc or clang in any standard build +process for third-party code. + +The instrumentation has a fairly modest performance impact; in conjunction with +other optimizations implemented by afl-fuzz, most programs can be fuzzed as fast +or even faster than possible with traditional tools. + +The correct way to recompile the target program may vary depending on the +specifics of the build process, but a nearly-universal approach would be: + +```shell +$ CC=/path/to/afl/afl-gcc ./configure +$ make clean all +``` + +For C++ programs, you'd would also want to set `CXX=/path/to/afl/afl-g++`. + +The clang wrappers (afl-clang and afl-clang++) can be used in the same way; +clang users may also opt to leverage a higher-performance instrumentation mode, +as described in [llvm_mode/README.llvm](llvm_mode/README.llvm). +Clang/LLVM has a much better performance and works with LLVM version 4.0 to 8. + +Using the LAF Intel performance enhancements are also recommended, see +[llvm_mode/README.laf-intel](llvm_mode/README.laf-intel) + +Using partial instrumentation is also recommended, see +[llvm_mode/README.whitelist](llvm_mode/README.whitelist) + +When testing libraries, you need to find or write a simple program that reads +data from stdin or from a file and passes it to the tested library. In such a +case, it is essential to link this executable against a static version of the +instrumented library, or to make sure that the correct .so file is loaded at +runtime (usually by setting `LD_LIBRARY_PATH`). The simplest option is a static +build, usually possible via: + +```shell +$ CC=/path/to/afl/afl-gcc ./configure --disable-shared +``` + +Setting `AFL_HARDEN=1` when calling 'make' will cause the CC wrapper to +automatically enable code hardening options that make it easier to detect +simple memory bugs. Libdislocator, a helper library included with AFL (see +[libdislocator/README.dislocator](libdislocator/README.dislocator)) can help uncover heap corruption issues, too. + +PS. ASAN users are advised to review [docs/notes_for_asan.txt](docs/notes_for_asan.txt) +file for important caveats. + + +## 4) Instrumenting binary-only apps +--------------------------------- + +When source code is *NOT* available, the fuzzer offers experimental support for +fast, on-the-fly instrumentation of black-box binaries. This is accomplished +with a version of QEMU running in the lesser-known "user space emulation" mode. + +QEMU is a project separate from AFL, but you can conveniently build the +feature by doing: + +```shell +$ cd qemu_mode +$ ./build_qemu_support.sh +``` + +For additional instructions and caveats, see [qemu_mode/README.qemu](qemu_mode/README.qemu). + +The mode is approximately 2-5x slower than compile-time instrumentation, is +less conductive to parallelization, and may have some other quirks. + +If [afl-dyninst](https://github.com/vanhauser-thc/afl-dyninst) works for +your binary, then you can use afl-fuzz normally and it will have twice +the speed compared to qemu_mode. + +A more comprehensive description of these and other options can be found in +[docs/binaryonly_fuzzing.txt](docs/binaryonly_fuzzing.txt) + + +## 5) Power schedules +------------------ + +The power schedules were copied from Marcel Böhme's excellent AFLfast +implementation and expands on the ability to discover new paths and +therefore the coverage. + +The available schedules are: + + - explore (default) + - fast + - coe + - quad + - lin + - exploit + +In parallel mode (-M/-S, several instances with shared queue), we suggest to +run the master using the exploit schedule (-p exploit) and the slaves with a +combination of cut-off-exponential (-p coe), exponential (-p fast; default), +and explore (-p explore) schedules. + +In single mode, using -p fast is usually more beneficial than the default +explore mode. +(We don't want to change the default behaviour of afl, so "fast" has not been +made the default mode). + +More details can be found in the paper published at the 23rd ACM Conference on +Computer and Communications Security (CCS'16): + + (https://www.sigsac.org/ccs/CCS2016/accepted-papers/)[https://www.sigsac.org/ccs/CCS2016/accepted-papers/] + + +## 6) Choosing initial test cases +------------------------------ + +To operate correctly, the fuzzer requires one or more starting file that +contains a good example of the input data normally expected by the targeted +application. There are two basic rules: + + - Keep the files small. Under 1 kB is ideal, although not strictly necessary. + For a discussion of why size matters, see [perf_tips.txt](docs/perf_tips.txt). + + - Use multiple test cases only if they are functionally different from + each other. There is no point in using fifty different vacation photos + to fuzz an image library. + +You can find many good examples of starting files in the testcases/ subdirectory +that comes with this tool. + +PS. If a large corpus of data is available for screening, you may want to use +the afl-cmin utility to identify a subset of functionally distinct files that +exercise different code paths in the target binary. + + +## 7) Fuzzing binaries +------------------- + +The fuzzing process itself is carried out by the afl-fuzz utility. This program +requires a read-only directory with initial test cases, a separate place to +store its findings, plus a path to the binary to test. + +For target binaries that accept input directly from stdin, the usual syntax is: + +```shell +$ ./afl-fuzz -i testcase_dir -o findings_dir /path/to/program [...params...] +``` + +For programs that take input from a file, use '@@' to mark the location in +the target's command line where the input file name should be placed. The +fuzzer will substitute this for you: + +```shell +$ ./afl-fuzz -i testcase_dir -o findings_dir /path/to/program @@ +``` + +You can also use the -f option to have the mutated data written to a specific +file. This is useful if the program expects a particular file extension or so. + +Non-instrumented binaries can be fuzzed in the QEMU mode (add -Q in the command +line) or in a traditional, blind-fuzzer mode (specify -n). + +You can use -t and -m to override the default timeout and memory limit for the +executed process; rare examples of targets that may need these settings touched +include compilers and video decoders. + +Tips for optimizing fuzzing performance are discussed in [perf_tips.txt](docs/perf_tips.txt). + +Note that afl-fuzz starts by performing an array of deterministic fuzzing +steps, which can take several days, but tend to produce neat test cases. If you +want quick & dirty results right away - akin to zzuf and other traditional +fuzzers - add the -d option to the command line. + + +## 8) Interpreting output +---------------------- + +See the [docs/status_screen.txt](docs/status_screen.txt) file for information on +how to interpret the displayed stats and monitor the health of the process. Be +sure to consult this file especially if any UI elements are highlighted in red. + +The fuzzing process will continue until you press Ctrl-C. At minimum, you want +to allow the fuzzer to complete one queue cycle, which may take anywhere from a +couple of hours to a week or so. + +There are three subdirectories created within the output directory and updated +in real time: + + - queue/ - test cases for every distinctive execution path, plus all the + starting files given by the user. This is the synthesized corpus + mentioned in section 2. + + Before using this corpus for any other purposes, you can shrink + it to a smaller size using the afl-cmin tool. The tool will find + a smaller subset of files offering equivalent edge coverage. + + - crashes/ - unique test cases that cause the tested program to receive a + fatal signal (e.g., SIGSEGV, SIGILL, SIGABRT). The entries are + grouped by the received signal. + + - hangs/ - unique test cases that cause the tested program to time out. The + default time limit before something is classified as a hang is + the larger of 1 second and the value of the -t parameter. + The value can be fine-tuned by setting AFL_HANG_TMOUT, but this + is rarely necessary. + +Crashes and hangs are considered "unique" if the associated execution paths +involve any state transitions not seen in previously-recorded faults. If a +single bug can be reached in multiple ways, there will be some count inflation +early in the process, but this should quickly taper off. + +The file names for crashes and hangs are correlated with parent, non-faulting +queue entries. This should help with debugging. + +When you can't reproduce a crash found by afl-fuzz, the most likely cause is +that you are not setting the same memory limit as used by the tool. Try: + +```shell +$ LIMIT_MB=50 +$ ( ulimit -Sv $[LIMIT_MB << 10]; /path/to/tested_binary ... ) +``` + +Change LIMIT_MB to match the -m parameter passed to afl-fuzz. On OpenBSD, +also change -Sv to -Sd. + +Any existing output directory can be also used to resume aborted jobs; try: + +```shell +$ ./afl-fuzz -i- -o existing_output_dir [...etc...] +``` + +If you have gnuplot installed, you can also generate some pretty graphs for any +active fuzzing task using afl-plot. For an example of how this looks like, +see [http://lcamtuf.coredump.cx/afl/plot/](http://lcamtuf.coredump.cx/afl/plot/). + + +## 9) Parallelized fuzzing +----------------------- + +Every instance of afl-fuzz takes up roughly one core. This means that on +multi-core systems, parallelization is necessary to fully utilize the hardware. +For tips on how to fuzz a common target on multiple cores or multiple networked +machines, please refer to [parallel_fuzzing.txt](docs/parallel_fuzzing.txt). + +The parallel fuzzing mode also offers a simple way for interfacing AFL to other +fuzzers, to symbolic or concolic execution engines, and so forth; again, see the +last section of [parallel_fuzzing.txt](docs/parallel_fuzzing.txt) for tips. + + +## 10) Fuzzer dictionaries +---------------------- + +By default, afl-fuzz mutation engine is optimized for compact data formats - +say, images, multimedia, compressed data, regular expression syntax, or shell +scripts. It is somewhat less suited for languages with particularly verbose and +redundant verbiage - notably including HTML, SQL, or JavaScript. + +To avoid the hassle of building syntax-aware tools, afl-fuzz provides a way to +seed the fuzzing process with an optional dictionary of language keywords, +magic headers, or other special tokens associated with the targeted data type +-- and use that to reconstruct the underlying grammar on the go: + + [http://lcamtuf.blogspot.com/2015/01/afl-fuzz-making-up-grammar-with.html](http://lcamtuf.blogspot.com/2015/01/afl-fuzz-making-up-grammar-with.html) + +To use this feature, you first need to create a dictionary in one of the two +formats discussed in [dictionaries/README.dictionaries](ictionaries/README.dictionaries); +and then point the fuzzer to it via the -x option in the command line. + +(Several common dictionaries are already provided in that subdirectory, too.) + +There is no way to provide more structured descriptions of the underlying +syntax, but the fuzzer will likely figure out some of this based on the +instrumentation feedback alone. This actually works in practice, say: + + [http://lcamtuf.blogspot.com/2015/04/finding-bugs-in-sqlite-easy-way.html](http://lcamtuf.blogspot.com/2015/04/finding-bugs-in-sqlite-easy-way.html) + +PS. Even when no explicit dictionary is given, afl-fuzz will try to extract +existing syntax tokens in the input corpus by watching the instrumentation +very closely during deterministic byte flips. This works for some types of +parsers and grammars, but isn't nearly as good as the -x mode. + +If a dictionary is really hard to come by, another option is to let AFL run +for a while, and then use the token capture library that comes as a companion +utility with AFL. For that, see [libtokencap/README.tokencap](libtokencap/README.tokencap). + + +## 11) Crash triage +---------------- + +The coverage-based grouping of crashes usually produces a small data set that +can be quickly triaged manually or with a very simple GDB or Valgrind script. +Every crash is also traceable to its parent non-crashing test case in the +queue, making it easier to diagnose faults. + +Having said that, it's important to acknowledge that some fuzzing crashes can be +difficult to quickly evaluate for exploitability without a lot of debugging and +code analysis work. To assist with this task, afl-fuzz supports a very unique +"crash exploration" mode enabled with the -C flag. + +In this mode, the fuzzer takes one or more crashing test cases as the input, +and uses its feedback-driven fuzzing strategies to very quickly enumerate all +code paths that can be reached in the program while keeping it in the +crashing state. + +Mutations that do not result in a crash are rejected; so are any changes that +do not affect the execution path. + +The output is a small corpus of files that can be very rapidly examined to see +what degree of control the attacker has over the faulting address, or whether +it is possible to get past an initial out-of-bounds read - and see what lies +beneath. + +Oh, one more thing: for test case minimization, give afl-tmin a try. The tool +can be operated in a very simple way: + +```shell +$ ./afl-tmin -i test_case -o minimized_result -- /path/to/program [...] +``` + +The tool works with crashing and non-crashing test cases alike. In the crash +mode, it will happily accept instrumented and non-instrumented binaries. In the +non-crashing mode, the minimizer relies on standard AFL instrumentation to make +the file simpler without altering the execution path. + +The minimizer accepts the -m, -t, -f and @@ syntax in a manner compatible with +afl-fuzz. + +Another recent addition to AFL is the afl-analyze tool. It takes an input +file, attempts to sequentially flip bytes, and observes the behavior of the +tested program. It then color-codes the input based on which sections appear to +be critical, and which are not; while not bulletproof, it can often offer quick +insights into complex file formats. More info about its operation can be found +near the end of [docs/technical_details.txt](docs/technical_details.txt). + + +## 12) Going beyond crashes +------------------------ + +Fuzzing is a wonderful and underutilized technique for discovering non-crashing +design and implementation errors, too. Quite a few interesting bugs have been +found by modifying the target programs to call abort() when, say: + + - Two bignum libraries produce different outputs when given the same + fuzzer-generated input, + + - An image library produces different outputs when asked to decode the same + input image several times in a row, + + - A serialization / deserialization library fails to produce stable outputs + when iteratively serializing and deserializing fuzzer-supplied data, + + - A compression library produces an output inconsistent with the input file + when asked to compress and then decompress a particular blob. + +Implementing these or similar sanity checks usually takes very little time; +if you are the maintainer of a particular package, you can make this code +conditional with `#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION` (a flag also +shared with libfuzzer) or `#ifdef __AFL_COMPILER` (this one is just for AFL). + + +## 13) Common-sense risks +---------------------- + +Please keep in mind that, similarly to many other computationally-intensive +tasks, fuzzing may put strain on your hardware and on the OS. In particular: + + - Your CPU will run hot and will need adequate cooling. In most cases, if + cooling is insufficient or stops working properly, CPU speeds will be + automatically throttled. That said, especially when fuzzing on less + suitable hardware (laptops, smartphones, etc), it's not entirely impossible + for something to blow up. + + - Targeted programs may end up erratically grabbing gigabytes of memory or + filling up disk space with junk files. AFL tries to enforce basic memory + limits, but can't prevent each and every possible mishap. The bottom line + is that you shouldn't be fuzzing on systems where the prospect of data loss + is not an acceptable risk. + + - Fuzzing involves billions of reads and writes to the filesystem. On modern + systems, this will be usually heavily cached, resulting in fairly modest + "physical" I/O - but there are many factors that may alter this equation. + It is your responsibility to monitor for potential trouble; with very heavy + I/O, the lifespan of many HDDs and SSDs may be reduced. + + A good way to monitor disk I/O on Linux is the 'iostat' command: + +```shell + $ iostat -d 3 -x -k [...optional disk ID...] +``` + + +## 14) Known limitations & areas for improvement +--------------------------------------------- + +Here are some of the most important caveats for AFL: + + - AFL detects faults by checking for the first spawned process dying due to + a signal (SIGSEGV, SIGABRT, etc). Programs that install custom handlers for + these signals may need to have the relevant code commented out. In the same + vein, faults in child processed spawned by the fuzzed target may evade + detection unless you manually add some code to catch that. + + - As with any other brute-force tool, the fuzzer offers limited coverage if + encryption, checksums, cryptographic signatures, or compression are used to + wholly wrap the actual data format to be tested. + + To work around this, you can comment out the relevant checks (see + experimental/libpng_no_checksum/ for inspiration); if this is not possible, + you can also write a postprocessor, as explained in + experimental/post_library/ (with AFL_POST_LIBRARY) + + - There are some unfortunate trade-offs with ASAN and 64-bit binaries. This + isn't due to any specific fault of afl-fuzz; see [docs/notes_for_asan.txt](docs/notes_for_asan.txt) + for tips. + + - There is no direct support for fuzzing network services, background + daemons, or interactive apps that require UI interaction to work. You may + need to make simple code changes to make them behave in a more traditional + way. Preeny may offer a relatively simple option, too - see: + [https://github.com/zardus/preeny](https://github.com/zardus/preeny) + + Some useful tips for modifying network-based services can be also found at: + [https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop](https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop) + + - AFL doesn't output human-readable coverage data. If you want to monitor + coverage, use afl-cov from Michael Rash: [https://github.com/mrash/afl-cov](https://github.com/mrash/afl-cov) + + - Occasionally, sentient machines rise against their creators. If this + happens to you, please consult [http://lcamtuf.coredump.cx/prep/](http://lcamtuf.coredump.cx/prep/). + +Beyond this, see INSTALL for platform-specific tips. + + +## 15) Special thanks +------------------ + +Many of the improvements to the original afl wouldn't be possible without +feedback, bug reports, or patches from: + +``` + Jann Horn Hanno Boeck + Felix Groebert Jakub Wilk + Richard W. M. Jones Alexander Cherepanov + Tom Ritter Hovik Manucharyan + Sebastian Roschke Eberhard Mattes + Padraig Brady Ben Laurie + @dronesec Luca Barbato + Tobias Ospelt Thomas Jarosch + Martin Carpenter Mudge Zatko + Joe Zbiciak Ryan Govostes + Michael Rash William Robinet + Jonathan Gray Filipe Cabecinhas + Nico Weber Jodie Cunningham + Andrew Griffiths Parker Thompson + Jonathan Neuschfer Tyler Nighswander + Ben Nagy Samir Aguiar + Aidan Thornton Aleksandar Nikolich + Sam Hakim Laszlo Szekeres + David A. Wheeler Turo Lamminen + Andreas Stieger Richard Godbee + Louis Dassy teor2345 + Alex Moneger Dmitry Vyukov + Keegan McAllister Kostya Serebryany + Richo Healey Martijn Bogaard + rc0r Jonathan Foote + Christian Holler Dominique Pelle + Jacek Wielemborek Leo Barnes + Jeremy Barnes Jeff Trull + Guillaume Endignoux ilovezfs + Daniel Godas-Lopez Franjo Ivancic + Austin Seipp Daniel Komaromy + Daniel Binderman Jonathan Metzman + Vegard Nossum Jan Kneschke + Kurt Roeckx Marcel Bohme + Van-Thuan Pham Abhik Roychoudhury + Joshua J. Drake Toby Hutton + Rene Freingruber Sergey Davidoff + Sami Liedes Craig Young + Andrzej Jackowski Daniel Hodson +``` + +Thank you! + + +## 16) Contact +----------- + +Questions? Concerns? Bug reports? The contributors can be reached via +[https://github.com/vanhauser-thc/AFLplusplus](https://github.com/vanhauser-thc/AFLplusplus) + +There is also a mailing list for the afl project; to join, send a mail to +. Or, if you prefer to browse +archives first, try: [https://groups.google.com/group/afl-users](https://groups.google.com/group/afl-users) diff --git a/afl-fuzz.c b/afl-fuzz.c index 8f4e1344..f974268f 100644 --- a/afl-fuzz.c +++ b/afl-fuzz.c @@ -23,7 +23,9 @@ #define AFL_MAIN #define MESSAGES_TO_STDOUT +#ifndef _GNU_SOURCE #define _GNU_SOURCE +#endif #define _FILE_OFFSET_BITS 64 #include "config.h" diff --git a/docs/README b/docs/README deleted file mode 100644 index c2c93f38..00000000 --- a/docs/README +++ /dev/null @@ -1,592 +0,0 @@ -============================ -american fuzzy lop plus plus -============================ - - Originally written by Michal Zalewski - - Repository: https://github.com/vanhauser-thc/AFLplusplus - - afl++ is maintained by Marc Heuse , Heiko Eissfeldt - and Andrea Fioraldi as - there have been no updates to afl since November 2017. - - - Many improvements were made, e.g. more performant llvm_mode, supporting - llvm up to version 8, Qemu 3.1, more speed and crashfixes for Qemu, - laf-intel feature for Qemu (with libcompcov) etc. - - Additionally AFLfast's power schedules by Marcel Boehme from - https://github.com/mboehme/aflfast have been incorporated. - - C. Hollers afl-fuzz Python mutator module and llvm_mode whitelist support - was added too (https://github.com/choller/afl) - - New is the excellent MOpt mutator from - https://github.com/puppet-meteor/MOpt-AFL - - Also newly integrated is instrim, a very effective CFG llvm_mode - instrumentation implementation from https://github.com/csienslab/instrim - - And finally the newest addition is the unicorn_mode which allows fuzzing - of binaries from completely different platforms - provided by domenukk! - The unicorn afl mode is not the stock version but like afl++ contains - various patches from forks that make it better :) - - A more thorough list is available in the PATCHES file. - - So all in all this is the best-of AFL that is currently out there :-) - - - Copyright 2013, 2014, 2015, 2016 Google Inc. All rights reserved. - Released under terms and conditions of Apache License, Version 2.0. - - For new versions and additional information, check out: - https://github.com/vanhauser-thc/AFLplusplus - - To compare notes with other users or get notified about major new features, - send a mail to . - - ** See QuickStartGuide.txt if you don't have time to read this file. ** - - -1) Challenges of guided fuzzing -------------------------------- - -Fuzzing is one of the most powerful and proven strategies for identifying -security issues in real-world software; it is responsible for the vast -majority of remote code execution and privilege escalation bugs found to date -in security-critical software. - -Unfortunately, fuzzing is also relatively shallow; blind, random mutations -make it very unlikely to reach certain code paths in the tested code, leaving -some vulnerabilities firmly outside the reach of this technique. - -There have been numerous attempts to solve this problem. One of the early -approaches - pioneered by Tavis Ormandy - is corpus distillation. The method -relies on coverage signals to select a subset of interesting seeds from a -massive, high-quality corpus of candidate files, and then fuzz them by -traditional means. The approach works exceptionally well, but requires such -a corpus to be readily available. In addition, block coverage measurements -provide only a very simplistic understanding of program state, and are less -useful for guiding the fuzzing effort in the long haul. - -Other, more sophisticated research has focused on techniques such as program -flow analysis ("concolic execution"), symbolic execution, or static analysis. -All these methods are extremely promising in experimental settings, but tend -to suffer from reliability and performance problems in practical uses - and -currently do not offer a viable alternative to "dumb" fuzzing techniques. - - -2) The afl-fuzz approach ------------------------- - -American Fuzzy Lop is a brute-force fuzzer coupled with an exceedingly simple -but rock-solid instrumentation-guided genetic algorithm. It uses a modified -form of edge coverage to effortlessly pick up subtle, local-scale changes to -program control flow. - -Simplifying a bit, the overall algorithm can be summed up as: - - 1) Load user-supplied initial test cases into the queue, - - 2) Take next input file from the queue, - - 3) Attempt to trim the test case to the smallest size that doesn't alter - the measured behavior of the program, - - 4) Repeatedly mutate the file using a balanced and well-researched variety - of traditional fuzzing strategies, - - 5) If any of the generated mutations resulted in a new state transition - recorded by the instrumentation, add mutated output as a new entry in the - queue. - - 6) Go to 2. - -The discovered test cases are also periodically culled to eliminate ones that -have been obsoleted by newer, higher-coverage finds; and undergo several other -instrumentation-driven effort minimization steps. - -As a side result of the fuzzing process, the tool creates a small, -self-contained corpus of interesting test cases. These are extremely useful -for seeding other, labor- or resource-intensive testing regimes - for example, -for stress-testing browsers, office applications, graphics suites, or -closed-source tools. - -The fuzzer is thoroughly tested to deliver out-of-the-box performance far -superior to blind fuzzing or coverage-only tools. - - -3) Instrumenting programs for use with AFL ------------------------------------------- - -PLEASE NOTE: llvm_mode compilation with afl-clang-fast/afl-clang-fast++ -instead of afl-gcc/afl-g++ is much faster and has a few cool features. -See llvm_mode/ - however few code does not compile with llvm. -We support llvm versions 4.0 to 8. - -When source code is available, instrumentation can be injected by a companion -tool that works as a drop-in replacement for gcc or clang in any standard build -process for third-party code. - -The instrumentation has a fairly modest performance impact; in conjunction with -other optimizations implemented by afl-fuzz, most programs can be fuzzed as fast -or even faster than possible with traditional tools. - -The correct way to recompile the target program may vary depending on the -specifics of the build process, but a nearly-universal approach would be: - -$ CC=/path/to/afl/afl-gcc ./configure -$ make clean all - -For C++ programs, you'd would also want to set CXX=/path/to/afl/afl-g++. - -The clang wrappers (afl-clang and afl-clang++) can be used in the same way; -clang users may also opt to leverage a higher-performance instrumentation mode, -as described in llvm_mode/README.llvm. -Clang/LLVM has a much better performance and works from LLVM version 4.0 to 8. -Using the LAF Intel performance enhancements are also recommended, see -llvm_mode/README.laf-intel -Using partial instrumentation is also recommended, see -llvm_mode/README.whitelist - -When testing libraries, you need to find or write a simple program that reads -data from stdin or from a file and passes it to the tested library. In such a -case, it is essential to link this executable against a static version of the -instrumented library, or to make sure that the correct .so file is loaded at -runtime (usually by setting LD_LIBRARY_PATH). The simplest option is a static -build, usually possible via: - -$ CC=/path/to/afl/afl-gcc ./configure --disable-shared - -Setting AFL_HARDEN=1 when calling 'make' will cause the CC wrapper to -automatically enable code hardening options that make it easier to detect -simple memory bugs. Libdislocator, a helper library included with AFL (see -libdislocator/README.dislocator) can help uncover heap corruption issues, too. - -PS. ASAN users are advised to docs/review notes_for_asan.txt file for -important caveats. - - -4) Instrumenting binary-only apps ---------------------------------- - -When source code is *NOT* available, the fuzzer offers experimental support for -fast, on-the-fly instrumentation of black-box binaries. This is accomplished -with a version of QEMU running in the lesser-known "user space emulation" mode. - -QEMU is a project separate from AFL, but you can conveniently build the -feature by doing: - -$ cd qemu_mode -$ ./build_qemu_support.sh - -For additional instructions and caveats, see qemu_mode/README.qemu. - -The mode is approximately 2-5x slower than compile-time instrumentation, is -less conductive to parallelization, and may have some other quirks. - -If [afl-dyninst](https://github.com/vanhauser-thc/afl-dyninst) works for -your binary, then you can use afl-fuzz normally and it will have twice -the speed compared to qemu_mode. - - -5) Power schedules ------------------- - -The power schedules were copied from Marcel Böhme's excellent AFLfast -implementation and expands on the ability to discover new paths and -therefore the coverage. - -The available schedules are: - - - explore (default) - - fast - - coe - - quad - - lin - - exploit - -In parallel mode (-M/-S, several instances with shared queue), we suggest to -run the master using the exploit schedule (-p exploit) and the slaves with a -combination of cut-off-exponential (-p coe), exponential (-p fast; default), -and explore (-p explore) schedules. - -In single mode, using -p fast is usually more beneficial than the default -explore mode. -(We don't want to change the default behaviour of afl, so "fast" has not been -made the default mode). - -More details can be found in the paper published at the 23rd ACM Conference on -Computer and Communications Security (CCS'16): - - https://www.sigsac.org/ccs/CCS2016/accepted-papers/ - -6) Choosing initial test cases ------------------------------- - -To operate correctly, the fuzzer requires one or more starting file that -contains a good example of the input data normally expected by the targeted -application. There are two basic rules: - - - Keep the files small. Under 1 kB is ideal, although not strictly necessary. - For a discussion of why size matters, see perf_tips.txt. - - - Use multiple test cases only if they are functionally different from - each other. There is no point in using fifty different vacation photos - to fuzz an image library. - -You can find many good examples of starting files in the testcases/ subdirectory -that comes with this tool. - -PS. If a large corpus of data is available for screening, you may want to use -the afl-cmin utility to identify a subset of functionally distinct files that -exercise different code paths in the target binary. - - -7) Fuzzing binaries -------------------- - -The fuzzing process itself is carried out by the afl-fuzz utility. This program -requires a read-only directory with initial test cases, a separate place to -store its findings, plus a path to the binary to test. - -For target binaries that accept input directly from stdin, the usual syntax is: - -$ ./afl-fuzz -i testcase_dir -o findings_dir /path/to/program [...params...] - -For programs that take input from a file, use '@@' to mark the location in -the target's command line where the input file name should be placed. The -fuzzer will substitute this for you: - -$ ./afl-fuzz -i testcase_dir -o findings_dir /path/to/program @@ - -You can also use the -f option to have the mutated data written to a specific -file. This is useful if the program expects a particular file extension or so. - -Non-instrumented binaries can be fuzzed in the QEMU mode (add -Q in the command -line) or in a traditional, blind-fuzzer mode (specify -n). - -You can use -t and -m to override the default timeout and memory limit for the -executed process; rare examples of targets that may need these settings touched -include compilers and video decoders. - -Tips for optimizing fuzzing performance are discussed in perf_tips.txt. - -Note that afl-fuzz starts by performing an array of deterministic fuzzing -steps, which can take several days, but tend to produce neat test cases. If you -want quick & dirty results right away - akin to zzuf and other traditional -fuzzers - add the -d option to the command line. - - -8) Interpreting output ----------------------- - -See the status_screen.txt file for information on how to interpret the -displayed stats and monitor the health of the process. Be sure to consult this -file especially if any UI elements are highlighted in red. - -The fuzzing process will continue until you press Ctrl-C. At minimum, you want -to allow the fuzzer to complete one queue cycle, which may take anywhere from a -couple of hours to a week or so. - -There are three subdirectories created within the output directory and updated -in real time: - - - queue/ - test cases for every distinctive execution path, plus all the - starting files given by the user. This is the synthesized corpus - mentioned in section 2. - - Before using this corpus for any other purposes, you can shrink - it to a smaller size using the afl-cmin tool. The tool will find - a smaller subset of files offering equivalent edge coverage. - - - crashes/ - unique test cases that cause the tested program to receive a - fatal signal (e.g., SIGSEGV, SIGILL, SIGABRT). The entries are - grouped by the received signal. - - - hangs/ - unique test cases that cause the tested program to time out. The - default time limit before something is classified as a hang is - the larger of 1 second and the value of the -t parameter. - The value can be fine-tuned by setting AFL_HANG_TMOUT, but this - is rarely necessary. - -Crashes and hangs are considered "unique" if the associated execution paths -involve any state transitions not seen in previously-recorded faults. If a -single bug can be reached in multiple ways, there will be some count inflation -early in the process, but this should quickly taper off. - -The file names for crashes and hangs are correlated with parent, non-faulting -queue entries. This should help with debugging. - -When you can't reproduce a crash found by afl-fuzz, the most likely cause is -that you are not setting the same memory limit as used by the tool. Try: - -$ LIMIT_MB=50 -$ ( ulimit -Sv $[LIMIT_MB << 10]; /path/to/tested_binary ... ) - -Change LIMIT_MB to match the -m parameter passed to afl-fuzz. On OpenBSD, -also change -Sv to -Sd. - -Any existing output directory can be also used to resume aborted jobs; try: - -$ ./afl-fuzz -i- -o existing_output_dir [...etc...] - -If you have gnuplot installed, you can also generate some pretty graphs for any -active fuzzing task using afl-plot. For an example of how this looks like, -see http://lcamtuf.coredump.cx/afl/plot/. - - -9) Parallelized fuzzing ------------------------ - -Every instance of afl-fuzz takes up roughly one core. This means that on -multi-core systems, parallelization is necessary to fully utilize the hardware. -For tips on how to fuzz a common target on multiple cores or multiple networked -machines, please refer to parallel_fuzzing.txt. - -The parallel fuzzing mode also offers a simple way for interfacing AFL to other -fuzzers, to symbolic or concolic execution engines, and so forth; again, see the -last section of parallel_fuzzing.txt for tips. - - -10) Fuzzer dictionaries ----------------------- - -By default, afl-fuzz mutation engine is optimized for compact data formats - -say, images, multimedia, compressed data, regular expression syntax, or shell -scripts. It is somewhat less suited for languages with particularly verbose and -redundant verbiage - notably including HTML, SQL, or JavaScript. - -To avoid the hassle of building syntax-aware tools, afl-fuzz provides a way to -seed the fuzzing process with an optional dictionary of language keywords, -magic headers, or other special tokens associated with the targeted data type -- and use that to reconstruct the underlying grammar on the go: - - http://lcamtuf.blogspot.com/2015/01/afl-fuzz-making-up-grammar-with.html - -To use this feature, you first need to create a dictionary in one of the two -formats discussed in dictionaries/README.dictionaries; and then point the fuzzer -to it via the -x option in the command line. - -(Several common dictionaries are already provided in that subdirectory, too.) - -There is no way to provide more structured descriptions of the underlying -syntax, but the fuzzer will likely figure out some of this based on the -instrumentation feedback alone. This actually works in practice, say: - - http://lcamtuf.blogspot.com/2015/04/finding-bugs-in-sqlite-easy-way.html - -PS. Even when no explicit dictionary is given, afl-fuzz will try to extract -existing syntax tokens in the input corpus by watching the instrumentation -very closely during deterministic byte flips. This works for some types of -parsers and grammars, but isn't nearly as good as the -x mode. - -If a dictionary is really hard to come by, another option is to let AFL run -for a while, and then use the token capture library that comes as a companion -utility with AFL. For that, see libtokencap/README.tokencap. - - -11) Crash triage ----------------- - -The coverage-based grouping of crashes usually produces a small data set that -can be quickly triaged manually or with a very simple GDB or Valgrind script. -Every crash is also traceable to its parent non-crashing test case in the -queue, making it easier to diagnose faults. - -Having said that, it's important to acknowledge that some fuzzing crashes can be -difficult to quickly evaluate for exploitability without a lot of debugging and -code analysis work. To assist with this task, afl-fuzz supports a very unique -"crash exploration" mode enabled with the -C flag. - -In this mode, the fuzzer takes one or more crashing test cases as the input, -and uses its feedback-driven fuzzing strategies to very quickly enumerate all -code paths that can be reached in the program while keeping it in the -crashing state. - -Mutations that do not result in a crash are rejected; so are any changes that -do not affect the execution path. - -The output is a small corpus of files that can be very rapidly examined to see -what degree of control the attacker has over the faulting address, or whether -it is possible to get past an initial out-of-bounds read - and see what lies -beneath. - -Oh, one more thing: for test case minimization, give afl-tmin a try. The tool -can be operated in a very simple way: - -$ ./afl-tmin -i test_case -o minimized_result -- /path/to/program [...] - -The tool works with crashing and non-crashing test cases alike. In the crash -mode, it will happily accept instrumented and non-instrumented binaries. In the -non-crashing mode, the minimizer relies on standard AFL instrumentation to make -the file simpler without altering the execution path. - -The minimizer accepts the -m, -t, -f and @@ syntax in a manner compatible with -afl-fuzz. - -Another recent addition to AFL is the afl-analyze tool. It takes an input -file, attempts to sequentially flip bytes, and observes the behavior of the -tested program. It then color-codes the input based on which sections appear to -be critical, and which are not; while not bulletproof, it can often offer quick -insights into complex file formats. More info about its operation can be found -near the end of technical_details.txt. - - -12) Going beyond crashes ------------------------- - -Fuzzing is a wonderful and underutilized technique for discovering non-crashing -design and implementation errors, too. Quite a few interesting bugs have been -found by modifying the target programs to call abort() when, say: - - - Two bignum libraries produce different outputs when given the same - fuzzer-generated input, - - - An image library produces different outputs when asked to decode the same - input image several times in a row, - - - A serialization / deserialization library fails to produce stable outputs - when iteratively serializing and deserializing fuzzer-supplied data, - - - A compression library produces an output inconsistent with the input file - when asked to compress and then decompress a particular blob. - -Implementing these or similar sanity checks usually takes very little time; -if you are the maintainer of a particular package, you can make this code -conditional with #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION (a flag also -shared with libfuzzer) or #ifdef __AFL_COMPILER (this one is just for AFL). - - -13) Common-sense risks ----------------------- - -Please keep in mind that, similarly to many other computationally-intensive -tasks, fuzzing may put strain on your hardware and on the OS. In particular: - - - Your CPU will run hot and will need adequate cooling. In most cases, if - cooling is insufficient or stops working properly, CPU speeds will be - automatically throttled. That said, especially when fuzzing on less - suitable hardware (laptops, smartphones, etc), it's not entirely impossible - for something to blow up. - - - Targeted programs may end up erratically grabbing gigabytes of memory or - filling up disk space with junk files. AFL tries to enforce basic memory - limits, but can't prevent each and every possible mishap. The bottom line - is that you shouldn't be fuzzing on systems where the prospect of data loss - is not an acceptable risk. - - - Fuzzing involves billions of reads and writes to the filesystem. On modern - systems, this will be usually heavily cached, resulting in fairly modest - "physical" I/O - but there are many factors that may alter this equation. - It is your responsibility to monitor for potential trouble; with very heavy - I/O, the lifespan of many HDDs and SSDs may be reduced. - - A good way to monitor disk I/O on Linux is the 'iostat' command: - - $ iostat -d 3 -x -k [...optional disk ID...] - - -14) Known limitations & areas for improvement ---------------------------------------------- - -Here are some of the most important caveats for AFL: - - - AFL detects faults by checking for the first spawned process dying due to - a signal (SIGSEGV, SIGABRT, etc). Programs that install custom handlers for - these signals may need to have the relevant code commented out. In the same - vein, faults in child processed spawned by the fuzzed target may evade - detection unless you manually add some code to catch that. - - - As with any other brute-force tool, the fuzzer offers limited coverage if - encryption, checksums, cryptographic signatures, or compression are used to - wholly wrap the actual data format to be tested. - - To work around this, you can comment out the relevant checks (see - experimental/libpng_no_checksum/ for inspiration); if this is not possible, - you can also write a postprocessor, as explained in - experimental/post_library/ (with AFL_POST_LIBRARY) - - - There are some unfortunate trade-offs with ASAN and 64-bit binaries. This - isn't due to any specific fault of afl-fuzz; see notes_for_asan.txt for - tips. - - - There is no direct support for fuzzing network services, background - daemons, or interactive apps that require UI interaction to work. You may - need to make simple code changes to make them behave in a more traditional - way. Preeny may offer a relatively simple option, too - see: - https://github.com/zardus/preeny - - Some useful tips for modifying network-based services can be also found at: - https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop - - - AFL doesn't output human-readable coverage data. If you want to monitor - coverage, use afl-cov from Michael Rash: https://github.com/mrash/afl-cov - - - Occasionally, sentient machines rise against their creators. If this - happens to you, please consult http://lcamtuf.coredump.cx/prep/. - -Beyond this, see INSTALL for platform-specific tips. - - -15) Special thanks ------------------- - -Many of the improvements to afl-fuzz wouldn't be possible without feedback, -bug reports, or patches from: - - Jann Horn Hanno Boeck - Felix Groebert Jakub Wilk - Richard W. M. Jones Alexander Cherepanov - Tom Ritter Hovik Manucharyan - Sebastian Roschke Eberhard Mattes - Padraig Brady Ben Laurie - @dronesec Luca Barbato - Tobias Ospelt Thomas Jarosch - Martin Carpenter Mudge Zatko - Joe Zbiciak Ryan Govostes - Michael Rash William Robinet - Jonathan Gray Filipe Cabecinhas - Nico Weber Jodie Cunningham - Andrew Griffiths Parker Thompson - Jonathan Neuschfer Tyler Nighswander - Ben Nagy Samir Aguiar - Aidan Thornton Aleksandar Nikolich - Sam Hakim Laszlo Szekeres - David A. Wheeler Turo Lamminen - Andreas Stieger Richard Godbee - Louis Dassy teor2345 - Alex Moneger Dmitry Vyukov - Keegan McAllister Kostya Serebryany - Richo Healey Martijn Bogaard - rc0r Jonathan Foote - Christian Holler Dominique Pelle - Jacek Wielemborek Leo Barnes - Jeremy Barnes Jeff Trull - Guillaume Endignoux ilovezfs - Daniel Godas-Lopez Franjo Ivancic - Austin Seipp Daniel Komaromy - Daniel Binderman Jonathan Metzman - Vegard Nossum Jan Kneschke - Kurt Roeckx Marcel Bohme - Van-Thuan Pham Abhik Roychoudhury - Joshua J. Drake Toby Hutton - Rene Freingruber Sergey Davidoff - Sami Liedes Craig Young - Andrzej Jackowski Daniel Hodson - -Thank you! - - -16) Contact ------------ - -Questions? Concerns? Bug reports? The contributors can be reached via -https://github.com/vanhauser-thc/AFLplusplus - -There is also a mailing list for the afl project; to join, send a mail to -. Or, if you prefer to browse -archives first, try: - - https://groups.google.com/group/afl-users diff --git a/llvm_mode/README.llvm b/llvm_mode/README.llvm index 00528a46..a0c40211 100644 --- a/llvm_mode/README.llvm +++ b/llvm_mode/README.llvm @@ -205,10 +205,8 @@ post-process the assembly or install any compiler plugins. See: http://clang.llvm.org/docs/SanitizerCoverage.html#tracing-pcs-with-guards -As of this writing, the feature is only available on SVN trunk, and is yet to -make it to an official release of LLVM. Nevertheless, if you have a -sufficiently recent compiler and want to give it a try, build afl-clang-fast -this way: +If you have a sufficiently recent compiler and want to give it a try, build +afl-clang-fast this way: AFL_TRACE_PC=1 make clean all diff --git a/types.h b/types.h index 3497bb2b..7606d4ed 100644 --- a/types.h +++ b/types.h @@ -78,7 +78,7 @@ typedef int64_t s64; #define STRINGIFY(x) STRINGIFY_INTERNAL(x) #define MEM_BARRIER() \ - asm volatile("" ::: "memory") + __asm__ volatile("" ::: "memory") #if __GNUC__ < 6 #define likely(_x) (_x) -- cgit 1.4.1