From b0492ba642e7afb9a1269c27a264d11e5c2e6ec5 Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Tue, 14 Jan 2020 13:58:36 +0100 Subject: Fixed description of unicorn harness.c --- unicorn_mode/samples/c/harness.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/unicorn_mode/samples/c/harness.c b/unicorn_mode/samples/c/harness.c index cc81ba7f..a987b8e1 100644 --- a/unicorn_mode/samples/c/harness.c +++ b/unicorn_mode/samples/c/harness.c @@ -1,11 +1,10 @@ /* Simple test harness for AFL++'s unicornafl c mode. - This loads the simple_target.bin binary (precompiled as MIPS code) into + This loads the simple_target_x86_64 binary into Unicorn's memory map for emulation, places the specified input into - simple_target's buffer (hardcoded to be at 0x300000), and executes 'main()'. - If any crashes occur during emulation, this script throws a matching signal - to tell AFL that a crash occurred. + argv[1], sets up argv, and argc and executes 'main()'. + If run inside AFL, afl_fuzz automatically does the "right thing" Run under AFL as follows: -- cgit 1.4.1 From d0ea8f84334a36133c289c711c233d1f5a1a8045 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Tue, 14 Jan 2020 14:45:55 +0100 Subject: todo update --- TODO | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO b/TODO index 8a8fdd41..259fbd0a 100644 --- a/TODO +++ b/TODO @@ -2,6 +2,9 @@ Roadmap 2.61+: ============== +afl-fuzz: + - sync_fuzzers(): only masters sync from all, slaves only sync from master + gcc_plugin: - laf-intel - better instrumentation -- cgit 1.4.1 From d1d5e7c02a4b17f42b6334f258899d6543c9142c Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 15 Jan 2020 10:10:25 +0100 Subject: blacklist function support for llvm_mode --- TODO | 3 +++ docs/ChangeLog | 3 ++- llvm_mode/LLVMInsTrim.so.cc | 24 ++++++++++++++++++++++++ llvm_mode/afl-clang-fast.c | 3 ++- llvm_mode/afl-llvm-pass.so.cc | 27 ++++++++++++++++++++++++--- src/afl-fuzz-stats.c | 14 +++++++------- 6 files changed, 62 insertions(+), 12 deletions(-) diff --git a/TODO b/TODO index 259fbd0a..e935eafa 100644 --- a/TODO +++ b/TODO @@ -2,6 +2,9 @@ Roadmap 2.61+: ============== +Makefile: + - -march=native -Ofast -flto=full + afl-fuzz: - sync_fuzzers(): only masters sync from all, slaves only sync from master diff --git a/docs/ChangeLog b/docs/ChangeLog index f034d251..a94a2038 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -20,7 +20,8 @@ Version ++2.60d (develop): - afl-fuzz: - now prints the real python version support compiled in - afl-clang-fast now shows in the help output for which llvm version it - was compiled for. + was compiled for + - added blacklisted function check in llvm_mode - added fix from Debian project to compile libdislocator and libtokencap diff --git a/llvm_mode/LLVMInsTrim.so.cc b/llvm_mode/LLVMInsTrim.so.cc index 552cf580..11451b43 100644 --- a/llvm_mode/LLVMInsTrim.so.cc +++ b/llvm_mode/LLVMInsTrim.so.cc @@ -94,6 +94,28 @@ struct InsTrim : public ModulePass { } + // ripped from aflgo + static bool isBlacklisted(const Function *F) { + + static const SmallVector Blacklist = { + + "asan.", + "llvm.", + "sancov.", + "__ubsan_handle_", + + }; + + for (auto const &BlacklistFunc : Blacklist) { + + if (F->getName().startswith(BlacklistFunc)) { return true; } + + } + + return false; + + } + bool runOnModule(Module &M) override { char be_quiet = 0; @@ -240,6 +262,8 @@ struct InsTrim : public ModulePass { } + if (isBlacklisted(&F)) continue; + std::unordered_set MS; if (!MarkSetOpt) { diff --git a/llvm_mode/afl-clang-fast.c b/llvm_mode/afl-clang-fast.c index 2b359cdf..b322b762 100644 --- a/llvm_mode/afl-clang-fast.c +++ b/llvm_mode/afl-clang-fast.c @@ -444,7 +444,8 @@ int main(int argc, char** argv) { "You can specify custom next-stage toolchain via AFL_CC and AFL_CXX. " "Setting\n" "AFL_HARDEN enables hardening optimizations in the compiled code.\n\n" - "afl-clang-fast was built for llvm %s with the llvm binary path of \"%s\".\n\n", + "afl-clang-fast was built for llvm %s with the llvm binary path of " + "\"%s\".\n\n", BIN_PATH, BIN_PATH, LLVM_VERSION, LLVM_BINDIR); exit(1); diff --git a/llvm_mode/afl-llvm-pass.so.cc b/llvm_mode/afl-llvm-pass.so.cc index 0b3db4ed..15cc6127 100644 --- a/llvm_mode/afl-llvm-pass.so.cc +++ b/llvm_mode/afl-llvm-pass.so.cc @@ -75,6 +75,28 @@ class AFLCoverage : public ModulePass { } + // ripped from aflgo + static bool isBlacklisted(const Function *F) { + + static const SmallVector Blacklist = { + + "asan.", + "llvm.", + "sancov.", + "__ubsan_handle_", + + }; + + for (auto const &BlacklistFunc : Blacklist) { + + if (F->getName().startswith(BlacklistFunc)) { return true; } + + } + + return false; + + } + bool runOnModule(Module &M) override; // StringRef getPassName() const override { @@ -156,13 +178,11 @@ bool AFLCoverage::runOnModule(Module &M) { /* Instrument all the things! */ - const char *IntrinsicPrefix = "llvm."; int inst_blocks = 0; for (auto &F : M) { - auto Fname = F.getName(); - if (Fname.startswith(IntrinsicPrefix)) continue; + if (isBlacklisted(&F)) continue; for (auto &BB : F) { @@ -377,6 +397,7 @@ bool AFLCoverage::runOnModule(Module &M) { inst_blocks++; } + } /* Say something nice. */ diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 18f32ae8..7679403b 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -334,9 +334,9 @@ void show_stats(void) { /* Lord, forgive me this. */ - SAYF(SET_G1 bSTG bLT bH bSTOP cCYA + SAYF(SET_G1 bSTG bLT bH bSTOP cCYA " process timing " bSTG bH30 bH5 bH bHB bH bSTOP cCYA - " overall results " bSTG bH2 bH2 bRT "\n"); + " overall results " bSTG bH2 bH2 bRT "\n"); if (dumb_mode) { @@ -413,9 +413,9 @@ void show_stats(void) { " uniq hangs : " cRST "%-6s" bSTG bV "\n", DTD(cur_ms, last_hang_time), tmp); - SAYF(bVR bH bSTOP cCYA + SAYF(bVR bH bSTOP cCYA " cycle progress " bSTG bH10 bH5 bH2 bH2 bHB bH bSTOP cCYA - " map coverage " bSTG bH bHT bH20 bH2 bVL "\n"); + " map coverage " bSTG bH bHT bH20 bH2 bVL "\n"); /* This gets funny because we want to print several variable-length variables together, but then cram them into a fixed-width field - so we need to @@ -443,9 +443,9 @@ void show_stats(void) { SAYF(bSTOP " count coverage : " cRST "%-21s" bSTG bV "\n", tmp); - SAYF(bVR bH bSTOP cCYA + SAYF(bVR bH bSTOP cCYA " stage progress " bSTG bH10 bH5 bH2 bH2 bX bH bSTOP cCYA - " findings in depth " bSTG bH10 bH5 bH2 bH2 bVL "\n"); + " findings in depth " bSTG bH10 bH5 bH2 bH2 bVL "\n"); sprintf(tmp, "%s (%0.02f%%)", DI(queued_favored), ((double)queued_favored) * 100 / queued_paths); @@ -514,7 +514,7 @@ void show_stats(void) { /* Aaaalmost there... hold on! */ - SAYF(bVR bH cCYA bSTOP + SAYF(bVR bH cCYA bSTOP " fuzzing strategy yields " bSTG bH10 bHT bH10 bH5 bHB bH bSTOP cCYA " path geometry " bSTG bH5 bH2 bVL "\n"); -- cgit 1.4.1 From e673dc6dbebedc1374cd4fb9201cf784408c00f3 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 15 Jan 2020 10:23:28 +0100 Subject: stronger afl-fuzz performance compilation --- Makefile | 22 +++++++++++++++++++--- docs/ChangeLog | 1 + src/afl-fuzz-bitmap.c | 2 +- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0c0b73c5..57d4c8c7 100644 --- a/Makefile +++ b/Makefile @@ -32,9 +32,25 @@ PROGS = afl-gcc afl-fuzz afl-showmap afl-tmin afl-gotcpu afl-analyze SH_PROGS = afl-plot afl-cmin afl-whatsup afl-system-config MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8) -CFLAGS ?= -O3 -funroll-loops -CFLAGS += -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign -I include/ \ - -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ +ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + CFLAGS_PERFORMANCE += -march=native +endif + +ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto=full -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + CFLAGS_PERFORMANCE += -flto=full +else + ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto=thin -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + CFLAGS_PERFORMANCE += -flto=thin + else + ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + CFLAGS_PERFORMANCE += -flto + endif + endif +endif + +CFLAGS ?= -funroll-loops -Ofast $(CFLAGS_PERFORMANCE) +CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ + -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ -DDOC_PATH=\"$(DOC_PATH)\" -Wno-unused-function AFL_FUZZ_FILES = $(wildcard src/afl-fuzz*.c) diff --git a/docs/ChangeLog b/docs/ChangeLog index a94a2038..594f75e9 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -19,6 +19,7 @@ Version ++2.60d (develop): - afl-fuzz: - now prints the real python version support compiled in + - set stronger performance compile options and little tweaks - afl-clang-fast now shows in the help output for which llvm version it was compiled for - added blacklisted function check in llvm_mode diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 0f611cdf..515a7a79 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -524,7 +524,7 @@ u8 save_if_interesting(char** argv, void* mem, u32 len, u8 fault) { struct queue_entry* q = queue; while (q) { - if (q->exec_cksum == cksum) q->n_fuzz = q->n_fuzz + 1; + if (q->exec_cksum == cksum) { q->n_fuzz = q->n_fuzz + 1; break ; } q = q->next; -- cgit 1.4.1 From d5dff8960cf72f097a5d8e178ea28dfe9a72c0b3 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 15 Jan 2020 12:00:25 +0100 Subject: O3 is faster than Ofast ... duh ... --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 57d4c8c7..893e51d1 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ else endif endif -CFLAGS ?= -funroll-loops -Ofast $(CFLAGS_PERFORMANCE) +CFLAGS ?= -O3 -funroll-loops $(CFLAGS_PERFORMANCE) CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ -DDOC_PATH=\"$(DOC_PATH)\" -Wno-unused-function -- cgit 1.4.1 From b5c19a58f6a5badfea74f267a3c032cbd89b4bea Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 15 Jan 2020 15:54:54 +0100 Subject: fix for cc=clang and libradamsa optimization --- Makefile | 26 +++++++++++--------------- src/third_party/libradamsa/Makefile | 8 +++++--- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 893e51d1..56b77999 100644 --- a/Makefile +++ b/Makefile @@ -32,23 +32,19 @@ PROGS = afl-gcc afl-fuzz afl-showmap afl-tmin afl-gotcpu afl-analyze SH_PROGS = afl-plot afl-cmin afl-whatsup afl-system-config MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8) -ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" - CFLAGS_PERFORMANCE += -march=native -endif - ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto=full -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" - CFLAGS_PERFORMANCE += -flto=full + CFLAGS_FLTO ?= -flto=full else ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto=thin -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" - CFLAGS_PERFORMANCE += -flto=thin + CFLAGS_FLTO ?= -flto=thin else ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" - CFLAGS_PERFORMANCE += -flto + CFLAGS_FLTO ?= -flto endif endif endif -CFLAGS ?= -O3 -funroll-loops $(CFLAGS_PERFORMANCE) +CFLAGS ?= -O3 -funroll-loops -march=native CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ -DDOC_PATH=\"$(DOC_PATH)\" -Wno-unused-function @@ -245,13 +241,13 @@ afl-as: src/afl-as.c include/afl-as.h $(COMM_HDR) | test_x86 ln -sf afl-as as src/afl-common.o : src/afl-common.c include/common.h - $(CC) $(CFLAGS) -c src/afl-common.c -o src/afl-common.o + $(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-common.c -o src/afl-common.o src/afl-forkserver.o : src/afl-forkserver.c include/forkserver.h - $(CC) $(CFLAGS) -c src/afl-forkserver.c -o src/afl-forkserver.o + $(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-forkserver.c -o src/afl-forkserver.o src/afl-sharedmem.o : src/afl-sharedmem.c include/sharedmem.h - $(CC) $(CFLAGS) -c src/afl-sharedmem.c -o src/afl-sharedmem.o + $(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-sharedmem.c -o src/afl-sharedmem.o radamsa: src/third_party/libradamsa/libradamsa.so cp src/third_party/libradamsa/libradamsa.so . @@ -260,16 +256,16 @@ src/third_party/libradamsa/libradamsa.so: src/third_party/libradamsa/libradamsa. $(MAKE) -C src/third_party/libradamsa/ afl-fuzz: include/afl-fuzz.h $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o -o $@ $(PYFLAGS) $(LDFLAGS) + $(CC) $(CFLAGS) $(CFLAGS_FLTO) $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o -o $@ $(PYFLAGS) $(LDFLAGS) afl-showmap: src/afl-showmap.c src/afl-common.o src/afl-sharedmem.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) src/$@.c src/afl-common.o src/afl-sharedmem.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(CFLAGS_FLTO) src/$@.c src/afl-common.o src/afl-sharedmem.o -o $@ $(LDFLAGS) afl-tmin: src/afl-tmin.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(CFLAGS_FLTO) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o -o $@ $(LDFLAGS) afl-analyze: src/afl-analyze.c src/afl-common.o src/afl-sharedmem.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) src/$@.c src/afl-common.o src/afl-sharedmem.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(CFLAGS_FLTO) src/$@.c src/afl-common.o src/afl-sharedmem.o -o $@ $(LDFLAGS) afl-gotcpu: src/afl-gotcpu.c $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) src/$@.c -o $@ $(LDFLAGS) diff --git a/src/third_party/libradamsa/Makefile b/src/third_party/libradamsa/Makefile index 0f56fa18..b061f8e1 100644 --- a/src/third_party/libradamsa/Makefile +++ b/src/third_party/libradamsa/Makefile @@ -2,17 +2,19 @@ CUR_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) all: libradamsa.so +CFLAGS ?= -O3 -funroll-loops -march=native $(CFLAGS_FLTO) + libradamsa.so: libradamsa.a - $(CC) -O3 -shared libradamsa.a -o libradamsa.so + $(CC) $(CFLAGS) -shared libradamsa.a -o libradamsa.so libradamsa.a: libradamsa.c radamsa.h @echo " ***************************************************************" @echo " * Compiling libradamsa, wait some minutes (~3 on modern CPUs) *" @echo " ***************************************************************" - $(CC) -fPIC -O3 -I $(CUR_DIR) -o libradamsa.a -c libradamsa.c + $(CC) -fPIC $(CFLAGS) -I $(CUR_DIR) -o libradamsa.a -c libradamsa.c test: libradamsa.a libradamsa-test.c - cc -O3 -I $(CUR_DIR) -o libradamsa-test libradamsa-test.c libradamsa.a + cc $(CLFAGS) -I $(CUR_DIR) -o libradamsa-test libradamsa-test.c libradamsa.a ./libradamsa-test libradamsa-test.c | grep "library test passed" rm /tmp/libradamsa-*.fuzz -- cgit 1.4.1 From bd58094dbc87463680a54d99ffcff7ae2a591353 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 16 Jan 2020 13:38:04 +0100 Subject: fix libradamsa Makefile typo --- src/third_party/libradamsa/Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/third_party/libradamsa/Makefile b/src/third_party/libradamsa/Makefile index b061f8e1..c5a78ead 100644 --- a/src/third_party/libradamsa/Makefile +++ b/src/third_party/libradamsa/Makefile @@ -2,7 +2,11 @@ CUR_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) all: libradamsa.so -CFLAGS ?= -O3 -funroll-loops -march=native $(CFLAGS_FLTO) +# These can be overriden: +CFLAGS ?= -march=native $(CFLAGS_FLTO) + +# These are required: (otherwise radamsa gets very very slooooow) +CFLAGS += -O3 -funroll-loops libradamsa.so: libradamsa.a $(CC) $(CFLAGS) -shared libradamsa.a -o libradamsa.so @@ -14,7 +18,7 @@ libradamsa.a: libradamsa.c radamsa.h $(CC) -fPIC $(CFLAGS) -I $(CUR_DIR) -o libradamsa.a -c libradamsa.c test: libradamsa.a libradamsa-test.c - cc $(CLFAGS) -I $(CUR_DIR) -o libradamsa-test libradamsa-test.c libradamsa.a + $(CC) $(CFLAGS) -I $(CUR_DIR) -o libradamsa-test libradamsa-test.c libradamsa.a ./libradamsa-test libradamsa-test.c | grep "library test passed" rm /tmp/libradamsa-*.fuzz -- cgit 1.4.1 From 3671d7eb8a9dddc5be29ae5e3d4ba7669a898e53 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Fri, 17 Jan 2020 16:00:11 +0100 Subject: fixes for gcc 5.5 (does not understand -march=native) on NetBSD --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 56b77999..d208f509 100644 --- a/Makefile +++ b/Makefile @@ -44,11 +44,15 @@ else endif endif -CFLAGS ?= -O3 -funroll-loops -march=native +CFLAGS ?= -O3 -funroll-loops CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ -DDOC_PATH=\"$(DOC_PATH)\" -Wno-unused-function +ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + CFLAGS += -march=native +endif + AFL_FUZZ_FILES = $(wildcard src/afl-fuzz*.c) ifneq "($filter %3.7m, $(shell python3.7m-config --includes 2>/dev/null)" "" @@ -253,7 +257,7 @@ radamsa: src/third_party/libradamsa/libradamsa.so cp src/third_party/libradamsa/libradamsa.so . src/third_party/libradamsa/libradamsa.so: src/third_party/libradamsa/libradamsa.c src/third_party/libradamsa/radamsa.h - $(MAKE) -C src/third_party/libradamsa/ + $(MAKE) -C src/third_party/libradamsa/ CFLAGS="$(CFLAGS)" afl-fuzz: include/afl-fuzz.h $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) $(CFLAGS_FLTO) $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o -o $@ $(PYFLAGS) $(LDFLAGS) -- cgit 1.4.1 From 7c0704b30c85b5fdaba582bfb4299623c255f498 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Fri, 17 Jan 2020 16:01:44 +0100 Subject: use default memory limit for afl-tmin (again) in test.sh, fix it in afl-tmin --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index c763e91f..8f40773c 100755 --- a/test/test.sh +++ b/test/test.sh @@ -158,7 +158,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { CODE=1 ;; esac - ../afl-tmin -m200 -i in/in2 -o in2/in2 -- ./test-instr.plain > /dev/null 2>&1 + ../afl-tmin -i in/in2 -o in2/in2 -- ./test-instr.plain > /dev/null 2>&1 SIZE=`ls -l in2/in2 2> /dev/null | awk '{print$5}'` test "$SIZE" = 1 && $ECHO "$GREEN[+] afl-tmin correctly minimized the testcase" test "$SIZE" = 1 || { -- cgit 1.4.1 From cb23fe2aba8487464755abe2debfb78980cc5eba Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Fri, 17 Jan 2020 16:06:43 +0100 Subject: increased default memory limit (200megs) on NetBSD (due to kernel bug), fixes the afl-tmin test --- include/config.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/include/config.h b/include/config.h index c5139dbd..69380282 100644 --- a/include/config.h +++ b/include/config.h @@ -61,12 +61,15 @@ /* Default memory limit for child process (MB): */ -#ifndef WORD_SIZE_64 -#define MEM_LIMIT 25 +#ifndef __NetBSD__ +# ifndef WORD_SIZE_64 +# define MEM_LIMIT 25 +# else +# define MEM_LIMIT 50 +# endif /* ^!WORD_SIZE_64 */ #else -#define MEM_LIMIT 50 -#endif /* ^!WORD_SIZE_64 */ - +# define MEM_LIMIT 200 +#endif /* Default memory limit when running in QEMU mode (MB): */ #define MEM_LIMIT_QEMU 200 -- cgit 1.4.1 From 800d43b84666461cb682be20fdc5df48c13c252b Mon Sep 17 00:00:00 2001 From: van Hauser Date: Fri, 17 Jan 2020 16:17:08 +0100 Subject: small change for march=native --- Makefile | 10 +++++----- docs/ChangeLog | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index d208f509..703ed673 100644 --- a/Makefile +++ b/Makefile @@ -44,15 +44,15 @@ else endif endif -CFLAGS ?= -O3 -funroll-loops +ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + CFLAGS_OPT = -march=native +endif + +CFLAGS ?= -O3 -funroll-loops $(CFLAGS_OPT) CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ -DDOC_PATH=\"$(DOC_PATH)\" -Wno-unused-function -ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" - CFLAGS += -march=native -endif - AFL_FUZZ_FILES = $(wildcard src/afl-fuzz*.c) ifneq "($filter %3.7m, $(shell python3.7m-config --includes 2>/dev/null)" "" diff --git a/docs/ChangeLog b/docs/ChangeLog index 594f75e9..5347d244 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -17,6 +17,7 @@ sending a mail to . Version ++2.60d (develop): -------------------------- + - use -march=native if available - afl-fuzz: - now prints the real python version support compiled in - set stronger performance compile options and little tweaks -- cgit 1.4.1 From 1ac31361ca61f71b6a419064de5063aef80203e5 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Fri, 17 Jan 2020 20:41:30 +0100 Subject: as suggested, added a comment, why NetBSD needs a higher memory limit --- include/config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/config.h b/include/config.h index 69380282..4eac82e0 100644 --- a/include/config.h +++ b/include/config.h @@ -67,7 +67,7 @@ # else # define MEM_LIMIT 50 # endif /* ^!WORD_SIZE_64 */ -#else +#else /* NetBSD's kernel needs more space for stack, see discussion for issue #165 */ # define MEM_LIMIT 200 #endif /* Default memory limit when running in QEMU mode (MB): */ -- cgit 1.4.1 From 858b5da24e3b060e2ebf6ab48ded22fbdd7d3ceb Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sat, 18 Jan 2020 14:28:31 +0000 Subject: libdislocator: reallocarray API introduction --- libdislocator/libdislocator.so.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/libdislocator/libdislocator.so.c b/libdislocator/libdislocator.so.c index 20649470..b9ba8967 100644 --- a/libdislocator/libdislocator.so.c +++ b/libdislocator/libdislocator.so.c @@ -397,6 +397,28 @@ void* aligned_alloc(size_t align, size_t len) { } +/* specific BSD api mainly checking possible overflow for the size */ + +void* reallocarray(void* ptr, size_t elem_len, size_t elem_cnt) { + + const size_t elem_lim = 1UL << (sizeof(size_t) * 4); + const size_t elem_tot = elem_len * elem_cnt; + void* ret = NULL; + + if ((elem_len >= elem_lim || elem_cnt >= elem_lim) && elem_len > 0 && + elem_cnt > (SIZE_MAX / elem_len)) { + + DEBUGF("reallocarray size overflow (%zu)", elem_tot); + + } else { + + ret = realloc(ptr, elem_tot); + + } + + return ret; +} + __attribute__((constructor)) void __dislocator_init(void) { u8* tmp = (u8*)getenv("AFL_LD_LIMIT_MB"); -- cgit 1.4.1 From 6b0950b03d8a9fd0c21b4be71fd4a4bd6ab68547 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sat, 18 Jan 2020 16:13:57 +0100 Subject: fix some syntax errors regarding $(filter ...) --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 703ed673..df5ad048 100644 --- a/Makefile +++ b/Makefile @@ -55,17 +55,17 @@ CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ AFL_FUZZ_FILES = $(wildcard src/afl-fuzz*.c) -ifneq "($filter %3.7m, $(shell python3.7m-config --includes 2>/dev/null)" "" +ifneq "$(filter %3.7m, $(shell python3.7m-config --includes 2>/dev/null))" "" PYTHON_INCLUDE ?= $(shell python3.7m-config --includes) PYTHON_LIB ?= $(shell python3.7m-config --ldflags) PYTHON_VERSION = 3.7m else - ifneq "($filter %3.7, $(shell python3.7-config --includes) 2> /dev/null" "" + ifneq "$(filter %3.7, $(shell python3.7-config --includes) 2> /dev/null)" "" PYTHON_INCLUDE ?= $(shell python3.7-config --includes) PYTHON_LIB ?= $(shell python3.7-config --ldflags) PYTHON_VERSION = 3.7 else - ifneq "($filter %2.7, $(shell python2.7-config --includes) 2> /dev/null" "" + ifneq "$(filter %2.7, $(shell python2.7-config --includes) 2> /dev/null)" "" PYTHON_INCLUDE ?= $(shell python2.7-config --includes) PYTHON_LIB ?= $(shell python2.7-config --ldflags) PYTHON_VERSION = 2.7 @@ -77,14 +77,14 @@ PYTHON_INCLUDE ?= $(shell test -e /usr/include/python3.7m && echo /usr/include/p PYTHON_INCLUDE ?= $(shell test -e /usr/include/python3.7 && echo /usr/include/python3.7) PYTHON_INCLUDE ?= $(shell test -e /usr/include/python2.7 && echo /usr/include/python2.7) -ifneq "($filter %3.7m, $(PYTHON_INCLUDE))" "" +ifneq "$(filter %3.7m, $(PYTHON_INCLUDE))" "" PYTHON_VERSION ?= 3.7m PYTHON_LIB ?= -lpython3.7m else - ifneq "($filter %3.7, $(PYTHON_INCLUDE))" "" + ifneq "$(filter %3.7, $(PYTHON_INCLUDE))" "" PYTHON_VERSION ?= 3.7 else - ifneq "($filter %2.7, $(PYTHON_INCLUDE))" "" + ifneq "$(filter %2.7, $(PYTHON_INCLUDE))" "" PYTHON_VERSION ?= 2.7 PYTHON_LIB ?= -lpython2.7 else -- cgit 1.4.1 From 00b1d16ac61e9f86cd0c1defec6299e0a5e3fdde Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sat, 18 Jan 2020 16:28:13 +0100 Subject: more fixes for python checks --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index df5ad048..dbb37feb 100644 --- a/Makefile +++ b/Makefile @@ -60,12 +60,12 @@ ifneq "$(filter %3.7m, $(shell python3.7m-config --includes 2>/dev/null))" "" PYTHON_LIB ?= $(shell python3.7m-config --ldflags) PYTHON_VERSION = 3.7m else - ifneq "$(filter %3.7, $(shell python3.7-config --includes) 2> /dev/null)" "" + ifneq "$(filter %3.7, $(shell python3.7-config --includes 2>/dev/null))" "" PYTHON_INCLUDE ?= $(shell python3.7-config --includes) PYTHON_LIB ?= $(shell python3.7-config --ldflags) PYTHON_VERSION = 3.7 else - ifneq "$(filter %2.7, $(shell python2.7-config --includes) 2> /dev/null)" "" + ifneq "$(filter %2.7, $(shell python2.7-config --includes 2>/dev/null))" "" PYTHON_INCLUDE ?= $(shell python2.7-config --includes) PYTHON_LIB ?= $(shell python2.7-config --ldflags) PYTHON_VERSION = 2.7 -- cgit 1.4.1 From db5d5017155a24cb04bef97a0cf97d45456e7901 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sat, 18 Jan 2020 16:46:14 +0100 Subject: set AFL_CC for libradamsa test (needed on FreeBSD) --- test/test.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 8f40773c..5bab0d7a 100755 --- a/test/test.sh +++ b/test/test.sh @@ -457,7 +457,13 @@ test -e ../libdislocator.so && { } rm -f test-compcov test -e ../libradamsa.so && { - test -e test-instr.plain || ../afl-clang-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 + # on FreeBSD need to set AFL_CC + if which clang >/dev/null; then + export AFL_CC=`which clang` + else + export AFL_CC=`$LLVM_CONFIG --bindir`/clang + fi + test -e test-instr.plain || ../afl-clang-fast -o test-instr.plain ../test-instr.c test -e test-instr.plain || ../afl-gcc-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain || ../${AFL_GCC} -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain && { -- cgit 1.4.1 From 08691fcc974a9fcf2df3e926959b21199df7e946 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sat, 18 Jan 2020 16:58:20 +0100 Subject: add forgotten stderr redirect --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 5bab0d7a..c770c1b7 100755 --- a/test/test.sh +++ b/test/test.sh @@ -463,7 +463,7 @@ test -e ../libradamsa.so && { else export AFL_CC=`$LLVM_CONFIG --bindir`/clang fi - test -e test-instr.plain || ../afl-clang-fast -o test-instr.plain ../test-instr.c + test -e test-instr.plain || ../afl-clang-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain || ../afl-gcc-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain || ../${AFL_GCC} -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain && { -- cgit 1.4.1 From 0eec6221554c260b2d93de73e88c2279c4479753 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sat, 18 Jan 2020 16:35:21 +0100 Subject: Intel test taken from lto branch, extended (as in test.sh), and tested on RaspberryPi --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index dbb37feb..7260ee47 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,14 @@ ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -march=native -o .te CFLAGS_OPT = -march=native endif +ifneq "$(shell uname -m)" "x86_64" + ifneq "$(shell uname -m)" "i386" + ifneq "$(shell uname -m)" "amd64" + AFL_NO_X86=1 + endif + endif +endif + CFLAGS ?= -O3 -funroll-loops $(CFLAGS_OPT) CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ -- cgit 1.4.1 From e7770a70023381bc7ff96b1d346b0ff9741f62de Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sun, 19 Jan 2020 12:25:32 +0100 Subject: make exporting AFL_CC FreeBSD specific, since it seems to harm the libradamsa test on travis/arm64 --- test/test.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/test.sh b/test/test.sh index c770c1b7..43b278b4 100755 --- a/test/test.sh +++ b/test/test.sh @@ -458,11 +458,14 @@ test -e ../libdislocator.so && { rm -f test-compcov test -e ../libradamsa.so && { # on FreeBSD need to set AFL_CC - if which clang >/dev/null; then - export AFL_CC=`which clang` - else - export AFL_CC=`$LLVM_CONFIG --bindir`/clang - fi + + test `uname -s` = 'FreeBSD' && { + if which clang >/dev/null; then + export AFL_CC=`which clang` + else + export AFL_CC=`$LLVM_CONFIG --bindir`/clang + fi + } test -e test-instr.plain || ../afl-clang-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain || ../afl-gcc-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain || ../${AFL_GCC} -o test-instr.plain ../test-instr.c > /dev/null 2>&1 -- cgit 1.4.1 From f706e210ec07d8797850781ed82d2279df9a88b9 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sun, 19 Jan 2020 21:20:51 +0100 Subject: add missing test cases for qemu_mode unsigaction library --- test/test.sh | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 43b278b4..97cc1511 100755 --- a/test/test.sh +++ b/test/test.sh @@ -569,8 +569,64 @@ test -e ../afl-qemu-trace && { CODE=1 exit 1 } - $ECHO "$YELLOW[-] we need a test case for qemu_mode unsigaction library" rm -rf in out errors + test -e ../qemu_mode/unsigaction/unsigaction32.so && { + ${AFL_CC} -o test-unsigaction32 -m32 test-unsigaction.c >> errors 2>&1 && { + ./test-unsigaction32 + RETVAL_NORMAL32=$? + LD_PRELOAD=../qemu_mode/unsigaction/unsigaction32.so ./test-unsigaction32 + RETVAL_LIBUNSIGACTION32=$? + test $RETVAL_NORMAL32 = "2" -a $RETVAL_LIBUNSIGACTION32 = "0" && { + $ECHO "$GREEN[+] qemu_mode unsigaction library (32 bit) ignores signals" + } || { + test $RETVAL_NORMAL32 != "2" && { + $ECHO "$RED[!] cannot trigger signal in test program (32 bit)" + } + test $RETVAL_LIBUNSIGACTION32 != "0" && { + $ECHO "$RED[!] signal in test program (32 bit) is not ignored with unsigaction" + } + CODE=1 + } + } || { + echo CUT------------------------------------------------------------------CUT + cat errors + echo CUT------------------------------------------------------------------CUT + $ECHO "$RED[!] cannot compile test program (32 bit) for unsigaction library" + CODE=1 + } + } || { + $ECHO "$YELLOW[-] we cannot test qemu_mode unsigaction library (32 bit) because it is not present" + INCOMPLETE=1 + } + test -e ../qemu_mode/unsigaction/unsigaction64.so && { + ${AFL_CC} -o test-unsigaction64 -m64 test-unsigaction.c >> errors 2>&1 && { + ./test-unsigaction64 + RETVAL_NORMAL64=$? + LD_PRELOAD=../qemu_mode/unsigaction/unsigaction64.so ./test-unsigaction64 + RETVAL_LIBUNSIGACTION64=$? + test $RETVAL_NORMAL64 = "2" -a $RETVAL_LIBUNSIGACTION64 = "0" && { + $ECHO "$GREEN[+] qemu_mode unsigaction library (64 bit) ignores signals" + } || { + test $RETVAL_NORMAL64 != "2" && { + $ECHO "$RED[!] cannot trigger signal in test program (64 bit)" + } + test $RETVAL_LIBUNSIGACTION64 != "0" && { + $ECHO "$RED[!] signal in test program (64 bit) is not ignored with unsigaction" + } + CODE=1 + } + } || { + echo CUT------------------------------------------------------------------CUT + cat errors + echo CUT------------------------------------------------------------------CUT + $ECHO "$RED[!] cannot compile test program (64 bit) for unsigaction library" + CODE=1 + } + } || { + $ECHO "$YELLOW[-] we cannot test qemu_mode unsigaction library (64 bit) because it is not present" + INCOMPLETE=1 + } + rm -rf errors test-unsigaction32 test-unsigaction64 } } || { $ECHO "$RED[!] gcc compilation of test targets failed - what is going on??" -- cgit 1.4.1 From 274c8d7d3cff7ad61f2a57c7f69914a3948711d2 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sun, 19 Jan 2020 21:22:41 +0100 Subject: add missing test program (oops) --- test/test-unsigaction.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 test/test-unsigaction.c diff --git a/test/test-unsigaction.c b/test/test-unsigaction.c new file mode 100644 index 00000000..1a5e4b26 --- /dev/null +++ b/test/test-unsigaction.c @@ -0,0 +1,25 @@ +#include /* sigemptyset(), sigaction(), kill(), SIGUSR1 */ +#include /* exit() */ +#include /* getpid() */ +#include /* errno */ +#include /* fprintf() */ + +static void mysig_handler(int sig) +{ + exit(2); +} + +int main() +{ + /* setup sig handler */ + struct sigaction sa; + sa.sa_handler = mysig_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + if (sigaction(SIGCHLD, &sa, NULL)) { + fprintf(stderr, "could not set signal handler %d, aborted\n", errno); + exit(1); + } + kill(getpid(), SIGCHLD); + return 0; +} -- cgit 1.4.1 From 72058fdcbcdc707824bd4211ce528237afc1140e Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 20 Jan 2020 12:56:55 +0100 Subject: another freebsd fix in test.sh --- test/test.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/test.sh b/test/test.sh index 97cc1511..23d98278 100755 --- a/test/test.sh +++ b/test/test.sh @@ -179,11 +179,13 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { $ECHO "$BLUE[*] Testing: llvm_mode" test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { # on FreeBSD need to set AFL_CC - if which clang >/dev/null; then - export AFL_CC=`which clang` - else - export AFL_CC=`$LLVM_CONFIG --bindir`/clang - fi + test `uname -s` = 'FreeBSD' && { + if which clang >/dev/null; then + export AFL_CC=`which clang` + else + export AFL_CC=`$LLVM_CONFIG --bindir`/clang + fi + } ../afl-clang-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 AFL_HARDEN=1 ../afl-clang-fast -o test-compcov.harden test-compcov.c > /dev/null 2>&1 test -e test-instr.plain && { -- cgit 1.4.1 From 0d5a8f69e9785cdaec4c9b62e186050112a7cb8f Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 20 Jan 2020 19:21:44 +0100 Subject: fixed Heiko's global search-replace :) --- include/afl-as.h | 2 +- include/afl-fuzz.h | 2 +- include/alloc-inl.h | 2 +- include/android-ashmem.h | 2 +- include/common.h | 2 +- include/config.h | 17 +++++++++-------- include/debug.h | 2 +- include/forkserver.h | 2 +- include/sharedmem.h | 2 +- include/types.h | 2 +- libdislocator/libdislocator.so.c | 1 + src/afl-analyze.c | 2 +- src/afl-as.c | 2 +- src/afl-common.c | 2 +- src/afl-forkserver.c | 2 +- src/afl-fuzz-bitmap.c | 9 +++++++-- src/afl-fuzz-extras.c | 2 +- src/afl-fuzz-globals.c | 2 +- src/afl-fuzz-init.c | 2 +- src/afl-fuzz-misc.c | 2 +- src/afl-fuzz-one.c | 2 +- src/afl-fuzz-python.c | 2 +- src/afl-fuzz-queue.c | 2 +- src/afl-fuzz-run.c | 2 +- src/afl-fuzz-stats.c | 2 +- src/afl-fuzz.c | 2 +- src/afl-gcc.c | 2 +- src/afl-gotcpu.c | 2 +- src/afl-sharedmem.c | 2 +- src/afl-showmap.c | 2 +- src/afl-tmin.c | 2 +- 31 files changed, 45 insertions(+), 38 deletions(-) diff --git a/include/afl-as.h b/include/afl-as.h index 3af42205..bd5e734a 100644 --- a/include/afl-as.h +++ b/include/afl-as.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 9ecf1f29..00d29f76 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/alloc-inl.h b/include/alloc-inl.h index 48598ed3..5592b295 100644 --- a/include/alloc-inl.h +++ b/include/alloc-inl.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/android-ashmem.h b/include/android-ashmem.h index 35a5ba5e..adddc05f 100755 --- a/include/android-ashmem.h +++ b/include/android-ashmem.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/common.h b/include/common.h index 8ab78b41..3b953470 100644 --- a/include/common.h +++ b/include/common.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/config.h b/include/config.h index 4eac82e0..83fcb8f9 100644 --- a/include/config.h +++ b/include/config.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi @@ -62,13 +62,14 @@ /* Default memory limit for child process (MB): */ #ifndef __NetBSD__ -# ifndef WORD_SIZE_64 -# define MEM_LIMIT 25 -# else -# define MEM_LIMIT 50 -# endif /* ^!WORD_SIZE_64 */ -#else /* NetBSD's kernel needs more space for stack, see discussion for issue #165 */ -# define MEM_LIMIT 200 +#ifndef WORD_SIZE_64 +#define MEM_LIMIT 25 +#else +#define MEM_LIMIT 50 +#endif /* ^!WORD_SIZE_64 */ +#else /* NetBSD's kernel needs more space for stack, see discussion for issue \ + #165 */ +#define MEM_LIMIT 200 #endif /* Default memory limit when running in QEMU mode (MB): */ diff --git a/include/debug.h b/include/debug.h index 68109927..d6c04935 100644 --- a/include/debug.h +++ b/include/debug.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/forkserver.h b/include/forkserver.h index 17bc65af..0fdcba48 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -6,7 +6,7 @@ Forkserver design by Jann Horn - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/sharedmem.h b/include/sharedmem.h index 69291330..7604d64c 100644 --- a/include/sharedmem.h +++ b/include/sharedmem.h @@ -6,7 +6,7 @@ Forkserver design by Jann Horn - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/types.h b/include/types.h index eba47be7..9e681e81 100644 --- a/include/types.h +++ b/include/types.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/libdislocator/libdislocator.so.c b/libdislocator/libdislocator.so.c index b9ba8967..221a629b 100644 --- a/libdislocator/libdislocator.so.c +++ b/libdislocator/libdislocator.so.c @@ -417,6 +417,7 @@ void* reallocarray(void* ptr, size_t elem_len, size_t elem_cnt) { } return ret; + } __attribute__((constructor)) void __dislocator_init(void) { diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 3d4e636e..3de8c037 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-as.c b/src/afl-as.c index 77ac2f97..8d689385 100644 --- a/src/afl-as.c +++ b/src/afl-as.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-common.c b/src/afl-common.c index 8c2f2b9a..6cb97cdf 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index de50c73c..77e1d648 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -6,7 +6,7 @@ Forkserver design by Jann Horn - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 515a7a79..3ffda284 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi @@ -524,7 +524,12 @@ u8 save_if_interesting(char** argv, void* mem, u32 len, u8 fault) { struct queue_entry* q = queue; while (q) { - if (q->exec_cksum == cksum) { q->n_fuzz = q->n_fuzz + 1; break ; } + if (q->exec_cksum == cksum) { + + q->n_fuzz = q->n_fuzz + 1; + break; + + } q = q->next; diff --git a/src/afl-fuzz-extras.c b/src/afl-fuzz-extras.c index fcc7749d..6c6dc28c 100644 --- a/src/afl-fuzz-extras.c +++ b/src/afl-fuzz-extras.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-globals.c b/src/afl-fuzz-globals.c index b3476778..f0d98192 100644 --- a/src/afl-fuzz-globals.c +++ b/src/afl-fuzz-globals.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 5fe3689e..219be822 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-misc.c b/src/afl-fuzz-misc.c index b8f376be..0da0cb0a 100644 --- a/src/afl-fuzz-misc.c +++ b/src/afl-fuzz-misc.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 74123300..b04683be 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index f1cdecde..f06c8e25 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 1b51e3aa..0880de75 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index fa7a872a..a006194d 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 7679403b..f2afb295 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 0af8b35f..9a7495ef 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-gcc.c b/src/afl-gcc.c index 301e2034..e46fe5cd 100644 --- a/src/afl-gcc.c +++ b/src/afl-gcc.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-gotcpu.c b/src/afl-gotcpu.c index 9a56159c..5be30238 100644 --- a/src/afl-gotcpu.c +++ b/src/afl-gotcpu.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-sharedmem.c b/src/afl-sharedmem.c index 16eb14a7..04fcaa1c 100644 --- a/src/afl-sharedmem.c +++ b/src/afl-sharedmem.c @@ -6,7 +6,7 @@ Forkserver design by Jann Horn - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 8c899c9d..b9da3208 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -6,7 +6,7 @@ Forkserver design by Jann Horn - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 3e33b72f..7ce0ccaa 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -6,7 +6,7 @@ Forkserver design by Jann Horn - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi -- cgit 1.4.1 From 00d086f816d6b517a6817d6093a83ed8a65b18fa Mon Sep 17 00:00:00 2001 From: van Hauser Date: Tue, 21 Jan 2020 12:53:36 +0100 Subject: USE_TRACE_PC unnecessary, set env AFL_LLVM_USE_TRACE_PC instead --- docs/ChangeLog | 8 +++++--- llvm_mode/README.md | 19 +++++++++---------- llvm_mode/afl-clang-fast.c | 33 ++++++++++++++++++++++++--------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/docs/ChangeLog b/docs/ChangeLog index 5347d244..bb3537dd 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -21,9 +21,11 @@ Version ++2.60d (develop): - afl-fuzz: - now prints the real python version support compiled in - set stronger performance compile options and little tweaks - - afl-clang-fast now shows in the help output for which llvm version it - was compiled for - - added blacklisted function check in llvm_mode + - afl-clang-fast: + - show in the help output for which llvm version it was compiled for + - now does not need to be recompiled between trace-pc and pass + instrumentation. compile normally and set AFL_LLVM_USE_TRACE_PC :) + - added blacklisted function check in all modules of llvm_mode - added fix from Debian project to compile libdislocator and libtokencap diff --git a/llvm_mode/README.md b/llvm_mode/README.md index 5afa4dfd..150d1a17 100644 --- a/llvm_mode/README.md +++ b/llvm_mode/README.md @@ -198,24 +198,23 @@ PS. Because there are task switches still involved, the mode isn't as fast as faster than the normal fork() model, and compared to in-process fuzzing, should be a lot more robust. -## 8) Bonus feature #3: new 'trace-pc-guard' mode +## 8) Bonus feature #3: 'trace-pc-guard' mode -Recent versions of LLVM are shipping with a built-in execution tracing feature +LLVM is shipping with a built-in execution tracing feature that provides AFL with the necessary tracing data without the need to post-process the assembly or install any compiler plugins. See: http://clang.llvm.org/docs/SanitizerCoverage.html#tracing-pcs-with-guards -If you have a sufficiently recent compiler and want to give it a try, build -afl-clang-fast this way: +If you have not an outdated compiler and want to give it a try, build +targets this way: ``` - AFL_TRACE_PC=1 make clean all + libtarget-1.0 $ AFL_LLVM_USE_TRACE_PC=1 make ``` -Note that this mode is currently about 20% slower than "vanilla" afl-clang-fast, +Note that this mode is about 20% slower than "vanilla" afl-clang-fast, and about 5-10% slower than afl-clang. This is likely because the -instrumentation is not inlined, and instead involves a function call. On systems -that support it, compiling your target with -flto should help. - - +instrumentation is not inlined, and instead involves a function call. +On systems that support it, compiling your target with -flto can help +a bit. diff --git a/llvm_mode/afl-clang-fast.c b/llvm_mode/afl-clang-fast.c index b322b762..7da7c5a3 100644 --- a/llvm_mode/afl-clang-fast.c +++ b/llvm_mode/afl-clang-fast.c @@ -204,13 +204,24 @@ static void edit_params(u32 argc, char** argv) { // "-fsanitize-coverage=trace-cmp,trace-div,trace-gep"; // cc_params[cc_par_cnt++] = "-sanitizer-coverage-block-threshold=0"; #else - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - if (getenv("AFL_LLVM_INSTRIM") != NULL || getenv("INSTRIM_LIB") != NULL) - cc_params[cc_par_cnt++] = alloc_printf("%s/libLLVMInsTrim.so", obj_path); - else - cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); + if (getenv("USE_TRACE_PC") || getenv("AFL_USE_TRACE_PC") || + getenv("AFL_LLVM_USE_TRACE_PC") || getenv("AFL_TRACE_PC")) { + + cc_params[cc_par_cnt++] = + "-fsanitize-coverage=trace-pc-guard"; // edge coverage by default + + } else { + + cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = "-load"; + cc_params[cc_par_cnt++] = "-Xclang"; + if (getenv("AFL_LLVM_INSTRIM") != NULL || getenv("INSTRIM_LIB") != NULL) + cc_params[cc_par_cnt++] = alloc_printf("%s/libLLVMInsTrim.so", obj_path); + else + cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); + + } + #endif /* ^USE_TRACE_PC */ cc_params[cc_par_cnt++] = "-Qunused-arguments"; @@ -282,8 +293,10 @@ static void edit_params(u32 argc, char** argv) { #ifdef USE_TRACE_PC - if (getenv("AFL_INST_RATIO")) - FATAL("AFL_INST_RATIO not available at compile time with 'trace-pc'."); + if (getenv("USE_TRACE_PC") || getenv("AFL_USE_TRACE_PC") || + getenv("AFL_LLVM_USE_TRACE_PC") || getenv("AFL_TRACE_PC")) + if (getenv("AFL_INST_RATIO")) + FATAL("AFL_INST_RATIO not available at compile time with 'trace-pc'."); #endif /* USE_TRACE_PC */ @@ -455,6 +468,8 @@ int main(int argc, char** argv) { #ifdef USE_TRACE_PC SAYF(cCYA "afl-clang-fast" VERSION cRST " [tpcg] by \n"); +#warning \ + "You do not need to specifically compile with USE_TRACE_PC anymore, setting the environment variable AFL_LLVM_USE_TRACE_PC is enough." #else SAYF(cCYA "afl-clang-fast" VERSION cRST " by \n"); #endif /* ^USE_TRACE_PC */ -- cgit 1.4.1 From fb221db8ae4d640aa6261633ca249a86305292c4 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 22 Jan 2020 08:35:41 +0100 Subject: clarify gcc plugin test case result --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 23d98278..0ae6fd09 100755 --- a/test/test.sh +++ b/test/test.sh @@ -336,7 +336,7 @@ test -e ../afl-gcc-fast -a -e ../afl-gcc-rt.o && { $ECHO "$GREEN[+] gcc_plugin run reported $TUPLES instrumented locations which is fine" } || { $ECHO "$RED[!] gcc_plugin instrumentation produces a weird number of instrumented locations: $TUPLES" - $ECHO "$YELLOW[-] the gcc_plugin instrumentation issue is not flagged as an error because travis builds would all fail otherwise :-(" + $ECHO "$YELLOW[-] this is a known issue in gcc, not afl++. It is not flagged as an error because travis builds would all fail otherwise :-(" #CODE=1 } } -- cgit 1.4.1 From 4fbcc37f8450136759913875b6234d2e3ab2f032 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Wed, 22 Jan 2020 09:26:54 +0100 Subject: awk version for portability, tested on linux and FreeBSD so far --- afl-cmin.awk | 440 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 440 insertions(+) create mode 100755 afl-cmin.awk diff --git a/afl-cmin.awk b/afl-cmin.awk new file mode 100755 index 00000000..021f7059 --- /dev/null +++ b/afl-cmin.awk @@ -0,0 +1,440 @@ +#!/usr/bin/awk -f + +# getopt.awk --- Do C library getopt(3) function in awk + +# External variables: +# Optind -- index in ARGV of first nonoption argument +# Optarg -- string value of argument to current option +# Opterr -- if nonzero, print our own diagnostic +# Optopt -- current option letter + +# Returns: +# -1 at end of options +# "?" for unrecognized option +# a character representing the current option + +# Private Data: +# _opti -- index in multiflag option, e.g., -abc + +function getopt(argc, argv, options, thisopt, i) +{ + if (length(options) == 0) # no options given + return -1 + + if (argv[Optind] == "--") { # all done + Optind++ + _opti = 0 + return -1 + } else if (argv[Optind] !~ /^-[^:[:space:]]/) { + _opti = 0 + return -1 + } + if (_opti == 0) + _opti = 2 + thisopt = substr(argv[Optind], _opti, 1) + Optopt = thisopt + i = index(options, thisopt) + if (i == 0) { + if (Opterr) + printf("%c -- invalid option\n", thisopt) > "/dev/stderr" + if (_opti >= length(argv[Optind])) { + Optind++ + _opti = 0 + } else + _opti++ + return "?" + } + if (substr(options, i + 1, 1) == ":") { + # get option argument + if (length(substr(argv[Optind], _opti + 1)) > 0) + Optarg = substr(argv[Optind], _opti + 1) + else + Optarg = argv[++Optind] + _opti = 0 + } else + Optarg = "" + if (_opti == 0 || _opti >= length(argv[Optind])) { + Optind++ + _opti = 0 + } else + _opti++ + return thisopt +} + +BEGIN { + Opterr = 1 # default is to diagnose + Optind = 1 # skip ARGV[0] + + # test program + if (_getopt_test) { + while ((_go_c = getopt(ARGC, ARGV, "ab:cd")) != -1) + printf("c = <%c>, Optarg = <%s>\n", + _go_c, Optarg) + printf("non-option arguments:\n") + for (; Optind < ARGC; Optind++) + printf("\tARGV[%d] = <%s>\n", + Optind, ARGV[Optind]) + } +} + +function usage() { + print \ +"Usage: afl-cmin [ options ] -- /path/to/target_app [ ... ]\n" \ +"\n" \ +"Required parameters:\n" \ +"\n" \ +" -i dir - input directory with starting corpus\n" \ +" -o dir - output directory for minimized files\n" \ +"\n" \ +"Execution control settings:\n" \ +"\n" \ +" -f file - location read by the fuzzed program (stdin)\n" \ +" -m megs - memory limit for child process ("mem_limit" MB)\n" \ +" -t msec - run time limit for child process (none)\n" \ +" -Q - use binary-only instrumentation (QEMU mode)\n" \ +" -U - use unicorn-based instrumentation (unicorn mode)\n" \ +"\n" \ +"Minimization settings:\n" \ +" -C - keep crashing inputs, reject everything else\n" \ +" -e - solve for edge coverage only, ignore hit counts\n" \ +"\n" \ +"For additional tips, please consult docs/README.md\n" \ +"\n" \ + > "/dev/stderr" + exit 1 +} + +function exists_and_is_executable(binarypath) { + return 0 == system("test -f "binarypath" -a -x "binarypath) +} + +BEGIN { + print "corpus minimization tool for afl-fuzz++ (awk version)\n" +print "PATH="ENVIRON["PATH"] + + # defaults + extra_par = "" + # process options + Opterr = 1 # default is to diagnose + Optind = 1 # skip ARGV[0] + while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eCQU?")) != -1) { + if (_go_c == "i") { + if (!Optarg) usage() + if (in_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + in_dir = Optarg + continue + } else + if (_go_c == "o") { + if (!Optarg) usage() + if (out_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + out_dir = Optarg + continue + } else + if (_go_c == "f") { + if (!Optarg) usage() + if (stdin_file) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + stdin_file = Optarg + continue + } else + if (_go_c == "m") { + if (!Optarg) usage() + if (mem_limit) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + mem_limit = Optarg + mem_limit_given = 1 + continue + } else + if (_go_c == "t") { + if (!Optarg) usage() + if (timeout) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + timeout = Optarg + continue + } else + if (_go_c == "C") { + ENVIRON["AFL_CMIN_CRASHES_ONLY"] = 1 + continue + } else + if (_go_c == "e") { + extra_par = extra_par " -e" + continue + } else + if (_go_c == "Q") { + if (qemu_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + extra_par = extra_par " -Q" + if ( !mem_limit_given ) mem_limit = "250" + qemu_mode = 1 + continue + } else + if (_go_c == "U") { + if (unicorn_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + extra_par = extra_par " -U" + if ( !mem_limit_given ) mem_limit = "250" + unicorn_mode = 1 + continue + } else + if (_go_c == "?") { + exit 1 + } else + usage() + } # while options + + if (!mem_limit) mem_limit = 200 + if (!timeout) timeout = "none" + + # get program args + i = 0 + prog_args_string = "" + for (; Optind < ARGC; Optind++) { + prog_args[i++] = ARGV[Optind] + if (i > 1) + prog_args_string = prog_args_string" "ARGV[Optind] + } + + # sanity checks + if (!prog_args[0] || !in_dir || !out_dir) usage() + + target_bin = prog_args[0] + + # Do a sanity check to discourage the use of /tmp, since we can't really + # handle this safely from an awk script. + + if (!ENVIRON["AFL_ALLOW_TMP"]) { + dirlist[0] = in_dir + dirlist[1] = target_bin + dirlist[2] = out_dir + dirlist[3] = stdin_file + "pwd" | getline dirlist[4] # current directory + for (dirind in dirlist) { + dir = dirlist[dirind] + + if (dir ~ /^(\/var)?\/tmp/) { + print "[-] Error: do not use this script in /tmp or /var/tmp." > "/dev/stderr" + exit 1 + } + } + delete dirlist + } + + # If @@ is specified, but there's no -f, let's come up with a temporary input + # file name. + + trace_dir = out_dir "/.traces" + + if (!stdin_file) { + found_atat = 0 + for (prog_args_ind in prog_args) { + if ("@@" == prog_args[prog_args_ind]) { + found_atat = 1 + break + } + } + if (found_atat) { + stdin_file = trace_dir "/.cur_input" + } + } + + # Check for obvious errors. + + if (mem_limit && mem_limit != "none" && mem_limit < 5) { + print "[-] Error: dangerously low memory limit." > "/dev/stderr" + exit 1 + } + + if (timeout && timeout != "none" && timeout < 10) { + print "[-] Error: dangerously low timeout." > "/dev/stderr" + exit 1 + } + + if (target_bin && !exists_and_is_executable(target_bin)) { + + "which "target_bin" 2>/dev/null" | getline tnew + if (!tnew || !exists_and_is_executable(tnew)) { + print "[-] Error: binary '"target_bin"' not found or not executable." > "/dev/stderr" + exit 1 + } + target_bin = tnew + } + + if (!ENVIRON["AFL_SKIP_BIN_CHECK"] && !qemu_mode && !unicorn_mode) { + if (0 != system( "grep -q __AFL_SHM_ID "target_bin )) { + print "[-] Error: binary '"target_bin"' doesn't appear to be instrumented." > "/dev/stderr" + exit 1 + } + } + + if (0 != system( "test -d "in_dir )) { + print "[-] Error: directory '"in_dir"' not found." > "/dev/stderr" + exit 1 + } + + if (0 == system( "test -d "in_dir"/queue" )) { + in_dir = in_dir "/queue" + } + + system("rm -rf "trace_dir" 2>/dev/null"); + system("rm "out_dir"/id[:_]* 2>/dev/null") + + if (0 == system( "test -d "out_dir" -a -e "out_dir"/*" )) { + print "[-] Error: directory '"out_dir"' exists and is not empty - delete it first." > "/dev/stderr" + exit 1 + } + + if (stdin_file) { + # truncate input file + printf "" > stdin_file + close( stdin_file ) + } + + if (!ENVIRON["AFL_PATH"]) { + if (0 == system("test -f afl-cmin.awk")) { + path = "." + } else { + "which afl-showmap 2>/dev/null" | getline path + } + showmap = path + } else { + showmap = ENVIRON["AFL_PATH"] "/afl-showmap" + } + + if (!showmap || 0 != system("test -x "showmap )) { + print "[-] Error: can't find 'afl-showmap' - please set AFL_PATH." > "/dev/stderr" + exit 1 + } + + # get list of input filenames sorted by size + i = 0 + while ("find "in_dir" -type f -exec stat -f '%z %N' \{\} \; | sort -n | cut -d' ' -f2-" | getline) { + infilesSmallToBig[i++] = $0 + } + in_count = i + + first_file = infilesSmallToBig[0] + + # Make sure that we're not dealing with a directory. + + if (0 == system("test -d "in_dir"/"first_file)) { + print "[-] Error: The input directory contains subdirectories - please fix." > "/dev/stderr" + exit 1 + } + + # Check for the more efficient way to copy files... + if (0 != system("mkdir -p -m 0700 "trace_dir)) { + print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" + exit 1 + } + + if (0 == system("ln "in_dir"/"first_file" "trace_dir"/.link_test")) { + cp_tool = "ln" + } else { + cp_tool = "cp" + } + + # Make sure that we can actually get anything out of afl-showmap before we + # waste too much time. + + print "[*] Testing the target binary..." + + if (!stdin_file) { + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") + } else { + system("cp "in_dir"/"first_file" "stdin_file) + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 0) { + ++first_count + } + + if (first_count) { + print "[+] OK, "first_count" tuples recorded." + } else { + print "[-] Error: no instrumentation output detected (perhaps crash or timeout)." > "/dev/stderr" + if (!ENVIRON["AFL_KEEP_TRACES"]) { + system("rm -rf "trace_dir" 2>/dev/null") + } + exit 1 + } + + # Let's roll! + + ############################# + # STEP 1: Collecting traces # + ############################# + + print "[*] Obtaining traces for "in_count" input files in '"in_dir"'." + + cur = 0; + if (!stdin_file) { + while (cur < in_count) { + fn = infilesSmallToBig[cur] + ++cur; + printf "\r Processing file "cur"/"in_count + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/"fn"\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"fn"\"") + } + } else { + while (cur < in_count) { + fn = infilesSmallToBig[cur] + ++cur + printf "\r Processing file "cur"/"in_count + system("cp "in_dir"/"fn" "stdin_file) + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/"fn"\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 0) { + key = line + if (!(key in key_count)) { + ++tuple_count + } + ++key_count[key] + if (! (key in best_file)) { + # this is the best file for this key + best_file[key] = fn + # copy file unless already done + if (! (fn in file_already_copied)) { + system(cp_tool" "in_dir"/"fn" "out_dir"/"fn) + file_already_copied[fn] = "" + ++out_count + } + } + } + close(tracefile_path) + } + + print "" + print "[+] Found "tuple_count" unique tuples across "in_count" files." + + if (out_count == 1) { + print "[!] WARNING: All test cases had the same traces, check syntax!" + } + print "[+] Narrowed down to "out_count" files, saved in '"out_dir"'." + + if (!ENVIRON["AFL_KEEP_TRACES"]) { + system("rm -rf "trace_dir" 2>/dev/null") + } + + exit 0 +} -- cgit 1.4.1 From 7ce627c92e9b0536e254422d5ef604c3f58e43ce Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Wed, 22 Jan 2020 18:38:41 +0100 Subject: Oops, only this version works with FreeBSD, OpenBSD, NetBSD, MacOS, raspbian --- afl-cmin.awk | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/afl-cmin.awk b/afl-cmin.awk index 021f7059..28c460e8 100755 --- a/afl-cmin.awk +++ b/afl-cmin.awk @@ -110,7 +110,6 @@ function exists_and_is_executable(binarypath) { BEGIN { print "corpus minimization tool for afl-fuzz++ (awk version)\n" -print "PATH="ENVIRON["PATH"] # defaults extra_par = "" @@ -302,7 +301,13 @@ print "PATH="ENVIRON["PATH"] # get list of input filenames sorted by size i = 0 - while ("find "in_dir" -type f -exec stat -f '%z %N' \{\} \; | sort -n | cut -d' ' -f2-" | getline) { + # yuck, gnu stat is incompatible to bsd stat + if ("stat --version 2>/dev/null" !~ /GNU coreutils/) { + stat_format = "-f '%z %N'" + } else { + stat_format = "-c '%s %n'" + } + while ("cd "in_dir" && find . -type f -exec stat "stat_format" \{\} \\; | sort -n | cut -d' ' -f2-" | getline) { infilesSmallToBig[i++] = $0 } in_count = i -- cgit 1.4.1 From ce0b9dae5971f22cd0ae0b468322f78ee2a8a766 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Wed, 22 Jan 2020 19:07:02 +0100 Subject: final step: rename afl-cmin to afl-cmin.bash and add a wrapper afl-cmin for afl-cmin.awk --- afl-cmin | 474 +--------------------------------------------------------- afl-cmin.bash | 470 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 474 insertions(+), 470 deletions(-) create mode 100755 afl-cmin.bash diff --git a/afl-cmin b/afl-cmin index 1dd782d8..75dc63a7 100755 --- a/afl-cmin +++ b/afl-cmin @@ -1,470 +1,4 @@ -#!/usr/bin/env bash -# -# american fuzzy lop++ - corpus minimization tool -# --------------------------------------------- -# -# Originally written by Michal Zalewski -# -# Copyright 2014, 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# This tool tries to find the smallest subset of files in the input directory -# that still trigger the full range of instrumentation data points seen in -# the starting corpus. This has two uses: -# -# - Screening large corpora of input files before using them as a seed for -# afl-fuzz. The tool will remove functionally redundant files and likely -# leave you with a much smaller set. -# -# (In this case, you probably also want to consider running afl-tmin on -# the individual files later on to reduce their size.) -# -# - Minimizing the corpus generated organically by afl-fuzz, perhaps when -# planning to feed it to more resource-intensive tools. The tool achieves -# this by removing all entries that used to trigger unique behaviors in the -# past, but have been made obsolete by later finds. -# -# Note that the tool doesn't modify the files themselves. For that, you want -# afl-tmin. -# -# This script must use bash because other shells may have hardcoded limits on -# array sizes. -# - -echo "corpus minimization tool for afl-fuzz by Michal Zalewski" -echo - -######### -# SETUP # -######### - -# Process command-line options... - -MEM_LIMIT=200 -TIMEOUT=none - -unset IN_DIR OUT_DIR STDIN_FILE EXTRA_PAR MEM_LIMIT_GIVEN \ - AFL_CMIN_CRASHES_ONLY AFL_CMIN_ALLOW_ANY QEMU_MODE UNICORN_MODE - -while getopts "+i:o:f:m:t:eQUCh" opt; do - - case "$opt" in - - "h") - ;; - - "i") - IN_DIR="$OPTARG" - ;; - - "o") - OUT_DIR="$OPTARG" - ;; - "f") - STDIN_FILE="$OPTARG" - ;; - "m") - MEM_LIMIT="$OPTARG" - MEM_LIMIT_GIVEN=1 - ;; - "t") - TIMEOUT="$OPTARG" - ;; - "e") - EXTRA_PAR="$EXTRA_PAR -e" - ;; - "C") - export AFL_CMIN_CRASHES_ONLY=1 - ;; - "Q") - EXTRA_PAR="$EXTRA_PAR -Q" - test "$MEM_LIMIT_GIVEN" = "" && MEM_LIMIT=250 - QEMU_MODE=1 - ;; - "U") - EXTRA_PAR="$EXTRA_PAR -U" - test "$MEM_LIMIT_GIVEN" = "" && MEM_LIMIT=250 - UNICORN_MODE=1 - ;; - "?") - exit 1 - ;; - - esac - -done - -shift $((OPTIND-1)) - -TARGET_BIN="$1" - -if [ "$TARGET_BIN" = "" -o "$IN_DIR" = "" -o "$OUT_DIR" = "" ]; then - - cat 1>&2 <<_EOF_ -Usage: $0 [ options ] -- /path/to/target_app [ ... ] - -Required parameters: - - -i dir - input directory with the starting corpus - -o dir - output directory for minimized files - -Execution control settings: - - -f file - location read by the fuzzed program (stdin) - -m megs - memory limit for child process ($MEM_LIMIT MB) - -t msec - run time limit for child process (none) - -Q - use binary-only instrumentation (QEMU mode) - -U - use unicorn-based instrumentation (Unicorn mode) - -Minimization settings: - - -C - keep crashing inputs, reject everything else - -e - solve for edge coverage only, ignore hit counts - -For additional tips, please consult docs/README. - -_EOF_ - exit 1 -fi - -# Do a sanity check to discourage the use of /tmp, since we can't really -# handle this safely from a shell script. - -if [ "$AFL_ALLOW_TMP" = "" ]; then - - echo "$IN_DIR" | grep -qE '^(/var)?/tmp/' - T1="$?" - - echo "$TARGET_BIN" | grep -qE '^(/var)?/tmp/' - T2="$?" - - echo "$OUT_DIR" | grep -qE '^(/var)?/tmp/' - T3="$?" - - echo "$STDIN_FILE" | grep -qE '^(/var)?/tmp/' - T4="$?" - - echo "$PWD" | grep -qE '^(/var)?/tmp/' - T5="$?" - - if [ "$T1" = "0" -o "$T2" = "0" -o "$T3" = "0" -o "$T4" = "0" -o "$T5" = "0" ]; then - echo "[-] Error: do not use this script in /tmp or /var/tmp." 1>&2 - exit 1 - fi - -fi - -# If @@ is specified, but there's no -f, let's come up with a temporary input -# file name. - -TRACE_DIR="$OUT_DIR/.traces" - -if [ "$STDIN_FILE" = "" ]; then - - if echo "$*" | grep -qF '@@'; then - STDIN_FILE="$TRACE_DIR/.cur_input" - fi - -fi - -# Check for obvious errors. - -if [ ! "$MEM_LIMIT" = "none" ]; then - - if [ "$MEM_LIMIT" -lt "5" ]; then - echo "[-] Error: dangerously low memory limit." 1>&2 - exit 1 - fi - -fi - -if [ ! "$TIMEOUT" = "none" ]; then - - if [ "$TIMEOUT" -lt "10" ]; then - echo "[-] Error: dangerously low timeout." 1>&2 - exit 1 - fi - -fi - -if [ ! -f "$TARGET_BIN" -o ! -x "$TARGET_BIN" ]; then - - TNEW="`which "$TARGET_BIN" 2>/dev/null`" - - if [ ! -f "$TNEW" -o ! -x "$TNEW" ]; then - echo "[-] Error: binary '$TARGET_BIN' not found or not executable." 1>&2 - exit 1 - fi - - TARGET_BIN="$TNEW" - -fi - -if [ "$AFL_SKIP_BIN_CHECK" = "" -a "$QEMU_MODE" = "" -a "$UNICORN_MODE" = "" ]; then - - if ! grep -qF "__AFL_SHM_ID" "$TARGET_BIN"; then - echo "[-] Error: binary '$TARGET_BIN' doesn't appear to be instrumented." 1>&2 - exit 1 - fi - -fi - -if [ ! -d "$IN_DIR" ]; then - echo "[-] Error: directory '$IN_DIR' not found." 1>&2 - exit 1 -fi - -test -d "$IN_DIR/queue" && IN_DIR="$IN_DIR/queue" - -find "$OUT_DIR" -name 'id[:_]*' -maxdepth 1 -exec rm -- {} \; 2>/dev/null -rm -rf "$TRACE_DIR" 2>/dev/null - -rmdir "$OUT_DIR" 2>/dev/null - -if [ -d "$OUT_DIR" ]; then - echo "[-] Error: directory '$OUT_DIR' exists and is not empty - delete it first." 1>&2 - exit 1 -fi - -mkdir -m 700 -p "$TRACE_DIR" || exit 1 - -if [ ! "$STDIN_FILE" = "" ]; then - rm -f "$STDIN_FILE" || exit 1 - touch "$STDIN_FILE" || exit 1 -fi - -if [ "$AFL_PATH" = "" ]; then - SHOWMAP="${0%/afl-cmin}/afl-showmap" -else - SHOWMAP="$AFL_PATH/afl-showmap" -fi - -if [ ! -x "$SHOWMAP" ]; then - echo "[-] Error: can't find 'afl-showmap' - please set AFL_PATH." 1>&2 - rm -rf "$TRACE_DIR" - exit 1 -fi - -IN_COUNT=$((`ls -- "$IN_DIR" 2>/dev/null | wc -l`)) - -if [ "$IN_COUNT" = "0" ]; then - echo "[+] Hmm, no inputs in the target directory. Nothing to be done." - rm -rf "$TRACE_DIR" - exit 1 -fi - -FIRST_FILE=`ls "$IN_DIR" | head -1` - -# Make sure that we're not dealing with a directory. - -if [ -d "$IN_DIR/$FIRST_FILE" ]; then - echo "[-] Error: The target directory contains subdirectories - please fix." 1>&2 - rm -rf "$TRACE_DIR" - exit 1 -fi - -# Check for the more efficient way to copy files... - -if ln "$IN_DIR/$FIRST_FILE" "$TRACE_DIR/.link_test" 2>/dev/null; then - CP_TOOL=ln -else - CP_TOOL=cp -fi - -# Make sure that we can actually get anything out of afl-showmap before we -# waste too much time. - -echo "[*] Testing the target binary..." - -if [ "$STDIN_FILE" = "" ]; then - - AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$FIRST_FILE" - -else - - cp "$IN_DIR/$FIRST_FILE" "$STDIN_FILE" - AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_PAR -A "$STDIN_FILE" -- "$@" &2 - test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR" - exit 1 - -fi - -# Let's roll! - -############################# -# STEP 1: COLLECTING TRACES # -############################# - -echo "[*] Obtaining traces for input files in '$IN_DIR'..." - -( - - CUR=0 - - if [ "$STDIN_FILE" = "" ]; then - - ls "$IN_DIR" | while read -r fn; do - - CUR=$((CUR+1)) - printf "\\r Processing file $CUR/$IN_COUNT... " - - "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$fn" - - done - - else - - ls "$IN_DIR" | while read -r fn; do - - CUR=$((CUR+1)) - printf "\\r Processing file $CUR/$IN_COUNT... " - - cp "$IN_DIR/$fn" "$STDIN_FILE" - - "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -A "$STDIN_FILE" -- "$@" "$TRACE_DIR/.all_uniq" - -TUPLE_COUNT=$((`grep -c . "$TRACE_DIR/.all_uniq"`)) - -echo "[+] Found $TUPLE_COUNT unique tuples across $IN_COUNT files." - -##################################### -# STEP 3: SELECTING CANDIDATE FILES # -##################################### - -# The next step is to find the best candidate for each tuple. The "best" -# part is understood simply as the smallest input that includes a particular -# tuple in its trace. Empirical evidence suggests that this produces smaller -# datasets than more involved algorithms that could be still pulled off in -# a shell script. - -echo "[*] Finding best candidates for each tuple..." - -CUR=0 - -ls -rS "$IN_DIR" | while read -r fn; do - - CUR=$((CUR+1)) - printf "\\r Processing file $CUR/$IN_COUNT... " - - sed "s#\$# $fn#" "$TRACE_DIR/$fn" >>"$TRACE_DIR/.candidate_list" - -done - -echo - -############################## -# STEP 4: LOADING CANDIDATES # -############################## - -# At this point, we have a file of tuple-file pairs, sorted by file size -# in ascending order (as a consequence of ls -rS). By doing sort keyed -# only by tuple (-k 1,1) and configured to output only the first line for -# every key (-s -u), we end up with the smallest file for each tuple. - -echo "[*] Sorting candidate list (be patient)..." - -sort -k1,1 -s -u "$TRACE_DIR/.candidate_list" | \ - sed 's/^/BEST_FILE[/;s/ /]="/;s/$/"/' >"$TRACE_DIR/.candidate_script" - -if [ ! -s "$TRACE_DIR/.candidate_script" ]; then - echo "[-] Error: no traces obtained from test cases, check syntax!" 1>&2 - test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR" - exit 1 -fi - -# The sed command converted the sorted list to a shell script that populates -# BEST_FILE[tuple]="fname". Let's load that! - -. "$TRACE_DIR/.candidate_script" - -########################## -# STEP 5: WRITING OUTPUT # -########################## - -# The final trick is to grab the top pick for each tuple, unless said tuple is -# already set due to the inclusion of an earlier candidate; and then put all -# tuples associated with the newly-added file to the "already have" list. The -# loop works from least popular tuples and toward the most common ones. - -echo "[*] Processing candidates and writing output files..." - -CUR=0 - -touch "$TRACE_DIR/.already_have" - -while read -r cnt tuple; do - - CUR=$((CUR+1)) - printf "\\r Processing tuple $CUR/$TUPLE_COUNT... " - - # If we already have this tuple, skip it. - - grep -q "^$tuple\$" "$TRACE_DIR/.already_have" && continue - - FN=${BEST_FILE[tuple]} - - $CP_TOOL "$IN_DIR/$FN" "$OUT_DIR/$FN" - - if [ "$((CUR % 5))" = "0" ]; then - sort -u "$TRACE_DIR/$FN" "$TRACE_DIR/.already_have" >"$TRACE_DIR/.tmp" - mv -f "$TRACE_DIR/.tmp" "$TRACE_DIR/.already_have" - else - cat "$TRACE_DIR/$FN" >>"$TRACE_DIR/.already_have" - fi - -done <"$TRACE_DIR/.all_uniq" - -echo - -OUT_COUNT=`ls -- "$OUT_DIR" | wc -l` - -if [ "$OUT_COUNT" = "1" ]; then - echo "[!] WARNING: All test cases had the same traces, check syntax!" -fi - -echo "[+] Narrowed down to $OUT_COUNT files, saved in '$OUT_DIR'." -echo - -test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR" - -exit 0 +#!/usr/bin/env sh +THISPATH=`dirname ${0}` +export PATH=${THISPATH}:$PATH +awk -f ${0}.awk -- ${@+"$@"} diff --git a/afl-cmin.bash b/afl-cmin.bash new file mode 100755 index 00000000..1dd782d8 --- /dev/null +++ b/afl-cmin.bash @@ -0,0 +1,470 @@ +#!/usr/bin/env bash +# +# american fuzzy lop++ - corpus minimization tool +# --------------------------------------------- +# +# Originally written by Michal Zalewski +# +# Copyright 2014, 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# This tool tries to find the smallest subset of files in the input directory +# that still trigger the full range of instrumentation data points seen in +# the starting corpus. This has two uses: +# +# - Screening large corpora of input files before using them as a seed for +# afl-fuzz. The tool will remove functionally redundant files and likely +# leave you with a much smaller set. +# +# (In this case, you probably also want to consider running afl-tmin on +# the individual files later on to reduce their size.) +# +# - Minimizing the corpus generated organically by afl-fuzz, perhaps when +# planning to feed it to more resource-intensive tools. The tool achieves +# this by removing all entries that used to trigger unique behaviors in the +# past, but have been made obsolete by later finds. +# +# Note that the tool doesn't modify the files themselves. For that, you want +# afl-tmin. +# +# This script must use bash because other shells may have hardcoded limits on +# array sizes. +# + +echo "corpus minimization tool for afl-fuzz by Michal Zalewski" +echo + +######### +# SETUP # +######### + +# Process command-line options... + +MEM_LIMIT=200 +TIMEOUT=none + +unset IN_DIR OUT_DIR STDIN_FILE EXTRA_PAR MEM_LIMIT_GIVEN \ + AFL_CMIN_CRASHES_ONLY AFL_CMIN_ALLOW_ANY QEMU_MODE UNICORN_MODE + +while getopts "+i:o:f:m:t:eQUCh" opt; do + + case "$opt" in + + "h") + ;; + + "i") + IN_DIR="$OPTARG" + ;; + + "o") + OUT_DIR="$OPTARG" + ;; + "f") + STDIN_FILE="$OPTARG" + ;; + "m") + MEM_LIMIT="$OPTARG" + MEM_LIMIT_GIVEN=1 + ;; + "t") + TIMEOUT="$OPTARG" + ;; + "e") + EXTRA_PAR="$EXTRA_PAR -e" + ;; + "C") + export AFL_CMIN_CRASHES_ONLY=1 + ;; + "Q") + EXTRA_PAR="$EXTRA_PAR -Q" + test "$MEM_LIMIT_GIVEN" = "" && MEM_LIMIT=250 + QEMU_MODE=1 + ;; + "U") + EXTRA_PAR="$EXTRA_PAR -U" + test "$MEM_LIMIT_GIVEN" = "" && MEM_LIMIT=250 + UNICORN_MODE=1 + ;; + "?") + exit 1 + ;; + + esac + +done + +shift $((OPTIND-1)) + +TARGET_BIN="$1" + +if [ "$TARGET_BIN" = "" -o "$IN_DIR" = "" -o "$OUT_DIR" = "" ]; then + + cat 1>&2 <<_EOF_ +Usage: $0 [ options ] -- /path/to/target_app [ ... ] + +Required parameters: + + -i dir - input directory with the starting corpus + -o dir - output directory for minimized files + +Execution control settings: + + -f file - location read by the fuzzed program (stdin) + -m megs - memory limit for child process ($MEM_LIMIT MB) + -t msec - run time limit for child process (none) + -Q - use binary-only instrumentation (QEMU mode) + -U - use unicorn-based instrumentation (Unicorn mode) + +Minimization settings: + + -C - keep crashing inputs, reject everything else + -e - solve for edge coverage only, ignore hit counts + +For additional tips, please consult docs/README. + +_EOF_ + exit 1 +fi + +# Do a sanity check to discourage the use of /tmp, since we can't really +# handle this safely from a shell script. + +if [ "$AFL_ALLOW_TMP" = "" ]; then + + echo "$IN_DIR" | grep -qE '^(/var)?/tmp/' + T1="$?" + + echo "$TARGET_BIN" | grep -qE '^(/var)?/tmp/' + T2="$?" + + echo "$OUT_DIR" | grep -qE '^(/var)?/tmp/' + T3="$?" + + echo "$STDIN_FILE" | grep -qE '^(/var)?/tmp/' + T4="$?" + + echo "$PWD" | grep -qE '^(/var)?/tmp/' + T5="$?" + + if [ "$T1" = "0" -o "$T2" = "0" -o "$T3" = "0" -o "$T4" = "0" -o "$T5" = "0" ]; then + echo "[-] Error: do not use this script in /tmp or /var/tmp." 1>&2 + exit 1 + fi + +fi + +# If @@ is specified, but there's no -f, let's come up with a temporary input +# file name. + +TRACE_DIR="$OUT_DIR/.traces" + +if [ "$STDIN_FILE" = "" ]; then + + if echo "$*" | grep -qF '@@'; then + STDIN_FILE="$TRACE_DIR/.cur_input" + fi + +fi + +# Check for obvious errors. + +if [ ! "$MEM_LIMIT" = "none" ]; then + + if [ "$MEM_LIMIT" -lt "5" ]; then + echo "[-] Error: dangerously low memory limit." 1>&2 + exit 1 + fi + +fi + +if [ ! "$TIMEOUT" = "none" ]; then + + if [ "$TIMEOUT" -lt "10" ]; then + echo "[-] Error: dangerously low timeout." 1>&2 + exit 1 + fi + +fi + +if [ ! -f "$TARGET_BIN" -o ! -x "$TARGET_BIN" ]; then + + TNEW="`which "$TARGET_BIN" 2>/dev/null`" + + if [ ! -f "$TNEW" -o ! -x "$TNEW" ]; then + echo "[-] Error: binary '$TARGET_BIN' not found or not executable." 1>&2 + exit 1 + fi + + TARGET_BIN="$TNEW" + +fi + +if [ "$AFL_SKIP_BIN_CHECK" = "" -a "$QEMU_MODE" = "" -a "$UNICORN_MODE" = "" ]; then + + if ! grep -qF "__AFL_SHM_ID" "$TARGET_BIN"; then + echo "[-] Error: binary '$TARGET_BIN' doesn't appear to be instrumented." 1>&2 + exit 1 + fi + +fi + +if [ ! -d "$IN_DIR" ]; then + echo "[-] Error: directory '$IN_DIR' not found." 1>&2 + exit 1 +fi + +test -d "$IN_DIR/queue" && IN_DIR="$IN_DIR/queue" + +find "$OUT_DIR" -name 'id[:_]*' -maxdepth 1 -exec rm -- {} \; 2>/dev/null +rm -rf "$TRACE_DIR" 2>/dev/null + +rmdir "$OUT_DIR" 2>/dev/null + +if [ -d "$OUT_DIR" ]; then + echo "[-] Error: directory '$OUT_DIR' exists and is not empty - delete it first." 1>&2 + exit 1 +fi + +mkdir -m 700 -p "$TRACE_DIR" || exit 1 + +if [ ! "$STDIN_FILE" = "" ]; then + rm -f "$STDIN_FILE" || exit 1 + touch "$STDIN_FILE" || exit 1 +fi + +if [ "$AFL_PATH" = "" ]; then + SHOWMAP="${0%/afl-cmin}/afl-showmap" +else + SHOWMAP="$AFL_PATH/afl-showmap" +fi + +if [ ! -x "$SHOWMAP" ]; then + echo "[-] Error: can't find 'afl-showmap' - please set AFL_PATH." 1>&2 + rm -rf "$TRACE_DIR" + exit 1 +fi + +IN_COUNT=$((`ls -- "$IN_DIR" 2>/dev/null | wc -l`)) + +if [ "$IN_COUNT" = "0" ]; then + echo "[+] Hmm, no inputs in the target directory. Nothing to be done." + rm -rf "$TRACE_DIR" + exit 1 +fi + +FIRST_FILE=`ls "$IN_DIR" | head -1` + +# Make sure that we're not dealing with a directory. + +if [ -d "$IN_DIR/$FIRST_FILE" ]; then + echo "[-] Error: The target directory contains subdirectories - please fix." 1>&2 + rm -rf "$TRACE_DIR" + exit 1 +fi + +# Check for the more efficient way to copy files... + +if ln "$IN_DIR/$FIRST_FILE" "$TRACE_DIR/.link_test" 2>/dev/null; then + CP_TOOL=ln +else + CP_TOOL=cp +fi + +# Make sure that we can actually get anything out of afl-showmap before we +# waste too much time. + +echo "[*] Testing the target binary..." + +if [ "$STDIN_FILE" = "" ]; then + + AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$FIRST_FILE" + +else + + cp "$IN_DIR/$FIRST_FILE" "$STDIN_FILE" + AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_PAR -A "$STDIN_FILE" -- "$@" &2 + test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR" + exit 1 + +fi + +# Let's roll! + +############################# +# STEP 1: COLLECTING TRACES # +############################# + +echo "[*] Obtaining traces for input files in '$IN_DIR'..." + +( + + CUR=0 + + if [ "$STDIN_FILE" = "" ]; then + + ls "$IN_DIR" | while read -r fn; do + + CUR=$((CUR+1)) + printf "\\r Processing file $CUR/$IN_COUNT... " + + "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$fn" + + done + + else + + ls "$IN_DIR" | while read -r fn; do + + CUR=$((CUR+1)) + printf "\\r Processing file $CUR/$IN_COUNT... " + + cp "$IN_DIR/$fn" "$STDIN_FILE" + + "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -A "$STDIN_FILE" -- "$@" "$TRACE_DIR/.all_uniq" + +TUPLE_COUNT=$((`grep -c . "$TRACE_DIR/.all_uniq"`)) + +echo "[+] Found $TUPLE_COUNT unique tuples across $IN_COUNT files." + +##################################### +# STEP 3: SELECTING CANDIDATE FILES # +##################################### + +# The next step is to find the best candidate for each tuple. The "best" +# part is understood simply as the smallest input that includes a particular +# tuple in its trace. Empirical evidence suggests that this produces smaller +# datasets than more involved algorithms that could be still pulled off in +# a shell script. + +echo "[*] Finding best candidates for each tuple..." + +CUR=0 + +ls -rS "$IN_DIR" | while read -r fn; do + + CUR=$((CUR+1)) + printf "\\r Processing file $CUR/$IN_COUNT... " + + sed "s#\$# $fn#" "$TRACE_DIR/$fn" >>"$TRACE_DIR/.candidate_list" + +done + +echo + +############################## +# STEP 4: LOADING CANDIDATES # +############################## + +# At this point, we have a file of tuple-file pairs, sorted by file size +# in ascending order (as a consequence of ls -rS). By doing sort keyed +# only by tuple (-k 1,1) and configured to output only the first line for +# every key (-s -u), we end up with the smallest file for each tuple. + +echo "[*] Sorting candidate list (be patient)..." + +sort -k1,1 -s -u "$TRACE_DIR/.candidate_list" | \ + sed 's/^/BEST_FILE[/;s/ /]="/;s/$/"/' >"$TRACE_DIR/.candidate_script" + +if [ ! -s "$TRACE_DIR/.candidate_script" ]; then + echo "[-] Error: no traces obtained from test cases, check syntax!" 1>&2 + test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR" + exit 1 +fi + +# The sed command converted the sorted list to a shell script that populates +# BEST_FILE[tuple]="fname". Let's load that! + +. "$TRACE_DIR/.candidate_script" + +########################## +# STEP 5: WRITING OUTPUT # +########################## + +# The final trick is to grab the top pick for each tuple, unless said tuple is +# already set due to the inclusion of an earlier candidate; and then put all +# tuples associated with the newly-added file to the "already have" list. The +# loop works from least popular tuples and toward the most common ones. + +echo "[*] Processing candidates and writing output files..." + +CUR=0 + +touch "$TRACE_DIR/.already_have" + +while read -r cnt tuple; do + + CUR=$((CUR+1)) + printf "\\r Processing tuple $CUR/$TUPLE_COUNT... " + + # If we already have this tuple, skip it. + + grep -q "^$tuple\$" "$TRACE_DIR/.already_have" && continue + + FN=${BEST_FILE[tuple]} + + $CP_TOOL "$IN_DIR/$FN" "$OUT_DIR/$FN" + + if [ "$((CUR % 5))" = "0" ]; then + sort -u "$TRACE_DIR/$FN" "$TRACE_DIR/.already_have" >"$TRACE_DIR/.tmp" + mv -f "$TRACE_DIR/.tmp" "$TRACE_DIR/.already_have" + else + cat "$TRACE_DIR/$FN" >>"$TRACE_DIR/.already_have" + fi + +done <"$TRACE_DIR/.all_uniq" + +echo + +OUT_COUNT=`ls -- "$OUT_DIR" | wc -l` + +if [ "$OUT_COUNT" = "1" ]; then + echo "[!] WARNING: All test cases had the same traces, check syntax!" +fi + +echo "[+] Narrowed down to $OUT_COUNT files, saved in '$OUT_DIR'." +echo + +test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR" + +exit 0 -- cgit 1.4.1 From 9da167dffdc14468d17ac3c1c942e483baf17433 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 22 Jan 2020 21:08:47 +0100 Subject: fix for modern linux --- afl-cmin.awk | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/afl-cmin.awk b/afl-cmin.awk index 28c460e8..fcdfb71f 100755 --- a/afl-cmin.awk +++ b/afl-cmin.awk @@ -109,7 +109,7 @@ function exists_and_is_executable(binarypath) { } BEGIN { - print "corpus minimization tool for afl-fuzz++ (awk version)\n" + print "corpus minimization tool for afl++ (awk version)\n" # defaults extra_par = "" @@ -289,7 +289,7 @@ BEGIN { } else { "which afl-showmap 2>/dev/null" | getline path } - showmap = path + showmap = path "/afl-showmap" } else { showmap = ENVIRON["AFL_PATH"] "/afl-showmap" } @@ -303,11 +303,12 @@ BEGIN { i = 0 # yuck, gnu stat is incompatible to bsd stat if ("stat --version 2>/dev/null" !~ /GNU coreutils/) { - stat_format = "-f '%z %N'" - } else { + # I dont get it why this does not work, output is "stat (GNU coreutils) 8.30" and still it goes here ... stat_format = "-c '%s %n'" + } else { + stat_format = "-f '%z %N'" } - while ("cd "in_dir" && find . -type f -exec stat "stat_format" \{\} \\; | sort -n | cut -d' ' -f2-" | getline) { + while ("cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" | getline) { infilesSmallToBig[i++] = $0 } in_count = i -- cgit 1.4.1 From c51f89b58e56338a5a430344548d1385432d173e Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Wed, 22 Jan 2020 21:50:35 +0100 Subject: rectification of vanhauser's fix, made it a bit more robust, enabled error output for travis debugging --- afl-cmin.awk | 40 ++++++++++++++++++++++++++++++++-------- test/test.sh | 2 +- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/afl-cmin.awk b/afl-cmin.awk index fcdfb71f..967c4e87 100755 --- a/afl-cmin.awk +++ b/afl-cmin.awk @@ -1,5 +1,28 @@ #!/usr/bin/awk -f +# awk script to minimize a test corpus of input files +# +# based on afl-cmin bash script written by Michal Zalewski +# rewritten by Heiko Eißfeldt (hexcoder-) +# +# uses getopt.awk package from Arnold Robbins +# +# external tools used by this script: +# test +# grep +# rm +# mkdir +# ln +# cp +# pwd +# which +# cd +# find +# stat +# sort +# cut +# and afl-showmap from this project :-) + # getopt.awk --- Do C library getopt(3) function in awk # External variables: @@ -285,11 +308,10 @@ BEGIN { if (!ENVIRON["AFL_PATH"]) { if (0 == system("test -f afl-cmin.awk")) { - path = "." + showmap = "./afl-showmap" } else { - "which afl-showmap 2>/dev/null" | getline path + "which afl-showmap 2>/dev/null" | getline showmap } - showmap = path "/afl-showmap" } else { showmap = ENVIRON["AFL_PATH"] "/afl-showmap" } @@ -301,12 +323,14 @@ BEGIN { # get list of input filenames sorted by size i = 0 - # yuck, gnu stat is incompatible to bsd stat - if ("stat --version 2>/dev/null" !~ /GNU coreutils/) { - # I dont get it why this does not work, output is "stat (GNU coreutils) 8.30" and still it goes here ... - stat_format = "-c '%s %n'" + # yuck, gnu stat is option incompatible to bsd stat + # we use a heuristic to differentiate between + # GNU stat and other stats + "stat --version 2>/dev/null" | getline statversion + if (statversion ~ /GNU coreutils/) { + stat_format = "-c '%s %n'" # GNU } else { - stat_format = "-f '%z %N'" + stat_format = "-f '%z %N'" # *BSD, MacOS } while ("cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" | getline) { infilesSmallToBig[i++] = $0 diff --git a/test/test.sh b/test/test.sh index 0ae6fd09..cc7fe224 100755 --- a/test/test.sh +++ b/test/test.sh @@ -150,7 +150,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { } echo 000000000000000000000000 > in/in2 mkdir -p in2 - ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null 2>&1 + ../afl-cmin -i in -o in2 -- ./test-instr.plain CNT=`ls in2/ | wc -l` case "$CNT" in 1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized testcase numbers" ;; -- cgit 1.4.1 From 7e7ab8f5415409fd1bb643f4dfef44c5a3935006 Mon Sep 17 00:00:00 2001 From: hexcoder Date: Wed, 22 Jan 2020 22:24:00 +0100 Subject: Update binaryonly_fuzzing.txt --- docs/binaryonly_fuzzing.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/binaryonly_fuzzing.txt b/docs/binaryonly_fuzzing.txt index 239fb4b0..f8d68cd8 100644 --- a/docs/binaryonly_fuzzing.txt +++ b/docs/binaryonly_fuzzing.txt @@ -5,10 +5,10 @@ Fuzzing binary-only programs with afl++ afl++, libfuzzer and others are great if you have the source code, and it allows for very fast and coverage guided fuzzing. -However, if there is only the binary program and not source code available, -then standard afl++ (dumb mode) is not effective. +However, if there is only the binary program and no source code available, +then standard `afl-fuzz -n` (dumb mode) is not effective. -The following is a description of how these can be fuzzed with afl++ +The following is a description of how these binaries can be fuzzed with afl++ !!!!! TL;DR: try DYNINST with afl-dyninst. If it produces too many crashes then @@ -28,7 +28,7 @@ As it is included in afl++ this needs no URL. WINE+QEMU --------- -Wine mode can run Win32 PE with the QEMU instrumentation. +Wine mode can run Win32 PE binaries with the QEMU instrumentation. It needs Wine, python3 and the pefile python package installed. UNICORN @@ -37,7 +37,7 @@ Unicorn is a fork of QEMU. The instrumentation is, therefore, very similar. In contrast to QEMU, Unicorn does not offer a full system or even userland emulation. Runtime environment and/or loaders have to be written from scratch, if needed. On top, block chaining has been removed. This means the speed boost introduced in -to the patched QEMU Mode of afl++ cannot simply be ported over to Unicorn. +the patched QEMU Mode of afl++ cannot simply be ported over to Unicorn. For further information, check out ./unicorn_mode.txt. -- cgit 1.4.1 From c490b9aa3694ba9c33ba0657ddd5e19dd979f2ed Mon Sep 17 00:00:00 2001 From: hexcoder Date: Thu, 23 Jan 2020 09:11:35 +0100 Subject: afl-cmin debugging is done now, so suppress stdout messages again (but not stderr) --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index cc7fe224..3473155f 100755 --- a/test/test.sh +++ b/test/test.sh @@ -150,7 +150,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { } echo 000000000000000000000000 > in/in2 mkdir -p in2 - ../afl-cmin -i in -o in2 -- ./test-instr.plain + ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null CNT=`ls in2/ | wc -l` case "$CNT" in 1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized testcase numbers" ;; -- cgit 1.4.1 From 8b17cac71c5196bae11a5a7ee8f6a17bdb3917e0 Mon Sep 17 00:00:00 2001 From: hexcoder Date: Thu, 23 Jan 2020 09:46:07 +0100 Subject: add socket_fuzz description --- experimental/README.experiments | 3 +++ 1 file changed, 3 insertions(+) diff --git a/experimental/README.experiments b/experimental/README.experiments index af9739bd..5a505ad7 100644 --- a/experimental/README.experiments +++ b/experimental/README.experiments @@ -28,6 +28,9 @@ Here's a quick overview of the stuff you can find in this directory: mode to speed up certain fuzzing jobs. - post_library - an example of how to build postprocessors for AFL. + + - socketfuzz - a LD_PRELOAD library 'redirects' a socket to stdin + for fuzzing access with afl++ Note that the minimize_corpus.sh tool has graduated from the experimental/ directory and is now available as ../afl-cmin. The LLVM mode has likewise -- cgit 1.4.1 From a58800b90122f3d612a0badb243d2c1b6fc9c742 Mon Sep 17 00:00:00 2001 From: hexcoder Date: Thu, 23 Jan 2020 09:46:59 +0100 Subject: typo --- experimental/README.experiments | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experimental/README.experiments b/experimental/README.experiments index 5a505ad7..543c078c 100644 --- a/experimental/README.experiments +++ b/experimental/README.experiments @@ -29,7 +29,7 @@ Here's a quick overview of the stuff you can find in this directory: - post_library - an example of how to build postprocessors for AFL. - - socketfuzz - a LD_PRELOAD library 'redirects' a socket to stdin + - socket_fuzzing - a LD_PRELOAD library 'redirects' a socket to stdin for fuzzing access with afl++ Note that the minimize_corpus.sh tool has graduated from the experimental/ -- cgit 1.4.1 From e7c95ebf5a4828b662252b10052a89923dd25030 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 23 Jan 2020 10:15:33 +0100 Subject: afl-cmin final touches --- Makefile | 2 +- afl-cmin | 473 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- afl-cmin.awk | 470 -------------------------------------------------------- docs/ChangeLog | 2 + test/test.sh | 2 +- 5 files changed, 476 insertions(+), 473 deletions(-) delete mode 100755 afl-cmin.awk diff --git a/Makefile b/Makefile index 7260ee47..459cae5f 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ VERSION = $(shell grep '^\#define VERSION ' ../config.h | cut -d '"' -f2) # PROGS intentionally omit afl-as, which gets installed elsewhere. PROGS = afl-gcc afl-fuzz afl-showmap afl-tmin afl-gotcpu afl-analyze -SH_PROGS = afl-plot afl-cmin afl-whatsup afl-system-config +SH_PROGS = afl-plot afl-cmin afl-cmin.bash afl-whatsup afl-system-config MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8) ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto=full -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" diff --git a/afl-cmin b/afl-cmin index 75dc63a7..a072a62a 100755 --- a/afl-cmin +++ b/afl-cmin @@ -1,4 +1,475 @@ #!/usr/bin/env sh THISPATH=`dirname ${0}` export PATH=${THISPATH}:$PATH -awk -f ${0}.awk -- ${@+"$@"} +awk -f - -- ${@+"$@"} <<'EOF' +#!/usr/bin/awk -f + +# awk script to minimize a test corpus of input files +# +# based on afl-cmin bash script written by Michal Zalewski +# rewritten by Heiko Eißfeldt (hexcoder-) +# +# uses getopt.awk package from Arnold Robbins +# +# external tools used by this script: +# test +# grep +# rm +# mkdir +# ln +# cp +# pwd +# which +# cd +# find +# stat +# sort +# cut +# and afl-showmap from this project :-) + +# getopt.awk --- Do C library getopt(3) function in awk + +# External variables: +# Optind -- index in ARGV of first nonoption argument +# Optarg -- string value of argument to current option +# Opterr -- if nonzero, print our own diagnostic +# Optopt -- current option letter + +# Returns: +# -1 at end of options +# "?" for unrecognized option +# a character representing the current option + +# Private Data: +# _opti -- index in multiflag option, e.g., -abc + +function getopt(argc, argv, options, thisopt, i) +{ + if (length(options) == 0) # no options given + return -1 + + if (argv[Optind] == "--") { # all done + Optind++ + _opti = 0 + return -1 + } else if (argv[Optind] !~ /^-[^:[:space:]]/) { + _opti = 0 + return -1 + } + if (_opti == 0) + _opti = 2 + thisopt = substr(argv[Optind], _opti, 1) + Optopt = thisopt + i = index(options, thisopt) + if (i == 0) { + if (Opterr) + printf("%c -- invalid option\n", thisopt) > "/dev/stderr" + if (_opti >= length(argv[Optind])) { + Optind++ + _opti = 0 + } else + _opti++ + return "?" + } + if (substr(options, i + 1, 1) == ":") { + # get option argument + if (length(substr(argv[Optind], _opti + 1)) > 0) + Optarg = substr(argv[Optind], _opti + 1) + else + Optarg = argv[++Optind] + _opti = 0 + } else + Optarg = "" + if (_opti == 0 || _opti >= length(argv[Optind])) { + Optind++ + _opti = 0 + } else + _opti++ + return thisopt +} + +BEGIN { + Opterr = 1 # default is to diagnose + Optind = 1 # skip ARGV[0] + + # test program + if (_getopt_test) { + while ((_go_c = getopt(ARGC, ARGV, "ab:cd")) != -1) + printf("c = <%c>, Optarg = <%s>\n", + _go_c, Optarg) + printf("non-option arguments:\n") + for (; Optind < ARGC; Optind++) + printf("\tARGV[%d] = <%s>\n", + Optind, ARGV[Optind]) + } +} + +function usage() { + print \ +"Usage: afl-cmin [ options ] -- /path/to/target_app [ ... ]\n" \ +"\n" \ +"Required parameters:\n" \ +"\n" \ +" -i dir - input directory with starting corpus\n" \ +" -o dir - output directory for minimized files\n" \ +"\n" \ +"Execution control settings:\n" \ +"\n" \ +" -f file - location read by the fuzzed program (stdin)\n" \ +" -m megs - memory limit for child process ("mem_limit" MB)\n" \ +" -t msec - run time limit for child process (none)\n" \ +" -Q - use binary-only instrumentation (QEMU mode)\n" \ +" -U - use unicorn-based instrumentation (unicorn mode)\n" \ +"\n" \ +"Minimization settings:\n" \ +" -C - keep crashing inputs, reject everything else\n" \ +" -e - solve for edge coverage only, ignore hit counts\n" \ +"\n" \ +"For additional tips, please consult docs/README.md\n" \ +"\n" \ + > "/dev/stderr" + exit 1 +} + +function exists_and_is_executable(binarypath) { + return 0 == system("test -f "binarypath" -a -x "binarypath) +} + +BEGIN { + print "corpus minimization tool for afl++ (awk version)\n" + + # defaults + extra_par = "" + # process options + Opterr = 1 # default is to diagnose + Optind = 1 # skip ARGV[0] + while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eCQU?")) != -1) { + if (_go_c == "i") { + if (!Optarg) usage() + if (in_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + in_dir = Optarg + continue + } else + if (_go_c == "o") { + if (!Optarg) usage() + if (out_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + out_dir = Optarg + continue + } else + if (_go_c == "f") { + if (!Optarg) usage() + if (stdin_file) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + stdin_file = Optarg + continue + } else + if (_go_c == "m") { + if (!Optarg) usage() + if (mem_limit) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + mem_limit = Optarg + mem_limit_given = 1 + continue + } else + if (_go_c == "t") { + if (!Optarg) usage() + if (timeout) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + timeout = Optarg + continue + } else + if (_go_c == "C") { + ENVIRON["AFL_CMIN_CRASHES_ONLY"] = 1 + continue + } else + if (_go_c == "e") { + extra_par = extra_par " -e" + continue + } else + if (_go_c == "Q") { + if (qemu_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + extra_par = extra_par " -Q" + if ( !mem_limit_given ) mem_limit = "250" + qemu_mode = 1 + continue + } else + if (_go_c == "U") { + if (unicorn_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + extra_par = extra_par " -U" + if ( !mem_limit_given ) mem_limit = "250" + unicorn_mode = 1 + continue + } else + if (_go_c == "?") { + exit 1 + } else + usage() + } # while options + + if (!mem_limit) mem_limit = 200 + if (!timeout) timeout = "none" + + # get program args + i = 0 + prog_args_string = "" + for (; Optind < ARGC; Optind++) { + prog_args[i++] = ARGV[Optind] + if (i > 1) + prog_args_string = prog_args_string" "ARGV[Optind] + } + + # sanity checks + if (!prog_args[0] || !in_dir || !out_dir) usage() + + target_bin = prog_args[0] + + # Do a sanity check to discourage the use of /tmp, since we can't really + # handle this safely from an awk script. + + if (!ENVIRON["AFL_ALLOW_TMP"]) { + dirlist[0] = in_dir + dirlist[1] = target_bin + dirlist[2] = out_dir + dirlist[3] = stdin_file + "pwd" | getline dirlist[4] # current directory + for (dirind in dirlist) { + dir = dirlist[dirind] + + if (dir ~ /^(\/var)?\/tmp/) { + print "[-] Error: do not use this script in /tmp or /var/tmp." > "/dev/stderr" + exit 1 + } + } + delete dirlist + } + + # If @@ is specified, but there's no -f, let's come up with a temporary input + # file name. + + trace_dir = out_dir "/.traces" + + if (!stdin_file) { + found_atat = 0 + for (prog_args_ind in prog_args) { + if ("@@" == prog_args[prog_args_ind]) { + found_atat = 1 + break + } + } + if (found_atat) { + stdin_file = trace_dir "/.cur_input" + } + } + + # Check for obvious errors. + + if (mem_limit && mem_limit != "none" && mem_limit < 5) { + print "[-] Error: dangerously low memory limit." > "/dev/stderr" + exit 1 + } + + if (timeout && timeout != "none" && timeout < 10) { + print "[-] Error: dangerously low timeout." > "/dev/stderr" + exit 1 + } + + if (target_bin && !exists_and_is_executable(target_bin)) { + + "which "target_bin" 2>/dev/null" | getline tnew + if (!tnew || !exists_and_is_executable(tnew)) { + print "[-] Error: binary '"target_bin"' not found or not executable." > "/dev/stderr" + exit 1 + } + target_bin = tnew + } + + if (!ENVIRON["AFL_SKIP_BIN_CHECK"] && !qemu_mode && !unicorn_mode) { + if (0 != system( "grep -q __AFL_SHM_ID "target_bin )) { + print "[-] Error: binary '"target_bin"' doesn't appear to be instrumented." > "/dev/stderr" + exit 1 + } + } + + if (0 != system( "test -d "in_dir )) { + print "[-] Error: directory '"in_dir"' not found." > "/dev/stderr" + exit 1 + } + + if (0 == system( "test -d "in_dir"/queue" )) { + in_dir = in_dir "/queue" + } + + system("rm -rf "trace_dir" 2>/dev/null"); + system("rm "out_dir"/id[:_]* 2>/dev/null") + + if (0 == system( "test -d "out_dir" -a -e "out_dir"/*" )) { + print "[-] Error: directory '"out_dir"' exists and is not empty - delete it first." > "/dev/stderr" + exit 1 + } + + if (stdin_file) { + # truncate input file + printf "" > stdin_file + close( stdin_file ) + } + + if (!ENVIRON["AFL_PATH"]) { + if (0 == system("test -f afl-cmin.awk")) { + showmap = "./afl-showmap" + } else { + "which afl-showmap 2>/dev/null" | getline showmap + } + } else { + showmap = ENVIRON["AFL_PATH"] "/afl-showmap" + } + + if (!showmap || 0 != system("test -x "showmap )) { + print "[-] Error: can't find 'afl-showmap' - please set AFL_PATH." > "/dev/stderr" + exit 1 + } + + # get list of input filenames sorted by size + i = 0 + # yuck, gnu stat is option incompatible to bsd stat + # we use a heuristic to differentiate between + # GNU stat and other stats + "stat --version 2>/dev/null" | getline statversion + if (statversion ~ /GNU coreutils/) { + stat_format = "-c '%s %n'" # GNU + } else { + stat_format = "-f '%z %N'" # *BSD, MacOS + } + while ("cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" | getline) { + infilesSmallToBig[i++] = $0 + } + in_count = i + + first_file = infilesSmallToBig[0] + + # Make sure that we're not dealing with a directory. + + if (0 == system("test -d "in_dir"/"first_file)) { + print "[-] Error: The input directory contains subdirectories - please fix." > "/dev/stderr" + exit 1 + } + + # Check for the more efficient way to copy files... + if (0 != system("mkdir -p -m 0700 "trace_dir)) { + print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" + exit 1 + } + + if (0 == system("ln "in_dir"/"first_file" "trace_dir"/.link_test")) { + cp_tool = "ln" + } else { + cp_tool = "cp" + } + + # Make sure that we can actually get anything out of afl-showmap before we + # waste too much time. + + print "[*] Testing the target binary..." + + if (!stdin_file) { + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") + } else { + system("cp "in_dir"/"first_file" "stdin_file) + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 0) { + ++first_count + } + + if (first_count) { + print "[+] OK, "first_count" tuples recorded." + } else { + print "[-] Error: no instrumentation output detected (perhaps crash or timeout)." > "/dev/stderr" + if (!ENVIRON["AFL_KEEP_TRACES"]) { + system("rm -rf "trace_dir" 2>/dev/null") + } + exit 1 + } + + # Let's roll! + + ############################# + # STEP 1: Collecting traces # + ############################# + + print "[*] Obtaining traces for "in_count" input files in '"in_dir"'." + + cur = 0; + if (!stdin_file) { + while (cur < in_count) { + fn = infilesSmallToBig[cur] + ++cur; + printf "\r Processing file "cur"/"in_count + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/"fn"\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"fn"\"") + } + } else { + while (cur < in_count) { + fn = infilesSmallToBig[cur] + ++cur + printf "\r Processing file "cur"/"in_count + system("cp "in_dir"/"fn" "stdin_file) + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/"fn"\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 0) { + key = line + if (!(key in key_count)) { + ++tuple_count + } + ++key_count[key] + if (! (key in best_file)) { + # this is the best file for this key + best_file[key] = fn + # copy file unless already done + if (! (fn in file_already_copied)) { + system(cp_tool" "in_dir"/"fn" "out_dir"/"fn) + file_already_copied[fn] = "" + ++out_count + } + } + } + close(tracefile_path) + } + + print "" + print "[+] Found "tuple_count" unique tuples across "in_count" files." + + if (out_count == 1) { + print "[!] WARNING: All test cases had the same traces, check syntax!" + } + print "[+] Narrowed down to "out_count" files, saved in '"out_dir"'." + + if (!ENVIRON["AFL_KEEP_TRACES"]) { + system("rm -rf "trace_dir" 2>/dev/null") + } + + exit 0 +} +EOF diff --git a/afl-cmin.awk b/afl-cmin.awk deleted file mode 100755 index 967c4e87..00000000 --- a/afl-cmin.awk +++ /dev/null @@ -1,470 +0,0 @@ -#!/usr/bin/awk -f - -# awk script to minimize a test corpus of input files -# -# based on afl-cmin bash script written by Michal Zalewski -# rewritten by Heiko Eißfeldt (hexcoder-) -# -# uses getopt.awk package from Arnold Robbins -# -# external tools used by this script: -# test -# grep -# rm -# mkdir -# ln -# cp -# pwd -# which -# cd -# find -# stat -# sort -# cut -# and afl-showmap from this project :-) - -# getopt.awk --- Do C library getopt(3) function in awk - -# External variables: -# Optind -- index in ARGV of first nonoption argument -# Optarg -- string value of argument to current option -# Opterr -- if nonzero, print our own diagnostic -# Optopt -- current option letter - -# Returns: -# -1 at end of options -# "?" for unrecognized option -# a character representing the current option - -# Private Data: -# _opti -- index in multiflag option, e.g., -abc - -function getopt(argc, argv, options, thisopt, i) -{ - if (length(options) == 0) # no options given - return -1 - - if (argv[Optind] == "--") { # all done - Optind++ - _opti = 0 - return -1 - } else if (argv[Optind] !~ /^-[^:[:space:]]/) { - _opti = 0 - return -1 - } - if (_opti == 0) - _opti = 2 - thisopt = substr(argv[Optind], _opti, 1) - Optopt = thisopt - i = index(options, thisopt) - if (i == 0) { - if (Opterr) - printf("%c -- invalid option\n", thisopt) > "/dev/stderr" - if (_opti >= length(argv[Optind])) { - Optind++ - _opti = 0 - } else - _opti++ - return "?" - } - if (substr(options, i + 1, 1) == ":") { - # get option argument - if (length(substr(argv[Optind], _opti + 1)) > 0) - Optarg = substr(argv[Optind], _opti + 1) - else - Optarg = argv[++Optind] - _opti = 0 - } else - Optarg = "" - if (_opti == 0 || _opti >= length(argv[Optind])) { - Optind++ - _opti = 0 - } else - _opti++ - return thisopt -} - -BEGIN { - Opterr = 1 # default is to diagnose - Optind = 1 # skip ARGV[0] - - # test program - if (_getopt_test) { - while ((_go_c = getopt(ARGC, ARGV, "ab:cd")) != -1) - printf("c = <%c>, Optarg = <%s>\n", - _go_c, Optarg) - printf("non-option arguments:\n") - for (; Optind < ARGC; Optind++) - printf("\tARGV[%d] = <%s>\n", - Optind, ARGV[Optind]) - } -} - -function usage() { - print \ -"Usage: afl-cmin [ options ] -- /path/to/target_app [ ... ]\n" \ -"\n" \ -"Required parameters:\n" \ -"\n" \ -" -i dir - input directory with starting corpus\n" \ -" -o dir - output directory for minimized files\n" \ -"\n" \ -"Execution control settings:\n" \ -"\n" \ -" -f file - location read by the fuzzed program (stdin)\n" \ -" -m megs - memory limit for child process ("mem_limit" MB)\n" \ -" -t msec - run time limit for child process (none)\n" \ -" -Q - use binary-only instrumentation (QEMU mode)\n" \ -" -U - use unicorn-based instrumentation (unicorn mode)\n" \ -"\n" \ -"Minimization settings:\n" \ -" -C - keep crashing inputs, reject everything else\n" \ -" -e - solve for edge coverage only, ignore hit counts\n" \ -"\n" \ -"For additional tips, please consult docs/README.md\n" \ -"\n" \ - > "/dev/stderr" - exit 1 -} - -function exists_and_is_executable(binarypath) { - return 0 == system("test -f "binarypath" -a -x "binarypath) -} - -BEGIN { - print "corpus minimization tool for afl++ (awk version)\n" - - # defaults - extra_par = "" - # process options - Opterr = 1 # default is to diagnose - Optind = 1 # skip ARGV[0] - while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eCQU?")) != -1) { - if (_go_c == "i") { - if (!Optarg) usage() - if (in_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - in_dir = Optarg - continue - } else - if (_go_c == "o") { - if (!Optarg) usage() - if (out_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - out_dir = Optarg - continue - } else - if (_go_c == "f") { - if (!Optarg) usage() - if (stdin_file) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - stdin_file = Optarg - continue - } else - if (_go_c == "m") { - if (!Optarg) usage() - if (mem_limit) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - mem_limit = Optarg - mem_limit_given = 1 - continue - } else - if (_go_c == "t") { - if (!Optarg) usage() - if (timeout) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - timeout = Optarg - continue - } else - if (_go_c == "C") { - ENVIRON["AFL_CMIN_CRASHES_ONLY"] = 1 - continue - } else - if (_go_c == "e") { - extra_par = extra_par " -e" - continue - } else - if (_go_c == "Q") { - if (qemu_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - extra_par = extra_par " -Q" - if ( !mem_limit_given ) mem_limit = "250" - qemu_mode = 1 - continue - } else - if (_go_c == "U") { - if (unicorn_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - extra_par = extra_par " -U" - if ( !mem_limit_given ) mem_limit = "250" - unicorn_mode = 1 - continue - } else - if (_go_c == "?") { - exit 1 - } else - usage() - } # while options - - if (!mem_limit) mem_limit = 200 - if (!timeout) timeout = "none" - - # get program args - i = 0 - prog_args_string = "" - for (; Optind < ARGC; Optind++) { - prog_args[i++] = ARGV[Optind] - if (i > 1) - prog_args_string = prog_args_string" "ARGV[Optind] - } - - # sanity checks - if (!prog_args[0] || !in_dir || !out_dir) usage() - - target_bin = prog_args[0] - - # Do a sanity check to discourage the use of /tmp, since we can't really - # handle this safely from an awk script. - - if (!ENVIRON["AFL_ALLOW_TMP"]) { - dirlist[0] = in_dir - dirlist[1] = target_bin - dirlist[2] = out_dir - dirlist[3] = stdin_file - "pwd" | getline dirlist[4] # current directory - for (dirind in dirlist) { - dir = dirlist[dirind] - - if (dir ~ /^(\/var)?\/tmp/) { - print "[-] Error: do not use this script in /tmp or /var/tmp." > "/dev/stderr" - exit 1 - } - } - delete dirlist - } - - # If @@ is specified, but there's no -f, let's come up with a temporary input - # file name. - - trace_dir = out_dir "/.traces" - - if (!stdin_file) { - found_atat = 0 - for (prog_args_ind in prog_args) { - if ("@@" == prog_args[prog_args_ind]) { - found_atat = 1 - break - } - } - if (found_atat) { - stdin_file = trace_dir "/.cur_input" - } - } - - # Check for obvious errors. - - if (mem_limit && mem_limit != "none" && mem_limit < 5) { - print "[-] Error: dangerously low memory limit." > "/dev/stderr" - exit 1 - } - - if (timeout && timeout != "none" && timeout < 10) { - print "[-] Error: dangerously low timeout." > "/dev/stderr" - exit 1 - } - - if (target_bin && !exists_and_is_executable(target_bin)) { - - "which "target_bin" 2>/dev/null" | getline tnew - if (!tnew || !exists_and_is_executable(tnew)) { - print "[-] Error: binary '"target_bin"' not found or not executable." > "/dev/stderr" - exit 1 - } - target_bin = tnew - } - - if (!ENVIRON["AFL_SKIP_BIN_CHECK"] && !qemu_mode && !unicorn_mode) { - if (0 != system( "grep -q __AFL_SHM_ID "target_bin )) { - print "[-] Error: binary '"target_bin"' doesn't appear to be instrumented." > "/dev/stderr" - exit 1 - } - } - - if (0 != system( "test -d "in_dir )) { - print "[-] Error: directory '"in_dir"' not found." > "/dev/stderr" - exit 1 - } - - if (0 == system( "test -d "in_dir"/queue" )) { - in_dir = in_dir "/queue" - } - - system("rm -rf "trace_dir" 2>/dev/null"); - system("rm "out_dir"/id[:_]* 2>/dev/null") - - if (0 == system( "test -d "out_dir" -a -e "out_dir"/*" )) { - print "[-] Error: directory '"out_dir"' exists and is not empty - delete it first." > "/dev/stderr" - exit 1 - } - - if (stdin_file) { - # truncate input file - printf "" > stdin_file - close( stdin_file ) - } - - if (!ENVIRON["AFL_PATH"]) { - if (0 == system("test -f afl-cmin.awk")) { - showmap = "./afl-showmap" - } else { - "which afl-showmap 2>/dev/null" | getline showmap - } - } else { - showmap = ENVIRON["AFL_PATH"] "/afl-showmap" - } - - if (!showmap || 0 != system("test -x "showmap )) { - print "[-] Error: can't find 'afl-showmap' - please set AFL_PATH." > "/dev/stderr" - exit 1 - } - - # get list of input filenames sorted by size - i = 0 - # yuck, gnu stat is option incompatible to bsd stat - # we use a heuristic to differentiate between - # GNU stat and other stats - "stat --version 2>/dev/null" | getline statversion - if (statversion ~ /GNU coreutils/) { - stat_format = "-c '%s %n'" # GNU - } else { - stat_format = "-f '%z %N'" # *BSD, MacOS - } - while ("cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" | getline) { - infilesSmallToBig[i++] = $0 - } - in_count = i - - first_file = infilesSmallToBig[0] - - # Make sure that we're not dealing with a directory. - - if (0 == system("test -d "in_dir"/"first_file)) { - print "[-] Error: The input directory contains subdirectories - please fix." > "/dev/stderr" - exit 1 - } - - # Check for the more efficient way to copy files... - if (0 != system("mkdir -p -m 0700 "trace_dir)) { - print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" - exit 1 - } - - if (0 == system("ln "in_dir"/"first_file" "trace_dir"/.link_test")) { - cp_tool = "ln" - } else { - cp_tool = "cp" - } - - # Make sure that we can actually get anything out of afl-showmap before we - # waste too much time. - - print "[*] Testing the target binary..." - - if (!stdin_file) { - system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") - } else { - system("cp "in_dir"/"first_file" "stdin_file) - system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 0) { - ++first_count - } - - if (first_count) { - print "[+] OK, "first_count" tuples recorded." - } else { - print "[-] Error: no instrumentation output detected (perhaps crash or timeout)." > "/dev/stderr" - if (!ENVIRON["AFL_KEEP_TRACES"]) { - system("rm -rf "trace_dir" 2>/dev/null") - } - exit 1 - } - - # Let's roll! - - ############################# - # STEP 1: Collecting traces # - ############################# - - print "[*] Obtaining traces for "in_count" input files in '"in_dir"'." - - cur = 0; - if (!stdin_file) { - while (cur < in_count) { - fn = infilesSmallToBig[cur] - ++cur; - printf "\r Processing file "cur"/"in_count - system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/"fn"\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"fn"\"") - } - } else { - while (cur < in_count) { - fn = infilesSmallToBig[cur] - ++cur - printf "\r Processing file "cur"/"in_count - system("cp "in_dir"/"fn" "stdin_file) - system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/"fn"\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 0) { - key = line - if (!(key in key_count)) { - ++tuple_count - } - ++key_count[key] - if (! (key in best_file)) { - # this is the best file for this key - best_file[key] = fn - # copy file unless already done - if (! (fn in file_already_copied)) { - system(cp_tool" "in_dir"/"fn" "out_dir"/"fn) - file_already_copied[fn] = "" - ++out_count - } - } - } - close(tracefile_path) - } - - print "" - print "[+] Found "tuple_count" unique tuples across "in_count" files." - - if (out_count == 1) { - print "[!] WARNING: All test cases had the same traces, check syntax!" - } - print "[+] Narrowed down to "out_count" files, saved in '"out_dir"'." - - if (!ENVIRON["AFL_KEEP_TRACES"]) { - system("rm -rf "trace_dir" 2>/dev/null") - } - - exit 0 -} diff --git a/docs/ChangeLog b/docs/ChangeLog index bb3537dd..33c6f618 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -25,6 +25,8 @@ Version ++2.60d (develop): - show in the help output for which llvm version it was compiled for - now does not need to be recompiled between trace-pc and pass instrumentation. compile normally and set AFL_LLVM_USE_TRACE_PC :) + - afl-cmin is now a sh script (invoking awk) instead of bash for portability + the original script is still present as afl-cmin.bash - added blacklisted function check in all modules of llvm_mode - added fix from Debian project to compile libdislocator and libtokencap diff --git a/test/test.sh b/test/test.sh index 3473155f..0ae6fd09 100755 --- a/test/test.sh +++ b/test/test.sh @@ -150,7 +150,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { } echo 000000000000000000000000 > in/in2 mkdir -p in2 - ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null + ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null 2>&1 CNT=`ls in2/ | wc -l` case "$CNT" in 1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized testcase numbers" ;; -- cgit 1.4.1 From 436873a19abe5858e56555db02095f4eb7e6febd Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 23 Jan 2020 11:55:53 +0100 Subject: show stderr on afl-cmin test.sh --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 0ae6fd09..3473155f 100755 --- a/test/test.sh +++ b/test/test.sh @@ -150,7 +150,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { } echo 000000000000000000000000 > in/in2 mkdir -p in2 - ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null 2>&1 + ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null CNT=`ls in2/ | wc -l` case "$CNT" in 1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized testcase numbers" ;; -- cgit 1.4.1 From 6abe33030396c8f15f00b4fe3d083f3841de3212 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Fri, 24 Jan 2020 20:58:15 +0100 Subject: afl-cmin more awk portability (mawk), add afl-cmin/afl-tmin tests for non-x86 platforms --- afl-cmin | 27 ++++++++------------------- test/test.sh | 26 +++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/afl-cmin b/afl-cmin index a072a62a..de5a66ed 100755 --- a/afl-cmin +++ b/afl-cmin @@ -8,6 +8,10 @@ awk -f - -- ${@+"$@"} <<'EOF' # # based on afl-cmin bash script written by Michal Zalewski # rewritten by Heiko Eißfeldt (hexcoder-) +# tested with: +# gnu awk (x86 Linux) +# bsd awk (x86 *BSD) +# mawk (arm32 raspbian) # # uses getopt.awk package from Arnold Robbins # @@ -52,7 +56,7 @@ function getopt(argc, argv, options, thisopt, i) Optind++ _opti = 0 return -1 - } else if (argv[Optind] !~ /^-[^:[:space:]]/) { + } else if (argv[Optind] !~ /^-[^:\t ]/) { _opti = 0 return -1 } @@ -88,22 +92,6 @@ function getopt(argc, argv, options, thisopt, i) return thisopt } -BEGIN { - Opterr = 1 # default is to diagnose - Optind = 1 # skip ARGV[0] - - # test program - if (_getopt_test) { - while ((_go_c = getopt(ARGC, ARGV, "ab:cd")) != -1) - printf("c = <%c>, Optarg = <%s>\n", - _go_c, Optarg) - printf("non-option arguments:\n") - for (; Optind < ARGC; Optind++) - printf("\tARGV[%d] = <%s>\n", - Optind, ARGV[Optind]) - } -} - function usage() { print \ "Usage: afl-cmin [ options ] -- /path/to/target_app [ ... ]\n" \ @@ -311,7 +299,7 @@ BEGIN { } if (!ENVIRON["AFL_PATH"]) { - if (0 == system("test -f afl-cmin.awk")) { + if (0 == system("test -f afl-cmin")) { showmap = "./afl-showmap" } else { "which afl-showmap 2>/dev/null" | getline showmap @@ -336,7 +324,8 @@ BEGIN { } else { stat_format = "-f '%z %N'" # *BSD, MacOS } - while ("cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" | getline) { + cmdline = "cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" + while (cmdline | getline) { infilesSmallToBig[i++] = $0 } in_count = i diff --git a/test/test.sh b/test/test.sh index 3473155f..93a4e008 100755 --- a/test/test.sh +++ b/test/test.sh @@ -153,8 +153,8 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null CNT=`ls in2/ | wc -l` case "$CNT" in -1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized testcase numbers" ;; -*) $ECHO "$RED[!] afl-cmin did not correctly minimize testcase numbers" +1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; +*) $ECHO "$RED[!] afl-cmin did not correctly minimizethe number of testcases" CODE=1 ;; esac @@ -176,7 +176,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { $ECHO "$YELLOW[-] not an intel platform, cannot test afl-gcc" } -$ECHO "$BLUE[*] Testing: llvm_mode" +$ECHO "$BLUE[*] Testing: llvm_mode, afl-showmap, afl-fuzz, afl-cmin and afl-tmin" test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { # on FreeBSD need to set AFL_CC test `uname -s` = 'FreeBSD' && { @@ -253,6 +253,26 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { $ECHO "$RED[!] afl-fuzz is not working correctly with llvm_mode" CODE=1 } + test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" || { + echo 000000000000000000000000 > in/in2 + mkdir -p in2 + ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null + CNT=`ls in2/ | wc -l` + case "$CNT" in +1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; +*) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases" + CODE=1 + ;; + esac + ../afl-tmin -i in/in2 -o in2/in2 -- ./test-instr.plain > /dev/null 2>&1 + SIZE=`ls -l in2/in2 2> /dev/null | awk '{print$5}'` + test "$SIZE" = 1 && $ECHO "$GREEN[+] afl-tmin correctly minimized the testcase" + test "$SIZE" = 1 || { + $ECHO "$RED[!] afl-tmin did incorrectly minimize the testcase to $SIZE" + CODE=1 + } + rm -rf in2 + } rm -rf in out errors } rm -f test-instr.plain -- cgit 1.4.1 From 5d2330f04e45225588a11c64b26a7dbb1a2fbe1a Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sat, 25 Jan 2020 05:27:10 +0100 Subject: nicer output for afl-system-config --- afl-system-config | 93 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 38 deletions(-) diff --git a/afl-system-config b/afl-system-config index 2a7df17f..1e180d8b 100755 --- a/afl-system-config +++ b/afl-system-config @@ -1,6 +1,6 @@ #!/bin/sh test "$1" = "-h" && { - echo afl-system-config by Marc Heuse + echo 'afl-system-config by Marc Heuse ' echo echo $0 echo @@ -12,55 +12,72 @@ test "$1" = "-h" && { exit 1 } +DONE= PLATFORM=`uname -s` -echo This reconfigures the system to have a better fuzzing performance +echo This reconfigures the system to have a better fuzzing performance. if [ '!' "$EUID" = 0 ] && [ '!' `id -u` = 0 ] ; then - echo Error you need to be root to run this - exit 1 + echo "Warning: you need to be root to run this!" + # we do not exit as other mechanisms exist that allows to do this than + # being root. let the errors speak for themselves. fi if [ "$PLATFORM" = "Linux" ] ; then -sysctl -w kernel.core_pattern=core -sysctl -w kernel.randomize_va_space=0 -sysctl -w kernel.sched_child_runs_first=1 -sysctl -w kernel.sched_autogroup_enabled=1 -sysctl -w kernel.sched_migration_cost_ns=50000000 -sysctl -w kernel.sched_latency_ns=250000000 -echo never > /sys/kernel/mm/transparent_hugepage/enabled -test -e /sys/devices/system/cpu/cpufreq/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpufreq/scaling_governor -test -e /sys/devices/system/cpu/cpufreq/policy0/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor -test -e /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -test -e /sys/devices/system/cpu/intel_pstate/no_turbo && echo 0 > /sys/devices/system/cpu/intel_pstate/no_turbo -test -e /sys/devices/system/cpu/cpufreq/boost && echo 1 > /sys/devices/system/cpu/cpufreq/boost -echo -echo It is recommended to boot the kernel with lots of security off - if you are running a machine that is in a secured network - so set this: -echo '/etc/default/grub:GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"' +{ + sysctl -w kernel.core_pattern=core + sysctl -w kernel.randomize_va_space=0 + sysctl -w kernel.sched_child_runs_first=1 + sysctl -w kernel.sched_autogroup_enabled=1 + sysctl -w kernel.sched_migration_cost_ns=50000000 + sysctl -w kernel.sched_latency_ns=250000000 + echo never > /sys/kernel/mm/transparent_hugepage/enabled + test -e /sys/devices/system/cpu/cpufreq/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpufreq/scaling_governor + test -e /sys/devices/system/cpu/cpufreq/policy0/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor + test -e /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + test -e /sys/devices/system/cpu/intel_pstate/no_turbo && echo 0 > /sys/devices/system/cpu/intel_pstate/no_turbo + test -e /sys/devices/system/cpu/cpufreq/boost && echo 1 > /sys/devices/system/cpu/cpufreq/boost +} > /dev/null + echo Settings applied. + dmesg | egrep -q 'nospectre_v2|spectre_v2=off' || { + echo It is recommended to boot the kernel with lots of security off - if you are running a machine that is in a secured network - so set this: + echo ' /etc/default/grub:GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"' + } + DONE=1 fi if [ "$PLATFORM" = "FreeBSD" ] ; then -sysctl kern.elf32.aslr.enable=0 -sysctl kern.elf64.aslr.enable=0 -echo -echo It is recommended to boot the kernel with lots of security off - if you are running a machine that is in a secured network - so set this: -echo 'sysctl hw.ibrs_disable=1' -echo -echo 'Setting kern.pmap.pg_ps_enabled=0 into /boot/loader.conf might be helpful too.' +{ + sysctl kern.elf32.aslr.enable=0 + sysctl kern.elf64.aslr.enable=0 +} > /dev/null + echo Settings applied. + echo It is recommended to boot the kernel with lots of security off - if you are running a machine that is in a secured network - so set this: + echo ' sysctl hw.ibrs_disable=1' + echo 'Setting kern.pmap.pg_ps_enabled=0 into /boot/loader.conf might be helpful too.' + DONE=1 fi if [ "$PLATFORM" = "OpenBSD" ] ; then -echo -echo 'System security features cannot be disabled on OpenBSD.' + echo + echo 'System security features cannot be disabled on OpenBSD.' + DONE=1 fi if [ "$PLATFORM" = "NetBSD" ] ; then -echo -echo It is recommended to enable unprivileged users to set cpu affinity -echo to be able to use afl-gotcpu meaningfully. -/sbin/sysctl -w security.models.extensions.user_set_cpu_affinity=1 +{ + #echo It is recommended to enable unprivileged users to set cpu affinity + #echo to be able to use afl-gotcpu meaningfully. + /sbin/sysctl -w security.models.extensions.user_set_cpu_affinity=1 +} > /dev/null + echo Settings applied. + DONE=1 fi if [ "$PLATFORM" = "Darwin" ] ; then if [ $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') ] ; then -echo We unload the default crash reporter here -SL=/System/Library; PL=com.apple.ReportCrash -launchctl unload -w ${SL}/LaunchAgents/${PL}.plist -sudo launchctl unload -w ${SL}/LaunchDaemons/${PL}.Root.plist + echo We unload the default crash reporter here + SL=/System/Library; PL=com.apple.ReportCrash + launchctl unload -w ${SL}/LaunchAgents/${PL}.plist + sudo launchctl unload -w ${SL}/LaunchDaemons/${PL}.Root.plist + echo Settings applied. + else + echo Nothing to do. fi + DONE=1 fi -echo -echo Also use AFL_TMPDIR to use a tmpfs for the input file +test -z "$DONE" && echo Error: Unknown platform: $PLATFORM +test -z "$AFL_TMPDIR" && echo Also use AFL_TMPDIR and point it to a tmpfs for the input file caching -- cgit 1.4.1 From 2c6847bfa0b57f3330b1aab9b91d935757db51b7 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sat, 25 Jan 2020 16:11:42 +0100 Subject: added whitelist+blacklist to all llvm_mode passes --- docs/ChangeLog | 2 +- llvm_mode/LLVMInsTrim.so.cc | 29 +------- llvm_mode/MarkNodes.cc | 19 ++---- llvm_mode/compare-transform-pass.so.cc | 94 ++++++++++++++++++++++++++ llvm_mode/split-compares-pass.so.cc | 118 +++++++++++++++++++++++++++++++++ llvm_mode/split-switches-pass.so.cc | 113 +++++++++++++++++++++++++++++++ test/test.sh | 16 ++--- 7 files changed, 343 insertions(+), 48 deletions(-) diff --git a/docs/ChangeLog b/docs/ChangeLog index 33c6f618..c1d53379 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -27,7 +27,7 @@ Version ++2.60d (develop): instrumentation. compile normally and set AFL_LLVM_USE_TRACE_PC :) - afl-cmin is now a sh script (invoking awk) instead of bash for portability the original script is still present as afl-cmin.bash - - added blacklisted function check in all modules of llvm_mode + - added blacklist and whitelisting function check in all modules of llvm_mode - added fix from Debian project to compile libdislocator and libtokencap diff --git a/llvm_mode/LLVMInsTrim.so.cc b/llvm_mode/LLVMInsTrim.so.cc index 11451b43..24df6d42 100644 --- a/llvm_mode/LLVMInsTrim.so.cc +++ b/llvm_mode/LLVMInsTrim.so.cc @@ -144,19 +144,6 @@ struct InsTrim : public ModulePass { // this is our default MarkSetOpt = true; - /* // I dont think this makes sense to port into LLVMInsTrim - char* inst_ratio_str = getenv("AFL_INST_RATIO"); - unsigned int inst_ratio = 100; - if (inst_ratio_str) { - - if (sscanf(inst_ratio_str, "%u", &inst_ratio) != 1 || !inst_ratio || - inst_ratio > 100) FATAL("Bad value of AFL_INST_RATIO (must be between 1 - and 100)"); - - } - - */ - LLVMContext &C = M.getContext(); IntegerType *Int8Ty = IntegerType::getInt8Ty(C); IntegerType *Int32Ty = IntegerType::getInt32Ty(C); @@ -203,8 +190,7 @@ struct InsTrim : public ModulePass { if (instFilename.str().empty()) { - /* If the original location is empty, try using the inlined location - */ + /* If the original location is empty, try using the inlined location */ DILocation *oDILoc = cDILoc->getInlinedAt(); if (oDILoc) { @@ -432,28 +418,19 @@ struct InsTrim : public ModulePass { IRB.CreateStore(Incr, MapPtrIdx) ->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - /* Set prev_loc to cur_loc >> 1 */ - /* - StoreInst *Store = IRB.CreateStore(ConstantInt::get(Int32Ty, L >> 1), - OldPrev); Store->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, - None)); - */ - total_instr++; } } - OKF("Instrumented %u locations (%llu, %llu) (%s mode)\n" /*", ratio - %u%%)."*/ - , + OKF("Instrumented %u locations (%llu, %llu) (%s mode)\n", total_instr, total_rs, total_hs, getenv("AFL_HARDEN") ? "hardened" : ((getenv("AFL_USE_ASAN") || getenv("AFL_USE_MSAN")) ? "ASAN/MSAN" - : "non-hardened") /*, inst_ratio*/); + : "non-hardened")); return false; } diff --git a/llvm_mode/MarkNodes.cc b/llvm_mode/MarkNodes.cc index 2aeeda8d..caa8cede 100644 --- a/llvm_mode/MarkNodes.cc +++ b/llvm_mode/MarkNodes.cc @@ -65,16 +65,11 @@ void buildCFG(Function *F) { } - // uint32_t FakeID = 0; for (auto S = F->begin(), E = F->end(); S != E; ++S) { BasicBlock *BB = &*S; uint32_t MyID = LMap[BB]; - // if (succ_begin(BB) == succ_end(BB)) { - // Succs[MyID].push_back(FakeID); - // Marked.insert(MyID); - //} for (auto I = succ_begin(BB), E = succ_end(BB); I != E; ++I) { Succs[MyID].push_back(LMap[*I]); @@ -113,7 +108,7 @@ void DFStree(size_t now_id) { } -void turnCFGintoDAG(Function *F) { +void turnCFGintoDAG() { tSuccs = Succs; tag.resize(Blocks.size()); @@ -176,7 +171,7 @@ void DFS(uint32_t now) { } -void DominatorTree(Function *F) { +void DominatorTree() { if (Blocks.empty()) return; uint32_t s = start_point; @@ -390,7 +385,7 @@ void MarkSubGraph(uint32_t ss, uint32_t tt) { } -void MarkVertice(Function *F) { +void MarkVertice() { uint32_t s = start_point; @@ -411,8 +406,6 @@ void MarkVertice(Function *F) { timeStamp = 0; uint32_t t = 0; - // MarkSubGraph(s, t); - // return; while (s != t) { @@ -432,9 +425,9 @@ std::pair, std::vector > markNodes( reset(); labelEachBlock(F); buildCFG(F); - turnCFGintoDAG(F); - DominatorTree::DominatorTree(F); - MarkVertice(F); + turnCFGintoDAG(); + DominatorTree::DominatorTree(); + MarkVertice(); std::vector Result, ResultAbove; for (uint32_t x : Markabove) { diff --git a/llvm_mode/compare-transform-pass.so.cc b/llvm_mode/compare-transform-pass.so.cc index 0ccce875..5d924b63 100644 --- a/llvm_mode/compare-transform-pass.so.cc +++ b/llvm_mode/compare-transform-pass.so.cc @@ -18,7 +18,13 @@ #include #include +#include +#include +#include +#include + #include "llvm/ADT/Statistic.h" +#include "llvm/IR/DebugInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" @@ -42,6 +48,23 @@ class CompareTransform : public ModulePass { static char ID; CompareTransform() : ModulePass(ID) { + char *instWhiteListFilename = getenv("AFL_LLVM_WHITELIST"); + if (instWhiteListFilename) { + + std::string line; + std::ifstream fileStream; + fileStream.open(instWhiteListFilename); + if (!fileStream) report_fatal_error("Unable to open AFL_LLVM_WHITELIST"); + getline(fileStream, line); + while (fileStream) { + + myWhitelist.push_back(line); + getline(fileStream, line); + + } + + } + } bool runOnModule(Module &M) override; @@ -57,6 +80,9 @@ class CompareTransform : public ModulePass { } + protected: + std::list myWhitelist; + private: bool transformCmps(Module &M, const bool processStrcmp, const bool processMemcmp, const bool processStrncmp, @@ -104,6 +130,74 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, for (auto &BB : F) { + if (!myWhitelist.empty()) { + + BasicBlock::iterator IP = BB.getFirstInsertionPt(); + + bool instrumentBlock = false; + + /* Get the current location using debug information. + * For now, just instrument the block if we are not able + * to determine our location. */ + DebugLoc Loc = IP->getDebugLoc(); + if (Loc) { + + DILocation *cDILoc = dyn_cast(Loc.getAsMDNode()); + + unsigned int instLine = cDILoc->getLine(); + StringRef instFilename = cDILoc->getFilename(); + + if (instFilename.str().empty()) { + + /* If the original location is empty, try using the inlined location + */ + DILocation *oDILoc = cDILoc->getInlinedAt(); + if (oDILoc) { + + instFilename = oDILoc->getFilename(); + instLine = oDILoc->getLine(); + + } + + } + + (void)instLine; + + /* Continue only if we know where we actually are */ + if (!instFilename.str().empty()) { + + for (std::list::iterator it = myWhitelist.begin(); + it != myWhitelist.end(); ++it) { + + /* We don't check for filename equality here because + * filenames might actually be full paths. Instead we + * check that the actual filename ends in the filename + * specified in the list. */ + if (instFilename.str().length() >= it->length()) { + + if (instFilename.str().compare( + instFilename.str().length() - it->length(), + it->length(), *it) == 0) { + + instrumentBlock = true; + break; + + } + + } + + } + + } + + } + + /* Either we couldn't figure out our location or the location is + * not whitelisted, so we skip instrumentation. */ + if (!instrumentBlock) continue; + + } + for (auto &IN : BB) { CallInst *callInst = nullptr; diff --git a/llvm_mode/split-compares-pass.so.cc b/llvm_mode/split-compares-pass.so.cc index eeac4a55..bc25b322 100644 --- a/llvm_mode/split-compares-pass.so.cc +++ b/llvm_mode/split-compares-pass.so.cc @@ -15,7 +15,17 @@ * limitations under the License. */ +#include +#include +#include + +#include +#include +#include +#include + #include "llvm/Pass.h" +#include "llvm/IR/DebugInfo.h" #include "llvm/Support/raw_ostream.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" @@ -35,6 +45,41 @@ class SplitComparesTransform : public ModulePass { static char ID; SplitComparesTransform() : ModulePass(ID) { + char *instWhiteListFilename = getenv("AFL_LLVM_WHITELIST"); + if (instWhiteListFilename) { + + std::string line; + std::ifstream fileStream; + fileStream.open(instWhiteListFilename); + if (!fileStream) report_fatal_error("Unable to open AFL_LLVM_WHITELIST"); + getline(fileStream, line); + while (fileStream) { + + myWhitelist.push_back(line); + getline(fileStream, line); + + } + + } + + } + + static bool isBlacklisted(const Function *F) { + + static const SmallVector Blacklist = { + + "asan.", "llvm.", "sancov.", "__ubsan_handle_", "ign." + + }; + + for (auto const &BlacklistFunc : Blacklist) { + + if (F->getName().startswith(BlacklistFunc)) { return true; } + + } + + return false; + } bool runOnModule(Module &M) override; @@ -49,6 +94,9 @@ class SplitComparesTransform : public ModulePass { } + protected: + std::list myWhitelist; + private: int enableFPSplit; @@ -77,8 +125,78 @@ bool SplitComparesTransform::simplifyCompares(Module &M) { * all integer comparisons with >= and <= predicates to the icomps vector */ for (auto &F : M) { + if (isBlacklisted(&F)) continue; + for (auto &BB : F) { + if (!myWhitelist.empty()) { + + bool instrumentBlock = false; + + BasicBlock::iterator IP = BB.getFirstInsertionPt(); + + /* Get the current location using debug information. + * For now, just instrument the block if we are not able + * to determine our location. */ + DebugLoc Loc = IP->getDebugLoc(); + if (Loc) { + + DILocation *cDILoc = dyn_cast(Loc.getAsMDNode()); + + unsigned int instLine = cDILoc->getLine(); + StringRef instFilename = cDILoc->getFilename(); + + if (instFilename.str().empty()) { + + /* If the original location is empty, try using the inlined location + */ + DILocation *oDILoc = cDILoc->getInlinedAt(); + if (oDILoc) { + + instFilename = oDILoc->getFilename(); + instLine = oDILoc->getLine(); + + } + + } + + (void)instLine; + + /* Continue only if we know where we actually are */ + if (!instFilename.str().empty()) { + + for (std::list::iterator it = myWhitelist.begin(); + it != myWhitelist.end(); ++it) { + + /* We don't check for filename equality here because + * filenames might actually be full paths. Instead we + * check that the actual filename ends in the filename + * specified in the list. */ + if (instFilename.str().length() >= it->length()) { + + if (instFilename.str().compare( + instFilename.str().length() - it->length(), + it->length(), *it) == 0) { + + instrumentBlock = true; + break; + + } + + } + + } + + } + + } + + /* Either we couldn't figure out our location or the location is + * not whitelisted, so we skip instrumentation. */ + if (!instrumentBlock) continue; + + } + for (auto &IN : BB) { CmpInst *selectcmpInst = nullptr; diff --git a/llvm_mode/split-switches-pass.so.cc b/llvm_mode/split-switches-pass.so.cc index 2743a71a..3a2838c0 100644 --- a/llvm_mode/split-switches-pass.so.cc +++ b/llvm_mode/split-switches-pass.so.cc @@ -18,7 +18,13 @@ #include #include +#include +#include +#include +#include + #include "llvm/ADT/Statistic.h" +#include "llvm/IR/DebugInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" @@ -42,6 +48,41 @@ class SplitSwitchesTransform : public ModulePass { static char ID; SplitSwitchesTransform() : ModulePass(ID) { + char *instWhiteListFilename = getenv("AFL_LLVM_WHITELIST"); + if (instWhiteListFilename) { + + std::string line; + std::ifstream fileStream; + fileStream.open(instWhiteListFilename); + if (!fileStream) report_fatal_error("Unable to open AFL_LLVM_WHITELIST"); + getline(fileStream, line); + while (fileStream) { + + myWhitelist.push_back(line); + getline(fileStream, line); + + } + + } + + } + + static bool isBlacklisted(const Function *F) { + + static const SmallVector Blacklist = { + + "asan.", "llvm.", "sancov.", "__ubsan_handle_", "ign." + + }; + + for (auto const &BlacklistFunc : Blacklist) { + + if (F->getName().startswith(BlacklistFunc)) { return true; } + + } + + return false; + } bool runOnModule(Module &M) override; @@ -71,6 +112,9 @@ class SplitSwitchesTransform : public ModulePass { typedef std::vector CaseVector; + protected: + std::list myWhitelist; + private: bool splitSwitches(Module &M); bool transformCmps(Module &M, const bool processStrcmp, @@ -268,10 +312,79 @@ bool SplitSwitchesTransform::splitSwitches(Module &M) { * all switches to switches vector for later processing */ for (auto &F : M) { + if (isBlacklisted(&F)) continue; + for (auto &BB : F) { SwitchInst *switchInst = nullptr; + if (!myWhitelist.empty()) { + + bool instrumentBlock = false; + BasicBlock::iterator IP = BB.getFirstInsertionPt(); + + /* Get the current location using debug information. + * For now, just instrument the block if we are not able + * to determine our location. */ + DebugLoc Loc = IP->getDebugLoc(); + if (Loc) { + + DILocation *cDILoc = dyn_cast(Loc.getAsMDNode()); + + unsigned int instLine = cDILoc->getLine(); + StringRef instFilename = cDILoc->getFilename(); + + if (instFilename.str().empty()) { + + /* If the original location is empty, try using the inlined location + */ + DILocation *oDILoc = cDILoc->getInlinedAt(); + if (oDILoc) { + + instFilename = oDILoc->getFilename(); + instLine = oDILoc->getLine(); + + } + + } + + (void)instLine; + + /* Continue only if we know where we actually are */ + if (!instFilename.str().empty()) { + + for (std::list::iterator it = myWhitelist.begin(); + it != myWhitelist.end(); ++it) { + + /* We don't check for filename equality here because + * filenames might actually be full paths. Instead we + * check that the actual filename ends in the filename + * specified in the list. */ + if (instFilename.str().length() >= it->length()) { + + if (instFilename.str().compare( + instFilename.str().length() - it->length(), + it->length(), *it) == 0) { + + instrumentBlock = true; + break; + + } + + } + + } + + } + + } + + /* Either we couldn't figure out our location or the location is + * not whitelisted, so we skip instrumentation. */ + if (!instrumentBlock) continue; + + } + if ((switchInst = dyn_cast(BB.getTerminator()))) { if (switchInst->getNumCases() < 1) continue; diff --git a/test/test.sh b/test/test.sh index 93a4e008..9676d22d 100755 --- a/test/test.sh +++ b/test/test.sh @@ -153,10 +153,10 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null CNT=`ls in2/ | wc -l` case "$CNT" in -1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; -*) $ECHO "$RED[!] afl-cmin did not correctly minimizethe number of testcases" - CODE=1 - ;; + *1) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; + *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases" + CODE=1 + ;; esac ../afl-tmin -i in/in2 -o in2/in2 -- ./test-instr.plain > /dev/null 2>&1 SIZE=`ls -l in2/in2 2> /dev/null | awk '{print$5}'` @@ -259,10 +259,10 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null CNT=`ls in2/ | wc -l` case "$CNT" in -1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; -*) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases" - CODE=1 - ;; + *1) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; + *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases" + CODE=1 + ;; esac ../afl-tmin -i in/in2 -o in2/in2 -- ./test-instr.plain > /dev/null 2>&1 SIZE=`ls -l in2/in2 2> /dev/null | awk '{print$5}'` -- cgit 1.4.1 From 3561a1b775989a0cf37221f810eec601cdb14bcf Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 27 Jan 2020 00:19:59 +0100 Subject: dockerfile update --- Dockerfile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1947f211..7bb60610 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,9 @@ RUN apt-get update && apt-get install -y \ clang \ clang-9 \ flex \ + git \ + python3.7 \ + python3.7-dev \ gcc-9 \ gcc-9-plugin-dev \ gcc-9-multilib \ @@ -23,10 +26,12 @@ RUN apt-get update && apt-get install -y \ ca-certificates \ libpixman-1-dev \ && rm -rf /var/lib/apt/lists/* + ARG CC=gcc-9 ARG CXX=g++-9 ARG LLVM_CONFIG=llvm-config-9 -COPY . /app -RUN cd /app && make clean && make distrib && \ - make install && cd .. && rm -rf /app -WORKDIR /work + +RUN git clone https://github.com/vanhauser-thc/AFLplusplus + +RUN cd AFLplusplus && make clean && make distrib && \ + make install && cd .. && rm -rf AFLplusplus -- cgit 1.4.1 From fa64c0d4a5a6eb1eddd13071e3b326778bf6db5a Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 27 Jan 2020 10:40:13 +0100 Subject: important fixes for afl-cmin --- afl-cmin | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/afl-cmin b/afl-cmin index de5a66ed..865809e1 100755 --- a/afl-cmin +++ b/afl-cmin @@ -292,6 +292,24 @@ BEGIN { exit 1 } + # Check for the more efficient way to copy files... + if (0 != system("mkdir -p -m 0700 "trace_dir)) { + print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" + exit 1 + } + if (0 != system("mkdir -p -m 0700 "trace_dir"/.state/auto_extras")) { + print "[-] Error: Cannot create directory "trace_dir"/.state/auto_extras" > "/dev/stderr" + exit 1 + } + if (0 != system("mkdir -p -m 0700 "trace_dir"/.state/redundant_edges")) { + print "[-] Error: Cannot create directory "trace_dir"/.state/redundant_edges" > "/dev/stderr" + exit 1 + } + if (0 != system("mkdir -p -m 0700 "trace_dir"/.state/deterministic_done")) { + print "[-] Error: Cannot create directory "trace_dir"/.state/deterministic_done" > "/dev/stderr" + exit 1 + } + if (stdin_file) { # truncate input file printf "" > stdin_file @@ -339,12 +357,6 @@ BEGIN { exit 1 } - # Check for the more efficient way to copy files... - if (0 != system("mkdir -p -m 0700 "trace_dir)) { - print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" - exit 1 - } - if (0 == system("ln "in_dir"/"first_file" "trace_dir"/.link_test")) { cp_tool = "ln" } else { @@ -360,7 +372,7 @@ BEGIN { system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") } else { system("cp "in_dir"/"first_file" "stdin_file) - system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" Date: Mon, 27 Jan 2020 11:47:39 +0100 Subject: nearing afl-cmin perfection :-) --- afl-cmin | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/afl-cmin b/afl-cmin index 865809e1..f6e76263 100755 --- a/afl-cmin +++ b/afl-cmin @@ -297,18 +297,6 @@ BEGIN { print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" exit 1 } - if (0 != system("mkdir -p -m 0700 "trace_dir"/.state/auto_extras")) { - print "[-] Error: Cannot create directory "trace_dir"/.state/auto_extras" > "/dev/stderr" - exit 1 - } - if (0 != system("mkdir -p -m 0700 "trace_dir"/.state/redundant_edges")) { - print "[-] Error: Cannot create directory "trace_dir"/.state/redundant_edges" > "/dev/stderr" - exit 1 - } - if (0 != system("mkdir -p -m 0700 "trace_dir"/.state/deterministic_done")) { - print "[-] Error: Cannot create directory "trace_dir"/.state/deterministic_done" > "/dev/stderr" - exit 1 - } if (stdin_file) { # truncate input file @@ -342,7 +330,7 @@ BEGIN { } else { stat_format = "-f '%z %N'" # *BSD, MacOS } - cmdline = "cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" + cmdline = "cd "in_dir" && find . -maxdepth 1 -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" while (cmdline | getline) { infilesSmallToBig[i++] = $0 } -- cgit 1.4.1 From 3374ada561e5dcfe052c41837fc15bd29287b285 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 27 Jan 2020 11:48:49 +0100 Subject: nearing afl-cmin perfection :-) --- afl-cmin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/afl-cmin b/afl-cmin index f6e76263..e9d713aa 100755 --- a/afl-cmin +++ b/afl-cmin @@ -360,7 +360,7 @@ BEGIN { system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") } else { system("cp "in_dir"/"first_file" "stdin_file) - system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" < /dev/null") + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" Date: Mon, 27 Jan 2020 13:06:00 +0100 Subject: updated binary_fuzzing document --- docs/binaryonly_fuzzing.md | 153 ++++++++++++++++++++++++++++++++++++++++++++ docs/binaryonly_fuzzing.txt | 144 ----------------------------------------- 2 files changed, 153 insertions(+), 144 deletions(-) create mode 100644 docs/binaryonly_fuzzing.md delete mode 100644 docs/binaryonly_fuzzing.txt diff --git a/docs/binaryonly_fuzzing.md b/docs/binaryonly_fuzzing.md new file mode 100644 index 00000000..d22e4ce2 --- /dev/null +++ b/docs/binaryonly_fuzzing.md @@ -0,0 +1,153 @@ +#Fuzzing binary-only programs with afl++ + +afl++, libfuzzer and others are great if you have the source code, and +it allows for very fast and coverage guided fuzzing. + +However, if there is only the binary program and no source code available, +then standard `afl-fuzz -n` (dumb mode) is not effective. + +The following is a description of how these binaries can be fuzzed with afl++ + +!!!!! +TL;DR: try DYNINST with afl-dyninst. If it produces too many crashes then + use afl -Q qemu_mode, or better: use both in parallel. +!!!!! + + +##QEMU +Qemu is the "native" solution to the program. +It is available in the ./qemu_mode/ directory and once compiled it can +be accessed by the afl-fuzz -Q command line option. +The speed decrease is at about 50% +It is the easiest to use alternative and even works for cross-platform binaries. + +Note that there is also honggfuzz: [https://github.com/google/honggfuzz](https://github.com/google/honggfuzz) +which now has a qemu_mode, but its performance is just 1.5%! + +As it is included in afl++ this needs no URL. + + +##WINE+QEMU +Wine mode can run Win32 PE binaries with the QEMU instrumentation. +It needs Wine, python3 and the pefile python package installed. + +As it is included in afl++ this needs no URL. + + +##UNICORN +Unicorn is a fork of QEMU. The instrumentation is, therefore, very similar. +In contrast to QEMU, Unicorn does not offer a full system or even userland +emulation. Runtime environment and/or loaders have to be written from scratch, +if needed. On top, block chaining has been removed. This means the speed boost +introduced in the patched QEMU Mode of afl++ cannot simply be ported over to +Unicorn. For further information, check out ./unicorn_mode.txt. + +As it is included in afl++ this needs no URL. + + +##DYNINST +Dyninst is a binary instrumentation framework similar to Pintool and Dynamorio +(see far below). However whereas Pintool and Dynamorio work at runtime, dyninst +instruments the target at load time, and then let it run - or save the +binary with the changes. +This is great for some things, e.g. fuzzing, and not so effective for others, +e.g. malware analysis. + +So what we can do with dyninst is taking every basic block, and put afl's +instrumention code in there - and then save the binary. +Afterwards we can just fuzz the newly saved target binary with afl-fuzz. +Sounds great? It is. The issue though - it is a non-trivial problem to +insert instructions, which change addresses in the process space, so that +everything is still working afterwards. Hence more often than not binaries +crash when they are run. + +The speed decrease is about 15-35%, depending on the optimization options +used with afl-dyninst. + +So if dyninst works, it is the best option available. Otherwise it just doesn't +work well. + +[https://github.com/vanhauser-thc/afl-dyninst](https://github.com/vanhauser-thc/afl-dyninst) + + +##INTEL-PT +If you have a newer Intel CPU, you can make use of Intels processor trace. +The big issue with Intel's PT is the small buffer size and the complex +encoding of the debug information collected through PT. +This makes the decoding very CPU intensive and hence slow. +As a result, the overall speed decrease is about 70-90% (depending on +the implementation and other factors). + +There are two afl intel-pt implementations: + +1. [https://github.com/junxzm1990/afl-pt](https://github.com/junxzm1990/afl-pt) + => this needs Ubuntu 14.04.05 without any updates and the 4.4 kernel. + +2. [https://github.com/hunter-ht-2018/ptfuzzer](https://github.com/hunter-ht-2018/ptfuzzer) + => this needs a 4.14 or 4.15 kernel. the "nopti" kernel boot option must + be used. This one is faster than the other. + +Note that there is also honggfuzz: https://github.com/google/honggfuzz +But its IPT performance is just 6%! + + +##CORESIGHT +Coresight is ARM's answer to Intel's PT. +There is no implementation so far which handle coresight and getting +it working on an ARM Linux is very difficult due to custom kernel building +on embedded systems is difficult. And finding one that has coresight in +the ARM chip is difficult too. +My guess is that it is slower than Qemu, but faster than Intel PT. + +If anyone finds any coresight implementation for afl please ping me: +vh@thc.org + + +##FRIDA +Frida is a dynamic instrumentation engine like Pintool, Dyninst and Dynamorio. +What is special is that it is written Python, and scripted with Javascript. +It is mostly used to reverse binaries on mobile phones however can be used +everywhere. + +There is a WIP fuzzer available at [https://github.com/andreafioraldi/frida-fuzzer](https://github.com/andreafioraldi/frida-fuzzer) + + +##PIN & DYNAMORIO +Pintool and Dynamorio are dynamic instrumentation engines, and they can be +used for getting basic block information at runtime. +Pintool is only available for Intel x32/x64 on Linux, Mac OS and Windows +whereas Dynamorio is additionally available for ARM and AARCH64. +Dynamorio is also 10x faster than Pintool. + +The big issue with Dynamorio (and therefore Pintool too) is speed. +Dynamorio has a speed decrease of 98-99% +Pintool has a speed decrease of 99.5% + +Hence Dynamorio is the option to go for if everything fails, and Pintool +only if Dynamorio fails too. + +Dynamorio solutions: + * [https://github.com/vanhauser-thc/afl-dynamorio](https://github.com/vanhauser-thc/afl-dynamorio) + * [https://github.com/mxmssh/drAFL](https://github.com/mxmssh/drAFL) + * [https://github.com/googleprojectzero/winafl/](https://github.com/googleprojectzero/winafl/) <= very good but windows only + +Pintool solutions: + * [https://github.com/vanhauser-thc/afl-pin](https://github.com/vanhauser-thc/afl-pin) + * [https://github.com/mothran/aflpin](https://github.com/mothran/aflpin) + * [https://github.com/spinpx/afl_pin_mode](https://github.com/spinpx/afl_pin_mode) <= only old Pintool version supported + + +##Non-AFL solutions +There are many binary-only fuzzing frameworks. +Some are great for CTFs but don't work with large binaries, others are very +slow but have good path discovery, some are very hard to set-up ... + +* QSYM: [https://github.com/sslab-gatech/qsym](https://github.com/sslab-gatech/qsym) +* Manticore: [https://github.com/trailofbits/manticore](https://github.com/trailofbits/manticore) +* S2E: [https://github.com/S2E](https://github.com/S2E) +* + + +## Closing words + +That's it! News, corrections, updates? Send an email to vh@thc.org diff --git a/docs/binaryonly_fuzzing.txt b/docs/binaryonly_fuzzing.txt deleted file mode 100644 index f8d68cd8..00000000 --- a/docs/binaryonly_fuzzing.txt +++ /dev/null @@ -1,144 +0,0 @@ - -Fuzzing binary-only programs with afl++ -======================================= - -afl++, libfuzzer and others are great if you have the source code, and -it allows for very fast and coverage guided fuzzing. - -However, if there is only the binary program and no source code available, -then standard `afl-fuzz -n` (dumb mode) is not effective. - -The following is a description of how these binaries can be fuzzed with afl++ - -!!!!! -TL;DR: try DYNINST with afl-dyninst. If it produces too many crashes then - use afl -Q qemu_mode, or better: use both in parallel. -!!!!! - - -QEMU ----- -Qemu is the "native" solution to the program. -It is available in the ./qemu_mode/ directory and once compiled it can -be accessed by the afl-fuzz -Q command line option. -The speed decrease is at about 50% -It is the easiest to use alternative and even works for cross-platform binaries. - -As it is included in afl++ this needs no URL. - -WINE+QEMU ---------- -Wine mode can run Win32 PE binaries with the QEMU instrumentation. -It needs Wine, python3 and the pefile python package installed. - -UNICORN -------- -Unicorn is a fork of QEMU. The instrumentation is, therefore, very similar. -In contrast to QEMU, Unicorn does not offer a full system or even userland emulation. -Runtime environment and/or loaders have to be written from scratch, if needed. -On top, block chaining has been removed. This means the speed boost introduced in -the patched QEMU Mode of afl++ cannot simply be ported over to Unicorn. -For further information, check out ./unicorn_mode.txt. - - -DYNINST -------- -Dyninst is a binary instrumentation framework similar to Pintool and Dynamorio -(see far below). However whereas Pintool and Dynamorio work at runtime, dyninst -instruments the target at load time, and then let it run. -This is great for some things, e.g. fuzzing, and not so effective for others, -e.g. malware analysis. - -So what we can do with dyninst is taking every basic block, and put afl's -instrumention code in there - and then save the binary. -Afterwards we can just fuzz the newly saved target binary with afl-fuzz. -Sounds great? It is. The issue though - it is a non-trivial problem to -insert instructions, which change addresses in the process space, so -everything is still working afterwards. Hence more often than not binaries -crash when they are run (because of instrumentation). - -The speed decrease is about 15-35%, depending on the optimization options -used with afl-dyninst. - -So if dyninst works, it is the best option available. Otherwise it just doesn't -work well. - -https://github.com/vanhauser-thc/afl-dyninst - - -INTEL-PT --------- -If you have a newer Intel CPU, you can make use of Intels processor trace. -The big issue with Intel's PT is the small buffer size and the complex -encoding of the debug information collected through PT. -This makes the decoding very CPU intensive and hence slow. -As a result, the overall speed decrease is about 70-90% (depending on -the implementation and other factors). - -There are two afl intel-pt implementations: - -1. https://github.com/junxzm1990/afl-pt - => this needs Ubuntu 14.04.05 without any updates and the 4.4 kernel. - -2. https://github.com/hunter-ht-2018/ptfuzzer - => this needs a 4.14 or 4.15 kernel. the "nopti" kernel boot option must - be used. This one is faster than the other. - - -CORESIGHT ---------- - -Coresight is ARM's answer to Intel's PT. -There is no implementation so far which handle coresight and getting -it working on an ARM Linux is very difficult due to custom kernel building -on embedded systems is difficult. And finding one that has coresight in -the ARM chip is difficult too. -My guess is that it is slower than Qemu, but faster than Intel PT. -If anyone finds any coresight implementation for afl please ping me: -vh@thc.org - - -PIN & DYNAMORIO ---------------- - -Pintool and Dynamorio are dynamic instrumentation engines, and they can be -used for getting basic block information at runtime. -Pintool is only available for Intel x32/x64 on Linux, Mac OS and Windows -whereas Dynamorio is additionally available for ARM and AARCH64. -Dynamorio is also 10x faster than Pintool. - -The big issue with Dynamorio (and therefore Pintool too) is speed. -Dynamorio has a speed decrease of 98-99% -Pintool has a speed decrease of 99.5% - -Hence Dynamorio is the option to go for if everything fails, and Pintool -only if Dynamorio fails too. - -Dynamorio solutions: - https://github.com/vanhauser-thc/afl-dynamorio - https://github.com/mxmssh/drAFL - https://github.com/googleprojectzero/winafl/ <= very good but windows only - -Pintool solutions: - https://github.com/vanhauser-thc/afl-pin - https://github.com/mothran/aflpin - https://github.com/spinpx/afl_pin_mode <= only old Pintool version supported - - -Non-AFL solutions ------------------ - -There are many binary-only fuzzing frameworks. Some are great for CTFs but don't -work with large binaries, others are very slow but have good path discovery, -some are very hard to set-up ... - -QSYM: https://github.com/sslab-gatech/qsym -Manticore: https://github.com/trailofbits/manticore -S2E: https://github.com/S2E - - - - -That's it! -News, corrections, updates? -Email vh@thc.org -- cgit 1.4.1