From 4b3ad5f037ee9a36aa057bf55a69acca1f573922 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 5 Sep 2020 12:32:10 +0200 Subject: add cull queue, -i subdir traversal --- src/afl-fuzz-queue.c | 7 ------- 1 file changed, 7 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index c6d8225f..db91813b 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -239,13 +239,6 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { afl->cycles_wo_finds = 0; - if (!(afl->queued_paths % 100)) { - - afl->q_prev100->next_100 = q; - afl->q_prev100 = q; - - } - struct queue_entry **queue_buf = afl_realloc( AFL_BUF_PARAM(queue), afl->queued_paths * sizeof(struct queue_entry *)); if (unlikely(!queue_buf)) { PFATAL("alloc"); } -- cgit 1.4.1 From e30b2c6af6e369844c92c00a20ebdd53473a747c Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 5 Sep 2020 13:18:28 +0200 Subject: final changes for pre-3.0 --- .gitignore | 6 ++ Android.bp | 6 +- GNUmakefile | 3 +- GNUmakefile.gcc_plugin | 1 + GNUmakefile.llvm | 8 ++- README.md | 2 +- docs/Changelog.md | 14 ++++ docs/FAQ.md | 104 +++++++++++++++--------------- docs/INSTALL.md | 19 +++--- docs/env_variables.md | 121 ++++++++++++++++++----------------- docs/ideas.md | 57 ----------------- docs/life_pro_tips.md | 4 +- docs/perf_tips.md | 8 +-- docs/sister_projects.md | 4 +- docs/status_screen.md | 2 +- examples/README.md | 2 +- examples/aflpp_driver/aflpp_driver.c | 2 +- include/afl-fuzz.h | 3 +- include/config.h | 4 +- include/envs.h | 1 + qemu_mode/patches/afl-qemu-cpu-inl.h | 2 +- src/afl-fuzz-init.c | 3 +- src/afl-fuzz-queue.c | 13 ++-- src/afl-fuzz-stats.c | 8 +-- src/afl-fuzz.c | 3 +- test/test-gcc-plugin.sh | 2 +- test/test-unittests.sh | 2 + 27 files changed, 188 insertions(+), 216 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/.gitignore b/.gitignore index 0527a0b2..e3adb6ef 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,12 @@ afl-showmap.8 afl-system-config.8 afl-tmin.8 afl-whatsup.8 +afl-c++ +afl-cc +afl-lto +afl-lto++ +afl-lto++.8 +afl-lto.8 qemu_mode/libcompcov/compcovtest qemu_mode/qemu-* unicorn_mode/samples/*/\.test-* diff --git a/Android.bp b/Android.bp index e59129db..2c2114b2 100644 --- a/Android.bp +++ b/Android.bp @@ -101,7 +101,7 @@ cc_binary_host { ], srcs: [ - "llvm_mode/afl-clang-fast.c", + "src/afl-cc.c", ], } @@ -119,7 +119,7 @@ cc_binary_host { ], srcs: [ - "llvm_mode/afl-clang-fast.c", + "src/afl-cc.c", ], } @@ -136,6 +136,6 @@ cc_library_static { ], srcs: [ - "llvm_mode/afl-llvm-rt.o.c", + "instrumentation/afl-llvm-rt.o.c", ], } diff --git a/GNUmakefile b/GNUmakefile index 0046a481..7455483c 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -533,7 +533,7 @@ endif deepclean: clean rm -rf qemu_mode/qemu-3.1.1.tar.xz rm -rf unicorn_mode/unicornafl - git reset --hard >/dev/null 2>&1 || true + # NEVER EVER ACTIVATE THAT!!!!! git reset --hard >/dev/null 2>&1 || true .PHONY: distrib distrib: all @@ -591,6 +591,7 @@ install: all $(MANPAGES) @install -d -m 755 $${DESTDIR}$(BIN_PATH) $${DESTDIR}$(HELPER_PATH) $${DESTDIR}$(DOC_PATH) $${DESTDIR}$(MISC_PATH) @rm -f $${DESTDIR}$(BIN_PATH)/afl-plot.sh @rm -f $${DESTDIR}$(BIN_PATH)/afl-as + @rm -f $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt-32.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt-64.o $${DESTDIR}$(HELPER_PATH)/afl-gcc-rt.o install -m 755 $(PROGS) $(SH_PROGS) $${DESTDIR}$(BIN_PATH) @if [ -f afl-qemu-trace ]; then install -m 755 afl-qemu-trace $${DESTDIR}$(BIN_PATH); fi @if [ -f libdislocator.so ]; then set -e; install -m 755 libdislocator.so $${DESTDIR}$(HELPER_PATH); fi diff --git a/GNUmakefile.gcc_plugin b/GNUmakefile.gcc_plugin index aeb1ef16..b73fcfda 100644 --- a/GNUmakefile.gcc_plugin +++ b/GNUmakefile.gcc_plugin @@ -158,6 +158,7 @@ vpath % .. install: all ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-gcc-fast ln -sf afl-c++ $${DESTDIR}$(BIN_PATH)/afl-g++-fast + ln -sf afl-compiler-rt.o $${DESTDIR}$(HELPER_PATH)/afl-gcc-rt.o install -m 755 ./afl-gcc-pass.so $${DESTDIR}$(HELPER_PATH) install -m 644 -T instrumentation/README.gcc_plugin.md $${DESTDIR}$(DOC_PATH)/README.gcc_plugin.md diff --git a/GNUmakefile.llvm b/GNUmakefile.llvm index d76e0b28..1bb3d265 100644 --- a/GNUmakefile.llvm +++ b/GNUmakefile.llvm @@ -423,10 +423,12 @@ document: ./afl-compiler-rt-32.o: instrumentation/afl-compiler-rt.o.c | test_deps @printf "[*] Building 32-bit variant of the runtime (-m32)... " @$(CLANG_BIN) $(CLANG_CFL) $(CFLAGS_SAFE) $(CPPFLAGS) -O3 -Wno-unused-result -m32 -fPIC -c $< -o $@ 2>/dev/null; if [ "$$?" = "0" ]; then echo "success!"; else echo "failed (that's fine)"; fi + @test -e afl-compiler-rt-32.o && ln -sf afl-compiler-rt-32.o afl-llvm-rt-64.o ./afl-compiler-rt-64.o: instrumentation/afl-compiler-rt.o.c | test_deps @printf "[*] Building 64-bit variant of the runtime (-m64)... " @$(CLANG_BIN) $(CLANG_CFL) $(CFLAGS_SAFE) $(CPPFLAGS) -O3 -Wno-unused-result -m64 -fPIC -c $< -o $@ 2>/dev/null; if [ "$$?" = "0" ]; then echo "success!"; else echo "failed (that's fine)"; fi + @test -e afl-compiler-rt-64.o && ln -sf afl-compiler-rt-64.o afl-llvm-rt-64.o .PHONY: test_build test_build: $(PROGS) @@ -448,11 +450,11 @@ all_done: test_build install: all @install -d -m 755 $${DESTDIR}$(BIN_PATH) $${DESTDIR}$(HELPER_PATH) $${DESTDIR}$(DOC_PATH) $${DESTDIR}$(MISC_PATH) @if [ -f ./afl-cc ]; then set -e; install -m 755 ./afl-cc $${DESTDIR}$(BIN_PATH); ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-c++; fi - @if [ -f ./afl-compiler-rt.o ]; then set -e; install -m 755 ./afl-compiler-rt.o $${DESTDIR}$(HELPER_PATH); fi + @if [ -f ./afl-compiler-rt.o ]; then set -e; install -m 755 ./afl-compiler-rt.o $${DESTDIR}$(HELPER_PATH); ln -sf afl-compiler-rt.o $${DESTDIR}$(HELPER_PATH)afl-llvm-rt.o ;fi @if [ -f ./afl-lto ]; then set -e; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-lto; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-lto++; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang-lto; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang-lto++; install -m 755 ./afl-llvm-lto-instrumentation.so ./afl-llvm-rt-lto*.o ./afl-llvm-lto-instrumentlist.so $${DESTDIR}$(HELPER_PATH); fi @if [ -f ./afl-ld-lto ]; then set -e; install -m 755 ./afl-ld-lto $${DESTDIR}$(BIN_PATH); fi - @if [ -f ./afl-compiler-rt-32.o ]; then set -e; install -m 755 ./afl-compiler-rt-32.o $${DESTDIR}$(HELPER_PATH); fi - @if [ -f ./afl-compiler-rt-64.o ]; then set -e; install -m 755 ./afl-compiler-rt-64.o $${DESTDIR}$(HELPER_PATH); fi + @if [ -f ./afl-compiler-rt-32.o ]; then set -e; install -m 755 ./afl-compiler-rt-32.o $${DESTDIR}$(HELPER_PATH); ln -sf afl-compiler-rt-32.o $${DESTDIR}$(HELPER_PATH)afl-llvm-rt-32.o ;fi + @if [ -f ./afl-compiler-rt-64.o ]; then set -e; install -m 755 ./afl-compiler-rt-64.o $${DESTDIR}$(HELPER_PATH); ln -sf afl-compiler-rt-64.o $${DESTDIR}$(HELPER_PATH)afl-llvm-rt-64.o ; fi @if [ -f ./compare-transform-pass.so ]; then set -e; install -m 755 ./*.so $${DESTDIR}$(HELPER_PATH); fi @if [ -f ./compare-transform-pass.so ]; then set -e; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang-fast ; ln -sf ./afl-c++ $${DESTDIR}$(BIN_PATH)/afl-clang-fast++ ; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang ; ln -sf ./afl-c++ $${DESTDIR}$(BIN_PATH)/afl-clang++ ; fi @if [ -f ./SanitizerCoverageLTO.so ]; then set -e; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang-lto ; ln -sf ./afl-c++ $${DESTDIR}$(BIN_PATH)/afl-clang-lto++ ; fi diff --git a/README.md b/README.md index 96b34260..c886489d 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Release Version: [2.67c](https://github.com/AFLplusplus/AFLplusplus/releases) - Github Version: 2.67d + Github Version: 3.00a Repository: [https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) diff --git a/docs/Changelog.md b/docs/Changelog.md index 6321aee4..9de03e78 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -9,6 +9,20 @@ Want to stay in the loop on major new features? Join our mailing list by sending a mail to . +### Version ++3.00a (develop) + - llvm_mode/ and gcc_plugin/ moved to instrumentation/ + - all compilers combined to afl-cc which emulates the previous ones + - afl-llvm/gcc-rt.o merged into afl-compiler-rt.o + - afl-fuzz + - reading testcases from -i now descends into subdirectories + - allow up to 4 -x command line options + - loaded extras now have a duplicate protection + - instrumentation + - new llvm pass: dict2file via AFL_LLVM_DICT2FILE, create afl-fuzz + -x dictionary of string comparisons found during compilation + - not overriding -Ox or -fno-unroll-loops anymore + + ### Version ++2.68c (release) - added the GSoC excellent afl++ grammar mutator by Shengtuo to our custom_mutators/ (see custom_mutators/README.md) - or get it here: diff --git a/docs/FAQ.md b/docs/FAQ.md index 064638f4..24942492 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -4,11 +4,11 @@ * [What is the difference between afl and afl++?](#what-is-the-difference-between-afl-and-afl) * [How to improve the fuzzing speed?](#how-to-improve-the-fuzzing-speed) - * [How do I fuzz a network service?](#how-do-i-fuzz-a-network-service) - * [How do I fuzz a GUI program?](#how-do-i-fuzz-a-gui-program) + * [How do I fuzz a network service?](#how-to-fuzz-a-network-service) + * [How do I fuzz a GUI program?](#how-to-fuzz-a-gui-program) * [What is an edge?](#what-is-an-edge) * [Why is my stability below 100%?](#why-is-my-stability-below-100) - * [How can I improve the stability value?](#how-can-i-improve-the-stability-value) + * [How can I improve the stability value](#how-can-i-improve-the-stability-value) If you find an interesting or important question missing, submit it via [https://github.com/AFLplusplus/AFLplusplus/issues](https://github.com/AFLplusplus/AFLplusplus/issues) @@ -18,52 +18,51 @@ If you find an interesting or important question missing, submit it via American Fuzzy Lop (AFL) was developed by Michał "lcamtuf" Zalewski starting in 2013/2014, and when he left Google end of 2017 he stopped developing it. -At the end of 2019 the Google fuzzing team took over maintenance of AFL, however -it is only accepting PRs from the community and is not developing enhancements +At the end of 2019 the Google fuzzing team took over maintance of AFL, however +it is only accepting PR from the community and is not developing enhancements anymore. -In the second quarter of 2019, 1 1/2 year later when no further development of -AFL had happened and it became clear there would none be coming, afl++ -was born, where initially community patches were collected and applied -for bug fixes and enhancements. Then from various AFL spin-offs - mostly academic +In the second quarter of 2019, 1 1/2 years after no further development of +AFL had happened and it became clear there would be none coming, afl++ +was born, where initially first community patches were collected and applied +for bugs and enhancements. Then from various AFL spin-offs - mostly academic research - features were integrated. This already resulted in a much advanced AFL. Until the end of 2019 the afl++ team had grown to four active developers which -then implemented their own research and features, making it now by far the most +then implemented their own research and feature, making it now by far the most flexible and feature rich guided fuzzer available as open source. And in independent fuzzing benchmarks it is one of the best fuzzers available, e.g. [Fuzzbench Report](https://www.fuzzbench.com/reports/2020-08-03/index.html) -## How to improve the fuzzing speed? +## How to improve the fuzzing speed - 1. Use [llvm_mode](docs/llvm_mode/README.md): afl-clang-lto (llvm >= 11) or afl-clang-fast (llvm >= 9 recommended) - 2. Use [persistent mode](llvm_mode/README.persistent_mode.md) (x2-x20 speed increase) + 1. use [instrumentation](docs/README.llvm.md): afl-clang-lto (llvm >= 11) or afl-clang-fast (llvm >= 9 recommended) + 2. Use [persistent mode](instrumentation/README.persistent_mode.md) (x2-x20 speed increase) 3. Use the [afl++ snapshot module](https://github.com/AFLplusplus/AFL-Snapshot-LKM) (x2 speed increase) - 4. If you do not use shmem persistent mode, use `AFL_TMPDIR` to put the input file directory on a tempfs location, see [docs/env_variables.md](docs/env_variables.md) - 5. Improve Linux kernel performance: modify `/etc/default/grub`, set `GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"`; then `update-grub` and `reboot` (warning: makes the system less secure) + 4. If you do not use shmem persistent mode, use `AFL_TMPDIR` to point the input file on a tempfs location, see [docs/env_variables.md](docs/env_variables.md) + 5. Improve kernel performance: modify `/etc/default/grub`, set `GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"`; then `update-grub` and `reboot` (warning: makes the system more insecure) 6. Running on an `ext2` filesystem with `noatime` mount option will be a bit faster than on any other journaling filesystem 7. Use your cores! [README.md:3.b) Using multiple cores/threads](../README.md#b-using-multiple-coresthreads) ## How do I fuzz a network service? -The short answer is - you cannot, at least not "out of the box". +The short answer is - you cannot, at least "out of the box". -Using a network channel is inadequate for several reasons: -- it has a slow-down of x10-20 on the fuzzing speed -- it does not scale to fuzzing multiple instances easily, -- instead of one initial data packet often a back-and-forth interplay of packets is needed for stateful protocols (which is totally unsupported by most coverage aware fuzzers). +Using network has a slow-down of x10-20 on the fuzzing speed, does not scale, +and finally usually it is more than one initial data packet but a back-and-forth +which is totally unsupported by most coverage aware fuzzers. The established method to fuzz network services is to modify the source code to read from a file or stdin (fd 0) (or even faster via shared memory, combine -this with persistent mode [llvm_mode/README.persistent_mode.md](llvm_mode/README.persistent_mode.md) +this with persistent mode [instrumentation/README.persistent_mode.md](instrumentation/README.persistent_mode.md) and you have a performance gain of x10 instead of a performance loss of over -x10 - that is a x100 difference!). +x10 - that is a x100 difference! If modifying the source is not an option (e.g. because you only have a binary and perform binary fuzzing) you can also use a shared library with AFL_PRELOAD -to emulate the network. This is also much faster than the real network would be. -See [examples/socket_fuzzing/](../examples/socket_fuzzing/). +to emulate the network. This is also much faster than network would be. +See [examples/socket_fuzzing/](../examples/socket_fuzzing/) There is an outdated afl++ branch that implements networking if you are desperate though: [https://github.com/AFLplusplus/AFLplusplus/tree/networking](https://github.com/AFLplusplus/AFLplusplus/tree/networking) - @@ -74,7 +73,7 @@ which allows you to define network state with different type of data packets. If the GUI program can read the fuzz data from a file (via the command line, a fixed location or via an environment variable) without needing any user -interaction then it would be suitable for fuzzing. +interaction then then yes. Otherwise it is not possible without modifying the source code - which is a very good idea anyway as the GUI functionality is a huge CPU/time overhead @@ -83,13 +82,13 @@ for the fuzzing. So create a new `main()` that just reads the test case and calls the functionality for processing the input that the GUI program is using. -## What is an "edge"? +## What is an "edge" A program contains `functions`, `functions` contain the compiled machine code. The compiled machine code in a `function` can be in a single or many `basic blocks`. A `basic block` is the largest possible number of subsequent machine code -instructions that has exactly one entrypoint (which can be be entered by multiple other basic blocks) -and runs linearly without branching or jumping to other addresses (except at the end). +instructions that runs independent, meaning it does not split up to different +locations nor is it jumped into it from a different location: ``` function() { A: @@ -99,7 +98,7 @@ function() { if (x) goto C; else goto D; C: some code - goto E + goto D D: some code goto B @@ -109,7 +108,7 @@ function() { ``` Every code block between two jump locations is a `basic block`. -An `edge` is then the unique relationship between two directly connected `basic blocks` (from the +An `edge` is then the unique relationship between two `basic blocks` (from the code example above): ``` Block A @@ -124,9 +123,8 @@ code example above): Block E ``` Every line between two blocks is an `edge`. -Note that a few basic block loop to itself, this too would be an edge. -## Why is my stability below 100%? +## Why is my stability below 100% Stability is measured by how many percent of the edges in the target are "stable". Sending the same input again and again should take the exact same @@ -134,37 +132,37 @@ path through the target every time. If that is the case, the stability is 100%. If however randomness happens, e.g. a thread reading other external data, reaction to timing, etc. then in some of the re-executions with the same data -the edge coverage result will be different accross runs. +the result in the edge information will be different accross runs. Those edges that change are then flagged "unstable". The more "unstable" edges, the more difficult for afl++ to identify valid new paths. A value above 90% is usually fine and a value above 80% is also still ok, and -even a value above 20% can still result in successful finds of bugs. -However, it is recommended that for values below 90% or 80% you should take -countermeasures to improve stability. +even above 20% can still result in successful finds of bugs. +However, it is recommended that below 90% or 80% you should take measures to +improve the stability. -## How can I improve the stability value? +## How can I improve the stability value -For fuzzing a 100% stable target that covers all edges is the best case. +For fuzzing a 100% stable target that covers all edges is the best. A 90% stable target that covers all edges is however better than a 100% stable target that ignores 10% of the edges. With instability you basically have a partial coverage loss on an edge, with -ignored functions you have a full loss on that edges. +ignore you have a full loss on that edge. There are functions that are unstable, but also provide value to coverage, eg init functions that use fuzz data as input for example. -If however a function that has nothing to do with the input data is the -source of instability, e.g. checking jitter, or is a hash map function etc. -then it should not be instrumented. +If however it is a function that has nothing to do with the input data is the +source, e.g. checking jitter, or is a hash map function etc. then it should +not be instrumented. -To be able to exclude these functions (based on AFL++'s measured stability) -the following process will allow to identify functions with variable edges. +To be able to make this decision the following process will allow you to +identify the functions with variable edges so you can make this decision. -Four steps are required to do this and it also requires quite some knowledge -of coding and/or disassembly and is effectively possible only with +Four steps are required to do this and requires quite some knowledge of +coding and/or disassembly and it is only effectively possible with afl-clang-fast PCGUARD and afl-clang-lto LTO instrumentation. 1. First step: Identify which edge ID numbers are unstable @@ -173,7 +171,7 @@ afl-clang-fast PCGUARD and afl-clang-lto LTO instrumentation. The out/fuzzer_stats file will then show the edge IDs that were identified as unstable. - 2. Second step: Find the responsible function(s). + 2. Second step: Find the responsible function. a) For LTO instrumented binaries this can be documented during compile time, just set `export AFL_LLVM_DOCUMENT_IDS=/path/to/a/file`. @@ -182,10 +180,10 @@ afl-clang-fast PCGUARD and afl-clang-lto LTO instrumentation. b) For PCGUARD instrumented binaries it is much more difficult. Here you can either modify the __sanitizer_cov_trace_pc_guard function in - llvm_mode/afl-llvm-rt.o.c to write a backtrace to a file if the ID in + instrumentation/afl-llvm-rt.o.c to write a backtrace to a file if the ID in __afl_area_ptr[*guard] is one of the unstable edge IDs. (Example code is already there). - Then recompile and reinstall llvm_mode and rebuild your target. + Then recompile and reinstall instrumentation and rebuild your target. Run the recompiled target with afl-fuzz for a while and then check the file that you wrote with the backtrace information. Alternatively you can use `gdb` to hook __sanitizer_cov_trace_pc_guard_init @@ -193,20 +191,20 @@ afl-clang-fast PCGUARD and afl-clang-lto LTO instrumentation. and set a write breakpoint to that address (`watch 0x.....`). c) in all other instrumentation types this is not possible. So just - recompile with the two mentioned above. This is just for + recompile with the the two mentioned above. This is just for identifying the functions that have unstable edges. 3. Third step: create a text file with the filenames/functions Identify which source code files contain the functions that you need to remove from instrumentation, or just specify the functions you want to - skip for instrumentation. Note that optimization might inline functions! + skip instrumenting. Note that optimization might inline functions! - Simply follow this document on how to do this: [llvm_mode/README.instrument_list.md](llvm_mode/README.instrument_list.md) + Simply follow this document on how to do this: [instrumentation/README.instrument_list.md](instrumentation/README.instrument_list.md) If PCGUARD is used, then you need to follow this guide (needs llvm 12+!): [http://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation](http://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation) - Only exclude those functions from instrumentation that provide no value + Only deny those functions from instrumentation that provide no value for coverage - that is if it does not process any fuzz data directly or indirectly (e.g. hash maps, thread management etc.). If however a function directly or indirectly handles fuzz data then you diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 766f24d7..fb7b5642 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -24,7 +24,7 @@ There are no special dependencies to speak of; you will need GNU make and a working compiler (gcc or clang). Some of the optional scripts bundled with the program may depend on bash, gdb, and similar basic tools. -If you are using clang, please review llvm_mode/README.md; the LLVM +If you are using clang, please review README.llvm.md; the LLVM integration mode can offer substantial performance gains compared to the traditional approach. @@ -52,10 +52,10 @@ sudo gmake install Keep in mind that if you are using csh as your shell, the syntax of some of the shell commands given in the README.md and other docs will be different. -The `llvm_mode` requires a dynamically linked, fully-operational installation of +The `llvm` requires a dynamically linked, fully-operational installation of clang. At least on FreeBSD, the clang binaries are static and do not include some of the essential tools, so if you want to make it work, you may need to -follow the instructions in llvm_mode/README.md. +follow the instructions in README.llvm.md. Beyond that, everything should work as advertised. @@ -97,27 +97,24 @@ and definitely don't look POSIX-compliant. This means two things: User emulation mode of QEMU does not appear to be supported on MacOS X, so black-box instrumentation mode (`-Q`) will not work. -The llvm_mode requires a fully-operational installation of clang. The one that +The llvm instrumentation requires a fully-operational installation of clang. The one that comes with Xcode is missing some of the essential headers and helper tools. -See llvm_mode/README.md for advice on how to build the compiler from scratch. +See README.llvm.md for advice on how to build the compiler from scratch. ## 4. Linux or *BSD on non-x86 systems Standard build will fail on non-x86 systems, but you should be able to leverage two other options: - - The LLVM mode (see llvm_mode/README.md), which does not rely on + - The LLVM mode (see README.llvm.md), which does not rely on x86-specific assembly shims. It's fast and robust, but requires a complete installation of clang. - The QEMU mode (see qemu_mode/README.md), which can be also used for fuzzing cross-platform binaries. It's slower and more fragile, but can be used even when you don't have the source for the tested app. -If you're not sure what you need, you need the LLVM mode. To get it, try: - -```bash -AFL_NO_X86=1 gmake && gmake -C llvm_mode -``` +If you're not sure what you need, you need the LLVM mode, which is built by +default. ...and compile your target program with afl-clang-fast or afl-clang-fast++ instead of the traditional afl-gcc or afl-clang wrappers. diff --git a/docs/env_variables.md b/docs/env_variables.md index c47d10e8..9d289f6d 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -5,13 +5,25 @@ users or for some types of custom fuzzing setups. See README.md for the general instruction manual. -## 1) Settings for afl-gcc, afl-clang, and afl-as - and gcc_plugin afl-gcc-fast +## 1) Settings for all compilers -Because they can't directly accept command-line options, the compile-time -tools make fairly broad use of environmental variables: +Starting with afl++ 3.0 there is only one compiler: afl-cc +To select the different instrumentation modes this can be done by + 1. passing --afl-MODE command line options to the compiler + 2. use a symlink to afl-cc: afl-gcc, afl-g++, afl-clang, afl-clang++, + afl-clang-fast, afl-clang-fast++, afl-clang-lto, afl-clang-lto++, + afl-gcc-fast, afl-g++-fast + 3. using the environment variable AFL_CC_COMPILER with MODE - - Most afl tools do not print any output if stdout/stderr are redirected. - If you want to save the output in a file then set the AFL_DEBUG +MODE can one of LTO (afl-clang-lto*), LLVM (afl-clang-fast*), GCC_PLUGIN +(afl-g*-fast) or GCC (afl-gcc/afl-g++). + +Because beside the --afl-MODE command no afl specific command-line options +are accepted, the compile-time tools make fairly broad use of environmental +variables: + + - Most afl tools do not print any ouput if stout/stderr are redirected. + If you want to have the output into a file then set the AFL_DEBUG environment variable. This is sadly necessary for various build processes which fail otherwise. @@ -24,6 +36,8 @@ tools make fairly broad use of environmental variables: will cause problems in programs built with -Werror, simply because -O3 enables more thorough code analysis and can spew out additional warnings. To disable optimizations, set AFL_DONT_OPTIMIZE. + However if -O... and/or -fno-unroll-loops are set, these are not + overriden. - Setting AFL_USE_ASAN automatically enables ASAN, provided that your compiler supports that. Note that fuzzing with ASAN is mildly challenging @@ -44,7 +58,7 @@ tools make fairly broad use of environmental variables: you instrument hand-written assembly when compiling clang code by plugging a normalizer into the chain. (There is no equivalent feature for GCC.) - - Setting AFL_INST_RATIO to a percentage between 0% and 100% controls the + - Setting AFL_INST_RATIO to a percentage between 0 and 100% controls the probability of instrumenting every branch. This is (very rarely) useful when dealing with exceptionally complex programs that saturate the output bitmap. Examples include v8, ffmpeg, and perl. @@ -55,19 +69,16 @@ tools make fairly broad use of environmental variables: Setting AFL_INST_RATIO to 0 is a valid choice. This will instrument only the transitions between function entry points, but not individual branches. + Note that this is an outdated variable. A few instances (e.g. afl-gcc) + still support these, but state-of-the-art (e.g. LLVM LTO and LLVM PCGUARD) + do not need this. + - AFL_NO_BUILTIN causes the compiler to generate code suitable for use with libtokencap.so (but perhaps running a bit slower than without the flag). - TMPDIR is used by afl-as for temporary files; if this variable is not set, the tool defaults to /tmp. - - Setting AFL_KEEP_ASSEMBLY prevents afl-as from deleting instrumented - assembly files. Useful for troubleshooting problems or understanding how - the tool works. To get them in a predictable place, try something like: - - mkdir assembly_here - TMPDIR=$PWD/assembly_here AFL_KEEP_ASSEMBLY=1 make clean all - - If you are a weird person that wants to compile and instrument asm text files then use the AFL_AS_FORCE_INSTRUMENT variable: AFL_AS_FORCE_INSTRUMENT=1 afl-gcc foo.s -o foo @@ -78,19 +89,24 @@ tools make fairly broad use of environmental variables: - Setting AFL_CAL_FAST will speed up the initial calibration, if the application is very slow -## 2) Settings for afl-clang-fast / afl-clang-fast++ / afl-gcc-fast / afl-g++-fast +## 2) Settings for LLVM and LTO: afl-clang-fast / afl-clang-fast++ / afl-clang-lto / afl-clang-lto++ -The native instrumentation helpers (llvm_mode and gcc_plugin) accept a subset +The native instrumentation helpers (instrumentation and gcc_plugin) accept a subset of the settings discussed in section #1, with the exception of: + - LLVM modes support `AFL_LLVM_DICT2FILE=/absolute/path/file.txt` which will + write all constant string comparisons to this file to be used with + afl-fuzz' `-x` option. + - AFL_AS, since this toolchain does not directly invoke GNU as. - TMPDIR and AFL_KEEP_ASSEMBLY, since no temporary assembly files are created. - - AFL_INST_RATIO, as we by default use collision free instrumentation. + - AFL_INST_RATIO, as we by default collision free instrumentation is used. + Not all passes support this option though as it is an outdated feature. -Then there are a few specific features that are only available in llvm_mode: +Then there are a few specific features that are only available in instrumentation: ### Select the instrumentation mode @@ -121,7 +137,7 @@ Then there are a few specific features that are only available in llvm_mode: None of the following options are necessary to be used and are rather for manual use (which only ever the author of this LTO implementation will use). - These are used if several seperated instrumentations are performed which + These are used if several seperated instrumentation are performed which are then later combined. - AFL_LLVM_DOCUMENT_IDS=file will document to a file which edge ID was given @@ -136,7 +152,7 @@ Then there are a few specific features that are only available in llvm_mode: - AFL_LLVM_LTO_DONTWRITEID prevents that the highest location ID written into the instrumentation is set in a global variable - See llvm_mode/README.LTO.md for more information. + See instrumentation/README.LTO.md for more information. ### INSTRIM @@ -154,7 +170,7 @@ Then there are a few specific features that are only available in llvm_mode: afl-fuzz will only be able to see the path the loop took, but not how many times it was called (unless it is a complex loop). - See llvm_mode/README.instrim.md + See instrumentation/README.instrim.md ### NGRAM @@ -165,7 +181,7 @@ Then there are a few specific features that are only available in llvm_mode: config.h to at least 18 and maybe up to 20 for this as otherwise too many map collisions occur. - See llvm_mode/README.ctx.md + See instrumentation/README.ctx.md ### CTX @@ -176,7 +192,7 @@ Then there are a few specific features that are only available in llvm_mode: config.h to at least 18 and maybe up to 20 for this as otherwise too many map collisions occur. - See llvm_mode/README.ngram.md + See instrumentation/README.ngram.md ### LAF-INTEL @@ -196,17 +212,17 @@ Then there are a few specific features that are only available in llvm_mode: - Setting AFL_LLVM_LAF_ALL sets all of the above - See llvm_mode/README.laf-intel.md for more information. + See instrumentation/README.laf-intel.md for more information. ### INSTRUMENT LIST (selectively instrument files and functions) - This feature allows selective instrumentation of the source + This feature allows selectively instrumentation of the source - Setting AFL_LLVM_ALLOWLIST or AFL_LLVM_DENYLIST with a filenames and/or function will only instrument (or skip) those files that match the names listed in the specified file. - See llvm_mode/README.instrument_list.md for more information. + See instrumentation/README.instrument_list.md for more information. ### NOT_ZERO @@ -220,27 +236,34 @@ Then there are a few specific features that are only available in llvm_mode: test. If the target performs only few loops then this will give a small performance boost. - See llvm_mode/README.neverzero.md + See instrumentation/README.neverzero.md ### CMPLOG - Setting AFL_LLVM_CMPLOG=1 during compilation will tell afl-clang-fast to - produce a CmpLog binary. See llvm_mode/README.cmplog.md + produce a CmpLog binary. See instrumentation/README.cmplog.md - See llvm_mode/README.neverzero.md + See instrumentation/README.neverzero.md -Then there are a few specific features that are only available in the gcc_plugin: +## 3) Settings for GCC / GCC_PLUGIN modes -### INSTRUMENT_FILE +Then there are a few specific features that are only available in GCC and +GCC_PLUGIN mode. - This feature allows selective instrumentation of the source + - Setting AFL_KEEP_ASSEMBLY prevents afl-as from deleting instrumented + assembly files. Useful for troubleshooting problems or understanding how + the tool works. (GCC mode only) + To get them in a predictable place, try something like: - - Setting AFL_GCC_INSTRUMENT_FILE with a filename will only instrument those - files that match the names listed in this file (one filename per line). + mkdir assembly_here + TMPDIR=$PWD/assembly_here AFL_KEEP_ASSEMBLY=1 make clean all + - Setting AFL_GCC_INSTRUMENT_FILE with a filename will only instrument those + files that match the names listed in this file (one filename per line). See gcc_plugin/README.instrument_list.md for more information. + (GCC_PLUGIN mode only) -## 3) Settings for afl-fuzz +## 4) Settings for afl-fuzz The main fuzzer binary accepts several options that disable a couple of sanity checks or alter some of the more exotic semantics of the tool: @@ -278,14 +301,6 @@ checks or alter some of the more exotic semantics of the tool: don't want AFL to spend too much time classifying that stuff and just rapidly put all timeouts in that bin. - - Setting AFL_FORKSRV_INIT_TMOUT allows yout to specify a different timeout - to wait for the forkserver to spin up. The default is the `-t` value times - `FORK_WAIT_MULT` from `config.h` (usually 10), so for a `-t 100`, the - default would wait `1000` milis. Setting a different time here is useful - if the target has a very slow startup time, for example when doing - full-system fuzzing or emulation, but you don't want the actual runs - to wait too long for timeouts. - - AFL_NO_ARITH causes AFL to skip most of the deterministic arithmetics. This can be useful to speed up the fuzzing of text-based file formats. @@ -377,22 +392,12 @@ checks or alter some of the more exotic semantics of the tool: Note that this setting inhibits some of the user-friendly diagnostics normally done when starting up the forkserver and causes a pretty significant performance drop. - - - Setting AFL_MAX_DET_EXTRAS changes the count of dictionary entries/extras - (default 200), after which the entries will be used probabilistically. - So, if the dict/extras file (`-x`) contains more tokens than this threshold, - not all of the tokens will be used in each fuzzing step, every time. - Instead, there is a chance that the entry will be skipped during fuzzing. - This makes sure that the fuzzer doesn't spend all its time only inserting - the extras, but will still do other mutations. However, it decreases the - likelihood for each token to be inserted, before the next queue entry is fuzzed. - Either way, all tokens will be used eventually, in a longer fuzzing campaign. - Outdated environment variables that are that not supported anymore: AFL_DEFER_FORKSRV AFL_PERSISTENT -## 4) Settings for afl-qemu-trace +## 5) Settings for afl-qemu-trace The QEMU wrapper used to instrument binary-only code supports several settings: @@ -446,7 +451,7 @@ The QEMU wrapper used to instrument binary-only code supports several settings: stack pointer in which QEMU can find the return address when `start addr` is hitted. -## 5) Settings for afl-cmin +## 6) Settings for afl-cmin The corpus minimization script offers very little customization: @@ -472,12 +477,12 @@ to match when minimizing crashes. This will make minimization less useful, but may prevent the tool from "jumping" from one crashing condition to another in very buggy software. You probably want to combine it with the -e flag. -## 7) Settings for afl-analyze +## 8) Settings for afl-analyze You can set AFL_ANALYZE_HEX to get file offsets printed as hexadecimal instead of decimal. -## 8) Settings for libdislocator +## 9) Settings for libdislocator The library honors these environmental variables: @@ -499,12 +504,12 @@ The library honors these environmental variables: - AFL_ALIGNED_ALLOC=1 will force the alignment of the allocation size to max_align_t to be compliant with the C standard. -## 9) Settings for libtokencap +## 10) Settings for libtokencap This library accepts AFL_TOKEN_FILE to indicate the location to which the discovered tokens should be written. -## 10) Third-party variables set by afl-fuzz & other tools +## 11) Third-party variables set by afl-fuzz & other tools Several variables are not directly interpreted by afl-fuzz, but are set to optimal values if not already present in the environment: diff --git a/docs/ideas.md b/docs/ideas.md index 65e2e8e6..a5d40963 100644 --- a/docs/ideas.md +++ b/docs/ideas.md @@ -3,49 +3,6 @@ In the following, we describe a variety of ideas that could be implemented for future AFL++ versions. -For GSOC2020 interested students please see -[https://github.com/AFLplusplus/AFLplusplus/issues/208](https://github.com/AFLplusplus/AFLplusplus/issues/208) - -## Flexible Grammar Mutator (currently in development) - -Currently, AFL++'s mutation does not have deeper knowledge about the fuzzed -binary, apart from feedback, even though the developer may have insights -about the target. - -A developer may choose to provide dictionaries and implement own mutations -in python or C, but an easy mutator that behaves according to a given grammar, -does not exist. - -State-of-the-art research on grammar fuzzing has some problems in their -implementations like code quality, scalability, or ease of use and other -common issues of the academic code. - -We aim to develop a pluggable grammar mutator for afl++ that combines -various results. - -Mentor: andreafioraldi - -## perf-fuzz Linux Kernel Module - -Expand on [snapshot LKM](https://github.com/AFLplusplus/AFL-Snapshot-LKM) -To make it thread safe, can snapshot several processes at once and increase -overall performance. - -Mentor: any - -## QEMU 5-based Instrumentation - -First tests to use QEMU 4 for binary-only AFL++ showed that caching behavior -changed, which vastly decreases fuzzing speeds. - -In this task test if QEMU 5 performs better and port the afl++ QEMU 3.1 -patches to QEMU 5. - -Understanding the current instrumentation and fixing the current caching -issues will be needed. - -Mentor: andreafioraldi - ## WASM Instrumentation Currently, AFL++ can be used for source code fuzzing and traditional binaries. @@ -66,20 +23,6 @@ Either improve a single mutator thorugh learning of many different bugs Mentor: domenukk -## Reengineer `afl-fuzz` as Thread Safe, Embeddable Library (currently in development) - -Right now, afl-fuzz is single threaded, cannot safely be embedded in tools, -and not multi-threaded. It makes use of a large number of globals, must always -be the parent process and exec child processes. -Instead, afl-fuzz could be refactored to contain no global state and globals. -This allows for different use cases that could be implemented during this -project. -Note that in the mean time a lot has happened here already, but e.g. making -it all work and implement multithreading in afl-fuzz ... there is still quite -some work to do. - -Mentor: hexcoder- or vanhauser-thc - ## Collision-free Binary-Only Maps AFL++ supports collison-free maps using an LTO (link-time-optimization) pass. diff --git a/docs/life_pro_tips.md b/docs/life_pro_tips.md index a5bd7286..0004c297 100644 --- a/docs/life_pro_tips.md +++ b/docs/life_pro_tips.md @@ -30,10 +30,10 @@ Check out the `fuzzer_stats` file in the AFL output dir or try `afl-whatsup`. It could be important - consult docs/status_screen.md right away! ## Know your target? Convert it to persistent mode for a huge performance gain! -Consult section #5 in llvm_mode/README.md for tips. +Consult section #5 in README.llvm.md for tips. ## Using clang? -Check out llvm_mode/ for a faster alternative to afl-gcc! +Check out instrumentation/ for a faster alternative to afl-gcc! ## Did you know that AFL can fuzz closed-source or cross-platform binaries? Check out qemu_mode/README.md and unicorn_mode/README.md for more. diff --git a/docs/perf_tips.md b/docs/perf_tips.md index 731dc238..fbcb4d8d 100644 --- a/docs/perf_tips.md +++ b/docs/perf_tips.md @@ -51,7 +51,7 @@ a file. ## 3. Use LLVM instrumentation When fuzzing slow targets, you can gain 20-100% performance improvement by -using the LLVM-based instrumentation mode described in [the llvm_mode README](../llvm_mode/README.md). +using the LLVM-based instrumentation mode described in [the instrumentation README](../instrumentation/README.llvm.md). Note that this mode requires the use of clang and will not work with GCC. The LLVM mode also offers a "persistent", in-process fuzzing mode that can @@ -62,12 +62,12 @@ modes require you to edit the source code of the fuzzed program, but the changes often amount to just strategically placing a single line or two. If there are important data comparisons performed (e.g. `strcmp(ptr, MAGIC_HDR)`) -then using laf-intel (see llvm_mode/README.laf-intel.md) will help `afl-fuzz` a lot +then using laf-intel (see instrumentation/README.laf-intel.md) will help `afl-fuzz` a lot to get to the important parts in the code. If you are only interested in specific parts of the code being fuzzed, you can instrument_files the files that are actually relevant. This improves the speed and -accuracy of afl. See llvm_mode/README.instrument_list.md +accuracy of afl. See instrumentation/README.instrument_list.md Also use the InsTrim mode on larger binaries, this improves performance and coverage a lot. @@ -110,7 +110,7 @@ e.g.: https://launchpad.net/libeatmydata In programs that are slow due to unavoidable initialization overhead, you may -want to try the LLVM deferred forkserver mode (see llvm_mode/README.md), +want to try the LLVM deferred forkserver mode (see README.llvm.md), which can give you speed gains up to 10x, as mentioned above. Last but not least, if you are using ASAN and the performance is unacceptable, diff --git a/docs/sister_projects.md b/docs/sister_projects.md index a501ecbd..640e59f7 100644 --- a/docs/sister_projects.md +++ b/docs/sister_projects.md @@ -52,7 +52,7 @@ options. Provides an evolutionary instrumentation-guided fuzzing harness that allows some programs to be fuzzed without the fork / execve overhead. (Similar functionality is now available as the "persistent" feature described in -[the llvm_mode readme](../llvm_mode/README.md)) +[the llvm_mode readme](../instrumentation/README.llvm.md)) http://llvm.org/docs/LibFuzzer.html @@ -245,7 +245,7 @@ https://code.google.com/p/address-sanitizer/wiki/AsanCoverage#Coverage_counters ### AFL JS (Han Choongwoo) One-off optimizations to speed up the fuzzing of JavaScriptCore (now likely -superseded by LLVM deferred forkserver init - see llvm_mode/README.md). +superseded by LLVM deferred forkserver init - see README.llvm.md). https://github.com/tunz/afl-fuzz-js diff --git a/docs/status_screen.md b/docs/status_screen.md index b89468ce..2eeb8f3f 100644 --- a/docs/status_screen.md +++ b/docs/status_screen.md @@ -324,7 +324,7 @@ there are several things to look at: - Multiple threads executing at once in semi-random order. This is harmless when the 'stability' metric stays over 90% or so, but can become an issue if not. Here's what to try: - * Use afl-clang-fast from [llvm_mode](../llvm_mode/) - it uses a thread-local tracking + * Use afl-clang-fast from [instrumentation](../instrumentation/) - it uses a thread-local tracking model that is less prone to concurrency issues, * See if the target can be compiled or run without threads. Common `./configure` options include `--without-threads`, `--disable-pthreads`, or diff --git a/examples/README.md b/examples/README.md index d28aadbe..46a92c6e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -47,7 +47,7 @@ Here's a quick overview of the stuff you can find in this directory: Note that the minimize_corpus.sh tool has graduated from the examples/ directory and is now available as ../afl-cmin. The LLVM mode has likewise -graduated to ../llvm_mode/*. +graduated to ../instrumentation/*. Most of the tools in this directory are meant chiefly as examples that need to be tweaked for your specific needs. They come with some basic documentation, diff --git a/examples/aflpp_driver/aflpp_driver.c b/examples/aflpp_driver/aflpp_driver.c index ff5446e9..82e55fc4 100644 --- a/examples/aflpp_driver/aflpp_driver.c +++ b/examples/aflpp_driver/aflpp_driver.c @@ -27,7 +27,7 @@ EOF # Build your target with -fsanitize-coverage=trace-pc-guard using fresh clang. clang -g -fsanitize-coverage=trace-pc-guard test_fuzzer.cc -c # Build afl-llvm-rt.o.c from the AFL distribution. -clang -c -w $AFL_HOME/llvm_mode/afl-llvm-rt.o.c +clang -c -w $AFL_HOME/instrumentation/afl-llvm-rt.o.c # Build this file, link it with afl-llvm-rt.o.o and the target code. clang++ afl_driver.cpp test_fuzzer.o afl-llvm-rt.o.o # Run AFL: diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 4281c554..9e469864 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -935,6 +935,7 @@ u8 has_new_bits(afl_state_t *, u8 *); void load_extras_file(afl_state_t *, u8 *, u32 *, u32 *, u32); void load_extras(afl_state_t *, u8 *); +void dedup_extras(afl_state_t *); void add_extra(afl_state_t *afl, u8 *mem, u32 len); void maybe_add_auto(afl_state_t *, u8 *, u32); void save_auto(afl_state_t *); @@ -972,7 +973,7 @@ u8 fuzz_one(afl_state_t *); void bind_to_free_cpu(afl_state_t *); #endif void setup_post(afl_state_t *); -void read_testcases(afl_state_t *); +void read_testcases(afl_state_t *, u8 *); void perform_dry_run(afl_state_t *); void pivot_inputs(afl_state_t *); u32 find_start_position(afl_state_t *); diff --git a/include/config.h b/include/config.h index 77407d50..8cc70075 100644 --- a/include/config.h +++ b/include/config.h @@ -28,7 +28,7 @@ /* Version string: */ // c = release, d = volatile github dev, e = experimental branch -#define VERSION "++2.68c" +#define VERSION "++3.00a" /****************************************************** * * @@ -195,7 +195,7 @@ steps; past this point, the "extras/user" step will be still carried out, but with proportionally lower odds: */ -#define MAX_DET_EXTRAS 200 +#define MAX_DET_EXTRAS 256 /* Maximum number of auto-extracted dictionary tokens to actually use in fuzzing (first value), and to keep in memory as candidates. The latter should be much diff --git a/include/envs.h b/include/envs.h index 2dc1dbbf..d9968fcd 100644 --- a/include/envs.h +++ b/include/envs.h @@ -69,6 +69,7 @@ static char *afl_environment_variables[] = { "AFL_LLVM_CMPLOG", "AFL_LLVM_INSTRIM", "AFL_LLVM_CTX", + "AFL_LLVM_DICT2FILE", "AFL_LLVM_DOCUMENT_IDS", "AFL_LLVM_INSTRUMENT", "AFL_LLVM_INSTRIM_LOOPHEAD", diff --git a/qemu_mode/patches/afl-qemu-cpu-inl.h b/qemu_mode/patches/afl-qemu-cpu-inl.h index 63b7581d..0e38f38b 100644 --- a/qemu_mode/patches/afl-qemu-cpu-inl.h +++ b/qemu_mode/patches/afl-qemu-cpu-inl.h @@ -466,7 +466,7 @@ void afl_forkserver(CPUState *cpu) { } /* A simplified persistent mode handler, used as explained in - * llvm_mode/README.md. */ + * instrumentation/README.llvm.md */ void afl_persistent_loop(void) { diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 713849a1..1351d274 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -626,6 +626,7 @@ if (dir == NULL) { if (!access(fn1, F_OK)) { afl->in_dir = fn1; + subdirs = 0; } else { @@ -1063,7 +1064,7 @@ restart_outer_cull_loop: } - afl->q_prev100 = afl->queue = afl->queue_top = afl->queue; + afl->queue = afl->queue_top = afl->queue; } diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index db91813b..af52aa45 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -138,8 +138,7 @@ static u8 check_if_text(struct queue_entry *q) { } // non-overlong 2-byte - if (len - offset > 1 && - ((0xC2 <= buf[offset + 0] && buf[offset + 0] <= 0xDF) && + if (len - offset > 1 && ((0xC2 <= buf[offset + 0] && buf[offset + 0] <= 0xDF) && (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF))) { offset += 2; @@ -230,7 +229,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { } else { - afl->q_prev100 = afl->queue = afl->queue_top = q; + afl->queue = afl->queue_top = q; } @@ -274,15 +273,15 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { void destroy_queue(afl_state_t *afl) { - struct queue_entry *q = afl->queue, *n; + struct queue_entry *q; + u32 i; - while (q) { + for (i = 0; i < afl->queued_paths; i++) { - n = q->next; + q = afl->queue_buf[i]; ck_free(q->fname); ck_free(q->trace_mini); ck_free(q); - q = n; } diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 51eed14b..c60c65aa 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -982,10 +982,9 @@ void show_stats(afl_state_t *afl) { void show_init_stats(afl_state_t *afl) { struct queue_entry *q = afl->queue; - u32 min_bits = 0, max_bits = 0; + u32 min_bits = 0, max_bits = 0, max_len = 0, count = 0; u64 min_us = 0, max_us = 0; u64 avg_us = 0; - u32 max_len = 0; u8 val_bufs[4][STRINGIFY_VAL_SIZE_MAX]; #define IB(i) val_bufs[(i)], sizeof(val_bufs[(i)]) @@ -1006,6 +1005,7 @@ void show_init_stats(afl_state_t *afl) { if (q->len > max_len) { max_len = q->len; } + ++count; q = q->next; } @@ -1072,10 +1072,10 @@ void show_init_stats(afl_state_t *afl) { OKF("Here are some useful stats:\n\n" cGRA " Test case count : " cRST - "%u favored, %u variable, %u total\n" cGRA " Bitmap range : " cRST + "%u favored, %u variable, %u ignored, %u total\n" cGRA " Bitmap range : " cRST "%u to %u bits (average: %0.02f bits)\n" cGRA " Exec timing : " cRST "%s to %s us (average: %s us)\n", - afl->queued_favored, afl->queued_variable, afl->queued_paths, min_bits, + afl->queued_favored, afl->queued_variable, afl->queued_paths - count, afl->queued_paths, min_bits, max_bits, ((double)afl->total_bitmap_size) / (afl->total_bitmap_entries ? afl->total_bitmap_entries : 1), diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index bfaa22e8..73ca6aaa 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1133,8 +1133,9 @@ int main(int argc, char **argv_orig, char **envp) { setup_cmdline_file(afl, argv + optind); - read_testcases(afl); + read_testcases(afl, NULL); // read_foreign_testcases(afl, 1); for the moment dont do this + OKF("Loaded a total of %u seeds.", afl->queued_paths); load_auto(afl); diff --git a/test/test-gcc-plugin.sh b/test/test-gcc-plugin.sh index 2ed10a72..8b8cbd8e 100755 --- a/test/test-gcc-plugin.sh +++ b/test/test-gcc-plugin.sh @@ -3,7 +3,7 @@ . ./test-pre.sh $ECHO "$BLUE[*] Testing: gcc_plugin" -test -e ../afl-gcc-fast -a -e ../afl-gcc-rt.o && { +test -e ../afl-gcc-fast -a -e ../afl-compiler-rt.o && { SAVE_AFL_CC=${AFL_CC} export AFL_CC=`command -v gcc` ../afl-gcc-fast -o test-instr.plain.gccpi ../test-instr.c > /dev/null 2>&1 diff --git a/test/test-unittests.sh b/test/test-unittests.sh index f540b5f8..58c2eea9 100755 --- a/test/test-unittests.sh +++ b/test/test-unittests.sh @@ -7,3 +7,5 @@ unset AFL_CC make -C .. unit || CODE=1 INCOMPLETE=1 : . ./test-post.sh + +rm -rf unittests/unit_hash unittests/unit_rand -- cgit 1.4.1 From 2f90f2faba92c0ef5e081ff74b54fb07eb1faaa9 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 5 Sep 2020 13:19:19 +0200 Subject: code-format --- src/afl-fuzz-init.c | 23 +++++++++++++---------- src/afl-fuzz-queue.c | 3 ++- src/afl-fuzz-stats.c | 16 +++++++++------- 3 files changed, 24 insertions(+), 18 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 1351d274..c834e5db 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -617,24 +617,27 @@ void read_testcases(afl_state_t *afl, u8 *directory) { s32 nl_cnt, subdirs = 1; u32 i; u8 * fn1, *dir = directory; - u8 val_buf[2][STRINGIFY_VAL_SIZE_MAX]; + u8 val_buf[2][STRINGIFY_VAL_SIZE_MAX]; /* Auto-detect non-in-place resumption attempts. */ -if (dir == NULL) { - fn1 = alloc_printf("%s/queue", afl->in_dir); - if (!access(fn1, F_OK)) { + if (dir == NULL) { - afl->in_dir = fn1; - subdirs = 0; + fn1 = alloc_printf("%s/queue", afl->in_dir); + if (!access(fn1, F_OK)) { - } else { + afl->in_dir = fn1; + subdirs = 0; + + } else { - ck_free(fn1); + ck_free(fn1); + + } + + dir = afl->in_dir; } - dir = afl->in_dir; -} ACTF("Scanning '%s'...", dir); diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index af52aa45..8c7bfc55 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -138,7 +138,8 @@ static u8 check_if_text(struct queue_entry *q) { } // non-overlong 2-byte - if (len - offset > 1 && ((0xC2 <= buf[offset + 0] && buf[offset + 0] <= 0xDF) && + if (len - offset > 1 && + ((0xC2 <= buf[offset + 0] && buf[offset + 0] <= 0xDF) && (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF))) { offset += 2; diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index c60c65aa..dfc0cd97 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -35,12 +35,12 @@ void write_setup_file(afl_state_t *afl, u32 argc, char **argv) { u8 fn[PATH_MAX]; snprintf(fn, PATH_MAX, "%s/fuzzer_setup", afl->out_dir); FILE *f = create_ffile(fn); - u32 i; + u32 i; fprintf(f, "# environment variables:\n"); - u32 s_afl_env = (u32) - sizeof(afl_environment_variables) / sizeof(afl_environment_variables[0]) - - 1U; + u32 s_afl_env = (u32)sizeof(afl_environment_variables) / + sizeof(afl_environment_variables[0]) - + 1U; for (i = 0; i < s_afl_env; ++i) { @@ -75,6 +75,7 @@ void write_setup_file(afl_state_t *afl, u32 argc, char **argv) { } } + fprintf(f, "\n"); fclose(f); @@ -1072,11 +1073,12 @@ void show_init_stats(afl_state_t *afl) { OKF("Here are some useful stats:\n\n" cGRA " Test case count : " cRST - "%u favored, %u variable, %u ignored, %u total\n" cGRA " Bitmap range : " cRST + "%u favored, %u variable, %u ignored, %u total\n" cGRA + " Bitmap range : " cRST "%u to %u bits (average: %0.02f bits)\n" cGRA " Exec timing : " cRST "%s to %s us (average: %s us)\n", - afl->queued_favored, afl->queued_variable, afl->queued_paths - count, afl->queued_paths, min_bits, - max_bits, + afl->queued_favored, afl->queued_variable, afl->queued_paths - count, + afl->queued_paths, min_bits, max_bits, ((double)afl->total_bitmap_size) / (afl->total_bitmap_entries ? afl->total_bitmap_entries : 1), stringify_int(IB(0), min_us), stringify_int(IB(1), max_us), -- cgit 1.4.1 From 6404abd7d609350ffd6c6f221cbf56e60b2ef030 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 7 Sep 2020 17:30:28 +0200 Subject: bugfix for fixed seeds --- src/afl-fuzz-queue.c | 2 +- src/afl-fuzz-stats.c | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 8c7bfc55..336b7f4f 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -502,7 +502,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { // Longer execution time means longer work on the input, the deeper in // coverage, the better the fuzzing, right? -mh - if (afl->schedule >= RARE && likely(!afl->fixed_seed)) { + if (likely(afl->schedule < RARE) && likely(!afl->fixed_seed)) { if (q->exec_us * 0.1 > avg_exec_us) { diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index dfc0cd97..1d5b169d 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -1022,7 +1022,11 @@ void show_init_stats(afl_state_t *afl) { /* Let's keep things moving with slow binaries. */ - if (avg_us > 50000) { + if (unlikely(afl->fixed_seed)) { + + afl->havoc_div = 1; + + } else if (avg_us > 50000) { afl->havoc_div = 10; /* 0-19 execs/sec */ @@ -1093,7 +1097,11 @@ void show_init_stats(afl_state_t *afl) { random scheduler jitter is less likely to have any impact, and because our patience is wearing thin =) */ - if (avg_us > 50000) { + if (unlikely(afl->fixed_seed)) { + + afl->fsrv.exec_tmout = avg_us * 5 / 1000; + + } else if (avg_us > 50000) { afl->fsrv.exec_tmout = avg_us * 2 / 1000; -- cgit 1.4.1 From 7ff9800804f6f4f88b70d8aaf882b66e0ce40c8f Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sat, 19 Sep 2020 15:13:15 +0200 Subject: fix exploit, mmopt and rare schedule --- include/afl-fuzz.h | 14 +++++++------- src/afl-fuzz-queue.c | 2 +- src/afl-fuzz.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 9e469864..9404c417 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -243,13 +243,13 @@ enum { enum { /* 00 */ EXPLORE, /* AFL default, Exploration-based constant schedule */ - /* 01 */ EXPLOIT, /* AFL's exploitation-based const. */ - /* 02 */ FAST, /* Exponential schedule */ - /* 03 */ COE, /* Cut-Off Exponential schedule */ - /* 04 */ LIN, /* Linear schedule */ - /* 05 */ QUAD, /* Quadratic schedule */ - /* 06 */ RARE, /* Rare edges */ - /* 07 */ MMOPT, /* Modified MOPT schedule */ + /* 01 */ MMOPT, /* Modified MOPT schedule */ + /* 02 */ EXPLOIT, /* AFL's exploitation-based const. */ + /* 03 */ FAST, /* Exponential schedule */ + /* 04 */ COE, /* Cut-Off Exponential schedule */ + /* 05 */ LIN, /* Linear schedule */ + /* 06 */ QUAD, /* Quadratic schedule */ + /* 07 */ RARE, /* Rare edges */ /* 08 */ SEEK, /* EXPLORE that ignores timings */ POWER_SCHEDULES_NUM diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 336b7f4f..ddd08f1c 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -710,7 +710,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { } - if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) { + if (unlikely(afl->schedule >= EXPLOIT && afl->schedule <= QUAD)) { if (factor > MAX_FACTOR) { factor = MAX_FACTOR; } perf_score *= factor / POWER_BETA; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index ea24011e..11037f73 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1366,7 +1366,7 @@ int main(int argc, char **argv_orig, char **envp) { afl->expand_havoc = 2; break; case 2: - if (!have_p) afl->schedule = EXPLOIT; + //if (!have_p) afl->schedule = EXPLOIT; afl->expand_havoc = 3; break; case 3: -- cgit 1.4.1 From 888d63748a3c6aafd974cb9d96cdb8d3916e82bb Mon Sep 17 00:00:00 2001 From: Vitalii Akolzin Date: Thu, 24 Sep 2020 18:25:32 +0300 Subject: Fix potential endless loop in custom_mutator_stage Co-authored-by: Ivan Gulakov --- include/afl-fuzz.h | 3 +++ src/afl-fuzz-one.c | 64 ++++++++++++++++++++++++++++++++-------------------- src/afl-fuzz-queue.c | 2 ++ src/afl-fuzz-state.c | 2 ++ 4 files changed, 46 insertions(+), 25 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 9404c417..0efd48ec 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -657,6 +657,9 @@ typedef struct afl_state { * they do not call another function */ u8 *map_tmp_buf; + /* queue entries ready for splicing count (len > 1) */ + u32 ready_for_splicing_count; + } afl_state_t; struct custom_mutator { diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 5737c1f5..edae2a88 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -1696,50 +1696,58 @@ custom_mutator_stage: struct queue_entry *target; u32 tid; - u8 * new_buf; + u8 * new_buf = NULL; + u32 target_len = 0; - retry_external_pick: - /* Pick a random other queue entry for passing to external API */ + if (afl->ready_for_splicing_count > 1 || + (afl->ready_for_splicing_count == 1 && + afl->queue_cur->len == 1)) { - do { + retry_external_pick: + /* Pick a random other queue entry for passing to external API */ - tid = rand_below(afl, afl->queued_paths); + do { - } while (tid == afl->current_entry && afl->queued_paths > 1); + tid = rand_below(afl, afl->queued_paths); - afl->splicing_with = tid; - target = afl->queue_buf[tid]; + } while (tid == afl->current_entry && afl->queued_paths > 1); - /* Make sure that the target has a reasonable length. */ + afl->splicing_with = tid; + target = afl->queue_buf[tid]; - while (target && (target->len < 2 || target == afl->queue_cur) && - afl->queued_paths > 3) { + /* Make sure that the target has a reasonable length. */ - target = target->next; - ++afl->splicing_with; + while (target && (target->len < 2 || target == afl->queue_cur) && + afl->queued_paths > 2) { - } + target = target->next; + ++afl->splicing_with; - if (!target) { goto retry_external_pick; } + } - /* Read the additional testcase into a new buffer. */ - fd = open(target->fname, O_RDONLY); - if (unlikely(fd < 0)) { + if (!target) { goto retry_external_pick; } - PFATAL("Unable to open '%s'", target->fname); + /* Read the additional testcase into a new buffer. */ + fd = open(target->fname, O_RDONLY); + if (unlikely(fd < 0)) { - } + PFATAL("Unable to open '%s'", target->fname); - new_buf = afl_realloc(AFL_BUF_PARAM(out_scratch), target->len); - if (unlikely(!new_buf)) { PFATAL("alloc"); } - ck_read(fd, new_buf, target->len, target->fname); - close(fd); + } + + new_buf = afl_realloc(AFL_BUF_PARAM(out_scratch), target->len); + if (unlikely(!new_buf)) { PFATAL("alloc"); } + ck_read(fd, new_buf, target->len, target->fname); + close(fd); + target_len = target->len; + + } u8 *mutated_buf = NULL; size_t mutated_size = el->afl_custom_fuzz(el->data, out_buf, len, &mutated_buf, new_buf, - target->len, max_seed_size); + target_len, max_seed_size); if (unlikely(!mutated_buf)) { @@ -2738,6 +2746,8 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { if (!afl->non_instrumented_mode && !afl->queue_cur->trim_done) { + u32 old_len = afl->queue_cur->len; + u8 res = trim_case(afl, afl->queue_cur, in_buf); if (res == FSRV_RUN_ERROR) { @@ -2759,6 +2769,10 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { len = afl->queue_cur->len; + /* maybe current entry stop being ready for splicing */ + if (old_len > 1 && afl->queue_cur->len == 1) + afl->ready_for_splicing_count--; + } memcpy(out_buf, in_buf, len); diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index ddd08f1c..14aa34fc 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -234,6 +234,8 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { } + if (q->len > 1) afl->ready_for_splicing_count++; + ++afl->queued_paths; ++afl->pending_not_fuzzed; diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index ae45d571..9f68bb51 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -155,6 +155,8 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) { afl->stats_last_execs = 0; afl->stats_avg_exec = -1; + afl->ready_for_splicing_count = 0; + init_mopt_globals(afl); list_append(&afl_states, afl); -- cgit 1.4.1 From 6b3b1775b6b274bc62f9c79f686fc79fa110d0a8 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Fri, 25 Sep 2020 12:03:24 +0200 Subject: improving on splice candidate check patch --- include/afl-fuzz.h | 4 ++-- include/config.h | 2 +- src/afl-fuzz-one.c | 33 +++++++++------------------------ src/afl-fuzz-queue.c | 2 +- src/afl-fuzz-state.c | 48 +++++++++++++----------------------------------- 5 files changed, 26 insertions(+), 63 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 0efd48ec..441ecc61 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -483,7 +483,7 @@ typedef struct afl_state { disable_trim, /* Never trim in fuzz_one */ shmem_testcase_mode, /* If sharedmem testcases are used */ expand_havoc, /* perform expensive havoc after no find */ - cycle_schedules; /* cycle power schedules ? */ + cycle_schedules; /* cycle power schedules? */ u8 *virgin_bits, /* Regions yet untouched by fuzzing */ *virgin_tmout, /* Bits we haven't seen in tmouts */ @@ -657,7 +657,7 @@ typedef struct afl_state { * they do not call another function */ u8 *map_tmp_buf; - /* queue entries ready for splicing count (len > 1) */ + /* queue entries ready for splicing count (len > 4) */ u32 ready_for_splicing_count; } afl_state_t; diff --git a/include/config.h b/include/config.h index a01491e7..7c8e0c7d 100644 --- a/include/config.h +++ b/include/config.h @@ -136,7 +136,7 @@ two cycles where smaller blocks are favored: */ #define HAVOC_BLK_SMALL 32 -#define HAVOC_BLK_MEDIUM 128 +#define HAVOC_BLK_MEDIUM 128 #define HAVOC_BLK_LARGE 1500 /* Extra-large blocks, selected very rarely (<5% of the time): */ diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 8c1aa179..e96c4311 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -1699,36 +1699,22 @@ custom_mutator_stage: u8 * new_buf = NULL; u32 target_len = 0; - /* check if splicing is possible (if the only entry has len > 1 - * check it is not current entry) - */ - if (afl->ready_for_splicing_count > 1 || - (afl->ready_for_splicing_count == 1 && - afl->queue_cur->len == 1)) { + /* check if splicing makes sense yet (enough entries) */ + if (likely(afl->ready_for_splicing_count > 1)) { - retry_external_pick: - /* Pick a random other queue entry for passing to external API */ + /* Pick a random other queue entry for passing to external API + that has the necessary length */ do { tid = rand_below(afl, afl->queued_paths); - } while (tid == afl->current_entry && afl->queued_paths > 1); - - afl->splicing_with = tid; - target = afl->queue_buf[tid]; - - /* Make sure that the target has a reasonable length. */ - - while (target && (target->len < 2 || target == afl->queue_cur) && - afl->queued_paths > 2) { - - target = target->next; - ++afl->splicing_with; + } while (unlikely(tid == afl->current_entry && - } + afl->queue_buf[tid]->len >= 4)); - if (!target) { goto retry_external_pick; } + target = afl->queue_buf[tid]; + afl->splicing_with = tid; /* Read the additional testcase into a new buffer. */ fd = open(target->fname, O_RDONLY); @@ -2773,8 +2759,7 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { len = afl->queue_cur->len; /* maybe current entry is not ready for splicing anymore */ - if (old_len > 1 && afl->queue_cur->len == 1) - afl->ready_for_splicing_count--; + if (unlikely(len <= 4 && old_len > 4)) afl->ready_for_splicing_count--; } diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 14aa34fc..53c3e984 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -234,7 +234,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { } - if (q->len > 1) afl->ready_for_splicing_count++; + if (likely(q->len > 4)) afl->ready_for_splicing_count++; ++afl->queued_paths; ++afl->pending_not_fuzzed; diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index 9f68bb51..5e0995fe 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -95,6 +95,11 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) { afl->stage_name = "init"; /* Name of the current fuzz stage */ afl->splicing_with = -1; /* Splicing with which test case? */ afl->cpu_to_bind = -1; + afl->cal_cycles = CAL_CYCLES; + afl->cal_cycles_long = CAL_CYCLES_LONG; + afl->hang_tmout = EXEC_TIMEOUT; + afl->stats_update_freq = 1; + afl->stats_avg_exec = -1; #ifdef HAVE_AFFINITY afl->cpu_aff = -1; /* Selected CPU core */ @@ -115,48 +120,13 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) { // afl_state_t is not available in forkserver.c afl->fsrv.afl_ptr = (void *)afl; afl->fsrv.add_extra_func = (void (*)(void *, u8 *, u32)) & add_extra; - - afl->cal_cycles = CAL_CYCLES; - afl->cal_cycles_long = CAL_CYCLES_LONG; - afl->fsrv.exec_tmout = EXEC_TIMEOUT; - afl->hang_tmout = EXEC_TIMEOUT; - afl->fsrv.mem_limit = MEM_LIMIT; - - afl->stats_update_freq = 1; - afl->fsrv.dev_urandom_fd = -1; afl->fsrv.dev_null_fd = -1; - afl->fsrv.child_pid = -1; afl->fsrv.out_dir_fd = -1; - afl->cmplog_prev_timed_out = 0; - - /* statis file */ - afl->last_bitmap_cvg = 0; - afl->last_stability = 0; - afl->last_eps = 0; - - /* plot file saves from last run */ - afl->plot_prev_qp = 0; - afl->plot_prev_pf = 0; - afl->plot_prev_pnf = 0; - afl->plot_prev_ce = 0; - afl->plot_prev_md = 0; - afl->plot_prev_qc = 0; - afl->plot_prev_uc = 0; - afl->plot_prev_uh = 0; - - afl->stats_last_stats_ms = 0; - afl->stats_last_plot_ms = 0; - afl->stats_last_ms = 0; - afl->stats_last_execs = 0; - afl->stats_avg_exec = -1; - - afl->ready_for_splicing_count = 0; - init_mopt_globals(afl); list_append(&afl_states, afl); @@ -177,6 +147,14 @@ void read_afl_environment(afl_state_t *afl, char **envp) { WARNF("Potentially mistyped AFL environment variable: %s", env); issue_detected = 1; + } else if (strncmp(env, "USE_", 4) == 0) { + + WARNF( + "Potentially mistyped AFL environment variable: %s, did you mean " + "AFL_%s?", + env, env); + issue_detected = 1; + } else if (strncmp(env, "AFL_", 4) == 0) { int i = 0, match = 0; -- cgit 1.4.1 From e87eca7fe8ec3ed0ba79e7722350ad502b67218b Mon Sep 17 00:00:00 2001 From: Marcel Boehme Date: Tue, 29 Sep 2020 11:53:27 +0000 Subject: Patching and improving AFLFast schedules. --- GNUmakefile | 2 +- include/afl-fuzz.h | 4 ++- src/afl-fuzz-bitmap.c | 18 +++-------- src/afl-fuzz-init.c | 8 +++++ src/afl-fuzz-queue.c | 84 +++++++++++++++++++++++++++++++++------------------ src/afl-fuzz.c | 7 +++++ 6 files changed, 78 insertions(+), 45 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/GNUmakefile b/GNUmakefile index 889c0e7d..c885a935 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -205,7 +205,7 @@ endif ifneq "$(filter Linux GNU%,$(shell uname))" "" override CFLAGS += -D_FORTIFY_SOURCE=2 - LDFLAGS += -ldl -lrt + LDFLAGS += -ldl -lrt -lm endif ifneq "$(findstring FreeBSD, $(shell uname))" "" diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index aa278820..f65fc40f 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -155,7 +155,6 @@ struct queue_entry { u64 exec_us, /* Execution time (us) */ handicap, /* Number of queue cycles behind */ - n_fuzz, /* Number of fuzz, does not overflow*/ depth, /* Path depth */ exec_cksum; /* Checksum of the execution trace */ @@ -492,6 +491,9 @@ typedef struct afl_state { u8 *var_bytes; /* Bytes that appear to be variable */ + #define n_fuzz_size (1 << 21) + u32 *n_fuzz; + volatile u8 stop_soon, /* Ctrl-C pressed? */ clear_screen; /* Window resized? */ diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 1b9df624..64de86a2 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -555,19 +555,9 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); - struct queue_entry *q = afl->queue; - while (q) { - - if (q->exec_cksum == cksum) { - - ++q->n_fuzz; - break; - - } - - q = q->next; - - } + /* Saturated increment */ + if (afl->n_fuzz[cksum % n_fuzz_size] < 0xFFFFFFFF) + afl->n_fuzz[cksum % n_fuzz_size]++; } @@ -610,6 +600,8 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { afl->queue_top->exec_cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); + afl->n_fuzz[cksum % n_fuzz_size] = 1; + /* Try to calibrate inline; this also calls update_bitmap_score() when successful. */ diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index cbac3822..b825837f 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -729,6 +729,14 @@ void read_testcases(afl_state_t *afl, u8 *directory) { add_to_queue(afl, fn2, st.st_size >= MAX_FILE ? MAX_FILE : st.st_size, passed_det); + if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) { + + u64 cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); + + afl->n_fuzz[cksum % n_fuzz_size] = 1; + + } + } free(nl); /* not tracked */ diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 53c3e984..dfabba7b 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -25,6 +25,7 @@ #include "afl-fuzz.h" #include #include +#include /* Mark deterministic checks as done for a particular queue entry. We use the .state file to avoid repeating deterministic fuzzing when resuming aborted @@ -218,7 +219,6 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { q->len = len; q->depth = afl->cur_depth + 1; q->passed_det = passed_det; - q->n_fuzz = 1; q->trace_mini = NULL; if (q->depth > afl->max_depth) { afl->max_depth = q->depth; } @@ -307,8 +307,10 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { u64 fav_factor; u64 fuzz_p2; - if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) - fuzz_p2 = next_pow2(q->n_fuzz); + if (unlikely(afl->schedule >= FAST && afl->schedule < RARE)) + fuzz_p2 = 0; // Skip the fuzz_p2 comparison + else if (unlikely(afl->schedule == RARE)) + fuzz_p2 = next_pow2(afl->n_fuzz[q->exec_cksum % n_fuzz_size]); else fuzz_p2 = q->fuzz_level; @@ -334,7 +336,7 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { u64 top_rated_fav_factor; u64 top_rated_fuzz_p2; if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) - top_rated_fuzz_p2 = next_pow2(afl->top_rated[i]->n_fuzz); + top_rated_fuzz_p2 = next_pow2(afl->n_fuzz[afl->top_rated[i]->exec_cksum % n_fuzz_size]); else top_rated_fuzz_p2 = afl->top_rated[i]->fuzz_level; @@ -605,11 +607,10 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { } - u64 fuzz = q->n_fuzz; - u64 fuzz_total; + u32 n_paths; + double factor = 1.0; + long double fuzz_mu; - u32 n_paths, fuzz_mu; - u32 factor = 1; switch (afl->schedule) { @@ -624,60 +625,83 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { break; case COE: - fuzz_total = 0; + fuzz_mu = 0.0; n_paths = 0; + // Don't modify perf_score for unfuzzed seeds + if (q->fuzz_level == 0) break; + struct queue_entry *queue_it = afl->queue; while (queue_it) { - fuzz_total += queue_it->n_fuzz; + fuzz_mu += log2(afl->n_fuzz[q->exec_cksum % n_fuzz_size]); n_paths++; + queue_it = queue_it->next; } if (unlikely(!n_paths)) { FATAL("Queue state corrupt"); } - fuzz_mu = fuzz_total / n_paths; - if (fuzz <= fuzz_mu) { + fuzz_mu = fuzz_mu / n_paths; - if (q->fuzz_level < 16) { + if (log2(afl->n_fuzz[q->exec_cksum % n_fuzz_size]) > fuzz_mu) { - factor = ((u32)(1 << q->fuzz_level)); + /* Never skip favourites */ + if (!q->favored) factor = 0; - } else { + break; - factor = MAX_FACTOR; + } - } + // Fall through + case FAST: - } else { + // Don't modify unfuzzed seeds + if (q->fuzz_level == 0) break; - factor = 0; + switch ((u32)log2(afl->n_fuzz[q->exec_cksum % n_fuzz_size])) { - } + case 0 ... 1: + factor = 4; + break; - break; + case 2 ... 3: + factor = 3; + break; - case FAST: - if (q->fuzz_level < 16) { + case 4: + factor = 2; + break; - factor = ((u32)(1 << q->fuzz_level)) / (fuzz == 0 ? 1 : fuzz); + case 5: + break; - } else { + case 6: + if (!q->favored) factor = 0.8; + break; - factor = MAX_FACTOR / (fuzz == 0 ? 1 : next_pow2(fuzz)); + case 7: + if (!q->favored) factor = 0.6; + break; + + default: + if (!q->favored) factor = 0.4; + break; } + if (q->favored) + factor *= 1.15; + break; case LIN: - factor = q->fuzz_level / (fuzz == 0 ? 1 : fuzz); + factor = q->fuzz_level / (afl->n_fuzz[q->exec_cksum % n_fuzz_size] + 1); break; case QUAD: - factor = q->fuzz_level * q->fuzz_level / (fuzz == 0 ? 1 : fuzz); + factor = q->fuzz_level * q->fuzz_level / (afl->n_fuzz[q->exec_cksum % n_fuzz_size] + 1); break; case MMOPT: @@ -703,7 +727,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { // the more often fuzz result paths are equal to this queue entry, // reduce its value perf_score *= - (1 - (double)((double)q->n_fuzz / (double)afl->fsrv.total_execs)); + (1 - (double)((double)afl->n_fuzz[q->exec_cksum % n_fuzz_size] / (double)afl->fsrv.total_execs)); break; @@ -724,7 +748,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { perf_score *= 2; - } else if (perf_score < 1) { + } else if (afl->schedule != COE && perf_score < 1) { // Add a lower bound to AFLFast's energy assignment strategies perf_score = 1; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 28507857..889f753d 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -936,6 +936,13 @@ int main(int argc, char **argv_orig, char **envp) { } + /* Dynamically allocate memory for AFLFast schedules */ + if (afl->schedule >= FAST && afl->schedule <= RARE) { + + afl->n_fuzz = ck_alloc(n_fuzz_size * sizeof(u32)); + + } + if (get_afl_env("AFL_NO_FORKSRV")) { afl->no_forkserver = 1; } if (get_afl_env("AFL_NO_CPU_RED")) { afl->no_cpu_meter_red = 1; } if (get_afl_env("AFL_NO_ARITH")) { afl->no_arith = 1; } -- cgit 1.4.1 From 383cd487a2c28012c80341f8517e473120af4d19 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Tue, 29 Sep 2020 15:02:57 +0200 Subject: small improvements to Marcel's patch, fix laf-intel + redqueen crashes --- docs/Changelog.md | 1 + include/afl-fuzz.h | 5 +-- instrumentation/afl-llvm-dict2file.so.cc | 14 ++++++--- instrumentation/cmplog-instructions-pass.cc | 6 ++-- instrumentation/cmplog-routines-pass.cc | 11 ++++--- instrumentation/compare-transform-pass.so.cc | 24 ++++++++------ instrumentation/split-compares-pass.so.cc | 47 +++++++++++++++++----------- instrumentation/split-switches-pass.so.cc | 21 +++++++------ src/afl-fuzz-bitmap.c | 13 +++++--- src/afl-fuzz-init.c | 4 +-- src/afl-fuzz-queue.c | 30 +++++++++--------- src/afl-fuzz.c | 2 +- 12 files changed, 104 insertions(+), 74 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/docs/Changelog.md b/docs/Changelog.md index 789b1f74..0f923423 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -14,6 +14,7 @@ sending a mail to . - all compilers combined to afl-cc which emulates the previous ones - afl-llvm/gcc-rt.o merged into afl-compiler-rt.o - afl-fuzz + - Marcel Boehme submitted a patch that improves all AFFast schedules :) - reading testcases from -i now descends into subdirectories - allow up to 4 -x command line options - loaded extras now have a duplicate protection diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index f65fc40f..fb661ce5 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -151,7 +151,8 @@ struct queue_entry { is_ascii; /* Is the input just ascii text? */ u32 bitmap_size, /* Number of bits set in bitmap */ - fuzz_level; /* Number of fuzzing iterations */ + fuzz_level, /* Number of fuzzing iterations */ + n_fuzz_entry; /* offset in n_fuzz */ u64 exec_us, /* Execution time (us) */ handicap, /* Number of queue cycles behind */ @@ -491,7 +492,7 @@ typedef struct afl_state { u8 *var_bytes; /* Bytes that appear to be variable */ - #define n_fuzz_size (1 << 21) +#define N_FUZZ_SIZE (1 << 21) u32 *n_fuzz; volatile u8 stop_soon, /* Ctrl-C pressed? */ diff --git a/instrumentation/afl-llvm-dict2file.so.cc b/instrumentation/afl-llvm-dict2file.so.cc index e04ebda8..bd8eb27a 100644 --- a/instrumentation/afl-llvm-dict2file.so.cc +++ b/instrumentation/afl-llvm-dict2file.so.cc @@ -381,8 +381,9 @@ bool AFLdict2filePass::runOnModule(Module &M) { if (debug) fprintf(stderr, "F:%s %p(%s)->\"%s\"(%s) %p(%s)->\"%s\"(%s)\n", - FuncName.c_str(), (void*)Str1P, Str1P->getName().str().c_str(), - Str1.c_str(), HasStr1 == true ? "true" : "false", (void*)Str2P, + FuncName.c_str(), (void *)Str1P, + Str1P->getName().str().c_str(), Str1.c_str(), + HasStr1 == true ? "true" : "false", (void *)Str2P, Str2P->getName().str().c_str(), Str2.c_str(), HasStr2 == true ? "true" : "false"); @@ -436,7 +437,8 @@ bool AFLdict2filePass::runOnModule(Module &M) { valueMap[Str1P] = new std::string(Str2); if (debug) - fprintf(stderr, "Saved: %s for %p\n", Str2.c_str(), (void*)Str1P); + fprintf(stderr, "Saved: %s for %p\n", Str2.c_str(), + (void *)Str1P); continue; } @@ -455,7 +457,8 @@ bool AFLdict2filePass::runOnModule(Module &M) { Str2 = *strng; HasStr2 = true; if (debug) - fprintf(stderr, "Filled2: %s for %p\n", strng->c_str(), (void*)Str2P); + fprintf(stderr, "Filled2: %s for %p\n", strng->c_str(), + (void *)Str2P); } @@ -497,7 +500,8 @@ bool AFLdict2filePass::runOnModule(Module &M) { Str1 = *strng; HasStr1 = true; if (debug) - fprintf(stderr, "Filled1: %s for %p\n", strng->c_str(), (void*)Str1P); + fprintf(stderr, "Filled1: %s for %p\n", strng->c_str(), + (void *)Str1P); } diff --git a/instrumentation/cmplog-instructions-pass.cc b/instrumentation/cmplog-instructions-pass.cc index d5de3dbb..9921de0c 100644 --- a/instrumentation/cmplog-instructions-pass.cc +++ b/instrumentation/cmplog-instructions-pass.cc @@ -210,7 +210,8 @@ bool CmpLogInstructions::hookInstrs(Module &M) { } if (!icomps.size()) return false; - if (!be_quiet) errs() << "Hooking " << icomps.size() << " cmp instructions\n"; + // if (!be_quiet) errs() << "Hooking " << icomps.size() << " cmp + // instructions\n"; for (auto &selectcmpInst : icomps) { @@ -259,8 +260,7 @@ bool CmpLogInstructions::hookInstrs(Module &M) { bool CmpLogInstructions::runOnModule(Module &M) { if (getenv("AFL_QUIET") == NULL) - llvm::errs() - << "Running cmplog-instructions-pass by andreafioraldi@gmail.com\n"; + printf("Running cmplog-instructions-pass by andreafioraldi@gmail.com\n"); else be_quiet = 1; hookInstrs(M); diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index c44f38c4..e92883ae 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -149,9 +149,11 @@ bool CmpLogRoutines::hookRtns(Module &M) { } if (!calls.size()) return false; - if (!be_quiet) - errs() << "Hooking " << calls.size() - << " calls with pointers as arguments\n"; + /* + if (!be_quiet) + errs() << "Hooking " << calls.size() + << " calls with pointers as arguments\n"; + */ for (auto &callInst : calls) { @@ -179,8 +181,7 @@ bool CmpLogRoutines::hookRtns(Module &M) { bool CmpLogRoutines::runOnModule(Module &M) { if (getenv("AFL_QUIET") == NULL) - llvm::errs() - << "Running cmplog-routines-pass by andreafioraldi@gmail.com\n"; + printf("Running cmplog-routines-pass by andreafioraldi@gmail.com\n"); else be_quiet = 1; hookRtns(M); diff --git a/instrumentation/compare-transform-pass.so.cc b/instrumentation/compare-transform-pass.so.cc index 9d2f4a92..3a4abd6e 100644 --- a/instrumentation/compare-transform-pass.so.cc +++ b/instrumentation/compare-transform-pass.so.cc @@ -339,8 +339,9 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, if (!calls.size()) return false; if (!be_quiet) - errs() << "Replacing " << calls.size() - << " calls to strcmp/memcmp/strncmp/strcasecmp/strncasecmp\n"; + printf( + "Replacing %lu calls to strcmp/memcmp/strncmp/strcasecmp/strncasecmp\n", + calls.size()); for (auto &callInst : calls) { @@ -426,11 +427,14 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, else unrollLen = constStrLen; - if (!be_quiet) - errs() << callInst->getCalledFunction()->getName() << ": unroll len " - << unrollLen - << ((isSizedcmp && !isConstSized) ? ", variable n" : "") << ": " - << ConstStr << "\n"; + /* + if (!be_quiet) + errs() << callInst->getCalledFunction()->getName() << ": unroll len " + << unrollLen + << ((isSizedcmp && !isConstSized) ? ", variable n" : "") << ": + " + << ConstStr << "\n"; + */ /* split before the call instruction */ BasicBlock *bb = callInst->getParent(); @@ -556,10 +560,12 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, bool CompareTransform::runOnModule(Module &M) { if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) - llvm::errs() << "Running compare-transform-pass by laf.intel@gmail.com, " - "extended by heiko@hexco.de\n"; + printf( + "Running compare-transform-pass by laf.intel@gmail.com, extended by " + "heiko@hexco.de\n"); else be_quiet = 1; + transformCmps(M, true, true, true, true, true); verifyModule(M); diff --git a/instrumentation/split-compares-pass.so.cc b/instrumentation/split-compares-pass.so.cc index 2fb90e5e..6d0c52a4 100644 --- a/instrumentation/split-compares-pass.so.cc +++ b/instrumentation/split-compares-pass.so.cc @@ -1262,8 +1262,9 @@ bool SplitComparesTransform::runOnModule(Module &M) { if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) { - errs() << "Split-compare-pass by laf.intel@gmail.com, extended by " - "heiko@hexco.de\n"; + printf( + "Split-compare-pass by laf.intel@gmail.com, extended by " + "heiko@hexco.de\n"); } else { @@ -1275,13 +1276,15 @@ bool SplitComparesTransform::runOnModule(Module &M) { count = splitFPCompares(M); - if (!be_quiet) { + /* + if (!be_quiet) { - errs() << "Split-floatingpoint-compare-pass: " << count - << " FP comparisons split\n"; + errs() << "Split-floatingpoint-compare-pass: " << count + << " FP comparisons split\n"; - } + } + */ simplifyFPCompares(M); } @@ -1294,10 +1297,12 @@ bool SplitComparesTransform::runOnModule(Module &M) { case 64: count = splitIntCompares(M, bitw); - if (!be_quiet) - errs() << "Split-integer-compare-pass " << bitw << "bit: " << count - << " split\n"; - + /* + if (!be_quiet) + errs() << "Split-integer-compare-pass " << bitw << "bit: " << + count + << " split\n"; + */ bitw >>= 1; #if LLVM_VERSION_MAJOR > 3 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 7) @@ -1305,10 +1310,12 @@ bool SplitComparesTransform::runOnModule(Module &M) { #endif case 32: count = splitIntCompares(M, bitw); - if (!be_quiet) - errs() << "Split-integer-compare-pass " << bitw << "bit: " << count - << " split\n"; - + /* + if (!be_quiet) + errs() << "Split-integer-compare-pass " << bitw << "bit: " << + count + << " split\n"; + */ bitw >>= 1; #if LLVM_VERSION_MAJOR > 3 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 7) @@ -1316,15 +1323,17 @@ bool SplitComparesTransform::runOnModule(Module &M) { #endif case 16: count = splitIntCompares(M, bitw); - if (!be_quiet) - errs() << "Split-integer-compare-pass " << bitw << "bit: " << count - << " split\n"; - + /* + if (!be_quiet) + errs() << "Split-integer-compare-pass " << bitw << "bit: " << + count + << " split\n"; + */ bitw >>= 1; break; default: - if (!be_quiet) errs() << "NOT Running split-compare-pass \n"; + // if (!be_quiet) errs() << "NOT Running split-compare-pass \n"; return false; break; diff --git a/instrumentation/split-switches-pass.so.cc b/instrumentation/split-switches-pass.so.cc index a79d4114..97ab04a4 100644 --- a/instrumentation/split-switches-pass.so.cc +++ b/instrumentation/split-switches-pass.so.cc @@ -327,10 +327,11 @@ bool SplitSwitchesTransform::splitSwitches(Module &M) { } if (!switches.size()) return false; - if (!be_quiet) - errs() << "Rewriting " << switches.size() << " switch statements " - << "\n"; - + /* + if (!be_quiet) + errs() << "Rewriting " << switches.size() << " switch statements " + << "\n"; + */ for (auto &SI : switches) { BasicBlock *CurBlock = SI->getParent(); @@ -341,15 +342,17 @@ bool SplitSwitchesTransform::splitSwitches(Module &M) { BasicBlock *Default = SI->getDefaultDest(); unsigned bitw = Val->getType()->getIntegerBitWidth(); - if (!be_quiet) - errs() << "switch: " << SI->getNumCases() << " cases " << bitw - << " bit\n"; + /* + if (!be_quiet) + errs() << "switch: " << SI->getNumCases() << " cases " << bitw + << " bit\n"; + */ /* If there is only the default destination or the condition checks 8 bit or * less, don't bother with the code below. */ if (!SI->getNumCases() || bitw <= 8) { - if (!be_quiet) errs() << "skip trivial switch..\n"; + // if (!be_quiet) errs() << "skip trivial switch..\n"; continue; } @@ -415,7 +418,7 @@ bool SplitSwitchesTransform::splitSwitches(Module &M) { bool SplitSwitchesTransform::runOnModule(Module &M) { if ((isatty(2) && getenv("AFL_QUIET") == NULL) || getenv("AFL_DEBUG") != NULL) - llvm::errs() << "Running split-switches-pass by laf.intel@gmail.com\n"; + printf("Running split-switches-pass by laf.intel@gmail.com\n"); else be_quiet = 1; splitSwitches(M); diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 64de86a2..a22223b9 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -556,8 +556,8 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); /* Saturated increment */ - if (afl->n_fuzz[cksum % n_fuzz_size] < 0xFFFFFFFF) - afl->n_fuzz[cksum % n_fuzz_size]++; + if (afl->n_fuzz[cksum % N_FUZZ_SIZE] < 0xFFFFFFFF) + afl->n_fuzz[cksum % N_FUZZ_SIZE]++; } @@ -597,10 +597,15 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { if (cksum) afl->queue_top->exec_cksum = cksum; else - afl->queue_top->exec_cksum = + cksum = afl->queue_top->exec_cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); - afl->n_fuzz[cksum % n_fuzz_size] = 1; + if (afl->schedule >= FAST && afl->schedule <= RARE) { + + afl->queue_top->n_fuzz_entry = cksum % N_FUZZ_SIZE; + afl->n_fuzz[afl->queue_top->n_fuzz_entry] = 1; + + } /* Try to calibrate inline; this also calls update_bitmap_score() when successful. */ diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index b825837f..65478a78 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -732,8 +732,8 @@ void read_testcases(afl_state_t *afl, u8 *directory) { if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) { u64 cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); - - afl->n_fuzz[cksum % n_fuzz_size] = 1; + afl->queue_top->n_fuzz_entry = cksum % N_FUZZ_SIZE; + afl->n_fuzz[afl->queue_top->n_fuzz_entry] = 1; } diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index dfabba7b..0d7d0314 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -308,9 +308,9 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { u64 fuzz_p2; if (unlikely(afl->schedule >= FAST && afl->schedule < RARE)) - fuzz_p2 = 0; // Skip the fuzz_p2 comparison + fuzz_p2 = 0; // Skip the fuzz_p2 comparison else if (unlikely(afl->schedule == RARE)) - fuzz_p2 = next_pow2(afl->n_fuzz[q->exec_cksum % n_fuzz_size]); + fuzz_p2 = next_pow2(afl->n_fuzz[q->n_fuzz_entry]); else fuzz_p2 = q->fuzz_level; @@ -336,7 +336,8 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { u64 top_rated_fav_factor; u64 top_rated_fuzz_p2; if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) - top_rated_fuzz_p2 = next_pow2(afl->n_fuzz[afl->top_rated[i]->exec_cksum % n_fuzz_size]); + top_rated_fuzz_p2 = + next_pow2(afl->n_fuzz[afl->top_rated[i]->n_fuzz_entry]); else top_rated_fuzz_p2 = afl->top_rated[i]->fuzz_level; @@ -607,11 +608,10 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { } - u32 n_paths; - double factor = 1.0; + u32 n_paths; + double factor = 1.0; long double fuzz_mu; - switch (afl->schedule) { case EXPLORE: @@ -634,7 +634,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { struct queue_entry *queue_it = afl->queue; while (queue_it) { - fuzz_mu += log2(afl->n_fuzz[q->exec_cksum % n_fuzz_size]); + fuzz_mu += log2(afl->n_fuzz[q->n_fuzz_entry]); n_paths++; queue_it = queue_it->next; @@ -645,7 +645,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { fuzz_mu = fuzz_mu / n_paths; - if (log2(afl->n_fuzz[q->exec_cksum % n_fuzz_size]) > fuzz_mu) { + if (log2(afl->n_fuzz[q->n_fuzz_entry]) > fuzz_mu) { /* Never skip favourites */ if (!q->favored) factor = 0; @@ -660,7 +660,7 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { // Don't modify unfuzzed seeds if (q->fuzz_level == 0) break; - switch ((u32)log2(afl->n_fuzz[q->exec_cksum % n_fuzz_size])) { + switch ((u32)log2(afl->n_fuzz[q->n_fuzz_entry])) { case 0 ... 1: factor = 4; @@ -691,17 +691,17 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { } - if (q->favored) - factor *= 1.15; + if (q->favored) factor *= 1.15; break; case LIN: - factor = q->fuzz_level / (afl->n_fuzz[q->exec_cksum % n_fuzz_size] + 1); + factor = q->fuzz_level / (afl->n_fuzz[q->n_fuzz_entry] + 1); break; case QUAD: - factor = q->fuzz_level * q->fuzz_level / (afl->n_fuzz[q->exec_cksum % n_fuzz_size] + 1); + factor = + q->fuzz_level * q->fuzz_level / (afl->n_fuzz[q->n_fuzz_entry] + 1); break; case MMOPT: @@ -726,8 +726,8 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { perf_score += (q->tc_ref * 10); // the more often fuzz result paths are equal to this queue entry, // reduce its value - perf_score *= - (1 - (double)((double)afl->n_fuzz[q->exec_cksum % n_fuzz_size] / (double)afl->fsrv.total_execs)); + perf_score *= (1 - (double)((double)afl->n_fuzz[q->n_fuzz_entry] / + (double)afl->fsrv.total_execs)); break; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 889f753d..273d1c14 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -939,7 +939,7 @@ int main(int argc, char **argv_orig, char **envp) { /* Dynamically allocate memory for AFLFast schedules */ if (afl->schedule >= FAST && afl->schedule <= RARE) { - afl->n_fuzz = ck_alloc(n_fuzz_size * sizeof(u32)); + afl->n_fuzz = ck_alloc(N_FUZZ_SIZE * sizeof(u32)); } -- cgit 1.4.1 From a4b60ca5b61c9bca5fa7b67528baeb3a8ea9320e Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Tue, 6 Oct 2020 15:37:59 +0200 Subject: testcase cache added --- include/afl-fuzz.h | 15 ++++++ include/config.h | 8 +++ src/afl-fuzz-one.c | 148 ++++++++++++++++++--------------------------------- src/afl-fuzz-queue.c | 65 ++++++++++++++++++++++ src/afl-fuzz.c | 3 +- 5 files changed, 142 insertions(+), 97 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index fb661ce5..46da8c7d 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -162,6 +162,9 @@ struct queue_entry { u8 *trace_mini; /* Trace bytes, if kept */ u32 tc_ref; /* Trace bytes ref count */ + u8 *testcase_buf; /* The testcase buffer, if loaded. */ + u32 testcase_refs; /* count of users of testcase buf */ + struct queue_entry *next; /* Next element, if any */ }; @@ -664,6 +667,11 @@ typedef struct afl_state { /* queue entries ready for splicing count (len > 4) */ u32 ready_for_splicing_count; + /* How many queue entries currently have cached testcases */ + u32 q_testcase_cache_count; + /* Refs to each queue entry with cached testcase (for eviction, if cache_count is too large) */ + struct queue_entry *q_testcase_cache[TESTCASE_CACHE_SIZE]; + } afl_state_t; struct custom_mutator { @@ -1101,5 +1109,12 @@ static inline u64 next_p2(u64 val) { } +/* Returns the testcase buf from the file behind this queue entry. + Increases the refcount. */ +u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q); + +/* Tell afl that this testcase may be evicted from the cache */ +void queue_testcase_release(afl_state_t *afl, struct queue_entry *q); + #endif diff --git a/include/config.h b/include/config.h index 7c8e0c7d..38a734ce 100644 --- a/include/config.h +++ b/include/config.h @@ -295,6 +295,14 @@ #define RESEED_RNG 100000 +/* The amount of entries in the testcase cache, held in memory. +Decrease if RAM usage is high. */ +#define TESTCASE_CACHE_SIZE 2048 + +#if TESTCASE_CACHE_SIZE < 4 + #error "Dangerously low cache size: Set TESTCASE_CACHE_SIZE to 4 or more in config.h! +#endif + /* Maximum line length passed from GCC to 'as' and used for parsing configuration files: */ diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index c04b492b..20558618 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -370,7 +370,7 @@ static void locate_diffs(u8 *ptr1, u8 *ptr2, u32 len, s32 *first, s32 *last) { u8 fuzz_one_original(afl_state_t *afl) { - s32 len, fd, temp_len; + s32 len, temp_len; u32 j; u32 i; u8 *in_buf, *out_buf, *orig_in, *ex_tmp, *eff_map = 0; @@ -453,28 +453,9 @@ u8 fuzz_one_original(afl_state_t *afl) { } - /* Map the test case into memory. */ - - fd = open(afl->queue_cur->fname, O_RDONLY); - - if (unlikely(fd < 0)) { - - PFATAL("Unable to open '%s'", afl->queue_cur->fname); - - } - + orig_in = in_buf = queue_testcase_take(afl, afl->queue_cur); len = afl->queue_cur->len; - orig_in = in_buf = mmap(0, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - - if (unlikely(orig_in == MAP_FAILED)) { - - PFATAL("Unable to mmap '%s' with len %d", afl->queue_cur->fname, len); - - } - - close(fd); - /* We could mmap() out_buf as MAP_PRIVATE, but we end up clobbering every single byte anyway, so it wouldn't give us any performance or memory usage benefits. */ @@ -1694,7 +1675,7 @@ custom_mutator_stage: for (afl->stage_cur = 0; afl->stage_cur < afl->stage_max; ++afl->stage_cur) { - struct queue_entry *target; + struct queue_entry *target = NULL; u32 tid; u8 * new_buf = NULL; u32 target_len = 0; @@ -1717,17 +1698,7 @@ custom_mutator_stage: afl->splicing_with = tid; /* Read the additional testcase into a new buffer. */ - fd = open(target->fname, O_RDONLY); - if (unlikely(fd < 0)) { - - PFATAL("Unable to open '%s'", target->fname); - - } - - new_buf = afl_realloc(AFL_BUF_PARAM(out_scratch), target->len); - if (unlikely(!new_buf)) { PFATAL("alloc"); } - ck_read(fd, new_buf, target->len, target->fname); - close(fd); + new_buf = queue_testcase_take(afl, target); target_len = target->len; } @@ -1738,6 +1709,11 @@ custom_mutator_stage: el->afl_custom_fuzz(el->data, out_buf, len, &mutated_buf, new_buf, target_len, max_seed_size); + if (new_buf) { + queue_testcase_release(afl, target); + new_buf = NULL; + } + if (unlikely(!mutated_buf)) { FATAL("Error in custom_fuzz. Size returned: %zd", mutated_size); @@ -2320,51 +2296,44 @@ havoc_stage: /* Overwrite bytes with a randomly selected chunk from another testcase or insert that chunk. */ - if (afl->queued_paths < 4) break; + if (afl->queued_paths < 4) { break; } /* Pick a random queue entry and seek to it. */ u32 tid; - do + do { tid = rand_below(afl, afl->queued_paths); - while (tid == afl->current_entry); + } while (tid == afl->current_entry); struct queue_entry *target = afl->queue_buf[tid]; /* Make sure that the target has a reasonable length. */ - while (target && (target->len < 2 || target == afl->queue_cur)) + while (target && (target->len < 2 || target == afl->queue_cur)) { target = target->next; + } - if (!target) break; - - /* Read the testcase into a new buffer. */ - - fd = open(target->fname, O_RDONLY); - - if (unlikely(fd < 0)) { + if (!target) { break; } - PFATAL("Unable to open '%s'", target->fname); - - } u32 new_len = target->len; - u8 *new_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), new_len); - if (unlikely(!new_buf)) { PFATAL("alloc"); } - - ck_read(fd, new_buf, new_len, target->fname); - close(fd); + /* Get the testcase contents for splicing. */ + u8 *new_buf = queue_testcase_take(afl, target); u8 overwrite = 0; - if (temp_len >= 2 && rand_below(afl, 2)) + if (temp_len >= 2 && rand_below(afl, 2)) { overwrite = 1; + } else if (temp_len + HAVOC_BLK_XL >= MAX_FILE) { - if (temp_len >= 2) + if (temp_len >= 2) { overwrite = 1; - else + } else { + queue_testcase_release(afl, target); + new_buf = NULL; break; + } } @@ -2411,6 +2380,9 @@ havoc_stage: } + /* We don't need this splice testcase anymore */ + queue_testcase_release(afl, target); + new_buf = NULL; break; } @@ -2516,24 +2488,16 @@ retry_splicing: if (!target) { goto retry_splicing; } - /* Read the testcase into a new buffer. */ - - fd = open(target->fname, O_RDONLY); - - if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", target->fname); } - + /* Get the testcase buffer */ + u8 *splice_buf = queue_testcase_take(afl, target); new_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), target->len); if (unlikely(!new_buf)) { PFATAL("alloc"); } - ck_read(fd, new_buf, target->len, target->fname); - - close(fd); - /* Find a suitable splicing location, somewhere between the first and the last differing byte. Bail out if the difference is just a single byte or so. */ - locate_diffs(in_buf, new_buf, MIN(len, (s64)target->len), &f_diff, &l_diff); + locate_diffs(in_buf, splice_buf, MIN(len, (s64)target->len), &f_diff, &l_diff); if (f_diff < 0 || l_diff < 2 || f_diff == l_diff) { goto retry_splicing; } @@ -2545,6 +2509,7 @@ retry_splicing: len = target->len; memcpy(new_buf, in_buf, split_at); + memcpy(new_buf + split_at, splice_buf + split_at, target->len - split_at); afl_swap_bufs(AFL_BUF_PARAM(in), AFL_BUF_PARAM(in_scratch)); in_buf = new_buf; @@ -2552,6 +2517,9 @@ retry_splicing: if (unlikely(!out_buf)) { PFATAL("alloc"); } memcpy(out_buf, in_buf, len); + queue_testcase_release(afl, target); + splice_buf = NULL; + goto custom_mutator_stage; /* ???: While integrating Python module, the author decided to jump to python stage, but the reason behind this is not clear.*/ @@ -2582,7 +2550,8 @@ abandon_entry: ++afl->queue_cur->fuzz_level; - munmap(orig_in, afl->queue_cur->len); + queue_testcase_release(afl, afl->queue_cur); + orig_in = NULL; return ret_val; @@ -2604,7 +2573,7 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { } - s32 len, fd, temp_len; + s32 len, temp_len; u32 i; u32 j; u8 *in_buf, *out_buf, *orig_in, *ex_tmp, *eff_map = 0; @@ -2669,23 +2638,9 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { } /* Map the test case into memory. */ - - fd = open(afl->queue_cur->fname, O_RDONLY); - - if (fd < 0) { PFATAL("Unable to open '%s'", afl->queue_cur->fname); } - + orig_in = in_buf = queue_testcase_take(afl, afl->queue_cur); len = afl->queue_cur->len; - orig_in = in_buf = mmap(0, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - - if (orig_in == MAP_FAILED) { - - PFATAL("Unable to mmap '%s'", afl->queue_cur->fname); - - } - - close(fd); - /* We could mmap() out_buf as MAP_PRIVATE, but we end up clobbering every single byte anyway, so it wouldn't give us any performance or memory usage benefits. */ @@ -4522,31 +4477,24 @@ pacemaker_fuzzing: if (!target) { goto retry_splicing_puppet; } /* Read the testcase into a new buffer. */ - - fd = open(target->fname, O_RDONLY); - - if (fd < 0) { PFATAL("Unable to open '%s'", target->fname); } - - new_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), target->len); - if (unlikely(!new_buf)) { PFATAL("alloc"); } - - ck_read(fd, new_buf, target->len, target->fname); - - close(fd); + u8 *splicing_buf = queue_testcase_take(afl, target); /* Find a suitable splicin g location, somewhere between the first and the last differing byte. Bail out if the difference is just a single byte or so. */ - locate_diffs(in_buf, new_buf, MIN(len, (s32)target->len), &f_diff, + locate_diffs(in_buf, splicing_buf, MIN(len, (s32)target->len), &f_diff, &l_diff); if (f_diff < 0 || l_diff < 2 || f_diff == l_diff) { + queue_testcase_release(afl, target); goto retry_splicing_puppet; } + new_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), target->len); + /* Split somewhere between the first and last differing byte. */ split_at = f_diff + rand_below(afl, l_diff - f_diff); @@ -4555,12 +4503,16 @@ pacemaker_fuzzing: len = target->len; memcpy(new_buf, in_buf, split_at); + memcpy(new_buf + split_at, splicing_buf + split_at, target->len - split_at); afl_swap_bufs(AFL_BUF_PARAM(in), AFL_BUF_PARAM(in_scratch)); in_buf = new_buf; out_buf = afl_realloc(AFL_BUF_PARAM(out), len); if (unlikely(!out_buf)) { PFATAL("alloc"); } memcpy(out_buf, in_buf, len); + queue_testcase_release(afl, target); + splicing_buf = NULL; + goto havoc_stage_puppet; } /* if splice_cycle */ @@ -4594,7 +4546,8 @@ pacemaker_fuzzing: // if (afl->queue_cur->favored) --afl->pending_favored; // } - munmap(orig_in, afl->queue_cur->len); + queue_testcase_release(afl, afl->queue_cur); + orig_in = NULL; if (afl->key_puppet == 1) { @@ -4730,6 +4683,9 @@ pacemaker_fuzzing: } /* block */ + queue_testcase_release(afl, afl->queue_cur); + orig_in = NULL; + return ret_val; } diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 0d7d0314..e2387aaa 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -220,6 +220,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { q->depth = afl->cur_depth + 1; q->passed_det = passed_det; q->trace_mini = NULL; + q->testcase_buf = NULL; if (q->depth > afl->max_depth) { afl->max_depth = q->depth; } @@ -767,3 +768,67 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { } +/* Tell afl that this testcase may be evicted from the cache */ +inline void queue_testcase_release(afl_state_t *afl, struct queue_entry *q) { + (void) afl; + q->testcase_refs--; + if (unlikely(q->testcase_refs < 0)) { FATAL("Testcase refcount smaller than 0"); } +} + +/* Returns the testcase buf from the file behind this queue entry. + Increases the refcount. */ +u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q) { + if (!q->testcase_buf) { + u32 tid = 0; + /* Buf not cached, let's do that now */ + + if (likely(afl->q_testcase_cache_count == TESTCASE_CACHE_SIZE)) { + /* Cache full. We neet to evict one to map one. + Get a random one which is not in use */ + do { + + tid = rand_below(afl, afl->q_testcase_cache_count); + + } while (afl->q_testcase_cache[tid]->testcase_refs > 0); + + struct queue_entry *old_cached = afl->q_testcase_cache[tid]; + /* free the current buf from cache */ + munmap(old_cached->testcase_buf, old_cached->len); + old_cached->testcase_buf = NULL; + + } else { + tid = afl->q_testcase_cache_count; + afl->q_testcase_cache_count++; + } + + /* Map the test case into memory. */ + + int fd = open(q->fname, O_RDONLY); + + if (unlikely(fd < 0)) { + + PFATAL("Unable to open '%s'", q->fname); + + } + + u32 len = q->len; + + q->testcase_buf = mmap(0, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + + if (unlikely(q->testcase_buf == MAP_FAILED)) { + + PFATAL("Unable to mmap '%s' with len %d", q->fname, len); + + } + + close(fd); + + /* Register us as cached */ + afl->q_testcase_cache[tid] = q; + + } + q->testcase_refs++; + if (!q->testcase_buf) { FATAL("Testcase buf is NULL, this should never happen"); } + return q->testcase_buf; + +} diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 2f8aa3fd..dd9aaa8f 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1152,8 +1152,9 @@ int main(int argc, char **argv_orig, char **envp) { if (extras_dir_cnt) { - for (i = 0; i < extras_dir_cnt; i++) + for (i = 0; i < extras_dir_cnt; i++) { load_extras(afl, extras_dir[i]); + } dedup_extras(afl); OKF("Loaded a total of %u extras.", afl->extras_cnt); -- cgit 1.4.1 From 74dc227c4412d0121c9b972e5d89db89f54c6b3a Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Tue, 6 Oct 2020 15:38:36 +0200 Subject: code format --- include/afl-fuzz.h | 5 +++-- include/config.h | 3 ++- src/afl-fuzz-one.c | 22 +++++++++++++++++----- src/afl-fuzz-queue.c | 34 ++++++++++++++++++++++++---------- src/afl-fuzz.c | 2 ++ 5 files changed, 48 insertions(+), 18 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 46da8c7d..5ab787e0 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -163,7 +163,7 @@ struct queue_entry { u32 tc_ref; /* Trace bytes ref count */ u8 *testcase_buf; /* The testcase buffer, if loaded. */ - u32 testcase_refs; /* count of users of testcase buf */ + u32 testcase_refs; /* count of users of testcase buf */ struct queue_entry *next; /* Next element, if any */ @@ -669,7 +669,8 @@ typedef struct afl_state { /* How many queue entries currently have cached testcases */ u32 q_testcase_cache_count; - /* Refs to each queue entry with cached testcase (for eviction, if cache_count is too large) */ + /* Refs to each queue entry with cached testcase (for eviction, if cache_count + * is too large) */ struct queue_entry *q_testcase_cache[TESTCASE_CACHE_SIZE]; } afl_state_t; diff --git a/include/config.h b/include/config.h index 38a734ce..ec378036 100644 --- a/include/config.h +++ b/include/config.h @@ -300,7 +300,8 @@ Decrease if RAM usage is high. */ #define TESTCASE_CACHE_SIZE 2048 #if TESTCASE_CACHE_SIZE < 4 - #error "Dangerously low cache size: Set TESTCASE_CACHE_SIZE to 4 or more in config.h! + #error \ + "Dangerously low cache size: Set TESTCASE_CACHE_SIZE to 4 or more in config.h! #endif /* Maximum line length passed from GCC to 'as' and used for parsing diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 20558618..a5f77f11 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -1710,8 +1710,10 @@ custom_mutator_stage: target_len, max_seed_size); if (new_buf) { + queue_testcase_release(afl, target); new_buf = NULL; + } if (unlikely(!mutated_buf)) { @@ -2302,7 +2304,9 @@ havoc_stage: u32 tid; do { + tid = rand_below(afl, afl->queued_paths); + } while (tid == afl->current_entry); struct queue_entry *target = afl->queue_buf[tid]; @@ -2310,12 +2314,13 @@ havoc_stage: /* Make sure that the target has a reasonable length. */ while (target && (target->len < 2 || target == afl->queue_cur)) { + target = target->next; + } if (!target) { break; } - u32 new_len = target->len; /* Get the testcase contents for splicing. */ @@ -2323,16 +2328,21 @@ havoc_stage: u8 overwrite = 0; if (temp_len >= 2 && rand_below(afl, 2)) { + overwrite = 1; - } - else if (temp_len + HAVOC_BLK_XL >= MAX_FILE) { + + } else if (temp_len + HAVOC_BLK_XL >= MAX_FILE) { if (temp_len >= 2) { + overwrite = 1; + } else { + queue_testcase_release(afl, target); new_buf = NULL; break; + } } @@ -2497,7 +2507,8 @@ retry_splicing: the last differing byte. Bail out if the difference is just a single byte or so. */ - locate_diffs(in_buf, splice_buf, MIN(len, (s64)target->len), &f_diff, &l_diff); + locate_diffs(in_buf, splice_buf, MIN(len, (s64)target->len), &f_diff, + &l_diff); if (f_diff < 0 || l_diff < 2 || f_diff == l_diff) { goto retry_splicing; } @@ -4503,7 +4514,8 @@ pacemaker_fuzzing: len = target->len; memcpy(new_buf, in_buf, split_at); - memcpy(new_buf + split_at, splicing_buf + split_at, target->len - split_at); + memcpy(new_buf + split_at, splicing_buf + split_at, + target->len - split_at); afl_swap_bufs(AFL_BUF_PARAM(in), AFL_BUF_PARAM(in_scratch)); in_buf = new_buf; out_buf = afl_realloc(AFL_BUF_PARAM(out), len); diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index e2387aaa..721f9ac7 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -770,24 +770,33 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { /* Tell afl that this testcase may be evicted from the cache */ inline void queue_testcase_release(afl_state_t *afl, struct queue_entry *q) { - (void) afl; + + (void)afl; q->testcase_refs--; - if (unlikely(q->testcase_refs < 0)) { FATAL("Testcase refcount smaller than 0"); } + if (unlikely(q->testcase_refs < 0)) { + + FATAL("Testcase refcount smaller than 0"); + + } + } /* Returns the testcase buf from the file behind this queue entry. Increases the refcount. */ u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q) { + if (!q->testcase_buf) { + u32 tid = 0; /* Buf not cached, let's do that now */ if (likely(afl->q_testcase_cache_count == TESTCASE_CACHE_SIZE)) { + /* Cache full. We neet to evict one to map one. Get a random one which is not in use */ do { - tid = rand_below(afl, afl->q_testcase_cache_count); + tid = rand_below(afl, afl->q_testcase_cache_count); } while (afl->q_testcase_cache[tid]->testcase_refs > 0); @@ -795,21 +804,19 @@ u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q) { /* free the current buf from cache */ munmap(old_cached->testcase_buf, old_cached->len); old_cached->testcase_buf = NULL; - + } else { + tid = afl->q_testcase_cache_count; afl->q_testcase_cache_count++; + } /* Map the test case into memory. */ int fd = open(q->fname, O_RDONLY); - if (unlikely(fd < 0)) { - - PFATAL("Unable to open '%s'", q->fname); - - } + if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); } u32 len = q->len; @@ -827,8 +834,15 @@ u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q) { afl->q_testcase_cache[tid] = q; } + q->testcase_refs++; - if (!q->testcase_buf) { FATAL("Testcase buf is NULL, this should never happen"); } + if (!q->testcase_buf) { + + FATAL("Testcase buf is NULL, this should never happen"); + + } + return q->testcase_buf; } + diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index dd9aaa8f..9b7c1445 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1153,7 +1153,9 @@ int main(int argc, char **argv_orig, char **envp) { if (extras_dir_cnt) { for (i = 0; i < extras_dir_cnt; i++) { + load_extras(afl, extras_dir[i]); + } dedup_extras(afl); -- cgit 1.4.1 From 4f207b4eba26c2b268ba2fd0a51298d6ab88f110 Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Tue, 6 Oct 2020 16:20:32 +0200 Subject: fixed ref check --- src/afl-fuzz-queue.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 721f9ac7..58e026f5 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -772,13 +772,14 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { inline void queue_testcase_release(afl_state_t *afl, struct queue_entry *q) { (void)afl; - q->testcase_refs--; - if (unlikely(q->testcase_refs < 0)) { + if (unlikely(q->testcase_refs == 0)) { - FATAL("Testcase refcount smaller than 0"); + FATAL("Testcase refcount reduced past 0"); } + q->testcase_refs--; + } /* Returns the testcase buf from the file behind this queue entry. -- cgit 1.4.1 From 2d5fadc1e6a684b5e3e527a64b614f6b1ba8415f Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Tue, 6 Oct 2020 16:45:25 +0200 Subject: hunting ref underflow --- src/afl-fuzz-one.c | 3 --- src/afl-fuzz-queue.c | 11 +++++++++-- 2 files changed, 9 insertions(+), 5 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index a5f77f11..f25ab4ee 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -4695,9 +4695,6 @@ pacemaker_fuzzing: } /* block */ - queue_testcase_release(afl, afl->queue_cur); - orig_in = NULL; - return ret_val; } diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 58e026f5..0b491202 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -837,10 +837,17 @@ u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q) { } q->testcase_refs++; - if (!q->testcase_buf) { + if (unlikely(!q->testcase_buf || !q->testcase_refs)) { + if (!q->testcase_buf) { + + FATAL("Testcase buf is NULL, this should never happen"); - FATAL("Testcase buf is NULL, this should never happen"); + } + if (!q->testcase_refs) { + FATAL("Testcase ref overflow. Missing a testcase release somwhere?"); + + } } return q->testcase_buf; -- cgit 1.4.1 From 6a397d6111a21ebbf736237609c1c69d47c40f03 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sun, 11 Oct 2020 14:31:31 +0200 Subject: add new seed selection algo and make it the default --- docs/Changelog.md | 4 ++ include/afl-fuzz.h | 14 +++++- src/afl-fuzz-init.c | 8 ++++ src/afl-fuzz-one.c | 10 ++++- src/afl-fuzz-queue.c | 124 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/afl-fuzz.c | 100 +++++++++++++++++++++++++++++------------ 6 files changed, 227 insertions(+), 33 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/docs/Changelog.md b/docs/Changelog.md index 9eb47e18..f15f1d93 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -17,6 +17,10 @@ sending a mail to . - memory limits are now disabled by default, set them with -m if required - deterministic fuzzing is now disabled by default and can be enabled with -D. It is still enabled by default for -M. + - a new seed selection was implemented that uses weighted randoms based on + a schedule performance score, which is much better that the previous + walk the whole queue approach. Select the old mode with -Z (auto enabled + with -M) - statsd support by Edznux, thanks a lot! - Marcel Boehme submitted a patch that improves all AFFast schedules :) - reading testcases from -i now descends into subdirectories diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index e9d148e9..45de197d 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -151,7 +151,8 @@ struct queue_entry { favored, /* Currently favored? */ fs_redundant, /* Marked as redundant in the fs? */ fully_colorized, /* Do not run redqueen stage again */ - is_ascii; /* Is the input just ascii text? */ + is_ascii, /* Is the input just ascii text? */ + disabled; /* Is disabled from fuzz selection */ u32 bitmap_size, /* Number of bits set in bitmap */ fuzz_level, /* Number of fuzzing iterations */ @@ -165,6 +166,8 @@ struct queue_entry { u8 *trace_mini; /* Trace bytes, if kept */ u32 tc_ref; /* Trace bytes ref count */ + double perf_score; /* performance score */ + struct queue_entry *next; /* Next element, if any */ }; @@ -488,12 +491,17 @@ typedef struct afl_state { disable_trim, /* Never trim in fuzz_one */ shmem_testcase_mode, /* If sharedmem testcases are used */ expand_havoc, /* perform expensive havoc after no find */ - cycle_schedules; /* cycle power schedules? */ + cycle_schedules, /* cycle power schedules? */ + old_seed_selection; /* use vanilla afl seed selection */ u8 *virgin_bits, /* Regions yet untouched by fuzzing */ *virgin_tmout, /* Bits we haven't seen in tmouts */ *virgin_crash; /* Bits we haven't seen in crashes */ + double *alias_probability; /* alias weighted probabilities */ + u32 * alias_table; /* alias weighted random lookup table */ + u32 active_paths; /* enabled entries in the queue */ + u8 *var_bytes; /* Bytes that appear to be variable */ #define N_FUZZ_SIZE (1 << 21) @@ -1009,6 +1017,8 @@ void find_timeout(afl_state_t *); double get_runnable_processes(void); void nuke_resume_dir(afl_state_t *); int check_main_node_exists(afl_state_t *); +u32 select_next_queue_entry(afl_state_t *afl); +void create_alias_table(afl_state_t *afl); void setup_dirs_fds(afl_state_t *); void setup_cmdline_file(afl_state_t *, char **); void setup_stdio_file(afl_state_t *); diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 65478a78..881bf10f 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -959,6 +959,8 @@ void perform_dry_run(afl_state_t *afl) { /* Remove from fuzzing queue but keep for splicing */ struct queue_entry *p = afl->queue; + p->disabled = 1; + p->perf_score = 0; while (p && p->next != q) p = p->next; @@ -968,6 +970,7 @@ void perform_dry_run(afl_state_t *afl) { afl->queue = q->next; --afl->pending_not_fuzzed; + --afl->active_paths; afl->max_depth = 0; p = afl->queue; @@ -1054,6 +1057,7 @@ restart_outer_cull_loop: duplicates = 1; --afl->pending_not_fuzzed; + afl->active_paths--; // We do not remove any of the memory allocated because for // splicing the data might still be interesting. @@ -1063,11 +1067,15 @@ restart_outer_cull_loop: // we keep the shorter file if (p->len >= q->len) { + p->disabled = 1; + p->perf_score = 0; q->next = p->next; goto restart_inner_cull_loop; } else { + q->disabled = 1; + q->perf_score = 0; if (prev) prev->next = q = p; else diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index c04b492b..6ef728e0 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -554,7 +554,10 @@ u8 fuzz_one_original(afl_state_t *afl) { * PERFORMANCE SCORE * *********************/ - orig_perf = perf_score = calculate_score(afl, afl->queue_cur); + if (likely(!afl->old_seed_selection)) + orig_perf = perf_score = afl->queue_cur->perf_score; + else + orig_perf = perf_score = calculate_score(afl, afl->queue_cur); if (unlikely(perf_score == 0)) { goto abandon_entry; } @@ -2769,7 +2772,10 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { * PERFORMANCE SCORE * *********************/ - orig_perf = perf_score = calculate_score(afl, afl->queue_cur); + if (likely(!afl->old_seed_selection)) + orig_perf = perf_score = afl->queue_cur->perf_score; + else + orig_perf = perf_score = calculate_score(afl, afl->queue_cur); if (unlikely(afl->shm.cmplog_mode && !afl->queue_cur->fully_colorized)) { diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 0d7d0314..d608e890 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -27,6 +27,129 @@ #include #include +inline u32 select_next_queue_entry(afl_state_t *afl) { + + u32 r = rand_below(afl, 0xffffffff); + u32 s = r % afl->queued_paths; + // fprintf(stderr, "select: r=%u s=%u ... r < prob[s]=%f ? s=%u : + // alias[%u]=%u\n", r, s, afl->alias_probability[s], s, s, + // afl->alias_table[s]); + return (r < afl->alias_probability[s] ? s : afl->alias_table[s]); + +} + +void create_alias_table(afl_state_t *afl) { + + u32 n = afl->queued_paths, i = 0, a, g; + + afl->alias_table = + (u32 *)afl_realloc((void **)&afl->alias_table, n * sizeof(u32)); + afl->alias_probability = (double *)afl_realloc( + (void **)&afl->alias_probability, n * sizeof(double)); + double *P = (double *)afl_realloc(AFL_BUF_PARAM(out), n * sizeof(double)); + int * S = (u32 *)afl_realloc(AFL_BUF_PARAM(out_scratch), n * sizeof(u32)); + int * L = (u32 *)afl_realloc(AFL_BUF_PARAM(in_scratch), n * sizeof(u32)); + + if (!P || !S || !L) FATAL("could not aquire memory for alias table"); + memset((void *)afl->alias_table, 0, n * sizeof(u32)); + memset((void *)afl->alias_probability, 0, n * sizeof(double)); + + double sum = 0; + + for (i = 0; i < n; i++) { + + struct queue_entry *q = afl->queue_buf[i]; + + if (!q->disabled) q->perf_score = calculate_score(afl, q); + + sum += q->perf_score; + /* + if (afl->debug) + fprintf(stderr, "entry %u: score=%f %s (sum: %f)\n", i, q->perf_score, + q->disabled ? "disabled" : "", sum); + */ + + } + + for (i = 0; i < n; i++) { + + struct queue_entry *q = afl->queue_buf[i]; + + P[i] = q->perf_score * n / sum; + + } + + int nS = 0, nL = 0, s; + for (s = (s32)n - 1; s >= 0; --s) { + + if (P[s] < 1) + S[nS++] = s; + else + L[nL++] = s; + + } + + while (nS && nL) { + + a = S[--nS]; + g = L[--nL]; + afl->alias_probability[a] = P[a]; + afl->alias_table[a] = g; + P[g] = P[g] + P[a] - 1; + if (P[g] < 1) + S[nS++] = g; + else + L[nL++] = g; + + } + + while (nL) + afl->alias_probability[L[--nL]] = 1; + + while (nS) + afl->alias_probability[S[--nS]] = 1; + + /* + if (afl->debug) { + + fprintf(stderr, " %-3s %-3s %-9s\n", "entry", "alias", "prob"); + for (u32 i = 0; i < n; ++i) + fprintf(stderr, " %3i %3i %9.7f\n", i, afl->alias_table[i], + afl->alias_probability[i]); + + } + + int prob = 0; + fprintf(stderr, "Alias:"); + for (i = 0; i < n; i++) { + + fprintf(stderr, " [%u]=%u", i, afl->alias_table[i]); + if (afl->alias_table[i] >= n) + prob = i; + + } + + fprintf(stderr, "\n"); + + if (prob) { + + fprintf(stderr, "PROBLEM! alias[%u] = %u\n", prob, + afl->alias_table[prob]); + + for (i = 0; i < n; i++) { + + struct queue_entry *q = afl->queue_buf[i]; + + fprintf(stderr, "%u: score=%f\n", i, q->perf_score); + + } + + } + + */ + +} + /* Mark deterministic checks as done for a particular queue entry. We use the .state file to avoid repeating deterministic fuzzing when resuming aborted scans. */ @@ -237,6 +360,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { if (likely(q->len > 4)) afl->ready_for_splicing_count++; ++afl->queued_paths; + ++afl->active_paths; ++afl->pending_not_fuzzed; afl->cycles_wo_finds = 0; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 24df2997..004adffe 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -115,6 +115,8 @@ static void usage(u8 *argv0, int more_help) { " if using QEMU, just use -c 0.\n\n" "Fuzzing behavior settings:\n" + " -Z - sequential queue selection instead of weighted " + "random\n" " -N - do not unlink the fuzzing input file (for devices " "etc.)\n" " -n - fuzz without instrumentation (non-instrumented mode)\n" @@ -131,8 +133,7 @@ static void usage(u8 *argv0, int more_help) { "Other stuff:\n" " -M/-S id - distributed mode (see docs/parallel_fuzzing.md)\n" - " use -D to force -S secondary to perform deterministic " - "fuzzing\n" + " -M auto-sets -D and -Z (use -d to disable -D)\n" " -F path - sync to a foreign fuzzer queue directory (requires " "-M, can\n" " be specified up to %u times)\n" @@ -250,7 +251,7 @@ int main(int argc, char **argv_orig, char **envp) { s32 opt, i; u64 prev_queued = 0; - u32 sync_interval_cnt = 0, seek_to, show_help = 0, map_size = MAP_SIZE; + u32 sync_interval_cnt = 0, seek_to = 0, show_help = 0, map_size = MAP_SIZE; u8 *extras_dir[4]; u8 mem_limit_given = 0, exit_1 = 0, debug = 0, extras_dir_cnt = 0 /*, have_p = 0*/; @@ -287,10 +288,14 @@ int main(int argc, char **argv_orig, char **envp) { while ((opt = getopt( argc, argv, - "+b:c:i:I:o:f:F:m:t:T:dDnCB:S:M:x:QNUWe:p:s:V:E:L:hRP:")) > 0) { + "+b:c:i:I:o:f:F:m:t:T:dDnCB:S:M:x:QNUWe:p:s:V:E:L:hRP:Z")) > 0) { switch (opt) { + case 'Z': + afl->old_seed_selection = 1; + break; + case 'I': afl->infoexec = optarg; break; @@ -355,14 +360,16 @@ int main(int argc, char **argv_orig, char **envp) { afl->schedule = RARE; - } else if (!stricmp(optarg, "explore") || !stricmp(optarg, "afl")) { - - afl->schedule = EXPLORE; + } else if (!stricmp(optarg, "explore") || !stricmp(optarg, "afl") || - } else if (!stricmp(optarg, "seek") || !stricmp(optarg, "default") || + !stricmp(optarg, "default") || !stricmp(optarg, "normal")) { + afl->schedule = EXPLORE; + + } else if (!stricmp(optarg, "seek")) { + afl->schedule = SEEK; } else { @@ -404,7 +411,8 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->sync_id) { FATAL("Multiple -S or -M options not supported"); } afl->sync_id = ck_strdup(optarg); - afl->skip_deterministic = 0; + afl->skip_deterministic = 0; // force determinsitic fuzzing + afl->old_seed_selection = 1; // force old queue walking seed selection if ((c = strchr(afl->sync_id, ':'))) { @@ -1131,8 +1139,10 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->is_secondary_node && check_main_node_exists(afl) == 0) { - WARNF("no -M main node found. You need to run one main instance!"); - sleep(3); + WARNF( + "no -M main node found. It is recommended to run exactly one main " + "instance."); + sleep(1); } @@ -1302,7 +1312,7 @@ int main(int argc, char **argv_orig, char **envp) { show_init_stats(afl); - seek_to = find_start_position(afl); + if (unlikely(afl->old_seed_selection)) seek_to = find_start_position(afl); write_stats_file(afl, 0, 0, 0); maybe_update_plot_file(afl, 0, 0); @@ -1324,28 +1334,37 @@ int main(int argc, char **argv_orig, char **envp) { // real start time, we reset, so this works correctly with -V afl->start_time = get_cur_time(); + u32 runs_in_current_cycle = (u32)-1; + u32 prev_queued_paths = 0; + while (1) { u8 skipped_fuzz; cull_queue(afl); - if (!afl->queue_cur) { + if (unlikely((!afl->old_seed_selection && + runs_in_current_cycle > afl->queued_paths) || + (afl->old_seed_selection && !afl->queue_cur))) { ++afl->queue_cycle; - afl->current_entry = 0; + runs_in_current_cycle = 0; afl->cur_skipped_paths = 0; - afl->queue_cur = afl->queue; - if (seek_to) { + if (unlikely(afl->old_seed_selection)) { - afl->current_entry = seek_to; - afl->queue_cur = afl->queue_buf[seek_to]; - seek_to = 0; + afl->current_entry = 0; + afl->queue_cur = afl->queue; - } + if (unlikely(seek_to)) { - // show_stats(afl); + afl->current_entry = seek_to; + afl->queue_cur = afl->queue_buf[seek_to]; + seek_to = 0; + + } + + } if (unlikely(afl->not_on_tty)) { @@ -1366,9 +1385,11 @@ int main(int argc, char **argv_orig, char **envp) { switch (afl->expand_havoc) { case 0: + // this adds extra splicing mutation options to havoc mode afl->expand_havoc = 1; break; case 1: + // add MOpt mutator if (afl->limit_time_sig == 0 && !afl->custom_only && !afl->python_only) { @@ -1381,25 +1402,26 @@ int main(int argc, char **argv_orig, char **envp) { break; case 2: // if (!have_p) afl->schedule = EXPLOIT; + // increase havoc mutations per fuzz attempt afl->havoc_stack_pow2++; afl->expand_havoc = 3; break; case 3: + // further increase havoc mutations per fuzz attempt afl->havoc_stack_pow2++; afl->expand_havoc = 4; break; case 4: + // if not in sync mode, enable deterministic mode? + // if (!afl->sync_dir) afl->skip_deterministic = 0; + afl->expand_havoc = 5; + break; + case 5: // nothing else currently break; } - if (afl->expand_havoc) { - - } else - - afl->expand_havoc = 1; - } else { afl->use_splicing = 1; @@ -1470,6 +1492,22 @@ int main(int argc, char **argv_orig, char **envp) { } + if (likely(!afl->old_seed_selection)) { + + ++runs_in_current_cycle; + if (unlikely(prev_queued_paths < afl->queued_paths)) { + + // we have new queue entries since the last run, recreate alias table + prev_queued_paths = afl->queued_paths; + create_alias_table(afl); + + } + + afl->current_entry = select_next_queue_entry(afl); + afl->queue_cur = afl->queue_buf[afl->current_entry]; + + } + skipped_fuzz = fuzz_one(afl); if (!skipped_fuzz && !afl->stop_soon && afl->sync_id) { @@ -1490,8 +1528,12 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->stop_soon) { break; } - afl->queue_cur = afl->queue_cur->next; - ++afl->current_entry; + if (unlikely(afl->old_seed_selection)) { + + afl->queue_cur = afl->queue_cur->next; + ++afl->current_entry; + + } } -- cgit 1.4.1 From b7e0490bcdaa7fa792a9dccfa5983e03af92730e Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 12 Oct 2020 03:44:34 +0200 Subject: Revert "Merge branch 'memcache_marc' into dev" This reverts commit c03fbcedaa68db5324423975a34331287426f7c2, reversing changes made to dab017dddaaab6d836a590f7bba3eea3549758d2. --- include/afl-fuzz.h | 16 ------ include/config.h | 9 --- src/afl-fuzz-one.c | 157 +++++++++++++++++++++++++++++++-------------------- src/afl-fuzz-queue.c | 87 ---------------------------- src/afl-fuzz.c | 5 +- 5 files changed, 97 insertions(+), 177 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index a3e87129..45de197d 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -168,9 +168,6 @@ struct queue_entry { double perf_score; /* performance score */ - u8 *testcase_buf; /* The testcase buffer, if loaded. */ - u32 testcase_refs; /* count of users of testcase buf */ - struct queue_entry *next; /* Next element, if any */ }; @@ -689,12 +686,6 @@ typedef struct afl_state { /* queue entries ready for splicing count (len > 4) */ u32 ready_for_splicing_count; - /* How many queue entries currently have cached testcases */ - u32 q_testcase_cache_count; - /* Refs to each queue entry with cached testcase (for eviction, if cache_count - * is too large) */ - struct queue_entry *q_testcase_cache[TESTCASE_CACHE_SIZE]; - } afl_state_t; struct custom_mutator { @@ -1141,12 +1132,5 @@ static inline u64 next_p2(u64 val) { } -/* Returns the testcase buf from the file behind this queue entry. - Increases the refcount. */ -u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q); - -/* Tell afl that this testcase may be evicted from the cache */ -void queue_testcase_release(afl_state_t *afl, struct queue_entry *q); - #endif diff --git a/include/config.h b/include/config.h index 3f498275..7dd045e3 100644 --- a/include/config.h +++ b/include/config.h @@ -295,15 +295,6 @@ #define RESEED_RNG 100000 -/* The amount of entries in the testcase cache, held in memory. -Decrease if RAM usage is high. */ -#define TESTCASE_CACHE_SIZE 3072 - -#if TESTCASE_CACHE_SIZE < 4 - #error \ - "Dangerously low cache size: Set TESTCASE_CACHE_SIZE to 4 or more in config.h!" -#endif - /* Maximum line length passed from GCC to 'as' and used for parsing configuration files: */ diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index ebe541a2..6ef728e0 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -370,7 +370,7 @@ static void locate_diffs(u8 *ptr1, u8 *ptr2, u32 len, s32 *first, s32 *last) { u8 fuzz_one_original(afl_state_t *afl) { - s32 len, temp_len; + s32 len, fd, temp_len; u32 j; u32 i; u8 *in_buf, *out_buf, *orig_in, *ex_tmp, *eff_map = 0; @@ -453,9 +453,28 @@ u8 fuzz_one_original(afl_state_t *afl) { } - orig_in = in_buf = queue_testcase_take(afl, afl->queue_cur); + /* Map the test case into memory. */ + + fd = open(afl->queue_cur->fname, O_RDONLY); + + if (unlikely(fd < 0)) { + + PFATAL("Unable to open '%s'", afl->queue_cur->fname); + + } + len = afl->queue_cur->len; + orig_in = in_buf = mmap(0, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + + if (unlikely(orig_in == MAP_FAILED)) { + + PFATAL("Unable to mmap '%s' with len %d", afl->queue_cur->fname, len); + + } + + close(fd); + /* We could mmap() out_buf as MAP_PRIVATE, but we end up clobbering every single byte anyway, so it wouldn't give us any performance or memory usage benefits. */ @@ -1678,7 +1697,7 @@ custom_mutator_stage: for (afl->stage_cur = 0; afl->stage_cur < afl->stage_max; ++afl->stage_cur) { - struct queue_entry *target = NULL; + struct queue_entry *target; u32 tid; u8 * new_buf = NULL; u32 target_len = 0; @@ -1701,7 +1720,17 @@ custom_mutator_stage: afl->splicing_with = tid; /* Read the additional testcase into a new buffer. */ - new_buf = queue_testcase_take(afl, target); + fd = open(target->fname, O_RDONLY); + if (unlikely(fd < 0)) { + + PFATAL("Unable to open '%s'", target->fname); + + } + + new_buf = afl_realloc(AFL_BUF_PARAM(out_scratch), target->len); + if (unlikely(!new_buf)) { PFATAL("alloc"); } + ck_read(fd, new_buf, target->len, target->fname); + close(fd); target_len = target->len; } @@ -1712,13 +1741,6 @@ custom_mutator_stage: el->afl_custom_fuzz(el->data, out_buf, len, &mutated_buf, new_buf, target_len, max_seed_size); - if (new_buf) { - - queue_testcase_release(afl, target); - new_buf = NULL; - - } - if (unlikely(!mutated_buf)) { FATAL("Error in custom_fuzz. Size returned: %zd", mutated_size); @@ -2301,53 +2323,52 @@ havoc_stage: /* Overwrite bytes with a randomly selected chunk from another testcase or insert that chunk. */ - if (afl->queued_paths < 4) { break; } + if (afl->queued_paths < 4) break; /* Pick a random queue entry and seek to it. */ u32 tid; - do { - + do tid = rand_below(afl, afl->queued_paths); - - } while (tid == afl->current_entry); + while (tid == afl->current_entry); struct queue_entry *target = afl->queue_buf[tid]; /* Make sure that the target has a reasonable length. */ - while (target && (target->len < 2 || target == afl->queue_cur)) { - + while (target && (target->len < 2 || target == afl->queue_cur)) target = target->next; - } + if (!target) break; - if (!target) { break; } + /* Read the testcase into a new buffer. */ - u32 new_len = target->len; + fd = open(target->fname, O_RDONLY); - /* Get the testcase contents for splicing. */ - u8 *new_buf = queue_testcase_take(afl, target); + if (unlikely(fd < 0)) { - u8 overwrite = 0; - if (temp_len >= 2 && rand_below(afl, 2)) { + PFATAL("Unable to open '%s'", target->fname); - overwrite = 1; + } - } else if (temp_len + HAVOC_BLK_XL >= MAX_FILE) { + u32 new_len = target->len; + u8 *new_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), new_len); + if (unlikely(!new_buf)) { PFATAL("alloc"); } - if (temp_len >= 2) { + ck_read(fd, new_buf, new_len, target->fname); - overwrite = 1; + close(fd); - } else { + u8 overwrite = 0; + if (temp_len >= 2 && rand_below(afl, 2)) + overwrite = 1; + else if (temp_len + HAVOC_BLK_XL >= MAX_FILE) { - queue_testcase_release(afl, target); - new_buf = NULL; + if (temp_len >= 2) + overwrite = 1; + else break; - } - } if (overwrite) { @@ -2393,9 +2414,6 @@ havoc_stage: } - /* We don't need this splice testcase anymore */ - queue_testcase_release(afl, target); - new_buf = NULL; break; } @@ -2501,17 +2519,24 @@ retry_splicing: if (!target) { goto retry_splicing; } - /* Get the testcase buffer */ - u8 *splice_buf = queue_testcase_take(afl, target); + /* Read the testcase into a new buffer. */ + + fd = open(target->fname, O_RDONLY); + + if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", target->fname); } + new_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), target->len); if (unlikely(!new_buf)) { PFATAL("alloc"); } + ck_read(fd, new_buf, target->len, target->fname); + + close(fd); + /* Find a suitable splicing location, somewhere between the first and the last differing byte. Bail out if the difference is just a single byte or so. */ - locate_diffs(in_buf, splice_buf, MIN(len, (s64)target->len), &f_diff, - &l_diff); + locate_diffs(in_buf, new_buf, MIN(len, (s64)target->len), &f_diff, &l_diff); if (f_diff < 0 || l_diff < 2 || f_diff == l_diff) { goto retry_splicing; } @@ -2523,7 +2548,6 @@ retry_splicing: len = target->len; memcpy(new_buf, in_buf, split_at); - memcpy(new_buf + split_at, splice_buf + split_at, target->len - split_at); afl_swap_bufs(AFL_BUF_PARAM(in), AFL_BUF_PARAM(in_scratch)); in_buf = new_buf; @@ -2531,9 +2555,6 @@ retry_splicing: if (unlikely(!out_buf)) { PFATAL("alloc"); } memcpy(out_buf, in_buf, len); - queue_testcase_release(afl, target); - splice_buf = NULL; - goto custom_mutator_stage; /* ???: While integrating Python module, the author decided to jump to python stage, but the reason behind this is not clear.*/ @@ -2564,8 +2585,7 @@ abandon_entry: ++afl->queue_cur->fuzz_level; - queue_testcase_release(afl, afl->queue_cur); - orig_in = NULL; + munmap(orig_in, afl->queue_cur->len); return ret_val; @@ -2587,7 +2607,7 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { } - s32 len, temp_len; + s32 len, fd, temp_len; u32 i; u32 j; u8 *in_buf, *out_buf, *orig_in, *ex_tmp, *eff_map = 0; @@ -2652,9 +2672,23 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { } /* Map the test case into memory. */ - orig_in = in_buf = queue_testcase_take(afl, afl->queue_cur); + + fd = open(afl->queue_cur->fname, O_RDONLY); + + if (fd < 0) { PFATAL("Unable to open '%s'", afl->queue_cur->fname); } + len = afl->queue_cur->len; + orig_in = in_buf = mmap(0, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + + if (orig_in == MAP_FAILED) { + + PFATAL("Unable to mmap '%s'", afl->queue_cur->fname); + + } + + close(fd); + /* We could mmap() out_buf as MAP_PRIVATE, but we end up clobbering every single byte anyway, so it wouldn't give us any performance or memory usage benefits. */ @@ -4494,24 +4528,31 @@ pacemaker_fuzzing: if (!target) { goto retry_splicing_puppet; } /* Read the testcase into a new buffer. */ - u8 *splicing_buf = queue_testcase_take(afl, target); + + fd = open(target->fname, O_RDONLY); + + if (fd < 0) { PFATAL("Unable to open '%s'", target->fname); } + + new_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), target->len); + if (unlikely(!new_buf)) { PFATAL("alloc"); } + + ck_read(fd, new_buf, target->len, target->fname); + + close(fd); /* Find a suitable splicin g location, somewhere between the first and the last differing byte. Bail out if the difference is just a single byte or so. */ - locate_diffs(in_buf, splicing_buf, MIN(len, (s32)target->len), &f_diff, + locate_diffs(in_buf, new_buf, MIN(len, (s32)target->len), &f_diff, &l_diff); if (f_diff < 0 || l_diff < 2 || f_diff == l_diff) { - queue_testcase_release(afl, target); goto retry_splicing_puppet; } - new_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), target->len); - /* Split somewhere between the first and last differing byte. */ split_at = f_diff + rand_below(afl, l_diff - f_diff); @@ -4520,17 +4561,12 @@ pacemaker_fuzzing: len = target->len; memcpy(new_buf, in_buf, split_at); - memcpy(new_buf + split_at, splicing_buf + split_at, - target->len - split_at); afl_swap_bufs(AFL_BUF_PARAM(in), AFL_BUF_PARAM(in_scratch)); in_buf = new_buf; out_buf = afl_realloc(AFL_BUF_PARAM(out), len); if (unlikely(!out_buf)) { PFATAL("alloc"); } memcpy(out_buf, in_buf, len); - queue_testcase_release(afl, target); - splicing_buf = NULL; - goto havoc_stage_puppet; } /* if splice_cycle */ @@ -4564,8 +4600,7 @@ pacemaker_fuzzing: // if (afl->queue_cur->favored) --afl->pending_favored; // } - queue_testcase_release(afl, afl->queue_cur); - orig_in = NULL; + munmap(orig_in, afl->queue_cur->len); if (afl->key_puppet == 1) { diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index a034b168..d608e890 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -343,7 +343,6 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { q->depth = afl->cur_depth + 1; q->passed_det = passed_det; q->trace_mini = NULL; - q->testcase_buf = NULL; if (q->depth > afl->max_depth) { afl->max_depth = q->depth; } @@ -892,89 +891,3 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { } -/* Tell afl that this testcase may be evicted from the cache */ -inline void queue_testcase_release(afl_state_t *afl, struct queue_entry *q) { - - (void)afl; - if (unlikely(q->testcase_refs == 0)) { - - FATAL("Testcase refcount reduced past 0"); - - } - - q->testcase_refs--; - -} - -/* Returns the testcase buf from the file behind this queue entry. - Increases the refcount. */ -u8 *queue_testcase_take(afl_state_t *afl, struct queue_entry *q) { - - if (!q->testcase_buf) { - - u32 tid = 0; - /* Buf not cached, let's do that now */ - - if (likely(afl->q_testcase_cache_count == TESTCASE_CACHE_SIZE)) { - - /* Cache full. We neet to evict one to map one. - Get a random one which is not in use */ - do { - - tid = rand_below(afl, afl->q_testcase_cache_count); - - } while (afl->q_testcase_cache[tid]->testcase_refs > 0); - - struct queue_entry *old_cached = afl->q_testcase_cache[tid]; - /* free the current buf from cache */ - munmap(old_cached->testcase_buf, old_cached->len); - old_cached->testcase_buf = NULL; - - } else { - - tid = afl->q_testcase_cache_count; - afl->q_testcase_cache_count++; - - } - - /* Map the test case into memory. */ - - int fd = open(q->fname, O_RDONLY); - - if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); } - - u32 len = q->len; - - q->testcase_buf = mmap(0, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - - if (unlikely(q->testcase_buf == MAP_FAILED)) { - - PFATAL("Unable to mmap '%s' with len %d", q->fname, len); - - } - - close(fd); - - /* Register us as cached */ - afl->q_testcase_cache[tid] = q; - - } - - q->testcase_refs++; - if (unlikely(!q->testcase_buf || !q->testcase_refs)) { - if (!q->testcase_buf) { - - FATAL("Testcase buf is NULL, this should never happen"); - - } - if (!q->testcase_refs) { - - FATAL("Testcase ref overflow. Missing a testcase release somwhere?"); - - } - } - - return q->testcase_buf; - -} - diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index cb5eb37a..d42a0d36 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1174,12 +1174,9 @@ int main(int argc, char **argv_orig, char **envp) { if (extras_dir_cnt) { - for (i = 0; i < extras_dir_cnt; i++) { - + for (i = 0; i < extras_dir_cnt; i++) load_extras(afl, extras_dir[i]); - } - dedup_extras(afl); OKF("Loaded a total of %u extras.", afl->extras_cnt); -- cgit 1.4.1 From d9b63766dfdb8feeb1dc6f7c51c17abf07ee4086 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 12 Oct 2020 11:12:16 +0200 Subject: fix new seed selection algo --- include/afl-fuzz.h | 3 +++ src/afl-fuzz-queue.c | 59 ++++++++++++--------------------------------------- src/afl-performance.c | 10 ++++++++- 3 files changed, 26 insertions(+), 46 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 45de197d..85597150 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -1045,6 +1045,9 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len, /* xoshiro256** */ uint64_t rand_next(afl_state_t *afl); +/* probability between 0.0 and 1.0 */ +double rand_next_percent(afl_state_t *afl); + /**** Inline routines ****/ /* Generate a random number (from 0 to limit - 1). This may diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index d608e890..f224d851 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -27,17 +27,22 @@ #include #include +/* select next queue entry based on alias algo - fast! */ + inline u32 select_next_queue_entry(afl_state_t *afl) { - u32 r = rand_below(afl, 0xffffffff); - u32 s = r % afl->queued_paths; - // fprintf(stderr, "select: r=%u s=%u ... r < prob[s]=%f ? s=%u : - // alias[%u]=%u\n", r, s, afl->alias_probability[s], s, s, - // afl->alias_table[s]); - return (r < afl->alias_probability[s] ? s : afl->alias_table[s]); + u32 s = rand_below(afl, afl->queued_paths); + double p = rand_next_percent(afl); + /* + fprintf(stderr, "select: p=%f s=%u ... p < prob[s]=%f ? s=%u : alias[%u]=%u" + " ==> %u\n", p, s, afl->alias_probability[s], s, s, afl->alias_table[s], p < afl->alias_probability[s] ? s : afl->alias_table[s]); + */ + return (p < afl->alias_probability[s] ? s : afl->alias_table[s]); } +/* create the alias table that allows weighted random selection - expensive */ + void create_alias_table(afl_state_t *afl) { u32 n = afl->queued_paths, i = 0, a, g; @@ -63,11 +68,6 @@ void create_alias_table(afl_state_t *afl) { if (!q->disabled) q->perf_score = calculate_score(afl, q); sum += q->perf_score; - /* - if (afl->debug) - fprintf(stderr, "entry %u: score=%f %s (sum: %f)\n", i, q->perf_score, - q->disabled ? "disabled" : "", sum); - */ } @@ -110,41 +110,10 @@ void create_alias_table(afl_state_t *afl) { afl->alias_probability[S[--nS]] = 1; /* - if (afl->debug) { - - fprintf(stderr, " %-3s %-3s %-9s\n", "entry", "alias", "prob"); + fprintf(stderr, " %-3s %-3s %-9s %-9s\n", "entry", "alias", "prob", "perf"); for (u32 i = 0; i < n; ++i) - fprintf(stderr, " %3i %3i %9.7f\n", i, afl->alias_table[i], - afl->alias_probability[i]); - - } - - int prob = 0; - fprintf(stderr, "Alias:"); - for (i = 0; i < n; i++) { - - fprintf(stderr, " [%u]=%u", i, afl->alias_table[i]); - if (afl->alias_table[i] >= n) - prob = i; - - } - - fprintf(stderr, "\n"); - - if (prob) { - - fprintf(stderr, "PROBLEM! alias[%u] = %u\n", prob, - afl->alias_table[prob]); - - for (i = 0; i < n; i++) { - - struct queue_entry *q = afl->queue_buf[i]; - - fprintf(stderr, "%u: score=%f\n", i, q->perf_score); - - } - - } + fprintf(stderr, " %3i %3i %9.7f %9.7f\n", i, afl->alias_table[i], + afl->alias_probability[i], afl->queue_buf[i]->perf_score); */ diff --git a/src/afl-performance.c b/src/afl-performance.c index 7a80ac4b..6fa95dea 100644 --- a/src/afl-performance.c +++ b/src/afl-performance.c @@ -47,7 +47,7 @@ void rand_set_seed(afl_state_t *afl, s64 init_seed) { } -uint64_t rand_next(afl_state_t *afl) { +inline uint64_t rand_next(afl_state_t *afl) { const uint64_t result = rotl(afl->rand_seed[0] + afl->rand_seed[3], 23) + afl->rand_seed[0]; @@ -67,6 +67,14 @@ uint64_t rand_next(afl_state_t *afl) { } +/* returns a double between 0.000000000 and 1.000000000 */ + +inline double rand_next_percent(afl_state_t *afl) { + + return (double)(((double)rand_next(afl)) / (double) 0xffffffffffffffff); + +} + /* This is the jump function for the generator. It is equivalent to 2^128 calls to rand_next(); it can be used to generate 2^128 non-overlapping subsequences for parallel computations. */ -- cgit 1.4.1 From 56ac3fcdc511d124ad058412021ead21bbbcf4bf Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 14 Oct 2020 15:30:30 +0200 Subject: configurable testcache with malloc (#581) * cache item number to cache memory size * reload testcase if trimming changed the size * fix splicing selection * slim splicing * import sync fix * write testcache stats to fuzzer_stats * fix new seed selection algo * malloc+read instead of mmap * fix * testcache is configurable now and no reference counts * fixes compilation, test script * fixes * switch TEST_CC to afl-cc in makefile * code format * fix * fix crash * fix crash * fix env help output * remove unnecessary pointer resets * fix endless loop bug * actually use the cache if set * one more fix * increase default cache entries, add default cache size value to config.h Co-authored-by: hexcoder- --- GNUmakefile | 4 +- include/afl-fuzz.h | 31 +++++++- include/config.h | 9 +++ include/envs.h | 1 + src/afl-fuzz-init.c | 4 +- src/afl-fuzz-one.c | 169 +++++++++---------------------------------- src/afl-fuzz-queue.c | 167 ++++++++++++++++++++++++++++++++++++++---- src/afl-fuzz-run.c | 6 ++ src/afl-fuzz-state.c | 8 ++ src/afl-fuzz-stats.c | 7 +- src/afl-fuzz.c | 29 ++++++-- src/afl-performance.c | 2 +- test/test-custom-mutators.sh | 2 +- 13 files changed, 276 insertions(+), 163 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/GNUmakefile b/GNUmakefile index c885a935..80b7b68b 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -223,8 +223,6 @@ ifneq "$(findstring OpenBSD, $(shell uname))" "" LDFLAGS += -lpthread endif -TEST_CC = afl-gcc - COMM_HDR = include/alloc-inl.h include/config.h include/debug.h include/types.h ifeq "$(shell echo '$(HASH)include @int main() {return 0; }' | tr @ '\n' | $(CC) $(CFLAGS) -x c - -o .test $(PYTHON_INCLUDE) $(LDFLAGS) $(PYTHON_LIB) 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" @@ -488,7 +486,7 @@ code-format: ifndef AFL_NO_X86 test_build: afl-cc afl-as afl-showmap @echo "[*] Testing the CC wrapper and instrumentation output..." - @unset AFL_USE_ASAN AFL_USE_MSAN AFL_CC; AFL_DEBUG=1 AFL_INST_RATIO=100 AFL_PATH=. ./$(TEST_CC) $(CFLAGS) test-instr.c -o test-instr $(LDFLAGS) 2>&1 | grep 'afl-as' >/dev/null || (echo "Oops, afl-as did not get called from "$(TEST_CC)". This is normally achieved by "$(CC)" honoring the -B option."; exit 1 ) + @unset AFL_MAP_SIZE AFL_USE_UBSAN AFL_USE_CFISAN AFL_USE_ASAN AFL_USE_MSAN AFL_CC; AFL_INST_RATIO=100 AFL_PATH=. ./afl-cc $(CFLAGS) test-instr.c -o test-instr $(LDFLAGS) 2>&1 || (echo "Oops, afl-cc failed"; exit 1 ) ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -q -o .test-instr0 ./test-instr < /dev/null echo 1 | ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -q -o .test-instr1 ./test-instr @rm -f test-instr diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 85597150..940c5602 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -168,6 +168,8 @@ struct queue_entry { double perf_score; /* performance score */ + u8 *testcase_buf; /* The testcase buffer, if loaded. */ + struct queue_entry *next; /* Next element, if any */ }; @@ -363,7 +365,7 @@ typedef struct afl_env_vars { u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path, *afl_hang_tmout, *afl_forksrv_init_tmout, *afl_skip_crashes, *afl_preload, *afl_max_det_extras, *afl_statsd_host, *afl_statsd_port, - *afl_statsd_tags_flavor; + *afl_statsd_tags_flavor, *afl_testcache_size; } afl_env_vars_t; @@ -675,6 +677,9 @@ typedef struct afl_state { u8 *in_scratch_buf; u8 *ex_buf; + + u8 *testcase_buf, *splicecase_buf; + u32 custom_mutators_count; list_t custom_mutator_list; @@ -686,6 +691,22 @@ typedef struct afl_state { /* queue entries ready for splicing count (len > 4) */ u32 ready_for_splicing_count; + /* This is the user specified maximum size to use for the testcase cache */ + u64 q_testcase_max_cache_size; + + /* How much of the testcase cache is used so far */ + u64 q_testcase_cache_size; + + /* highest cache count so far */ + u32 q_testcase_max_cache_count; + + /* How many queue entries currently have cached testcases */ + u32 q_testcase_cache_count; + + /* Refs to each queue entry with cached testcase (for eviction, if cache_count + * is too large) */ + struct queue_entry *q_testcase_cache[TESTCASE_ENTRIES]; + } afl_state_t; struct custom_mutator { @@ -1135,5 +1156,13 @@ static inline u64 next_p2(u64 val) { } +/* Returns the testcase buf from the file behind this queue entry. + Increases the refcount. */ +u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q); + +/* If trimming changes the testcase size we have to reload it */ +void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q, + u32 old_len); + #endif diff --git a/include/config.h b/include/config.h index 7dd045e3..b4f3a775 100644 --- a/include/config.h +++ b/include/config.h @@ -295,6 +295,15 @@ #define RESEED_RNG 100000 +/* The maximum number of testcases to cache */ + +#define TESTCASE_ENTRIES 16384 + +/* The default maximum testcase cache size in MB, 0 = disable. + A value between 50 and 250 is a good default value. */ + +#define TESTCASE_CACHE 0 + /* Maximum line length passed from GCC to 'as' and used for parsing configuration files: */ diff --git a/include/envs.h b/include/envs.h index 51520312..a1b3ad12 100644 --- a/include/envs.h +++ b/include/envs.h @@ -139,6 +139,7 @@ static char *afl_environment_variables[] = { "AFL_STATSD_HOST", "AFL_STATSD_PORT", "AFL_STATSD_TAGS_FLAVOR", + "AFL_TESTCACHE_SIZE", "AFL_TMIN_EXACT", "AFL_TMPDIR", "AFL_TOKEN_FILE", diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 881bf10f..607b652f 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -1045,7 +1045,7 @@ restart_outer_cull_loop: while (q) { - if (q->cal_failed || !q->exec_cksum) continue; + if (q->cal_failed || !q->exec_cksum) { goto next_entry; } restart_inner_cull_loop: @@ -1090,6 +1090,8 @@ restart_outer_cull_loop: } + next_entry: + prev = q; q = q->next; diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index fc092f8d..154e4b45 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -370,7 +370,7 @@ static void locate_diffs(u8 *ptr1, u8 *ptr2, u32 len, s32 *first, s32 *last) { u8 fuzz_one_original(afl_state_t *afl) { - s32 len, fd, temp_len; + s32 len, temp_len; u32 j; u32 i; u8 *in_buf, *out_buf, *orig_in, *ex_tmp, *eff_map = 0; @@ -453,32 +453,9 @@ u8 fuzz_one_original(afl_state_t *afl) { } - /* Map the test case into memory. */ - - fd = open(afl->queue_cur->fname, O_RDONLY); - - if (unlikely(fd < 0)) { - - PFATAL("Unable to open '%s'", afl->queue_cur->fname); - - } - + orig_in = in_buf = queue_testcase_get(afl, afl->queue_cur); len = afl->queue_cur->len; - orig_in = in_buf = mmap(0, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - - if (unlikely(orig_in == MAP_FAILED)) { - - PFATAL("Unable to mmap '%s' with len %d", afl->queue_cur->fname, len); - - } - - close(fd); - - /* We could mmap() out_buf as MAP_PRIVATE, but we end up clobbering every - single byte anyway, so it wouldn't give us any performance or memory usage - benefits. */ - out_buf = afl_realloc(AFL_BUF_PARAM(out), len); if (unlikely(!out_buf)) { PFATAL("alloc"); } @@ -526,6 +503,7 @@ u8 fuzz_one_original(afl_state_t *afl) { !afl->disable_trim)) { u8 res = trim_case(afl, afl->queue_cur, in_buf); + orig_in = in_buf = queue_testcase_get(afl, afl->queue_cur); if (unlikely(res == FSRV_RUN_ERROR)) { @@ -1720,17 +1698,7 @@ custom_mutator_stage: afl->splicing_with = tid; /* Read the additional testcase into a new buffer. */ - fd = open(target->fname, O_RDONLY); - if (unlikely(fd < 0)) { - - PFATAL("Unable to open '%s'", target->fname); - - } - - new_buf = afl_realloc(AFL_BUF_PARAM(out_scratch), target->len); - if (unlikely(!new_buf)) { PFATAL("alloc"); } - ck_read(fd, new_buf, target->len, target->fname); - close(fd); + new_buf = queue_testcase_get(afl, target); target_len = target->len; } @@ -2182,7 +2150,6 @@ havoc_stage: afl_swap_bufs(AFL_BUF_PARAM(out), AFL_BUF_PARAM(out_scratch)); out_buf = new_buf; - new_buf = NULL; temp_len += clone_len; } @@ -2326,43 +2293,21 @@ havoc_stage: /* Pick a random queue entry and seek to it. */ u32 tid; - do - tid = rand_below(afl, afl->queued_paths); - while (tid == afl->current_entry || afl->queue_buf[tid]->len < 4); - - struct queue_entry *target = afl->queue_buf[tid]; - - /* Read the testcase into a new buffer. */ - - fd = open(target->fname, O_RDONLY); - - if (unlikely(fd < 0)) { - - PFATAL("Unable to open '%s'", target->fname); - - } - - u32 new_len = target->len; - u8 *new_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), new_len); - if (unlikely(!new_buf)) { PFATAL("alloc"); } - - ck_read(fd, new_buf, new_len, target->fname); + do { - close(fd); + tid = rand_below(afl, afl->queued_paths); - u8 overwrite = 0; - if (temp_len >= 2 && rand_below(afl, 2)) - overwrite = 1; - else if (temp_len + HAVOC_BLK_XL >= MAX_FILE) { + } while (tid == afl->current_entry || afl->queue_buf[tid]->len < 4); - if (temp_len >= 2) - overwrite = 1; - else - break; + /* Get the testcase for splicing. */ + struct queue_entry *target = afl->queue_buf[tid]; + u32 new_len = target->len; + u8 * new_buf = queue_testcase_get(afl, target); - } + if ((temp_len >= 2 && rand_below(afl, 2)) || + temp_len + HAVOC_BLK_XL >= MAX_FILE) { - if (overwrite) { + /* overwrite mode */ u32 copy_from, copy_to, copy_len; @@ -2376,15 +2321,16 @@ havoc_stage: } else { + /* insert mode */ + u32 clone_from, clone_to, clone_len; clone_len = choose_block_len(afl, new_len); clone_from = rand_below(afl, new_len - clone_len + 1); + clone_to = rand_below(afl, temp_len + 1); - clone_to = rand_below(afl, temp_len); - - u8 *temp_buf = - afl_realloc(AFL_BUF_PARAM(out_scratch), temp_len + clone_len); + u8 *temp_buf = afl_realloc(AFL_BUF_PARAM(out_scratch), + temp_len + clone_len + 1); if (unlikely(!temp_buf)) { PFATAL("alloc"); } /* Head */ @@ -2496,21 +2442,10 @@ retry_splicing: } while (tid == afl->current_entry || afl->queue_buf[tid]->len < 4); + /* Get the testcase */ afl->splicing_with = tid; target = afl->queue_buf[tid]; - - /* Read the testcase into a new buffer. */ - - fd = open(target->fname, O_RDONLY); - - if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", target->fname); } - - new_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), target->len); - if (unlikely(!new_buf)) { PFATAL("alloc"); } - - ck_read(fd, new_buf, target->len, target->fname); - - close(fd); + new_buf = queue_testcase_get(afl, target); /* Find a suitable splicing location, somewhere between the first and the last differing byte. Bail out if the difference is just a single @@ -2527,18 +2462,16 @@ retry_splicing: /* Do the thing. */ len = target->len; - memcpy(new_buf, in_buf, split_at); - afl_swap_bufs(AFL_BUF_PARAM(in), AFL_BUF_PARAM(in_scratch)); - in_buf = new_buf; + afl->in_scratch_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), len); + memcpy(afl->in_scratch_buf, in_buf, split_at); + memcpy(afl->in_scratch_buf + split_at, new_buf, len - split_at); + in_buf = afl->in_scratch_buf; out_buf = afl_realloc(AFL_BUF_PARAM(out), len); if (unlikely(!out_buf)) { PFATAL("alloc"); } memcpy(out_buf, in_buf, len); goto custom_mutator_stage; - /* ???: While integrating Python module, the author decided to jump to - python stage, but the reason behind this is not clear.*/ - // goto havoc_stage; } @@ -2564,9 +2497,7 @@ abandon_entry: } ++afl->queue_cur->fuzz_level; - - munmap(orig_in, afl->queue_cur->len); - + orig_in = NULL; return ret_val; #undef FLIP_BIT @@ -2587,7 +2518,7 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { } - s32 len, fd, temp_len; + s32 len, temp_len; u32 i; u32 j; u8 *in_buf, *out_buf, *orig_in, *ex_tmp, *eff_map = 0; @@ -2652,32 +2583,11 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { } /* Map the test case into memory. */ - - fd = open(afl->queue_cur->fname, O_RDONLY); - - if (fd < 0) { PFATAL("Unable to open '%s'", afl->queue_cur->fname); } - + orig_in = in_buf = queue_testcase_get(afl, afl->queue_cur); len = afl->queue_cur->len; - - orig_in = in_buf = mmap(0, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - - if (orig_in == MAP_FAILED) { - - PFATAL("Unable to mmap '%s'", afl->queue_cur->fname); - - } - - close(fd); - - /* We could mmap() out_buf as MAP_PRIVATE, but we end up clobbering every - single byte anyway, so it wouldn't give us any performance or memory usage - benefits. */ - out_buf = afl_realloc(AFL_BUF_PARAM(out), len); if (unlikely(!out_buf)) { PFATAL("alloc"); } - afl->subseq_tmouts = 0; - afl->cur_depth = afl->queue_cur->depth; /******************************************* @@ -2721,6 +2631,7 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { u32 old_len = afl->queue_cur->len; u8 res = trim_case(afl, afl->queue_cur, in_buf); + orig_in = in_buf = queue_testcase_get(afl, afl->queue_cur); if (res == FSRV_RUN_ERROR) { @@ -4497,17 +4408,7 @@ pacemaker_fuzzing: target = afl->queue_buf[tid]; /* Read the testcase into a new buffer. */ - - fd = open(target->fname, O_RDONLY); - - if (fd < 0) { PFATAL("Unable to open '%s'", target->fname); } - - new_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), target->len); - if (unlikely(!new_buf)) { PFATAL("alloc"); } - - ck_read(fd, new_buf, target->len, target->fname); - - close(fd); + new_buf = queue_testcase_get(afl, target); /* Find a suitable splicin g location, somewhere between the first and the last differing byte. Bail out if the difference is just a single @@ -4529,9 +4430,11 @@ pacemaker_fuzzing: /* Do the thing. */ len = target->len; - memcpy(new_buf, in_buf, split_at); - afl_swap_bufs(AFL_BUF_PARAM(in), AFL_BUF_PARAM(in_scratch)); - in_buf = new_buf; + afl->in_scratch_buf = afl_realloc(AFL_BUF_PARAM(in_scratch), len); + memcpy(afl->in_scratch_buf, in_buf, split_at); + memcpy(afl->in_scratch_buf + split_at, new_buf, len - split_at); + in_buf = afl->in_scratch_buf; + out_buf = afl_realloc(AFL_BUF_PARAM(out), len); if (unlikely(!out_buf)) { PFATAL("alloc"); } memcpy(out_buf, in_buf, len); @@ -4569,7 +4472,7 @@ pacemaker_fuzzing: // if (afl->queue_cur->favored) --afl->pending_favored; // } - munmap(orig_in, afl->queue_cur->len); + orig_in = NULL; if (afl->key_puppet == 1) { diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index f224d851..c634328f 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -31,11 +31,12 @@ inline u32 select_next_queue_entry(afl_state_t *afl) { - u32 s = rand_below(afl, afl->queued_paths); + u32 s = rand_below(afl, afl->queued_paths); double p = rand_next_percent(afl); /* fprintf(stderr, "select: p=%f s=%u ... p < prob[s]=%f ? s=%u : alias[%u]=%u" - " ==> %u\n", p, s, afl->alias_probability[s], s, s, afl->alias_table[s], p < afl->alias_probability[s] ? s : afl->alias_table[s]); + " ==> %u\n", p, s, afl->alias_probability[s], s, s, afl->alias_table[s], p < + afl->alias_probability[s] ? s : afl->alias_table[s]); */ return (p < afl->alias_probability[s] ? s : afl->alias_table[s]); @@ -55,7 +56,7 @@ void create_alias_table(afl_state_t *afl) { int * S = (u32 *)afl_realloc(AFL_BUF_PARAM(out_scratch), n * sizeof(u32)); int * L = (u32 *)afl_realloc(AFL_BUF_PARAM(in_scratch), n * sizeof(u32)); - if (!P || !S || !L) FATAL("could not aquire memory for alias table"); + if (!P || !S || !L) { FATAL("could not aquire memory for alias table"); } memset((void *)afl->alias_table, 0, n * sizeof(u32)); memset((void *)afl->alias_probability, 0, n * sizeof(double)); @@ -65,7 +66,7 @@ void create_alias_table(afl_state_t *afl) { struct queue_entry *q = afl->queue_buf[i]; - if (!q->disabled) q->perf_score = calculate_score(afl, q); + if (!q->disabled) { q->perf_score = calculate_score(afl, q); } sum += q->perf_score; @@ -74,19 +75,23 @@ void create_alias_table(afl_state_t *afl) { for (i = 0; i < n; i++) { struct queue_entry *q = afl->queue_buf[i]; - - P[i] = q->perf_score * n / sum; + P[i] = (q->perf_score * n) / sum; } int nS = 0, nL = 0, s; for (s = (s32)n - 1; s >= 0; --s) { - if (P[s] < 1) + if (P[s] < 1) { + S[nS++] = s; - else + + } else { + L[nL++] = s; + } + } while (nS && nL) { @@ -96,11 +101,16 @@ void create_alias_table(afl_state_t *afl) { afl->alias_probability[a] = P[a]; afl->alias_table[a] = g; P[g] = P[g] + P[a] - 1; - if (P[g] < 1) + if (P[g] < 1) { + S[nS++] = g; - else + + } else { + L[nL++] = g; + } + } while (nL) @@ -110,11 +120,10 @@ void create_alias_table(afl_state_t *afl) { afl->alias_probability[S[--nS]] = 1; /* - fprintf(stderr, " %-3s %-3s %-9s %-9s\n", "entry", "alias", "prob", "perf"); - for (u32 i = 0; i < n; ++i) - fprintf(stderr, " %3i %3i %9.7f %9.7f\n", i, afl->alias_table[i], - afl->alias_probability[i], afl->queue_buf[i]->perf_score); - + fprintf(stderr, " entry alias probability perf_score\n"); + for (u32 i = 0; i < n; ++i) + fprintf(stderr, " %5u %5u %11u %0.9f\n", i, afl->alias_table[i], + afl->alias_probability[i], afl->queue_buf[i]->perf_score); */ } @@ -860,3 +869,131 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { } +void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q, + u32 old_len) { + + if (likely(q->testcase_buf)) { + + free(q->testcase_buf); + int fd = open(q->fname, O_RDONLY); + + if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); } + + u32 len = q->len; + q->testcase_buf = malloc(len); + + if (unlikely(!q->testcase_buf)) { + + PFATAL("Unable to mmap '%s' with len %d", q->fname, len); + + } + + close(fd); + afl->q_testcase_cache_size = afl->q_testcase_cache_size + q->len - old_len; + + } + +} + +/* Returns the testcase buf from the file behind this queue entry. + Increases the refcount. */ +inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { + + u32 len = q->len; + + /* first handle if no testcase cache is configured */ + + if (unlikely(!afl->q_testcase_max_cache_size)) { + + u8 *buf; + + if (q == afl->queue_cur) { + + buf = afl_realloc((void **)&afl->testcase_buf, len); + + } else { + + buf = afl_realloc((void **)&afl->splicecase_buf, len); + + } + + if (unlikely(!buf)) { + + PFATAL("Unable to malloc '%s' with len %u", q->fname, len); + + } + + int fd = open(q->fname, O_RDONLY); + + if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); } + + ck_read(fd, buf, len, q->fname); + close(fd); + return buf; + + } + + /* now handle the testcase cache */ + + if (unlikely(!q->testcase_buf)) { + + /* Buf not cached, let's load it */ + u32 tid = 0; + + while (unlikely(afl->q_testcase_cache_size + len >= + afl->q_testcase_max_cache_size || + afl->q_testcase_cache_count >= TESTCASE_ENTRIES - 1)) { + + /* Cache full. We neet to evict one to map one. + Get a random one which is not in use */ + + do { + + tid = rand_below(afl, afl->q_testcase_max_cache_count); + + } while (afl->q_testcase_cache[tid] == NULL || + + afl->q_testcase_cache[tid] == afl->queue_cur); + + struct queue_entry *old_cached = afl->q_testcase_cache[tid]; + free(old_cached->testcase_buf); + old_cached->testcase_buf = NULL; + afl->q_testcase_cache_size -= old_cached->len; + afl->q_testcase_cache[tid] = NULL; + --afl->q_testcase_cache_count; + + } + + while (likely(afl->q_testcase_cache[tid] != NULL)) + ++tid; + + /* Map the test case into memory. */ + + int fd = open(q->fname, O_RDONLY); + + if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); } + + q->testcase_buf = malloc(len); + + if (unlikely(!q->testcase_buf)) { + + PFATAL("Unable to malloc '%s' with len %u", q->fname, len); + + } + + ck_read(fd, q->testcase_buf, len, q->fname); + close(fd); + + /* Register testcase as cached */ + afl->q_testcase_cache[tid] = q; + afl->q_testcase_cache_size += q->len; + ++afl->q_testcase_cache_count; + if (tid >= afl->q_testcase_max_cache_count) + afl->q_testcase_max_cache_count = tid + 1; + + } + + return q->testcase_buf; + +} + diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index ee22b0f6..ab870319 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -692,6 +692,8 @@ void sync_fuzzers(afl_state_t *afl) { u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { + u32 orig_len = q->len; + /* Custom mutator trimmer */ if (afl->custom_mutators_count) { @@ -709,6 +711,8 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { }); + if (orig_len != q->len) { queue_testcase_retake(afl, q, orig_len); } + if (custom_trimmed) return trimmed_case; } @@ -842,6 +846,8 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { close(fd); + if (orig_len != q->len) queue_testcase_retake(afl, q, orig_len); + memcpy(afl->fsrv.trace_bits, afl->clean_trace, afl->fsrv.map_size); update_bitmap_score(afl, q); diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index a0a2795e..0824b77f 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -103,6 +103,7 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) { afl->stats_avg_exec = -1; afl->skip_deterministic = 1; afl->use_splicing = 1; + afl->q_testcase_max_cache_size = TESTCASE_CACHE * 1024000; #ifdef HAVE_AFFINITY afl->cpu_aff = -1; /* Selected CPU core */ @@ -353,6 +354,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_forksrv_init_tmout = (u8 *)get_afl_env(afl_environment_variables[i]); + } else if (!strncmp(env, "AFL_TESTCACHE_SIZE", + + afl_environment_variable_len)) { + + afl->afl_env.afl_testcache_size = + (u8 *)get_afl_env(afl_environment_variables[i]); + } else if (!strncmp(env, "AFL_STATSD_HOST", afl_environment_variable_len)) { diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 76f24977..4f0cab4c 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -165,6 +165,8 @@ void write_stats_file(afl_state_t *afl, double bitmap_cvg, double stability, "edges_found : %u\n" "var_byte_count : %u\n" "havoc_expansion : %u\n" + "testcache_size : %llu\n" + "testcache_count : %u\n" "afl_banner : %s\n" "afl_version : " VERSION "\n" @@ -198,8 +200,9 @@ void write_stats_file(afl_state_t *afl, double bitmap_cvg, double stability, #else -1, #endif - t_bytes, afl->var_byte_count, afl->expand_havoc, afl->use_banner, - afl->unicorn_mode ? "unicorn" : "", + t_bytes, afl->var_byte_count, afl->expand_havoc, + afl->q_testcase_cache_size, afl->q_testcase_cache_count, + afl->use_banner, afl->unicorn_mode ? "unicorn" : "", afl->fsrv.qemu_mode ? "qemu " : "", afl->non_instrumented_mode ? " non_instrumented " : "", afl->no_forkserver ? "no_fsrv " : "", afl->crash_mode ? "crash " : "", diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 6498eb30..a59abb7d 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -196,11 +196,13 @@ static void usage(u8 *argv0, int more_help) { "AFL_SKIP_BIN_CHECK: skip the check, if the target is an executable\n" "AFL_SKIP_CPUFREQ: do not warn about variable cpu clocking\n" "AFL_SKIP_CRASHES: during initial dry run do not terminate for crashing inputs\n" - "AFL_STATSD: enables StatsD metrics collection" - "AFL_STATSD_HOST: change default statsd host (default 127.0.0.1)" - "AFL_STATSD_PORT: change default statsd port (default: 8125)" - "AFL_STATSD_TAGS_FLAVOR: change default statsd tags format (default will disable tags)." - " Supported formats are: 'dogstatsd', 'librato', 'signalfx' and 'influxdb'" + "AFL_STATSD: enables StatsD metrics collection\n" + "AFL_STATSD_HOST: change default statsd host (default 127.0.0.1)\n" + "AFL_STATSD_PORT: change default statsd port (default: 8125)\n" + "AFL_STATSD_TAGS_FLAVOR: set statsd tags format (default: disable tags)\n" + " Supported formats are: 'dogstatsd', 'librato', 'signalfx'\n" + " and 'influxdb'\n" + "AFL_TESTCACHE_SIZE: use a cache for testcases, improves performance (in MB)\n" "AFL_TMPDIR: directory to use for input file generation (ramdisk recommended)\n" //"AFL_PERSISTENT: not supported anymore -> no effect, just a warning\n" //"AFL_DEFER_FORKSRV: not supported anymore -> no effect, just a warning\n" @@ -885,7 +887,7 @@ int main(int argc, char **argv_orig, char **envp) { auto_sync = 1; afl->sync_id = ck_strdup("default"); afl->is_secondary_node = 1; - OKF("no -M/-S set, autoconfiguring for \"-S %s\"", afl->sync_id); + OKF("No -M/-S set, autoconfiguring for \"-S %s\"", afl->sync_id); } @@ -1006,6 +1008,21 @@ int main(int argc, char **argv_orig, char **envp) { } + if (afl->afl_env.afl_testcache_size) { + + afl->q_testcase_max_cache_size = + (u64)atoi(afl->afl_env.afl_testcache_size) * 1024000; + OKF("Enabled testcache with %llu MB", + afl->q_testcase_max_cache_size / 1024000); + + } else { + + ACTF( + "No testcache was configured. it is recommended to use a testcache, it " + "improves performance: set AFL_TESTCACHE_SIZE=(value in MB)"); + + } + if (afl->afl_env.afl_forksrv_init_tmout) { afl->fsrv.init_tmout = atoi(afl->afl_env.afl_forksrv_init_tmout); diff --git a/src/afl-performance.c b/src/afl-performance.c index 6fa95dea..e070a05e 100644 --- a/src/afl-performance.c +++ b/src/afl-performance.c @@ -71,7 +71,7 @@ inline uint64_t rand_next(afl_state_t *afl) { inline double rand_next_percent(afl_state_t *afl) { - return (double)(((double)rand_next(afl)) / (double) 0xffffffffffffffff); + return (double)(((double)rand_next(afl)) / (double)0xffffffffffffffff); } diff --git a/test/test-custom-mutators.sh b/test/test-custom-mutators.sh index f7677ac5..d4d21048 100755 --- a/test/test-custom-mutators.sh +++ b/test/test-custom-mutators.sh @@ -1,4 +1,4 @@ -f#!/bin/sh +#!/bin/sh . ./test-pre.sh -- cgit 1.4.1 From 0f8529a3db242131486cc3bf4a66c024c2b3e126 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 15 Oct 2020 15:22:40 +0200 Subject: prepare halloween fuzzbench run --- src/afl-fuzz-queue.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index c634328f..38d7f77e 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -120,10 +120,11 @@ void create_alias_table(afl_state_t *afl) { afl->alias_probability[S[--nS]] = 1; /* - fprintf(stderr, " entry alias probability perf_score\n"); + fprintf(stderr, " entry alias probability perf_score filename\n"); for (u32 i = 0; i < n; ++i) - fprintf(stderr, " %5u %5u %11u %0.9f\n", i, afl->alias_table[i], - afl->alias_probability[i], afl->queue_buf[i]->perf_score); + fprintf(stderr, " %5u %5u %11u %0.9f %s\n", i, afl->alias_table[i], + afl->alias_probability[i], afl->queue_buf[i]->perf_score, + afl->queue_buf[i]->fname); */ } -- cgit 1.4.1 From 354bda28465588e424c0a93b413af01a603191ce Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 15 Oct 2020 15:33:47 +0200 Subject: fix reget of testcase after trim --- src/afl-fuzz-queue.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 38d7f77e..095a391f 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -875,22 +875,27 @@ void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q, if (likely(q->testcase_buf)) { - free(q->testcase_buf); - int fd = open(q->fname, O_RDONLY); + u32 len = q->len; - if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); } + if (len != old_len) { - u32 len = q->len; - q->testcase_buf = malloc(len); + afl->q_testcase_cache_size = + afl->q_testcase_cache_size + q->len - old_len; + q->testcase_buf = realloc(q->testcase_buf, len); + if (unlikely(!q->testcase_buf)) { - if (unlikely(!q->testcase_buf)) { + PFATAL("Unable to malloc '%s' with len %d", q->fname, len); - PFATAL("Unable to mmap '%s' with len %d", q->fname, len); + } } + int fd = open(q->fname, O_RDONLY); + + if (unlikely(fd < 0)) { PFATAL("Unable to open '%s'", q->fname); } + + ck_read(fd, q->testcase_buf, len, q->fname); close(fd); - afl->q_testcase_cache_size = afl->q_testcase_cache_size + q->len - old_len; } -- cgit 1.4.1 From f41aafa4f7aa446c3cb1cbe6d77364cf32a6c6cb Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 15 Oct 2020 15:48:39 +0200 Subject: retake from mem if possible --- include/afl-fuzz.h | 4 ++++ src/afl-fuzz-queue.c | 39 ++++++++++++++++++++++++++++++++++----- src/afl-fuzz-run.c | 8 ++++++-- 3 files changed, 44 insertions(+), 7 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index acded98f..6204c81b 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -1165,6 +1165,10 @@ u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q); void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q, u32 old_len); +/* If trimming changes the testcase size we have to replace it */ +void queue_testcase_retake_mem(afl_state_t *afl, struct queue_entry *q, u8 *in, + u32 len, u32 old_len); + #if TESTCASE_CACHE == 1 #error define of TESTCASE_CACHE must be zero or larger than 1 #endif diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 095a391f..92b722f6 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -870,8 +870,10 @@ u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { } -void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q, - u32 old_len) { +/* after a custom trim we need to reload the testcase from disk */ + +inline void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q, + u32 old_len) { if (likely(q->testcase_buf)) { @@ -879,9 +881,9 @@ void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q, if (len != old_len) { - afl->q_testcase_cache_size = - afl->q_testcase_cache_size + q->len - old_len; + afl->q_testcase_cache_size = afl->q_testcase_cache_size + len - old_len; q->testcase_buf = realloc(q->testcase_buf, len); + if (unlikely(!q->testcase_buf)) { PFATAL("Unable to malloc '%s' with len %d", q->fname, len); @@ -901,8 +903,35 @@ void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q, } +/* after a normal trim we need to replace the testcase with the new data */ + +inline void queue_testcase_retake_mem(afl_state_t *afl, struct queue_entry *q, + u8 *in, u32 len, u32 old_len) { + + if (likely(q->testcase_buf)) { + + if (len != old_len) { + + afl->q_testcase_cache_size = afl->q_testcase_cache_size + len - old_len; + q->testcase_buf = realloc(q->testcase_buf, len); + + if (unlikely(!q->testcase_buf)) { + + PFATAL("Unable to malloc '%s' with len %d", q->fname, len); + + } + + } + + memcpy(q->testcase_buf, in, len); + + } + +} + /* Returns the testcase buf from the file behind this queue entry. Increases the refcount. */ + inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { u32 len = q->len; @@ -913,7 +942,7 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { u8 *buf; - if (q == afl->queue_cur) { + if (unlikely(q == afl->queue_cur)) { buf = afl_realloc((void **)&afl->testcase_buf, len); diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index ab870319..dfd3abfb 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -711,7 +711,11 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { }); - if (orig_len != q->len) { queue_testcase_retake(afl, q, orig_len); } + if (orig_len != q->len || custom_trimmed) { + + queue_testcase_retake(afl, q, orig_len); + + } if (custom_trimmed) return trimmed_case; @@ -846,7 +850,7 @@ u8 trim_case(afl_state_t *afl, struct queue_entry *q, u8 *in_buf) { close(fd); - if (orig_len != q->len) queue_testcase_retake(afl, q, orig_len); + queue_testcase_retake_mem(afl, q, in_buf, q->len, orig_len); memcpy(afl->fsrv.trace_bits, afl->clean_trace, afl->fsrv.map_size); update_bitmap_score(afl, q); -- cgit 1.4.1 From d5c3b4bafdae8a68e7f63c0afdd1cc5820636f2d Mon Sep 17 00:00:00 2001 From: van Hauser Date: Fri, 16 Oct 2020 09:35:35 +0200 Subject: directly add new queue to cache --- include/afl-fuzz.h | 4 ++++ src/afl-fuzz-bitmap.c | 6 ++++++ src/afl-fuzz-queue.c | 43 ++++++++++++++++++++++++++++++++++++++++++- src/afl-showmap.c | 2 +- 4 files changed, 53 insertions(+), 2 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 6204c81b..e94f389a 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -1169,6 +1169,10 @@ void queue_testcase_retake(afl_state_t *afl, struct queue_entry *q, void queue_testcase_retake_mem(afl_state_t *afl, struct queue_entry *q, u8 *in, u32 len, u32 old_len); +/* Add a new queue entry directly to the cache */ + +void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q, u8 *mem); + #if TESTCASE_CACHE == 1 #error define of TESTCASE_CACHE must be zero or larger than 1 #endif diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index a22223b9..2653b9fd 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -623,6 +623,12 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { ck_write(fd, mem, len, queue_fn); close(fd); + if (likely(afl->q_testcase_max_cache_size)) { + + queue_testcase_store_mem(afl, afl->queue_top, mem); + + } + keeping = 1; } diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 92b722f6..f8034ebd 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -1021,7 +1021,7 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { /* Register testcase as cached */ afl->q_testcase_cache[tid] = q; - afl->q_testcase_cache_size += q->len; + afl->q_testcase_cache_size += len; ++afl->q_testcase_cache_count; if (tid >= afl->q_testcase_max_cache_count) afl->q_testcase_max_cache_count = tid + 1; @@ -1032,3 +1032,44 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { } +/* Adds the new queue entry to the cache. */ + +inline void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q, + u8 *mem) { + + u32 len = q->len; + + if (unlikely(afl->q_testcase_cache_size + len >= + afl->q_testcase_max_cache_size || + afl->q_testcase_cache_count >= TESTCASE_ENTRIES - 1)) { + + return; + + } + + u32 tid = 0; + + while (likely(afl->q_testcase_cache[tid] != NULL)) + ++tid; + + /* Map the test case into memory. */ + + q->testcase_buf = malloc(len); + + if (unlikely(!q->testcase_buf)) { + + PFATAL("Unable to malloc '%s' with len %u", q->fname, len); + + } + + memcpy(q->testcase_buf, mem, len); + + /* Register testcase as cached */ + afl->q_testcase_cache[tid] = q; + afl->q_testcase_cache_size += len; + ++afl->q_testcase_cache_count; + if (tid >= afl->q_testcase_max_cache_count) + afl->q_testcase_max_cache_count = tid + 1; + +} + diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 822e62df..6213c447 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1017,7 +1017,7 @@ int main(int argc, char **argv_orig, char **envp) { alloc_printf("%s/.afl-showmap-temp-%u", use_dir, (u32)getpid()); unlink(stdin_file); atexit(at_exit_handler); - afl->fsrv.out_file = stdin_file; + fsrv->out_file = stdin_file; fsrv->out_fd = open(stdin_file, O_RDWR | O_CREAT | O_EXCL, 0600); if (fsrv->out_fd < 0) { PFATAL("Unable to create '%s'", out_file); } -- cgit 1.4.1 From fcea01a8ea7ec507b675d839035eb0fed2f06867 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sat, 17 Oct 2020 11:38:55 +0200 Subject: add eviction stat --- include/afl-fuzz.h | 3 +++ src/afl-fuzz-queue.c | 1 + src/afl-fuzz-stats.c | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index e94f389a..880b8d50 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -704,6 +704,9 @@ typedef struct afl_state { /* How many queue entries currently have cached testcases */ u32 q_testcase_cache_count; + /* How often did we evict from the cache */ + u32 q_testcase_evictions; + /* Refs to each queue entry with cached testcase (for eviction, if cache_count * is too large) */ struct queue_entry *q_testcase_cache[TESTCASE_ENTRIES]; diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index f8034ebd..e0df7206 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -996,6 +996,7 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { afl->q_testcase_cache_size -= old_cached->len; afl->q_testcase_cache[tid] = NULL; --afl->q_testcase_cache_count; + ++afl->q_testcase_evictions; } diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 4f0cab4c..d213d054 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -167,6 +167,7 @@ void write_stats_file(afl_state_t *afl, double bitmap_cvg, double stability, "havoc_expansion : %u\n" "testcache_size : %llu\n" "testcache_count : %u\n" + "testcache_evict : %u\n" "afl_banner : %s\n" "afl_version : " VERSION "\n" @@ -202,7 +203,8 @@ void write_stats_file(afl_state_t *afl, double bitmap_cvg, double stability, #endif t_bytes, afl->var_byte_count, afl->expand_havoc, afl->q_testcase_cache_size, afl->q_testcase_cache_count, - afl->use_banner, afl->unicorn_mode ? "unicorn" : "", + afl->q_testcase_evictions, afl->use_banner, + afl->unicorn_mode ? "unicorn" : "", afl->fsrv.qemu_mode ? "qemu " : "", afl->non_instrumented_mode ? " non_instrumented " : "", afl->no_forkserver ? "no_fsrv " : "", afl->crash_mode ? "crash " : "", -- cgit 1.4.1 From ac1c3b87015dd2c9b1bae0198f7925816aa63aec Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 19 Oct 2020 11:34:57 +0200 Subject: mini improvements --- TODO.md | 1 + src/afl-fuzz-one.c | 2 +- src/afl-fuzz-queue.c | 8 ++++---- src/afl-fuzz.c | 56 +++++++++++++++++++++++++++------------------------- src/afl-showmap.c | 10 ++++++---- 5 files changed, 41 insertions(+), 36 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/TODO.md b/TODO.md index 6b43d6be..7e203d26 100644 --- a/TODO.md +++ b/TODO.md @@ -7,6 +7,7 @@ - afl-plot to support multiple plot_data - afl_custom_fuzz_splice_optin() - intel-pt tracer + - own sancov for llvm 12 ## Further down the road diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 02550d36..1e63abc7 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -2540,7 +2540,7 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { #else - if (afl->pending_favored) { + if (likely(afl->pending_favored)) { /* If we have any favored, non-fuzzed new arrivals in the queue, possibly skip to them at the expense of already-fuzzed or non-favored diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index e0df7206..7f157121 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -179,9 +179,9 @@ void mark_as_variable(afl_state_t *afl, struct queue_entry *q) { void mark_as_redundant(afl_state_t *afl, struct queue_entry *q, u8 state) { - u8 fn[PATH_MAX]; + if (likely(state == q->fs_redundant)) { return; } - if (state == q->fs_redundant) { return; } + u8 fn[PATH_MAX]; q->fs_redundant = state; @@ -521,13 +521,13 @@ void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { void cull_queue(afl_state_t *afl) { + if (likely(!afl->score_changed || afl->non_instrumented_mode)) { return; } + struct queue_entry *q; u32 len = (afl->fsrv.map_size >> 3); u32 i; u8 * temp_v = afl->map_tmp_buf; - if (afl->non_instrumented_mode || !afl->score_changed) { return; } - afl->score_changed = 0; memset(temp_v, 255, len); diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 9a82edeb..7215ecec 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1378,10 +1378,9 @@ int main(int argc, char **argv_orig, char **envp) { u32 runs_in_current_cycle = (u32)-1; u32 prev_queued_paths = 0; + u8 skipped_fuzz; - while (1) { - - u8 skipped_fuzz; + while (likely(!afl->stop_soon)) { cull_queue(afl); @@ -1418,8 +1417,8 @@ int main(int argc, char **argv_orig, char **envp) { /* If we had a full queue cycle with no new finds, try recombination strategies next. */ - if (afl->queued_paths == prev_queued && - (get_cur_time() - afl->start_time) >= 3600) { + if (unlikely(afl->queued_paths == prev_queued && + (get_cur_time() - afl->start_time) >= 3600)) { if (afl->use_splicing) { @@ -1534,46 +1533,49 @@ int main(int argc, char **argv_orig, char **envp) { } - if (likely(!afl->old_seed_selection)) { + ++runs_in_current_cycle; - ++runs_in_current_cycle; - if (unlikely(prev_queued_paths < afl->queued_paths)) { + do { - // we have new queue entries since the last run, recreate alias table - prev_queued_paths = afl->queued_paths; - create_alias_table(afl); + if (likely(!afl->old_seed_selection)) { - } + if (unlikely(prev_queued_paths < afl->queued_paths)) { - afl->current_entry = select_next_queue_entry(afl); - afl->queue_cur = afl->queue_buf[afl->current_entry]; + // we have new queue entries since the last run, recreate alias table + prev_queued_paths = afl->queued_paths; + create_alias_table(afl); - } + } - skipped_fuzz = fuzz_one(afl); + afl->current_entry = select_next_queue_entry(afl); + afl->queue_cur = afl->queue_buf[afl->current_entry]; - if (!skipped_fuzz && !afl->stop_soon && afl->sync_id) { + } - if (unlikely(afl->is_main_node)) { + skipped_fuzz = fuzz_one(afl); - if (!(sync_interval_cnt++ % (SYNC_INTERVAL / 3))) { sync_fuzzers(afl); } + if (unlikely(!afl->stop_soon && exit_1)) { afl->stop_soon = 2; } - } else { + if (unlikely(afl->old_seed_selection)) { - if (!(sync_interval_cnt++ % SYNC_INTERVAL)) { sync_fuzzers(afl); } + afl->queue_cur = afl->queue_cur->next; + ++afl->current_entry; } - } + } while (skipped_fuzz && afl->queue_cur && !afl->stop_soon); - if (!afl->stop_soon && exit_1) { afl->stop_soon = 2; } + if (!afl->stop_soon && afl->sync_id) { - if (afl->stop_soon) { break; } + if (unlikely(afl->is_main_node)) { - if (unlikely(afl->old_seed_selection)) { + if (!(sync_interval_cnt++ % (SYNC_INTERVAL / 3))) { sync_fuzzers(afl); } - afl->queue_cur = afl->queue_cur->next; - ++afl->current_entry; + } else { + + if (!(sync_interval_cnt++ % SYNC_INTERVAL)) { sync_fuzzers(afl); } + + } } diff --git a/src/afl-showmap.c b/src/afl-showmap.c index bd0d1a29..4b357254 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -209,10 +209,10 @@ static u32 write_results_to_file(afl_forkserver_t *fsrv, u8 *outfile) { if (!outfile) { FATAL("Output filename not set (Bug in AFL++?)"); } - if (cmin_mode && (fsrv->last_run_timed_out - || (!caa && child_crashed != cco))) { + if (cmin_mode && + (fsrv->last_run_timed_out || (!caa && child_crashed != cco))) { - return ret; + return ret; } @@ -298,7 +298,8 @@ static void showmap_run_target_forkserver(afl_forkserver_t *fsrv, u8 *mem, if (!quiet_mode) { SAYF(cRST "-- Program output ends --\n"); } - if (!fsrv->last_run_timed_out && !stop_soon && WIFSIGNALED(fsrv->child_status)) { + if (!fsrv->last_run_timed_out && !stop_soon && + WIFSIGNALED(fsrv->child_status)) { child_crashed = 1; @@ -1202,6 +1203,7 @@ int main(int argc, char **argv_orig, char **envp) { ret = child_crashed * 2 + fsrv->last_run_timed_out; } + if (fsrv->target_path) { ck_free(fsrv->target_path); } afl_fsrv_deinit(fsrv); -- cgit 1.4.1 From d0cdbc48aebf88bc0cc519db553ca762f794296e Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Tue, 20 Oct 2020 00:07:40 +0200 Subject: always close file descriptor when opened, use standard types --- src/afl-fuzz-queue.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 7f157121..1bda0835 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -211,13 +211,16 @@ static u8 check_if_text(struct queue_entry *q) { if (q->len < AFL_TXT_MIN_LEN) return 0; u8 buf[MAX_FILE]; - s32 fd, len = q->len, offset = 0, ascii = 0, utf8 = 0, comp; + int fd; + u32 len = q->len, offset = 0, ascii = 0, utf8 = 0; + ssize_t comp; if (len >= MAX_FILE) len = MAX_FILE - 1; if ((fd = open(q->fname, O_RDONLY)) < 0) return 0; - if ((comp = read(fd, buf, len)) != len) return 0; - buf[len] = 0; + comp = read(fd, buf, len); close(fd); + if (comp != (ssize_t)len) return 0; + buf[len] = 0; while (offset < len) { -- cgit 1.4.1 From 8d75c089384a0975e39e54f3ba23c37a6101b61c Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 22 Oct 2020 15:48:09 +0200 Subject: change queue cache algo --- src/afl-fuzz-queue.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 1bda0835..3e6deb0c 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -210,9 +210,9 @@ static u8 check_if_text(struct queue_entry *q) { if (q->len < AFL_TXT_MIN_LEN) return 0; - u8 buf[MAX_FILE]; - int fd; - u32 len = q->len, offset = 0, ascii = 0, utf8 = 0; + u8 buf[MAX_FILE]; + int fd; + u32 len = q->len, offset = 0, ascii = 0, utf8 = 0; ssize_t comp; if (len >= MAX_FILE) len = MAX_FILE - 1; @@ -913,7 +913,7 @@ inline void queue_testcase_retake_mem(afl_state_t *afl, struct queue_entry *q, if (likely(q->testcase_buf)) { - if (len != old_len) { + if (likely(len != old_len)) { afl->q_testcase_cache_size = afl->q_testcase_cache_size + len - old_len; q->testcase_buf = realloc(q->testcase_buf, len); @@ -976,17 +976,21 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { if (unlikely(!q->testcase_buf)) { /* Buf not cached, let's load it */ - u32 tid = 0; + u32 tid = afl->q_testcase_max_cache_count; while (unlikely(afl->q_testcase_cache_size + len >= afl->q_testcase_max_cache_size || afl->q_testcase_cache_count >= TESTCASE_ENTRIES - 1)) { - /* Cache full. We neet to evict one to map one. + /* Cache full. We neet to evict one or more to map one. Get a random one which is not in use */ do { + // if the cache (MB) is not enough for the queue then this gets + // undesirable because q_testcase_max_cache_count grows sometimes + // although the number of items in the cache will not change hence + // more and more loops tid = rand_below(afl, afl->q_testcase_max_cache_count); } while (afl->q_testcase_cache[tid] == NULL || @@ -1003,7 +1007,16 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { } - while (likely(afl->q_testcase_cache[tid] != NULL)) + if (tid >= TESTCASE_ENTRIES) { + + // uh we were full, so now we have to search from start + tid = 0; + + } + + // we need this while loop in case there were ever previous evictions but + // not in this call. + while (unlikely(afl->q_testcase_cache[tid] != NULL)) ++tid; /* Map the test case into memory. */ @@ -1047,6 +1060,7 @@ inline void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q, afl->q_testcase_max_cache_size || afl->q_testcase_cache_count >= TESTCASE_ENTRIES - 1)) { + // no space? will be loaded regularly later. return; } -- cgit 1.4.1 From c866aef37fcf799506d93b9a47d4eb2b77c75f5b Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 22 Oct 2020 21:07:37 +0200 Subject: maybe enhancement to cache algo --- include/afl-fuzz.h | 5 ++++- src/afl-fuzz-queue.c | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 880b8d50..220380b9 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -704,7 +704,10 @@ typedef struct afl_state { /* How many queue entries currently have cached testcases */ u32 q_testcase_cache_count; - /* How often did we evict from the cache */ + /* the smallest id currently known free entry */ + u32 q_testcase_smallest_free; + + /* How often did we evict from the cache (for statistics only) */ u32 q_testcase_evictions; /* Refs to each queue entry with cached testcase (for eviction, if cache_count diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 3e6deb0c..db387c33 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -1004,13 +1004,15 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { afl->q_testcase_cache[tid] = NULL; --afl->q_testcase_cache_count; ++afl->q_testcase_evictions; + if (tid < afl->q_testcase_smallest_free) + afl->q_testcase_smallest_free = tid; } - if (tid >= TESTCASE_ENTRIES) { + if (unlikely(tid >= TESTCASE_ENTRIES)) { // uh we were full, so now we have to search from start - tid = 0; + tid = afl->q_testcase_smallest_free; } @@ -1042,6 +1044,8 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { ++afl->q_testcase_cache_count; if (tid >= afl->q_testcase_max_cache_count) afl->q_testcase_max_cache_count = tid + 1; + if (tid == afl->q_testcase_smallest_free) + afl->q_testcase_smallest_free = tid + 1; } -- cgit 1.4.1 From 0e748ccda713708de6a501d23a58788aba9d0b03 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Fri, 23 Oct 2020 14:05:34 +0200 Subject: set max testcache entries automated if not specified by the user --- include/afl-fuzz.h | 7 ++++-- include/config.h | 10 ++++---- include/envs.h | 1 + src/afl-fuzz-queue.c | 11 +++++---- src/afl-fuzz-state.c | 10 +++++++- src/afl-fuzz-stats.c | 2 +- src/afl-fuzz.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++++--- 7 files changed, 91 insertions(+), 18 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 220380b9..11feb9f7 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -366,7 +366,7 @@ typedef struct afl_env_vars { u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path, *afl_hang_tmout, *afl_forksrv_init_tmout, *afl_skip_crashes, *afl_preload, *afl_max_det_extras, *afl_statsd_host, *afl_statsd_port, - *afl_statsd_tags_flavor, *afl_testcache_size; + *afl_statsd_tags_flavor, *afl_testcache_size, *afl_testcache_entries; } afl_env_vars_t; @@ -695,6 +695,9 @@ typedef struct afl_state { /* This is the user specified maximum size to use for the testcase cache */ u64 q_testcase_max_cache_size; + /* This is the user specified maximum entries in the testcase cache */ + u32 q_testcase_max_cache_entries; + /* How much of the testcase cache is used so far */ u64 q_testcase_cache_size; @@ -712,7 +715,7 @@ typedef struct afl_state { /* Refs to each queue entry with cached testcase (for eviction, if cache_count * is too large) */ - struct queue_entry *q_testcase_cache[TESTCASE_ENTRIES]; + struct queue_entry **q_testcase_cache; } afl_state_t; diff --git a/include/config.h b/include/config.h index b4f3a775..491d8132 100644 --- a/include/config.h +++ b/include/config.h @@ -295,14 +295,12 @@ #define RESEED_RNG 100000 -/* The maximum number of testcases to cache */ - -#define TESTCASE_ENTRIES 16384 - /* The default maximum testcase cache size in MB, 0 = disable. - A value between 50 and 250 is a good default value. */ + A value between 50 and 250 is a good default value. Note that the + number of entries will be auto assigned if not specified via the + AFL_TESTCACHE_ENTRIES env variable */ -#define TESTCASE_CACHE 0 +#define TESTCASE_CACHE_SIZE 50 /* Maximum line length passed from GCC to 'as' and used for parsing configuration files: */ diff --git a/include/envs.h b/include/envs.h index a1b3ad12..b753d5f8 100644 --- a/include/envs.h +++ b/include/envs.h @@ -140,6 +140,7 @@ static char *afl_environment_variables[] = { "AFL_STATSD_PORT", "AFL_STATSD_TAGS_FLAVOR", "AFL_TESTCACHE_SIZE", + "AFL_TESTCACHE_ENTRIES", "AFL_TMIN_EXACT", "AFL_TMPDIR", "AFL_TOKEN_FILE", diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index db387c33..baa80e61 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -978,9 +978,9 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { /* Buf not cached, let's load it */ u32 tid = afl->q_testcase_max_cache_count; - while (unlikely(afl->q_testcase_cache_size + len >= - afl->q_testcase_max_cache_size || - afl->q_testcase_cache_count >= TESTCASE_ENTRIES - 1)) { + while (unlikely( + afl->q_testcase_cache_size + len >= afl->q_testcase_max_cache_size || + afl->q_testcase_cache_count >= afl->q_testcase_max_cache_entries - 1)) { /* Cache full. We neet to evict one or more to map one. Get a random one which is not in use */ @@ -1009,7 +1009,7 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { } - if (unlikely(tid >= TESTCASE_ENTRIES)) { + if (unlikely(tid >= afl->q_testcase_max_cache_entries)) { // uh we were full, so now we have to search from start tid = afl->q_testcase_smallest_free; @@ -1062,7 +1062,8 @@ inline void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q, if (unlikely(afl->q_testcase_cache_size + len >= afl->q_testcase_max_cache_size || - afl->q_testcase_cache_count >= TESTCASE_ENTRIES - 1)) { + afl->q_testcase_cache_count >= + afl->q_testcase_max_cache_entries - 1)) { // no space? will be loaded regularly later. return; diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index 0824b77f..ae7d410b 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -103,7 +103,8 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) { afl->stats_avg_exec = -1; afl->skip_deterministic = 1; afl->use_splicing = 1; - afl->q_testcase_max_cache_size = TESTCASE_CACHE * 1024000; + afl->q_testcase_max_cache_size = TESTCASE_CACHE_SIZE * 1048576UL; + afl->q_testcase_max_cache_entries = 4096; #ifdef HAVE_AFFINITY afl->cpu_aff = -1; /* Selected CPU core */ @@ -361,6 +362,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_testcache_size = (u8 *)get_afl_env(afl_environment_variables[i]); + } else if (!strncmp(env, "AFL_TESTCACHE_ENTRIES", + + afl_environment_variable_len)) { + + afl->afl_env.afl_testcache_entries = + (u8 *)get_afl_env(afl_environment_variables[i]); + } else if (!strncmp(env, "AFL_STATSD_HOST", afl_environment_variable_len)) { diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index d213d054..bec90519 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -1028,7 +1028,7 @@ void show_init_stats(afl_state_t *afl) { } - SAYF("\n"); + // SAYF("\n"); if (avg_us > ((afl->fsrv.qemu_mode || afl->unicorn_mode) ? 50000 : 10000)) { diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 7215ecec..637e1985 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -251,7 +251,7 @@ static int stricmp(char const *a, char const *b) { int main(int argc, char **argv_orig, char **envp) { - s32 opt, i, auto_sync = 0; + s32 opt, i, auto_sync = 0, user_set_cache = 0; u64 prev_queued = 0; u32 sync_interval_cnt = 0, seek_to = 0, show_help = 0, map_size = MAP_SIZE; u8 *extras_dir[4]; @@ -1015,6 +1015,22 @@ int main(int argc, char **argv_orig, char **envp) { } + if (afl->afl_env.afl_testcache_entries) { + + afl->q_testcase_max_cache_entries = + (u32)atoi(afl->afl_env.afl_testcache_entries); + + user_set_cache = 1; + + } + + if (!afl->afl_env.afl_testcache_size || !afl->afl_env.afl_testcache_entries) { + + afl->afl_env.afl_testcache_entries = 0; + afl->afl_env.afl_testcache_size = 0; + + } + if (!afl->q_testcase_max_cache_size) { ACTF( @@ -1347,6 +1363,52 @@ int main(int argc, char **argv_orig, char **envp) { perform_dry_run(afl); + if (!user_set_cache && afl->q_testcase_max_cache_size) { + + /* The user defined not a fixed number of entries for the cache. + Hence we autodetect a good value. After the dry run inputs are + trimmed and we know the average and max size of the input seeds. + We use this information to set a fitting size to max entries + based on the cache size. */ + + struct queue_entry *q = afl->queue; + u64 size = 0, count = 0, avg = 0, max = 0; + + while (q) { + + ++count; + size += q->len; + if (max < q->len) { max = q->len; } + q = q->next; + + } + + if (count) { + + avg = size / count; + avg = ((avg + max) / 2) + 1; + + } + + if (avg < 10240) { avg = 10240; } + + afl->q_testcase_max_cache_entries = afl->q_testcase_max_cache_size / avg; + + if (afl->q_testcase_max_cache_entries > 32768) + afl->q_testcase_max_cache_entries = 32768; + + } + + if (afl->q_testcase_max_cache_entries) { + + OKF("Setting %u maximum entries for the testcase cache", + afl->q_testcase_max_cache_entries); + afl->q_testcase_cache = + ck_alloc(afl->q_testcase_max_cache_entries * sizeof(size_t)); + if (!afl->q_testcase_cache) { PFATAL("malloc failed for cache entries"); } + + } + cull_queue(afl); if (!afl->pending_not_fuzzed) @@ -1366,8 +1428,7 @@ int main(int argc, char **argv_orig, char **envp) { if (!afl->not_on_tty) { - sleep(4); - afl->start_time += 4000; + sleep(1); if (afl->stop_soon) { goto stop_fuzzing; } } @@ -1654,6 +1715,7 @@ stop_fuzzing: ck_free(afl->fsrv.target_path); ck_free(afl->fsrv.out_file); ck_free(afl->sync_id); + if (afl->q_testcase_cache) { ck_free(afl->q_testcase_cache); } afl_state_deinit(afl); free(afl); /* not tracked */ -- cgit 1.4.1 From aa0d3785206d52c1815aff850817d55bf50f3598 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Fri, 23 Oct 2020 15:21:21 +0200 Subject: better cache entry algo --- src/afl-fuzz-queue.c | 18 +++++++++++++++++ src/afl-fuzz-state.c | 2 +- src/afl-fuzz.c | 55 ++++++++++++++++++++++++++-------------------------- 3 files changed, 47 insertions(+), 28 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index baa80e61..4989a0ba 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -985,6 +985,24 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { /* Cache full. We neet to evict one or more to map one. Get a random one which is not in use */ + if (unlikely(afl->q_testcase_cache_size + len >= afl->q_testcase_max_cache_size && + (afl->q_testcase_cache_count < afl->q_testcase_max_cache_entries && + afl->q_testcase_max_cache_count < + afl->q_testcase_max_cache_entries))) { + + if (afl->q_testcase_max_cache_count > afl->q_testcase_cache_count) { + + afl->q_testcase_max_cache_entries = + afl->q_testcase_max_cache_count + 1; + + } else { + + afl->q_testcase_max_cache_entries = afl->q_testcase_cache_count + 1; + + } + + } + do { // if the cache (MB) is not enough for the queue then this gets diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index ae7d410b..3ce16cad 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -104,7 +104,7 @@ void afl_state_init(afl_state_t *afl, uint32_t map_size) { afl->skip_deterministic = 1; afl->use_splicing = 1; afl->q_testcase_max_cache_size = TESTCASE_CACHE_SIZE * 1048576UL; - afl->q_testcase_max_cache_entries = 4096; + afl->q_testcase_max_cache_entries = 64 * 1024; #ifdef HAVE_AFFINITY afl->cpu_aff = -1; /* Selected CPU core */ diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 637e1985..70e21c0f 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -251,7 +251,7 @@ static int stricmp(char const *a, char const *b) { int main(int argc, char **argv_orig, char **envp) { - s32 opt, i, auto_sync = 0, user_set_cache = 0; + s32 opt, i, auto_sync = 0 /*, user_set_cache = 0*/; u64 prev_queued = 0; u32 sync_interval_cnt = 0, seek_to = 0, show_help = 0, map_size = MAP_SIZE; u8 *extras_dir[4]; @@ -1020,7 +1020,7 @@ int main(int argc, char **argv_orig, char **envp) { afl->q_testcase_max_cache_entries = (u32)atoi(afl->afl_env.afl_testcache_entries); - user_set_cache = 1; + // user_set_cache = 1; } @@ -1363,46 +1363,47 @@ int main(int argc, char **argv_orig, char **envp) { perform_dry_run(afl); - if (!user_set_cache && afl->q_testcase_max_cache_size) { + /* + if (!user_set_cache && afl->q_testcase_max_cache_size) { - /* The user defined not a fixed number of entries for the cache. - Hence we autodetect a good value. After the dry run inputs are - trimmed and we know the average and max size of the input seeds. - We use this information to set a fitting size to max entries - based on the cache size. */ + / * The user defined not a fixed number of entries for the cache. + Hence we autodetect a good value. After the dry run inputs are + trimmed and we know the average and max size of the input seeds. + We use this information to set a fitting size to max entries + based on the cache size. * / - struct queue_entry *q = afl->queue; - u64 size = 0, count = 0, avg = 0, max = 0; + struct queue_entry *q = afl->queue; + u64 size = 0, count = 0, avg = 0, max = 0; - while (q) { + while (q) { - ++count; - size += q->len; - if (max < q->len) { max = q->len; } - q = q->next; + ++count; + size += q->len; + if (max < q->len) { max = q->len; } + q = q->next; - } + } - if (count) { + if (count) { - avg = size / count; - avg = ((avg + max) / 2) + 1; + avg = size / count; + avg = ((avg + max) / 2) + 1; - } + } - if (avg < 10240) { avg = 10240; } + if (avg < 10240) { avg = 10240; } - afl->q_testcase_max_cache_entries = afl->q_testcase_max_cache_size / avg; + afl->q_testcase_max_cache_entries = afl->q_testcase_max_cache_size / avg; - if (afl->q_testcase_max_cache_entries > 32768) - afl->q_testcase_max_cache_entries = 32768; + if (afl->q_testcase_max_cache_entries > 32768) + afl->q_testcase_max_cache_entries = 32768; - } + } + + */ if (afl->q_testcase_max_cache_entries) { - OKF("Setting %u maximum entries for the testcase cache", - afl->q_testcase_max_cache_entries); afl->q_testcase_cache = ck_alloc(afl->q_testcase_max_cache_entries * sizeof(size_t)); if (!afl->q_testcase_cache) { PFATAL("malloc failed for cache entries"); } -- cgit 1.4.1 From 4e99e3b36c8af6488fde10fac8cdb0797c95e02a Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Sat, 24 Oct 2020 03:16:22 +0200 Subject: initializing testcase_buf --- src/afl-fuzz-queue.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 4989a0ba..02e66a4e 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -325,6 +325,7 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) { q->depth = afl->cur_depth + 1; q->passed_det = passed_det; q->trace_mini = NULL; + q->testcase_buf = NULL; if (q->depth > afl->max_depth) { afl->max_depth = q->depth; } -- cgit 1.4.1 From e5f30c690822d69b4d99117024ae2570b1572481 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sat, 24 Oct 2020 16:28:46 +0200 Subject: fix testcache bug --- src/afl-fuzz-init.c | 1 + src/afl-fuzz-queue.c | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 1bccff8f..19a8d77b 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -2358,6 +2358,7 @@ void check_asan_opts(afl_state_t *afl) { FATAL("Custom ASAN_OPTIONS set without symbolize=0 - please fix!"); } + #endif } diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 02e66a4e..d107dbc8 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -914,20 +914,22 @@ inline void queue_testcase_retake_mem(afl_state_t *afl, struct queue_entry *q, if (likely(q->testcase_buf)) { + u32 is_same = in == q->testcase_buf; + if (likely(len != old_len)) { - afl->q_testcase_cache_size = afl->q_testcase_cache_size + len - old_len; - q->testcase_buf = realloc(q->testcase_buf, len); + u8 *ptr = realloc(q->testcase_buf, len); - if (unlikely(!q->testcase_buf)) { + if (likely(ptr)) { - PFATAL("Unable to malloc '%s' with len %d", q->fname, len); + q->testcase_buf = ptr; + afl->q_testcase_cache_size = afl->q_testcase_cache_size + len - old_len; } } - memcpy(q->testcase_buf, in, len); + if (unlikely(!is_same)) { memcpy(q->testcase_buf, in, len); } } @@ -986,10 +988,12 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { /* Cache full. We neet to evict one or more to map one. Get a random one which is not in use */ - if (unlikely(afl->q_testcase_cache_size + len >= afl->q_testcase_max_cache_size && - (afl->q_testcase_cache_count < afl->q_testcase_max_cache_entries && - afl->q_testcase_max_cache_count < - afl->q_testcase_max_cache_entries))) { + if (unlikely(afl->q_testcase_cache_size + len >= + afl->q_testcase_max_cache_size && + (afl->q_testcase_cache_count < + afl->q_testcase_max_cache_entries && + afl->q_testcase_max_cache_count < + afl->q_testcase_max_cache_entries))) { if (afl->q_testcase_max_cache_count > afl->q_testcase_cache_count) { -- cgit 1.4.1 From 868cb61ea6a2949e80e8a94fe7b19229bebecd10 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 26 Oct 2020 15:24:33 +0100 Subject: hopeful the final testcache improvement ... --- src/afl-fuzz-queue.c | 56 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 9 deletions(-) (limited to 'src/afl-fuzz-queue.c') diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index d107dbc8..c78df8be 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -979,21 +979,23 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { if (unlikely(!q->testcase_buf)) { /* Buf not cached, let's load it */ - u32 tid = afl->q_testcase_max_cache_count; + u32 tid = afl->q_testcase_max_cache_count; + static u32 do_once = 0; // because even threaded we would want this. WIP while (unlikely( afl->q_testcase_cache_size + len >= afl->q_testcase_max_cache_size || afl->q_testcase_cache_count >= afl->q_testcase_max_cache_entries - 1)) { - /* Cache full. We neet to evict one or more to map one. - Get a random one which is not in use */ + /* We want a max number of entries to the cache that we learn. + Very simple: once the cache is filled by size - that is the max. */ if (unlikely(afl->q_testcase_cache_size + len >= afl->q_testcase_max_cache_size && (afl->q_testcase_cache_count < afl->q_testcase_max_cache_entries && afl->q_testcase_max_cache_count < - afl->q_testcase_max_cache_entries))) { + afl->q_testcase_max_cache_entries) && + !do_once)) { if (afl->q_testcase_max_cache_count > afl->q_testcase_cache_count) { @@ -1006,8 +1008,19 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { } + do_once = 1; + // release unneeded memory + u8 *ptr = ck_realloc( + afl->q_testcase_cache, + (afl->q_testcase_max_cache_entries + 1) * sizeof(size_t)); + + if (ptr) { afl->q_testcase_cache = (struct queue_entry **)ptr; } + } + /* Cache full. We neet to evict one or more to map one. + Get a random one which is not in use */ + do { // if the cache (MB) is not enough for the queue then this gets @@ -1065,11 +1078,16 @@ inline u8 *queue_testcase_get(afl_state_t *afl, struct queue_entry *q) { afl->q_testcase_cache[tid] = q; afl->q_testcase_cache_size += len; ++afl->q_testcase_cache_count; - if (tid >= afl->q_testcase_max_cache_count) + if (likely(tid >= afl->q_testcase_max_cache_count)) { + afl->q_testcase_max_cache_count = tid + 1; - if (tid == afl->q_testcase_smallest_free) + + } else if (unlikely(tid == afl->q_testcase_smallest_free)) { + afl->q_testcase_smallest_free = tid + 1; + } + } return q->testcase_buf; @@ -1093,9 +1111,21 @@ inline void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q, } - u32 tid = 0; + u32 tid; + + if (unlikely(afl->q_testcase_max_cache_count >= + afl->q_testcase_max_cache_entries)) { - while (likely(afl->q_testcase_cache[tid] != NULL)) + // uh we were full, so now we have to search from start + tid = afl->q_testcase_smallest_free; + + } else { + + tid = afl->q_testcase_max_cache_count; + + } + + while (unlikely(afl->q_testcase_cache[tid] != NULL)) ++tid; /* Map the test case into memory. */ @@ -1114,8 +1144,16 @@ inline void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q, afl->q_testcase_cache[tid] = q; afl->q_testcase_cache_size += len; ++afl->q_testcase_cache_count; - if (tid >= afl->q_testcase_max_cache_count) + + if (likely(tid >= afl->q_testcase_max_cache_count)) { + afl->q_testcase_max_cache_count = tid + 1; + } else if (unlikely(tid == afl->q_testcase_smallest_free)) { + + afl->q_testcase_smallest_free = tid + 1; + + } + } -- cgit 1.4.1