diff options
256 files changed, 12820 insertions, 10045 deletions
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 31152cd2..0d80f4a3 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -8,8 +8,9 @@ assignees: '' --- **IMPORTANT** -1. You have verified that the issue to be present in the current `dev` branch -2. Please supply the command line options and relevant environment variables, e.g. a copy-paste of the contents of `out/default/fuzzer_setup` +1. You have verified that the issue to be present in the current `dev` branch. +2. Please supply the command line options and relevant environment variables, + e.g., a copy-paste of the contents of `out/default/fuzzer_setup`. Thank you for making AFL++ better! diff --git a/.gitignore b/.gitignore index 5268bb37..22ee6bf1 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ afl-g++-fast afl-gotcpu afl-ld afl-ld-lto +afl-cs-proxy afl-qemu-trace afl-showmap afl-tmin @@ -94,3 +95,5 @@ utils/optimin/optimin utils/persistent_mode/persistent_demo utils/persistent_mode/persistent_demo_new utils/persistent_mode/test-instr +!coresight_mode +!coresight_mode/coresight-trace diff --git a/.gitmodules b/.gitmodules index 200f3ecc..6569c0b1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,9 @@ [submodule "utils/optimin/EvalMaxSAT"] path = utils/optimin/EvalMaxSAT url = https://github.com/FlorentAvellaneda/EvalMaxSAT +[submodule "coresight_mode/patchelf"] + path = coresight_mode/patchelf + url = https://github.com/NixOS/patchelf.git +[submodule "coresight_mode/coresight-trace"] + path = coresight_mode/coresight-trace + url = https://github.com/RICSecLab/coresight-trace.git diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c36ed9d8..fb13b91a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# How to submit a Pull Request to AFLplusplus +# How to submit a Pull Request to AFL++ All contributions (pull requests) must be made against our `dev` branch. @@ -15,10 +15,9 @@ project, or added a file in a directory we already format, otherwise run: ./.custom-format.py -i file-that-you-have-created.c ``` -Regarding the coding style, please follow the AFL style. -No camel case at all and use AFL's macros wherever possible -(e.g. WARNF, FATAL, MAP_SIZE, ...). +Regarding the coding style, please follow the AFL style. No camel case at all +and use AFL's macros wherever possible (e.g., WARNF, FATAL, MAP_SIZE, ...). -Remember that AFLplusplus has to build and run on many platforms, so -generalize your Makefiles/GNUmakefile (or your patches to our pre-existing -Makefiles) to be as generic as possible. +Remember that AFL++ has to build and run on many platforms, so generalize your +Makefiles/GNUmakefile (or your patches to our pre-existing Makefiles) to be as +generic as possible. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 18fb6367..a3c70746 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,8 @@ env NO_ARCH_OPT 1 RUN apt-get update && \ apt-get -y install --no-install-suggests --no-install-recommends \ automake \ + cmake \ + meson \ ninja-build \ bison flex \ build-essential \ diff --git a/GNUmakefile b/GNUmakefile index 376f6e9a..673d2bf8 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -10,7 +10,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # For Heiko: @@ -32,7 +32,7 @@ VERSION = $(shell grep '^$(HASH)define VERSION ' ../config.h | cut -d '"' -f # PROGS intentionally omit afl-as, which gets installed elsewhere. PROGS = afl-fuzz afl-showmap afl-tmin afl-gotcpu afl-analyze -SH_PROGS = afl-plot afl-cmin afl-cmin.bash afl-whatsup afl-system-config afl-persistent-config +SH_PROGS = afl-plot afl-cmin afl-cmin.bash afl-whatsup afl-system-config afl-persistent-config afl-cc MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8) afl-as.8 ASAN_OPTIONS=detect_leaks=0 @@ -346,7 +346,7 @@ help: @echo "HELP --- the following make targets exist:" @echo "==========================================" @echo "all: just the main afl++ binaries" - @echo "binary-only: everything for binary-only fuzzing: qemu_mode, unicorn_mode, libdislocator, libtokencap" + @echo "binary-only: everything for binary-only fuzzing: qemu_mode, frida_mode, unicorn_mode, coresight_mode, libdislocator, libtokencap" @echo "source-only: everything for source code fuzzing: gcc_plugin, libdislocator, libtokencap" @echo "distrib: everything (for both binary-only and source code fuzzing)" @echo "man: creates simple man pages from the help option of the programs" @@ -541,7 +541,7 @@ test_build: afl-cc afl-gcc afl-as afl-showmap # echo 1 | ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -q -o .test-instr1 ./test-instr # @rm -f test-instr # @cmp -s .test-instr0 .test-instr1; DR="$$?"; rm -f .test-instr0 .test-instr1; if [ "$$DR" = "0" ]; then echo; echo "Oops, the instrumentation of afl-gcc does not seem to be behaving correctly!"; \ -# gcc -v 2>&1 | grep -q -- --with-as= && ( echo; echo "Gcc is configured not to use an external assembler with the -B option."; echo "See docs/INSTALL.md section 5 how to build a -B enabled gcc." ) || \ +# gcc -v 2>&1 | grep -q -- --with-as= && ( echo; echo "Gcc is configured not to use an external assembler with the -B option." ) || \ # ( echo; echo "Please post to https://github.com/AFLplusplus/AFLplusplus/issues to troubleshoot the issue." ); echo; exit 0; fi # @echo # @echo "[+] All right, the instrumentation of afl-gcc seems to be working!" @@ -564,7 +564,7 @@ all_done: test_build .PHONY: clean clean: - rm -rf $(PROGS) libradamsa.so afl-fuzz-document afl-as as afl-g++ afl-clang afl-clang++ *.o src/*.o *~ a.out core core.[1-9][0-9]* *.stackdump .test .test1 .test2 test-instr .test-instr0 .test-instr1 afl-qemu-trace afl-gcc-fast afl-gcc-pass.so afl-g++-fast ld *.so *.8 test/unittests/*.o test/unittests/unit_maybe_alloc test/unittests/preallocable .afl-* afl-gcc afl-g++ afl-clang afl-clang++ test/unittests/unit_hash test/unittests/unit_rand *.dSYM + rm -rf $(PROGS) libradamsa.so afl-fuzz-document afl-as as afl-g++ afl-clang afl-clang++ *.o src/*.o *~ a.out core core.[1-9][0-9]* *.stackdump .test .test1 .test2 test-instr .test-instr0 .test-instr1 afl-cs-proxy afl-qemu-trace afl-gcc-fast afl-gcc-pass.so afl-g++-fast ld *.so *.8 test/unittests/*.o test/unittests/unit_maybe_alloc test/unittests/preallocable .afl-* afl-gcc afl-g++ afl-clang afl-clang++ test/unittests/unit_hash test/unittests/unit_rand *.dSYM -$(MAKE) -f GNUmakefile.llvm clean -$(MAKE) -f GNUmakefile.gcc_plugin clean $(MAKE) -C utils/libdislocator clean @@ -579,19 +579,23 @@ clean: $(MAKE) -C qemu_mode/libqasan clean -$(MAKE) -C frida_mode clean ifeq "$(IN_REPO)" "1" + -test -e coresight_mode/coresight-trace/Makefile && $(MAKE) -C coresight_mode/coresight-trace clean || true -test -e qemu_mode/qemuafl/Makefile && $(MAKE) -C qemu_mode/qemuafl clean || true test -e unicorn_mode/unicornafl/Makefile && $(MAKE) -C unicorn_mode/unicornafl clean || true else + rm -rf coresight_mode/coresight_trace rm -rf qemu_mode/qemuafl rm -rf unicorn_mode/unicornafl endif .PHONY: deepclean deepclean: clean + rm -rf coresight_mode/coresight-trace rm -rf unicorn_mode/unicornafl rm -rf qemu_mode/qemuafl ifeq "$(IN_REPO)" "1" # NEVER EVER ACTIVATE THAT!!!!! git reset --hard >/dev/null 2>&1 || true + git checkout coresight_mode/coresight-trace git checkout unicorn_mode/unicornafl git checkout qemu_mode/qemuafl endif @@ -610,6 +614,9 @@ endif # -$(MAKE) -C utils/plot_ui -$(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" +ifeq "$(ARCH)" "aarch64" + -$(MAKE) -C coresight_mode +endif -cd qemu_mode && sh ./build_qemu_support.sh -cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh endif @@ -624,6 +631,9 @@ binary-only: test_shm test_python ready $(PROGS) # -$(MAKE) -C utils/plot_ui -$(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" +ifeq "$(ARCH)" "aarch64" + -$(MAKE) -C coresight_mode +endif -cd qemu_mode && sh ./build_qemu_support.sh -cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh endif @@ -695,7 +705,7 @@ endif .PHONY: uninstall uninstall: - -cd $${DESTDIR}$(BIN_PATH) && rm -f $(PROGS) $(SH_PROGS) afl-qemu-trace afl-plot-ui afl-fuzz-document afl-network-server afl-g* afl-plot.sh afl-as afl-ld-lto afl-c* afl-lto* + -cd $${DESTDIR}$(BIN_PATH) && rm -f $(PROGS) $(SH_PROGS) afl-cs-proxy afl-qemu-trace afl-plot-ui afl-fuzz-document afl-network-server afl-g* afl-plot.sh afl-as afl-ld-lto afl-c* afl-lto* -cd $${DESTDIR}$(HELPER_PATH) && rm -f afl-g*.*o afl-llvm-*.*o afl-compiler-*.*o libdislocator.so libtokencap.so libcompcov.so libqasan.so afl-frida-trace.so socketfuzz*.so argvfuzz*.so libAFLDriver.a libAFLQemuDriver.a as afl-as SanitizerCoverage*.so compare-transform-pass.so cmplog-*-pass.so split-*-pass.so dynamic_list.txt -rm -rf $${DESTDIR}$(MISC_PATH)/testcases $${DESTDIR}$(MISC_PATH)/dictionaries -sh -c "ls docs/*.md | sed 's|^docs/|$${DESTDIR}$(DOC_PATH)/|' | xargs rm -f" diff --git a/GNUmakefile.gcc_plugin b/GNUmakefile.gcc_plugin index bce97b2f..ed2725d7 100644 --- a/GNUmakefile.gcc_plugin +++ b/GNUmakefile.gcc_plugin @@ -17,7 +17,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # #TEST_MMAP=1 PREFIX ?= /usr/local diff --git a/GNUmakefile.llvm b/GNUmakefile.llvm index b802ef16..1e2c411d 100644 --- a/GNUmakefile.llvm +++ b/GNUmakefile.llvm @@ -12,7 +12,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # For Heiko: @@ -308,7 +308,7 @@ ifeq "$(TEST_MMAP)" "1" endif PROGS_ALWAYS = ./afl-cc ./afl-compiler-rt.o ./afl-compiler-rt-32.o ./afl-compiler-rt-64.o -PROGS = $(PROGS_ALWAYS) ./afl-llvm-pass.so ./SanitizerCoveragePCGUARD.so ./split-compares-pass.so ./split-switches-pass.so ./cmplog-routines-pass.so ./cmplog-instructions-pass.so ./cmplog-switches-pass.so ./afl-llvm-dict2file.so ./compare-transform-pass.so ./afl-ld-lto ./afl-llvm-lto-instrumentlist.so ./afl-llvm-lto-instrumentation.so ./SanitizerCoverageLTO.so +PROGS = $(PROGS_ALWAYS) ./afl-llvm-pass.so ./SanitizerCoveragePCGUARD.so ./split-compares-pass.so ./split-switches-pass.so ./cmplog-routines-pass.so ./cmplog-instructions-pass.so ./cmplog-switches-pass.so ./afl-llvm-dict2file.so ./compare-transform-pass.so ./afl-ld-lto ./afl-llvm-lto-instrumentlist.so ./SanitizerCoverageLTO.so # If prerequisites are not given, warn, do not build anything, and exit with code 0 ifeq "$(LLVMVER)" "" @@ -410,11 +410,6 @@ endif ./SanitizerCoverageLTO.so: instrumentation/SanitizerCoverageLTO.so.cc ifeq "$(LLVM_LTO)" "1" $(CXX) $(CLANG_CPPFL) -Wno-writable-strings -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) instrumentation/afl-llvm-common.o -endif - -./afl-llvm-lto-instrumentation.so: instrumentation/afl-llvm-lto-instrumentation.so.cc instrumentation/afl-llvm-common.o -ifeq "$(LLVM_LTO)" "1" - $(CXX) $(CLANG_CPPFL) -Wno-writable-strings -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) instrumentation/afl-llvm-common.o $(CLANG_BIN) $(CFLAGS_SAFE) $(CPPFLAGS) -Wno-unused-result -O0 $(AFL_CLANG_FLTO) -fPIC -c instrumentation/afl-llvm-rt-lto.o.c -o ./afl-llvm-rt-lto.o @$(CLANG_BIN) $(CFLAGS_SAFE) $(CPPFLAGS) -Wno-unused-result -O0 $(AFL_CLANG_FLTO) -m64 -fPIC -c instrumentation/afl-llvm-rt-lto.o.c -o ./afl-llvm-rt-lto-64.o 2>/dev/null; if [ "$$?" = "0" ]; then : ; fi @$(CLANG_BIN) $(CFLAGS_SAFE) $(CPPFLAGS) -Wno-unused-result -O0 $(AFL_CLANG_FLTO) -m32 -fPIC -c instrumentation/afl-llvm-rt-lto.o.c -o ./afl-llvm-rt-lto-32.o 2>/dev/null; if [ "$$?" = "0" ]; then : ; fi @@ -480,7 +475,7 @@ install: all @if [ -f ./afl-cc ]; then set -e; install -m 755 ./afl-cc $${DESTDIR}$(BIN_PATH); ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-c++; fi @rm -f $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt*.o $${DESTDIR}$(HELPER_PATH)/afl-gcc-rt*.o @if [ -f ./afl-compiler-rt.o ]; then set -e; install -m 755 ./afl-compiler-rt.o $${DESTDIR}$(HELPER_PATH); ln -sf afl-compiler-rt.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt.o ;fi - @if [ -f ./afl-lto ]; then set -e; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-lto; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-lto++; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang-lto; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang-lto++; install -m 755 ./afl-llvm-lto-instrumentation.so ./afl-llvm-rt-lto*.o ./afl-llvm-lto-instrumentlist.so $${DESTDIR}$(HELPER_PATH); fi + @if [ -f ./afl-lto ]; then set -e; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-lto; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-lto++; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang-lto; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang-lto++; install -m 755 ./afl-llvm-rt-lto*.o ./afl-llvm-lto-instrumentlist.so $${DESTDIR}$(HELPER_PATH); fi @if [ -f ./afl-ld-lto ]; then set -e; install -m 755 ./afl-ld-lto $${DESTDIR}$(BIN_PATH); fi @if [ -f ./afl-compiler-rt-32.o ]; then set -e; install -m 755 ./afl-compiler-rt-32.o $${DESTDIR}$(HELPER_PATH); ln -sf afl-compiler-rt-32.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt-32.o ;fi @if [ -f ./afl-compiler-rt-64.o ]; then set -e; install -m 755 ./afl-compiler-rt-64.o $${DESTDIR}$(HELPER_PATH); ln -sf afl-compiler-rt-64.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt-64.o ; fi diff --git a/QuickStartGuide.md b/QuickStartGuide.md deleted file mode 120000 index 8136d85e..00000000 --- a/QuickStartGuide.md +++ /dev/null @@ -1 +0,0 @@ -docs/QuickStartGuide.md \ No newline at end of file diff --git a/README.md b/README.md index 1f1fd3b2..21724696 100644 --- a/README.md +++ b/README.md @@ -1,1441 +1,246 @@ # American Fuzzy Lop plus plus (AFL++) - <img align="right" src="https://raw.githubusercontent.com/andreafioraldi/AFLplusplus-website/master/static/logo_256x256.png" alt="AFL++ Logo"> +<img align="right" src="https://raw.githubusercontent.com/andreafioraldi/AFLplusplus-website/master/static/logo_256x256.png" alt="AFL++ logo"> - Release Version: [3.14c](https://github.com/AFLplusplus/AFLplusplus/releases) +Release version: [3.14c](https://github.com/AFLplusplus/AFLplusplus/releases) - Github Version: 3.15a +GitHub version: 3.15a - Repository: [https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) - - AFL++ is maintained by: - - * Marc "van Hauser" Heuse <mh@mh-sec.de>, - * Heiko "hexcoder-" Eißfeldt <heiko.eissfeldt@hexco.de>, - * Andrea Fioraldi <andreafioraldi@gmail.com> and - * Dominik Maier <mail@dmnk.co>. - - Originally developed by Michał "lcamtuf" Zalewski. - - AFL++ is a superior fork to Google's AFL - more speed, more and better - mutations, more and better instrumentation, custom module support, etc. - - If you want to use AFL++ for your academic work, check the [papers page](https://aflplus.plus/papers/) - on the website. To cite our work, look at the [Cite](#cite) section. - For comparisons use the fuzzbench `aflplusplus` setup, or use `afl-clang-fast` - with `AFL_LLVM_CMPLOG=1`. - -## Major behaviour changes in AFL++ 3.00 onwards: - -With AFL++ 3.13-3.20 we introduce frida_mode (-O) to have an alternative for -binary-only fuzzing. It is slower than Qemu mode but works on MacOS, Android, -iOS etc. - -With AFL++ 3.15 we introduced the following changes from previous behaviours: - * Also -M main mode does not do deterministic fuzzing by default anymore - * afl-cmin and afl-showmap -Ci now descent into subdirectories like - afl-fuzz -i does (but note that afl-cmin.bash does not) - -With AFL++ 3.14 we introduced the following changes from previous behaviours: - * afl-fuzz: deterministic fuzzing it not a default for -M main anymore - * afl-cmin/afl-showmap -i now descends into subdirectories (afl-cmin.bash - however does not) - -With AFL++ 3.10 we introduced the following changes from previous behaviours: - * The '+' feature of the '-t' option now means to auto-calculate the timeout - with the value given being the maximum timeout. The original meaning of - "skipping timeouts instead of abort" is now inherent to the -t option. - -With AFL++ 3.00 we introduced changes that break some previous AFL and AFL++ -behaviours and defaults: - * There are no llvm_mode and gcc_plugin subdirectories anymore and there is - only one compiler: afl-cc. All previous compilers now symlink to this one. - All instrumentation source code is now in the `instrumentation/` folder. - * The gcc_plugin was replaced with a new version submitted by AdaCore that - supports more features. Thank you! - * qemu_mode got upgraded to QEMU 5.1, but to be able to build this a current - ninja build tool version and python3 setuptools are required. - qemu_mode also got new options like snapshotting, instrumenting specific - shared libraries, etc. Additionally QEMU 5.1 supports more CPU targets so - this is really worth it. - * When instrumenting targets, afl-cc will not supersede optimizations anymore - if any were given. This allows to fuzz targets build regularly like those - for debug or release versions. - * afl-fuzz: - * if neither -M or -S is specified, `-S default` is assumed, so more - fuzzers can easily be added later - * `-i` input directory option now descends into subdirectories. It also - does not fatal on crashes and too large files, instead it skips them - and uses them for splicing mutations - * -m none is now default, set memory limits (in MB) with e.g. -m 250 - * deterministic fuzzing is now disabled by default (unless using -M) and - can be enabled with -D - * a caching of testcases can now be performed and can be modified by - editing config.h for TESTCASE_CACHE or by specifying the env variable - `AFL_TESTCACHE_SIZE` (in MB). Good values are between 50-500 (default: 50). - * -M mains do not perform trimming - * examples/ got renamed to utils/ - * libtokencap/ libdislocator/ and qdbi_mode/ were moved to utils/ - * afl-cmin/afl-cmin.bash now search first in PATH and last in AFL_PATH - - -## Contents - - 1. [Features](#important-features-of-afl) - 2. [How to compile and install AFL++](#building-and-installing-afl) - 3. [How to fuzz a target](#how-to-fuzz-with-afl) - 4. [Fuzzing binary-only targets](#fuzzing-binary-only-targets) - 5. [Good examples and writeups of AFL++ usages](#good-examples-and-writeups) - 6. [CI Fuzzing](#ci-fuzzing) - 7. [Branches](#branches) - 8. [Want to help?](#help-wanted) - 9. [Detailed help and description of AFL++](#challenges-of-guided-fuzzing) - -## Important features of AFL++ - - AFL++ supports llvm from 3.8 up to version 13, very fast binary fuzzing with QEMU 5.1 - with laf-intel and redqueen, frida mode, unicorn mode, gcc plugin, full *BSD, - Mac OS, Solaris and Android support and much, much, much more. - - | Feature/Instrumentation | afl-gcc | llvm | gcc_plugin | frida_mode | qemu_mode |unicorn_mode | - | -------------------------|:-------:|:---------:|:----------:|:----------------:|:----------------:|:----------------:| - | Threadsafe counters | | x(3) | | | | | - | NeverZero | x86[_64]| x(1) | x | x | x | x | - | Persistent Mode | | x | x | x86[_64]/arm64 | x86[_64]/arm[64] | x | - | LAF-Intel / CompCov | | x | | | x86[_64]/arm[64] | x86[_64]/arm[64] | - | CmpLog | | x | | x86[_64]/arm64 | x86[_64]/arm[64] | | - | Selective Instrumentation| | x | x | x | x | | - | Non-Colliding Coverage | | x(4) | | | (x)(5) | | - | Ngram prev_loc Coverage | | x(6) | | | | | - | Context Coverage | | x(6) | | | | | - | Auto Dictionary | | x(7) | | | | | - | Snapshot LKM Support | | (x)(8) | (x)(8) | | (x)(5) | | - | Shared Memory Testcases | | x | x | x86[_64]/arm64 | x | x | - - 1. default for LLVM >= 9.0, env var for older version due an efficiency bug in previous llvm versions - 2. GCC creates non-performant code, hence it is disabled in gcc_plugin - 3. with `AFL_LLVM_THREADSAFE_INST`, disables NeverZero - 4. with pcguard mode and LTO mode for LLVM 11 and newer - 5. upcoming, development in the branch - 6. not compatible with LTO instrumentation and needs at least LLVM v4.1 - 7. automatic in LTO mode with LLVM 11 and newer, an extra pass for all LLVM versions that write to a file to use with afl-fuzz' `-x` - 8. the snapshot LKM is currently unmaintained due to too many kernel changes coming too fast :-( - - Among others, the following features and patches have been integrated: - - * NeverZero patch for afl-gcc, instrumentation, qemu_mode and unicorn_mode which prevents a wrapping map value to zero, increases coverage - * Persistent mode, deferred forkserver and in-memory fuzzing for qemu_mode - * Unicorn mode which allows fuzzing of binaries from completely different platforms (integration provided by domenukk) - * The new CmpLog instrumentation for LLVM and QEMU inspired by [Redqueen](https://www.syssec.ruhr-uni-bochum.de/media/emma/veroeffentlichungen/2018/12/17/NDSS19-Redqueen.pdf) - * Win32 PE binary-only fuzzing with QEMU and Wine - * AFLfast's power schedules by Marcel Böhme: [https://github.com/mboehme/aflfast](https://github.com/mboehme/aflfast) - * The MOpt mutator: [https://github.com/puppet-meteor/MOpt-AFL](https://github.com/puppet-meteor/MOpt-AFL) - * LLVM mode Ngram coverage by Adrian Herrera [https://github.com/adrianherrera/afl-ngram-pass](https://github.com/adrianherrera/afl-ngram-pass) - * LAF-Intel/CompCov support for instrumentation, qemu_mode and unicorn_mode (with enhanced capabilities) - * Radamsa and honggfuzz mutators (as custom mutators). - * QBDI mode to fuzz android native libraries via Quarkslab's [QBDI](https://github.com/QBDI/QBDI) framework - * Frida and ptrace mode to fuzz binary-only libraries, etc. - - So all in all this is the best-of AFL that is out there :-) - - For new versions and additional information, check out: - [https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) - - To compare notes with other users or get notified about major new features, - send a mail to <afl-users+subscribe@googlegroups.com>. - - See [docs/QuickStartGuide.md](docs/QuickStartGuide.md) if you don't have time to - read this file - however this is not recommended! - -## Branches +Repository: +[https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) - The following branches exist: +AFL++ is maintained by: - * [stable/trunk](https://github.com/AFLplusplus/AFLplusplus/) : stable state of AFL++ - it is synced from dev from time to - time when we are satisfied with its stability - * [dev](https://github.com/AFLplusplus/AFLplusplus/tree/dev) : development state of AFL++ - bleeding edge and you might catch a - checkout which does not compile or has a bug. *We only accept PRs in dev!!* - * [release](https://github.com/AFLplusplus/AFLplusplus/tree/release) : the latest release - * (any other) : experimental branches to work on specific features or testing - new functionality or changes. +* Marc "van Hauser" Heuse <mh@mh-sec.de>, +* Heiko "hexcoder-" Eißfeldt <heiko.eissfeldt@hexco.de>, +* Andrea Fioraldi <andreafioraldi@gmail.com> and +* Dominik Maier <mail@dmnk.co>. - For releases, please see the [Releases](https://github.com/AFLplusplus/AFLplusplus/releases) tab. +Originally developed by Michał "lcamtuf" Zalewski. -## Help wanted +AFL++ is a superior fork to Google's AFL - more speed, more and better +mutations, more and better instrumentation, custom module support, etc. -We have several ideas we would like to see in AFL++ to make it even better. -However, we already work on so many things that we do not have the time for -all the big ideas. +You are free to copy, modify, and distribute AFL++ with attribution under the +terms of the Apache-2.0 License. See the [LICENSE](LICENSE) for details. -This can be your way to support and contribute to AFL++ - extend it to do -something cool. +## Getting started -We have an idea list in [docs/ideas.md](docs/ideas.md). +Here is some information to get you started: -For everyone who wants to contribute (and send pull requests) please read -[CONTRIBUTING.md](CONTRIBUTING.md) before your submit. +* For releases, see the + [Releases tab](https://github.com/AFLplusplus/AFLplusplus/releases) and + [branches](#branches). Also take a look at the list of + [important changes in AFL++](docs/important_changes.md) and the list of + [features](docs/features.md). +* If you want to use AFL++ for your academic work, check the + [papers page](https://aflplus.plus/papers/) on the website. +* To cite our work, look at the [Cite](#cite) section. +* For comparisons, use the fuzzbench `aflplusplus` setup, or use + `afl-clang-fast` with `AFL_LLVM_CMPLOG=1`. You can find the `aflplusplus` + default configuration on Google's + [fuzzbench](https://github.com/google/fuzzbench/tree/master/fuzzers/aflplusplus). +* To get you started with tutorials, go to + [docs/tutorials.md](docs/tutorials.md). ## Building and installing AFL++ -An easy way to install AFL++ with everything compiled is available via docker: -You can use the [Dockerfile](Dockerfile) (which has gcc-10 and clang-11 - -hence afl-clang-lto is available!) or just pull directly from the docker hub: +To have AFL++ easily available with everything compiled, pull the image directly +from the Docker Hub: + ```shell docker pull aflplusplus/aflplusplus docker run -ti -v /location/of/your/target:/src aflplusplus/aflplusplus ``` -This image is automatically generated when a push to the stable repo happens. -You will find your target source code in /src in the container. - -If you want to build AFL++ yourself you have many options. -The easiest choice is to build and install everything: - -```shell -sudo apt-get update -sudo apt-get install -y build-essential python3-dev automake git flex bison libglib2.0-dev libpixman-1-dev python3-setuptools -# try to install llvm 11 and install the distro default if that fails -sudo apt-get install -y lld-11 llvm-11 llvm-11-dev clang-11 || sudo apt-get install -y lld llvm llvm-dev clang -sudo apt-get install -y gcc-$(gcc --version|head -n1|sed 's/.* //'|sed 's/\..*//')-plugin-dev libstdc++-$(gcc --version|head -n1|sed 's/.* //'|sed 's/\..*//')-dev -git clone https://github.com/AFLplusplus/AFLplusplus -cd AFLplusplus -make distrib -sudo make install -``` -It is recommended to install the newest available gcc, clang and llvm-dev -possible in your distribution! - -Note that "make distrib" also builds instrumentation, qemu_mode, unicorn_mode and -more. If you just want plain AFL++ then do "make all", however compiling and -using at least instrumentation is highly recommended for much better results - -hence in this case - -```shell -make source-only -``` -is what you should choose. - -These build targets exist: - -* all: just the main AFL++ binaries -* binary-only: everything for binary-only fuzzing: qemu_mode, unicorn_mode, libdislocator, libtokencap -* source-only: everything for source code fuzzing: instrumentation, libdislocator, libtokencap -* distrib: everything (for both binary-only and source code fuzzing) -* man: creates simple man pages from the help option of the programs -* install: installs everything you have compiled with the build options above -* clean: cleans everything compiled, not downloads (unless not on a checkout) -* deepclean: cleans everything including downloads -* code-format: format the code, do this before you commit and send a PR please! -* tests: runs test cases to ensure that all features are still working as they should -* unit: perform unit tests (based on cmocka) -* help: shows these build options - -[Unless you are on Mac OS X](https://developer.apple.com/library/archive/qa/qa1118/_index.html) you can also build statically linked versions of the -AFL++ binaries by passing the STATIC=1 argument to make: - -```shell -make STATIC=1 -``` - -These build options exist: - -* STATIC - compile AFL++ static -* ASAN_BUILD - compiles with memory sanitizer for debug purposes -* DEBUG - no optimization, -ggdb3, all warnings and -Werror -* PROFILING - compile with profiling information (gprof) -* INTROSPECTION - compile afl-fuzz with mutation introspection -* NO_PYTHON - disable python support -* NO_SPLICING - disables splicing mutation in afl-fuzz, not recommended for normal fuzzing -* AFL_NO_X86 - if compiling on non-intel/amd platforms -* LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config (e.g. Debian) - -e.g.: `make ASAN_BUILD=1` - -## Good examples and writeups - -Here are some good writeups to show how to effectively use AFL++: - - * [https://aflplus.plus/docs/tutorials/libxml2_tutorial/](https://aflplus.plus/docs/tutorials/libxml2_tutorial/) - * [https://bananamafia.dev/post/gb-fuzz/](https://bananamafia.dev/post/gb-fuzz/) - * [https://securitylab.github.com/research/fuzzing-challenges-solutions-1](https://securitylab.github.com/research/fuzzing-challenges-solutions-1) - * [https://securitylab.github.com/research/fuzzing-software-2](https://securitylab.github.com/research/fuzzing-software-2) - * [https://securitylab.github.com/research/fuzzing-sockets-FTP](https://securitylab.github.com/research/fuzzing-sockets-FTP) - * [https://securitylab.github.com/research/fuzzing-sockets-FreeRDP](https://securitylab.github.com/research/fuzzing-sockets-FreeRDP) - * [https://securitylab.github.com/research/fuzzing-apache-1](https://securitylab.github.com/research/fuzzing-apache-1) - -If you do not want to follow a tutorial but rather try an exercise type of -training then we can highly recommend the following: - * [https://github.com/antonio-morales/Fuzzing101](https://github.com/antonio-morales/Fuzzing101) - -If you are interested in fuzzing structured data (where you define what the -structure is), these links have you covered: - * Superion for AFL++: [https://github.com/adrian-rt/superion-mutator](https://github.com/adrian-rt/superion-mutator) - * libprotobuf for AFL++: [https://github.com/P1umer/AFLplusplus-protobuf-mutator](https://github.com/P1umer/AFLplusplus-protobuf-mutator) - * libprotobuf raw: [https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator](https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator) - * libprotobuf for old AFL++ API: [https://github.com/thebabush/afl-libprotobuf-mutator](https://github.com/thebabush/afl-libprotobuf-mutator) - -If you find other good ones, please send them to us :-) - -## How to fuzz with AFL++ - -The following describes how to fuzz with a target if source code is available. -If you have a binary-only target please skip to [#Instrumenting binary-only apps](#Instrumenting binary-only apps) - -Fuzzing source code is a three-step process. - -1. Compile the target with a special compiler that prepares the target to be - fuzzed efficiently. This step is called "instrumenting a target". -2. Prepare the fuzzing by selecting and optimizing the input corpus for the - target. -3. Perform the fuzzing of the target by randomly mutating input and assessing - if a generated input was processed in a new path in the target binary. - -### 1. Instrumenting that target - -#### a) Selecting the best AFL++ compiler for instrumenting the target - -AFL++ comes with a central compiler `afl-cc` that incorporates various different -kinds of compiler targets and and instrumentation options. -The following evaluation flow will help you to select the best possible. - -It is highly recommended to have the newest llvm version possible installed, -anything below 9 is not recommended. - -``` -+--------------------------------+ -| clang/clang++ 11+ is available | --> use LTO mode (afl-clang-lto/afl-clang-lto++) -+--------------------------------+ see [instrumentation/README.lto.md](instrumentation/README.lto.md) - | - | if not, or if the target fails with LTO afl-clang-lto/++ - | - v -+---------------------------------+ -| clang/clang++ 3.8+ is available | --> use LLVM mode (afl-clang-fast/afl-clang-fast++) -+---------------------------------+ see [instrumentation/README.llvm.md](instrumentation/README.llvm.md) - | - | if not, or if the target fails with LLVM afl-clang-fast/++ - | - v - +--------------------------------+ - | gcc 5+ is available | -> use GCC_PLUGIN mode (afl-gcc-fast/afl-g++-fast) - +--------------------------------+ see [instrumentation/README.gcc_plugin.md](instrumentation/README.gcc_plugin.md) and - [instrumentation/README.instrument_list.md](instrumentation/README.instrument_list.md) - | - | if not, or if you do not have a gcc with plugin support - | - v - use GCC mode (afl-gcc/afl-g++) (or afl-clang/afl-clang++ for clang) -``` - -Clickable README links for the chosen compiler: - - * [LTO mode - afl-clang-lto](instrumentation/README.lto.md) - * [LLVM mode - afl-clang-fast](instrumentation/README.llvm.md) - * [GCC_PLUGIN mode - afl-gcc-fast](instrumentation/README.gcc_plugin.md) - * GCC/CLANG modes (afl-gcc/afl-clang) have no README as they have no own features - -You can select the mode for the afl-cc compiler by: - 1. use a symlink to afl-cc: afl-gcc, afl-g++, afl-clang, afl-clang++, - afl-clang-fast, afl-clang-fast++, afl-clang-lto, afl-clang-lto++, - afl-gcc-fast, afl-g++-fast (recommended!) - 2. using the environment variable AFL_CC_COMPILER with MODE - 3. passing --afl-MODE command line options to the compiler via CFLAGS/CXXFLAGS/CPPFLAGS - -MODE can be one of: LTO (afl-clang-lto*), LLVM (afl-clang-fast*), GCC_PLUGIN -(afl-g*-fast) or GCC (afl-gcc/afl-g++) or CLANG(afl-clang/afl-clang++). - -Because no AFL specific command-line options are accepted (beside the ---afl-MODE command), the compile-time tools make fairly broad use of environment -variables, which can be listed with `afl-cc -hh` or by reading [docs/env_variables.md](docs/env_variables.md). - -#### b) Selecting instrumentation options - -The following options are available when you instrument with LTO mode (afl-clang-fast/afl-clang-lto): - - * Splitting integer, string, float and switch comparisons so AFL++ can easier - solve these. This is an important option if you do not have a very good - and large input corpus. This technique is called laf-intel or COMPCOV. - To use this set the following environment variable before compiling the - target: `export AFL_LLVM_LAF_ALL=1` - You can read more about this in [instrumentation/README.laf-intel.md](instrumentation/README.laf-intel.md) - * A different technique (and usually a better one than laf-intel) is to - instrument the target so that any compare values in the target are sent to - AFL++ which then tries to put these values into the fuzzing data at different - locations. This technique is very fast and good - if the target does not - transform input data before comparison. Therefore this technique is called - `input to state` or `redqueen`. - If you want to use this technique, then you have to compile the target - twice, once specifically with/for this mode, and pass this binary to afl-fuzz - via the `-c` parameter. - Note that you can compile also just a cmplog binary and use that for both - however there will be a performance penality. - You can read more about this in [instrumentation/README.cmplog.md](instrumentation/README.cmplog.md) - -If you use LTO, LLVM or GCC_PLUGIN mode (afl-clang-fast/afl-clang-lto/afl-gcc-fast) -you have the option to selectively only instrument parts of the target that you -are interested in: - - * To instrument only those parts of the target that you are interested in - create a file with all the filenames of the source code that should be - instrumented. - For afl-clang-lto and afl-gcc-fast - or afl-clang-fast if a mode other than - DEFAULT/PCGUARD is used or you have llvm > 10.0.0 - just put one - filename or function per line (no directory information necessary for - filenames9, and either set `export AFL_LLVM_ALLOWLIST=allowlist.txt` **or** - `export AFL_LLVM_DENYLIST=denylist.txt` - depending on if you want per - default to instrument unless noted (DENYLIST) or not perform instrumentation - unless requested (ALLOWLIST). - **NOTE:** During optimization functions might be inlined and then would not match! - See [instrumentation/README.instrument_list.md](instrumentation/README.instrument_list.md) - -There are many more options and modes available however these are most of the -time less effective. See: - * [instrumentation/README.ctx.md](instrumentation/README.ctx.md) - * [instrumentation/README.ngram.md](instrumentation/README.ngram.md) - -AFL++ performs "never zero" counting in its bitmap. You can read more about this -here: - * [instrumentation/README.neverzero.md](instrumentation/README.neverzero.md) - -#### c) Sanitizers - -It is possible to use sanitizers when instrumenting targets for fuzzing, -which allows you to find bugs that would not necessarily result in a crash. - -Note that sanitizers have a huge impact on CPU (= less executions per second) -and RAM usage. Also you should only run one afl-fuzz instance per sanitizer type. -This is enough because a use-after-free bug will be picked up, e.g. by -ASAN (address sanitizer) anyway when syncing to other fuzzing instances, -so not all fuzzing instances need to be instrumented with ASAN. - -The following sanitizers have built-in support in AFL++: - * ASAN = Address SANitizer, finds memory corruption vulnerabilities like - use-after-free, NULL pointer dereference, buffer overruns, etc. - Enabled with `export AFL_USE_ASAN=1` before compiling. - * MSAN = Memory SANitizer, finds read access to uninitialized memory, eg. - a local variable that is defined and read before it is even set. - Enabled with `export AFL_USE_MSAN=1` before compiling. - * UBSAN = Undefined Behaviour SANitizer, finds instances where - by the - C and C++ standards - undefined behaviour happens, e.g. adding two - signed integers together where the result is larger than a signed integer - can hold. - Enabled with `export AFL_USE_UBSAN=1` before compiling. - * CFISAN = Control Flow Integrity SANitizer, finds instances where the - control flow is found to be illegal. Originally this was rather to - prevent return oriented programming exploit chains from functioning, - in fuzzing this is mostly reduced to detecting type confusion - vulnerabilities - which is however one of the most important and dangerous - C++ memory corruption classes! - Enabled with `export AFL_USE_CFISAN=1` before compiling. - * LSAN = Leak SANitizer, finds memory leaks in a program. This is not really - a security issue, but for developers this can be very valuable. - Note that unlike the other sanitizers above this needs - `__AFL_LEAK_CHECK();` added to all areas of the target source code where you - find a leak check necessary! - Enabled with `export AFL_USE_LSAN=1` before compiling. - -It is possible to further modify the behaviour of the sanitizers at run-time -by setting `ASAN_OPTIONS=...`, `LSAN_OPTIONS` etc. - the available parameters -can be looked up in the sanitizer documentation of llvm/clang. -afl-fuzz however requires some specific parameters important for fuzzing to be -set. If you want to set your own, it might bail and report what it is missing. - -Note that some sanitizers cannot be used together, e.g. ASAN and MSAN, and -others often cannot work together because of target weirdness, e.g. ASAN and -CFISAN. You might need to experiment which sanitizers you can combine in a -target (which means more instances can be run without a sanitized target, -which is more effective). - -#### d) Modify the target - -If the target has features that make fuzzing more difficult, e.g. -checksums, HMAC, etc. then modify the source code so that checks for these -values are removed. -This can even be done safely for source code used in operational products -by eliminating these checks within these AFL specific blocks: - -``` -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - // say that the checksum or HMAC was fine - or whatever is required - // to eliminate the need for the fuzzer to guess the right checksum - return 0; -#endif -``` - -All AFL++ compilers will set this preprocessor definition automatically. - -#### e) Instrument the target - -In this step the target source code is compiled so that it can be fuzzed. - -Basically you have to tell the target build system that the selected AFL++ -compiler is used. Also - if possible - you should always configure the -build system such that the target is compiled statically and not dynamically. -How to do this is described below. - -The #1 rule when instrumenting a target is: avoid instrumenting shared -libraries at all cost. You would need to set LD_LIBRARY_PATH to point to -these, you could accidently type "make install" and install them system wide - -so don't. Really don't. -**Always compile libraries you want to have instrumented as static and link -these to the target program!** - -Then build the target. (Usually with `make`) - -**NOTES** - -1. sometimes configure and build systems are fickle and do not like - stderr output (and think this means a test failure) - which is something - AFL++ likes to do to show statistics. It is recommended to disable AFL++ - instrumentation reporting via `export AFL_QUIET=1`. - -2. sometimes configure and build systems error on warnings - these should be - disabled (e.g. `--disable-werror` for some configure scripts). - -3. in case the configure/build system complains about AFL++'s compiler and - aborts then set `export AFL_NOOPT=1` which will then just behave like the - real compiler. This option has to be unset again before building the target! - -##### configure - -For `configure` build systems this is usually done by: -`CC=afl-clang-fast CXX=afl-clang-fast++ ./configure --disable-shared` - -Note that if you are using the (better) afl-clang-lto compiler you also have to -set AR to llvm-ar[-VERSION] and RANLIB to llvm-ranlib[-VERSION] - as is -described in [instrumentation/README.lto.md](instrumentation/README.lto.md). - -##### cmake - -For `cmake` build systems this is usually done by: -`mkdir build; cd build; cmake -DCMAKE_C_COMPILER=afl-cc -DCMAKE_CXX_COMPILER=afl-c++ ..` - -Note that if you are using the (better) afl-clang-lto compiler you also have to -set AR to llvm-ar[-VERSION] and RANLIB to llvm-ranlib[-VERSION] - as is -described in [instrumentation/README.lto.md](instrumentation/README.lto.md). - -##### meson - -For meson you have to set the AFL++ compiler with the very first command! -`CC=afl-cc CXX=afl-c++ meson` - -##### other build systems or if configure/cmake didn't work - -Sometimes cmake and configure do not pick up the AFL++ compiler, or the -ranlib/ar that is needed - because this was just not foreseen by the developer -of the target. Or they have non-standard options. Figure out if there is a -non-standard way to set this, otherwise set up the build normally and edit the -generated build environment afterwards manually to point it to the right compiler -(and/or ranlib and ar). - -#### f) Better instrumentation - -If you just fuzz a target program as-is you are wasting a great opportunity for -much more fuzzing speed. - -This variant requires the usage of afl-clang-lto, afl-clang-fast or afl-gcc-fast. - -It is the so-called `persistent mode`, which is much, much faster but -requires that you code a source file that is specifically calling the target -functions that you want to fuzz, plus a few specific AFL++ functions around -it. See [instrumentation/README.persistent_mode.md](instrumentation/README.persistent_mode.md) for details. - -Basically if you do not fuzz a target in persistent mode then you are just -doing it for a hobby and not professionally :-). - -#### g) libfuzzer fuzzer harnesses with LLVMFuzzerTestOneInput() - -libfuzzer `LLVMFuzzerTestOneInput()` harnesses are the defacto standard -for fuzzing, and they can be used with AFL++ (and honggfuzz) as well! -Compiling them is as simple as: -``` -afl-clang-fast++ -fsanitize=fuzzer -o harness harness.cpp targetlib.a -``` -You can even use advanced libfuzzer features like `FuzzedDataProvider`, -`LLVMFuzzerMutate()` etc. and they will work! - -The generated binary is fuzzed with afl-fuzz like any other fuzz target. - -Bonus: the target is already optimized for fuzzing due to persistent mode and -shared-memory testcases and hence gives you the fastest speed possible. - -For more information see [utils/aflpp_driver/README.md](utils/aflpp_driver/README.md) - -### 2. Preparing the fuzzing campaign - -As you fuzz the target with mutated input, having as diverse inputs for the -target as possible improves the efficiency a lot. - -#### a) Collect inputs - -Try to gather valid inputs for the target from wherever you can. E.g. if it is -the PNG picture format try to find as many png files as possible, e.g. from -reported bugs, test suites, random downloads from the internet, unit test -case data - from all kind of PNG software. - -If the input format is not known, you can also modify a target program to write -normal data it receives and processes to a file and use these. - -#### b) Making the input corpus unique - -Use the AFL++ tool `afl-cmin` to remove inputs from the corpus that do not -produce a new path in the target. - -Put all files from step a) into one directory, e.g. INPUTS. - -If the target program is to be called by fuzzing as `bin/target -d INPUTFILE` -the run afl-cmin like this: -`afl-cmin -i INPUTS -o INPUTS_UNIQUE -- bin/target -d @@` -Note that the INPUTFILE argument that the target program would read from has to be set as `@@`. - -If the target reads from stdin instead, just omit the `@@` as this is the -default. - -This step is highly recommended! - -#### c) Minimizing all corpus files - -The shorter the input files that still traverse the same path -within the target, the better the fuzzing will be. This minimization -is done with `afl-tmin` however it is a long process as this has to -be done for every file: - -``` -mkdir input -cd INPUTS_UNIQUE -for i in *; do - afl-tmin -i "$i" -o "../input/$i" -- bin/target -d @@ -done -``` - -This step can also be parallelized, e.g. with `parallel`. -Note that this step is rather optional though. - -#### Done! - -The INPUTS_UNIQUE/ directory from step b) - or even better the directory input/ -if you minimized the corpus in step c) - is the resulting input corpus directory -to be used in fuzzing! :-) - -### 3. Fuzzing the target - -In this final step we fuzz the target. -There are not that many important options to run the target - unless you want -to use many CPU cores/threads for the fuzzing, which will make the fuzzing much -more useful. - -If you just use one CPU for fuzzing, then you are fuzzing just for fun and not -seriously :-) - -#### a) Running afl-fuzz - -Before you do even a test run of afl-fuzz execute `sudo afl-system-config` (on -the host if you execute afl-fuzz in a docker container). This reconfigures the -system for optimal speed - which afl-fuzz checks and bails otherwise. -Set `export AFL_SKIP_CPUFREQ=1` for afl-fuzz to skip this check if you cannot -run afl-system-config with root privileges on the host for whatever reason. - -Note there is also `sudo afl-persistent-config` which sets additional permanent -boot options for a much better fuzzing performance. - -Note that both scripts improve your fuzzing performance but also decrease your -system protection against attacks! So set strong firewall rules and only -expose SSH as a network service if you use these (which is highly recommended). - -If you have an input corpus from step 2 then specify this directory with the `-i` -option. Otherwise create a new directory and create a file with any content -as test data in there. - -If you do not want anything special, the defaults are already usually best, -hence all you need is to specify the seed input directory with the result of -step [2a. Collect inputs](#a-collect-inputs): -`afl-fuzz -i input -o output -- bin/target -d @@` -Note that the directory specified with -o will be created if it does not exist. - -It can be valuable to run afl-fuzz in a screen or tmux shell so you can log off, -or afl-fuzz is not aborted if you are running it in a remote ssh session where -the connection fails in between. -Only do that though once you have verified that your fuzzing setup works! -Simply run it like `screen -dmS afl-main -- afl-fuzz -M main-$HOSTNAME -i ...` -and it will start away in a screen session. To enter this session simply type -`screen -r afl-main`. You see - it makes sense to name the screen session -same as the afl-fuzz -M/-S naming :-) -For more information on screen or tmux please check their documentation. - -If you need to stop and re-start the fuzzing, use the same command line options -(or even change them by selecting a different power schedule or another -mutation mode!) and switch the input directory with a dash (`-`): -`afl-fuzz -i - -o output -- bin/target -d @@` - -Memory limits are not enforced by afl-fuzz by default and the system may run -out of memory. You can decrease the memory with the `-m` option, the value is -in MB. If this is too small for the target, you can usually see this by -afl-fuzz bailing with the message that it could not connect to the forkserver. - -Adding a dictionary is helpful. See the directory [dictionaries/](dictionaries/) if -something is already included for your data format, and tell afl-fuzz to load -that dictionary by adding `-x dictionaries/FORMAT.dict`. With afl-clang-lto -you have an autodictionary generation for which you need to do nothing except -to use afl-clang-lto as the compiler. You also have the option to generate -a dictionary yourself, see [utils/libtokencap/README.md](utils/libtokencap/README.md). - -afl-fuzz has a variety of options that help to workaround target quirks like -specific locations for the input file (`-f`), performing deterministic -fuzzing (`-D`) and many more. Check out `afl-fuzz -h`. - -We highly recommend that you set a memory limit for running the target with `-m` -which defines the maximum memory in MB. This prevents a potential -out-of-memory problem for your system plus helps you detect missing `malloc()` -failure handling in the target. -Play around with various -m values until you find one that safely works for all -your input seeds (if you have good ones and then double or quadrouple that. - -By default afl-fuzz never stops fuzzing. To terminate AFL++ simply press Control-C -or send a signal SIGINT. You can limit the number of executions or approximate runtime -in seconds with options also. - -When you start afl-fuzz you will see a user interface that shows what the status -is: - - -All labels are explained in [docs/status_screen.md](docs/status_screen.md). - -#### b) Using multiple cores - -If you want to seriously fuzz then use as many cores/threads as possible to -fuzz your target. - -On the same machine - due to the design of how AFL++ works - there is a maximum -number of CPU cores/threads that are useful, use more and the overall performance -degrades instead. This value depends on the target, and the limit is between 32 -and 64 cores per machine. - -If you have the RAM, it is highly recommended run the instances with a caching -of the testcases. Depending on the average testcase size (and those found -during fuzzing) and their number, a value between 50-500MB is recommended. -You can set the cache size (in MB) by setting the environment variable `AFL_TESTCACHE_SIZE`. - -There should be one main fuzzer (`-M main-$HOSTNAME` option) and as many secondary -fuzzers (eg `-S variant1`) as you have cores that you use. -Every -M/-S entry needs a unique name (that can be whatever), however the same --o output directory location has to be used for all instances. - -For every secondary fuzzer there should be a variation, e.g.: - * one should fuzz the target that was compiled differently: with sanitizers - activated (`export AFL_USE_ASAN=1 ; export AFL_USE_UBSAN=1 ; - export AFL_USE_CFISAN=1`) - * one or two should fuzz the target with CMPLOG/redqueen (see above), at - least one cmplog instance should follow transformations (`-l AT`) - * one to three fuzzers should fuzz a target compiled with laf-intel/COMPCOV - (see above). Important note: If you run more than one laf-intel/COMPCOV - fuzzer and you want them to share their intermediate results, the main - fuzzer (`-M`) must be one of the them! (Although this is not really - recommended.) - -All other secondaries should be used like this: - * A quarter to a third with the MOpt mutator enabled: `-L 0` - * run with a different power schedule, recommended are: - `fast (default), explore, coe, lin, quad, exploit and rare` - which you can set with e.g. `-p explore` - * a few instances should use the old queue cycling with `-Z` - -Also it is recommended to set `export AFL_IMPORT_FIRST=1` to load testcases -from other fuzzers in the campaign first. - -If you have a large corpus, a corpus from a previous run or are fuzzing in -a CI, then also set `export AFL_CMPLOG_ONLY_NEW=1` and `export AFL_FAST_CAL=1`. - -You can also use different fuzzers. -If you are using AFL spinoffs or AFL conforming fuzzers, then just use the -same -o directory and give it a unique `-S` name. -Examples are: - * [Fuzzolic](https://github.com/season-lab/fuzzolic) - * [symcc](https://github.com/eurecom-s/symcc/) - * [Eclipser](https://github.com/SoftSec-KAIST/Eclipser/) - * [AFLsmart](https://github.com/aflsmart/aflsmart) - * [FairFuzz](https://github.com/carolemieux/afl-rb) - * [Neuzz](https://github.com/Dongdongshe/neuzz) - * [Angora](https://github.com/AngoraFuzzer/Angora) - -A long list can be found at [https://github.com/Microsvuln/Awesome-AFL](https://github.com/Microsvuln/Awesome-AFL) - -However you can also sync AFL++ with honggfuzz, libfuzzer with `-entropic=1`, etc. -Just show the main fuzzer (-M) with the `-F` option where the queue/work -directory of a different fuzzer is, e.g. `-F /src/target/honggfuzz`. -Using honggfuzz (with `-n 1` or `-n 2`) and libfuzzer in parallel is highly -recommended! - -#### c) Using multiple machines for fuzzing - -Maybe you have more than one machine you want to fuzz the same target on. -Simply start the `afl-fuzz` (and perhaps libfuzzer, honggfuzz, ...) -orchestra as you like, just ensure that your have one and only one `-M` -instance per server, and that its name is unique, hence the recommendation -for `-M main-$HOSTNAME`. - -Now there are three strategies on how you can sync between the servers: - * never: sounds weird, but this makes every server an island and has the - chance the each follow different paths into the target. You can make - this even more interesting by even giving different seeds to each server. - * regularly (~4h): this ensures that all fuzzing campaigns on the servers - "see" the same thing. It is like fuzzing on a huge server. - * in intervals of 1/10th of the overall expected runtime of the fuzzing you - sync. This tries a bit to combine both. have some individuality of the - paths each campaign on a server explores, on the other hand if one - gets stuck where another found progress this is handed over making it - unstuck. - -The syncing process itself is very simple. -As the `-M main-$HOSTNAME` instance syncs to all `-S` secondaries as well -as to other fuzzers, you have to copy only this directory to the other -machines. - -Lets say all servers have the `-o out` directory in /target/foo/out, and -you created a file `servers.txt` which contains the hostnames of all -participating servers, plus you have an ssh key deployed to all of them, -then run: -```bash -for FROM in `cat servers.txt`; do - for TO in `cat servers.txt`; do - rsync -rlpogtz --rsh=ssh $FROM:/target/foo/out/main-$FROM $TO:target/foo/out/ - done -done -``` -You can run this manually, per cron job - as you need it. -There is a more complex and configurable script in `utils/distributed_fuzzing`. - -#### d) The status of the fuzz campaign - -AFL++ comes with the `afl-whatsup` script to show the status of the fuzzing -campaign. - -Just supply the directory that afl-fuzz is given with the -o option and -you will see a detailed status of every fuzzer in that campaign plus -a summary. - -To have only the summary use the `-s` switch e.g.: `afl-whatsup -s out/` - -If you have multiple servers then use the command after a sync, or you have -to execute this script per server. - -#### e) Stopping fuzzing, restarting fuzzing, adding new seeds - -To stop an afl-fuzz run, simply press Control-C. - -To restart an afl-fuzz run, just reuse the same command line but replace the -`-i directory` with `-i -` or set `AFL_AUTORESUME=1`. - -If you want to add new seeds to a fuzzing campaign you can run a temporary -fuzzing instance, e.g. when your main fuzzer is using `-o out` and the new -seeds are in `newseeds/` directory: -``` -AFL_BENCH_JUST_ONE=1 AFL_FAST_CAL=1 afl-fuzz -i newseeds -o out -S newseeds -- ./target -``` - -#### f) Checking the coverage of the fuzzing - -The `paths found` value is a bad indicator for checking how good the coverage is. - -A better indicator - if you use default llvm instrumentation with at least -version 9 - is to use `afl-showmap` with the collect coverage option `-C` on -the output directory: -``` -$ afl-showmap -C -i out -o /dev/null -- ./target -params @@ -... -[*] Using SHARED MEMORY FUZZING feature. -[*] Target map size: 9960 -[+] Processed 7849 input files. -[+] Captured 4331 tuples (highest value 255, total values 67130596) in '/dev/nul -l'. -[+] A coverage of 4331 edges were achieved out of 9960 existing (43.48%) with 7849 input files. -``` -It is even better to check out the exact lines of code that have been reached - -and which have not been found so far. - -An "easy" helper script for this is [https://github.com/vanhauser-thc/afl-cov](https://github.com/vanhauser-thc/afl-cov), -just follow the README of that separate project. - -If you see that an important area or a feature has not been covered so far then -try to find an input that is able to reach that and start a new secondary in -that fuzzing campaign with that seed as input, let it run for a few minutes, -then terminate it. The main node will pick it up and make it available to the -other secondary nodes over time. Set `export AFL_NO_AFFINITY=1` or -`export AFL_TRY_AFFINITY=1` if you have no free core. - -Note that in nearly all cases you can never reach full coverage. A lot of -functionality is usually dependent on exclusive options that would need individual -fuzzing campaigns each with one of these options set. E.g. if you fuzz a library to -convert image formats and your target is the png to tiff API then you will not -touch any of the other library APIs and features. - -#### g) How long to fuzz a target? - -This is a difficult question. -Basically if no new path is found for a long time (e.g. for a day or a week) -then you can expect that your fuzzing won't be fruitful anymore. -However often this just means that you should switch out secondaries for -others, e.g. custom mutator modules, sync to very different fuzzers, etc. - -Keep the queue/ directory (for future fuzzings of the same or similar targets) -and use them to seed other good fuzzers like libfuzzer with the -entropic -switch or honggfuzz. - -#### h) Improve the speed! - - * Use [persistent mode](instrumentation/README.persistent_mode.md) (x2-x20 speed increase) - * If you do not use shmem persistent mode, use `AFL_TMPDIR` to point the input file on a tempfs location, see [docs/env_variables.md](docs/env_variables.md) - * Linux: Improve kernel performance: modify `/etc/default/grub`, set `GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"`; then `update-grub` and `reboot` (warning: makes the system more insecure) - you can also just run `sudo afl-persistent-config` - * Linux: Running on an `ext2` filesystem with `noatime` mount option will be a bit faster than on any other journaling filesystem - * Use your cores! [3.b) Using multiple cores/threads](#b-using-multiple-coresthreads) - * Run `sudo afl-system-config` before starting the first afl-fuzz instance after a reboot - -### The End - -Check out the [docs/FAQ](docs/FAQ.md) if it maybe answers your question (that -you might not even have known you had ;-) ). - -This is basically all you need to know to professionally run fuzzing campaigns. -If you want to know more, the rest of this README and the tons of texts in -[docs/](docs/) will have you covered. - -Note that there are also a lot of tools out there that help fuzzing with AFL++ -(some might be deprecated or unsupported): - -Speeding up fuzzing: - * [libfiowrapper](https://github.com/marekzmyslowski/libfiowrapper) - if the function you want to fuzz requires loading a file, this allows using the shared memory testcase feature :-) - recommended. - -Minimization of test cases: - * [afl-pytmin](https://github.com/ilsani/afl-pytmin) - a wrapper for afl-tmin that tries to speed up the process of minimization of a single test case by using many CPU cores. - * [afl-ddmin-mod](https://github.com/MarkusTeufelberger/afl-ddmin-mod) - a variation of afl-tmin based on the ddmin algorithm. - * [halfempty](https://github.com/googleprojectzero/halfempty) - is a fast utility for minimizing test cases by Tavis Ormandy based on parallelization. - -Distributed execution: - * [disfuzz-afl](https://github.com/MartijnB/disfuzz-afl) - distributed fuzzing for AFL. - * [AFLDFF](https://github.com/quantumvm/AFLDFF) - AFL distributed fuzzing framework. - * [afl-launch](https://github.com/bnagy/afl-launch) - a tool for the execution of many AFL instances. - * [afl-mothership](https://github.com/afl-mothership/afl-mothership) - management and execution of many synchronized AFL fuzzers on AWS cloud. - * [afl-in-the-cloud](https://github.com/abhisek/afl-in-the-cloud) - another script for running AFL in AWS. - -Deployment, management, monitoring, reporting - * [afl-utils](https://gitlab.com/rc0r/afl-utils) - a set of utilities for automatic processing/analysis of crashes and reducing the number of test cases. - * [afl-other-arch](https://github.com/shellphish/afl-other-arch) - is a set of patches and scripts for easily adding support for various non-x86 architectures for AFL. - * [afl-trivia](https://github.com/bnagy/afl-trivia) - a few small scripts to simplify the management of AFL. - * [afl-monitor](https://github.com/reflare/afl-monitor) - a script for monitoring AFL. - * [afl-manager](https://github.com/zx1340/afl-manager) - a web server on Python for managing multi-afl. - * [afl-remote](https://github.com/block8437/afl-remote) - a web server for the remote management of AFL instances. - * [afl-extras](https://github.com/fekir/afl-extras) - shell scripts to parallelize afl-tmin, startup, and data collection. - -Crash processing - * [afl-crash-analyzer](https://github.com/floyd-fuh/afl-crash-analyzer) - another crash analyzer for AFL. - * [fuzzer-utils](https://github.com/ThePatrickStar/fuzzer-utils) - a set of scripts for the analysis of results. - * [atriage](https://github.com/Ayrx/atriage) - a simple triage tool. - * [afl-kit](https://github.com/kcwu/afl-kit) - afl-cmin on Python. - * [AFLize](https://github.com/d33tah/aflize) - a tool that automatically generates builds of debian packages suitable for AFL. - * [afl-fid](https://github.com/FoRTE-Research/afl-fid) - a set of tools for working with input data. - -## CI Fuzzing - -Some notes on CI Fuzzing - this fuzzing is different to normal fuzzing -campaigns as these are much shorter runnings. - -1. Always: - * LTO has a much longer compile time which is diametrical to short fuzzing - - hence use afl-clang-fast instead. - * If you compile with CMPLOG then you can save fuzzing time and reuse that - compiled target for both the -c option and the main fuzz target. - This will impact the speed by ~15% though. - * `AFL_FAST_CAL` - Enable fast calibration, this halfs the time the saturated - corpus needs to be loaded. - * `AFL_CMPLOG_ONLY_NEW` - only perform cmplog on new found paths, not the - initial corpus as this very likely has been done for them already. - * Keep the generated corpus, use afl-cmin and reuse it every time! - -2. Additionally randomize the AFL++ compilation options, e.g. - * 40% for `AFL_LLVM_CMPLOG` - * 10% for `AFL_LLVM_LAF_ALL` - -3. Also randomize the afl-fuzz runtime options, e.g. - * 65% for `AFL_DISABLE_TRIM` - * 50% use a dictionary generated by `AFL_LLVM_DICT2FILE` - * 40% use MOpt (`-L 0`) - * 40% for `AFL_EXPAND_HAVOC_NOW` - * 20% for old queue processing (`-Z`) - * for CMPLOG targets, 60% for `-l 2`, 40% for `-l 3` - -4. Do *not* run any `-M` modes, just running `-S` modes is better for CI fuzzing. - `-M` enables old queue handling etc. which is good for a fuzzing campaign but - not good for short CI runs. - -How this can look like can e.g. be seen at AFL++'s setup in Google's [oss-fuzz](https://github.com/google/oss-fuzz/blob/master/infra/base-images/base-builder/compile_afl) -and [clusterfuzz](https://github.com/google/clusterfuzz/blob/master/src/python/bot/fuzzers/afl/launcher.py). - -## Fuzzing binary-only targets - -When source code is *NOT* available, AFL++ offers various support for fast, -on-the-fly instrumentation of black-box binaries. - -If you do not have to use Unicorn the following setup is recommended to use -qemu_mode: - * run 1 afl-fuzz -Q instance with CMPLOG (`-c 0` + `AFL_COMPCOV_LEVEL=2`) - * run 1 afl-fuzz -Q instance with QASAN (`AFL_USE_QASAN=1`) - * run 1 afl-fuzz -Q instance with LAF (`AFL_PRELOAD=libcmpcov.so` + `AFL_COMPCOV_LEVEL=2`) -Alternatively you can use frida_mode, just switch `-Q` with `-O` and remove the -LAF instance. - -Then run as many instances as you have cores left with either -Q mode or - better - -use a binary rewriter like afl-dyninst, retrowrite, zafl, etc. - -For Qemu and Frida mode, check out the persistent mode, it gives a huge speed -improvement if it is possible to use. - -### QEMU - -For linux programs and its libraries this is accomplished with a version of -QEMU running in the lesser-known "user space emulation" mode. -QEMU is a project separate from AFL, but you can conveniently build the -feature by doing: -```shell -cd qemu_mode -./build_qemu_support.sh -``` -For additional instructions and caveats, see [qemu_mode/README.md](qemu_mode/README.md). -If possible you should use the persistent mode, see [qemu_mode/README.persistent.md](qemu_mode/README.persistent.md). -The mode is approximately 2-5x slower than compile-time instrumentation, and is -less conducive to parallelization. - -If [afl-dyninst](https://github.com/vanhauser-thc/afl-dyninst) works for -your binary, then you can use afl-fuzz normally and it will have twice -the speed compared to qemu_mode (but slower than qemu persistent mode). -Note that several other binary rewriters exist, all with their advantages and -caveats. - -### Frida - -Frida mode is sometimes faster and sometimes slower than Qemu mode. -It is also newer, lacks COMPCOV, but supports MacOS. - -```shell -cd frida_mode -make -``` -For additional instructions and caveats, see [frida_mode/README.md](frida_mode/README.md). -If possible you should use the persistent mode, see [qemu_frida/README.persistent.md](qemu_frida/README.persistent.md). -The mode is approximately 2-5x slower than compile-time instrumentation, and is -less conducive to parallelization. - -### Unicorn - -For non-Linux binaries you can use AFL++'s unicorn mode which can emulate -anything you want - for the price of speed and user written scripts. -See [unicorn_mode](unicorn_mode/README.md). - -It can be easily built by: -```shell -cd unicorn_mode -./build_unicorn_support.sh -``` - -### Shared libraries -If the goal is to fuzz a dynamic library then there are two options available. -For both you need to write a small harness that loads and calls the library. -Faster is the frida solution: [utils/afl_frida/README.md](utils/afl_frida/README.md) +This image is automatically generated when a push to the stable repo happens +(see [branches](#branches)). You will find your target source code in `/src` in +the container. -Another, less precise and slower option is using ptrace with debugger interrupt -instrumentation: [utils/afl_untracer/README.md](utils/afl_untracer/README.md). +To build AFL++ yourself, continue at [docs/INSTALL.md](docs/INSTALL.md). -### More +## Quick start: Fuzzing with AFL++ -A more comprehensive description of these and other options can be found in -[docs/binaryonly_fuzzing.md](docs/binaryonly_fuzzing.md). +*NOTE: Before you start, please read about the +[common sense risks of fuzzing](docs/fuzzing_in_depth.md#0-common-sense-risks).* -## Challenges of guided fuzzing +This is a quick start for fuzzing targets with the source code available. To +read about the process in detail, see +[docs/fuzzing_in_depth.md](docs/fuzzing_in_depth.md). -Fuzzing is one of the most powerful and proven strategies for identifying -security issues in real-world software; it is responsible for the vast -majority of remote code execution and privilege escalation bugs found to date -in security-critical software. +To learn about fuzzing other targets, see: +* Binary-only targets: + [docs/fuzzing_binary-only_targets.md](docs/fuzzing_binary-only_targets.md) +* Network services: + [docs/best_practices.md#fuzzing-a-network-service](docs/best_practices.md#fuzzing-a-network-service) +* GUI programs: + [docs/best_practices.md#fuzzing-a-gui-program](docs/best_practices.md#fuzzing-a-gui-program) -Unfortunately, fuzzing is also relatively shallow; blind, random mutations -make it very unlikely to reach certain code paths in the tested code, leaving -some vulnerabilities firmly outside the reach of this technique. +Step-by-step quick start: -There have been numerous attempts to solve this problem. One of the early -approaches - pioneered by Tavis Ormandy - is corpus distillation. The method -relies on coverage signals to select a subset of interesting seeds from a -massive, high-quality corpus of candidate files, and then fuzz them by -traditional means. The approach works exceptionally well but requires such -a corpus to be readily available. In addition, block coverage measurements -provide only a very simplistic understanding of the program state and are less -useful for guiding the fuzzing effort in the long haul. +1. Compile the program or library to be fuzzed using `afl-cc`. A common way to + do this would be: -Other, more sophisticated research has focused on techniques such as program -flow analysis ("concolic execution"), symbolic execution, or static analysis. -All these methods are extremely promising in experimental settings, but tend -to suffer from reliability and performance problems in practical uses - and -currently do not offer a viable alternative to "dumb" fuzzing techniques. + ``` + CC=/path/to/afl-cc CXX=/path/to/afl-c++ ./configure --disable-shared + make clean all + ``` -## Background: The afl-fuzz approach +2. Get a small but valid input file that makes sense to the program. When + fuzzing verbose syntax (SQL, HTTP, etc.), create a dictionary as described in + [dictionaries/README.md](dictionaries/README.md), too. -American Fuzzy Lop is a brute-force fuzzer coupled with an exceedingly simple -but rock-solid instrumentation-guided genetic algorithm. It uses a modified -form of edge coverage to effortlessly pick up subtle, local-scale changes to -program control flow. +3. If the program reads from stdin, run `afl-fuzz` like so: -Simplifying a bit, the overall algorithm can be summed up as: + ``` + ./afl-fuzz -i seeds_dir -o output_dir -- \ + /path/to/tested/program [...program's cmdline...] + ``` - 1) Load user-supplied initial test cases into the queue, + To add a dictionary, add `-x /path/to/dictionary.txt` to afl-fuzz. - 2) Take the next input file from the queue, + If the program takes input from a file, you can put `@@` in the program's + command line; AFL++ will put an auto-generated file name in there for you. - 3) Attempt to trim the test case to the smallest size that doesn't alter - the measured behavior of the program, +4. Investigate anything shown in red in the fuzzer UI by promptly consulting + [docs/afl-fuzz_approach.md#understanding-the-status-screen](docs/afl-fuzz_approach.md#understanding-the-status-screen). - 4) Repeatedly mutate the file using a balanced and well-researched variety - of traditional fuzzing strategies, +5. You will find found crashes and hangs in the subdirectories `crashes/` and + `hangs/` in the `-o output_dir` directory. You can replay the crashes by + feeding them to the target, e.g.: - 5) If any of the generated mutations resulted in a new state transition - recorded by the instrumentation, add mutated output as a new entry in the - queue. + ``` + cat output_dir/crashes/id:000000,* | /path/to/tested/program [...program's cmdline...] + ``` - 6) Go to 2. + You can generate cores or use gdb directly to follow up the crashes. -The discovered test cases are also periodically culled to eliminate ones that -have been obsoleted by newer, higher-coverage finds; and undergo several other -instrumentation-driven effort minimization steps. - -As a side result of the fuzzing process, the tool creates a small, -self-contained corpus of interesting test cases. These are extremely useful -for seeding other, labor- or resource-intensive testing regimes - for example, -for stress-testing browsers, office applications, graphics suites, or -closed-source tools. - -The fuzzer is thoroughly tested to deliver out-of-the-box performance far -superior to blind fuzzing or coverage-only tools. - -## Help: Choosing initial test cases - -To operate correctly, the fuzzer requires one or more starting file that -contains a good example of the input data normally expected by the targeted -application. There are two basic rules: - - - Keep the files small. Under 1 kB is ideal, although not strictly necessary. - For a discussion of why size matters, see [perf_tips.md](docs/perf_tips.md). - - - Use multiple test cases only if they are functionally different from - each other. There is no point in using fifty different vacation photos - to fuzz an image library. - -You can find many good examples of starting files in the testcases/ subdirectory -that comes with this tool. - -PS. If a large corpus of data is available for screening, you may want to use -the afl-cmin utility to identify a subset of functionally distinct files that -exercise different code paths in the target binary. - -## Help: Interpreting output - -See the [docs/status_screen.md](docs/status_screen.md) file for information on -how to interpret the displayed stats and monitor the health of the process. Be -sure to consult this file especially if any UI elements are highlighted in red. - -The fuzzing process will continue until you press Ctrl-C. At a minimum, you want -to allow the fuzzer to complete one queue cycle, which may take anywhere from a -couple of hours to a week or so. - -There are three subdirectories created within the output directory and updated -in real-time: - - - queue/ - test cases for every distinctive execution path, plus all the - starting files given by the user. This is the synthesized corpus - mentioned in section 2. - - Before using this corpus for any other purposes, you can shrink - it to a smaller size using the afl-cmin tool. The tool will find - a smaller subset of files offering equivalent edge coverage. - - - crashes/ - unique test cases that cause the tested program to receive a - fatal signal (e.g., SIGSEGV, SIGILL, SIGABRT). The entries are - grouped by the received signal. - - - hangs/ - unique test cases that cause the tested program to time out. The - default time limit before something is classified as a hang is - the larger of 1 second and the value of the -t parameter. - The value can be fine-tuned by setting AFL_HANG_TMOUT, but this - is rarely necessary. - -Crashes and hangs are considered "unique" if the associated execution paths -involve any state transitions not seen in previously-recorded faults. If a -single bug can be reached in multiple ways, there will be some count inflation -early in the process, but this should quickly taper off. - -The file names for crashes and hangs are correlated with the parent, non-faulting -queue entries. This should help with debugging. - -When you can't reproduce a crash found by afl-fuzz, the most likely cause is -that you are not setting the same memory limit as used by the tool. Try: - -```shell -LIMIT_MB=50 -( ulimit -Sv $[LIMIT_MB << 10]; /path/to/tested_binary ... ) -``` - -Change LIMIT_MB to match the -m parameter passed to afl-fuzz. On OpenBSD, -also change -Sv to -Sd. - -Any existing output directory can be also used to resume aborted jobs; try: - -```shell -./afl-fuzz -i- -o existing_output_dir [...etc...] -``` - -If you have gnuplot installed, you can also generate some pretty graphs for any -active fuzzing task using afl-plot. For an example of how this looks like, -see [http://lcamtuf.coredump.cx/afl/plot/](http://lcamtuf.coredump.cx/afl/plot/). - -You can also manually build and install afl-plot-ui, which is a helper utility -for showing the graphs generated by afl-plot in a graphical window using GTK. -You can build and install it as follows - -```shell -sudo apt install libgtk-3-0 libgtk-3-dev pkg-config -cd utils/plot_ui -make -cd ../../ -sudo make install -``` - -## Help: Crash triage - -The coverage-based grouping of crashes usually produces a small data set that -can be quickly triaged manually or with a very simple GDB or Valgrind script. -Every crash is also traceable to its parent non-crashing test case in the -queue, making it easier to diagnose faults. - -Having said that, it's important to acknowledge that some fuzzing crashes can be -difficult to quickly evaluate for exploitability without a lot of debugging and -code analysis work. To assist with this task, afl-fuzz supports a very unique -"crash exploration" mode enabled with the -C flag. - -In this mode, the fuzzer takes one or more crashing test cases as the input -and uses its feedback-driven fuzzing strategies to very quickly enumerate all -code paths that can be reached in the program while keeping it in the -crashing state. - -Mutations that do not result in a crash are rejected; so are any changes that -do not affect the execution path. - -The output is a small corpus of files that can be very rapidly examined to see -what degree of control the attacker has over the faulting address, or whether -it is possible to get past an initial out-of-bounds read - and see what lies -beneath. - -Oh, one more thing: for test case minimization, give afl-tmin a try. The tool -can be operated in a very simple way: - -```shell -./afl-tmin -i test_case -o minimized_result -- /path/to/program [...] -``` - -The tool works with crashing and non-crashing test cases alike. In the crash -mode, it will happily accept instrumented and non-instrumented binaries. In the -non-crashing mode, the minimizer relies on standard AFL++ instrumentation to make -the file simpler without altering the execution path. - -The minimizer accepts the -m, -t, -f and @@ syntax in a manner compatible with -afl-fuzz. - -Another tool in AFL++ is the afl-analyze tool. It takes an input -file, attempts to sequentially flip bytes, and observes the behavior of the -tested program. It then color-codes the input based on which sections appear to -be critical, and which are not; while not bulletproof, it can often offer quick -insights into complex file formats. More info about its operation can be found -near the end of [docs/technical_details.md](docs/technical_details.md). - -## Going beyond crashes - -Fuzzing is a wonderful and underutilized technique for discovering non-crashing -design and implementation errors, too. Quite a few interesting bugs have been -found by modifying the target programs to call abort() when say: - - - Two bignum libraries produce different outputs when given the same - fuzzer-generated input, - - - An image library produces different outputs when asked to decode the same - input image several times in a row, - - - A serialization / deserialization library fails to produce stable outputs - when iteratively serializing and deserializing fuzzer-supplied data, - - - A compression library produces an output inconsistent with the input file - when asked to compress and then decompress a particular blob. - -Implementing these or similar sanity checks usually takes very little time; -if you are the maintainer of a particular package, you can make this code -conditional with `#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION` (a flag also -shared with libfuzzer and honggfuzz) or `#ifdef __AFL_COMPILER` (this one is -just for AFL). - -## Common-sense risks - -Please keep in mind that, similarly to many other computationally-intensive -tasks, fuzzing may put a strain on your hardware and on the OS. In particular: - - - Your CPU will run hot and will need adequate cooling. In most cases, if - cooling is insufficient or stops working properly, CPU speeds will be - automatically throttled. That said, especially when fuzzing on less - suitable hardware (laptops, smartphones, etc), it's not entirely impossible - for something to blow up. - - - Targeted programs may end up erratically grabbing gigabytes of memory or - filling up disk space with junk files. AFL++ tries to enforce basic memory - limits, but can't prevent each and every possible mishap. The bottom line - is that you shouldn't be fuzzing on systems where the prospect of data loss - is not an acceptable risk. - - - Fuzzing involves billions of reads and writes to the filesystem. On modern - systems, this will be usually heavily cached, resulting in fairly modest - "physical" I/O - but there are many factors that may alter this equation. - It is your responsibility to monitor for potential trouble; with very heavy - I/O, the lifespan of many HDDs and SSDs may be reduced. - - A good way to monitor disk I/O on Linux is the 'iostat' command: - -```shell - $ iostat -d 3 -x -k [...optional disk ID...] -``` - - Using the `AFL_TMPDIR` environment variable and a RAM-disk you can have the - heavy writing done in RAM to prevent the aforementioned wear and tear. For - example the following line will run a Docker container with all this preset: - - ```shell - # docker run -ti --mount type=tmpfs,destination=/ramdisk -e AFL_TMPDIR=/ramdisk aflplusplus/aflplusplus - ``` - -## Known limitations & areas for improvement +## Contact -Here are some of the most important caveats for AFL: +Questions? Concerns? Bug reports? - - AFL++ detects faults by checking for the first spawned process dying due to - a signal (SIGSEGV, SIGABRT, etc). Programs that install custom handlers for - these signals may need to have the relevant code commented out. In the same - vein, faults in child processes spawned by the fuzzed target may evade - detection unless you manually add some code to catch that. +* The contributors can be reached via + [https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus). +* Take a look at our [FAQ](docs/FAQ.md). If you find an interesting or important + question missing, submit it via + [https://github.com/AFLplusplus/AFLplusplus/discussions](https://github.com/AFLplusplus/AFLplusplus/discussions). +* There is a mailing list for the AFL/AFL++ project + ([browse archive](https://groups.google.com/group/afl-users)). To compare + notes with other users or to get notified about major new features, send an + email to <afl-users+subscribe@googlegroups.com>. +* Or join the [Awesome Fuzzing](https://discord.gg/gCraWct) Discord server. - - As with any other brute-force tool, the fuzzer offers limited coverage if - encryption, checksums, cryptographic signatures, or compression are used to - wholly wrap the actual data format to be tested. +## Branches - To work around this, you can comment out the relevant checks (see - utils/libpng_no_checksum/ for inspiration); if this is not possible, - you can also write a postprocessor, one of the hooks of custom mutators. - See [docs/custom_mutators.md](docs/custom_mutators.md) on how to use - `AFL_CUSTOM_MUTATOR_LIBRARY` +The following branches exist: - - There are some unfortunate trade-offs with ASAN and 64-bit binaries. This - isn't due to any specific fault of afl-fuzz. +* [release](https://github.com/AFLplusplus/AFLplusplus/tree/release): the latest + release +* [stable/trunk](https://github.com/AFLplusplus/AFLplusplus/): stable state of + AFL++ - it is synced from dev from time to time when we are satisfied with its + stability +* [dev](https://github.com/AFLplusplus/AFLplusplus/tree/dev): development state + of AFL++ - bleeding edge and you might catch a checkout which does not compile + or has a bug. *We only accept PRs in dev!!* +* (any other): experimental branches to work on specific features or testing new + functionality or changes. - - There is no direct support for fuzzing network services, background - daemons, or interactive apps that require UI interaction to work. You may - need to make simple code changes to make them behave in a more traditional - way. Preeny may offer a relatively simple option, too - see: - [https://github.com/zardus/preeny](https://github.com/zardus/preeny) +## Help wanted - Some useful tips for modifying network-based services can be also found at: - [https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop](https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop) +We have several [ideas](docs/ideas.md) we would like to see in AFL++ to make it +even better. However, we already work on so many things that we do not have the +time for all the big ideas. - - Occasionally, sentient machines rise against their creators. If this - happens to you, please consult [http://lcamtuf.coredump.cx/prep/](http://lcamtuf.coredump.cx/prep/). +This can be your way to support and contribute to AFL++ - extend it to do +something cool. -Beyond this, see INSTALL for platform-specific tips. +For everyone who wants to contribute (and send pull requests), please read our +[contributing guidelines](CONTRIBUTING.md) before your submit. ## Special thanks Many of the improvements to the original AFL and AFL++ wouldn't be possible -without feedback, bug reports, or patches from: - -``` - Jann Horn Hanno Boeck - Felix Groebert Jakub Wilk - Richard W. M. Jones Alexander Cherepanov - Tom Ritter Hovik Manucharyan - Sebastian Roschke Eberhard Mattes - Padraig Brady Ben Laurie - @dronesec Luca Barbato - Tobias Ospelt Thomas Jarosch - Martin Carpenter Mudge Zatko - Joe Zbiciak Ryan Govostes - Michael Rash William Robinet - Jonathan Gray Filipe Cabecinhas - Nico Weber Jodie Cunningham - Andrew Griffiths Parker Thompson - Jonathan Neuschaefer Tyler Nighswander - Ben Nagy Samir Aguiar - Aidan Thornton Aleksandar Nikolich - Sam Hakim Laszlo Szekeres - David A. Wheeler Turo Lamminen - Andreas Stieger Richard Godbee - Louis Dassy teor2345 - Alex Moneger Dmitry Vyukov - Keegan McAllister Kostya Serebryany - Richo Healey Martijn Bogaard - rc0r Jonathan Foote - Christian Holler Dominique Pelle - Jacek Wielemborek Leo Barnes - Jeremy Barnes Jeff Trull - Guillaume Endignoux ilovezfs - Daniel Godas-Lopez Franjo Ivancic - Austin Seipp Daniel Komaromy - Daniel Binderman Jonathan Metzman - Vegard Nossum Jan Kneschke - Kurt Roeckx Marcel Boehme - Van-Thuan Pham Abhik Roychoudhury - Joshua J. Drake Toby Hutton - Rene Freingruber Sergey Davidoff - Sami Liedes Craig Young - Andrzej Jackowski Daniel Hodson - Nathan Voss Dominik Maier - Andrea Biondo Vincent Le Garrec - Khaled Yakdan Kuang-che Wu - Josephine Calliotte Konrad Welc - Thomas Rooijakkers David Carlier - Ruben ten Hove Joey Jiao - fuzzah -``` - -Thank you! -(For people sending pull requests - please add yourself to this list :-) +without feedback, bug reports, or patches from our contributors. + +Thank you! (For people sending pull requests - please add yourself to this list +:-) + +<details> + + <summary>List of contributors</summary> + + ``` + Jann Horn Hanno Boeck + Felix Groebert Jakub Wilk + Richard W. M. Jones Alexander Cherepanov + Tom Ritter Hovik Manucharyan + Sebastian Roschke Eberhard Mattes + Padraig Brady Ben Laurie + @dronesec Luca Barbato + Tobias Ospelt Thomas Jarosch + Martin Carpenter Mudge Zatko + Joe Zbiciak Ryan Govostes + Michael Rash William Robinet + Jonathan Gray Filipe Cabecinhas + Nico Weber Jodie Cunningham + Andrew Griffiths Parker Thompson + Jonathan Neuschaefer Tyler Nighswander + Ben Nagy Samir Aguiar + Aidan Thornton Aleksandar Nikolich + Sam Hakim Laszlo Szekeres + David A. Wheeler Turo Lamminen + Andreas Stieger Richard Godbee + Louis Dassy teor2345 + Alex Moneger Dmitry Vyukov + Keegan McAllister Kostya Serebryany + Richo Healey Martijn Bogaard + rc0r Jonathan Foote + Christian Holler Dominique Pelle + Jacek Wielemborek Leo Barnes + Jeremy Barnes Jeff Trull + Guillaume Endignoux ilovezfs + Daniel Godas-Lopez Franjo Ivancic + Austin Seipp Daniel Komaromy + Daniel Binderman Jonathan Metzman + Vegard Nossum Jan Kneschke + Kurt Roeckx Marcel Boehme + Van-Thuan Pham Abhik Roychoudhury + Joshua J. Drake Toby Hutton + Rene Freingruber Sergey Davidoff + Sami Liedes Craig Young + Andrzej Jackowski Daniel Hodson + Nathan Voss Dominik Maier + Andrea Biondo Vincent Le Garrec + Khaled Yakdan Kuang-che Wu + Josephine Calliotte Konrad Welc + Thomas Rooijakkers David Carlier + Ruben ten Hove Joey Jiao + fuzzah + ``` + +</details> ## Cite -If you use AFLpluplus to compare to your work, please use either `afl-clang-lto` -or `afl-clang-fast` with `AFL_LLVM_CMPLOG=1` for building targets and -`afl-fuzz` with the command line option `-l 2` for fuzzing. -The most effective setup is the `aflplusplus` default configuration on Google's [fuzzbench](https://github.com/google/fuzzbench/tree/master/fuzzers/aflplusplus). +If you use AFL++ in scientific work, consider citing +[our paper](https://www.usenix.org/conference/woot20/presentation/fioraldi) +presented at WOOT'20: -If you use AFLplusplus in scientific work, consider citing [our paper](https://www.usenix.org/conference/woot20/presentation/fioraldi) presented at WOOT'20: + Andrea Fioraldi, Dominik Maier, Heiko Eißfeldt, and Marc Heuse. “AFL++: Combining incremental steps of fuzzing research”. In 14th USENIX Workshop on Offensive Technologies (WOOT 20). USENIX Association, Aug. 2020. -+ Andrea Fioraldi, Dominik Maier, Heiko Eißfeldt, and Marc Heuse. “AFL++: Combining incremental steps of fuzzing research”. In 14th USENIX Workshop on Offensive Technologies (WOOT 20). USENIX Association, Aug. 2020. +<details> -Bibtex: - -```bibtex -@inproceedings {AFLplusplus-Woot20, - author = {Andrea Fioraldi and Dominik Maier and Heiko Ei{\ss}feldt and Marc Heuse}, - title = {{AFL++}: Combining Incremental Steps of Fuzzing Research}, - booktitle = {14th {USENIX} Workshop on Offensive Technologies ({WOOT} 20)}, - year = {2020}, - publisher = {{USENIX} Association}, - month = aug, -} -``` - -## Contact - -Questions? Concerns? Bug reports? The contributors can be reached via -[https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) +<summary>BibTeX</summary> -There is also a mailing list for the AFL/AFL++ project; to join, send a mail to -<afl-users+subscribe@googlegroups.com>. Or, if you prefer to browse archives -first, try: [https://groups.google.com/group/afl-users](https://groups.google.com/group/afl-users) + ```bibtex + @inproceedings {AFLplusplus-Woot20, + author = {Andrea Fioraldi and Dominik Maier and Heiko Ei{\ss}feldt and Marc Heuse}, + title = {{AFL++}: Combining Incremental Steps of Fuzzing Research}, + booktitle = {14th {USENIX} Workshop on Offensive Technologies ({WOOT} 20)}, + year = {2020}, + publisher = {{USENIX} Association}, + month = aug, + } + ``` +</details> \ No newline at end of file diff --git a/TODO.md b/TODO.md index 1c616b4a..e6b095fc 100644 --- a/TODO.md +++ b/TODO.md @@ -1,38 +1,32 @@ # TODO list for AFL++ -## Roadmap 3.00+ +## Should + - better autodetection of shifting runtime timeout values - Update afl->pending_not_fuzzed for MOpt - - put fuzz target in top line of UI - afl-plot to support multiple plot_data - - afl_custom_fuzz_splice_optin() - - afl_custom_splice() - - better autodetection of shifting runtime timeout values - - cmplog: use colorization input for havoc? - parallel builds for source-only targets + - get rid of check_binary, replace with more forkserver communication +## Maybe -## Further down the road + - afl_custom_fuzz_splice_optin() + - afl_custom_splice() + - cmdline option from-to range for mutations -afl-fuzz: - - setting min_len/max_len/start_offset/end_offset limits for mutation output +## Further down the road -qemu_mode: +QEMU mode/FRIDA mode: - non colliding instrumentation - rename qemu specific envs to AFL_QEMU (AFL_ENTRYPOINT, AFL_CODE_START/END, AFL_COMPCOV_LEVEL?) - - add AFL_QEMU_EXITPOINT (maybe multiple?), maybe pointless as we have + - add AFL_QEMU_EXITPOINT (maybe multiple?), maybe pointless as there is persistent mode - - add/implement AFL_QEMU_INST_LIBLIST and AFL_QEMU_NOINST_PROGRAM - - add/implement AFL_QEMU_INST_REGIONS as a list of _START/_END addresses - ## Ideas - LTO/sancov: write current edge to prev_loc and use that information when - using cmplog or __sanitizer_cov_trace_cmp*. maybe we can deduct by follow - up edge numbers that both following cmp paths have been found and then - disable working on this edge id -> cmplog_intelligence branch - - use cmplog colorization taint result for havoc locations? - - new instrumentation option for a thread-safe variant of feedback to shared mem. - The user decides, if this is needed (eg the target is multithreaded). + using cmplog or __sanitizer_cov_trace_cmp*. maybe we can deduct by follow up + edge numbers that both following cmp paths have been found and then disable + working on this edge id -> cmplog_intelligence branch + - use cmplog colorization taint result for havoc locations? \ No newline at end of file diff --git a/afl-cmin b/afl-cmin index e6f8c175..879aead2 100755 --- a/afl-cmin +++ b/afl-cmin @@ -396,7 +396,7 @@ BEGIN { system( "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") } else { system("cp \""in_dir"/"first_file"\" "stdin_file) - system( "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null") + system( "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null") } first_count = 0 @@ -432,8 +432,8 @@ BEGIN { retval = system( AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string) } else { print " Processing "in_count" files (forkserver mode)..." -# print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null" - retval = system( AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null") +# print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null" + retval = system( AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null") } if (retval && !AFL_CMIN_CRASHES_ONLY) { diff --git a/afl-cmin.bash b/afl-cmin.bash index c77dfbc1..9ac65199 100755 --- a/afl-cmin.bash +++ b/afl-cmin.bash @@ -11,7 +11,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # This tool tries to find the smallest subset of files in the input directory # that still trigger the full range of instrumentation data points seen in @@ -310,7 +310,7 @@ if [ "$STDIN_FILE" = "" ]; then else cp "$IN_DIR/$FIRST_FILE" "$STDIN_FILE" - AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_PAR -A "$STDIN_FILE" -- "$@" </dev/null + AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_PAR -H "$STDIN_FILE" -- "$@" </dev/null fi @@ -360,7 +360,7 @@ echo "[*] Obtaining traces for input files in '$IN_DIR'..." cp "$IN_DIR/$fn" "$STDIN_FILE" - "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -A "$STDIN_FILE" -- "$@" </dev/null + "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -H "$STDIN_FILE" -- "$@" </dev/null done diff --git a/afl-plot b/afl-plot index 87b9caae..75981d7f 100755 --- a/afl-plot +++ b/afl-plot @@ -12,7 +12,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # get_abs_path() { @@ -195,15 +195,19 @@ exit 1 fi -mkdir -p "$outputdir/tmp" -afl-plot-ui > "$outputdir/tmp/win_ids" & +rm -rf "$outputdir/.tmp" +mkdir -p "$outputdir/.tmp" +mkfifo "$outputdir/.tmp/win_ids" || exit 1 -sleep 0.5 +afl-plot-ui > "$outputdir/.tmp/win_ids" & +W_IDS=$(cat "$outputdir/.tmp/win_ids") -W_ID1=$(cat $outputdir/tmp/win_ids | head -1) -W_ID2=$(cat $outputdir/tmp/win_ids | head -2 | tail -1) -W_ID3=$(cat $outputdir/tmp/win_ids | head -3 | tail -1) -W_ID4=$(cat $outputdir/tmp/win_ids | tail -1) +rm -rf "$outputdir/.tmp" + +W_ID1=$(echo "$W_IDS" | head -n 1) +W_ID2=$(echo "$W_IDS" | head -n 2 | tail -n 1) +W_ID3=$(echo "$W_IDS" | head -n 3 | tail -n 1) +W_ID4=$(echo "$W_IDS" | tail -n 1) echo "[*] Generating plots..." @@ -265,12 +269,6 @@ _EOF_ sleep 1 -rm "$outputdir/tmp/win_ids" - -if [ -z "$(ls -A $outputdir/tmp)" ]; then - rm -r "$outputdir/tmp" -fi - else echo "[*] Generating plots..." diff --git a/afl-system-config b/afl-system-config index dbdbbf1f..b222b2ad 100755 --- a/afl-system-config +++ b/afl-system-config @@ -34,8 +34,8 @@ if [ "$PLATFORM" = "Linux" ] ; then sysctl -w kernel.randomize_va_space=0 sysctl -w kernel.sched_child_runs_first=1 sysctl -w kernel.sched_autogroup_enabled=1 - sysctl -w kernel.sched_migration_cost_ns=50000000 - sysctl -w kernel.sched_latency_ns=250000000 + sysctl -w kernel.sched_migration_cost_ns=50000000 2>/dev/null + sysctl -w kernel.sched_latency_ns=250000000 2>/dev/null echo never > /sys/kernel/mm/transparent_hugepage/enabled test -e /sys/devices/system/cpu/cpufreq/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpufreq/scaling_governor test -e /sys/devices/system/cpu/cpufreq/policy0/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor @@ -52,7 +52,7 @@ if [ "$PLATFORM" = "Linux" ] ; then echo ' /etc/default/grub:GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=0 l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off srbds=off noexec=off noexec32=off tsx=on tsx_async_abort=off arm64.nopauth audit=0 hardened_usercopy=off ssbd=force-off"' echo } - echo If you run fuzzing instances in docker, run them with \"--security-opt seccomp=unconfined\" for more speed + echo If you run fuzzing instances in docker, run them with \"--security-opt seccomp=unconfined\" for more speed. echo DONE=1 fi diff --git a/afl-whatsup b/afl-whatsup index 9c2564c6..10a52f83 100755 --- a/afl-whatsup +++ b/afl-whatsup @@ -12,7 +12,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # This tool summarizes the status of any locally-running synchronized # instances of afl-fuzz. diff --git a/coresight_mode/.gitignore b/coresight_mode/.gitignore new file mode 100644 index 00000000..dedb1613 --- /dev/null +++ b/coresight_mode/.gitignore @@ -0,0 +1,2 @@ +.local +glibc* diff --git a/coresight_mode/GNUmakefile b/coresight_mode/GNUmakefile new file mode 100644 index 00000000..9ab30ff7 --- /dev/null +++ b/coresight_mode/GNUmakefile @@ -0,0 +1,62 @@ +#!/usr/bin/env make +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2021 Ricerca Security, Inc. All rights reserved. + +SHELL:=bash +PREFIX?=$(shell pwd)/.local + +CS_TRACE:=coresight-trace + +PATCHELF?=$(PREFIX)/bin/patchelf + +PATCH_DIR:=patches + +GLIBC_VER:=2.33 +GLIBC_NAME:=glibc-$(GLIBC_VER) +GLIBC_URL_BASE:=http://ftp.gnu.org/gnu/glibc +GLIBC_LDSO?=$(PREFIX)/lib/ld-linux-aarch64.so.1 + +OUTPUT?="$(TARGET).patched" + +all: build + +build: + git submodule update --init --recursive $(CS_TRACE) + $(MAKE) -C $(CS_TRACE) + cp $(CS_TRACE)/cs-proxy ../afl-cs-proxy + +patch: | $(PATCHELF) $(GLIBC_LDSO) + @if test -z "$(TARGET)"; then echo "TARGET is not set"; exit 1; fi + $(PATCHELF) \ + --set-interpreter $(GLIBC_LDSO) \ + --set-rpath $(dir $(GLIBC_LDSO)) \ + --output $(OUTPUT) \ + $(TARGET) + +$(PATCHELF): patchelf + git submodule update --init $< + cd $< && \ + ./bootstrap.sh && \ + ./configure --prefix=$(PREFIX) && \ + $(MAKE) && \ + $(MAKE) check && \ + $(MAKE) install + +$(GLIBC_LDSO): | $(GLIBC_NAME).tar.xz + tar -xf $(GLIBC_NAME).tar.xz + for file in $(shell find $(PATCH_DIR) -maxdepth 1 -type f); do \ + patch -p1 < $$file ; \ + done + mkdir -p $(GLIBC_NAME)/build + cd $(GLIBC_NAME)/build && \ + ../configure --prefix=$(PREFIX) && \ + $(MAKE) && \ + $(MAKE) install + +$(GLIBC_NAME).tar.xz: + wget -O $@ $(GLIBC_URL_BASE)/$@ + +clean: + $(MAKE) -C $(CS_TRACE) clean + +.PHONY: all build patch clean diff --git a/coresight_mode/Makefile b/coresight_mode/Makefile new file mode 100644 index 00000000..fb8990b9 --- /dev/null +++ b/coresight_mode/Makefile @@ -0,0 +1,21 @@ +#!/usr/bin/env make +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2021 Ricerca Security, Inc. All rights reserved. + +all: + @echo trying to use GNU make... + @gmake all || echo please install GNUmake + +build: + @echo trying to use GNU make... + @gmake build || echo please install GNUmake + +patch: + @echo trying to use GNU make... + @gmake patch || echo please install GNUmake + +clean: + @echo trying to use GNU make... + @gmake clean || echo please install GNUmake + +.PHONY: all build patch clean diff --git a/coresight_mode/README.md b/coresight_mode/README.md new file mode 100644 index 00000000..1a39d347 --- /dev/null +++ b/coresight_mode/README.md @@ -0,0 +1,70 @@ +# AFL++ CoreSight mode + +CoreSight mode enables binary-only fuzzing on ARM64 Linux using CoreSight (ARM's hardware tracing technology). + +NOTE: CoreSight mode is in the early development stage. Not applicable for production use. +Currently the following hardware boards are supported: +* NVIDIA Jetson TX2 (NVIDIA Parker) +* NVIDIA Jetson Nano (NVIDIA Tegra X1) +* GIGABYTE R181-T90 (Marvell ThunderX2 CN99XX) + +## Getting started + +Please read the [RICSec/coresight-trace README](https://github.com/RICSecLab/coresight-trace/blob/master/README.md) and check the prerequisites (capstone) before getting started. + +CoreSight mode supports the AFL++ fork server mode to reduce `exec` system call +overhead. To support it for binary-only fuzzing, it needs to modify the target +ELF binary to re-link to the patched glibc. We employ this design from +[PTrix](https://github.com/junxzm1990/afl-pt). + +Check out all the git submodules in the `cs_mode` directory: + +```bash +git submodule update --init --recursive +``` + +### Build coresight-trace + +There are some notes on building coresight-trace. Refer to the [README](https://github.com/RICSecLab/coresight-trace/blob/master/README.md) for the details. Run make in the `cs_mode` directory: + +```bash +make build +``` + +Make sure `cs-proxy` is placed in the AFL++ root directory as `afl-cs-proxy`. + +### Patch COTS binary + +The fork server mode requires patchelf and the patched glibc. The dependency build can be done by just run make: + +```bash +make patch TARGET=$BIN +``` + +The above make command builds and installs the dependencies to `$PREFIX` (default to `$PWD/.local`) at the first time. Then, it runs `patchelf` to `$BIN` with output `$OUTPUT` (`$BIN.patched` by default). + +### Run afl-fuzz + +Run `afl-fuzz` with `-A` option to use CoreSight mode. + +```bash +sudo afl-fuzz -A -i input -o output -- $OUTPUT @@ +``` + +## Environment Variables + +There are AFL++ CoreSight mode-specific environment variables for run-time configuration. + +* `AFL_CS_CUSTOM_BIN` overrides the proxy application path. `afl-cs-proxy` will be used if not defined. + +* `AFLCS_COV` specifies coverage type on CoreSight trace decoding. `edge` and `path` is supported. The default value is `edge`. +* `AFLCS_UDMABUF` is the u-dma-buf device number used to store trace data in the DMA region. The default value is `0`. + +## TODO List + +* Eliminate modified glibc dependency +* Support parallel fuzzing + +## Acknowledgements + +This project has received funding from the Acquisition, Technology & Logistics Agency (ATLA) under the National Security Technology Research Promotion Fund 2021 (JPJ004596). diff --git a/coresight_mode/coresight-trace b/coresight_mode/coresight-trace new file mode 160000 +Subproject ec0fd6104720ac0b59967616363dc18209adc02 diff --git a/coresight_mode/patchelf b/coresight_mode/patchelf new file mode 160000 +Subproject 7ec8edbe094ee13c91dadca191f92b9dfac8c0f diff --git a/coresight_mode/patches/0001-Add-AFL-forkserver.patch b/coresight_mode/patches/0001-Add-AFL-forkserver.patch new file mode 100644 index 00000000..51c242c4 --- /dev/null +++ b/coresight_mode/patches/0001-Add-AFL-forkserver.patch @@ -0,0 +1,117 @@ +diff --git a/glibc-2.33/elf/rtld.c b/glibc-2.33/elf/rtld.c +index 596b6ac3..2ee270d4 100644 +--- a/glibc-2.33/elf/rtld.c ++++ b/glibc-2.33/elf/rtld.c +@@ -169,6 +169,99 @@ uintptr_t __pointer_chk_guard_local + strong_alias (__pointer_chk_guard_local, __pointer_chk_guard) + #endif + ++#define AFLCS_RTLD 1 ++ ++#if AFLCS_RTLD ++ ++#include <sys/shm.h> ++#include <sys/types.h> ++#include <sys/wait.h> ++#include <dlfcn.h> ++#include <signal.h> ++ ++#include <asm/unistd.h> ++#include <unistd.h> ++ ++#define FORKSRV_FD 198 ++ ++#define AFLCS_ENABLE "__AFLCS_ENABLE" ++ ++/* We use this additional AFLCS_# AFLCS_#+1 pair to communicate with proxy */ ++#define AFLCS_FORKSRV_FD (FORKSRV_FD - 3) ++#define AFLCS_RTLD_SNIPPET do { __cs_start_forkserver(); } while(0) ++ ++/* Fork server logic, invoked before we return from _dl_start. */ ++ ++static void __cs_start_forkserver(void) { ++ int status; ++ pid_t child_pid; ++ static char tmp[4] = {0, 0, 0, 0}; ++ ++ if (!getenv(AFLCS_ENABLE)) { ++ return; ++ } ++ ++ if (write(AFLCS_FORKSRV_FD + 1, tmp, 4) != 4) { ++ _exit(-1); ++ } ++ ++ /* All right, let's await orders... */ ++ while (1) { ++ /* Whoops, parent dead? */ ++ if (read(AFLCS_FORKSRV_FD, tmp, 4) != 4) { ++ _exit(1); ++ } ++ ++ child_pid = INLINE_SYSCALL(clone, 5, ++ CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID | SIGCHLD, 0, ++ NULL, NULL, &THREAD_SELF->tid); ++ if (child_pid < 0) { ++ _exit(4); ++ } ++ if (!child_pid) { ++ /* Child process. Wait for parent start tracing */ ++ kill(getpid(), SIGSTOP); ++ /* Close descriptors and run free. */ ++ close(AFLCS_FORKSRV_FD); ++ close(AFLCS_FORKSRV_FD + 1); ++ return; ++ } ++ ++ /* Parent. */ ++ if (write(AFLCS_FORKSRV_FD + 1, &child_pid, 4) != 4) { ++ _exit(5); ++ } ++ ++ /* Wait until SIGCONT is signaled. */ ++ if (waitpid(child_pid, &status, WCONTINUED) < 0) { ++ _exit(6); ++ } ++ if (!WIFCONTINUED(status)) { ++ /* Relay status to proxy. */ ++ if (write(AFLCS_FORKSRV_FD + 1, &status, 4) != 4) { ++ _exit(7); ++ } ++ continue; ++ } ++ while (1) { ++ /* Get status. */ ++ if (waitpid(child_pid, &status, WUNTRACED) < 0) { ++ _exit(8); ++ } ++ /* Relay status to proxy. */ ++ if (write(AFLCS_FORKSRV_FD + 1, &status, 4) != 4) { ++ _exit(9); ++ } ++ if (!(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP)) { ++ /* The child process is exited. */ ++ break; ++ } ++ } ++ } ++} ++ ++#endif /* AFLCS_RTLD */ ++ + /* Check that AT_SECURE=0, or that the passed name does not contain + directories and is not overly long. Reject empty names + unconditionally. */ +@@ -588,6 +681,12 @@ _dl_start (void *arg) + # define ELF_MACHINE_START_ADDRESS(map, start) (start) + #endif + ++ /* AFL-CS-START */ ++#if AFLCS_RTLD ++ AFLCS_RTLD_SNIPPET; ++#endif ++ /* AFL-CS-END */ ++ + return ELF_MACHINE_START_ADDRESS (GL(dl_ns)[LM_ID_BASE]._ns_loaded, entry); + } + } diff --git a/custom_mutators/README.md b/custom_mutators/README.md index fa877f34..0289e150 100644 --- a/custom_mutators/README.md +++ b/custom_mutators/README.md @@ -15,6 +15,7 @@ In `./rust`, you will find rust bindings, including a simple example in `./rust/ If you use git to clone AFL++, then the following will incorporate our excellent grammar custom mutator: + ```sh git submodule update --init ``` diff --git a/custom_mutators/gramatron/README.md b/custom_mutators/gramatron/README.md index 91f93355..8aa0cc44 100644 --- a/custom_mutators/gramatron/README.md +++ b/custom_mutators/gramatron/README.md @@ -1,19 +1,19 @@ # GramaTron -Gramatron is a coverage-guided fuzzer that uses grammar automatons to perform -grammar-aware fuzzing. Technical details about our framework are available -in the [ISSTA'21 paper](https://nebelwelt.net/files/21ISSTA.pdf). -The artifact to reproduce the experiments presented in the paper are present -in `artifact/`. Instructions to run a sample campaign and incorporate new -grammars is presented below: +GramaTron is a coverage-guided fuzzer that uses grammar automatons to perform +grammar-aware fuzzing. Technical details about our framework are available in +the [ISSTA'21 paper](https://nebelwelt.net/files/21ISSTA.pdf). The artifact to +reproduce the experiments presented in the paper are present in `artifact/`. +Instructions to run a sample campaign and incorporate new grammars is presented +below: -# Compiling +## Compiling -Simply execute `./build_gramatron_mutator.sh` +Execute `./build_gramatron_mutator.sh`. -# Running +## Running -You have to set the grammar file to use with `GRAMMATRON_AUTOMATION`: +You have to set the grammar file to use with `GRAMATRON_AUTOMATION`: ``` export AFL_DISABLE_TRIM=1 @@ -23,23 +23,27 @@ export GRAMATRON_AUTOMATION=grammars/ruby/source_automata.json afl-fuzz -i in -o out -- ./target ``` -# Adding and testing a new grammar +## Adding and testing a new grammar -- Specify in a JSON format for CFG. Examples are correspond `source.json` files +- Specify in a JSON format for CFG. Examples are correspond `source.json` files. - Run the automaton generation script (in `src/gramfuzz-mutator/preprocess`) which will place the generated automaton in the same folder. -``` -./preprocess/prep_automaton.sh <grammar_file> <start_symbol> [stack_limit] -Eg. ./preprocess/prep_automaton.sh ~/grammars/ruby/source.json PROGRAM -``` -- If the grammar has no self-embedding rules then you do not need to pass the - stack limit parameter. However, if it does have self-embedding rules then you + ``` + ./preprocess/prep_automaton.sh <grammar_file> <start_symbol> [stack_limit] + + E.g., ./preprocess/prep_automaton.sh ~/grammars/ruby/source.json PROGRAM + ``` + +- If the grammar has no self-embedding rules, then you do not need to pass the + stack limit parameter. However, if it does have self-embedding rules, then you need to pass the stack limit parameter. We recommend starting with `5` and - then increasing it if you need more complexity -- To sanity-check that the automaton is generating inputs as expected you can use the `test` binary housed in `src/gramfuzz-mutator` -``` -./test SanityCheck <automaton_file> + then increasing it if you need more complexity. +- To sanity-check that the automaton is generating inputs as expected, you can + use the `test` binary housed in `src/gramfuzz-mutator`. -Eg. ./test SanityCheck ~/grammars/ruby/source_automata.json -``` + ``` + ./test SanityCheck <automaton_file> + + E.g., ./test SanityCheck ~/grammars/ruby/source_automata.json + ``` \ No newline at end of file diff --git a/custom_mutators/grammar_mutator/grammar_mutator b/custom_mutators/grammar_mutator/grammar_mutator -Subproject eedf07ddb0fb1f437f5e76b77cfd4064cf6a5d6 +Subproject b79d51a8daccbd7a693f9b6765c81ead14f28e2 diff --git a/custom_mutators/libfuzzer/FuzzerLoop.cpp b/custom_mutators/libfuzzer/FuzzerLoop.cpp index 08fda520..6716dbf5 100644 --- a/custom_mutators/libfuzzer/FuzzerLoop.cpp +++ b/custom_mutators/libfuzzer/FuzzerLoop.cpp @@ -1086,6 +1086,7 @@ ATTRIBUTE_INTERFACE size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) { assert(fuzzer::F); + fuzzer::F->GetMD().StartMutationSequence(); size_t r = fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize); #ifdef INTROSPECTION introspection_ptr = fuzzer::F->GetMD().WriteMutationSequence(); diff --git a/custom_mutators/libfuzzer/README.md b/custom_mutators/libfuzzer/README.md index fb3025f2..cb4773b7 100644 --- a/custom_mutators/libfuzzer/README.md +++ b/custom_mutators/libfuzzer/README.md @@ -11,9 +11,11 @@ Note that this is currently a simple implementation and it is missing two featur * Dictionary support To update the source, all that is needed is that FuzzerDriver.cpp has to receive + ``` #include "libfuzzer.inc" ``` + before the closing namespace bracket. It is also libfuzzer.inc where the configuration of the libfuzzer mutations @@ -21,4 +23,4 @@ are done. > Original repository: https://github.com/llvm/llvm-project > Path: compiler-rt/lib/fuzzer/*.{h|cpp} -> Source commit: df3e903655e2499968fc7af64fb5fa52b2ee79bb +> Source commit: df3e903655e2499968fc7af64fb5fa52b2ee79bb \ No newline at end of file diff --git a/custom_mutators/libfuzzer/libfuzzer.inc b/custom_mutators/libfuzzer/libfuzzer.inc index 01f21dbe..8c4bdbf6 100644 --- a/custom_mutators/libfuzzer/libfuzzer.inc +++ b/custom_mutators/libfuzzer/libfuzzer.inc @@ -2,7 +2,7 @@ extern "C" ATTRIBUTE_INTERFACE void LLVMFuzzerMyInit(int (*Callback)(const uint8_t *Data, size_t Size), unsigned int Seed) { - Random Rand(Seed); + auto *Rand = new Random(Seed); FuzzingOptions Options; Options.Verbosity = 3; Options.MaxLen = 1024000; @@ -30,7 +30,7 @@ LLVMFuzzerMyInit(int (*Callback)(const uint8_t *Data, size_t Size), unsigned int struct EntropicOptions Entropic; Entropic.Enabled = Options.Entropic; EF = new ExternalFunctions(); - auto *MD = new MutationDispatcher(Rand, Options); + auto *MD = new MutationDispatcher(*Rand, Options); auto *Corpus = new InputCorpus(Options.OutputCorpus, Entropic); auto *F = new Fuzzer(Callback, *Corpus, *MD, Options); } diff --git a/custom_mutators/libprotobuf-mutator-example/lpm_aflpp_custom_mutator_input.cc b/custom_mutators/libprotobuf-mutator-example/lpm_aflpp_custom_mutator_input.cc index e0273849..ecbfdd1c 100644 --- a/custom_mutators/libprotobuf-mutator-example/lpm_aflpp_custom_mutator_input.cc +++ b/custom_mutators/libprotobuf-mutator-example/lpm_aflpp_custom_mutator_input.cc @@ -99,10 +99,12 @@ extern "C" size_t afl_custom_fuzz(MyMutator *mutator, // return value from afl_c std::string s = ProtoToData(*p); // Copy to a new buffer ( mutated_out ) size_t mutated_size = s.size() <= max_size ? s.size() : max_size; // check if raw data's size is larger than max_size - uint8_t *mutated_out = new uint8_t[mutated_size+1]; - memcpy(mutated_out, s.c_str(), mutated_size); // copy the mutated data + + delete mutator->mutated_out; + mutator->mutated_out = new uint8_t[mutated_size+1]; + memcpy(mutator->mutated_out, s.c_str(), mutated_size); // copy the mutated data // Assign the mutated data and return mutated_size - *out_buf = mutated_out; + *out_buf = mutator->mutated_out; return mutated_size; } diff --git a/custom_mutators/libprotobuf-mutator-example/lpm_aflpp_custom_mutator_input.h b/custom_mutators/libprotobuf-mutator-example/lpm_aflpp_custom_mutator_input.h index ebd3ca65..0f5484ca 100644 --- a/custom_mutators/libprotobuf-mutator-example/lpm_aflpp_custom_mutator_input.h +++ b/custom_mutators/libprotobuf-mutator-example/lpm_aflpp_custom_mutator_input.h @@ -2,4 +2,6 @@ #include "test.pb.h" class MyMutator : public protobuf_mutator::Mutator { +public: + uint8_t *mutated_out = nullptr; }; diff --git a/custom_mutators/symcc/symcc.c b/custom_mutators/symcc/symcc.c index 19218449..86f23343 100644 --- a/custom_mutators/symcc/symcc.c +++ b/custom_mutators/symcc/symcc.c @@ -129,7 +129,7 @@ uint8_t afl_custom_queue_new_entry(my_mutator_t * data, int pid = fork(); - if (pid == -1) return; + if (pid == -1) return 0; if (pid) { @@ -147,7 +147,7 @@ uint8_t afl_custom_queue_new_entry(my_mutator_t * data, if (r <= 0) { close(pipefd[1]); - return; + return 0; } diff --git a/dictionaries/README.md b/dictionaries/README.md index 7c587abb..0b3b4d90 100644 --- a/dictionaries/README.md +++ b/dictionaries/README.md @@ -1,20 +1,18 @@ -# AFL dictionaries +# AFL++ dictionaries (See [../README.md](../README.md) for the general instruction manual.) -This subdirectory contains a set of dictionaries that can be used in -conjunction with the -x option to allow the fuzzer to effortlessly explore the -grammar of some of the more verbose data formats or languages. The basic -principle behind the operation of fuzzer dictionaries is outlined in section 10 -of the "main" README.md for the project. +This subdirectory contains a set of dictionaries that can be used in conjunction +with the -x option to allow the fuzzer to effortlessly explore the grammar of +some of the more verbose data formats or languages. -These sets were done by Michal Zalewski, various contributors, and imported -from oss-fuzz, go-fuzz and libfuzzer. +These sets were done by Michal Zalewski, various contributors, and imported from +oss-fuzz, go-fuzz and libfuzzer. Custom dictionaries can be added at will. They should consist of a reasonably-sized set of rudimentary syntax units that the fuzzer will then try -to clobber together in various ways. Snippets between 2 and 16 bytes are -usually the sweet spot. +to clobber together in various ways. Snippets between 2 and 16 bytes are usually +the sweet spot. Custom dictionaries can be created in two ways: @@ -36,9 +34,9 @@ In the file mode, every name field can be optionally followed by @<num>, e.g.: `keyword_foo@1 = "foo"` Such entries will be loaded only if the requested dictionary level is equal or -higher than this number. The default level is zero; a higher value can be set -by appending @<num> to the dictionary file name, like so: +higher than this number. The default level is zero; a higher value can be set by +appending @<num> to the dictionary file name, like so: `-x path/to/dictionary.dct@2` -Good examples of dictionaries can be found in xml.dict and png.dict. +Good examples of dictionaries can be found in xml.dict and png.dict. \ No newline at end of file diff --git a/docs/Changelog.md b/docs/Changelog.md index 0ffbef05..34b9affb 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -9,16 +9,46 @@ Want to stay in the loop on major new features? Join our mailing list by sending a mail to <afl-users+subscribe@googlegroups.com>. ### Version ++3.15a (dev) + - documentation restructuring, made possible by Google Season of Docs + - new binary-only fuzzing mode: coresight_mode for aarch64 CPUs :) + thanks to RICSecLab submitting! + - if instrumented libaries are dlopen()'ed after the forkserver you + will now see crashes. before you would have colliding coverage. + we changed this to force fixing a broken setup rather then allowing + ineffective fuzzing. + See docs/best_practices.md how to fix such setups. - afl-fuzz: - - added AFL_IGNORE_PROBLEMS plus checks to identify and abort on - incorrect LTO usage setups and enhanced the READMEs for better - information on how to deal with instrumenting libraries + - cmplog binaries will need to be recompiled for this version + (it is better!) - fix a regression introduced in 3.10 that resulted in less coverage being detected. thanks to Collin May for reporting! - + - added AFL_IGNORE_PROBLEMS, plus checks to identify and abort on + incorrect LTO usage setups and enhanced the READMEs for better + information on how to deal with instrumenting libraries + - fix -n dumb mode (nobody should use this) + - fix stability issue with LTO and cmplog + - better banner + - more effective cmplog mode + - more often update the UI when in input2stage mode + - frida_mode: + - better performance, bug fixes + - David Carlier added Android support :) + - afl-showmap, afl-tmin and afl-analyze: + - honor persistent mode for more speed. thanks to dloffre-snl + for reporting! + - fix bug where targets are not killed on timeouts + - moved hidden afl-showmap -A option to -H to be used for + coresight_mode + - Prevent accidently killing non-afl/fuzz services when aborting + afl-showmap and other tools. - afl-cc: + - new cmplog mode (incompatible with older afl++ versions) + - support llvm IR select instrumentation for default PCGUARD and LTO - fix for shared linking on MacOS - - llvm and LTO mode verified to work with new llvm 14-dev + - fixed a potential crash in targets for LAF string handling + - added AFL_USE_TSAN thread sanitizer support + - llvm and LTO mode modified to work with new llvm 14-dev (again) + - fix for AFL_REAL_LD - added the very good grammar mutator "GramaTron" to the custom_mutators - added optimin, a faster and better corpus minimizer by @@ -30,7 +60,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>. - fix AFL_PRELOAD issues on MacOS - removed utils/afl_frida because frida_mode/ is now so much better - added uninstall target to makefile (todo: update new readme!) - + - removed indirections in rust callbacks for unicornafl ### Version ++3.14c (release) - afl-fuzz: @@ -2748,7 +2778,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>. - Updated the documentation and added notes_for_asan.txt. Based on feedback from Hanno Boeck, Ben Laurie, and others. - - Moved the project to http://lcamtuf.coredump.cx/afl/. + - Moved the project to https://lcamtuf.coredump.cx/afl/. ### Version 0.46b: diff --git a/docs/FAQ.md b/docs/FAQ.md index 0e816062..7869ee61 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -1,243 +1,185 @@ -# Frequently asked questions about AFL++ - -## Contents - - * [What is the difference between AFL and AFL++?](#what-is-the-difference-between-afl-and-afl) - * [I got a weird compile error from clang](#i-got-a-weird-compile-error-from-clang) - * [How to improve the fuzzing speed?](#how-to-improve-the-fuzzing-speed) - * [How do I fuzz a network service?](#how-do-i-fuzz-a-network-service) - * [How do I fuzz a GUI program?](#how-do-i-fuzz-a-gui-program) - * [What is an edge?](#what-is-an-edge) - * [Why is my stability below 100%?](#why-is-my-stability-below-100) - * [How can I improve the stability value?](#how-can-i-improve-the-stability-value) +# Frequently asked questions (FAQ) If you find an interesting or important question missing, submit it via -[https://github.com/AFLplusplus/AFLplusplus/issues](https://github.com/AFLplusplus/AFLplusplus/issues) - -## What is the difference between AFL and AFL++? - -American Fuzzy Lop (AFL) was developed by Michał "lcamtuf" Zalewski starting in -2013/2014, and when he left Google end of 2017 he stopped developing it. - -At the end of 2019 the Google fuzzing team took over maintenance of AFL, however -it is only accepting PRs from the community and is not developing enhancements -anymore. - -In the second quarter of 2019, 1 1/2 year later when no further development of -AFL had happened and it became clear there would none be coming, AFL++ -was born, where initially community patches were collected and applied -for bug fixes and enhancements. Then from various AFL spin-offs - mostly academic -research - features were integrated. This already resulted in a much advanced -AFL. - -Until the end of 2019 the AFL++ team had grown to four active developers which -then implemented their own research and features, making it now by far the most -flexible and feature rich guided fuzzer available as open source. -And in independent fuzzing benchmarks it is one of the best fuzzers available, -e.g. [Fuzzbench Report](https://www.fuzzbench.com/reports/2020-08-03/index.html) - -## I got a weird compile error from clang - -If you see this kind of error when trying to instrument a target with afl-cc/ -afl-clang-fast/afl-clang-lto: -``` -/prg/tmp/llvm-project/build/bin/clang-13: symbol lookup error: /usr/local/bin/../lib/afl//cmplog-instructions-pass.so: undefined symbol: _ZNK4llvm8TypeSizecvmEv -clang-13: error: unable to execute command: No such file or directory -clang-13: error: clang frontend command failed due to signal (use -v to see invocation) -clang version 13.0.0 (https://github.com/llvm/llvm-project 1d7cf550721c51030144f3cd295c5789d51c4aad) -Target: x86_64-unknown-linux-gnu -Thread model: posix -InstalledDir: /prg/tmp/llvm-project/build/bin -clang-13: note: diagnostic msg: -******************** -``` -Then this means that your OS updated the clang installation from an upgrade -package and because of that the AFL++ llvm plugins do not match anymore. - -Solution: `git pull ; make clean install` of AFL++ - -## How to improve the fuzzing speed? - - 1. Use [llvm_mode](../instrumentation/README.llvm.md): afl-clang-lto (llvm >= 11) or afl-clang-fast (llvm >= 9 recommended) - 2. Use [persistent mode](../instrumentation/README.persistent_mode.md) (x2-x20 speed increase) - 3. Use the [AFL++ snapshot module](https://github.com/AFLplusplus/AFL-Snapshot-LKM) (x2 speed increase) - 4. If you do not use shmem persistent mode, use `AFL_TMPDIR` to put the input file directory on a tempfs location, see [docs/env_variables.md](docs/env_variables.md) - 5. Improve Linux kernel performance: modify `/etc/default/grub`, set `GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"`; then `update-grub` and `reboot` (warning: makes the system less secure) - 6. Running on an `ext2` filesystem with `noatime` mount option will be a bit faster than on any other journaling filesystem - 7. Use your cores! [README.md:3.b) Using multiple cores/threads](../README.md#b-using-multiple-coresthreads) - -## How do I fuzz a network service? - -The short answer is - you cannot, at least not "out of the box". - -Using a network channel is inadequate for several reasons: -- it has a slow-down of x10-20 on the fuzzing speed -- it does not scale to fuzzing multiple instances easily, -- instead of one initial data packet often a back-and-forth interplay of packets is needed for stateful protocols (which is totally unsupported by most coverage aware fuzzers). - -The established method to fuzz network services is to modify the source code -to read from a file or stdin (fd 0) (or even faster via shared memory, combine -this with persistent mode [instrumentation/README.persistent_mode.md](../instrumentation/README.persistent_mode.md) -and you have a performance gain of x10 instead of a performance loss of over -x10 - that is a x100 difference!). - -If modifying the source is not an option (e.g. because you only have a binary -and perform binary fuzzing) you can also use a shared library with AFL_PRELOAD -to emulate the network. This is also much faster than the real network would be. -See [utils/socket_fuzzing/](../utils/socket_fuzzing/). - -There is an outdated AFL++ branch that implements networking if you are -desperate though: [https://github.com/AFLplusplus/AFLplusplus/tree/networking](https://github.com/AFLplusplus/AFLplusplus/tree/networking) - -however a better option is AFLnet ([https://github.com/aflnet/aflnet](https://github.com/aflnet/aflnet)) -which allows you to define network state with different type of data packets. - -## How do I fuzz a GUI program? - -If the GUI program can read the fuzz data from a file (via the command line, -a fixed location or via an environment variable) without needing any user -interaction then it would be suitable for fuzzing. - -Otherwise it is not possible without modifying the source code - which is a -very good idea anyway as the GUI functionality is a huge CPU/time overhead -for the fuzzing. - -So create a new `main()` that just reads the test case and calls the -functionality for processing the input that the GUI program is using. - -## What is an "edge"? - -A program contains `functions`, `functions` contain the compiled machine code. -The compiled machine code in a `function` can be in a single or many `basic blocks`. -A `basic block` is the largest possible number of subsequent machine code -instructions that has exactly one entrypoint (which can be be entered by multiple other basic blocks) -and runs linearly without branching or jumping to other addresses (except at the end). -``` -function() { - A: - some - code - B: - if (x) goto C; else goto D; - C: - some code - goto E - D: - some code - goto B - E: - return -} -``` -Every code block between two jump locations is a `basic block`. - -An `edge` is then the unique relationship between two directly connected `basic blocks` (from the -code example above): -``` - Block A - | +[https://github.com/AFLplusplus/AFLplusplus/discussions](https://github.com/AFLplusplus/AFLplusplus/discussions). + +## General + +<details> + <summary id="what-is-the-difference-between-afl-and-aflplusplus">What is the difference between AFL and AFL++?</summary><p> + + AFL++ is a superior fork to Google's AFL - more speed, more and better + mutations, more and better instrumentation, custom module support, etc. + + American Fuzzy Lop (AFL) was developed by Michał "lcamtuf" Zalewski starting + in 2013/2014, and when he left Google end of 2017 he stopped developing it. + + At the end of 2019, the Google fuzzing team took over maintenance of AFL, + however, it is only accepting PRs from the community and is not developing + enhancements anymore. + + In the second quarter of 2019, 1 1/2 years later, when no further development + of AFL had happened and it became clear there would none be coming, AFL++ was + born, where initially community patches were collected and applied for bug + fixes and enhancements. Then from various AFL spin-offs - mostly academic + research - features were integrated. This already resulted in a much advanced + AFL. + + Until the end of 2019, the AFL++ team had grown to four active developers + which then implemented their own research and features, making it now by far + the most flexible and feature rich guided fuzzer available as open source. And + in independent fuzzing benchmarks it is one of the best fuzzers available, + e.g., [Fuzzbench + Report](https://www.fuzzbench.com/reports/2020-08-03/index.html). +</p></details> + +<details> + <summary id="where-can-i-find-tutorials">Where can I find tutorials?</summary><p> + + We compiled a list of tutorials and exercises, see + [tutorials.md](tutorials.md). +</p></details> + +<details> + <summary id="what-is-an-edge">What is an "edge"?</summary><p> + + A program contains `functions`, `functions` contain the compiled machine code. + The compiled machine code in a `function` can be in a single or many `basic + blocks`. A `basic block` is the largest possible number of subsequent machine + code instructions that has exactly one entry point (which can be be entered by + multiple other basic blocks) and runs linearly without branching or jumping to + other addresses (except at the end). + + ``` + function() { + A: + some + code + B: + if (x) goto C; else goto D; + C: + some code + goto E + D: + some code + goto B + E: + return + } + ``` + + Every code block between two jump locations is a `basic block`. + + An `edge` is then the unique relationship between two directly connected + `basic blocks` (from the code example above): + + ``` + Block A + | + v + Block B <------+ + / \ | + v v | + Block C Block D --+ + \ v - Block B <------+ - / \ | - v v | - Block C Block D --+ - \ - v - Block E -``` -Every line between two blocks is an `edge`. -Note that a few basic block loop to itself, this too would be an edge. - -## Why is my stability below 100%? - -Stability is measured by how many percent of the edges in the target are -"stable". Sending the same input again and again should take the exact same -path through the target every time. If that is the case, the stability is 100%. - -If however randomness happens, e.g. a thread reading other external data, -reaction to timing, etc. then in some of the re-executions with the same data -the edge coverage result will be different accross runs. -Those edges that change are then flagged "unstable". - -The more "unstable" edges, the more difficult for AFL++ to identify valid new -paths. - -A value above 90% is usually fine and a value above 80% is also still ok, and -even a value above 20% can still result in successful finds of bugs. -However, it is recommended that for values below 90% or 80% you should take -countermeasures to improve stability. - -## How can I improve the stability value? - -For fuzzing a 100% stable target that covers all edges is the best case. -A 90% stable target that covers all edges is however better than a 100% stable -target that ignores 10% of the edges. - -With instability you basically have a partial coverage loss on an edge, with -ignored functions you have a full loss on that edges. - -There are functions that are unstable, but also provide value to coverage, eg -init functions that use fuzz data as input for example. -If however a function that has nothing to do with the input data is the -source of instability, e.g. checking jitter, or is a hash map function etc. -then it should not be instrumented. - -To be able to exclude these functions (based on AFL++'s measured stability) -the following process will allow to identify functions with variable edges. - -Four steps are required to do this and it also requires quite some knowledge -of coding and/or disassembly and is effectively possible only with -afl-clang-fast PCGUARD and afl-clang-lto LTO instrumentation. - - 1. First step: Instrument to be able to find the responsible function(s). - - a) For LTO instrumented binaries this can be documented during compile - time, just set `export AFL_LLVM_DOCUMENT_IDS=/path/to/a/file`. - This file will have one assigned edge ID and the corresponding - function per line. - - b) For PCGUARD instrumented binaries it is much more difficult. Here you - can either modify the __sanitizer_cov_trace_pc_guard function in - instrumentation/afl-llvm-rt.o.c to write a backtrace to a file if the ID in - __afl_area_ptr[*guard] is one of the unstable edge IDs. - (Example code is already there). - Then recompile and reinstall llvm_mode and rebuild your target. - Run the recompiled target with afl-fuzz for a while and then check the - file that you wrote with the backtrace information. - Alternatively you can use `gdb` to hook __sanitizer_cov_trace_pc_guard_init - on start, check to which memory address the edge ID value is written - and set a write breakpoint to that address (`watch 0x.....`). - - c) in all other instrumentation types this is not possible. So just - recompile with the two mentioned above. This is just for - identifying the functions that have unstable edges. - - 2. Second step: Identify which edge ID numbers are unstable - - run the target with `export AFL_DEBUG=1` for a few minutes then terminate. - The out/fuzzer_stats file will then show the edge IDs that were identified - as unstable in the `var_bytes` entry. You can match these numbers - directly to the data you created in the first step. - Now you know which functions are responsible for the instability - - 3. Third step: create a text file with the filenames/functions - - Identify which source code files contain the functions that you need to - remove from instrumentation, or just specify the functions you want to - skip for instrumentation. Note that optimization might inline functions! - - Simply follow this document on how to do this: [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md) - If PCGUARD is used, then you need to follow this guide (needs llvm 12+!): - [http://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation](http://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation) - - Only exclude those functions from instrumentation that provide no value - for coverage - that is if it does not process any fuzz data directly - or indirectly (e.g. hash maps, thread management etc.). - If however a function directly or indirectly handles fuzz data then you - should not put the function in a deny instrumentation list and rather - live with the instability it comes with. - - 4. Fourth step: recompile the target - - Recompile, fuzz it, be happy :) - - This link explains this process for [Fuzzbench](https://github.com/google/fuzzbench/issues/677) + Block E + ``` + + Every line between two blocks is an `edge`. Note that a few basic block loop + to itself, this too would be an edge. +</p></details> + +## Targets + +<details> + <summary id="how-can-i-fuzz-a-binary-only-target">How can I fuzz a binary-only target?</summary><p> + + AFL++ is a great fuzzer if you have the source code available. + + However, if there is only the binary program and no source code available, + then the standard non-instrumented mode is not effective. + + To learn how these binaries can be fuzzed, read + [fuzzing_binary-only_targets.md](fuzzing_binary-only_targets.md). +</p></details> + +<details> + <summary id="how-can-i-fuzz-a-network-service">How can I fuzz a network service?</summary><p> + + The short answer is - you cannot, at least not "out of the box". + + For more information on fuzzing network services, see + [best_practices.md#fuzzing-a-network-service](best_practices.md#fuzzing-a-network-service). +</p></details> + +<details> + <summary id="how-can-i-fuzz-a-gui-program">How can I fuzz a GUI program?</summary><p> + + Not all GUI programs are suitable for fuzzing. If the GUI program can read the + fuzz data from a file without needing any user interaction, then it would be + suitable for fuzzing. + + For more information on fuzzing GUI programs, see + [best_practices.md#fuzzing-a-gui-program](best_practices.md#fuzzing-a-gui-program). +</p></details> + +## Performance + +<details> + <summary id="how-can-i-improve-the-fuzzing-speed">How can I improve the fuzzing speed?</summary><p> + + There are a few things you can do to improve the fuzzing speed, see + [best_practices.md#improving-speed](best_practices.md#improving-speed). +</p></details> + +<details> + <summary id="why-is-my-stability-below-100percent">Why is my stability below 100%?</summary><p> + + Stability is measured by how many percent of the edges in the target are + "stable". Sending the same input again and again should take the exact same + path through the target every time. If that is the case, the stability is + 100%. + + If, however, randomness happens, e.g., a thread reading other external data, + reaction to timing, etc., then in some of the re-executions with the same data + the edge coverage result will be different across runs. Those edges that + change are then flagged "unstable". + + The more "unstable" edges, the more difficult for AFL++ to identify valid new + paths. + + A value above 90% is usually fine and a value above 80% is also still ok, and + even a value above 20% can still result in successful finds of bugs. However, + it is recommended that for values below 90% or 80% you should take + countermeasures to improve stability. + + For more information on stability and how to improve the stability value, see + [best_practices.md#improving-stability](best_practices.md#improving-stability). +</p></details> + +## Troubleshooting + +<details> + <summary id="i-got-a-weird-compile-error-from-clang">I got a weird compile error from clang.</summary><p> + + If you see this kind of error when trying to instrument a target with + afl-cc/afl-clang-fast/afl-clang-lto: + + ``` + /prg/tmp/llvm-project/build/bin/clang-13: symbol lookup error: /usr/local/bin/../lib/afl//cmplog-instructions-pass.so: undefined symbol: _ZNK4llvm8TypeSizecvmEv + clang-13: error: unable to execute command: No such file or directory + clang-13: error: clang frontend command failed due to signal (use -v to see invocation) + clang version 13.0.0 (https://github.com/llvm/llvm-project 1d7cf550721c51030144f3cd295c5789d51c4aad) + Target: x86_64-unknown-linux-gnu + Thread model: posix + InstalledDir: /prg/tmp/llvm-project/build/bin + clang-13: note: diagnostic msg: + ******************** + ``` + + Then this means that your OS updated the clang installation from an upgrade + package and because of that the AFL++ llvm plugins do not match anymore. + + Solution: `git pull ; make clean install` of AFL++. +</p></details> \ No newline at end of file diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 17af532a..906d3f8e 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -1,82 +1,101 @@ -# Installation instructions +# Building and installing AFL++ - This document provides basic installation instructions and discusses known - issues for a variety of platforms. See README.md for the general instruction - manual. +## Linux on x86 -## 1. Linux on x86 ---------------- +An easy way to install AFL++ with everything compiled is available via docker: +You can use the [Dockerfile](../Dockerfile) (which has gcc-10 and clang-11 - +hence afl-clang-lto is available!) or just pull directly from the Docker Hub: -This platform is expected to work well. Compile the program with: - -```bash -make +```shell +docker pull aflplusplus/aflplusplus +docker run -ti -v /location/of/your/target:/src aflplusplus/aflplusplus ``` -You can start using the fuzzer without installation, but it is also possible to -install it with: - -```bash +This image is automatically generated when a push to the stable repo happens. +You will find your target source code in /src in the container. + +If you want to build AFL++ yourself, you have many options. The easiest choice +is to build and install everything: + +```shell +sudo apt-get update +sudo apt-get install -y build-essential python3-dev automake git flex bison libglib2.0-dev libpixman-1-dev python3-setuptools +# try to install llvm 11 and install the distro default if that fails +sudo apt-get install -y lld-11 llvm-11 llvm-11-dev clang-11 || sudo apt-get install -y lld llvm llvm-dev clang +sudo apt-get install -y gcc-$(gcc --version|head -n1|sed 's/.* //'|sed 's/\..*//')-plugin-dev libstdc++-$(gcc --version|head -n1|sed 's/.* //'|sed 's/\..*//')-dev +sudo apt-get install -y ninja-build # for QEMU mode +git clone https://github.com/AFLplusplus/AFLplusplus +cd AFLplusplus +make distrib sudo make install ``` -There are no special dependencies to speak of; you will need GNU make and a -working compiler (gcc or clang). Some of the optional scripts bundled with the -program may depend on bash, gdb, and similar basic tools. - -If you are using clang, please review README.llvm.md; the LLVM -integration mode can offer substantial performance gains compared to the -traditional approach. +It is recommended to install the newest available gcc, clang and llvm-dev +possible in your distribution! -Likewise, if you are using GCC, please review instrumentation/README.gcc_plugin.md. +Note that "make distrib" also builds instrumentation, QEMU mode, unicorn_mode +and more. If you just want plain AFL++, then do "make all". However, compiling +and using at least instrumentation is highly recommended for much better results +- hence in this case choose: -You may have to change several settings to get optimal results (most notably, -disable crash reporting utilities and switch to a different CPU governor), but -afl-fuzz will guide you through that if necessary. - -## 2. OpenBSD, FreeBSD, NetBSD on x86 - -Similarly to Linux, these platforms are expected to work well and are -regularly tested. Compile everything with GNU make: - -```bash -gmake +```shell +make source-only ``` -Note that BSD make will *not* work; if you do not have gmake on your system, -please install it first. As on Linux, you can use the fuzzer itself without -installation, or install it with: - -``` -sudo gmake install +These build targets exist: + +* all: just the main AFL++ binaries +* binary-only: everything for binary-only fuzzing: qemu_mode, unicorn_mode, + libdislocator, libtokencap +* source-only: everything for source code fuzzing: instrumentation, + libdislocator, libtokencap +* distrib: everything (for both binary-only and source code fuzzing) +* man: creates simple man pages from the help option of the programs +* install: installs everything you have compiled with the build options above +* clean: cleans everything compiled, not downloads (unless not on a checkout) +* deepclean: cleans everything including downloads +* code-format: format the code, do this before you commit and send a PR please! +* tests: runs test cases to ensure that all features are still working as they + should +* unit: perform unit tests (based on cmocka) +* help: shows these build options + +[Unless you are on Mac OS X](https://developer.apple.com/library/archive/qa/qa1118/_index.html), +you can also build statically linked versions of the AFL++ binaries by passing +the `STATIC=1` argument to make: + +```shell +make STATIC=1 ``` -Keep in mind that if you are using csh as your shell, the syntax of some of the -shell commands given in the README.md and other docs will be different. - -The `llvm` requires a dynamically linked, fully-operational installation of -clang. At least on FreeBSD, the clang binaries are static and do not include -some of the essential tools, so if you want to make it work, you may need to -follow the instructions in README.llvm.md. +These build options exist: -Beyond that, everything should work as advertised. +* STATIC - compile AFL++ static +* ASAN_BUILD - compiles with memory sanitizer for debug purposes +* DEBUG - no optimization, -ggdb3, all warnings and -Werror +* PROFILING - compile with profiling information (gprof) +* INTROSPECTION - compile afl-fuzz with mutation introspection +* NO_PYTHON - disable python support +* NO_SPLICING - disables splicing mutation in afl-fuzz, not recommended for + normal fuzzing +* AFL_NO_X86 - if compiling on non-intel/amd platforms +* LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config + (e.g., Debian) -The QEMU mode is currently supported only on Linux. I think it's just a QEMU -problem, I couldn't get a vanilla copy of user-mode emulation support working -correctly on BSD at all. +e.g.: `make ASAN_BUILD=1` -## 3. MacOS X on x86 and arm64 (M1) +## MacOS X on x86 and arm64 (M1) -MacOS X should work, but there are some gotchas due to the idiosyncrasies of -the platform. On top of this, I have limited release testing capabilities -and depend mostly on user feedback. +MacOS X should work, but there are some gotchas due to the idiosyncrasies of the +platform. On top of this, we have limited release testing capabilities and +depend mostly on user feedback. To build AFL, install llvm (and perhaps gcc) from brew and follow the general -instructions for Linux. If possible avoid Xcode at all cost. +instructions for Linux. If possible, avoid Xcode at all cost. `brew install wget git make cmake llvm gdb` -Be sure to setup PATH to point to the correct clang binaries and use the +Be sure to setup `PATH` to point to the correct clang binaries and use the freshly installed clang, clang++ and gmake, e.g.: ``` @@ -90,12 +109,13 @@ cd .. gmake install ``` -afl-gcc will fail unless you have GCC installed, but that is using outdated -instrumentation anyway. You don't want that. -Note that afl-clang-lto, afl-gcc-fast and qemu_mode are not working on MacOS. +`afl-gcc` will fail unless you have GCC installed, but that is using outdated +instrumentation anyway. You don't want that. Note that `afl-clang-lto`, +`afl-gcc-fast` and `qemu_mode` are not working on MacOS. The crash reporting daemon that comes by default with MacOS X will cause problems with fuzzing. You need to turn it off: + ``` launchctl unload -w /System/Library/LaunchAgents/com.apple.ReportCrash.plist sudo launchctl unload -w /System/Library/LaunchDaemons/com.apple.ReportCrash.Root.plist @@ -107,17 +127,17 @@ and definitely don't look POSIX-compliant. This means two things: - Fuzzing will be probably slower than on Linux. In fact, some folks report considerable performance gains by running the jobs inside a Linux VM on MacOS X. - - Some non-portable, platform-specific code may be incompatible with the - AFL forkserver. If you run into any problems, set `AFL_NO_FORKSRV=1` in the + - Some non-portable, platform-specific code may be incompatible with the AFL++ + forkserver. If you run into any problems, set `AFL_NO_FORKSRV=1` in the environment before starting afl-fuzz. User emulation mode of QEMU does not appear to be supported on MacOS X, so -black-box instrumentation mode (`-Q`) will not work. -However Frida mode (`-O`) should work on x86 and arm64 MacOS boxes. +black-box instrumentation mode (`-Q`) will not work. However, Frida mode (`-O`) +should work on x86 and arm64 MacOS boxes. MacOS X supports SYSV shared memory used by AFL's instrumentation, but the -default settings aren't usable with AFL++. The default settings on 10.14 seem -to be: +default settings aren't usable with AFL++. The default settings on 10.14 seem to +be: ```bash $ ipcs -M @@ -138,8 +158,8 @@ sysctl kern.sysv.shmmax=8388608 sysctl kern.sysv.shmall=4096 ``` -If you're running more than one instance of AFL you likely want to make `shmall` -bigger and increase `shmseg` as well: +If you're running more than one instance of AFL, you likely want to make +`shmall` bigger and increase `shmseg` as well: ```bash sysctl kern.sysv.shmmax=8388608 @@ -147,91 +167,6 @@ sysctl kern.sysv.shmseg=48 sysctl kern.sysv.shmall=98304 ``` -See http://www.spy-hill.com/help/apple/SharedMemory.html for documentation for -these settings and how to make them permanent. - -## 4. Linux or *BSD on non-x86 systems - -Standard build will fail on non-x86 systems, but you should be able to -leverage two other options: - - - The LLVM mode (see README.llvm.md), which does not rely on - x86-specific assembly shims. It's fast and robust, but requires a - complete installation of clang. - - The QEMU mode (see qemu_mode/README.md), which can be also used for - fuzzing cross-platform binaries. It's slower and more fragile, but - can be used even when you don't have the source for the tested app. - -If you're not sure what you need, you need the LLVM mode, which is built by -default. - -...and compile your target program with afl-clang-fast or afl-clang-fast++ -instead of the traditional afl-gcc or afl-clang wrappers. - -## 5. Solaris on x86 - -The fuzzer reportedly works on Solaris, but I have not tested this first-hand, -and the user base is fairly small, so I don't have a lot of feedback. - -To get the ball rolling, you will need to use GNU make and GCC or clang. I'm -being told that the stock version of GCC that comes with the platform does not -work properly due to its reliance on a hardcoded location for 'as' (completely -ignoring the `-B` parameter or `$PATH`). - -To fix this, you may want to build stock GCC from the source, like so: - -```sh -./configure --prefix=$HOME/gcc --with-gnu-as --with-gnu-ld \ - --with-gmp-include=/usr/include/gmp --with-mpfr-include=/usr/include/mpfr -make -sudo make install -``` - -Do *not* specify `--with-as=/usr/gnu/bin/as` - this will produce a GCC binary that -ignores the `-B` flag and you will be back to square one. - -Note that Solaris reportedly comes with crash reporting enabled, which causes -problems with crashes being misinterpreted as hangs, similarly to the gotchas -for Linux and MacOS X. AFL does not auto-detect crash reporting on this -particular platform, but you may need to run the following command: - -```sh -coreadm -d global -d global-setid -d process -d proc-setid \ - -d kzone -d log -``` - -User emulation mode of QEMU is not available on Solaris, so black-box -instrumentation mode (`-Q`) will not work. - -## 6. Everything else - -You're on your own. On POSIX-compliant systems, you may be able to compile and -run the fuzzer; and the LLVM and GCC plugin modes may offer a way to instrument -non-x86 code. - -The fuzzer will run on Windows in WSL only. It will not work under Cygwin on in the normal Windows world. It -could be ported to the latter platform fairly easily, but it's a pretty bad -idea, because Cygwin is extremely slow. It makes much more sense to use -VirtualBox or so to run a hardware-accelerated Linux VM; it will run around -20x faster or so. If you have a *really* compelling use case for Cygwin, let -me know. - -Although Android on x86 should theoretically work, the stock kernel may have -SHM support compiled out, and if so, you may have to address that issue first. -It's possible that all you need is this workaround: - - https://github.com/pelya/android-shmem - -Joshua J. Drake notes that the Android linker adds a shim that automatically -intercepts `SIGSEGV` and related signals. To fix this issue and be able to see -crashes, you need to put this at the beginning of the fuzzed program: - -```sh - signal(SIGILL, SIG_DFL); - signal(SIGABRT, SIG_DFL); - signal(SIGBUS, SIG_DFL); - signal(SIGFPE, SIG_DFL); - signal(SIGSEGV, SIG_DFL); -``` - -You may need to `#include <signal.h>` first. +See +[http://www.spy-hill.com/help/apple/SharedMemory.html](http://www.spy-hill.com/help/apple/SharedMemory.html) +for documentation for these settings and how to make them permanent. \ No newline at end of file diff --git a/docs/QuickStartGuide.md b/docs/QuickStartGuide.md deleted file mode 100644 index 2d056ecf..00000000 --- a/docs/QuickStartGuide.md +++ /dev/null @@ -1,50 +0,0 @@ -# AFL quick start guide - -You should read [README.md](../README.md) - it's pretty short. If you really can't, here's -how to hit the ground running: - -1) Compile AFL with 'make'. If build fails, see [INSTALL.md](INSTALL.md) for tips. - -2) Find or write a reasonably fast and simple program that takes data from - a file or stdin, processes it in a test-worthy way, then exits cleanly. - If testing a network service, modify it to run in the foreground and read - from stdin. When fuzzing a format that uses checksums, comment out the - checksum verification code, too. - - If this is not possible (e.g. in -Q(emu) mode) then use - AFL_CUSTOM_MUTATOR_LIBRARY to calculate the values with your own library. - - The program must crash properly when a fault is encountered. Watch out for - custom SIGSEGV or SIGABRT handlers and background processes. For tips on - detecting non-crashing flaws, see section 11 in [README.md](README.md) . - -3) Compile the program / library to be fuzzed using afl-cc. A common way to - do this would be: - - CC=/path/to/afl-cc CXX=/path/to/afl-c++ ./configure --disable-shared - make clean all - -4) Get a small but valid input file that makes sense to the program. When - fuzzing verbose syntax (SQL, HTTP, etc), create a dictionary as described in - dictionaries/README.md, too. - -5) If the program reads from stdin, run 'afl-fuzz' like so: - - ./afl-fuzz -i testcase_dir -o findings_dir -- \ - /path/to/tested/program [...program's cmdline...] - - If the program takes input from a file, you can put @@ in the program's - command line; AFL will put an auto-generated file name in there for you. - -6) Investigate anything shown in red in the fuzzer UI by promptly consulting - [status_screen.md](status_screen.md). - -8) There is a basic docker build with 'docker build -t aflplusplus .' - -That's it. Sit back, relax, and - time permitting - try to skim through the -following files: - - - README.md - A general introduction to AFL, - - docs/perf_tips.md - Simple tips on how to fuzz more quickly, - - docs/status_screen.md - An explanation of the tidbits shown in the UI, - - docs/parallel_fuzzing.md - Advice on running AFL on multiple cores. diff --git a/docs/afl-fuzz_approach.md b/docs/afl-fuzz_approach.md new file mode 100644 index 00000000..2da61cc4 --- /dev/null +++ b/docs/afl-fuzz_approach.md @@ -0,0 +1,543 @@ +# The afl-fuzz approach + +AFL++ is a brute-force fuzzer coupled with an exceedingly simple but rock-solid +instrumentation-guided genetic algorithm. It uses a modified form of edge +coverage to effortlessly pick up subtle, local-scale changes to program control +flow. + +Simplifying a bit, the overall algorithm can be summed up as: + +1) Load user-supplied initial test cases into the queue. + +2) Take the next input file from the queue. + +3) Attempt to trim the test case to the smallest size that doesn't alter the + measured behavior of the program. + +4) Repeatedly mutate the file using a balanced and well-researched variety of + traditional fuzzing strategies. + +5) If any of the generated mutations resulted in a new state transition recorded + by the instrumentation, add mutated output as a new entry in the queue. + +6) Go to 2. + +The discovered test cases are also periodically culled to eliminate ones that +have been obsoleted by newer, higher-coverage finds; and undergo several other +instrumentation-driven effort minimization steps. + +As a side result of the fuzzing process, the tool creates a small, +self-contained corpus of interesting test cases. These are extremely useful for +seeding other, labor- or resource-intensive testing regimes - for example, for +stress-testing browsers, office applications, graphics suites, or closed-source +tools. + +The fuzzer is thoroughly tested to deliver out-of-the-box performance far +superior to blind fuzzing or coverage-only tools. + +## Understanding the status screen + +This section provides an overview of the status screen - plus tips for +troubleshooting any warnings and red text shown in the UI. + +For the general instruction manual, see [README.md](../README.md). + +### A note about colors + +The status screen and error messages use colors to keep things readable and +attract your attention to the most important details. For example, red almost +always means "consult this doc" :-) + +Unfortunately, the UI will only render correctly if your terminal is using +traditional un*x palette (white text on black background) or something close to +that. + +If you are using inverse video, you may want to change your settings, say: + +- For GNOME Terminal, go to `Edit > Profile` preferences, select the "colors" + tab, and from the list of built-in schemes, choose "white on black". +- For the MacOS X Terminal app, open a new window using the "Pro" scheme via the + `Shell > New Window` menu (or make "Pro" your default). + +Alternatively, if you really like your current colors, you can edit config.h to +comment out USE_COLORS, then do `make clean all`. + +We are not aware of any other simple way to make this work without causing other +side effects - sorry about that. + +With that out of the way, let's talk about what's actually on the screen... + +### The status bar + +``` +american fuzzy lop ++3.01a (default) [fast] {0} +``` + +The top line shows you which mode afl-fuzz is running in (normal: "american +fuzzy lop", crash exploration mode: "peruvian rabbit mode") and the version of +AFL++. Next to the version is the banner, which, if not set with -T by hand, +will either show the binary name being fuzzed, or the -M/-S main/secondary name +for parallel fuzzing. Second to last is the power schedule mode being run +(default: fast). Finally, the last item is the CPU id. + +### Process timing + +``` + +----------------------------------------------------+ + | run time : 0 days, 8 hrs, 32 min, 43 sec | + | last new path : 0 days, 0 hrs, 6 min, 40 sec | + | last uniq crash : none seen yet | + | last uniq hang : 0 days, 1 hrs, 24 min, 32 sec | + +----------------------------------------------------+ +``` + +This section is fairly self-explanatory: it tells you how long the fuzzer has +been running and how much time has elapsed since its most recent finds. This is +broken down into "paths" (a shorthand for test cases that trigger new execution +patterns), crashes, and hangs. + +When it comes to timing: there is no hard rule, but most fuzzing jobs should be +expected to run for days or weeks; in fact, for a moderately complex project, +the first pass will probably take a day or so. Every now and then, some jobs +will be allowed to run for months. + +There's one important thing to watch out for: if the tool is not finding new +paths within several minutes of starting, you're probably not invoking the +target binary correctly and it never gets to parse the input files that are +thrown at it; other possible explanations are that the default memory limit +(`-m`) is too restrictive and the program exits after failing to allocate a +buffer very early on; or that the input files are patently invalid and always +fail a basic header check. + +If there are no new paths showing up for a while, you will eventually see a big +red warning in this section, too :-) + +### Overall results + +``` + +-----------------------+ + | cycles done : 0 | + | total paths : 2095 | + | uniq crashes : 0 | + | uniq hangs : 19 | + +-----------------------+ +``` + +The first field in this section gives you the count of queue passes done so far +- that is, the number of times the fuzzer went over all the interesting test + cases discovered so far, fuzzed them, and looped back to the very beginning. + Every fuzzing session should be allowed to complete at least one cycle; and + ideally, should run much longer than that. + +As noted earlier, the first pass can take a day or longer, so sit back and +relax. + +To help make the call on when to hit `Ctrl-C`, the cycle counter is color-coded. +It is shown in magenta during the first pass, progresses to yellow if new finds +are still being made in subsequent rounds, then blue when that ends - and +finally, turns green after the fuzzer hasn't been seeing any action for a longer +while. + +The remaining fields in this part of the screen should be pretty obvious: +there's the number of test cases ("paths") discovered so far, and the number of +unique faults. The test cases, crashes, and hangs can be explored in real-time +by browsing the output directory, see +[#interpreting-output](#interpreting-output). + +### Cycle progress + +``` + +-------------------------------------+ + | now processing : 1296 (61.86%) | + | paths timed out : 0 (0.00%) | + +-------------------------------------+ +``` + +This box tells you how far along the fuzzer is with the current queue cycle: it +shows the ID of the test case it is currently working on, plus the number of +inputs it decided to ditch because they were persistently timing out. + +The "*" suffix sometimes shown in the first line means that the currently +processed path is not "favored" (a property discussed later on). + +### Map coverage + +``` + +--------------------------------------+ + | map density : 10.15% / 29.07% | + | count coverage : 4.03 bits/tuple | + +--------------------------------------+ +``` + +The section provides some trivia about the coverage observed by the +instrumentation embedded in the target binary. + +The first line in the box tells you how many branch tuples already were hit, in +proportion to how much the bitmap can hold. The number on the left describes the +current input; the one on the right is the value for the entire input corpus. + +Be wary of extremes: + +- Absolute numbers below 200 or so suggest one of three things: that the program + is extremely simple; that it is not instrumented properly (e.g., due to being + linked against a non-instrumented copy of the target library); or that it is + bailing out prematurely on your input test cases. The fuzzer will try to mark + this in pink, just to make you aware. +- Percentages over 70% may very rarely happen with very complex programs that + make heavy use of template-generated code. Because high bitmap density makes + it harder for the fuzzer to reliably discern new program states, we recommend + recompiling the binary with `AFL_INST_RATIO=10` or so and trying again (see + [env_variables.md](env_variables.md)). The fuzzer will flag high percentages + in red. Chances are, you will never see that unless you're fuzzing extremely + hairy software (say, v8, perl, ffmpeg). + +The other line deals with the variability in tuple hit counts seen in the +binary. In essence, if every taken branch is always taken a fixed number of +times for all the inputs that were tried, this will read `1.00`. As we manage to +trigger other hit counts for every branch, the needle will start to move toward +`8.00` (every bit in the 8-bit map hit), but will probably never reach that +extreme. + +Together, the values can be useful for comparing the coverage of several +different fuzzing jobs that rely on the same instrumented binary. + +### Stage progress + +``` + +-------------------------------------+ + | now trying : interest 32/8 | + | stage execs : 3996/34.4k (11.62%) | + | total execs : 27.4M | + | exec speed : 891.7/sec | + +-------------------------------------+ +``` + +This part gives you an in-depth peek at what the fuzzer is actually doing right +now. It tells you about the current stage, which can be any of: + +- calibration - a pre-fuzzing stage where the execution path is examined to + detect anomalies, establish baseline execution speed, and so on. Executed very + briefly whenever a new find is being made. +- trim L/S - another pre-fuzzing stage where the test case is trimmed to the + shortest form that still produces the same execution path. The length (L) and + stepover (S) are chosen in general relationship to file size. +- bitflip L/S - deterministic bit flips. There are L bits toggled at any given + time, walking the input file with S-bit increments. The current L/S variants + are: `1/1`, `2/1`, `4/1`, `8/8`, `16/8`, `32/8`. +- arith L/8 - deterministic arithmetics. The fuzzer tries to subtract or add + small integers to 8-, 16-, and 32-bit values. The stepover is always 8 bits. +- interest L/8 - deterministic value overwrite. The fuzzer has a list of known + "interesting" 8-, 16-, and 32-bit values to try. The stepover is 8 bits. +- extras - deterministic injection of dictionary terms. This can be shown as + "user" or "auto", depending on whether the fuzzer is using a user-supplied + dictionary (`-x`) or an auto-created one. You will also see "over" or + "insert", depending on whether the dictionary words overwrite existing data or + are inserted by offsetting the remaining data to accommodate their length. +- havoc - a sort-of-fixed-length cycle with stacked random tweaks. The + operations attempted during this stage include bit flips, overwrites with + random and "interesting" integers, block deletion, block duplication, plus + assorted dictionary-related operations (if a dictionary is supplied in the + first place). +- splice - a last-resort strategy that kicks in after the first full queue cycle + with no new paths. It is equivalent to 'havoc', except that it first splices + together two random inputs from the queue at some arbitrarily selected + midpoint. +- sync - a stage used only when `-M` or `-S` is set (see + [fuzzing_in_depth.md:3c) Using multiple cores](fuzzing_in_depth.md#c-using-multiple-cores)). + No real fuzzing is involved, but the tool scans the output from other fuzzers + and imports test cases as necessary. The first time this is done, it may take + several minutes or so. + +The remaining fields should be fairly self-evident: there's the exec count +progress indicator for the current stage, a global exec counter, and a benchmark +for the current program execution speed. This may fluctuate from one test case +to another, but the benchmark should be ideally over 500 execs/sec most of the +time - and if it stays below 100, the job will probably take very long. + +The fuzzer will explicitly warn you about slow targets, too. If this happens, +see the [best_practices.md#improving-speed](best_practices.md#improving-speed) +for ideas on how to speed things up. + +### Findings in depth + +``` + +--------------------------------------+ + | favored paths : 879 (41.96%) | + | new edges on : 423 (20.19%) | + | total crashes : 0 (0 unique) | + | total tmouts : 24 (19 unique) | + +--------------------------------------+ +``` + +This gives you several metrics that are of interest mostly to complete nerds. +The section includes the number of paths that the fuzzer likes the most based on +a minimization algorithm baked into the code (these will get considerably more +air time), and the number of test cases that actually resulted in better edge +coverage (versus just pushing the branch hit counters up). There are also +additional, more detailed counters for crashes and timeouts. + +Note that the timeout counter is somewhat different from the hang counter; this +one includes all test cases that exceeded the timeout, even if they did not +exceed it by a margin sufficient to be classified as hangs. + +### Fuzzing strategy yields + +``` + +-----------------------------------------------------+ + | bit flips : 57/289k, 18/289k, 18/288k | + | byte flips : 0/36.2k, 4/35.7k, 7/34.6k | + | arithmetics : 53/2.54M, 0/537k, 0/55.2k | + | known ints : 8/322k, 12/1.32M, 10/1.70M | + | dictionary : 9/52k, 1/53k, 1/24k | + |havoc/splice : 1903/20.0M, 0/0 | + |py/custom/rq : unused, 53/2.54M, unused | + | trim/eff : 20.31%/9201, 17.05% | + +-----------------------------------------------------+ +``` + +This is just another nerd-targeted section keeping track of how many paths were +netted, in proportion to the number of execs attempted, for each of the fuzzing +strategies discussed earlier on. This serves to convincingly validate +assumptions about the usefulness of the various approaches taken by afl-fuzz. + +The trim strategy stats in this section are a bit different than the rest. The +first number in this line shows the ratio of bytes removed from the input files; +the second one corresponds to the number of execs needed to achieve this goal. +Finally, the third number shows the proportion of bytes that, although not +possible to remove, were deemed to have no effect and were excluded from some of +the more expensive deterministic fuzzing steps. + +Note that when deterministic mutation mode is off (which is the default because +it is not very efficient) the first five lines display "disabled (default, +enable with -D)". + +Only what is activated will have counter shown. + +### Path geometry + +``` + +---------------------+ + | levels : 5 | + | pending : 1570 | + | pend fav : 583 | + | own finds : 0 | + | imported : 0 | + | stability : 100.00% | + +---------------------+ +``` + +The first field in this section tracks the path depth reached through the guided +fuzzing process. In essence: the initial test cases supplied by the user are +considered "level 1". The test cases that can be derived from that through +traditional fuzzing are considered "level 2"; the ones derived by using these as +inputs to subsequent fuzzing rounds are "level 3"; and so forth. The maximum +depth is therefore a rough proxy for how much value you're getting out of the +instrumentation-guided approach taken by afl-fuzz. + +The next field shows you the number of inputs that have not gone through any +fuzzing yet. The same stat is also given for "favored" entries that the fuzzer +really wants to get to in this queue cycle (the non-favored entries may have to +wait a couple of cycles to get their chance). + +Next is the number of new paths found during this fuzzing section and imported +from other fuzzer instances when doing parallelized fuzzing; and the extent to +which identical inputs appear to sometimes produce variable behavior in the +tested binary. + +That last bit is actually fairly interesting: it measures the consistency of +observed traces. If a program always behaves the same for the same input data, +it will earn a score of 100%. When the value is lower but still shown in purple, +the fuzzing process is unlikely to be negatively affected. If it goes into red, +you may be in trouble, since AFL++ will have difficulty discerning between +meaningful and "phantom" effects of tweaking the input file. + +Now, most targets will just get a 100% score, but when you see lower figures, +there are several things to look at: + +- The use of uninitialized memory in conjunction with some intrinsic sources of + entropy in the tested binary. Harmless to AFL, but could be indicative of a + security bug. +- Attempts to manipulate persistent resources, such as left over temporary files + or shared memory objects. This is usually harmless, but you may want to + double-check to make sure the program isn't bailing out prematurely. Running + out of disk space, SHM handles, or other global resources can trigger this, + too. +- Hitting some functionality that is actually designed to behave randomly. + Generally harmless. For example, when fuzzing sqlite, an input like `select + random();` will trigger a variable execution path. +- Multiple threads executing at once in semi-random order. This is harmless when + the 'stability' metric stays over 90% or so, but can become an issue if not. + Here's what to try: + * Use afl-clang-fast from [instrumentation](../instrumentation/) - it uses a + thread-local tracking model that is less prone to concurrency issues, + * See if the target can be compiled or run without threads. Common + `./configure` options include `--without-threads`, `--disable-pthreads`, or + `--disable-openmp`. + * Replace pthreads with GNU Pth (https://www.gnu.org/software/pth/), which + allows you to use a deterministic scheduler. +- In persistent mode, minor drops in the "stability" metric can be normal, + because not all the code behaves identically when re-entered; but major dips + may signify that the code within `__AFL_LOOP()` is not behaving correctly on + subsequent iterations (e.g., due to incomplete clean-up or reinitialization of + the state) and that most of the fuzzing effort goes to waste. + +The paths where variable behavior is detected are marked with a matching entry +in the `<out_dir>/queue/.state/variable_behavior/` directory, so you can look +them up easily. + +### CPU load + +``` + [cpu: 25%] +``` + +This tiny widget shows the apparent CPU utilization on the local system. It is +calculated by taking the number of processes in the "runnable" state, and then +comparing it to the number of logical cores on the system. + +If the value is shown in green, you are using fewer CPU cores than available on +your system and can probably parallelize to improve performance; for tips on how +to do that, see +[fuzzing_in_depth.md:3c) Using multiple cores](fuzzing_in_depth.md#c-using-multiple-cores). + +If the value is shown in red, your CPU is *possibly* oversubscribed, and running +additional fuzzers may not give you any benefits. + +Of course, this benchmark is very simplistic; it tells you how many processes +are ready to run, but not how resource-hungry they may be. It also doesn't +distinguish between physical cores, logical cores, and virtualized CPUs; the +performance characteristics of each of these will differ quite a bit. + +If you want a more accurate measurement, you can run the `afl-gotcpu` utility +from the command line. + +## Interpreting output + +See [#understanding-the-status-screen](#understanding-the-status-screen) for +information on how to interpret the displayed stats and monitor the health of +the process. Be sure to consult this file especially if any UI elements are +highlighted in red. + +The fuzzing process will continue until you press Ctrl-C. At a minimum, you want +to allow the fuzzer to complete one queue cycle, which may take anywhere from a +couple of hours to a week or so. + +There are three subdirectories created within the output directory and updated +in real-time: + +- queue/ - test cases for every distinctive execution path, plus all the + starting files given by the user. This is the synthesized corpus. + + Before using this corpus for any other purposes, you can shrink + it to a smaller size using the afl-cmin tool. The tool will find + a smaller subset of files offering equivalent edge coverage. + +- crashes/ - unique test cases that cause the tested program to receive a fatal + signal (e.g., SIGSEGV, SIGILL, SIGABRT). The entries are grouped by + the received signal. + +- hangs/ - unique test cases that cause the tested program to time out. The + default time limit before something is classified as a hang is the + larger of 1 second and the value of the -t parameter. The value can + be fine-tuned by setting AFL_HANG_TMOUT, but this is rarely + necessary. + +Crashes and hangs are considered "unique" if the associated execution paths +involve any state transitions not seen in previously-recorded faults. If a +single bug can be reached in multiple ways, there will be some count inflation +early in the process, but this should quickly taper off. + +The file names for crashes and hangs are correlated with the parent, +non-faulting queue entries. This should help with debugging. + +## Visualizing + +If you have gnuplot installed, you can also generate some pretty graphs for any +active fuzzing task using afl-plot. For an example of how this looks like, see +[https://lcamtuf.coredump.cx/afl/plot/](https://lcamtuf.coredump.cx/afl/plot/). + +You can also manually build and install afl-plot-ui, which is a helper utility +for showing the graphs generated by afl-plot in a graphical window using GTK. +You can build and install it as follows: + +```shell +sudo apt install libgtk-3-0 libgtk-3-dev pkg-config +cd utils/plot_ui +make +cd ../../ +sudo make install +``` + +To learn more about remote monitoring and metrics visualization with StatsD, see +[rpc_statsd.md](rpc_statsd.md). + +### Addendum: status and plot files + +For unattended operation, some of the key status screen information can be also +found in a machine-readable format in the fuzzer_stats file in the output +directory. This includes: + +- `start_time` - unix time indicating the start time of afl-fuzz +- `last_update` - unix time corresponding to the last update of this file +- `run_time` - run time in seconds to the last update of this file +- `fuzzer_pid` - PID of the fuzzer process +- `cycles_done` - queue cycles completed so far +- `cycles_wo_finds` - number of cycles without any new paths found +- `execs_done` - number of execve() calls attempted +- `execs_per_sec` - overall number of execs per second +- `paths_total` - total number of entries in the queue +- `paths_favored` - number of queue entries that are favored +- `paths_found` - number of entries discovered through local fuzzing +- `paths_imported` - number of entries imported from other instances +- `max_depth` - number of levels in the generated data set +- `cur_path` - currently processed entry number +- `pending_favs` - number of favored entries still waiting to be fuzzed +- `pending_total` - number of all entries waiting to be fuzzed +- `variable_paths` - number of test cases showing variable behavior +- `stability` - percentage of bitmap bytes that behave consistently +- `bitmap_cvg` - percentage of edge coverage found in the map so far +- `unique_crashes` - number of unique crashes recorded +- `unique_hangs` - number of unique hangs encountered +- `last_path` - seconds since the last path was found +- `last_crash` - seconds since the last crash was found +- `last_hang` - seconds since the last hang was found +- `execs_since_crash` - execs since the last crash was found +- `exec_timeout` - the -t command line value +- `slowest_exec_ms` - real time of the slowest execution in ms +- `peak_rss_mb` - max rss usage reached during fuzzing in MB +- `edges_found` - how many edges have been found +- `var_byte_count` - how many edges are non-deterministic +- `afl_banner` - banner text (e.g., the target name) +- `afl_version` - the version of AFL++ used +- `target_mode` - default, persistent, qemu, unicorn, non-instrumented +- `command_line` - full command line used for the fuzzing session + +Most of these map directly to the UI elements discussed earlier on. + +On top of that, you can also find an entry called `plot_data`, containing a +plottable history for most of these fields. If you have gnuplot installed, you +can turn this into a nice progress report with the included `afl-plot` tool. + +### Addendum: automatically sending metrics with StatsD + +In a CI environment or when running multiple fuzzers, it can be tedious to log +into each of them or deploy scripts to read the fuzzer statistics. Using +`AFL_STATSD` (and the other related environment variables `AFL_STATSD_HOST`, +`AFL_STATSD_PORT`, `AFL_STATSD_TAGS_FLAVOR`) you can automatically send metrics +to your favorite StatsD server. Depending on your StatsD server, you will be +able to monitor, trigger alerts, or perform actions based on these metrics +(e.g.: alert on slow exec/s for a new build, threshold of crashes, time since +last crash > X, etc.). + +The selected metrics are a subset of all the metrics found in the status and in +the plot file. The list is the following: `cycle_done`, `cycles_wo_finds`, +`execs_done`,`execs_per_sec`, `paths_total`, `paths_favored`, `paths_found`, +`paths_imported`, `max_depth`, `cur_path`, `pending_favs`, `pending_total`, +`variable_paths`, `unique_crashes`, `unique_hangs`, `total_crashes`, +`slowest_exec_ms`, `edges_found`, `var_byte_count`, `havoc_expansion`. Their +definitions can be found in the addendum above. + +When using multiple fuzzer instances with StatsD, it is *strongly* recommended +to setup the flavor (`AFL_STATSD_TAGS_FLAVOR`) to match your StatsD server. This +will allow you to see individual fuzzer performance, detect bad ones, see the +progress of each strategy... \ No newline at end of file diff --git a/docs/best_practices.md b/docs/best_practices.md new file mode 100644 index 00000000..96c6e3c2 --- /dev/null +++ b/docs/best_practices.md @@ -0,0 +1,192 @@ +# Best practices + +## Contents + +### Targets + +* [Fuzzing a target with source code available](#fuzzing-a-target-with-source-code-available) +* [Fuzzing a target with dlopen() instrumented libraries](#fuzzing-a-target-with-dlopen-instrumented-libraries) +* [Fuzzing a binary-only target](#fuzzing-a-binary-only-target) +* [Fuzzing a GUI program](#fuzzing-a-gui-program) +* [Fuzzing a network service](#fuzzing-a-network-service) + +### Improvements + +* [Improving speed](#improving-speed) +* [Improving stability](#improving-stability) + +## Targets + +### Fuzzing a target with source code available + +To learn how to fuzz a target if source code is available, see +[fuzzing_in_depth.md](fuzzing_in_depth.md). + +### Fuzzing a target with dlopen instrumented libraries + +If a source code based fuzzing target loads instrumented libraries with +dlopen() after the forkserver has been activated and non-colliding coverage +instrumentation is used (PCGUARD (which is the default), or LTO), then this +an issue, because this would enlarge the coverage map, but afl-fuzz doesn't +know about it. + +The solution is to use `AFL_PRELOAD` for all dlopen()'ed libraries to +ensure that all coverage targets are present on startup in the target, +even if accessed only later with dlopen(). + +For PCGUARD instrumentation `abort()` is called if this is detected, for LTO +there will either be no coverage for the instrumented dlopen()'ed libraries or +you will see lots of crashes in the UI. + +Note that this is not an issue if you use the inferiour `afl-gcc-fast`, +`afl-gcc` or`AFL_LLVM_INSTRUMENT=CLASSIC/NGRAM/CTX afl-clang-fast` +instrumentation. + +### Fuzzing a binary-only target + +For a comprehensive guide, see +[fuzzing_binary-only_targets.md](fuzzing_binary-only_targets.md). + +### Fuzzing a GUI program + +If the GUI program can read the fuzz data from a file (via the command line, a +fixed location or via an environment variable) without needing any user +interaction, then it would be suitable for fuzzing. + +Otherwise, it is not possible without modifying the source code - which is a +very good idea anyway as the GUI functionality is a huge CPU/time overhead for +the fuzzing. + +So create a new `main()` that just reads the test case and calls the +functionality for processing the input that the GUI program is using. + +### Fuzzing a network service + +Fuzzing a network service does not work "out of the box". + +Using a network channel is inadequate for several reasons: +- it has a slow-down of x10-20 on the fuzzing speed +- it does not scale to fuzzing multiple instances easily, +- instead of one initial data packet often a back-and-forth interplay of packets + is needed for stateful protocols (which is totally unsupported by most + coverage aware fuzzers). + +The established method to fuzz network services is to modify the source code to +read from a file or stdin (fd 0) (or even faster via shared memory, combine this +with persistent mode +[instrumentation/README.persistent_mode.md](../instrumentation/README.persistent_mode.md) +and you have a performance gain of x10 instead of a performance loss of over x10 +- that is a x100 difference!). + +If modifying the source is not an option (e.g., because you only have a binary +and perform binary fuzzing) you can also use a shared library with AFL_PRELOAD +to emulate the network. This is also much faster than the real network would be. +See [utils/socket_fuzzing/](../utils/socket_fuzzing/). + +There is an outdated AFL++ branch that implements networking if you are +desperate though: +[https://github.com/AFLplusplus/AFLplusplus/tree/networking](https://github.com/AFLplusplus/AFLplusplus/tree/networking) +- however, a better option is AFLnet +([https://github.com/aflnet/aflnet](https://github.com/aflnet/aflnet)) which +allows you to define network state with different type of data packets. + +## Improvements + +### Improving speed + +1. Use [llvm_mode](../instrumentation/README.llvm.md): afl-clang-lto (llvm >= + 11) or afl-clang-fast (llvm >= 9 recommended). +2. Use [persistent mode](../instrumentation/README.persistent_mode.md) (x2-x20 + speed increase). +3. Instrument just what you are interested in, see + [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md). +4. If you do not use shmem persistent mode, use `AFL_TMPDIR` to put the input + file directory on a tempfs location, see + [env_variables.md](env_variables.md). +5. Improve Linux kernel performance: modify `/etc/default/grub`, set + `GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off + mitigations=off no_stf_barrier noibpb noibrs nopcid nopti + nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off + spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"`; then + `update-grub` and `reboot` (warning: makes the system less secure). +6. Running on an `ext2` filesystem with `noatime` mount option will be a bit + faster than on any other journaling filesystem. +7. Use your cores + ([fuzzing_in_depth.md:3c) Using multiple cores](fuzzing_in_depth.md#c-using-multiple-cores))! + +### Improving stability + +For fuzzing a 100% stable target that covers all edges is the best case. A 90% +stable target that covers all edges is, however, better than a 100% stable +target that ignores 10% of the edges. + +With instability, you basically have a partial coverage loss on an edge, with +ignored functions you have a full loss on that edges. + +There are functions that are unstable, but also provide value to coverage, e.g., +init functions that use fuzz data as input. If, however, a function that has +nothing to do with the input data is the source of instability, e.g., checking +jitter, or is a hash map function etc., then it should not be instrumented. + +To be able to exclude these functions (based on AFL++'s measured stability), the +following process will allow to identify functions with variable edges. + +Four steps are required to do this and it also requires quite some knowledge of +coding and/or disassembly and is effectively possible only with `afl-clang-fast` +`PCGUARD` and `afl-clang-lto` `LTO` instrumentation. + + 1. Instrument to be able to find the responsible function(s): + + a) For LTO instrumented binaries, this can be documented during compile + time, just set `export AFL_LLVM_DOCUMENT_IDS=/path/to/a/file`. This file + will have one assigned edge ID and the corresponding function per line. + + b) For PCGUARD instrumented binaries, it is much more difficult. Here you + can either modify the `__sanitizer_cov_trace_pc_guard` function in + `instrumentation/afl-llvm-rt.o.c` to write a backtrace to a file if the + ID in `__afl_area_ptr[*guard]` is one of the unstable edge IDs. (Example + code is already there). Then recompile and reinstall `llvm_mode` and + rebuild your target. Run the recompiled target with `afl-fuzz` for a + while and then check the file that you wrote with the backtrace + information. Alternatively, you can use `gdb` to hook + `__sanitizer_cov_trace_pc_guard_init` on start, check to which memory + address the edge ID value is written, and set a write breakpoint to that + address (`watch 0x.....`). + + c) In other instrumentation types, this is not possible. So just recompile + with the two mentioned above. This is just for identifying the functions + that have unstable edges. + + 2. Identify which edge ID numbers are unstable. + + Run the target with `export AFL_DEBUG=1` for a few minutes then terminate. + The out/fuzzer_stats file will then show the edge IDs that were identified + as unstable in the `var_bytes` entry. You can match these numbers directly + to the data you created in the first step. Now you know which functions are + responsible for the instability + + 3. Create a text file with the filenames/functions + + Identify which source code files contain the functions that you need to + remove from instrumentation, or just specify the functions you want to skip + for instrumentation. Note that optimization might inline functions! + + Follow this document on how to do this: + [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md). + + If `PCGUARD` is used, then you need to follow this guide (needs llvm 12+!): + [https://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation](https://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation) + + Only exclude those functions from instrumentation that provide no value for + coverage - that is if it does not process any fuzz data directly or + indirectly (e.g., hash maps, thread management etc.). If, however, a + function directly or indirectly handles fuzz data, then you should not put + the function in a deny instrumentation list and rather live with the + instability it comes with. + + 4. Recompile the target + + Recompile, fuzz it, be happy :) + + This link explains this process for + [Fuzzbench](https://github.com/google/fuzzbench/issues/677). \ No newline at end of file diff --git a/docs/binaryonly_fuzzing.md b/docs/binaryonly_fuzzing.md deleted file mode 100644 index 90ea3b66..00000000 --- a/docs/binaryonly_fuzzing.md +++ /dev/null @@ -1,223 +0,0 @@ -# Fuzzing binary-only programs with AFL++ - - AFL++, libfuzzer and others are great if you have the source code, and - it allows for very fast and coverage guided fuzzing. - - However, if there is only the binary program and no source code available, - then standard `afl-fuzz -n` (non-instrumented mode) is not effective. - - The following is a description of how these binaries can be fuzzed with AFL++. - - -## TL;DR: - - qemu_mode in persistent mode is the fastest - if the stability is - high enough. Otherwise try retrowrite, afl-dyninst and if these - fail too then try standard qemu_mode with AFL_ENTRYPOINT to where you need it. - - If your target is a library use utils/afl_frida/. - - If your target is non-linux then use unicorn_mode/. - - -## QEMU - - Qemu is the "native" solution to the program. - It is available in the ./qemu_mode/ directory and once compiled it can - be accessed by the afl-fuzz -Q command line option. - It is the easiest to use alternative and even works for cross-platform binaries. - - The speed decrease is at about 50%. - However various options exist to increase the speed: - - using AFL_ENTRYPOINT to move the forkserver entry to a later basic block in - the binary (+5-10% speed) - - using persistent mode [qemu_mode/README.persistent.md](../qemu_mode/README.persistent.md) - this will result in 150-300% overall speed increase - so 3-8x the original - qemu_mode speed! - - using AFL_CODE_START/AFL_CODE_END to only instrument specific parts - - Note that there is also honggfuzz: [https://github.com/google/honggfuzz](https://github.com/google/honggfuzz) - which now has a qemu_mode, but its performance is just 1.5% ... - - As it is included in AFL++ this needs no URL. - - If you like to code a customized fuzzer without much work, we highly - recommend to check out our sister project libafl which will support QEMU - too: - [https://github.com/AFLplusplus/LibAFL](https://github.com/AFLplusplus/LibAFL) - - -## AFL FRIDA - - In frida_mode you can fuzz binary-only targets easily like with QEMU, - with the advantage that frida_mode also works on MacOS (both intel and M1). - - If you want to fuzz a binary-only library then you can fuzz it with - frida-gum via utils/afl_frida/, you will have to write a harness to - call the target function in the library, use afl-frida.c as a template. - - Both come with AFL++ so this needs no URL. - - You can also perform remote fuzzing with frida, e.g. if you want to fuzz - on iPhone or Android devices, for this you can use - [https://github.com/ttdennis/fpicker/](https://github.com/ttdennis/fpicker/) - as an intermediate that uses AFL++ for fuzzing. - - If you like to code a customized fuzzer without much work, we highly - recommend to check out our sister project libafl which supports Frida too: - [https://github.com/AFLplusplus/LibAFL](https://github.com/AFLplusplus/LibAFL) - Working examples already exist :-) - - -## WINE+QEMU - - Wine mode can run Win32 PE binaries with the QEMU instrumentation. - It needs Wine, python3 and the pefile python package installed. - - As it is included in AFL++ this needs no URL. - - -## UNICORN - - Unicorn is a fork of QEMU. The instrumentation is, therefore, very similar. - In contrast to QEMU, Unicorn does not offer a full system or even userland - emulation. Runtime environment and/or loaders have to be written from scratch, - if needed. On top, block chaining has been removed. This means the speed boost - introduced in the patched QEMU Mode of AFL++ cannot simply be ported over to - Unicorn. For further information, check out [unicorn_mode/README.md](../unicorn_mode/README.md). - - As it is included in AFL++ this needs no URL. - - -## AFL UNTRACER - - If you want to fuzz a binary-only shared library then you can fuzz it with - utils/afl_untracer/, use afl-untracer.c as a template. - It is slower than AFL FRIDA (see above). - - -## DYNINST - - Dyninst is a binary instrumentation framework similar to Pintool and - Dynamorio (see far below). However whereas Pintool and Dynamorio work at - runtime, dyninst instruments the target at load time, and then let it run - - or save the binary with the changes. - This is great for some things, e.g. fuzzing, and not so effective for others, - e.g. malware analysis. - - So what we can do with dyninst is taking every basic block, and put afl's - instrumention code in there - and then save the binary. - Afterwards we can just fuzz the newly saved target binary with afl-fuzz. - Sounds great? It is. The issue though - it is a non-trivial problem to - insert instructions, which change addresses in the process space, so that - everything is still working afterwards. Hence more often than not binaries - crash when they are run. - - The speed decrease is about 15-35%, depending on the optimization options - used with afl-dyninst. - - So if Dyninst works, it is the best option available. Otherwise it just - doesn't work well. - - [https://github.com/vanhauser-thc/afl-dyninst](https://github.com/vanhauser-thc/afl-dyninst) - - -## RETROWRITE, ZAFL, ... other binary rewriter - - If you have an x86/x86_64 binary that still has its symbols, is compiled - with position independant code (PIC/PIE) and does not use most of the C++ - features then the retrowrite solution might be for you. - It decompiles to ASM files which can then be instrumented with afl-gcc. - - It is at about 80-85% performance. - - [https://git.zephyr-software.com/opensrc/zafl](https://git.zephyr-software.com/opensrc/zafl) - [https://github.com/HexHive/retrowrite](https://github.com/HexHive/retrowrite) - - -## MCSEMA - - Theoretically you can also decompile to llvm IR with mcsema, and then - use llvm_mode to instrument the binary. - Good luck with that. - - [https://github.com/lifting-bits/mcsema](https://github.com/lifting-bits/mcsema) - - -## INTEL-PT - - If you have a newer Intel CPU, you can make use of Intels processor trace. - The big issue with Intel's PT is the small buffer size and the complex - encoding of the debug information collected through PT. - This makes the decoding very CPU intensive and hence slow. - As a result, the overall speed decrease is about 70-90% (depending on - the implementation and other factors). - - There are two AFL intel-pt implementations: - - 1. [https://github.com/junxzm1990/afl-pt](https://github.com/junxzm1990/afl-pt) - => this needs Ubuntu 14.04.05 without any updates and the 4.4 kernel. - - 2. [https://github.com/hunter-ht-2018/ptfuzzer](https://github.com/hunter-ht-2018/ptfuzzer) - => this needs a 4.14 or 4.15 kernel. the "nopti" kernel boot option must - be used. This one is faster than the other. - - Note that there is also honggfuzz: https://github.com/google/honggfuzz - But its IPT performance is just 6%! - - -## CORESIGHT - - Coresight is ARM's answer to Intel's PT. - There is no implementation so far which handles coresight and getting - it working on an ARM Linux is very difficult due to custom kernel building - on embedded systems is difficult. And finding one that has coresight in - the ARM chip is difficult too. - My guess is that it is slower than Qemu, but faster than Intel PT. - - If anyone finds any coresight implementation for AFL please ping me: vh@thc.org - - -## PIN & DYNAMORIO - - Pintool and Dynamorio are dynamic instrumentation engines, and they can be - used for getting basic block information at runtime. - Pintool is only available for Intel x32/x64 on Linux, Mac OS and Windows, - whereas Dynamorio is additionally available for ARM and AARCH64. - Dynamorio is also 10x faster than Pintool. - - The big issue with Dynamorio (and therefore Pintool too) is speed. - Dynamorio has a speed decrease of 98-99% - Pintool has a speed decrease of 99.5% - - Hence Dynamorio is the option to go for if everything else fails, and Pintool - only if Dynamorio fails too. - - Dynamorio solutions: - * [https://github.com/vanhauser-thc/afl-dynamorio](https://github.com/vanhauser-thc/afl-dynamorio) - * [https://github.com/mxmssh/drAFL](https://github.com/mxmssh/drAFL) - * [https://github.com/googleprojectzero/winafl/](https://github.com/googleprojectzero/winafl/) <= very good but windows only - - Pintool solutions: - * [https://github.com/vanhauser-thc/afl-pin](https://github.com/vanhauser-thc/afl-pin) - * [https://github.com/mothran/aflpin](https://github.com/mothran/aflpin) - * [https://github.com/spinpx/afl_pin_mode](https://github.com/spinpx/afl_pin_mode) <= only old Pintool version supported - - -## Non-AFL solutions - - There are many binary-only fuzzing frameworks. - Some are great for CTFs but don't work with large binaries, others are very - slow but have good path discovery, some are very hard to set-up ... - - * QSYM: [https://github.com/sslab-gatech/qsym](https://github.com/sslab-gatech/qsym) - * Manticore: [https://github.com/trailofbits/manticore](https://github.com/trailofbits/manticore) - * S2E: [https://github.com/S2E](https://github.com/S2E) - * Tinyinst: [https://github.com/googleprojectzero/TinyInst](https://github.com/googleprojectzero/TinyInst) (Mac/Windows only) - * Jackalope: [https://github.com/googleprojectzero/Jackalope](https://github.com/googleprojectzero/Jackalope) - * ... please send me any missing that are good - - -## Closing words - - That's it! News, corrections, updates? Send an email to vh@thc.org diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index 8b5a4068..7b4e0516 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -1,16 +1,16 @@ # Custom Mutators in AFL++ This file describes how you can implement custom mutations to be used in AFL. -For now, we support C/C++ library and Python module, collectivelly named as the +For now, we support C/C++ library and Python module, collectively named as the custom mutator. -There is also experimental support for Rust in `custom_mutators/rust`. -Please refer to that directory for documentation. -Run ```cargo doc -p custom_mutator --open``` in that directory to view the -documentation in your web browser. +There is also experimental support for Rust in `custom_mutators/rust`. For +documentation, refer to that directory. Run `cargo doc -p custom_mutator --open` +in that directory to view the documentation in your web browser. Implemented by -- C/C++ library (`*.so`): Khaled Yakdan from Code Intelligence (<yakdan@code-intelligence.de>) +- C/C++ library (`*.so`): Khaled Yakdan from Code Intelligence + (<yakdan@code-intelligence.de>) - Python module: Christian Holler from Mozilla (<choller@mozilla.com>) ## 1) Introduction @@ -21,13 +21,17 @@ fuzzing by using libraries that perform mutations according to a given grammar. The custom mutator is passed to `afl-fuzz` via the `AFL_CUSTOM_MUTATOR_LIBRARY` or `AFL_PYTHON_MODULE` environment variable, and must export a fuzz function. -Now AFL also supports multiple custom mutators which can be specified in the same `AFL_CUSTOM_MUTATOR_LIBRARY` environment variable like this. +Now AFL++ also supports multiple custom mutators which can be specified in the +same `AFL_CUSTOM_MUTATOR_LIBRARY` environment variable like this. + ```bash export AFL_CUSTOM_MUTATOR_LIBRARY="full/path/to/mutator_first.so;full/path/to/mutator_second.so" ``` -Please see [APIs](#2-apis) and [Usage](#3-usage) for detail. -The custom mutation stage is set to be the first non-deterministic stage (right before the havoc stage). +For details, see [APIs](#2-apis) and [Usage](#3-usage). + +The custom mutation stage is set to be the first non-deterministic stage (right +before the havoc stage). Note: If `AFL_CUSTOM_MUTATOR_ONLY` is set, all mutations will solely be performed with the custom mutator. @@ -35,6 +39,7 @@ performed with the custom mutator. ## 2) APIs C/C++: + ```c void *afl_custom_init(afl_state_t *afl, unsigned int seed); unsigned int afl_custom_fuzz_count(void *data, const unsigned char *buf, size_t buf_size); @@ -53,6 +58,7 @@ void afl_custom_deinit(void *data); ``` Python: + ```python def init(seed): pass @@ -101,7 +107,8 @@ def deinit(): # optional for Python - `init`: - This method is called when AFL++ starts up and is used to seed RNG and set up buffers and state. + This method is called when AFL++ starts up and is used to seed RNG and set + up buffers and state. - `queue_get` (optional): @@ -110,27 +117,26 @@ def deinit(): # optional for Python - `fuzz_count` (optional): - When a queue entry is selected to be fuzzed, afl-fuzz selects the number - of fuzzing attempts with this input based on a few factors. - If however the custom mutator wants to set this number instead on how often - it is called for a specific queue entry, use this function. - This function is most useful if `AFL_CUSTOM_MUTATOR_ONLY` is **not** used. + When a queue entry is selected to be fuzzed, afl-fuzz selects the number of + fuzzing attempts with this input based on a few factors. If, however, the + custom mutator wants to set this number instead on how often it is called + for a specific queue entry, use this function. This function is most useful + if `AFL_CUSTOM_MUTATOR_ONLY` is **not** used. - `fuzz` (optional): This method performs custom mutations on a given input. It also accepts an - additional test case. - Note that this function is optional - but it makes sense to use it. - You would only skip this if `post_process` is used to fix checksums etc. - so if you are using it e.g. as a post processing library. + additional test case. Note that this function is optional - but it makes + sense to use it. You would only skip this if `post_process` is used to fix + checksums etc. so if you are using it, e.g., as a post processing library. Note that a length > 0 *must* be returned! - `describe` (optional): - When this function is called, it shall describe the current testcase, - generated by the last mutation. This will be called, for example, - to name the written testcase file after a crash occurred. - Using it can help to reproduce crashing mutations. + When this function is called, it shall describe the current test case, + generated by the last mutation. This will be called, for example, to name + the written test case file after a crash occurred. Using it can help to + reproduce crashing mutations. - `havoc_mutation` and `havoc_mutation_probability` (optional): @@ -142,21 +148,21 @@ def deinit(): # optional for Python - `post_process` (optional): For some cases, the format of the mutated data returned from the custom - mutator is not suitable to directly execute the target with this input. - For example, when using libprotobuf-mutator, the data returned is in a - protobuf format which corresponds to a given grammar. In order to execute - the target, the protobuf data must be converted to the plain-text format - expected by the target. In such scenarios, the user can define the - `post_process` function. This function is then transforming the data into the - format expected by the API before executing the target. + mutator is not suitable to directly execute the target with this input. For + example, when using libprotobuf-mutator, the data returned is in a protobuf + format which corresponds to a given grammar. In order to execute the target, + the protobuf data must be converted to the plain-text format expected by the + target. In such scenarios, the user can define the `post_process` function. + This function is then transforming the data into the format expected by the + API before executing the target. This can return any python object that implements the buffer protocol and supports PyBUF_SIMPLE. These include bytes, bytearray, etc. - `queue_new_entry` (optional): - This methods is called after adding a new test case to the queue. - If the contents of the file was changed return True, False otherwise. + This methods is called after adding a new test case to the queue. If the + contents of the file was changed, return True, False otherwise. - `introspection` (optional): @@ -168,8 +174,8 @@ def deinit(): # optional for Python The last method to be called, deinitializing the state. -Note that there are also three functions for trimming as described in the -next section. +Note that there are also three functions for trimming as described in the next +section. ### Trimming Support @@ -177,8 +183,8 @@ The generic trimming routines implemented in AFL++ can easily destroy the structure of complex formats, possibly leading to a point where you have a lot of test cases in the queue that your Python module cannot process anymore but your target application still accepts. This is especially the case when your -target can process a part of the input (causing coverage) and then errors out -on the remaining input. +target can process a part of the input (causing coverage) and then errors out on +the remaining input. In such cases, it makes sense to implement a custom trimming routine. The API consists of multiple methods because after each trimming step, we have to go @@ -189,8 +195,9 @@ trimmed input. Here's a quick API description: This method is called at the start of each trimming operation and receives the initial buffer. It should return the amount of iteration steps possible - on this input (e.g. if your input has n elements and you want to remove them - one by one, return n, if you do a binary search, return log(n), and so on). + on this input (e.g., if your input has n elements and you want to remove + them one by one, return n, if you do a binary search, return log(n), and so + on). If your trimming algorithm doesn't allow to determine the amount of (remaining) steps easily (esp. while running), then you can alternatively @@ -202,21 +209,21 @@ trimmed input. Here's a quick API description: - `trim` (optional) This method is called for each trimming operation. It doesn't have any - arguments because we already have the initial buffer from `init_trim` and we - can memorize the current state in the data variables. This can also save + arguments because there is already the initial buffer from `init_trim` and + we can memorize the current state in the data variables. This can also save reparsing steps for each iteration. It should return the trimmed input buffer. - `post_trim` (optional) This method is called after each trim operation to inform you if your - trimming step was successful or not (in terms of coverage). If you receive - a failure here, you should reset your input to the last known good state. - In any case, this method must return the next trim iteration index (from 0 - to the maximum amount of steps you returned in `init_trim`). + trimming step was successful or not (in terms of coverage). If you receive a + failure here, you should reset your input to the last known good state. In + any case, this method must return the next trim iteration index (from 0 to + the maximum amount of steps you returned in `init_trim`). Omitting any of three trimming methods will cause the trimming to be disabled -and trigger a fallback to the builtin default trimming routine. +and trigger a fallback to the built-in default trimming routine. ### Environment Variables @@ -224,11 +231,10 @@ Optionally, the following environment variables are supported: - `AFL_CUSTOM_MUTATOR_ONLY` - Disable all other mutation stages. This can prevent broken testcases - (those that your Python module can't work with anymore) to fill up your - queue. Best combined with a custom trimming routine (see below) because - trimming can cause the same test breakage like havoc and splice. - + Disable all other mutation stages. This can prevent broken test cases (those + that your Python module can't work with anymore) to fill up your queue. Best + combined with a custom trimming routine (see below) because trimming can + cause the same test breakage like havoc and splice. - `AFL_PYTHON_ONLY` @@ -264,22 +270,27 @@ In case your setup is different, set the necessary variables like this: ### Custom Mutator Preparation For C/C++ mutators, the source code must be compiled as a shared object: + ```bash gcc -shared -Wall -O3 example.c -o example.so ``` -Note that if you specify multiple custom mutators, the corresponding functions will -be called in the order in which they are specified. e.g first `post_process` function of -`example_first.so` will be called and then that of `example_second.so`. + +Note that if you specify multiple custom mutators, the corresponding functions +will be called in the order in which they are specified. E.g., the first +`post_process` function of `example_first.so` will be called and then that of +`example_second.so`. ### Run C/C++ + ```bash export AFL_CUSTOM_MUTATOR_LIBRARY="/full/path/to/example_first.so;/full/path/to/example_second.so" afl-fuzz /path/to/program ``` Python + ```bash export PYTHONPATH=`dirname /full/path/to/example.py` export AFL_PYTHON_MODULE=example @@ -288,8 +299,8 @@ afl-fuzz /path/to/program ## 4) Example -Please see [example.c](../custom_mutators/examples/example.c) and -[example.py](../custom_mutators/examples/example.py) +See [example.c](../custom_mutators/examples/example.c) and +[example.py](../custom_mutators/examples/example.py). ## 5) Other Resources @@ -297,4 +308,4 @@ Please see [example.c](../custom_mutators/examples/example.c) and - [bruce30262/libprotobuf-mutator_fuzzing_learning](https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator) - [thebabush/afl-libprotobuf-mutator](https://github.com/thebabush/afl-libprotobuf-mutator) - [XML Fuzzing@NullCon 2017](https://www.agarri.fr/docs/XML_Fuzzing-NullCon2017-PUBLIC.pdf) - - [A bug detected by AFL + XML-aware mutators](https://bugs.chromium.org/p/chromium/issues/detail?id=930663) + - [A bug detected by AFL + XML-aware mutators](https://bugs.chromium.org/p/chromium/issues/detail?id=930663) \ No newline at end of file diff --git a/docs/docs2.md b/docs/docs2.md new file mode 100644 index 00000000..23ef61c5 --- /dev/null +++ b/docs/docs2.md @@ -0,0 +1,124 @@ +# Restructure AFL++'s documentation - Case Study + +## Problem statement + +AFL++ inherited it's documentation from the original Google AFL project. +Since then it has been massively improved - feature and performance wise - +and although the documenation has likewise been continued it has grown out +of proportion. +The documentation is done by non-natives to the English language, plus +none of us has a writer background. + +We see questions on AFL++ usage on mailing lists (e.g. afl-users), discord +channels, web forums and as issues in our repository. +Most of them could be answered if people would read through all the +documentation. + +This only increases as AFL++ has been on the top of Google's fuzzbench +statistics (which measures the performance of fuzzers) and has been +integrated in Google's oss-fuzz and clusterfuzz - and is in many Unix +packaging repositories, e.g. Debian, FreeBSD, etc. + +AFL++ had 44 (!) documentation files with 13k total lines of content. +This was way too much. + +## Proposal abstract + +AFL++'s documentatin needs a complete overhaul, both on a +organisation/structural level as well as the content. + +Overall the following actions have to be performed: + * Create a better structure of documentation so it is easier to find the + information that is being looked for, combining and/or splitting up the + existing documents as needed. + * Rewrite some documentation to remove duplication. Several information is + present several times in the documentation. These should be removed to + where needed so that we have as little bloat as possible. + * The documents have been written and modified by a lot of different people, + most of them non-native English speaker. Hence an overall review where + parts should be rewritten has to be performed and then the rewrite done. + * Create a cheat-sheet for a very short best-setup build and run of AFL++ + * Pictures explain more than 1000 words. We need at least 4 images that + explain the workflow with AFL++: + - the build workflow + - the fuzzing workflow + - the fuzzing campaign management workflow + - the overall workflow that is an overview of the above + - maybe more? where the technical writes seems it necessary for + understanding. + +Requirements: + * Documentation has to be in Markdown format + * Images have to be either in SVG or PNG format. + * All documentation should be (moved) in(to) docs/ + +## Project description + +We created our proposal by discussing in the team what the issues are and +what was needed to fix it. +This resulted in the [project proposal](https://github.com/AFLplusplus/AFLplusplus/blob/stable/docs/docs.md). + +We did not want to be selected by a writer but select a writer ourselves, so +we combed through the list and reviewed every single one of them. +We were not looking for coders writing technical documentation, but rather +someone who is an experienced writer and has documented experience with +structuring documentation. +Few fit that profile and we sent out messages to 6 people. +We finally decided on Jana because she had a strong background in technical +documentation and structuring information. +She had no technical experience in fuzzing whatsoever, but we saw that as +a plus - of course this made the whole process longer to explain details, +but overall ensured that the documentation can be read by (mostly) everyone. + +We communicated via video calls every few weeks and she kept a public kanban +board about her todos, additional we used a Signal channel. +Her changes were imported via PRs where we discussed details. + +The project was off to a good start, but then Jana got pregnant with serious +side effects that made working impossible for her for a longer time, hence +the schedule was thrown back. +She offered to rescind the payment and we select a new writer, but we saw +little opportunity in that, as that would mean a new selection of a writer, +someone else with a different vision on how the result should look like so +basically a full restart of the project and a large impact on our own time. +So we agreed on - after discussion with the Google GSoD team - that she +continues the project after the GSoD completion deadline as best as she can. + +End of November she took one week off from work and fully dedicated her time +for the documenation which brought the project a big step forward. + +Originally the project should have been ended begin of October, but now - at +nearing the end of November, we are at about 85% completion, with the end +being expected around mid of December. + +## Metrics + +We merged most of the changes in our development branch and are getting +close to a state where the user documentation part is completed and we +can create a new release. Only then the new documentatin is actually visible +to users. Therefore no metrics could be collected so far. + +We plan on a user-assisted QA review end of November/begin of December. + +The documentation was reviewed by a few test users so far however who gave +it a thumbs up. + +## Summary + +The GSoD project itself is great. It helps to get the documentation back in +line. +It was and is a larger time investment from our side, but we expected that. +When the project is done, the documentation will be more accessible by users +and also need less maintenance by us. +There is still follow-up work to be done by us afterwards (web site for the +docs, etc.). + +Not sure what we would do differently next time. I think we prepared best as +possible and reacted best as possible to the unexpected. + +Recommendations for other organizations who would like to participate in GSoD: + - expect the process to take a larger part of your time. the writer needs + your full support. + - have someone dedicated from the dev/org side to support, educate and + supervice the writer + - set clear goals and expectations diff --git a/docs/env_variables.md b/docs/env_variables.md index 0686f1a8..c45f4ab9 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -1,88 +1,78 @@ -# Environmental variables +# Environment variables - This document discusses the environment variables used by American Fuzzy Lop++ - to expose various exotic functions that may be (rarely) useful for power - users or for some types of custom fuzzing setups. See [README.md](README.md) for the general - instruction manual. + This document discusses the environment variables used by AFL++ to expose + various exotic functions that may be (rarely) useful for power users or for + some types of custom fuzzing setups. For general information about AFL++, see + [README.md](../README.md). - Note that most tools will warn on any unknown AFL environment variables. - This is for warning on typos that can happen. If you want to disable this - check then set the `AFL_IGNORE_UNKNOWN_ENVS` environment variable. + Note: Most tools will warn on any unknown AFL++ environment variables; for + example, because of typos. If you want to disable this check, then set the + `AFL_IGNORE_UNKNOWN_ENVS` environment variable. ## 1) Settings for all compilers -Starting with AFL++ 3.0 there is only one compiler: afl-cc -To select the different instrumentation modes this can be done by - 1. passing the --afl-MODE command line option to the compiler - 2. or using a symlink to afl-cc: afl-gcc, afl-g++, afl-clang, afl-clang++, - afl-clang-fast, afl-clang-fast++, afl-clang-lto, afl-clang-lto++, - afl-gcc-fast, afl-g++-fast - 3. or using the environment variable `AFL_CC_COMPILER` with `MODE` - -`MODE` can be one of `LTO` (afl-clang-lto*), `LLVM` (afl-clang-fast*), `GCC_PLUGIN` -(afl-g*-fast) or `GCC` (afl-gcc/afl-g++). - -Because (with the exception of the --afl-MODE command line option) the -compile-time tools do not accept AFL specific command-line options, they -make fairly broad use of environmental variables instead: - - - Some build/configure scripts break with AFL++ compilers. To be able to - pass them, do: -``` - export CC=afl-cc - export CXX=afl-c++ - export AFL_NOOPT=1 - ./configure --disable-shared --disabler-werror - unset AFL_NOOPT - make -``` - - - Most AFL tools do not print any output if stdout/stderr are redirected. - If you want to get the output into a file then set the `AFL_DEBUG` - environment variable. - This is sadly necessary for various build processes which fail otherwise. +Starting with AFL++ 3.0, there is only one compiler: afl-cc. + +To select the different instrumentation modes, use one of the following options: + + - Pass the --afl-MODE command-line option to the compiler. Only this option + accepts further AFL-specific command-line options. + - Use a symlink to afl-cc: afl-clang, afl-clang++, afl-clang-fast, + afl-clang-fast++, afl-clang-lto, afl-clang-lto++, afl-g++, afl-g++-fast, + afl-gcc, afl-gcc-fast. This option does not accept AFL-specific command-line + options. Instead, use environment variables. + - Use the `AFL_CC_COMPILER` environment variable with `MODE`. To select + `MODE`, use one of the following values: + + - `GCC` (afl-gcc/afl-g++) + - `GCC_PLUGIN` (afl-g*-fast) + - `LLVM` (afl-clang-fast*) + - `LTO` (afl-clang-lto*). + +The compile-time tools do not accept AFL-specific command-line options. The +--afl-MODE command line option is the only exception. The other options make +fairly broad use of environment variables instead: + + - Some build/configure scripts break with AFL++ compilers. To be able to pass + them, do: + + ``` + export CC=afl-cc + export CXX=afl-c++ + export AFL_NOOPT=1 + ./configure --disable-shared --disabler-werror + unset AFL_NOOPT + make + ``` + + - Setting `AFL_AS`, `AFL_CC`, and `AFL_CXX` lets you use alternate downstream + compilation tools, rather than the default 'as', 'clang', or 'gcc' binaries + in your `$PATH`. + + - If you are a weird person that wants to compile and instrument asm text + files, then use the `AFL_AS_FORCE_INSTRUMENT` variable: + `AFL_AS_FORCE_INSTRUMENT=1 afl-gcc foo.s -o foo` + + - Most AFL tools do not print any output if stdout/stderr are redirected. If + you want to get the output into a file, then set the `AFL_DEBUG` environment + variable. This is sadly necessary for various build processes which fail + otherwise. + + - By default, the wrapper appends `-O3` to optimize builds. Very rarely, this + will cause problems in programs built with -Werror, because `-O3` enables + more thorough code analysis and can spew out additional warnings. To disable + optimizations, set `AFL_DONT_OPTIMIZE`. However, if `-O...` and/or + `-fno-unroll-loops` are set, these are not overridden. - Setting `AFL_HARDEN` automatically adds code hardening options when invoking the downstream compiler. This currently includes `-D_FORTIFY_SOURCE=2` and `-fstack-protector-all`. The setting is useful for catching non-crashing memory bugs at the expense of a very slight (sub-5%) performance loss. - - By default, the wrapper appends `-O3` to optimize builds. Very rarely, this - will cause problems in programs built with -Werror, simply because `-O3` - enables more thorough code analysis and can spew out additional warnings. - To disable optimizations, set `AFL_DONT_OPTIMIZE`. - However if `-O...` and/or `-fno-unroll-loops` are set, these are not - overridden. - - - Setting `AFL_USE_ASAN` automatically enables ASAN, provided that your - compiler supports it. - - (You can also enable MSAN via `AFL_USE_MSAN`; ASAN and MSAN come with the - same gotchas; the modes are mutually exclusive. UBSAN can be enabled - similarly by setting the environment variable `AFL_USE_UBSAN=1`. Finally - there is the Control Flow Integrity sanitizer that can be activated by - `AFL_USE_CFISAN=1`) - - - Setting `AFL_USE_LSAN` automatically enables Leak-Sanitizer, provided - that your compiler supports it. To perform a leak check within your - program at a certain point (such as at the end of an __AFL_LOOP), - you can run the macro __AFL_LEAK_CHECK(); which will cause - an abort if any memory is leaked (you can combine this with the - LSAN_OPTIONS=suppressions option to supress some known leaks). - - - Setting `AFL_CC`, `AFL_CXX`, and `AFL_AS` lets you use alternate downstream - compilation tools, rather than the default 'clang', 'gcc', or 'as' binaries - in your `$PATH`. - - - `AFL_PATH` can be used to point afl-gcc to an alternate location of afl-as. - One possible use of this is utils/clang_asm_normalize/, which lets - you instrument hand-written assembly when compiling clang code by plugging - a normalizer into the chain. (There is no equivalent feature for GCC.) - - Setting `AFL_INST_RATIO` to a percentage between 0 and 100 controls the - probability of instrumenting every branch. This is (very rarely) useful - when dealing with exceptionally complex programs that saturate the output - bitmap. Examples include v8, ffmpeg, and perl. + probability of instrumenting every branch. This is (very rarely) useful when + dealing with exceptionally complex programs that saturate the output bitmap. + Examples include ffmpeg, perl, and v8. (If this ever happens, afl-fuzz will warn you ahead of the time by displaying the "bitmap density" field in fiery red.) @@ -90,493 +80,612 @@ make fairly broad use of environmental variables instead: Setting `AFL_INST_RATIO` to 0 is a valid choice. This will instrument only the transitions between function entry points, but not individual branches. - Note that this is an outdated variable. A few instances (e.g. afl-gcc) - still support these, but state-of-the-art (e.g. LLVM LTO and LLVM PCGUARD) + Note that this is an outdated variable. A few instances (e.g., afl-gcc) + still support these, but state-of-the-art (e.g., LLVM LTO and LLVM PCGUARD) do not need this. - `AFL_NO_BUILTIN` causes the compiler to generate code suitable for use with libtokencap.so (but perhaps running a bit slower than without the flag). + - `AFL_PATH` can be used to point afl-gcc to an alternate location of afl-as. + One possible use of this is utils/clang_asm_normalize/, which lets you + instrument hand-written assembly when compiling clang code by plugging a + normalizer into the chain. (There is no equivalent feature for GCC.) + + - Setting `AFL_QUIET` will prevent afl-as and afl-cc banners from being + displayed during compilation, in case you find them distracting. + + - Setting `AFL_USE_...` automatically enables supported sanitizers - provided + that your compiler supports it. Available are: + - `AFL_USE_ASAN=1` - activates the address sanitizer (memory corruption + detection) + - `AFL_USE_CFISAN=1` - activates the Control Flow Integrity sanitizer (e.g. + type confusion vulnerabilities) + - `AFL_USE_LSAN` - activates the leak sanitizer. To perform a leak check + within your program at a certain point (such as at the end of an + `__AFL_LOOP()`), you can run the macro `__AFL_LEAK_CHECK();` which will + cause an abort if any memory is leaked (you can combine this with the + `LSAN_OPTIONS=...` suppression option to suppress some known leaks). + - `AFL_USE_MSAN=1` - activates the memory sanitizer (uninitialized memory) + - `AFL_USE_TSAN=1` - activates the thread sanitizer to find thread race + conditions + - `AFL_USE_UBSAN=1` - activates the undefined behavior sanitizer + - `TMPDIR` is used by afl-as for temporary files; if this variable is not set, the tool defaults to /tmp. - - If you are a weird person that wants to compile and instrument asm - text files then use the `AFL_AS_FORCE_INSTRUMENT` variable: - `AFL_AS_FORCE_INSTRUMENT=1 afl-gcc foo.s -o foo` +## 2) Settings for LLVM and LTO: afl-clang-fast / afl-clang-fast++ / afl-clang-lto / afl-clang-lto++ - - Setting `AFL_QUIET` will prevent afl-cc and afl-as banners from being - displayed during compilation, in case you find them distracting. +The native instrumentation helpers (instrumentation and gcc_plugin) accept a +subset of the settings discussed in section 1, with the exception of: -## 2) Settings for LLVM and LTO: afl-clang-fast / afl-clang-fast++ / afl-clang-lto / afl-clang-lto++ + - `AFL_AS`, since this toolchain does not directly invoke GNU `as`. -The native instrumentation helpers (instrumentation and gcc_plugin) accept a subset -of the settings discussed in section 1, with the exception of: + - `AFL_INST_RATIO`, as we use collision free instrumentation by default. Not + all passes support this option though as it is an outdated feature. - LLVM modes support `AFL_LLVM_DICT2FILE=/absolute/path/file.txt` which will - write all constant string comparisons to this file to be used later with + write all constant string comparisons to this file to be used later with afl-fuzz' `-x` option. - - `AFL_AS`, since this toolchain does not directly invoke GNU as. - - `TMPDIR` and `AFL_KEEP_ASSEMBLY`, since no temporary assembly files are created. - - `AFL_INST_RATIO`, as we by default use collision free instrumentation. - Not all passes support this option though as it is an outdated feature. - -Then there are a few specific features that are only available in instrumentation mode: +Then there are a few specific features that are only available in +instrumentation mode: ### Select the instrumentation mode - - `AFL_LLVM_INSTRUMENT` - this configures the instrumentation mode. - Available options: - PCGUARD - our own pcgard based instrumentation (default) - NATIVE - clang's original pcguard based instrumentation - CLASSIC - classic AFL (map[cur_loc ^ prev_loc >> 1]++) (default) - LTO - LTO instrumentation (see below) - CTX - context sensitive instrumentation (see below) - NGRAM-x - deeper previous location coverage (from NGRAM-2 up to NGRAM-16) - GCC - outdated gcc instrumentation - CLANG - outdated clang instrumentation - In CLASSIC you can also specify CTX and/or NGRAM, seperate the options - with a comma "," then, e.g.: - `AFL_LLVM_INSTRUMENT=CLASSIC,CTX,NGRAM-4` - Note that this is actually not a good idea to use both CTX and NGRAM :) +`AFL_LLVM_INSTRUMENT` - this configures the instrumentation mode. + +Available options: + + - CLANG - outdated clang instrumentation + - CLASSIC - classic AFL (map[cur_loc ^ prev_loc >> 1]++) (default) + + You can also specify CTX and/or NGRAM, separate the options with a comma "," + then, e.g.: `AFL_LLVM_INSTRUMENT=CLASSIC,CTX,NGRAM-4` -### LTO + Note: It is actually not a good idea to use both CTX and NGRAM. :) + - CTX - context sensitive instrumentation + - GCC - outdated gcc instrumentation + - LTO - LTO instrumentation + - NATIVE - clang's original pcguard based instrumentation + - NGRAM-x - deeper previous location coverage (from NGRAM-2 up to NGRAM-16) + - PCGUARD - our own pcgard based instrumentation (default) - This is a different kind way of instrumentation: first it compiles all - code in LTO (link time optimization) and then performs an edge inserting - instrumentation which is 100% collision free (collisions are a big issue - in AFL and AFL-like instrumentations). This is performed by using - afl-clang-lto/afl-clang-lto++ instead of afl-clang-fast, but is only - built if LLVM 11 or newer is used. +#### CMPLOG - - `AFL_LLVM_INSTRUMENT=CFG` will use Control Flow Graph instrumentation. - (not recommended for afl-clang-fast, default for afl-clang-lto as there - it is a different and better kind of instrumentation.) +Setting `AFL_LLVM_CMPLOG=1` during compilation will tell afl-clang-fast to +produce a CmpLog binary. - None of the following options are necessary to be used and are rather for - manual use (which only ever the author of this LTO implementation will use). - These are used if several separated instrumentations are performed which - are then later combined. +For more information, see +[instrumentation/README.cmplog.md](../instrumentation/README.cmplog.md). - - `AFL_LLVM_DOCUMENT_IDS=file` will document to a file which edge ID was given - to which function. This helps to identify functions with variable bytes - or which functions were touched by an input. - - `AFL_LLVM_MAP_ADDR` sets the fixed map address to a different address than - the default `0x10000`. A value of 0 or empty sets the map address to be - dynamic (the original AFL way, which is slower) - - `AFL_LLVM_MAP_DYNAMIC` sets the shared memory address to be dynamic - - `AFL_LLVM_LTO_STARTID` sets the starting location ID for the instrumentation. - This defaults to 1 - - `AFL_LLVM_LTO_DONTWRITEID` prevents that the highest location ID written - into the instrumentation is set in a global variable +#### CTX - See [instrumentation/README.lto.md](../instrumentation/README.lto.md) for more information. +Setting `AFL_LLVM_CTX` or `AFL_LLVM_INSTRUMENT=CTX` activates context sensitive +branch coverage - meaning that each edge is additionally combined with its +caller. It is highly recommended to increase the `MAP_SIZE_POW2` definition in +config.h to at least 18 and maybe up to 20 for this as otherwise too many map +collisions occur. -### NGRAM +For more information, see +[instrumentation/README.llvm.md#6) AFL++ Context Sensitive Branch Coverage](../instrumentation/README.llvm.md#6-afl-context-sensitive-branch-coverage). - - Setting `AFL_LLVM_NGRAM_SIZE` or `AFL_LLVM_INSTRUMENT=NGRAM-{value}` - activates ngram prev_loc coverage, good values are 2, 4 or 8 - (any value between 2 and 16 is valid). - It is highly recommended to increase the `MAP_SIZE_POW2` definition in - config.h to at least 18 and maybe up to 20 for this as otherwise too - many map collisions occur. +#### INSTRUMENT LIST (selectively instrument files and functions) - See [instrumentation/README.ngram.md](../instrumentation/README.ngram.md) +This feature allows selective instrumentation of the source. -### CTX +Setting `AFL_LLVM_ALLOWLIST` or `AFL_LLVM_DENYLIST` with a file name and/or +function will only instrument (or skip) those files that match the names listed +in the specified file. - - Setting `AFL_LLVM_CTX` or `AFL_LLVM_INSTRUMENT=CTX` - activates context sensitive branch coverage - meaning that each edge - is additionally combined with its caller. - It is highly recommended to increase the `MAP_SIZE_POW2` definition in - config.h to at least 18 and maybe up to 20 for this as otherwise too - many map collisions occur. +For more information, see +[instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md). - See [instrumentation/README.ctx.md](../instrumentation/README.ctx.md) +#### LAF-INTEL -### LAF-INTEL +This great feature will split compares into series of single byte comparisons to +allow afl-fuzz to find otherwise rather impossible paths. It is not restricted +to Intel CPUs. ;-) - This great feature will split compares into series of single byte comparisons - to allow afl-fuzz to find otherwise rather impossible paths. It is not - restricted to Intel CPUs ;-) + - Setting `AFL_LLVM_LAF_TRANSFORM_COMPARES` will split string compare + functions. - - Setting `AFL_LLVM_LAF_TRANSFORM_COMPARES` will split string compare functions + - Setting `AFL_LLVM_LAF_SPLIT_COMPARES` will split all floating point and 64, + 32 and 16 bit integer CMP instructions. - - Setting `AFL_LLVM_LAF_SPLIT_SWITCHES` will split all `switch` constructs + - Setting `AFL_LLVM_LAF_SPLIT_FLOATS` will split floating points, needs + `AFL_LLVM_LAF_SPLIT_COMPARES` to be set. - - Setting `AFL_LLVM_LAF_SPLIT_COMPARES` will split all floating point and - 64, 32 and 16 bit integer CMP instructions + - Setting `AFL_LLVM_LAF_SPLIT_SWITCHES` will split all `switch` constructs. - - Setting `AFL_LLVM_LAF_SPLIT_FLOATS` will split floating points, needs - AFL_LLVM_LAF_SPLIT_COMPARES to be set + - Setting `AFL_LLVM_LAF_ALL` sets all of the above. - - Setting `AFL_LLVM_LAF_ALL` sets all of the above +For more information, see +[instrumentation/README.laf-intel.md](../instrumentation/README.laf-intel.md). - See [instrumentation/README.laf-intel.md](../instrumentation/README.laf-intel.md) for more information. +#### LTO -### INSTRUMENT LIST (selectively instrument files and functions) +This is a different way of instrumentation: first it compiles all code in LTO +(link time optimization) and then performs an edge inserting instrumentation +which is 100% collision free (collisions are a big issue in AFL and AFL-like +instrumentations). This is performed by using afl-clang-lto/afl-clang-lto++ +instead of afl-clang-fast, but is only built if LLVM 11 or newer is used. - This feature allows selective instrumentation of the source +`AFL_LLVM_INSTRUMENT=CFG` will use Control Flow Graph instrumentation. (Not +recommended for afl-clang-fast, default for afl-clang-lto as there it is a +different and better kind of instrumentation.) - - Setting `AFL_LLVM_ALLOWLIST` or `AFL_LLVM_DENYLIST` with a filenames and/or - function will only instrument (or skip) those files that match the names - listed in the specified file. +None of the following options are necessary to be used and are rather for manual +use (which only ever the author of this LTO implementation will use). These are +used if several separated instrumentations are performed which are then later +combined. - See [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md) for more information. + - `AFL_LLVM_DOCUMENT_IDS=file` will document to a file which edge ID was given + to which function. This helps to identify functions with variable bytes or + which functions were touched by an input. + - `AFL_LLVM_LTO_DONTWRITEID` prevents that the highest location ID written + into the instrumentation is set in a global variable. + - `AFL_LLVM_LTO_STARTID` sets the starting location ID for the + instrumentation. This defaults to 1. + - `AFL_LLVM_MAP_ADDR` sets the fixed map address to a different address than + the default `0x10000`. A value of 0 or empty sets the map address to be + dynamic (the original AFL way, which is slower). + - `AFL_LLVM_MAP_DYNAMIC` sets the shared memory address to be dynamic. -### Thread safe instrumentation counters (in all modes) + For more information, see + [instrumentation/README.lto.md](../instrumentation/README.lto.md). - - Setting `AFL_LLVM_THREADSAFE_INST` will inject code that implements thread - safe counters. The overhead is a little bit higher compared to the older - non-thread safe case. Note that this disables neverzero (see below). +#### NGRAM -### NOT_ZERO +Setting `AFL_LLVM_INSTRUMENT=NGRAM-{value}` or `AFL_LLVM_NGRAM_SIZE` activates +ngram prev_loc coverage. Good values are 2, 4, or 8 (any value between 2 and 16 +is valid). It is highly recommended to increase the `MAP_SIZE_POW2` definition +in config.h to at least 18 and maybe up to 20 for this as otherwise too many map +collisions occur. - - Setting `AFL_LLVM_NOT_ZERO=1` during compilation will use counters - that skip zero on overflow. This is the default for llvm >= 9, - however for llvm versions below that this will increase an unnecessary - slowdown due a performance issue that is only fixed in llvm 9+. - This feature increases path discovery by a little bit. +For more information, see +[instrumentation/README.llvm.md#7) AFL++ N-Gram Branch Coverage](../instrumentation/README.llvm.md#7-afl-n-gram-branch-coverage). - - Setting `AFL_LLVM_SKIP_NEVERZERO=1` will not implement the skip zero - test. If the target performs only few loops then this will give a - small performance boost. +#### NOT_ZERO - See [instrumentation/README.neverzero.md](../instrumentation/README.neverzero.md) + - Setting `AFL_LLVM_NOT_ZERO=1` during compilation will use counters that skip + zero on overflow. This is the default for llvm >= 9, however, for llvm + versions below that this will increase an unnecessary slowdown due a + performance issue that is only fixed in llvm 9+. This feature increases path + discovery by a little bit. -### CMPLOG + - Setting `AFL_LLVM_SKIP_NEVERZERO=1` will not implement the skip zero test. + If the target performs only a few loops, then this will give a small + performance boost. - - Setting `AFL_LLVM_CMPLOG=1` during compilation will tell afl-clang-fast to - produce a CmpLog binary. +#### Thread safe instrumentation counters (in all modes) - See [instrumentation/README.cmplog.md](../instrumentation/README.cmplog.md) +Setting `AFL_LLVM_THREADSAFE_INST` will inject code that implements thread safe +counters. The overhead is a little bit higher compared to the older non-thread +safe case. Note that this disables neverzero (see NOT_ZERO). ## 3) Settings for GCC / GCC_PLUGIN modes -Then there are a few specific features that are only available in GCC and -GCC_PLUGIN mode. +There are a few specific features that are only available in GCC and GCC_PLUGIN +mode. + + - GCC mode only: Setting `AFL_KEEP_ASSEMBLY` prevents afl-as from deleting + instrumented assembly files. Useful for troubleshooting problems or + understanding how the tool works. - - Setting `AFL_KEEP_ASSEMBLY` prevents afl-as from deleting instrumented - assembly files. Useful for troubleshooting problems or understanding how - the tool works. (GCC mode only) To get them in a predictable place, try something like: -``` + + ``` mkdir assembly_here TMPDIR=$PWD/assembly_here AFL_KEEP_ASSEMBLY=1 make clean all -``` - - Setting `AFL_GCC_INSTRUMENT_FILE` with a filename will only instrument those - files that match the names listed in this file (one filename per line). - See [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md) for more information. - (GCC_PLUGIN mode only) + ``` + + - GCC_PLUGIN mode only: Setting `AFL_GCC_INSTRUMENT_FILE` with a filename will + only instrument those files that match the names listed in this file (one + filename per line). See + [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md) + for more information. ## 4) Settings for afl-fuzz The main fuzzer binary accepts several options that disable a couple of sanity checks or alter some of the more exotic semantics of the tool: - - Setting `AFL_SKIP_CPUFREQ` skips the check for CPU scaling policy. This is - useful if you can't change the defaults (e.g., no root access to the - system) and are OK with some performance loss. - - - `AFL_EXIT_WHEN_DONE` causes afl-fuzz to terminate when all existing paths - have been fuzzed and there were no new finds for a while. This would be - normally indicated by the cycle counter in the UI turning green. May be - convenient for some types of automated jobs. - - - `AFL_EXIT_ON_TIME` Causes afl-fuzz to terminate if no new paths were - found within a specified period of time (in seconds). May be convenient - for some types of automated jobs. - - - `AFL_EXIT_ON_SEED_ISSUES` will restore the vanilla afl-fuzz behaviour - which does not allow crashes or timeout seeds in the initial -i corpus. + - Setting `AFL_AUTORESUME` will resume a fuzz run (same as providing `-i -`) + for an existing out folder, even if a different `-i` was provided. Without + this setting, afl-fuzz will refuse execution for a long-fuzzed out dir. - - `AFL_MAP_SIZE` sets the size of the shared map that afl-fuzz, afl-showmap, - afl-tmin and afl-analyze create to gather instrumentation data from - the target. This must be equal or larger than the size the target was - compiled with. + - Benchmarking only: `AFL_BENCH_JUST_ONE` causes the fuzzer to exit after + processing the first queue entry; and `AFL_BENCH_UNTIL_CRASH` causes it to + exit soon after the first crash is found. - `AFL_CMPLOG_ONLY_NEW` will only perform the expensive cmplog feature for - newly found testcases and not for testcases that are loaded on startup + newly found test cases and not for test cases that are loaded on startup (`-i in`). This is an important feature to set when resuming a fuzzing session. - - `AFL_TESTCACHE_SIZE` allows you to override the size of `#define TESTCASE_CACHE` - in config.h. Recommended values are 50-250MB - or more if your fuzzing - finds a huge amount of paths for large inputs. + - Setting `AFL_CRASH_EXITCODE` sets the exit code AFL++ treats as crash. For + example, if `AFL_CRASH_EXITCODE='-1'` is set, each input resulting in a `-1` + return code (i.e. `exit(-1)` got called), will be treated as if a crash had + occurred. This may be beneficial if you look for higher-level faulty + conditions in which your target still exits gracefully. + + - Setting `AFL_CUSTOM_MUTATOR_LIBRARY` to a shared library with + afl_custom_fuzz() creates additional mutations through this library. If + afl-fuzz is compiled with Python (which is autodetected during building + afl-fuzz), setting `AFL_PYTHON_MODULE` to a Python module can also provide + additional mutations. If `AFL_CUSTOM_MUTATOR_ONLY` is also set, all + mutations will solely be performed with the custom mutator. This feature + allows to configure custom mutators which can be very helpful, e.g., fuzzing + XML or other highly flexible structured input. For details, see + [custom_mutators.md](custom_mutators.md). + + - Setting `AFL_CYCLE_SCHEDULES` will switch to a different schedule every time + a cycle is finished. + + - Setting `AFL_DEBUG_CHILD` will not suppress the child output. This lets you + see all output of the child, making setup issues obvious. For example, in an + unicornafl harness, you might see python stacktraces. You may also see other + logs that way, indicating why the forkserver won't start. Not pretty but + good for debugging purposes. Note that `AFL_DEBUG_CHILD_OUTPUT` is + deprecated. - Setting `AFL_DISABLE_TRIM` tells afl-fuzz not to trim test cases. This is usually a bad idea! - - Setting `AFL_NO_AFFINITY` disables attempts to bind to a specific CPU core - on Linux systems. This slows things down, but lets you run more instances - of afl-fuzz than would be prudent (if you really want to). + - `AFL_EXIT_ON_SEED_ISSUES` will restore the vanilla afl-fuzz behavior which + does not allow crashes or timeout seeds in the initial -i corpus. - - Setting `AFL_TRY_AFFINITY` tries to attempt binding to a specific CPU core - on Linux systems, but will not terminate if that fails. + - `AFL_EXIT_ON_TIME` causes afl-fuzz to terminate if no new paths were found + within a specified period of time (in seconds). May be convenient for some + types of automated jobs. - - Setting `AFL_NO_AUTODICT` will not load an LTO generated auto dictionary - that is compiled into the target. + - `AFL_EXIT_WHEN_DONE` causes afl-fuzz to terminate when all existing paths + have been fuzzed and there were no new finds for a while. This would be + normally indicated by the cycle counter in the UI turning green. May be + convenient for some types of automated jobs. - - Setting `AFL_HANG_TMOUT` allows you to specify a different timeout for - deciding if a particular test case is a "hang". The default is 1 second - or the value of the `-t` parameter, whichever is larger. Dialing the value - down can be useful if you are very concerned about slow inputs, or if you - don't want AFL++ to spend too much time classifying that stuff and just - rapidly put all timeouts in that bin. + - Setting `AFL_EXPAND_HAVOC_NOW` will start in the extended havoc mode that + includes costly mutations. afl-fuzz automatically enables this mode when + deemed useful otherwise. + + - `AFL_FAST_CAL` keeps the calibration stage about 2.5x faster (albeit less + precise), which can help when starting a session against a slow target. + `AFL_CAL_FAST` works too. + + - Setting `AFL_FORCE_UI` will force painting the UI on the screen even if no + valid terminal was detected (for virtual consoles). - Setting `AFL_FORKSRV_INIT_TMOUT` allows you to specify a different timeout to wait for the forkserver to spin up. The default is the `-t` value times `FORK_WAIT_MULT` from `config.h` (usually 10), so for a `-t 100`, the - default would wait for `1000` milliseconds. Setting a different time here is useful - if the target has a very slow startup time, for example when doing - full-system fuzzing or emulation, but you don't want the actual runs - to wait too long for timeouts. + default would wait for `1000` milliseconds. Setting a different time here is + useful if the target has a very slow startup time, for example, when doing + full-system fuzzing or emulation, but you don't want the actual runs to wait + too long for timeouts. - - `AFL_NO_ARITH` causes AFL++ to skip most of the deterministic arithmetics. - This can be useful to speed up the fuzzing of text-based file formats. - - - `AFL_NO_SNAPSHOT` will advice afl-fuzz not to use the snapshot feature - if the snapshot lkm is loaded - - - `AFL_SHUFFLE_QUEUE` randomly reorders the input queue on startup. Requested - by some users for unorthodox parallelized fuzzing setups, but not - advisable otherwise. + - Setting `AFL_HANG_TMOUT` allows you to specify a different timeout for + deciding if a particular test case is a "hang". The default is 1 second or + the value of the `-t` parameter, whichever is larger. Dialing the value down + can be useful if you are very concerned about slow inputs, or if you don't + want AFL++ to spend too much time classifying that stuff and just rapidly + put all timeouts in that bin. - - `AFL_TMPDIR` is used to write the `.cur_input` file to if exists, and in - the normal output directory otherwise. You would use this to point to - a ramdisk/tmpfs. This increases the speed by a small value but also - reduces the stress on SSDs. + - If you are Jakub, you may need `AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES`. + Others need not apply, unless they also want to disable the + `/proc/sys/kernel/core_pattern` check. - - When developing custom instrumentation on top of afl-fuzz, you can use - `AFL_SKIP_BIN_CHECK` to inhibit the checks for non-instrumented binaries - and shell scripts; and `AFL_DUMB_FORKSRV` in conjunction with the `-n` - setting to instruct afl-fuzz to still follow the fork server protocol - without expecting any instrumentation data in return. - Note that this also turns off auto map size detection. + - If afl-fuzz encounters an incorrect fuzzing setup during a fuzzing session + (not at startup), it will terminate. If you do not want this, then you can + set `AFL_IGNORE_PROBLEMS`. - When running in the `-M` or `-S` mode, setting `AFL_IMPORT_FIRST` causes the - fuzzer to import test cases from other instances before doing anything - else. This makes the "own finds" counter in the UI more accurate. - Beyond counter aesthetics, not much else should change. + fuzzer to import test cases from other instances before doing anything else. + This makes the "own finds" counter in the UI more accurate. Beyond counter + aesthetics, not much else should change. + + - `AFL_KILL_SIGNAL`: Set the signal ID to be delivered to child processes on + timeout. Unless you implement your own targets or instrumentation, you + likely don't have to set it. By default, on timeout and on exit, `SIGKILL` + (`AFL_KILL_SIGNAL=9`) will be delivered to the child. + + - `AFL_MAP_SIZE` sets the size of the shared map that afl-analyze, afl-fuzz, + afl-showmap, and afl-tmin create to gather instrumentation data from the + target. This must be equal or larger than the size the target was compiled + with. + + - Setting `AFL_MAX_DET_EXRAS` will change the threshold at what number of + elements in the `-x` dictionary and LTO autodict (combined) the + probabilistic mode will kick off. In probabilistic mode, not all dictionary + entries will be used all of the time for fuzzing mutations to not slow down + fuzzing. The default count is `200` elements. So for the 200 + 1st element, + there is a 1 in 201 chance, that one of the dictionary entries will not be + used directly. - - Note that `AFL_POST_LIBRARY` is deprecated, use `AFL_CUSTOM_MUTATOR_LIBRARY` - instead (see below). + - Setting `AFL_NO_AFFINITY` disables attempts to bind to a specific CPU core + on Linux systems. This slows things down, but lets you run more instances of + afl-fuzz than would be prudent (if you really want to). - - `AFL_KILL_SIGNAL`: Set the signal ID to be delivered to child processes on timeout. - Unless you implement your own targets or instrumentation, you likely don't have to set it. - By default, on timeout and on exit, `SIGKILL` (`AFL_KILL_SIGNAL=9`) will be delivered to the child. + - `AFL_NO_ARITH` causes AFL++ to skip most of the deterministic arithmetics. + This can be useful to speed up the fuzzing of text-based file formats. - - Setting `AFL_CUSTOM_MUTATOR_LIBRARY` to a shared library with - afl_custom_fuzz() creates additional mutations through this library. - If afl-fuzz is compiled with Python (which is autodetected during building - afl-fuzz), setting `AFL_PYTHON_MODULE` to a Python module can also provide - additional mutations. - If `AFL_CUSTOM_MUTATOR_ONLY` is also set, all mutations will solely be - performed with the custom mutator. - This feature allows to configure custom mutators which can be very helpful, - e.g. fuzzing XML or other highly flexible structured input. - Please see [custom_mutators.md](custom_mutators.md). + - Setting `AFL_NO_AUTODICT` will not load an LTO generated auto dictionary + that is compiled into the target. - - `AFL_FAST_CAL` keeps the calibration stage about 2.5x faster (albeit less - precise), which can help when starting a session against a slow target. - `AFL_CAL_FAST` works too. + - Setting `AFL_NO_COLOR` or `AFL_NO_COLOUR` will omit control sequences for + coloring console output when configured with USE_COLOR and not + ALWAYS_COLORED. - The CPU widget shown at the bottom of the screen is fairly simplistic and may complain of high load prematurely, especially on systems with low core - counts. To avoid the alarming red color, you can set `AFL_NO_CPU_RED`. - - - In QEMU mode (-Q) and Frida mode (-O), `AFL_PATH` will - be searched for afl-qemu-trace and afl-frida-trace.so. + counts. To avoid the alarming red color for very high CPU usages, you can + set `AFL_NO_CPU_RED`. - - In QEMU mode (-Q), setting `AFL_QEMU_CUSTOM_BIN` cause afl-fuzz to skip - prepending `afl-qemu-trace` to your command line. Use this if you wish to use a - custom afl-qemu-trace or if you need to modify the afl-qemu-trace arguments. - - - Setting `AFL_CYCLE_SCHEDULES` will switch to a different schedule everytime - a cycle is finished. - - - Setting `AFL_EXPAND_HAVOC_NOW` will start in the extended havoc mode that - includes costly mutations. afl-fuzz automatically enables this mode when - deemed useful otherwise. + - Setting `AFL_NO_FORKSRV` disables the forkserver optimization, reverting to + fork + execve() call for every tested input. This is useful mostly when + working with unruly libraries that create threads or do other crazy things + when initializing (before the instrumentation has a chance to run). - - Setting `AFL_PRELOAD` causes AFL++ to set `LD_PRELOAD` for the target binary - without disrupting the afl-fuzz process itself. This is useful, among other - things, for bootstrapping libdislocator.so. + Note that this setting inhibits some of the user-friendly diagnostics + normally done when starting up the forkserver and causes a pretty + significant performance drop. - - Setting `AFL_TARGET_ENV` causes AFL++ to set extra environment variables - for the target binary. Example: `AFL_TARGET_ENV="VAR1=1 VAR2='a b c'" afl-fuzz ... ` - This exists mostly for things like `LD_LIBRARY_PATH` but it would theoretically - allow fuzzing of AFL++ itself (with 'target' AFL++ using some AFL_ vars that - would disrupt work of 'fuzzer' AFL++). + - `AFL_NO_SNAPSHOT` will advice afl-fuzz not to use the snapshot feature if + the snapshot lkm is loaded. - - Setting `AFL_NO_UI` inhibits the UI altogether, and just periodically prints + - Setting `AFL_NO_UI` inhibits the UI altogether and just periodically prints some basic stats. This behavior is also automatically triggered when the output from afl-fuzz is redirected to a file or to a pipe. - - Setting `AFL_NO_COLOR` or `AFL_NO_COLOUR` will omit control sequences for - coloring console output when configured with USE_COLOR and not ALWAYS_COLORED. - - - Setting `AFL_FORCE_UI` will force painting the UI on the screen even if - no valid terminal was detected (for virtual consoles) + - In QEMU mode (-Q) and Frida mode (-O), `AFL_PATH` will be searched for + afl-qemu-trace and afl-frida-trace.so. - - If you are using persistent mode (you should, see [instrumentation/README.persistent_mode.md](instrumentation/README.persistent_mode.md)) - some targets keep inherent state due which a detected crash testcase does - not crash the target again when the testcase is given. To be able to still - re-trigger these crashes you can use the `AFL_PERSISTENT_RECORD` variable - with a value of how many previous fuzz cases to keep prio a crash. - if set to e.g. 10, then the 9 previous inputs are written to - out/default/crashes as RECORD:000000,cnt:000000 to RECORD:000000,cnt:000008 - and RECORD:000000,cnt:000009 being the crash case. - NOTE: This option needs to be enabled in config.h first! + - If you are using persistent mode (you should, see + [instrumentation/README.persistent_mode.md](../instrumentation/README.persistent_mode.md)), + some targets keep inherent state due which a detected crash test case does + not crash the target again when the test case is given. To be able to still + re-trigger these crashes, you can use the `AFL_PERSISTENT_RECORD` variable + with a value of how many previous fuzz cases to keep prio a crash. If set to + e.g., 10, then the 9 previous inputs are written to out/default/crashes as + RECORD:000000,cnt:000000 to RECORD:000000,cnt:000008 and + RECORD:000000,cnt:000009 being the crash case. NOTE: This option needs to be + enabled in config.h first! - - If afl-fuzz encounters an incorrect fuzzing setup during a fuzzing session - (not at startup), it will terminate. If you do not want this then you can - set `AFL_IGNORE_PROBLEMS`. - - - If you are Jakub, you may need `AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES`. - Others need not apply, unless they also want to disable the - `/proc/sys/kernel/core_pattern` check. - - - Benchmarking only: `AFL_BENCH_JUST_ONE` causes the fuzzer to exit after - processing the first queue entry; and `AFL_BENCH_UNTIL_CRASH` causes it to - exit soon after the first crash is found. - - - Setting `AFL_DEBUG_CHILD` will not suppress the child output. - This lets you see all output of the child, making setup issues obvious. - For example, in an unicornafl harness, you might see python stacktraces. - You may also see other logs that way, indicating why the forkserver won't start. - Not pretty but good for debugging purposes. - Note that `AFL_DEBUG_CHILD_OUTPUT` is deprecated. + - Note that `AFL_POST_LIBRARY` is deprecated, use `AFL_CUSTOM_MUTATOR_LIBRARY` + instead. - - Setting `AFL_NO_CPU_RED` will not display very high cpu usages in red color. + - Setting `AFL_PRELOAD` causes AFL++ to set `LD_PRELOAD` for the target binary + without disrupting the afl-fuzz process itself. This is useful, among other + things, for bootstrapping libdislocator.so. - - Setting `AFL_AUTORESUME` will resume a fuzz run (same as providing `-i -`) - for an existing out folder, even if a different `-i` was provided. - Without this setting, afl-fuzz will refuse execution for a long-fuzzed out dir. + - In QEMU mode (-Q), setting `AFL_QEMU_CUSTOM_BIN` will cause afl-fuzz to skip + prepending `afl-qemu-trace` to your command line. Use this if you wish to + use a custom afl-qemu-trace or if you need to modify the afl-qemu-trace + arguments. - - Setting `AFL_MAX_DET_EXRAS` will change the threshold at what number of elements - in the `-x` dictionary and LTO autodict (combined) the probabilistic mode will - kick off. In probabilistic mode, not all dictionary entries will be used all - of the time for fuzzing mutations to not slow down fuzzing. - The default count is `200` elements. So for the 200 + 1st element, there is a - 1 in 201 chance, that one of the dictionary entries will not be used directly. + - `AFL_SHUFFLE_QUEUE` randomly reorders the input queue on startup. Requested + by some users for unorthodox parallelized fuzzing setups, but not advisable + otherwise. - - Setting `AFL_NO_FORKSRV` disables the forkserver optimization, reverting to - fork + execve() call for every tested input. This is useful mostly when - working with unruly libraries that create threads or do other crazy - things when initializing (before the instrumentation has a chance to run). + - When developing custom instrumentation on top of afl-fuzz, you can use + `AFL_SKIP_BIN_CHECK` to inhibit the checks for non-instrumented binaries and + shell scripts; and `AFL_DUMB_FORKSRV` in conjunction with the `-n` setting + to instruct afl-fuzz to still follow the fork server protocol without + expecting any instrumentation data in return. Note that this also turns off + auto map size detection. - Note that this setting inhibits some of the user-friendly diagnostics - normally done when starting up the forkserver and causes a pretty - significant performance drop. + - Setting `AFL_SKIP_CPUFREQ` skips the check for CPU scaling policy. This is + useful if you can't change the defaults (e.g., no root access to the system) + and are OK with some performance loss. + + - Setting `AFL_STATSD` enables StatsD metrics collection. By default, AFL++ + will send these metrics over UDP to 127.0.0.1:8125. The host and port are + configurable with `AFL_STATSD_HOST` and `AFL_STATSD_PORT` respectively. To + enable tags (banner and afl_version), you should provide + `AFL_STATSD_TAGS_FLAVOR` that matches your StatsD server (see + `AFL_STATSD_TAGS_FLAVOR`). + + - Setting `AFL_STATSD_TAGS_FLAVOR` to one of `dogstatsd`, `influxdb`, + `librato`, or `signalfx` allows you to add tags to your fuzzing instances. + This is especially useful when running multiple instances (`-M/-S` for + example). Applied tags are `banner` and `afl_version`. `banner` corresponds + to the name of the fuzzer provided through `-M/-S`. `afl_version` + corresponds to the currently running AFL++ version (e.g., `++3.0c`). Default + (empty/non present) will add no tags to the metrics. For more information, + see [rpc_statsd.md](rpc_statsd.md). + + - Setting `AFL_TARGET_ENV` causes AFL++ to set extra environment variables for + the target binary. Example: `AFL_TARGET_ENV="VAR1=1 VAR2='a b c'" afl-fuzz + ... `. This exists mostly for things like `LD_LIBRARY_PATH` but it would + theoretically allow fuzzing of AFL++ itself (with 'target' AFL++ using some + AFL_ vars that would disrupt work of 'fuzzer' AFL++). + + - `AFL_TESTCACHE_SIZE` allows you to override the size of `#define + TESTCASE_CACHE` in config.h. Recommended values are 50-250MB - or more if + your fuzzing finds a huge amount of paths for large inputs. + + - `AFL_TMPDIR` is used to write the `.cur_input` file to if it exists, and in + the normal output directory otherwise. You would use this to point to a + ramdisk/tmpfs. This increases the speed by a small value but also reduces + the stress on SSDs. - - Setting `AFL_STATSD` enables StatsD metrics collection. - By default AFL++ will send these metrics over UDP to 127.0.0.1:8125. - The host and port are configurable with `AFL_STATSD_HOST` and `AFL_STATSD_PORT` respectively. - To enable tags (banner and afl_version) you should provide `AFL_STATSD_TAGS_FLAVOR` that matches - your StatsD server (see `AFL_STATSD_TAGS_FLAVOR`) - - - Setting `AFL_STATSD_TAGS_FLAVOR` to one of `dogstatsd`, `librato`, `signalfx` or `influxdb` - allows you to add tags to your fuzzing instances. This is especially useful when running - multiple instances (`-M/-S` for example). Applied tags are `banner` and `afl_version`. - `banner` corresponds to the name of the fuzzer provided through `-M/-S`. - `afl_version` corresponds to the currently running AFL version (e.g `++3.0c`). - Default (empty/non present) will add no tags to the metrics. - See [rpc_statsd.md](rpc_statsd.md) for more information. - - - Setting `AFL_CRASH_EXITCODE` sets the exit code AFL treats as crash. - For example, if `AFL_CRASH_EXITCODE='-1'` is set, each input resulting - in an `-1` return code (i.e. `exit(-1)` got called), will be treated - as if a crash had ocurred. - This may be beneficial if you look for higher-level faulty conditions in which your - target still exits gracefully. + - Setting `AFL_TRY_AFFINITY` tries to attempt binding to a specific CPU core + on Linux systems, but will not terminate if that fails. - Outdated environment variables that are not supported anymore: - `AFL_DEFER_FORKSRV` - `AFL_PERSISTENT` + - `AFL_DEFER_FORKSRV` + - `AFL_PERSISTENT` ## 5) Settings for afl-qemu-trace The QEMU wrapper used to instrument binary-only code supports several settings: - - It is possible to set `AFL_INST_RATIO` to skip the instrumentation on some - of the basic blocks, which can be useful when dealing with very complex - binaries. - - - Setting `AFL_INST_LIBS` causes the translator to also instrument the code - inside any dynamically linked libraries (notably including glibc). - - Setting `AFL_COMPCOV_LEVEL` enables the CompareCoverage tracing of all cmp - and sub in x86 and x86_64 and memory comparions functions (e.g. strcmp, - memcmp, ...) when libcompcov is preloaded using `AFL_PRELOAD`. - More info at qemu_mode/libcompcov/README.md. + and sub in x86 and x86_64 and memory comparison functions (e.g., strcmp, + memcmp, ...) when libcompcov is preloaded using `AFL_PRELOAD`. More info at + [qemu_mode/libcompcov/README.md](../qemu_mode/libcompcov/README.md). + There are two levels at the moment, `AFL_COMPCOV_LEVEL=1` that instruments only comparisons with immediate values / read-only memory and - `AFL_COMPCOV_LEVEL=2` that instruments all the comparions. Level 2 is more + `AFL_COMPCOV_LEVEL=2` that instruments all the comparisons. Level 2 is more accurate but may need a larger shared memory. - - Setting `AFL_QEMU_COMPCOV` enables the CompareCoverage tracing of all - cmp and sub in x86 and x86_64. - This is an alias of `AFL_COMPCOV_LEVEL=1` when `AFL_COMPCOV_LEVEL` is - not specified. + - `AFL_DEBUG` will print the found entry point for the binary to stderr. Use + this if you are unsure if the entry point might be wrong - but use it + directly, e.g., `afl-qemu-trace ./program`. - - The underlying QEMU binary will recognize any standard "user space - emulation" variables (e.g., `QEMU_STACK_SIZE`), but there should be no - reason to touch them. + - `AFL_ENTRYPOINT` allows you to specify a specific entry point into the + binary (this can be very good for the performance!). The entry point is + specified as hex address, e.g., `0x4004110`. Note that the address must be + the address of a basic block. + + - Setting `AFL_INST_LIBS` causes the translator to also instrument the code + inside any dynamically linked libraries (notably including glibc). + + - It is possible to set `AFL_INST_RATIO` to skip the instrumentation on some + of the basic blocks, which can be useful when dealing with very complex + binaries. - - `AFL_DEBUG` will print the found entrypoint for the binary to stderr. - Use this if you are unsure if the entrypoint might be wrong - but - use it directly, e.g. `afl-qemu-trace ./program` + - Setting `AFL_QEMU_COMPCOV` enables the CompareCoverage tracing of all cmp + and sub in x86 and x86_64. This is an alias of `AFL_COMPCOV_LEVEL=1` when + `AFL_COMPCOV_LEVEL` is not specified. - - `AFL_ENTRYPOINT` allows you to specify a specific entrypoint into the - binary (this can be very good for the performance!). - The entrypoint is specified as hex address, e.g. `0x4004110` - Note that the address must be the address of a basic block. + - With `AFL_QEMU_FORCE_DFL`, you force QEMU to ignore the registered signal + handlers of the target. - - When the target is i386/x86_64 you can specify the address of the function + - When the target is i386/x86_64, you can specify the address of the function that has to be the body of the persistent loop using `AFL_QEMU_PERSISTENT_ADDR=start addr`. - - Another modality to execute the persistent loop is to specify also the - `AFL_QEMU_PERSISTENT_RET=end addr` env variable. - With this variable assigned, instead of patching the return address, the - specified instruction is transformed to a jump towards `start addr`. + - With `AFL_QEMU_PERSISTENT_GPR=1`, QEMU will save the original value of + general purpose registers and restore them in each persistent cycle. - - `AFL_QEMU_PERSISTENT_GPR=1` QEMU will save the original value of general - purpose registers and restore them in each persistent cycle. + - Another modality to execute the persistent loop is to specify also the + `AFL_QEMU_PERSISTENT_RET=end addr` environment variable. With this variable + assigned, instead of patching the return address, the specified instruction + is transformed to a jump towards `start addr`. - - With `AFL_QEMU_PERSISTENT_RETADDR_OFFSET` you can specify the offset from the - stack pointer in which QEMU can find the return address when `start addr` is - hit. + - With `AFL_QEMU_PERSISTENT_RETADDR_OFFSET`, you can specify the offset from + the stack pointer in which QEMU can find the return address when `start + addr` is hit. - - With `AFL_USE_QASAN` you can enable QEMU AddressSanitizer for dynamically + - With `AFL_USE_QASAN`, you can enable QEMU AddressSanitizer for dynamically linked binaries. - - With `AFL_QEMU_FORCE_DFL` you force QEMU to ignore the registered signal - handlers of the target. + - The underlying QEMU binary will recognize any standard "user space + emulation" variables (e.g., `QEMU_STACK_SIZE`), but there should be no + reason to touch them. -## 6) Settings for afl-cmin +## 7) Settings for afl-frida-trace + +The FRIDA wrapper used to instrument binary-only code supports many of the same +options as `afl-qemu-trace`, but also has a number of additional advanced +options. These are listed in brief below (see +[frida_mode/README.md](../frida_mode/README.md) for more details). These +settings are provided for compatibility with QEMU mode, the preferred way to +configure FRIDA mode is through its [scripting](../frida_mode/Scripting.md) +support. + +* `AFL_FRIDA_DEBUG_MAPS` - See `AFL_QEMU_DEBUG_MAPS` +* `AFL_FRIDA_DRIVER_NO_HOOK` - See `AFL_QEMU_DRIVER_NO_HOOK`. When using the + QEMU driver to provide a `main` loop for a user provided + `LLVMFuzzerTestOneInput`, this option configures the driver to read input from + `stdin` rather than using in-memory test cases. +* `AFL_FRIDA_EXCLUDE_RANGES` - See `AFL_QEMU_EXCLUDE_RANGES` +* `AFL_FRIDA_INST_COVERAGE_FILE` - File to write DynamoRio format coverage + information (e.g., to be loaded within IDA lighthouse). +* `AFL_FRIDA_INST_DEBUG_FILE` - File to write raw assembly of original blocks + and their instrumented counterparts during block compilation. +* `AFL_FRIDA_INST_JIT` - Enable the instrumentation of Just-In-Time compiled + code. Code is considered to be JIT if the executable segment is not backed by + a file. +* `AFL_FRIDA_INST_NO_OPTIMIZE` - Don't use optimized inline assembly coverage + instrumentation (the default where available). Required to use + `AFL_FRIDA_INST_TRACE`. +* `AFL_FRIDA_INST_NO_BACKPATCH` - Disable backpatching. At the end of executing + each block, control will return to FRIDA to identify the next block to + execute. +* `AFL_FRIDA_INST_NO_PREFETCH` - Disable prefetching. By default, the child will + report instrumented blocks back to the parent so that it can also instrument + them and they be inherited by the next child on fork, implies + `AFL_FRIDA_INST_NO_PREFETCH_BACKPATCH`. +* `AFL_FRIDA_INST_NO_PREFETCH_BACKPATCH` - Disable prefetching of stalker + backpatching information. By default, the child will report applied + backpatches to the parent so that they can be applied and then be inherited by + the next child on fork. +* `AFL_FRIDA_INST_RANGES` - See `AFL_QEMU_INST_RANGES` +* `AFL_FRIDA_INST_SEED` - Sets the initial seed for the hash function used to + generate block (and hence edge) IDs. Setting this to a constant value may be + useful for debugging purposes, e.g., investigating unstable edges. +* `AFL_FRIDA_INST_TRACE` - Log to stdout the address of executed blocks, implies + `AFL_FRIDA_INST_NO_OPTIMIZE`. +* `AFL_FRIDA_INST_TRACE_UNIQUE` - As per `AFL_FRIDA_INST_TRACE`, but each edge + is logged only once, requires `AFL_FRIDA_INST_NO_OPTIMIZE`. +* `AFL_FRIDA_INST_UNSTABLE_COVERAGE_FILE` - File to write DynamoRio format + coverage information for unstable edges (e.g., to be loaded within IDA + lighthouse). +* `AFL_FRIDA_JS_SCRIPT` - Set the script to be loaded by the FRIDA scripting + engine. See [frida_mode/Scripting.md](../frida_mode/Scripting.md) for details. +* `AFL_FRIDA_OUTPUT_STDOUT` - Redirect the standard output of the target + application to the named file (supersedes the setting of `AFL_DEBUG_CHILD`) +* `AFL_FRIDA_OUTPUT_STDERR` - Redirect the standard error of the target + application to the named file (supersedes the setting of `AFL_DEBUG_CHILD`) +* `AFL_FRIDA_PERSISTENT_ADDR` - See `AFL_QEMU_PERSISTENT_ADDR` +* `AFL_FRIDA_PERSISTENT_CNT` - See `AFL_QEMU_PERSISTENT_CNT` +* `AFL_FRIDA_PERSISTENT_DEBUG` - Insert a Breakpoint into the instrumented code + at `AFL_FRIDA_PERSISTENT_HOOK` and `AFL_FRIDA_PERSISTENT_RET` to allow the + user to detect issues in the persistent loop using a debugger. +* `AFL_FRIDA_PERSISTENT_HOOK` - See `AFL_QEMU_PERSISTENT_HOOK` +* `AFL_FRIDA_PERSISTENT_RET` - See `AFL_QEMU_PERSISTENT_RET` +* `AFL_FRIDA_SECCOMP_FILE` - Write a log of any syscalls made by the target to + the specified file. +* `AFL_FRIDA_STALKER_ADJACENT_BLOCKS` - Configure the number of adjacent blocks + to fetch when generating instrumented code. By fetching blocks in the same + order they appear in the original program, rather than the order of execution + should help reduce locallity and adjacency. This includes allowing us to + vector between adjancent blocks using a NOP slide rather than an immediate + branch. +* `AFL_FRIDA_STALKER_IC_ENTRIES` - Configure the number of inline cache entries + stored along-side branch instructions which provide a cache to avoid having to + call back into FRIDA to find the next block. Default is 32. +* `AFL_FRIDA_STATS_FILE` - Write statistics information about the code being + instrumented to the given file name. The statistics are written only for the + child process when new block is instrumented (when the + `AFL_FRIDA_STATS_INTERVAL` has expired). Note that just because a new path is + found does not mean a new block needs to be compiled. It could be that the + existing blocks instrumented have been executed in a different order. +* `AFL_FRIDA_STATS_INTERVAL` - The maximum frequency to output statistics + information. Stats will be written whenever they are updated if the given + interval has elapsed since last time they were written. +* `AFL_FRIDA_TRACEABLE` - Set the child process to be traceable by any process + to aid debugging and overcome the restrictions imposed by YAMA. Supported on + Linux only. Permits a non-root user to use `gcore` or similar to collect a + core dump of the instrumented target. Note that in order to capture the core + dump you must set a sufficient timeout (using `-t`) to avoid `afl-fuzz` + killing the process whilst it is being dumped. + +## 8) Settings for afl-cmin The corpus minimization script offers very little customization: - - Setting `AFL_PATH` offers a way to specify the location of afl-showmap - and afl-qemu-trace (the latter only in `-Q` mode). + - `AFL_ALLOW_TMP` permits this and some other scripts to run in /tmp. This is + a modest security risk on multi-user systems with rogue users, but should be + safe on dedicated fuzzing boxes. - `AFL_KEEP_TRACES` makes the tool keep traces and other metadata used for minimization and normally deleted at exit. The files can be found in the `<out_dir>/.traces/` directory. - - `AFL_ALLOW_TMP` permits this and some other scripts to run in /tmp. This is - a modest security risk on multi-user systems with rogue users, but should - be safe on dedicated fuzzing boxes. + - Setting `AFL_PATH` offers a way to specify the location of afl-showmap and + afl-qemu-trace (the latter only in `-Q` mode). - `AFL_PRINT_FILENAMES` prints each filename to stdout, as it gets processed. - This can help when embedding `afl-cmin` or `afl-showmap` in other scripts scripting. + This can help when embedding `afl-cmin` or `afl-showmap` in other scripts. -## 7) Settings for afl-tmin +## 9) Settings for afl-tmin Virtually nothing to play with. Well, in QEMU mode (`-Q`), `AFL_PATH` will be searched for afl-qemu-trace. In addition to this, `TMPDIR` may be used if a @@ -587,77 +696,81 @@ to match when minimizing crashes. This will make minimization less useful, but may prevent the tool from "jumping" from one crashing condition to another in very buggy software. You probably want to combine it with the `-e` flag. -## 8) Settings for afl-analyze +## 10) Settings for afl-analyze You can set `AFL_ANALYZE_HEX` to get file offsets printed as hexadecimal instead of decimal. -## 9) Settings for libdislocator +## 11) Settings for libdislocator -The library honors these environmental variables: +The library honors these environment variables: - - `AFL_LD_LIMIT_MB` caps the size of the maximum heap usage permitted by the - library, in megabytes. The default value is 1 GB. Once this is exceeded, - allocations will return NULL. + - `AFL_ALIGNED_ALLOC=1` will force the alignment of the allocation size to + `max_align_t` to be compliant with the C standard. - `AFL_LD_HARD_FAIL` alters the behavior by calling `abort()` on excessive allocations, thus causing what AFL++ would perceive as a crash. Useful for programs that are supposed to maintain a specific memory footprint. - - `AFL_LD_VERBOSE` causes the library to output some diagnostic messages - that may be useful for pinpointing the cause of any observed issues. + - `AFL_LD_LIMIT_MB` caps the size of the maximum heap usage permitted by the + library, in megabytes. The default value is 1 GB. Once this is exceeded, + allocations will return NULL. - - `AFL_LD_NO_CALLOC_OVER` inhibits `abort()` on `calloc()` overflows. Most - of the common allocators check for that internally and return NULL, so - it's a security risk only in more exotic setups. + - `AFL_LD_NO_CALLOC_OVER` inhibits `abort()` on `calloc()` overflows. Most of + the common allocators check for that internally and return NULL, so it's a + security risk only in more exotic setups. - - `AFL_ALIGNED_ALLOC=1` will force the alignment of the allocation size to - `max_align_t` to be compliant with the C standard. + - `AFL_LD_VERBOSE` causes the library to output some diagnostic messages that + may be useful for pinpointing the cause of any observed issues. -## 10) Settings for libtokencap +## 11) Settings for libtokencap This library accepts `AFL_TOKEN_FILE` to indicate the location to which the discovered tokens should be written. -## 11) Third-party variables set by afl-fuzz & other tools +## 12) Third-party variables set by afl-fuzz & other tools Several variables are not directly interpreted by afl-fuzz, but are set to optimal values if not already present in the environment: - - By default, `LD_BIND_NOW` is set to speed up fuzzing by forcing the - linker to do all the work before the fork server kicks in. You can - override this by setting `LD_BIND_LAZY` beforehand, but it is almost - certainly pointless. - - By default, `ASAN_OPTIONS` are set to (among others): -``` + + ``` abort_on_error=1 detect_leaks=0 malloc_context_size=0 symbolize=0 allocator_may_return_null=1 -``` - If you want to set your own options, be sure to include `abort_on_error=1` - - otherwise, the fuzzer will not be able to detect crashes in the tested - app. Similarly, include `symbolize=0`, since without it, AFL++ may have + ``` + + If you want to set your own options, be sure to include `abort_on_error=1` - + otherwise, the fuzzer will not be able to detect crashes in the tested app. + Similarly, include `symbolize=0`, since without it, AFL++ may have difficulty telling crashes and hangs apart. + - Similarly, the default `LSAN_OPTIONS` are set to: + + ``` + exit_code=23 + fast_unwind_on_malloc=0 + symbolize=0 + print_suppressions=0 + ``` + + Be sure to include the first ones for LSAN and MSAN when customizing + anything, since some MSAN and LSAN versions don't call `abort()` on error, + and we need a way to detect faults. + - In the same vein, by default, `MSAN_OPTIONS` are set to: -``` + + ``` exit_code=86 (required for legacy reasons) abort_on_error=1 symbolize=0 msan_track_origins=0 allocator_may_return_null=1 -``` - - Similarly, the default `LSAN_OPTIONS` are set to: -``` - exit_code=23 - fast_unwind_on_malloc=0 - symbolize=0 - print_suppressions=0 -``` - Be sure to include the first ones for LSAN and MSAN when customizing - anything, since some MSAN and LSAN versions don't call `abort()` on - error, and we need a way to detect faults. + ``` + - By default, `LD_BIND_NOW` is set to speed up fuzzing by forcing the linker + to do all the work before the fork server kicks in. You can override this by + setting `LD_BIND_LAZY` beforehand, but it is almost certainly pointless. diff --git a/docs/features.md b/docs/features.md new file mode 100644 index 00000000..431d9eb1 --- /dev/null +++ b/docs/features.md @@ -0,0 +1,61 @@ +# Important features of AFL++ + +AFL++ supports llvm from 3.8 up to version 12, very fast binary fuzzing with +QEMU 5.1 with laf-intel and redqueen, FRIDA mode, unicorn mode, gcc plugin, full +*BSD, Mac OS, Solaris and Android support and much, much, much more. + +| Feature/Instrumentation | afl-gcc | llvm | gcc_plugin | FRIDA mode(9) | QEMU mode(10) |unicorn_mode(10) |coresight_mode(11)| +| -------------------------|:-------:|:---------:|:----------:|:----------------:|:----------------:|:----------------:|:----------------:| +| Threadsafe counters | | x(3) | | | | | | +| NeverZero | x86[_64]| x(1) | x | x | x | x | | +| Persistent Mode | | x | x | x86[_64]/arm64 | x86[_64]/arm[64] | x | | +| LAF-Intel / CompCov | | x | | | x86[_64]/arm[64] | x86[_64]/arm[64] | | +| CmpLog | | x | | x86[_64]/arm64 | x86[_64]/arm[64] | | | +| Selective Instrumentation| | x | x | x | x | | | +| Non-Colliding Coverage | | x(4) | | | (x)(5) | | | +| Ngram prev_loc Coverage | | x(6) | | | | | | +| Context Coverage | | x(6) | | | | | | +| Auto Dictionary | | x(7) | | | | | | +| Snapshot LKM Support | | (x)(8) | (x)(8) | | (x)(5) | | | +| Shared Memory Test cases | | x | x | x86[_64]/arm64 | x | x | | + +1. default for LLVM >= 9.0, environment variable for older version due an + efficiency bug in previous llvm versions +2. GCC creates non-performant code, hence it is disabled in gcc_plugin +3. with `AFL_LLVM_THREADSAFE_INST`, disables NeverZero +4. with pcguard mode and LTO mode for LLVM 11 and newer +5. upcoming, development in the branch +6. not compatible with LTO instrumentation and needs at least LLVM v4.1 +7. automatic in LTO mode with LLVM 11 and newer, an extra pass for all LLVM + versions that write to a file to use with afl-fuzz' `-x` +8. the snapshot LKM is currently unmaintained due to too many kernel changes + coming too fast :-( +9. FRIDA mode is supported on Linux and MacOS for Intel and ARM +10. QEMU/Unicorn is only supported on Linux +11. Coresight mode is only available on AARCH64 Linux with a CPU with Coresight + extension + +Among others, the following features and patches have been integrated: + +* NeverZero patch for afl-gcc, instrumentation, QEMU mode and unicorn_mode which + prevents a wrapping map value to zero, increases coverage +* Persistent mode, deferred forkserver and in-memory fuzzing for QEMU mode +* Unicorn mode which allows fuzzing of binaries from completely different + platforms (integration provided by domenukk) +* The new CmpLog instrumentation for LLVM and QEMU inspired by + [Redqueen](https://www.syssec.ruhr-uni-bochum.de/media/emma/veroeffentlichungen/2018/12/17/NDSS19-Redqueen.pdf) +* Win32 PE binary-only fuzzing with QEMU and Wine +* AFLfast's power schedules by Marcel Böhme: + [https://github.com/mboehme/aflfast](https://github.com/mboehme/aflfast) +* The MOpt mutator: + [https://github.com/puppet-meteor/MOpt-AFL](https://github.com/puppet-meteor/MOpt-AFL) +* LLVM mode Ngram coverage by Adrian Herrera + [https://github.com/adrianherrera/afl-ngram-pass](https://github.com/adrianherrera/afl-ngram-pass) +* LAF-Intel/CompCov support for instrumentation, QEMU mode and unicorn_mode + (with enhanced capabilities) +* Radamsa and honggfuzz mutators (as custom mutators). +* QBDI mode to fuzz android native libraries via Quarkslab's + [QBDI](https://github.com/QBDI/QBDI) framework +* Frida and ptrace mode to fuzz binary-only libraries, etc. + +So all in all this is the best-of AFL that is out there :-) \ No newline at end of file diff --git a/docs/fuzzing_binary-only_targets.md b/docs/fuzzing_binary-only_targets.md new file mode 100644 index 00000000..5434a22c --- /dev/null +++ b/docs/fuzzing_binary-only_targets.md @@ -0,0 +1,296 @@ +# Fuzzing binary-only targets + +AFL++, libfuzzer, and other fuzzers are great if you have the source code of the +target. This allows for very fast and coverage guided fuzzing. + +However, if there is only the binary program and no source code available, then +standard `afl-fuzz -n` (non-instrumented mode) is not effective. + +For fast, on-the-fly instrumentation of black-box binaries, AFL++ still offers +various support. The following is a description of how these binaries can be +fuzzed with AFL++. + +## TL;DR: + +QEMU mode in persistent mode is the fastest - if the stability is high enough. +Otherwise, try RetroWrite, Dyninst, and if these fail, too, then try standard +QEMU mode with `AFL_ENTRYPOINT` to where you need it. + +If your target is a library, then use FRIDA mode. + +If your target is non-linux, then use unicorn_mode. + +## Fuzzing binary-only targets with AFL++ + +### QEMU mode + +QEMU mode is the "native" solution to the program. It is available in the +./qemu_mode/ directory and, once compiled, it can be accessed by the afl-fuzz -Q +command line option. It is the easiest to use alternative and even works for +cross-platform binaries. + +For linux programs and its libraries, this is accomplished with a version of +QEMU running in the lesser-known "user space emulation" mode. QEMU is a project +separate from AFL++, but you can conveniently build the feature by doing: + +```shell +cd qemu_mode +./build_qemu_support.sh +``` + +The following setup to use QEMU mode is recommended: + +* run 1 afl-fuzz -Q instance with CMPLOG (`-c 0` + `AFL_COMPCOV_LEVEL=2`) +* run 1 afl-fuzz -Q instance with QASAN (`AFL_USE_QASAN=1`) +* run 1 afl-fuzz -Q instance with LAF (`AFL_PRELOAD=libcmpcov.so` + + `AFL_COMPCOV_LEVEL=2`), alternatively you can use FRIDA mode, just switch `-Q` + with `-O` and remove the LAF instance + +Then run as many instances as you have cores left with either -Q mode or - even +better - use a binary rewriter like Dyninst, RetroWrite, ZAFL, etc. + +If [afl-dyninst](https://github.com/vanhauser-thc/afl-dyninst) works for your +binary, then you can use afl-fuzz normally and it will have twice the speed +compared to QEMU mode (but slower than QEMU persistent mode). Note that several +other binary rewriters exist, all with their advantages and caveats. + +The speed decrease of QEMU mode is at about 50%. However, various options exist +to increase the speed: +- using AFL_ENTRYPOINT to move the forkserver entry to a later basic block in + the binary (+5-10% speed) +- using persistent mode + [qemu_mode/README.persistent.md](../qemu_mode/README.persistent.md) this will + result in a 150-300% overall speed increase - so 3-8x the original QEMU mode + speed! +- using AFL_CODE_START/AFL_CODE_END to only instrument specific parts + +For additional instructions and caveats, see +[qemu_mode/README.md](../qemu_mode/README.md). If possible, you should use the +persistent mode, see +[qemu_mode/README.persistent.md](../qemu_mode/README.persistent.md). The mode is +approximately 2-5x slower than compile-time instrumentation, and is less +conducive to parallelization. + +Note that there is also honggfuzz: +[https://github.com/google/honggfuzz](https://github.com/google/honggfuzz) which +now has a QEMU mode, but its performance is just 1.5% ... + +If you like to code a customized fuzzer without much work, we highly recommend +to check out our sister project libafl which supports QEMU, too: +[https://github.com/AFLplusplus/LibAFL](https://github.com/AFLplusplus/LibAFL) + +### WINE+QEMU + +Wine mode can run Win32 PE binaries with the QEMU instrumentation. It needs +Wine, python3, and the pefile python package installed. + +It is included in AFL++. + +For more information, see +[qemu_mode/README.wine.md](../qemu_mode/README.wine.md). + +### FRIDA mode + +In FRIDA mode, you can fuzz binary-only targets as easily as with QEMU mode. +FRIDA mode is sometimes faster and sometimes slower than QEMU mode. It is also +newer, lacks COMPCOV, and has the advantage that it works on MacOS (both intel +and M1). + +To build FRIDA mode: + +```shell +cd frida_mode +make +``` + +For additional instructions and caveats, see +[frida_mode/README.md](../frida_mode/README.md). + +If possible, you should use the persistent mode, see +[instrumentation/README.persistent_mode.md](../instrumentation/README.persistent_mode.md). +The mode is approximately 2-5x slower than compile-time instrumentation, and is +less conducive to parallelization. But for binary-only fuzzing, it gives a huge +speed improvement if it is possible to use. + +If you want to fuzz a binary-only library, then you can fuzz it with frida-gum +via frida_mode/. You will have to write a harness to call the target function in +the library, use afl-frida.c as a template. + +You can also perform remote fuzzing with frida, e.g., if you want to fuzz on +iPhone or Android devices, for this you can use +[https://github.com/ttdennis/fpicker/](https://github.com/ttdennis/fpicker/) as +an intermediate that uses AFL++ for fuzzing. + +If you like to code a customized fuzzer without much work, we highly recommend +to check out our sister project libafl which supports Frida, too: +[https://github.com/AFLplusplus/LibAFL](https://github.com/AFLplusplus/LibAFL). +Working examples already exist :-) + +### Unicorn + +Unicorn is a fork of QEMU. The instrumentation is, therefore, very similar. In +contrast to QEMU, Unicorn does not offer a full system or even userland +emulation. Runtime environment and/or loaders have to be written from scratch, +if needed. On top, block chaining has been removed. This means the speed boost +introduced in the patched QEMU Mode of AFL++ cannot be ported over to Unicorn. + +For non-Linux binaries, you can use AFL++'s unicorn_mode which can emulate +anything you want - for the price of speed and user written scripts. + +To build unicorn_mode: + +```shell +cd unicorn_mode +./build_unicorn_support.sh +``` + +For further information, check out +[unicorn_mode/README.md](../unicorn_mode/README.md). + +### Shared libraries + +If the goal is to fuzz a dynamic library, then there are two options available. +For both, you need to write a small harness that loads and calls the library. +Then you fuzz this with either FRIDA mode or QEMU mode and either use +`AFL_INST_LIBS=1` or `AFL_QEMU/FRIDA_INST_RANGES`. + +Another, less precise and slower option is to fuzz it with utils/afl_untracer/ +and use afl-untracer.c as a template. It is slower than FRIDA mode. + +For more information, see +[utils/afl_untracer/README.md](../utils/afl_untracer/README.md). + +### Coresight + +Coresight is ARM's answer to Intel's PT. With AFL++ v3.15, there is a coresight +tracer implementation available in `coresight_mode/` which is faster than QEMU, +however, cannot run in parallel. Currently, only one process can be traced, it +is WIP. + +Fore more information, see +[coresight_mode/README.md](../coresight_mode/README.md). + +## Binary rewriters + +An alternative solution are binary rewriters. They are faster than the solutions +native to AFL++ but don't always work. + +### ZAFL + +ZAFL is a static rewriting platform supporting x86-64 C/C++, +stripped/unstripped, and PIE/non-PIE binaries. Beyond conventional +instrumentation, ZAFL's API enables transformation passes (e.g., laf-Intel, +context sensitivity, InsTrim, etc.). + +Its baseline instrumentation speed typically averages 90-95% of +afl-clang-fast's. + +[https://git.zephyr-software.com/opensrc/zafl](https://git.zephyr-software.com/opensrc/zafl) + +### RetroWrite + +If you have an x86/x86_64 binary that still has its symbols, is compiled with +position independent code (PIC/PIE), and does not use most of the C++ features, +then the RetroWrite solution might be for you. It decompiles to ASM files which +can then be instrumented with afl-gcc. + +It is at about 80-85% performance. + +[https://github.com/HexHive/retrowrite](https://github.com/HexHive/retrowrite) + +### Dyninst + +Dyninst is a binary instrumentation framework similar to Pintool and DynamoRIO. +However, whereas Pintool and DynamoRIO work at runtime, Dyninst instruments the +target at load time and then let it run - or save the binary with the changes. +This is great for some things, e.g., fuzzing, and not so effective for others, +e.g., malware analysis. + +So, what you can do with Dyninst is taking every basic block and putting AFL++'s +instrumentation code in there - and then save the binary. Afterwards, just fuzz +the newly saved target binary with afl-fuzz. Sounds great? It is. The issue +though - it is a non-trivial problem to insert instructions, which change +addresses in the process space, so that everything is still working afterwards. +Hence, more often than not binaries crash when they are run. + +The speed decrease is about 15-35%, depending on the optimization options used +with afl-dyninst. + +[https://github.com/vanhauser-thc/afl-dyninst](https://github.com/vanhauser-thc/afl-dyninst) + +### Mcsema + +Theoretically, you can also decompile to llvm IR with mcsema, and then use +llvm_mode to instrument the binary. Good luck with that. + +[https://github.com/lifting-bits/mcsema](https://github.com/lifting-bits/mcsema) + +## Binary tracers + +### Pintool & DynamoRIO + +Pintool and DynamoRIO are dynamic instrumentation engines. They can be used for +getting basic block information at runtime. Pintool is only available for Intel +x32/x64 on Linux, Mac OS, and Windows, whereas DynamoRIO is additionally +available for ARM and AARCH64. DynamoRIO is also 10x faster than Pintool. + +The big issue with DynamoRIO (and therefore Pintool, too) is speed. DynamoRIO +has a speed decrease of 98-99%, Pintool has a speed decrease of 99.5%. + +Hence, DynamoRIO is the option to go for if everything else fails and Pintool +only if DynamoRIO fails, too. + +DynamoRIO solutions: +* [https://github.com/vanhauser-thc/afl-dynamorio](https://github.com/vanhauser-thc/afl-dynamorio) +* [https://github.com/mxmssh/drAFL](https://github.com/mxmssh/drAFL) +* [https://github.com/googleprojectzero/winafl/](https://github.com/googleprojectzero/winafl/) + <= very good but windows only + +Pintool solutions: +* [https://github.com/vanhauser-thc/afl-pin](https://github.com/vanhauser-thc/afl-pin) +* [https://github.com/mothran/aflpin](https://github.com/mothran/aflpin) +* [https://github.com/spinpx/afl_pin_mode](https://github.com/spinpx/afl_pin_mode) + <= only old Pintool version supported + +### Intel PT + +If you have a newer Intel CPU, you can make use of Intel's processor trace. The +big issue with Intel's PT is the small buffer size and the complex encoding of +the debug information collected through PT. This makes the decoding very CPU +intensive and hence slow. As a result, the overall speed decrease is about +70-90% (depending on the implementation and other factors). + +There are two AFL intel-pt implementations: + +1. [https://github.com/junxzm1990/afl-pt](https://github.com/junxzm1990/afl-pt) + => This needs Ubuntu 14.04.05 without any updates and the 4.4 kernel. + +2. [https://github.com/hunter-ht-2018/ptfuzzer](https://github.com/hunter-ht-2018/ptfuzzer) + => This needs a 4.14 or 4.15 kernel. The "nopti" kernel boot option must be + used. This one is faster than the other. + +Note that there is also honggfuzz: +[https://github.com/google/honggfuzz](https://github.com/google/honggfuzz). But +its IPT performance is just 6%! + +## Non-AFL++ solutions + +There are many binary-only fuzzing frameworks. Some are great for CTFs but don't +work with large binaries, others are very slow but have good path discovery, +some are very hard to set-up... + +* Jackalope: + [https://github.com/googleprojectzero/Jackalope](https://github.com/googleprojectzero/Jackalope) +* Manticore: + [https://github.com/trailofbits/manticore](https://github.com/trailofbits/manticore) +* QSYM: + [https://github.com/sslab-gatech/qsym](https://github.com/sslab-gatech/qsym) +* S2E: [https://github.com/S2E](https://github.com/S2E) +* TinyInst: + [https://github.com/googleprojectzero/TinyInst](https://github.com/googleprojectzero/TinyInst) + (Mac/Windows only) +* ... please send me any missing that are good + +## Closing words + +That's it! News, corrections, updates? Send an email to vh@thc.org. \ No newline at end of file diff --git a/docs/fuzzing_in_depth.md b/docs/fuzzing_in_depth.md new file mode 100644 index 00000000..aaceb600 --- /dev/null +++ b/docs/fuzzing_in_depth.md @@ -0,0 +1,861 @@ +# Fuzzing with AFL++ + +The following describes how to fuzz with a target if source code is available. +If you have a binary-only target, go to +[fuzzing_binary-only_targets.md](fuzzing_binary-only_targets.md). + +Fuzzing source code is a three-step process: + +1. Compile the target with a special compiler that prepares the target to be + fuzzed efficiently. This step is called "instrumenting a target". +2. Prepare the fuzzing by selecting and optimizing the input corpus for the + target. +3. Perform the fuzzing of the target by randomly mutating input and assessing if + a generated input was processed in a new path in the target binary. + +## 0. Common sense risks + +Please keep in mind that, similarly to many other computationally-intensive +tasks, fuzzing may put a strain on your hardware and on the OS. In particular: + +- Your CPU will run hot and will need adequate cooling. In most cases, if + cooling is insufficient or stops working properly, CPU speeds will be + automatically throttled. That said, especially when fuzzing on less suitable + hardware (laptops, smartphones, etc.), it's not entirely impossible for + something to blow up. + +- Targeted programs may end up erratically grabbing gigabytes of memory or + filling up disk space with junk files. AFL++ tries to enforce basic memory + limits, but can't prevent each and every possible mishap. The bottom line is + that you shouldn't be fuzzing on systems where the prospect of data loss is + not an acceptable risk. + +- Fuzzing involves billions of reads and writes to the filesystem. On modern + systems, this will be usually heavily cached, resulting in fairly modest + "physical" I/O - but there are many factors that may alter this equation. It + is your responsibility to monitor for potential trouble; with very heavy I/O, + the lifespan of many HDDs and SSDs may be reduced. + + A good way to monitor disk I/O on Linux is the `iostat` command: + + ```shell + $ iostat -d 3 -x -k [...optional disk ID...] + ``` + + Using the `AFL_TMPDIR` environment variable and a RAM-disk, you can have the + heavy writing done in RAM to prevent the aforementioned wear and tear. For + example, the following line will run a Docker container with all this preset: + + ```shell + # docker run -ti --mount type=tmpfs,destination=/ramdisk -e AFL_TMPDIR=/ramdisk aflplusplus/aflplusplus + ``` + +## 1. Instrumenting the target + +### a) Selecting the best AFL++ compiler for instrumenting the target + +AFL++ comes with a central compiler `afl-cc` that incorporates various different +kinds of compiler targets and and instrumentation options. The following +evaluation flow will help you to select the best possible. + +It is highly recommended to have the newest llvm version possible installed, +anything below 9 is not recommended. + +``` ++--------------------------------+ +| clang/clang++ 11+ is available | --> use LTO mode (afl-clang-lto/afl-clang-lto++) ++--------------------------------+ see [instrumentation/README.lto.md](instrumentation/README.lto.md) + | + | if not, or if the target fails with LTO afl-clang-lto/++ + | + v ++---------------------------------+ +| clang/clang++ 3.8+ is available | --> use LLVM mode (afl-clang-fast/afl-clang-fast++) ++---------------------------------+ see [instrumentation/README.llvm.md](instrumentation/README.llvm.md) + | + | if not, or if the target fails with LLVM afl-clang-fast/++ + | + v + +--------------------------------+ + | gcc 5+ is available | -> use GCC_PLUGIN mode (afl-gcc-fast/afl-g++-fast) + +--------------------------------+ see [instrumentation/README.gcc_plugin.md](instrumentation/README.gcc_plugin.md) and + [instrumentation/README.instrument_list.md](instrumentation/README.instrument_list.md) + | + | if not, or if you do not have a gcc with plugin support + | + v + use GCC mode (afl-gcc/afl-g++) (or afl-clang/afl-clang++ for clang) +``` + +Clickable README links for the chosen compiler: + +* [LTO mode - afl-clang-lto](../instrumentation/README.lto.md) +* [LLVM mode - afl-clang-fast](../instrumentation/README.llvm.md) +* [GCC_PLUGIN mode - afl-gcc-fast](../instrumentation/README.gcc_plugin.md) +* GCC/CLANG modes (afl-gcc/afl-clang) have no README as they have no own + features + +You can select the mode for the afl-cc compiler by: +1. use a symlink to afl-cc: afl-gcc, afl-g++, afl-clang, afl-clang++, + afl-clang-fast, afl-clang-fast++, afl-clang-lto, afl-clang-lto++, + afl-gcc-fast, afl-g++-fast (recommended!) +2. using the environment variable AFL_CC_COMPILER with MODE +3. passing --afl-MODE command line options to the compiler via + CFLAGS/CXXFLAGS/CPPFLAGS + +MODE can be one of: LTO (afl-clang-lto*), LLVM (afl-clang-fast*), GCC_PLUGIN +(afl-g*-fast) or GCC (afl-gcc/afl-g++) or CLANG(afl-clang/afl-clang++). + +Because no AFL++ specific command-line options are accepted (beside the +--afl-MODE command), the compile-time tools make fairly broad use of environment +variables, which can be listed with `afl-cc -hh` or by reading +[env_variables.md](env_variables.md). + +### b) Selecting instrumentation options + +The following options are available when you instrument with LTO mode +(afl-clang-fast/afl-clang-lto): + +* Splitting integer, string, float and switch comparisons so AFL++ can easier + solve these. This is an important option if you do not have a very good and + large input corpus. This technique is called laf-intel or COMPCOV. To use this + set the following environment variable before compiling the target: `export + AFL_LLVM_LAF_ALL=1` You can read more about this in + [instrumentation/README.laf-intel.md](../instrumentation/README.laf-intel.md). +* A different technique (and usually a better one than laf-intel) is to + instrument the target so that any compare values in the target are sent to + AFL++ which then tries to put these values into the fuzzing data at different + locations. This technique is very fast and good - if the target does not + transform input data before comparison. Therefore this technique is called + `input to state` or `redqueen`. If you want to use this technique, then you + have to compile the target twice, once specifically with/for this mode by + setting `AFL_LLVM_CMPLOG=1`, and pass this binary to afl-fuzz via the `-c` + parameter. Note that you can compile also just a cmplog binary and use that + for both, however, there will be a performance penalty. You can read more + about this in + [instrumentation/README.cmplog.md](../instrumentation/README.cmplog.md). + +If you use LTO, LLVM or GCC_PLUGIN mode +(afl-clang-fast/afl-clang-lto/afl-gcc-fast) you have the option to selectively +only instrument parts of the target that you are interested in: + +* To instrument only those parts of the target that you are interested in create + a file with all the filenames of the source code that should be instrumented. + For afl-clang-lto and afl-gcc-fast - or afl-clang-fast if a mode other than + DEFAULT/PCGUARD is used or you have llvm > 10.0.0 - just put one filename or + function per line (no directory information necessary for filenames9, and + either set `export AFL_LLVM_ALLOWLIST=allowlist.txt` **or** `export + AFL_LLVM_DENYLIST=denylist.txt` - depending on if you want per default to + instrument unless noted (DENYLIST) or not perform instrumentation unless + requested (ALLOWLIST). **NOTE:** During optimization functions might be + inlined and then would not match! See + [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md) + +There are many more options and modes available, however, these are most of the +time less effective. See: +* [instrumentation/README.llvm.md#6) AFL++ Context Sensitive Branch Coverage](../instrumentation/README.llvm.md#6-afl-context-sensitive-branch-coverage) +* [instrumentation/README.llvm.md#7) AFL++ N-Gram Branch Coverage](../instrumentation/README.llvm.md#7-afl-n-gram-branch-coverage) + +AFL++ performs "never zero" counting in its bitmap. You can read more about this +here: +* [instrumentation/README.llvm.md#8-neverzero-counters](../instrumentation/README.llvm.md#8-neverzero-counters) + +### c) Selecting sanitizers + +It is possible to use sanitizers when instrumenting targets for fuzzing, which +allows you to find bugs that would not necessarily result in a crash. + +Note that sanitizers have a huge impact on CPU (= less executions per second) +and RAM usage. Also you should only run one afl-fuzz instance per sanitizer +type. This is enough because a use-after-free bug will be picked up, e.g., by +ASAN (address sanitizer) anyway when syncing to other fuzzing instances, so not +all fuzzing instances need to be instrumented with ASAN. + +The following sanitizers have built-in support in AFL++: +* ASAN = Address SANitizer, finds memory corruption vulnerabilities like + use-after-free, NULL pointer dereference, buffer overruns, etc. Enabled with + `export AFL_USE_ASAN=1` before compiling. +* MSAN = Memory SANitizer, finds read access to uninitialized memory, e.g., a + local variable that is defined and read before it is even set. Enabled with + `export AFL_USE_MSAN=1` before compiling. +* UBSAN = Undefined Behavior SANitizer, finds instances where - by the C and C++ + standards - undefined behavior happens, e.g., adding two signed integers + together where the result is larger than a signed integer can hold. Enabled + with `export AFL_USE_UBSAN=1` before compiling. +* CFISAN = Control Flow Integrity SANitizer, finds instances where the control + flow is found to be illegal. Originally this was rather to prevent return + oriented programming exploit chains from functioning, in fuzzing this is + mostly reduced to detecting type confusion vulnerabilities - which is, + however, one of the most important and dangerous C++ memory corruption + classes! Enabled with `export AFL_USE_CFISAN=1` before compiling. +* TSAN = Thread SANitizer, finds thread race conditions. Enabled with `export + AFL_USE_TSAN=1` before compiling. +* LSAN = Leak SANitizer, finds memory leaks in a program. This is not really a + security issue, but for developers this can be very valuable. Note that unlike + the other sanitizers above this needs `__AFL_LEAK_CHECK();` added to all areas + of the target source code where you find a leak check necessary! Enabled with + `export AFL_USE_LSAN=1` before compiling. + +It is possible to further modify the behavior of the sanitizers at run-time by +setting `ASAN_OPTIONS=...`, `LSAN_OPTIONS` etc. - the available parameters can +be looked up in the sanitizer documentation of llvm/clang. afl-fuzz, however, +requires some specific parameters important for fuzzing to be set. If you want +to set your own, it might bail and report what it is missing. + +Note that some sanitizers cannot be used together, e.g., ASAN and MSAN, and +others often cannot work together because of target weirdness, e.g., ASAN and +CFISAN. You might need to experiment which sanitizers you can combine in a +target (which means more instances can be run without a sanitized target, which +is more effective). + +### d) Modifying the target + +If the target has features that make fuzzing more difficult, e.g., checksums, +HMAC, etc., then modify the source code so that checks for these values are +removed. This can even be done safely for source code used in operational +products by eliminating these checks within these AFL++ specific blocks: + +``` +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + // say that the checksum or HMAC was fine - or whatever is required + // to eliminate the need for the fuzzer to guess the right checksum + return 0; +#endif +``` + +All AFL++ compilers will set this preprocessor definition automatically. + +### e) Instrumenting the target + +In this step the target source code is compiled so that it can be fuzzed. + +Basically you have to tell the target build system that the selected AFL++ +compiler is used. Also - if possible - you should always configure the build +system such that the target is compiled statically and not dynamically. How to +do this is described below. + +The #1 rule when instrumenting a target is: avoid instrumenting shared libraries +at all cost. You would need to set LD_LIBRARY_PATH to point to these, you could +accidentally type "make install" and install them system wide - so don't. Really +don't. **Always compile libraries you want to have instrumented as static and +link these to the target program!** + +Then build the target. (Usually with `make`) + +**NOTES** + +1. Sometimes configure and build systems are fickle and do not like stderr + output (and think this means a test failure) - which is something AFL++ likes + to do to show statistics. It is recommended to disable AFL++ instrumentation + reporting via `export AFL_QUIET=1`. + +2. Sometimes configure and build systems error on warnings - these should be + disabled (e.g., `--disable-werror` for some configure scripts). + +3. In case the configure/build system complains about AFL++'s compiler and + aborts, then set `export AFL_NOOPT=1` which will then just behave like the + real compiler. This option has to be unset again before building the target! + +#### configure + +For `configure` build systems this is usually done by: + +`CC=afl-clang-fast CXX=afl-clang-fast++ ./configure --disable-shared` + +Note that if you are using the (better) afl-clang-lto compiler you also have to +set AR to llvm-ar[-VERSION] and RANLIB to llvm-ranlib[-VERSION] - as is +described in [instrumentation/README.lto.md](../instrumentation/README.lto.md). + +#### cmake + +For `cmake` build systems this is usually done by: + +`mkdir build; cd build; cmake -DCMAKE_C_COMPILER=afl-cc -DCMAKE_CXX_COMPILER=afl-c++ ..` + +Note that if you are using the (better) afl-clang-lto compiler you also have to +set AR to llvm-ar[-VERSION] and RANLIB to llvm-ranlib[-VERSION] - as is +described in [instrumentation/README.lto.md](../instrumentation/README.lto.md). + +#### meson + +For meson you have to set the AFL++ compiler with the very first command! +`CC=afl-cc CXX=afl-c++ meson` + +#### other build systems or if configure/cmake didn't work + +Sometimes cmake and configure do not pick up the AFL++ compiler, or the +ranlib/ar that is needed - because this was just not foreseen by the developer +of the target. Or they have non-standard options. Figure out if there is a +non-standard way to set this, otherwise set up the build normally and edit the +generated build environment afterwards manually to point it to the right +compiler (and/or ranlib and ar). + +### f) Better instrumentation + +If you just fuzz a target program as-is you are wasting a great opportunity for +much more fuzzing speed. + +This variant requires the usage of afl-clang-lto, afl-clang-fast or +afl-gcc-fast. + +It is the so-called `persistent mode`, which is much, much faster but requires +that you code a source file that is specifically calling the target functions +that you want to fuzz, plus a few specific AFL++ functions around it. See +[instrumentation/README.persistent_mode.md](../instrumentation/README.persistent_mode.md) +for details. + +Basically if you do not fuzz a target in persistent mode, then you are just +doing it for a hobby and not professionally :-). + +### g) libfuzzer fuzzer harnesses with LLVMFuzzerTestOneInput() + +libfuzzer `LLVMFuzzerTestOneInput()` harnesses are the defacto standard for +fuzzing, and they can be used with AFL++ (and honggfuzz) as well! + +Compiling them is as simple as: + +``` +afl-clang-fast++ -fsanitize=fuzzer -o harness harness.cpp targetlib.a +``` + +You can even use advanced libfuzzer features like `FuzzedDataProvider`, +`LLVMFuzzerMutate()` etc. and they will work! + +The generated binary is fuzzed with afl-fuzz like any other fuzz target. + +Bonus: the target is already optimized for fuzzing due to persistent mode and +shared-memory test cases and hence gives you the fastest speed possible. + +For more information, see +[utils/aflpp_driver/README.md](../utils/aflpp_driver/README.md). + +## 2. Preparing the fuzzing campaign + +As you fuzz the target with mutated input, having as diverse inputs for the +target as possible improves the efficiency a lot. + +### a) Collecting inputs + +To operate correctly, the fuzzer requires one or more starting files that +contain a good example of the input data normally expected by the targeted +application. + +Try to gather valid inputs for the target from wherever you can. E.g., if it is +the PNG picture format, try to find as many PNG files as possible, e.g., from +reported bugs, test suites, random downloads from the internet, unit test case +data - from all kind of PNG software. + +If the input format is not known, you can also modify a target program to write +normal data it receives and processes to a file and use these. + +You can find many good examples of starting files in the +[testcases/](../testcases) subdirectory that comes with this tool. + +### b) Making the input corpus unique + +Use the AFL++ tool `afl-cmin` to remove inputs from the corpus that do not +produce a new path in the target. + +Put all files from step a) into one directory, e.g., INPUTS. + +If the target program is to be called by fuzzing as `bin/target -d INPUTFILE` +the run afl-cmin like this: + +`afl-cmin -i INPUTS -o INPUTS_UNIQUE -- bin/target -d @@` + +Note that the INPUTFILE argument that the target program would read from has to +be set as `@@`. + +If the target reads from stdin instead, just omit the `@@` as this is the +default. + +This step is highly recommended! + +### c) Minimizing all corpus files + +The shorter the input files that still traverse the same path within the target, +the better the fuzzing will be. This minimization is done with `afl-tmin`, +however, it is a long process as this has to be done for every file: + +``` +mkdir input +cd INPUTS_UNIQUE +for i in *; do + afl-tmin -i "$i" -o "../input/$i" -- bin/target -d @@ +done +``` + +This step can also be parallelized, e.g., with `parallel`. Note that this step +is rather optional though. + +### Done! + +The INPUTS_UNIQUE/ directory from step b) - or even better the directory input/ +if you minimized the corpus in step c) - is the resulting input corpus directory +to be used in fuzzing! :-) + +## 3. Fuzzing the target + +In this final step, fuzz the target. There are not that many important options +to run the target - unless you want to use many CPU cores/threads for the +fuzzing, which will make the fuzzing much more useful. + +If you just use one CPU for fuzzing, then you are fuzzing just for fun and not +seriously :-) + +### a) Running afl-fuzz + +Before you do even a test run of afl-fuzz execute `sudo afl-system-config` (on +the host if you execute afl-fuzz in a docker container). This reconfigures the +system for optimal speed - which afl-fuzz checks and bails otherwise. Set +`export AFL_SKIP_CPUFREQ=1` for afl-fuzz to skip this check if you cannot run +afl-system-config with root privileges on the host for whatever reason. + +Note there is also `sudo afl-persistent-config` which sets additional permanent +boot options for a much better fuzzing performance. + +Note that both scripts improve your fuzzing performance but also decrease your +system protection against attacks! So set strong firewall rules and only expose +SSH as a network service if you use these (which is highly recommended). + +If you have an input corpus from step 2, then specify this directory with the +`-i` option. Otherwise, create a new directory and create a file with any +content as test data in there. + +If you do not want anything special, the defaults are already usually best, +hence all you need is to specify the seed input directory with the result of +step [2a) Collect inputs](#a-collect-inputs): + +`afl-fuzz -i input -o output -- bin/target -d @@` + +Note that the directory specified with `-o` will be created if it does not +exist. + +It can be valuable to run afl-fuzz in a screen or tmux shell so you can log off, +or afl-fuzz is not aborted if you are running it in a remote ssh session where +the connection fails in between. Only do that though once you have verified that +your fuzzing setup works! Run it like `screen -dmS afl-main -- afl-fuzz -M +main-$HOSTNAME -i ...` and it will start away in a screen session. To enter this +session, type `screen -r afl-main`. You see - it makes sense to name the screen +session same as the afl-fuzz -M/-S naming :-) For more information on screen or +tmux, check their documentation. + +If you need to stop and re-start the fuzzing, use the same command line options +(or even change them by selecting a different power schedule or another mutation +mode!) and switch the input directory with a dash (`-`): + +`afl-fuzz -i - -o output -- bin/target -d @@` + +Adding a dictionary is helpful. See the directory +[dictionaries/](../dictionaries/) if something is already included for your data +format, and tell afl-fuzz to load that dictionary by adding `-x +dictionaries/FORMAT.dict`. With afl-clang-lto, you have an autodictionary +generation for which you need to do nothing except to use afl-clang-lto as the +compiler. You also have the option to generate a dictionary yourself, see +[utils/libtokencap/README.md](../utils/libtokencap/README.md). + +afl-fuzz has a variety of options that help to workaround target quirks like +specific locations for the input file (`-f`), performing deterministic fuzzing +(`-D`) and many more. Check out `afl-fuzz -h`. + +We highly recommend that you set a memory limit for running the target with `-m` +which defines the maximum memory in MB. This prevents a potential out-of-memory +problem for your system plus helps you detect missing `malloc()` failure +handling in the target. Play around with various -m values until you find one +that safely works for all your input seeds (if you have good ones and then +double or quadruple that. + +By default, afl-fuzz never stops fuzzing. To terminate AFL++, press Control-C or +send a signal SIGINT. You can limit the number of executions or approximate +runtime in seconds with options also. + +When you start afl-fuzz you will see a user interface that shows what the status +is: + + + +All labels are explained in +[afl-fuzz_approach.md#understanding-the-status-screen](afl-fuzz_approach.md#understanding-the-status-screen). + +### b) Keeping memory use and timeouts in check + +Memory limits are not enforced by afl-fuzz by default and the system may run out +of memory. You can decrease the memory with the `-m` option, the value is in MB. +If this is too small for the target, you can usually see this by afl-fuzz +bailing with the message that it could not connect to the forkserver. + +Consider setting low values for `-m` and `-t`. + +For programs that are nominally very fast, but get sluggish for some inputs, you +can also try setting `-t` values that are more punishing than what `afl-fuzz` +dares to use on its own. On fast and idle machines, going down to `-t 5` may be +a viable plan. + +The `-m` parameter is worth looking at, too. Some programs can end up spending a +fair amount of time allocating and initializing megabytes of memory when +presented with pathological inputs. Low `-m` values can make them give up sooner +and not waste CPU time. + +### c) Using multiple cores + +If you want to seriously fuzz, then use as many cores/threads as possible to +fuzz your target. + +On the same machine - due to the design of how AFL++ works - there is a maximum +number of CPU cores/threads that are useful, use more and the overall +performance degrades instead. This value depends on the target, and the limit is +between 32 and 64 cores per machine. + +If you have the RAM, it is highly recommended run the instances with a caching +of the test cases. Depending on the average test case size (and those found +during fuzzing) and their number, a value between 50-500MB is recommended. You +can set the cache size (in MB) by setting the environment variable +`AFL_TESTCACHE_SIZE`. + +There should be one main fuzzer (`-M main-$HOSTNAME` option) and as many +secondary fuzzers (e.g., `-S variant1`) as you have cores that you use. Every +-M/-S entry needs a unique name (that can be whatever), however, the same -o +output directory location has to be used for all instances. + +For every secondary fuzzer there should be a variation, e.g.: +* one should fuzz the target that was compiled differently: with sanitizers + activated (`export AFL_USE_ASAN=1 ; export AFL_USE_UBSAN=1 ; export + AFL_USE_CFISAN=1`) +* one or two should fuzz the target with CMPLOG/redqueen (see above), at least + one cmplog instance should follow transformations (`-l AT`) +* one to three fuzzers should fuzz a target compiled with laf-intel/COMPCOV (see + above). Important note: If you run more than one laf-intel/COMPCOV fuzzer and + you want them to share their intermediate results, the main fuzzer (`-M`) must + be one of them! (Although this is not really recommended.) + +All other secondaries should be used like this: +* a quarter to a third with the MOpt mutator enabled: `-L 0` +* run with a different power schedule, recommended are: + `fast (default), explore, coe, lin, quad, exploit and rare` which you can set + with, e.g., `-p explore` +* a few instances should use the old queue cycling with `-Z` + +Also, it is recommended to set `export AFL_IMPORT_FIRST=1` to load test cases +from other fuzzers in the campaign first. + +If you have a large corpus, a corpus from a previous run or are fuzzing in a CI, +then also set `export AFL_CMPLOG_ONLY_NEW=1` and `export AFL_FAST_CAL=1`. + +You can also use different fuzzers. If you are using AFL spinoffs or AFL +conforming fuzzers, then just use the same -o directory and give it a unique +`-S` name. Examples are: +* [Fuzzolic](https://github.com/season-lab/fuzzolic) +* [symcc](https://github.com/eurecom-s3/symcc/) +* [Eclipser](https://github.com/SoftSec-KAIST/Eclipser/) +* [AFLsmart](https://github.com/aflsmart/aflsmart) +* [FairFuzz](https://github.com/carolemieux/afl-rb) +* [Neuzz](https://github.com/Dongdongshe/neuzz) +* [Angora](https://github.com/AngoraFuzzer/Angora) + +A long list can be found at +[https://github.com/Microsvuln/Awesome-AFL](https://github.com/Microsvuln/Awesome-AFL). + +However, you can also sync AFL++ with honggfuzz, libfuzzer with `-entropic=1`, +etc. Just show the main fuzzer (-M) with the `-F` option where the queue/work +directory of a different fuzzer is, e.g., `-F /src/target/honggfuzz`. Using +honggfuzz (with `-n 1` or `-n 2`) and libfuzzer in parallel is highly +recommended! + +### d) Using multiple machines for fuzzing + +Maybe you have more than one machine you want to fuzz the same target on. Start +the `afl-fuzz` (and perhaps libfuzzer, honggfuzz, ...) orchestra as you like, +just ensure that your have one and only one `-M` instance per server, and that +its name is unique, hence the recommendation for `-M main-$HOSTNAME`. + +Now there are three strategies on how you can sync between the servers: +* never: sounds weird, but this makes every server an island and has the chance + the each follow different paths into the target. You can make this even more + interesting by even giving different seeds to each server. +* regularly (~4h): this ensures that all fuzzing campaigns on the servers "see" + the same thing. It is like fuzzing on a huge server. +* in intervals of 1/10th of the overall expected runtime of the fuzzing you + sync. This tries a bit to combine both. have some individuality of the paths + each campaign on a server explores, on the other hand if one gets stuck where + another found progress this is handed over making it unstuck. + +The syncing process itself is very simple. As the `-M main-$HOSTNAME` instance +syncs to all `-S` secondaries as well as to other fuzzers, you have to copy only +this directory to the other machines. + +Lets say all servers have the `-o out` directory in /target/foo/out, and you +created a file `servers.txt` which contains the hostnames of all participating +servers, plus you have an ssh key deployed to all of them, then run: + +```bash +for FROM in `cat servers.txt`; do + for TO in `cat servers.txt`; do + rsync -rlpogtz --rsh=ssh $FROM:/target/foo/out/main-$FROM $TO:target/foo/out/ + done +done +``` + +You can run this manually, per cron job - as you need it. There is a more +complex and configurable script in `utils/distributed_fuzzing`. + +### e) The status of the fuzz campaign + +AFL++ comes with the `afl-whatsup` script to show the status of the fuzzing +campaign. + +Just supply the directory that afl-fuzz is given with the `-o` option and you +will see a detailed status of every fuzzer in that campaign plus a summary. + +To have only the summary, use the `-s` switch, e.g., `afl-whatsup -s out/`. + +If you have multiple servers, then use the command after a sync or you have to +execute this script per server. + +Another tool to inspect the current state and history of a specific instance is +afl-plot, which generates an index.html file and a graphs that show how the +fuzzing instance is performing. The syntax is `afl-plot instance_dir web_dir`, +e.g., `afl-plot out/default /srv/www/htdocs/plot`. + +### f) Stopping fuzzing, restarting fuzzing, adding new seeds + +To stop an afl-fuzz run, press Control-C. + +To restart an afl-fuzz run, just reuse the same command line but replace the `-i +directory` with `-i -` or set `AFL_AUTORESUME=1`. + +If you want to add new seeds to a fuzzing campaign you can run a temporary +fuzzing instance, e.g., when your main fuzzer is using `-o out` and the new +seeds are in `newseeds/` directory: + +``` +AFL_BENCH_JUST_ONE=1 AFL_FAST_CAL=1 afl-fuzz -i newseeds -o out -S newseeds -- ./target +``` + +### g) Checking the coverage of the fuzzing + +The `paths found` value is a bad indicator for checking how good the coverage +is. + +A better indicator - if you use default llvm instrumentation with at least +version 9 - is to use `afl-showmap` with the collect coverage option `-C` on the +output directory: + +``` +$ afl-showmap -C -i out -o /dev/null -- ./target -params @@ +... +[*] Using SHARED MEMORY FUZZING feature. +[*] Target map size: 9960 +[+] Processed 7849 input files. +[+] Captured 4331 tuples (highest value 255, total values 67130596) in '/dev/nul +l'. +[+] A coverage of 4331 edges were achieved out of 9960 existing (43.48%) with 7849 input files. +``` + +It is even better to check out the exact lines of code that have been reached - +and which have not been found so far. + +An "easy" helper script for this is +[https://github.com/vanhauser-thc/afl-cov](https://github.com/vanhauser-thc/afl-cov), +just follow the README of that separate project. + +If you see that an important area or a feature has not been covered so far, then +try to find an input that is able to reach that and start a new secondary in +that fuzzing campaign with that seed as input, let it run for a few minutes, +then terminate it. The main node will pick it up and make it available to the +other secondary nodes over time. Set `export AFL_NO_AFFINITY=1` or `export +AFL_TRY_AFFINITY=1` if you have no free core. + +Note that in nearly all cases you can never reach full coverage. A lot of +functionality is usually dependent on exclusive options that would need +individual fuzzing campaigns each with one of these options set. E.g., if you +fuzz a library to convert image formats and your target is the png to tiff API, +then you will not touch any of the other library APIs and features. + +### h) How long to fuzz a target? + +This is a difficult question. Basically, if no new path is found for a long time +(e.g., for a day or a week), then you can expect that your fuzzing won't be +fruitful anymore. However, often this just means that you should switch out +secondaries for others, e.g., custom mutator modules, sync to very different +fuzzers, etc. + +Keep the queue/ directory (for future fuzzings of the same or similar targets) +and use them to seed other good fuzzers like libfuzzer with the -entropic switch +or honggfuzz. + +### i) Improve the speed! + +* Use [persistent mode](../instrumentation/README.persistent_mode.md) (x2-x20 + speed increase) +* If you do not use shmem persistent mode, use `AFL_TMPDIR` to point the input + file on a tempfs location, see [env_variables.md](env_variables.md) +* Linux: Improve kernel performance: modify `/etc/default/grub`, set + `GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off + mitigations=off no_stf_barrier noibpb noibrs nopcid nopti + nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off + spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"`; then + `update-grub` and `reboot` (warning: makes the system more insecure) - you can + also just run `sudo afl-persistent-config` +* Linux: Running on an `ext2` filesystem with `noatime` mount option will be a + bit faster than on any other journaling filesystem +* Use your cores! [3c) Using multiple cores](#c-using-multiple-cores) +* Run `sudo afl-system-config` before starting the first afl-fuzz instance after + a reboot + +### j) Going beyond crashes + +Fuzzing is a wonderful and underutilized technique for discovering non-crashing +design and implementation errors, too. Quite a few interesting bugs have been +found by modifying the target programs to call `abort()` when say: + +- Two bignum libraries produce different outputs when given the same + fuzzer-generated input. + +- An image library produces different outputs when asked to decode the same + input image several times in a row. + +- A serialization/deserialization library fails to produce stable outputs when + iteratively serializing and deserializing fuzzer-supplied data. + +- A compression library produces an output inconsistent with the input file when + asked to compress and then decompress a particular blob. + +Implementing these or similar sanity checks usually takes very little time; if +you are the maintainer of a particular package, you can make this code +conditional with `#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION` (a flag also +shared with libfuzzer and honggfuzz) or `#ifdef __AFL_COMPILER` (this one is +just for AFL++). + +### k) Known limitations & areas for improvement + +Here are some of the most important caveats for AFL++: + +- AFL++ detects faults by checking for the first spawned process dying due to a + signal (SIGSEGV, SIGABRT, etc.). Programs that install custom handlers for + these signals may need to have the relevant code commented out. In the same + vein, faults in child processes spawned by the fuzzed target may evade + detection unless you manually add some code to catch that. + +- As with any other brute-force tool, the fuzzer offers limited coverage if + encryption, checksums, cryptographic signatures, or compression are used to + wholly wrap the actual data format to be tested. + + To work around this, you can comment out the relevant checks (see + utils/libpng_no_checksum/ for inspiration); if this is not possible, you can + also write a postprocessor, one of the hooks of custom mutators. See + [custom_mutators.md](custom_mutators.md) on how to use + `AFL_CUSTOM_MUTATOR_LIBRARY`. + +- There are some unfortunate trade-offs with ASAN and 64-bit binaries. This + isn't due to any specific fault of afl-fuzz. + +- There is no direct support for fuzzing network services, background daemons, + or interactive apps that require UI interaction to work. You may need to make + simple code changes to make them behave in a more traditional way. Preeny may + offer a relatively simple option, too - see: + [https://github.com/zardus/preeny](https://github.com/zardus/preeny) + + Some useful tips for modifying network-based services can be also found at: + [https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop](https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop) + +- Occasionally, sentient machines rise against their creators. If this happens + to you, please consult + [https://lcamtuf.coredump.cx/prep/](https://lcamtuf.coredump.cx/prep/). + +Beyond this, see [INSTALL.md](INSTALL.md) for platform-specific tips. + +## 4. Triaging crashes + +The coverage-based grouping of crashes usually produces a small data set that +can be quickly triaged manually or with a very simple GDB or Valgrind script. +Every crash is also traceable to its parent non-crashing test case in the queue, +making it easier to diagnose faults. + +Having said that, it's important to acknowledge that some fuzzing crashes can be +difficult to quickly evaluate for exploitability without a lot of debugging and +code analysis work. To assist with this task, afl-fuzz supports a very unique +"crash exploration" mode enabled with the -C flag. + +In this mode, the fuzzer takes one or more crashing test cases as the input and +uses its feedback-driven fuzzing strategies to very quickly enumerate all code +paths that can be reached in the program while keeping it in the crashing state. + +Mutations that do not result in a crash are rejected; so are any changes that do +not affect the execution path. + +The output is a small corpus of files that can be very rapidly examined to see +what degree of control the attacker has over the faulting address, or whether it +is possible to get past an initial out-of-bounds read - and see what lies +beneath. + +Oh, one more thing: for test case minimization, give afl-tmin a try. The tool +can be operated in a very simple way: + +```shell +./afl-tmin -i test_case -o minimized_result -- /path/to/program [...] +``` + +The tool works with crashing and non-crashing test cases alike. In the crash +mode, it will happily accept instrumented and non-instrumented binaries. In the +non-crashing mode, the minimizer relies on standard AFL++ instrumentation to +make the file simpler without altering the execution path. + +The minimizer accepts the -m, -t, -f and @@ syntax in a manner compatible with +afl-fuzz. + +Another tool in AFL++ is the afl-analyze tool. It takes an input file, attempts +to sequentially flip bytes, and observes the behavior of the tested program. It +then color-codes the input based on which sections appear to be critical, and +which are not; while not bulletproof, it can often offer quick insights into +complex file formats. + +## 5. CI fuzzing + +Some notes on CI fuzzing - this fuzzing is different to normal fuzzing campaigns +as these are much shorter runnings. + +1. Always: + * LTO has a much longer compile time which is diametrical to short fuzzing - + hence use afl-clang-fast instead. + * If you compile with CMPLOG, then you can save fuzzing time and reuse that + compiled target for both the `-c` option and the main fuzz target. This + will impact the speed by ~15% though. + * `AFL_FAST_CAL` - Enable fast calibration, this halves the time the + saturated corpus needs to be loaded. + * `AFL_CMPLOG_ONLY_NEW` - only perform cmplog on new found paths, not the + initial corpus as this very likely has been done for them already. + * Keep the generated corpus, use afl-cmin and reuse it every time! + +2. Additionally randomize the AFL++ compilation options, e.g.: + * 40% for `AFL_LLVM_CMPLOG` + * 10% for `AFL_LLVM_LAF_ALL` + +3. Also randomize the afl-fuzz runtime options, e.g.: + * 65% for `AFL_DISABLE_TRIM` + * 50% use a dictionary generated by `AFL_LLVM_DICT2FILE` + * 40% use MOpt (`-L 0`) + * 40% for `AFL_EXPAND_HAVOC_NOW` + * 20% for old queue processing (`-Z`) + * for CMPLOG targets, 60% for `-l 2`, 40% for `-l 3` + +4. Do *not* run any `-M` modes, just running `-S` modes is better for CI + fuzzing. `-M` enables old queue handling etc. which is good for a fuzzing + campaign but not good for short CI runs. + +How this can look like can, e.g., be seen at AFL++'s setup in Google's +[oss-fuzz](https://github.com/google/oss-fuzz/blob/master/infra/base-images/base-builder/compile_afl) +and +[clusterfuzz](https://github.com/google/clusterfuzz/blob/master/src/clusterfuzz/_internal/bot/fuzzers/afl/launcher.py). + +## The End + +Check out the [FAQ](FAQ.md) if it maybe answers your question (that you might +not even have known you had ;-) ). + +This is basically all you need to know to professionally run fuzzing campaigns. +If you want to know more, the tons of texts in [docs/](./) will have you +covered. + +Note that there are also a lot of tools out there that help fuzzing with AFL++ +(some might be deprecated or unsupported), see +[third_party_tools.md](third_party_tools.md). \ No newline at end of file diff --git a/docs/ideas.md b/docs/ideas.md index 325e7031..52b07c26 100644 --- a/docs/ideas.md +++ b/docs/ideas.md @@ -1,58 +1,57 @@ # Ideas for AFL++ -In the following, we describe a variety of ideas that could be implemented -for future AFL++ versions. +In the following, we describe a variety of ideas that could be implemented for +future AFL++ versions. ## Analysis software -Currently analysis is done by using afl-plot, which is rather outdated. -A GTK or browser tool to create run-time analysis based on fuzzer_stats, -queue/id* information and plot_data that allows for zooming in and out, -changing min/max display values etc. and doing that for a single run, -different runs and campaigns vs campaigns. -Interesting values are execs, and execs/s, edges discovered (total, when -each edge was discovered and which other fuzzer share finding that edge), -test cases executed. -It should be clickable which value is X and Y axis, zoom factor, log scaling -on-off, etc. +Currently analysis is done by using afl-plot, which is rather outdated. A GTK or +browser tool to create run-time analysis based on fuzzer_stats, queue/id* +information and plot_data that allows for zooming in and out, changing min/max +display values etc. and doing that for a single run, different runs and +campaigns vs. campaigns. Interesting values are execs, and execs/s, edges +discovered (total, when each edge was discovered and which other fuzzer share +finding that edge), test cases executed. It should be clickable which value is X +and Y axis, zoom factor, log scaling on-off, etc. Mentor: vanhauser-thc ## WASM Instrumentation Currently, AFL++ can be used for source code fuzzing and traditional binaries. -With the rise of WASM as compile target, however, a novel way of -instrumentation needs to be implemented for binaries compiled to Webassembly. -This can either be done by inserting instrumentation directly into the -WASM AST, or by patching feedback into a WASM VMs of choice, similar to -the current Unicorn instrumentation. +With the rise of WASM as compile target, however, a novel way of instrumentation +needs to be implemented for binaries compiled to Webassembly. This can either be +done by inserting instrumentation directly into the WASM AST, or by patching +feedback into a WASM VMs of choice, similar to the current Unicorn +instrumentation. Mentor: any ## Support other programming languages Other programming languages also use llvm hence they could (easily?) supported -for fuzzing, e.g. mono, swift, go, kotlin native, fortran, ... +for fuzzing, e.g., mono, swift, go, kotlin native, fortran, ... -GCC also supports: Objective-C, Fortran, Ada, Go, and D -(according to [Gcc homepage](https://gcc.gnu.org/)) +GCC also supports: Objective-C, Fortran, Ada, Go, and D (according to +[Gcc homepage](https://gcc.gnu.org/)) -LLVM is also used by: Rust, LLGo (Go), kaleidoscope (Haskell), flang (Fortran), emscripten (JavaScript, WASM), ilwasm (CIL (C#)) -(according to [LLVM frontends](https://gist.github.com/axic/62d66fb9d8bccca6cc48fa9841db9241)) +LLVM is also used by: Rust, LLGo (Go), kaleidoscope (Haskell), flang (Fortran), +emscripten (JavaScript, WASM), ilwasm (CIL (C#)) (according to +[LLVM frontends](https://gist.github.com/axic/62d66fb9d8bccca6cc48fa9841db9241)) Mentor: vanhauser-thc ## Machine Learning -Something with machine learning, better than [NEUZZ](https://github.com/dongdongshe/neuzz) :-) -Either improve a single mutator thorugh learning of many different bugs -(a bug class) or gather deep insights about a single target beforehand -(CFG, DFG, VFG, ...?) and improve performance for a single target. +Something with machine learning, better than +[NEUZZ](https://github.com/dongdongshe/neuzz) :-) Either improve a single +mutator through learning of many different bugs (a bug class) or gather deep +insights about a single target beforehand (CFG, DFG, VFG, ...?) and improve +performance for a single target. Mentor: domenukk ## Your idea! -Finally, we are open to proposals! -Create an issue at https://github.com/AFLplusplus/AFLplusplus/issues and let's discuss :-) - +Finally, we are open to proposals! Create an issue at +https://github.com/AFLplusplus/AFLplusplus/issues and let's discuss :-) \ No newline at end of file diff --git a/docs/important_changes.md b/docs/important_changes.md new file mode 100644 index 00000000..d5e67f75 --- /dev/null +++ b/docs/important_changes.md @@ -0,0 +1,58 @@ +# Important changes in AFL++ + +This document lists important changes in AFL++, for example, major behavior +changes. + +## From version 3.00 onwards + +With AFL++ 3.13-3.20, we introduce FRIDA mode (`-O`) to have an alternative for +binary-only fuzzing. It is slower than QEMU mode but works on MacOS, Android, +iOS etc. + +With AFL++ 3.15, we introduced the following changes from previous behaviors: + * Also -M main mode does not do deterministic fuzzing by default anymore + * afl-cmin and afl-showmap -Ci now descent into subdirectories like afl-fuzz + -i does (but note that afl-cmin.bash does not) + +With AFL++ 3.14, we introduced the following changes from previous behaviors: + * afl-fuzz: deterministic fuzzing it not a default for -M main anymore + * afl-cmin/afl-showmap -i now descends into subdirectories (afl-cmin.bash, + however, does not) + +With AFL++ 3.10, we introduced the following changes from previous behaviors: + * The '+' feature of the '-t' option now means to auto-calculate the timeout + with the value given being the maximum timeout. The original meaning of + "skipping timeouts instead of abort" is now inherent to the -t option. + +With AFL++ 3.00, we introduced changes that break some previous AFL and AFL++ +behaviors and defaults: + * There are no llvm_mode and gcc_plugin subdirectories anymore and there is + only one compiler: afl-cc. All previous compilers now symlink to this one. + All instrumentation source code is now in the `instrumentation/` folder. + * The gcc_plugin was replaced with a new version submitted by AdaCore that + supports more features. Thank you! + * QEMU mode got upgraded to QEMU 5.1, but to be able to build this a current + ninja build tool version and python3 setuptools are required. QEMU mode also + got new options like snapshotting, instrumenting specific shared libraries, + etc. Additionally QEMU 5.1 supports more CPU targets so this is really worth + it. + * When instrumenting targets, afl-cc will not supersede optimizations anymore + if any were given. This allows to fuzz targets build regularly like those + for debug or release versions. + * afl-fuzz: + * if neither -M or -S is specified, `-S default` is assumed, so more fuzzers + can easily be added later + * `-i` input directory option now descends into subdirectories. It also does + not fatal on crashes and too large files, instead it skips them and uses + them for splicing mutations + * -m none is now default, set memory limits (in MB) with, e.g., -m 250 + * deterministic fuzzing is now disabled by default (unless using -M) and can + be enabled with -D + * a caching of test cases can now be performed and can be modified by + editing config.h for TESTCASE_CACHE or by specifying the environment + variable `AFL_TESTCACHE_SIZE` (in MB). Good values are between 50-500 + (default: 50). + * -M mains do not perform trimming + * examples/ got renamed to utils/ + * libtokencap/ libdislocator/ and qdbi_mode/ were moved to utils/ + * afl-cmin/afl-cmin.bash now search first in PATH and last in AFL_PATH \ No newline at end of file diff --git a/docs/life_pro_tips.md b/docs/life_pro_tips.md deleted file mode 100644 index 13ffcea0..00000000 --- a/docs/life_pro_tips.md +++ /dev/null @@ -1,87 +0,0 @@ -# AFL "Life Pro Tips" - -Bite-sized advice for those who understand the basics, but can't be bothered -to read or memorize every other piece of documentation for AFL. - -## Get more bang for your buck by using fuzzing dictionaries. - -See [dictionaries/README.md](../dictionaries/README.md) to learn how. - -## You can get the most out of your hardware by parallelizing AFL jobs. - -See [parallel_fuzzing.md](parallel_fuzzing.md) for step-by-step tips. - -## Improve the odds of spotting memory corruption bugs with libdislocator.so! - -It's easy. Consult [utils/libdislocator/README.md](../utils/libdislocator/README.md) for usage tips. - -## Want to understand how your target parses a particular input file? - -Try the bundled `afl-analyze` tool; it's got colors and all! - -## You can visually monitor the progress of your fuzzing jobs. - -Run the bundled `afl-plot` utility to generate browser-friendly graphs. - -## Need to monitor AFL jobs programmatically? -Check out the `fuzzer_stats` file in the AFL output dir or try `afl-whatsup`. - -## Puzzled by something showing up in red or purple in the AFL UI? -It could be important - consult docs/status_screen.md right away! - -## Know your target? Convert it to persistent mode for a huge performance gain! -Consult section #5 in README.llvm.md for tips. - -## Using clang? -Check out instrumentation/ for a faster alternative to afl-gcc! - -## Did you know that AFL can fuzz closed-source or cross-platform binaries? -Check out qemu_mode/README.md and unicorn_mode/README.md for more. - -## Did you know that afl-fuzz can minimize any test case for you? -Try the bundled `afl-tmin` tool - and get small repro files fast! - -## Not sure if a crash is exploitable? AFL can help you figure it out. Specify -`-C` to enable the peruvian were-rabbit mode. - -## Trouble dealing with a machine uprising? Relax, we've all been there. - -Find essential survival tips at http://lcamtuf.coredump.cx/prep/. - -## Want to automatically spot non-crashing memory handling bugs? - -Try running an AFL-generated corpus through ASAN, MSAN, or Valgrind. - -## Good selection of input files is critical to a successful fuzzing job. - -See docs/perf_tips.md for pro tips. - -## You can improve the odds of automatically spotting stack corruption issues. - -Specify `AFL_HARDEN=1` in the environment to enable hardening flags. - -## Bumping into problems with non-reproducible crashes? -It happens, but usually -isn't hard to diagnose. See section #7 in README.md for tips. - -## Fuzzing is not just about memory corruption issues in the codebase. -Add some -sanity-checking `assert()` / `abort()` statements to effortlessly catch logic bugs. - -## Hey kid... pssst... want to figure out how AFL really works? - -Check out docs/technical_details.md for all the gory details in one place! - -## There's a ton of third-party helper tools designed to work with AFL! - -Be sure to check out docs/sister_projects.md before writing your own. - -## Need to fuzz the command-line arguments of a particular program? - -You can find a simple solution in utils/argv_fuzzing. - -## Attacking a format that uses checksums? - -Remove the checksum-checking code or use a postprocessor! -See `afl_custom_post_process` in custom_mutators/examples/example.c for more. - diff --git a/docs/parallel_fuzzing.md b/docs/parallel_fuzzing.md deleted file mode 100644 index 90e12e89..00000000 --- a/docs/parallel_fuzzing.md +++ /dev/null @@ -1,259 +0,0 @@ -# Tips for parallel fuzzing - -This document talks about synchronizing afl-fuzz jobs on a single machine -or across a fleet of systems. See README.md for the general instruction manual. - -Note that this document is rather outdated. please refer to the main document -section on multiple core usage [../README.md#Using multiple cores](../README.md#b-using-multiple-coresthreads) -for up to date strategies! - -## 1) Introduction - -Every copy of afl-fuzz will take up one CPU core. This means that on an -n-core system, you can almost always run around n concurrent fuzzing jobs with -virtually no performance hit (you can use the afl-gotcpu tool to make sure). - -In fact, if you rely on just a single job on a multi-core system, you will -be underutilizing the hardware. So, parallelization is always the right way to -go. - -When targeting multiple unrelated binaries or using the tool in -"non-instrumented" (-n) mode, it is perfectly fine to just start up several -fully separate instances of afl-fuzz. The picture gets more complicated when -you want to have multiple fuzzers hammering a common target: if a hard-to-hit -but interesting test case is synthesized by one fuzzer, the remaining instances -will not be able to use that input to guide their work. - -To help with this problem, afl-fuzz offers a simple way to synchronize test -cases on the fly. - -Note that AFL++ has AFLfast's power schedules implemented. -It is therefore a good idea to use different power schedules if you run -several instances in parallel. See [power_schedules.md](power_schedules.md) - -Alternatively running other AFL spinoffs in parallel can be of value, -e.g. Angora (https://github.com/AngoraFuzzer/Angora/) - -## 2) Single-system parallelization - -If you wish to parallelize a single job across multiple cores on a local -system, simply create a new, empty output directory ("sync dir") that will be -shared by all the instances of afl-fuzz; and then come up with a naming scheme -for every instance - say, "fuzzer01", "fuzzer02", etc. - -Run the first one ("main node", -M) like this: - -``` -./afl-fuzz -i testcase_dir -o sync_dir -M fuzzer01 [...other stuff...] -``` - -...and then, start up secondary (-S) instances like this: - -``` -./afl-fuzz -i testcase_dir -o sync_dir -S fuzzer02 [...other stuff...] -./afl-fuzz -i testcase_dir -o sync_dir -S fuzzer03 [...other stuff...] -``` - -Each fuzzer will keep its state in a separate subdirectory, like so: - - /path/to/sync_dir/fuzzer01/ - -Each instance will also periodically rescan the top-level sync directory -for any test cases found by other fuzzers - and will incorporate them into -its own fuzzing when they are deemed interesting enough. -For performance reasons only -M main node syncs the queue with everyone, the --S secondary nodes will only sync from the main node. - -The difference between the -M and -S modes is that the main instance will -still perform deterministic checks; while the secondary instances will -proceed straight to random tweaks. - -Note that you must always have one -M main instance! -Running multiple -M instances is wasteful! - -You can also monitor the progress of your jobs from the command line with the -provided afl-whatsup tool. When the instances are no longer finding new paths, -it's probably time to stop. - -WARNING: Exercise caution when explicitly specifying the -f option. Each fuzzer -must use a separate temporary file; otherwise, things will go south. One safe -example may be: - -``` -./afl-fuzz [...] -S fuzzer10 -f file10.txt ./fuzzed/binary @@ -./afl-fuzz [...] -S fuzzer11 -f file11.txt ./fuzzed/binary @@ -./afl-fuzz [...] -S fuzzer12 -f file12.txt ./fuzzed/binary @@ -``` - -This is not a concern if you use @@ without -f and let afl-fuzz come up with the -file name. - -## 3) Multiple -M mains - - -There is support for parallelizing the deterministic checks. -This is only needed where - - 1. many new paths are found fast over a long time and it looks unlikely that - main node will ever catch up, and - 2. deterministic fuzzing is actively helping path discovery (you can see this - in the main node for the first for lines in the "fuzzing strategy yields" - section. If the ration `found/attemps` is high, then it is effective. It - most commonly isn't.) - -Only if both are true it is beneficial to have more than one main. -You can leverage this by creating -M instances like so: - -``` -./afl-fuzz -i testcase_dir -o sync_dir -M mainA:1/3 [...] -./afl-fuzz -i testcase_dir -o sync_dir -M mainB:2/3 [...] -./afl-fuzz -i testcase_dir -o sync_dir -M mainC:3/3 [...] -``` - -... where the first value after ':' is the sequential ID of a particular main -instance (starting at 1), and the second value is the total number of fuzzers to -distribute the deterministic fuzzing across. Note that if you boot up fewer -fuzzers than indicated by the second number passed to -M, you may end up with -poor coverage. - -## 4) Syncing with non-AFL fuzzers or independant instances - -A -M main node can be told with the `-F other_fuzzer_queue_directory` option -to sync results from other fuzzers, e.g. libfuzzer or honggfuzz. - -Only the specified directory will by synced into afl, not subdirectories. -The specified directory does not need to exist yet at the start of afl. - -The `-F` option can be passed to the main node several times. - -## 5) Multi-system parallelization - -The basic operating principle for multi-system parallelization is similar to -the mechanism explained in section 2. The key difference is that you need to -write a simple script that performs two actions: - - - Uses SSH with authorized_keys to connect to every machine and retrieve - a tar archive of the /path/to/sync_dir/<main_node(s)> directory local to - the machine. - It is best to use a naming scheme that includes host name and it's being - a main node (e.g. main1, main2) in the fuzzer ID, so that you can do - something like: - - ```sh - for host in `cat HOSTLIST`; do - ssh user@$host "tar -czf - sync/$host_main*/" > $host.tgz - done - ``` - - - Distributes and unpacks these files on all the remaining machines, e.g.: - - ```sh - for srchost in `cat HOSTLIST`; do - for dsthost in `cat HOSTLIST`; do - test "$srchost" = "$dsthost" && continue - ssh user@$srchost 'tar -kxzf -' < $dsthost.tgz - done - done - ``` - -There is an example of such a script in utils/distributed_fuzzing/. - -There are other (older) more featured, experimental tools: - * https://github.com/richo/roving - * https://github.com/MartijnB/disfuzz-afl - -However these do not support syncing just main nodes (yet). - -When developing custom test case sync code, there are several optimizations -to keep in mind: - - - The synchronization does not have to happen very often; running the - task every 60 minutes or even less often at later fuzzing stages is - fine - - - There is no need to synchronize crashes/ or hangs/; you only need to - copy over queue/* (and ideally, also fuzzer_stats). - - - It is not necessary (and not advisable!) to overwrite existing files; - the -k option in tar is a good way to avoid that. - - - There is no need to fetch directories for fuzzers that are not running - locally on a particular machine, and were simply copied over onto that - system during earlier runs. - - - For large fleets, you will want to consolidate tarballs for each host, - as this will let you use n SSH connections for sync, rather than n*(n-1). - - You may also want to implement staged synchronization. For example, you - could have 10 groups of systems, with group 1 pushing test cases only - to group 2; group 2 pushing them only to group 3; and so on, with group - eventually 10 feeding back to group 1. - - This arrangement would allow test interesting cases to propagate across - the fleet without having to copy every fuzzer queue to every single host. - - - You do not want a "main" instance of afl-fuzz on every system; you should - run them all with -S, and just designate a single process somewhere within - the fleet to run with -M. - - - Syncing is only necessary for the main nodes on a system. It is possible - to run main-less with only secondaries. However then you need to find out - which secondary took over the temporary role to be the main node. Look for - the `is_main_node` file in the fuzzer directories, eg. `sync-dir/hostname-*/is_main_node` - -It is *not* advisable to skip the synchronization script and run the fuzzers -directly on a network filesystem; unexpected latency and unkillable processes -in I/O wait state can mess things up. - -## 6) Remote monitoring and data collection - -You can use screen, nohup, tmux, or something equivalent to run remote -instances of afl-fuzz. If you redirect the program's output to a file, it will -automatically switch from a fancy UI to more limited status reports. There is -also basic machine-readable information which is always written to the -fuzzer_stats file in the output directory. Locally, that information can be -interpreted with afl-whatsup. - -In principle, you can use the status screen of the main (-M) instance to -monitor the overall fuzzing progress and decide when to stop. In this -mode, the most important signal is just that no new paths are being found -for a longer while. If you do not have a main instance, just pick any -single secondary instance to watch and go by that. - -You can also rely on that instance's output directory to collect the -synthesized corpus that covers all the noteworthy paths discovered anywhere -within the fleet. Secondary (-S) instances do not require any special -monitoring, other than just making sure that they are up. - -Keep in mind that crashing inputs are *not* automatically propagated to the -main instance, so you may still want to monitor for crashes fleet-wide -from within your synchronization or health checking scripts (see afl-whatsup). - -## 7) Asymmetric setups - -It is perhaps worth noting that all of the following is permitted: - - - Running afl-fuzz with conjunction with other guided tools that can extend - coverage (e.g., via concolic execution). Third-party tools simply need to - follow the protocol described above for pulling new test cases from - out_dir/<fuzzer_id>/queue/* and writing their own finds to sequentially - numbered id:nnnnnn files in out_dir/<ext_tool_id>/queue/*. - - - Running some of the synchronized fuzzers with different (but related) - target binaries. For example, simultaneously stress-testing several - different JPEG parsers (say, IJG jpeg and libjpeg-turbo) while sharing - the discovered test cases can have synergistic effects and improve the - overall coverage. - - (In this case, running one -M instance per target is necessary.) - - - Having some of the fuzzers invoke the binary in different ways. - For example, 'djpeg' supports several DCT modes, configurable with - a command-line flag, while 'dwebp' supports incremental and one-shot - decoding. In some scenarios, going after multiple distinct modes and then - pooling test cases will improve coverage. - - - Much less convincingly, running the synchronized fuzzers with different - starting test cases (e.g., progressive and standard JPEG) or dictionaries. - The synchronization mechanism ensures that the test sets will get fairly - homogeneous over time, but it introduces some initial variability. diff --git a/docs/perf_tips.md b/docs/perf_tips.md deleted file mode 100644 index 1e8fd4d0..00000000 --- a/docs/perf_tips.md +++ /dev/null @@ -1,209 +0,0 @@ -## Tips for performance optimization - - This file provides tips for troubleshooting slow or wasteful fuzzing jobs. - See README.md for the general instruction manual. - -## 1. Keep your test cases small - -This is probably the single most important step to take! Large test cases do -not merely take more time and memory to be parsed by the tested binary, but -also make the fuzzing process dramatically less efficient in several other -ways. - -To illustrate, let's say that you're randomly flipping bits in a file, one bit -at a time. Let's assume that if you flip bit #47, you will hit a security bug; -flipping any other bit just results in an invalid document. - -Now, if your starting test case is 100 bytes long, you will have a 71% chance of -triggering the bug within the first 1,000 execs - not bad! But if the test case -is 1 kB long, the probability that we will randomly hit the right pattern in -the same timeframe goes down to 11%. And if it has 10 kB of non-essential -cruft, the odds plunge to 1%. - -On top of that, with larger inputs, the binary may be now running 5-10x times -slower than before - so the overall drop in fuzzing efficiency may be easily -as high as 500x or so. - -In practice, this means that you shouldn't fuzz image parsers with your -vacation photos. Generate a tiny 16x16 picture instead, and run it through -`jpegtran` or `pngcrunch` for good measure. The same goes for most other types -of documents. - -There's plenty of small starting test cases in ../testcases/ - try them out -or submit new ones! - -If you want to start with a larger, third-party corpus, run `afl-cmin` with an -aggressive timeout on that data set first. - -## 2. Use a simpler target - -Consider using a simpler target binary in your fuzzing work. For example, for -image formats, bundled utilities such as `djpeg`, `readpng`, or `gifhisto` are -considerably (10-20x) faster than the convert tool from ImageMagick - all while exercising roughly the same library-level image parsing code. - -Even if you don't have a lightweight harness for a particular target, remember -that you can always use another, related library to generate a corpus that will -be then manually fed to a more resource-hungry program later on. - -Also note that reading the fuzzing input via stdin is faster than reading from -a file. - -## 3. Use LLVM persistent instrumentation - -The LLVM mode offers a "persistent", in-process fuzzing mode that can -work well for certain types of self-contained libraries, and for fast targets, -can offer performance gains up to 5-10x; and a "deferred fork server" mode -that can offer huge benefits for programs with high startup overhead. Both -modes require you to edit the source code of the fuzzed program, but the -changes often amount to just strategically placing a single line or two. - -If there are important data comparisons performed (e.g. `strcmp(ptr, MAGIC_HDR)`) -then using laf-intel (see instrumentation/README.laf-intel.md) will help `afl-fuzz` a lot -to get to the important parts in the code. - -If you are only interested in specific parts of the code being fuzzed, you can -instrument_files the files that are actually relevant. This improves the speed and -accuracy of afl. See instrumentation/README.instrument_list.md - -## 4. Profile and optimize the binary - -Check for any parameters or settings that obviously improve performance. For -example, the djpeg utility that comes with IJG jpeg and libjpeg-turbo can be -called with: - -```bash - -dct fast -nosmooth -onepass -dither none -scale 1/4 -``` - -...and that will speed things up. There is a corresponding drop in the quality -of decoded images, but it's probably not something you care about. - -In some programs, it is possible to disable output altogether, or at least use -an output format that is computationally inexpensive. For example, with image -transcoding tools, converting to a BMP file will be a lot faster than to PNG. - -With some laid-back parsers, enabling "strict" mode (i.e., bailing out after -first error) may result in smaller files and improved run time without -sacrificing coverage; for example, for sqlite, you may want to specify -bail. - -If the program is still too slow, you can use `strace -tt` or an equivalent -profiling tool to see if the targeted binary is doing anything silly. -Sometimes, you can speed things up simply by specifying `/dev/null` as the -config file, or disabling some compile-time features that aren't really needed -for the job (try `./configure --help`). One of the notoriously resource-consuming -things would be calling other utilities via `exec*()`, `popen()`, `system()`, or -equivalent calls; for example, tar can invoke external decompression tools -when it decides that the input file is a compressed archive. - -Some programs may also intentionally call `sleep()`, `usleep()`, or `nanosleep()`; -vim is a good example of that. Other programs may attempt `fsync()` and so on. -There are third-party libraries that make it easy to get rid of such code, -e.g.: - - https://launchpad.net/libeatmydata - -In programs that are slow due to unavoidable initialization overhead, you may -want to try the LLVM deferred forkserver mode (see README.llvm.md), -which can give you speed gains up to 10x, as mentioned above. - -Last but not least, if you are using ASAN and the performance is unacceptable, -consider turning it off for now, and manually examining the generated corpus -with an ASAN-enabled binary later on. - -## 5. Instrument just what you need - -Instrument just the libraries you actually want to stress-test right now, one -at a time. Let the program use system-wide, non-instrumented libraries for -any functionality you don't actually want to fuzz. For example, in most -cases, it doesn't make to instrument `libgmp` just because you're testing a -crypto app that relies on it for bignum math. - -Beware of programs that come with oddball third-party libraries bundled with -their source code (Spidermonkey is a good example of this). Check `./configure` -options to use non-instrumented system-wide copies instead. - -## 6. Parallelize your fuzzers - -The fuzzer is designed to need ~1 core per job. This means that on a, say, -4-core system, you can easily run four parallel fuzzing jobs with relatively -little performance hit. For tips on how to do that, see parallel_fuzzing.md. - -The `afl-gotcpu` utility can help you understand if you still have idle CPU -capacity on your system. (It won't tell you about memory bandwidth, cache -misses, or similar factors, but they are less likely to be a concern.) - -## 7. Keep memory use and timeouts in check - -Consider setting low values for `-m` and `-t`. - -For programs that are nominally very fast, but get sluggish for some inputs, -you can also try setting `-t` values that are more punishing than what `afl-fuzz` -dares to use on its own. On fast and idle machines, going down to `-t 5` may be -a viable plan. - -The `-m` parameter is worth looking at, too. Some programs can end up spending -a fair amount of time allocating and initializing megabytes of memory when -presented with pathological inputs. Low `-m` values can make them give up sooner -and not waste CPU time. - -## 8. Check OS configuration - -There are several OS-level factors that may affect fuzzing speed: - - - If you have no risk of power loss then run your fuzzing on a tmpfs - partition. This increases the performance noticably. - Alternatively you can use `AFL_TMPDIR` to point to a tmpfs location to - just write the input file to a tmpfs. - - High system load. Use idle machines where possible. Kill any non-essential - CPU hogs (idle browser windows, media players, complex screensavers, etc). - - Network filesystems, either used for fuzzer input / output, or accessed by - the fuzzed binary to read configuration files (pay special attention to the - home directory - many programs search it for dot-files). - - Disable all the spectre, meltdown etc. security countermeasures in the - kernel if your machine is properly separated: - -``` -ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off -no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable -nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off -spectre_v2=off stf_barrier=off -``` - In most Linux distributions you can put this into a `/etc/default/grub` - variable. - You can use `sudo afl-persistent-config` to set these options for you. - -The following list of changes are made when executing `afl-system-config`: - - - On-demand CPU scaling. The Linux `ondemand` governor performs its analysis - on a particular schedule and is known to underestimate the needs of - short-lived processes spawned by `afl-fuzz` (or any other fuzzer). On Linux, - this can be fixed with: - -``` bash - cd /sys/devices/system/cpu - echo performance | tee cpu*/cpufreq/scaling_governor -``` - - On other systems, the impact of CPU scaling will be different; when fuzzing, - use OS-specific tools to find out if all cores are running at full speed. - - Transparent huge pages. Some allocators, such as `jemalloc`, can incur a - heavy fuzzing penalty when transparent huge pages (THP) are enabled in the - kernel. You can disable this via: - -```bash - echo never > /sys/kernel/mm/transparent_hugepage/enabled -``` - - - Suboptimal scheduling strategies. The significance of this will vary from - one target to another, but on Linux, you may want to make sure that the - following options are set: - -```bash - echo 1 >/proc/sys/kernel/sched_child_runs_first - echo 1 >/proc/sys/kernel/sched_autogroup_enabled -``` - - Setting a different scheduling policy for the fuzzer process - say - `SCHED_RR` - can usually speed things up, too, but needs to be done with - care. - diff --git a/docs/rpc_statsd.md b/docs/rpc_statsd.md index 898ad099..003b9c79 100644 --- a/docs/rpc_statsd.md +++ b/docs/rpc_statsd.md @@ -1,143 +1,190 @@ -# Remote monitoring with StatsD +# Remote monitoring and metrics visualization -StatsD allows you to receive and aggregate metrics from a wide range of applications and retransmit them to the backend of your choice. -This enables you to create nice and readable dashboards containing all the information you need on your fuzzer instances. -No need to write your own statistics parsing system, deploy and maintain it to all your instances, sync with your graph rendering system... +AFL++ can send out metrics as StatsD messages. For remote monitoring and +visualization of the metrics, you can set up a tool chain. For example, with +Prometheus and Grafana. All tools are free and open source. -The available metrics are : +This enables you to create nice and readable dashboards containing all the +information you need on your fuzzer instances. There is no need to write your +own statistics parsing system, deploy and maintain it to all your instances, and +sync with your graph rendering system. + +Compared to the default integrated UI of AFL++, this can help you to visualize +trends and the fuzzing state over time. You might be able to see when the +fuzzing process has reached a state of no progress and visualize what are the +"best strategies" for your targets (according to your own criteria). You can do +so without logging into each instance individually. + + + +This is an example visualization with Grafana. The dashboard can be imported +with [this JSON template](resources/grafana-afl++.json). + +## AFL++ metrics and StatsD + +StatsD allows you to receive and aggregate metrics from a wide range of +applications and retransmit them to a backend of your choice. + +From AFL++, StatsD can receive the following metrics: +- cur_path - cycle_done - cycles_wo_finds +- edges_found - execs_done - execs_per_sec -- paths_total +- havoc_expansion +- max_depth - paths_favored - paths_found - paths_imported -- max_depth -- cur_path +- paths_total - pending_favs - pending_total -- variable_paths +- slowest_exec_ms +- total_crashes - unique_crashes - unique_hangs -- total_crashes -- slowest_exec_ms -- edges_found - var_byte_count -- havoc_expansion +- variable_paths -Compared to the default integrated UI, these metrics give you the opportunity to visualize trends and fuzzing state over time. -By doing so, you might be able to see when the fuzzing process has reached a state of no progress, visualize what are the "best strategies" -(according to your own criteria) for your targets, etc. And doing so without requiring to log into each instance manually. +Depending on your StatsD server, you will be able to monitor, trigger alerts, or +perform actions based on these metrics (for example: alert on slow exec/s for a +new build, threshold of crashes, time since last crash > X, and so on). -An example visualisation may look like the following: - +## Setting environment variables in AFL++ -*Notes: The exact same dashboard can be imported with [this JSON template](resources/grafana-afl++.json).* +1. To enable the StatsD metrics collection on your fuzzer instances, set the + environment variable `AFL_STATSD=1`. By default, AFL++ will send the metrics + over UDP to 127.0.0.1:8125. -## How to use +2. To enable tags for each metric based on their format (banner and + afl_version), set the environment variable `AFL_STATSD_TAGS_FLAVOR`. By + default, no tags will be added to the metrics. -To enable the StatsD reporting on your fuzzer instances, you need to set the environment variable `AFL_STATSD=1`. + The available values are the following: + - `dogstatsd` + - `influxdb` + - `librato` + - `signalfx` -Setting `AFL_STATSD_TAGS_FLAVOR` to the provider of your choice will assign tags / labels to each metric based on their format. -The possible values are `dogstatsd`, `librato`, `signalfx` or `influxdb`. -For more information on these env vars, check out `docs/env_variables.md`. + For more information on environment variables, see + [env_variables.md](env_variables.md). -The simplest way of using this feature is to use any metric provider and change the host/port of your StatsD daemon, -with `AFL_STATSD_HOST` and `AFL_STATSD_PORT`, if required (defaults are `localhost` and port `8125`). -To get started, here are some instructions with free and open source tools. -The following setup is based on Prometheus, statsd_exporter and Grafana. -Grafana here is not mandatory, but gives you some nice graphs and features. + Note: When using multiple fuzzer instances with StatsD it is *strongly* + recommended to set up `AFL_STATSD_TAGS_FLAVOR` to match your StatsD server. + This will allow you to see individual fuzzer performance, detect bad ones, + and see the progress of each strategy. -Depending on your setup and infrastructure, you may want to run these applications not on your fuzzer instances. -Only one instance of these 3 application is required for all your fuzzers. +3. Optional: To set the host and port of your StatsD daemon, set + `AFL_STATSD_HOST` and `AFL_STATSD_PORT`. The default values are `localhost` + and `8125`. -To simplify everything, we will use Docker and docker-compose. -Make sure you have them both installed. On most common Linux distributions, it's as simple as: +## Installing and setting up StatsD, Prometheus, and Grafana -```sh -curl -fsSL https://get.docker.com -o get-docker.sh -sh get-docker.sh -``` +The easiest way to install and set up the infrastructure is with Docker and +Docker Compose. -Once that's done, we can create the infrastructure. -Create and move into the directory of your choice. This will store all the configurations files required. - -First, create a `docker-compose.yml` containing the following: -```yml -version: '3' - -networks: - statsd-net: - driver: bridge - -services: - prometheus: - image: prom/prometheus - container_name: prometheus - volumes: - - ./prometheus.yml:/prometheus.yml - command: - - '--config.file=/prometheus.yml' - restart: unless-stopped - ports: - - "9090:9090" - networks: - - statsd-net - - statsd_exporter: - image: prom/statsd-exporter - container_name: statsd_exporter - volumes: - - ./statsd_mapping.yml:/statsd_mapping.yml - command: - - "--statsd.mapping-config=/statsd_mapping.yml" - ports: - - "9102:9102/tcp" - - "8125:9125/udp" - networks: - - statsd-net - - grafana: - image: grafana/grafana - container_name: grafana - restart: unless-stopped - ports: - - "3000:3000" - networks: - - statsd-net -``` +Depending on your fuzzing setup and infrastructure, you may not want to run +these applications on your fuzzer instances. This setup may be modified before +use in a production environment; for example, adding passwords, creating volumes +for storage, tweaking the metrics gathering to get host metrics (CPU, RAM, and +so on). -Then `prometheus.yml` -```yml -global: - scrape_interval: 15s - evaluation_interval: 15s +For all your fuzzing instances, only one instance of Prometheus and Grafana is +required. The +[statsd exporter](https://registry.hub.docker.com/r/prom/statsd-exporter) +converts the StatsD metrics to Prometheus. If you are using a provider that +supports StatsD directly, you can skip this part of the setup." -scrape_configs: - - job_name: 'fuzzing_metrics' - static_configs: - - targets: ['statsd_exporter:9102'] -``` +You can create and move the infrastructure files into a directory of your +choice. The directory will store all the required configuration files. -And finally `statsd_mapping.yml` -```yml -mappings: -- match: "fuzzing.*" - name: "fuzzing" - labels: - type: "$1" -``` +To install and set up Prometheus and Grafana: + +1. Install Docker and Docker Compose: + + ```sh + curl -fsSL https://get.docker.com -o get-docker.sh + sh get-docker.sh + ``` -Run `docker-compose up -d`. +2. Create a `docker-compose.yml` containing the following: -Everything should now be setup, you are now able to run your fuzzers with + ```yml + version: '3' + + networks: + statsd-net: + driver: bridge + + services: + prometheus: + image: prom/prometheus + container_name: prometheus + volumes: + - ./prometheus.yml:/prometheus.yml + command: + - '--config.file=/prometheus.yml' + restart: unless-stopped + ports: + - "9090:9090" + networks: + - statsd-net + + statsd_exporter: + image: prom/statsd-exporter + container_name: statsd_exporter + volumes: + - ./statsd_mapping.yml:/statsd_mapping.yml + command: + - "--statsd.mapping-config=/statsd_mapping.yml" + ports: + - "9102:9102/tcp" + - "8125:9125/udp" + networks: + - statsd-net + + grafana: + image: grafana/grafana + container_name: grafana + restart: unless-stopped + ports: + - "3000:3000" + networks: + - statsd-net + ``` + +3. Create a `prometheus.yml` containing the following: + + ```yml + global: + scrape_interval: 15s + evaluation_interval: 15s + + scrape_configs: + - job_name: 'fuzzing_metrics' + static_configs: + - targets: ['statsd_exporter:9102'] + ``` + +4. Create a `statsd_mapping.yml` containing the following: + + ```yml + mappings: + - match: "fuzzing.*" + name: "fuzzing" + labels: + type: "$1" + ``` + +5. Run `docker-compose up -d`. + +## Running AFL++ with StatsD + +To run your fuzzing instances: ``` -AFL_STATSD_TAGS_FLAVOR=dogstatsd AFL_STATSD=1 afl-fuzz -M test-fuzzer-1 -i i -o o ./bin/my-application @@ -AFL_STATSD_TAGS_FLAVOR=dogstatsd AFL_STATSD=1 afl-fuzz -S test-fuzzer-2 -i i -o o ./bin/my-application @@ +AFL_STATSD_TAGS_FLAVOR=dogstatsd AFL_STATSD=1 afl-fuzz -M test-fuzzer-1 -i i -o o [./bin/my-application] @@ +AFL_STATSD_TAGS_FLAVOR=dogstatsd AFL_STATSD=1 afl-fuzz -S test-fuzzer-2 -i i -o o [./bin/my-application] @@ ... -``` - -This setup may be modified before use in a production environment. Depending on your needs: adding passwords, creating volumes for storage, -tweaking the metrics gathering to get host metrics (CPU, RAM ...). +``` \ No newline at end of file diff --git a/docs/sister_projects.md b/docs/sister_projects.md deleted file mode 100644 index 5cb3a102..00000000 --- a/docs/sister_projects.md +++ /dev/null @@ -1,319 +0,0 @@ -# Sister projects - -This doc lists some of the projects that are inspired by, derived from, -designed for, or meant to integrate with AFL. See README.md for the general -instruction manual. - -!!! -!!! This list is outdated and needs an update, missing: e.g. Angora, FairFuzz -!!! - -## Support for other languages / environments: - -### Python AFL (Jakub Wilk) - -Allows fuzz-testing of Python programs. Uses custom instrumentation and its -own forkserver. - -http://jwilk.net/software/python-afl - -### Go-fuzz (Dmitry Vyukov) - -AFL-inspired guided fuzzing approach for Go targets: - -https://github.com/dvyukov/go-fuzz - -### afl.rs (Keegan McAllister) - -Allows Rust features to be easily fuzzed with AFL (using the LLVM mode). - -https://github.com/kmcallister/afl.rs - -### OCaml support (KC Sivaramakrishnan) - -Adds AFL-compatible instrumentation to OCaml programs. - -https://github.com/ocamllabs/opam-repo-dev/pull/23 -http://canopy.mirage.io/Posts/Fuzzing - -### AFL for GCJ Java and other GCC frontends (-) - -GCC Java programs are actually supported out of the box - simply rename -afl-gcc to afl-gcj. Unfortunately, by default, unhandled exceptions in GCJ do -not result in abort() being called, so you will need to manually add a -top-level exception handler that exits with SIGABRT or something equivalent. - -Other GCC-supported languages should be fairly easy to get working, but may -face similar problems. See https://gcc.gnu.org/frontends.html for a list of -options. - -## AFL-style in-process fuzzer for LLVM (Kostya Serebryany) - -Provides an evolutionary instrumentation-guided fuzzing harness that allows -some programs to be fuzzed without the fork / execve overhead. (Similar -functionality is now available as the "persistent" feature described in -[the llvm_mode readme](../instrumentation/README.llvm.md)) - -http://llvm.org/docs/LibFuzzer.html - -## TriforceAFL (Tim Newsham and Jesse Hertz) - -Leverages QEMU full system emulation mode to allow AFL to target operating -systems and other alien worlds: - -https://www.nccgroup.trust/us/about-us/newsroom-and-events/blog/2016/june/project-triforce-run-afl-on-everything/ - -## WinAFL (Ivan Fratric) - -As the name implies, allows you to fuzz Windows binaries (using DynamoRio). - -https://github.com/ivanfratric/winafl - -Another Windows alternative may be: - -https://github.com/carlosgprado/BrundleFuzz/ - -## Network fuzzing - -### Preeny (Yan Shoshitaishvili) - -Provides a fairly simple way to convince dynamically linked network-centric -programs to read from a file or not fork. Not AFL-specific, but described as -useful by many users. Some assembly required. - -https://github.com/zardus/preeny - -## Distributed fuzzing and related automation - -### roving (Richo Healey) - -A client-server architecture for effortlessly orchestrating AFL runs across -a fleet of machines. You don't want to use this on systems that face the -Internet or live in other untrusted environments. - -https://github.com/richo/roving - -### Distfuzz-AFL (Martijn Bogaard) - -Simplifies the management of afl-fuzz instances on remote machines. The -author notes that the current implementation isn't secure and should not -be exposed on the Internet. - -https://github.com/MartijnB/disfuzz-afl - -### AFLDFF (quantumvm) - -A nice GUI for managing AFL jobs. - -https://github.com/quantumvm/AFLDFF - -### afl-launch (Ben Nagy) - -Batch AFL launcher utility with a simple CLI. - -https://github.com/bnagy/afl-launch - -### AFL Utils (rc0r) - -Simplifies the triage of discovered crashes, start parallel instances, etc. - -https://github.com/rc0r/afl-utils - -### AFL crash analyzer (floyd) - -Another crash triage tool: - -https://github.com/floyd-fuh/afl-crash-analyzer - -### afl-extras (fekir) - -Collect data, parallel afl-tmin, startup scripts. - -https://github.com/fekir/afl-extras - -### afl-fuzzing-scripts (Tobias Ospelt) - -Simplifies starting up multiple parallel AFL jobs. - -https://github.com/floyd-fuh/afl-fuzzing-scripts/ - -### afl-sid (Jacek Wielemborek) - -Allows users to more conveniently build and deploy AFL via Docker. - -https://github.com/d33tah/afl-sid - -Another Docker-related project: - -https://github.com/ozzyjohnson/docker-afl - -### afl-monitor (Paul S. Ziegler) - -Provides more detailed and versatile statistics about your running AFL jobs. - -https://github.com/reflare/afl-monitor - -### FEXM (Security in Telecommunications) - -Fully automated fuzzing framework, based on AFL - -https://github.com/fgsect/fexm - -## Crash triage, coverage analysis, and other companion tools: - -### afl-crash-analyzer (Tobias Ospelt) - -Makes it easier to navigate and annotate crashing test cases. - -https://github.com/floyd-fuh/afl-crash-analyzer/ - -### Crashwalk (Ben Nagy) - -AFL-aware tool to annotate and sort through crashing test cases. - -https://github.com/bnagy/crashwalk - -### afl-cov (Michael Rash) - -Produces human-readable coverage data based on the output queue of afl-fuzz. - -https://github.com/mrash/afl-cov - -### afl-sancov (Bhargava Shastry) - -Similar to afl-cov, but uses clang sanitizer instrumentation. - -https://github.com/bshastry/afl-sancov - -### RecidiVM (Jakub Wilk) - -Makes it easy to estimate memory usage limits when fuzzing with ASAN or MSAN. - -http://jwilk.net/software/recidivm - -### aflize (Jacek Wielemborek) - -Automatically build AFL-enabled versions of Debian packages. - -https://github.com/d33tah/aflize - -### afl-ddmin-mod (Markus Teufelberger) - -A variant of afl-tmin that uses a more sophisticated (but slower) -minimization algorithm. - -https://github.com/MarkusTeufelberger/afl-ddmin-mod - -### afl-kit (Kuang-che Wu) - -Replacements for afl-cmin and afl-tmin with additional features, such -as the ability to filter crashes based on stderr patterns. - -https://github.com/kcwu/afl-kit - -## Narrow-purpose or experimental: - -### Cygwin support (Ali Rizvi-Santiago) - -Pretty self-explanatory. As per the author, this "mostly" ports AFL to -Windows. Field reports welcome! - -https://github.com/arizvisa/afl-cygwin - -### Pause and resume scripts (Ben Nagy) - -Simple automation to suspend and resume groups of fuzzing jobs. - -https://github.com/bnagy/afl-trivia - -### Static binary-only instrumentation (Aleksandar Nikolich) - -Allows black-box binaries to be instrumented statically (i.e., by modifying -the binary ahead of the time, rather than translating it on the run). Author -reports better performance compared to QEMU, but occasional translation -errors with stripped binaries. - -https://github.com/vanhauser-thc/afl-dyninst - -### AFL PIN (Parker Thompson) - -Early-stage Intel PIN instrumentation support (from before we settled on -faster-running QEMU). - -https://github.com/mothran/aflpin - -### AFL-style instrumentation in llvm (Kostya Serebryany) - -Allows AFL-equivalent instrumentation to be injected at compiler level. -This is currently not supported by AFL as-is, but may be useful in other -projects. - -https://code.google.com/p/address-sanitizer/wiki/AsanCoverage#Coverage_counters - -### AFL JS (Han Choongwoo) - -One-off optimizations to speed up the fuzzing of JavaScriptCore (now likely -superseded by LLVM deferred forkserver init - see README.llvm.md). - -https://github.com/tunz/afl-fuzz-js - -### AFL harness for fwknop (Michael Rash) - -An example of a fairly involved integration with AFL. - -https://github.com/mrash/fwknop/tree/master/test/afl - -### Building harnesses for DNS servers (Jonathan Foote, Ron Bowes) - -Two articles outlining the general principles and showing some example code. - -https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop -https://goo.gl/j9EgFf - -### Fuzzer shell for SQLite (Richard Hipp) - -A simple SQL shell designed specifically for fuzzing the underlying library. - -http://www.sqlite.org/src/artifact/9e7e273da2030371 - -### Support for Python mutation modules (Christian Holler) - -now integrated in AFL++, originally from here -https://github.com/choller/afl/blob/master/docs/mozilla/python_modules.txt - -### Support for selective instrumentation (Christian Holler) - -now integrated in AFL++, originally from here -https://github.com/choller/afl/blob/master/docs/mozilla/partial_instrumentation.txt - -### Syzkaller (Dmitry Vyukov) - -A similar guided approach as applied to fuzzing syscalls: - -https://github.com/google/syzkaller/wiki/Found-Bugs -https://github.com/dvyukov/linux/commit/33787098ffaaa83b8a7ccf519913ac5fd6125931 -http://events.linuxfoundation.org/sites/events/files/slides/AFL%20filesystem%20fuzzing%2C%20Vault%202016_0.pdf - - -### Kernel Snapshot Fuzzing using Unicornafl (Security in Telecommunications) - -https://github.com/fgsect/unicorefuzz - -### Android support (ele7enxxh) - -Based on a somewhat dated version of AFL: - -https://github.com/ele7enxxh/android-afl - -### CGI wrapper (floyd) - -Facilitates the testing of CGI scripts. - -https://github.com/floyd-fuh/afl-cgi-wrapper - -### Fuzzing difficulty estimation (Marcel Boehme) - -A fork of AFL that tries to quantify the likelihood of finding additional -paths or crashes at any point in a fuzzing job. - -https://github.com/mboehme/pythia diff --git a/docs/status_screen.md b/docs/status_screen.md deleted file mode 100644 index b1cb9696..00000000 --- a/docs/status_screen.md +++ /dev/null @@ -1,444 +0,0 @@ -# Understanding the status screen - -This document provides an overview of the status screen - plus tips for -troubleshooting any warnings and red text shown in the UI. See README.md for -the general instruction manual. - -## A note about colors - -The status screen and error messages use colors to keep things readable and -attract your attention to the most important details. For example, red almost -always means "consult this doc" :-) - -Unfortunately, the UI will render correctly only if your terminal is using -traditional un*x palette (white text on black background) or something close -to that. - -If you are using inverse video, you may want to change your settings, say: - -- For GNOME Terminal, go to `Edit > Profile` preferences, select the "colors" tab, and from the list of built-in schemes, choose "white on black". -- For the MacOS X Terminal app, open a new window using the "Pro" scheme via the `Shell > New Window` menu (or make "Pro" your default). - -Alternatively, if you really like your current colors, you can edit config.h -to comment out USE_COLORS, then do `make clean all`. - -I'm not aware of any other simple way to make this work without causing -other side effects - sorry about that. - -With that out of the way, let's talk about what's actually on the screen... - -### The status bar - -``` -american fuzzy lop ++3.01a (default) [fast] {0} -``` - -The top line shows you which mode afl-fuzz is running in -(normal: "american fuzy lop", crash exploration mode: "peruvian rabbit mode") -and the version of AFL++. -Next to the version is the banner, which, if not set with -T by hand, will -either show the binary name being fuzzed, or the -M/-S main/secondary name for -parallel fuzzing. -Second to last is the power schedule mode being run (default: fast). -Finally, the last item is the CPU id. - -### Process timing - -``` - +----------------------------------------------------+ - | run time : 0 days, 8 hrs, 32 min, 43 sec | - | last new path : 0 days, 0 hrs, 6 min, 40 sec | - | last uniq crash : none seen yet | - | last uniq hang : 0 days, 1 hrs, 24 min, 32 sec | - +----------------------------------------------------+ -``` - -This section is fairly self-explanatory: it tells you how long the fuzzer has -been running and how much time has elapsed since its most recent finds. This is -broken down into "paths" (a shorthand for test cases that trigger new execution -patterns), crashes, and hangs. - -When it comes to timing: there is no hard rule, but most fuzzing jobs should be -expected to run for days or weeks; in fact, for a moderately complex project, the -first pass will probably take a day or so. Every now and then, some jobs -will be allowed to run for months. - -There's one important thing to watch out for: if the tool is not finding new -paths within several minutes of starting, you're probably not invoking the -target binary correctly and it never gets to parse the input files we're -throwing at it; another possible explanations are that the default memory limit -(`-m`) is too restrictive, and the program exits after failing to allocate a -buffer very early on; or that the input files are patently invalid and always -fail a basic header check. - -If there are no new paths showing up for a while, you will eventually see a big -red warning in this section, too :-) - -### Overall results - -``` - +-----------------------+ - | cycles done : 0 | - | total paths : 2095 | - | uniq crashes : 0 | - | uniq hangs : 19 | - +-----------------------+ -``` - -The first field in this section gives you the count of queue passes done so far - that is, the number of times the fuzzer went over all the interesting test -cases discovered so far, fuzzed them, and looped back to the very beginning. -Every fuzzing session should be allowed to complete at least one cycle; and -ideally, should run much longer than that. - -As noted earlier, the first pass can take a day or longer, so sit back and -relax. - -To help make the call on when to hit `Ctrl-C`, the cycle counter is color-coded. -It is shown in magenta during the first pass, progresses to yellow if new finds -are still being made in subsequent rounds, then blue when that ends - and -finally, turns green after the fuzzer hasn't been seeing any action for a -longer while. - -The remaining fields in this part of the screen should be pretty obvious: -there's the number of test cases ("paths") discovered so far, and the number of -unique faults. The test cases, crashes, and hangs can be explored in real-time -by browsing the output directory, as discussed in README.md. - -### Cycle progress - -``` - +-------------------------------------+ - | now processing : 1296 (61.86%) | - | paths timed out : 0 (0.00%) | - +-------------------------------------+ -``` - -This box tells you how far along the fuzzer is with the current queue cycle: it -shows the ID of the test case it is currently working on, plus the number of -inputs it decided to ditch because they were persistently timing out. - -The "*" suffix sometimes shown in the first line means that the currently -processed path is not "favored" (a property discussed later on). - -### Map coverage - -``` - +--------------------------------------+ - | map density : 10.15% / 29.07% | - | count coverage : 4.03 bits/tuple | - +--------------------------------------+ -``` - -The section provides some trivia about the coverage observed by the -instrumentation embedded in the target binary. - -The first line in the box tells you how many branch tuples we have already -hit, in proportion to how much the bitmap can hold. The number on the left -describes the current input; the one on the right is the value for the entire -input corpus. - -Be wary of extremes: - - - Absolute numbers below 200 or so suggest one of three things: that the - program is extremely simple; that it is not instrumented properly (e.g., - due to being linked against a non-instrumented copy of the target - library); or that it is bailing out prematurely on your input test cases. - The fuzzer will try to mark this in pink, just to make you aware. - - Percentages over 70% may very rarely happen with very complex programs - that make heavy use of template-generated code. - Because high bitmap density makes it harder for the fuzzer to reliably - discern new program states, I recommend recompiling the binary with - `AFL_INST_RATIO=10` or so and trying again (see env_variables.md). - The fuzzer will flag high percentages in red. Chances are, you will never - see that unless you're fuzzing extremely hairy software (say, v8, perl, - ffmpeg). - -The other line deals with the variability in tuple hit counts seen in the -binary. In essence, if every taken branch is always taken a fixed number of -times for all the inputs we have tried, this will read `1.00`. As we manage -to trigger other hit counts for every branch, the needle will start to move -toward `8.00` (every bit in the 8-bit map hit), but will probably never -reach that extreme. - -Together, the values can be useful for comparing the coverage of several -different fuzzing jobs that rely on the same instrumented binary. - -### Stage progress - -``` - +-------------------------------------+ - | now trying : interest 32/8 | - | stage execs : 3996/34.4k (11.62%) | - | total execs : 27.4M | - | exec speed : 891.7/sec | - +-------------------------------------+ -``` - -This part gives you an in-depth peek at what the fuzzer is actually doing right -now. It tells you about the current stage, which can be any of: - - - calibration - a pre-fuzzing stage where the execution path is examined - to detect anomalies, establish baseline execution speed, and so on. Executed - very briefly whenever a new find is being made. - - trim L/S - another pre-fuzzing stage where the test case is trimmed to the - shortest form that still produces the same execution path. The length (L) - and stepover (S) are chosen in general relationship to file size. - - bitflip L/S - deterministic bit flips. There are L bits toggled at any given - time, walking the input file with S-bit increments. The current L/S variants - are: `1/1`, `2/1`, `4/1`, `8/8`, `16/8`, `32/8`. - - arith L/8 - deterministic arithmetics. The fuzzer tries to subtract or add - small integers to 8-, 16-, and 32-bit values. The stepover is always 8 bits. - - interest L/8 - deterministic value overwrite. The fuzzer has a list of known - "interesting" 8-, 16-, and 32-bit values to try. The stepover is 8 bits. - - extras - deterministic injection of dictionary terms. This can be shown as - "user" or "auto", depending on whether the fuzzer is using a user-supplied - dictionary (`-x`) or an auto-created one. You will also see "over" or "insert", - depending on whether the dictionary words overwrite existing data or are - inserted by offsetting the remaining data to accommodate their length. - - havoc - a sort-of-fixed-length cycle with stacked random tweaks. The - operations attempted during this stage include bit flips, overwrites with - random and "interesting" integers, block deletion, block duplication, plus - assorted dictionary-related operations (if a dictionary is supplied in the - first place). - - splice - a last-resort strategy that kicks in after the first full queue - cycle with no new paths. It is equivalent to 'havoc', except that it first - splices together two random inputs from the queue at some arbitrarily - selected midpoint. - - sync - a stage used only when `-M` or `-S` is set (see parallel_fuzzing.md). - No real fuzzing is involved, but the tool scans the output from other - fuzzers and imports test cases as necessary. The first time this is done, - it may take several minutes or so. - -The remaining fields should be fairly self-evident: there's the exec count -progress indicator for the current stage, a global exec counter, and a -benchmark for the current program execution speed. This may fluctuate from -one test case to another, but the benchmark should be ideally over 500 execs/sec -most of the time - and if it stays below 100, the job will probably take very -long. - -The fuzzer will explicitly warn you about slow targets, too. If this happens, -see the [perf_tips.md](perf_tips.md) file included with the fuzzer for ideas on how to speed -things up. - -### Findings in depth - -``` - +--------------------------------------+ - | favored paths : 879 (41.96%) | - | new edges on : 423 (20.19%) | - | total crashes : 0 (0 unique) | - | total tmouts : 24 (19 unique) | - +--------------------------------------+ -``` - -This gives you several metrics that are of interest mostly to complete nerds. -The section includes the number of paths that the fuzzer likes the most based -on a minimization algorithm baked into the code (these will get considerably -more air time), and the number of test cases that actually resulted in better -edge coverage (versus just pushing the branch hit counters up). There are also -additional, more detailed counters for crashes and timeouts. - -Note that the timeout counter is somewhat different from the hang counter; this -one includes all test cases that exceeded the timeout, even if they did not -exceed it by a margin sufficient to be classified as hangs. - -### Fuzzing strategy yields - -``` - +-----------------------------------------------------+ - | bit flips : 57/289k, 18/289k, 18/288k | - | byte flips : 0/36.2k, 4/35.7k, 7/34.6k | - | arithmetics : 53/2.54M, 0/537k, 0/55.2k | - | known ints : 8/322k, 12/1.32M, 10/1.70M | - | dictionary : 9/52k, 1/53k, 1/24k | - |havoc/splice : 1903/20.0M, 0/0 | - |py/custom/rq : unused, 53/2.54M, unused | - | trim/eff : 20.31%/9201, 17.05% | - +-----------------------------------------------------+ -``` - -This is just another nerd-targeted section keeping track of how many paths we -have netted, in proportion to the number of execs attempted, for each of the -fuzzing strategies discussed earlier on. This serves to convincingly validate -assumptions about the usefulness of the various approaches taken by afl-fuzz. - -The trim strategy stats in this section are a bit different than the rest. -The first number in this line shows the ratio of bytes removed from the input -files; the second one corresponds to the number of execs needed to achieve this -goal. Finally, the third number shows the proportion of bytes that, although -not possible to remove, were deemed to have no effect and were excluded from -some of the more expensive deterministic fuzzing steps. - -Note that when deterministic mutation mode is off (which is the default -because it is not very efficient) the first five lines display -"disabled (default, enable with -D)". - -Only what is activated will have counter shown. - -### Path geometry - -``` - +---------------------+ - | levels : 5 | - | pending : 1570 | - | pend fav : 583 | - | own finds : 0 | - | imported : 0 | - | stability : 100.00% | - +---------------------+ -``` - -The first field in this section tracks the path depth reached through the -guided fuzzing process. In essence: the initial test cases supplied by the -user are considered "level 1". The test cases that can be derived from that -through traditional fuzzing are considered "level 2"; the ones derived by -using these as inputs to subsequent fuzzing rounds are "level 3"; and so forth. -The maximum depth is therefore a rough proxy for how much value you're getting -out of the instrumentation-guided approach taken by afl-fuzz. - -The next field shows you the number of inputs that have not gone through any -fuzzing yet. The same stat is also given for "favored" entries that the fuzzer -really wants to get to in this queue cycle (the non-favored entries may have to -wait a couple of cycles to get their chance). - -Next, we have the number of new paths found during this fuzzing section and -imported from other fuzzer instances when doing parallelized fuzzing; and the -extent to which identical inputs appear to sometimes produce variable behavior -in the tested binary. - -That last bit is actually fairly interesting: it measures the consistency of -observed traces. If a program always behaves the same for the same input data, -it will earn a score of 100%. When the value is lower but still shown in purple, -the fuzzing process is unlikely to be negatively affected. If it goes into red, -you may be in trouble, since AFL will have difficulty discerning between -meaningful and "phantom" effects of tweaking the input file. - -Now, most targets will just get a 100% score, but when you see lower figures, -there are several things to look at: - - - The use of uninitialized memory in conjunction with some intrinsic sources - of entropy in the tested binary. Harmless to AFL, but could be indicative - of a security bug. - - Attempts to manipulate persistent resources, such as left over temporary - files or shared memory objects. This is usually harmless, but you may want - to double-check to make sure the program isn't bailing out prematurely. - Running out of disk space, SHM handles, or other global resources can - trigger this, too. - - Hitting some functionality that is actually designed to behave randomly. - Generally harmless. For example, when fuzzing sqlite, an input like - `select random();` will trigger a variable execution path. - - Multiple threads executing at once in semi-random order. This is harmless - when the 'stability' metric stays over 90% or so, but can become an issue - if not. Here's what to try: - * Use afl-clang-fast from [instrumentation](../instrumentation/) - it uses a thread-local tracking - model that is less prone to concurrency issues, - * See if the target can be compiled or run without threads. Common - `./configure` options include `--without-threads`, `--disable-pthreads`, or - `--disable-openmp`. - * Replace pthreads with GNU Pth (https://www.gnu.org/software/pth/), which - allows you to use a deterministic scheduler. - - In persistent mode, minor drops in the "stability" metric can be normal, - because not all the code behaves identically when re-entered; but major - dips may signify that the code within `__AFL_LOOP()` is not behaving - correctly on subsequent iterations (e.g., due to incomplete clean-up or - reinitialization of the state) and that most of the fuzzing effort goes - to waste. - -The paths where variable behavior is detected are marked with a matching entry -in the `<out_dir>/queue/.state/variable_behavior/` directory, so you can look -them up easily. - -### CPU load - -``` - [cpu: 25%] -``` - -This tiny widget shows the apparent CPU utilization on the local system. It is -calculated by taking the number of processes in the "runnable" state, and then -comparing it to the number of logical cores on the system. - -If the value is shown in green, you are using fewer CPU cores than available on -your system and can probably parallelize to improve performance; for tips on -how to do that, see parallel_fuzzing.md. - -If the value is shown in red, your CPU is *possibly* oversubscribed, and -running additional fuzzers may not give you any benefits. - -Of course, this benchmark is very simplistic; it tells you how many processes -are ready to run, but not how resource-hungry they may be. It also doesn't -distinguish between physical cores, logical cores, and virtualized CPUs; the -performance characteristics of each of these will differ quite a bit. - -If you want a more accurate measurement, you can run the `afl-gotcpu` utility from the command line. - -### Addendum: status and plot files - -For unattended operation, some of the key status screen information can be also -found in a machine-readable format in the fuzzer_stats file in the output -directory. This includes: - - - `start_time` - unix time indicating the start time of afl-fuzz - - `last_update` - unix time corresponding to the last update of this file - - `run_time` - run time in seconds to the last update of this file - - `fuzzer_pid` - PID of the fuzzer process - - `cycles_done` - queue cycles completed so far - - `cycles_wo_finds` - number of cycles without any new paths found - - `execs_done` - number of execve() calls attempted - - `execs_per_sec` - overall number of execs per second - - `paths_total` - total number of entries in the queue - - `paths_favored` - number of queue entries that are favored - - `paths_found` - number of entries discovered through local fuzzing - - `paths_imported` - number of entries imported from other instances - - `max_depth` - number of levels in the generated data set - - `cur_path` - currently processed entry number - - `pending_favs` - number of favored entries still waiting to be fuzzed - - `pending_total` - number of all entries waiting to be fuzzed - - `variable_paths` - number of test cases showing variable behavior - - `stability` - percentage of bitmap bytes that behave consistently - - `bitmap_cvg` - percentage of edge coverage found in the map so far - - `unique_crashes` - number of unique crashes recorded - - `unique_hangs` - number of unique hangs encountered - - `last_path` - seconds since the last path was found - - `last_crash` - seconds since the last crash was found - - `last_hang` - seconds since the last hang was found - - `execs_since_crash` - execs since the last crash was found - - `exec_timeout` - the -t command line value - - `slowest_exec_ms` - real time of the slowest execution in ms - - `peak_rss_mb` - max rss usage reached during fuzzing in MB - - `edges_found` - how many edges have been found - - `var_byte_count` - how many edges are non-deterministic - - `afl_banner` - banner text (e.g. the target name) - - `afl_version` - the version of AFL used - - `target_mode` - default, persistent, qemu, unicorn, non-instrumented - - `command_line` - full command line used for the fuzzing session - -Most of these map directly to the UI elements discussed earlier on. - -On top of that, you can also find an entry called `plot_data`, containing a -plottable history for most of these fields. If you have gnuplot installed, you -can turn this into a nice progress report with the included `afl-plot` tool. - - -### Addendum: Automatically send metrics with StatsD - -In a CI environment or when running multiple fuzzers, it can be tedious to -log into each of them or deploy scripts to read the fuzzer statistics. -Using `AFL_STATSD` (and the other related environment variables `AFL_STATSD_HOST`, -`AFL_STATSD_PORT`, `AFL_STATSD_TAGS_FLAVOR`) you can automatically send metrics -to your favorite StatsD server. Depending on your StatsD server you will be able -to monitor, trigger alerts or perform actions based on these metrics (e.g: alert on -slow exec/s for a new build, threshold of crashes, time since last crash > X, etc). - -The selected metrics are a subset of all the metrics found in the status and in -the plot file. The list is the following: `cycle_done`, `cycles_wo_finds`, -`execs_done`,`execs_per_sec`, `paths_total`, `paths_favored`, `paths_found`, -`paths_imported`, `max_depth`, `cur_path`, `pending_favs`, `pending_total`, -`variable_paths`, `unique_crashes`, `unique_hangs`, `total_crashes`, -`slowest_exec_ms`, `edges_found`, `var_byte_count`, `havoc_expansion`. -Their definitions can be found in the addendum above. - -When using multiple fuzzer instances with StatsD it is *strongly* recommended to setup -the flavor (AFL_STATSD_TAGS_FLAVOR) to match your StatsD server. This will allow you -to see individual fuzzer performance, detect bad ones, see the progress of each -strategy... diff --git a/docs/technical_details.md b/docs/technical_details.md deleted file mode 100644 index b0ca493e..00000000 --- a/docs/technical_details.md +++ /dev/null @@ -1,550 +0,0 @@ -# Technical "whitepaper" for afl-fuzz - - -NOTE: this document is rather outdated! - - -This document provides a quick overview of the guts of American Fuzzy Lop. -See README.md for the general instruction manual; and for a discussion of -motivations and design goals behind AFL, see historical_notes.md. - -## 0. Design statement - -American Fuzzy Lop does its best not to focus on any singular principle of -operation and not be a proof-of-concept for any specific theory. The tool can -be thought of as a collection of hacks that have been tested in practice, -found to be surprisingly effective, and have been implemented in the simplest, -most robust way I could think of at the time. - -Many of the resulting features are made possible thanks to the availability of -lightweight instrumentation that served as a foundation for the tool, but this -mechanism should be thought of merely as a means to an end. The only true -governing principles are speed, reliability, and ease of use. - -## 1. Coverage measurements - -The instrumentation injected into compiled programs captures branch (edge) -coverage, along with coarse branch-taken hit counts. The code injected at -branch points is essentially equivalent to: - -```c - cur_location = <COMPILE_TIME_RANDOM>; - shared_mem[cur_location ^ prev_location]++; - prev_location = cur_location >> 1; -``` - -The `cur_location` value is generated randomly to simplify the process of -linking complex projects and keep the XOR output distributed uniformly. - -The `shared_mem[]` array is a 64 kB SHM region passed to the instrumented binary -by the caller. Every byte set in the output map can be thought of as a hit for -a particular (`branch_src`, `branch_dst`) tuple in the instrumented code. - -The size of the map is chosen so that collisions are sporadic with almost all -of the intended targets, which usually sport between 2k and 10k discoverable -branch points: - -``` - Branch cnt | Colliding tuples | Example targets - ------------+------------------+----------------- - 1,000 | 0.75% | giflib, lzo - 2,000 | 1.5% | zlib, tar, xz - 5,000 | 3.5% | libpng, libwebp - 10,000 | 7% | libxml - 20,000 | 14% | sqlite - 50,000 | 30% | - -``` - -At the same time, its size is small enough to allow the map to be analyzed -in a matter of microseconds on the receiving end, and to effortlessly fit -within L2 cache. - -This form of coverage provides considerably more insight into the execution -path of the program than simple block coverage. In particular, it trivially -distinguishes between the following execution traces: - -``` - A -> B -> C -> D -> E (tuples: AB, BC, CD, DE) - A -> B -> D -> C -> E (tuples: AB, BD, DC, CE) -``` - -This aids the discovery of subtle fault conditions in the underlying code, -because security vulnerabilities are more often associated with unexpected -or incorrect state transitions than with merely reaching a new basic block. - -The reason for the shift operation in the last line of the pseudocode shown -earlier in this section is to preserve the directionality of tuples (without -this, A ^ B would be indistinguishable from B ^ A) and to retain the identity -of tight loops (otherwise, A ^ A would be obviously equal to B ^ B). - -The absence of simple saturating arithmetic opcodes on Intel CPUs means that -the hit counters can sometimes wrap around to zero. Since this is a fairly -unlikely and localized event, it's seen as an acceptable performance trade-off. - -### 2. Detecting new behaviors - -The fuzzer maintains a global map of tuples seen in previous executions; this -data can be rapidly compared with individual traces and updated in just a couple -of dword- or qword-wide instructions and a simple loop. - -When a mutated input produces an execution trace containing new tuples, the -corresponding input file is preserved and routed for additional processing -later on (see section #3). Inputs that do not trigger new local-scale state -transitions in the execution trace (i.e., produce no new tuples) are discarded, -even if their overall control flow sequence is unique. - -This approach allows for a very fine-grained and long-term exploration of -program state while not having to perform any computationally intensive and -fragile global comparisons of complex execution traces, and while avoiding the -scourge of path explosion. - -To illustrate the properties of the algorithm, consider that the second trace -shown below would be considered substantially new because of the presence of -new tuples (CA, AE): - -``` - #1: A -> B -> C -> D -> E - #2: A -> B -> C -> A -> E -``` - -At the same time, with #2 processed, the following pattern will not be seen -as unique, despite having a markedly different overall execution path: - -``` - #3: A -> B -> C -> A -> B -> C -> A -> B -> C -> D -> E -``` - -In addition to detecting new tuples, the fuzzer also considers coarse tuple -hit counts. These are divided into several buckets: - -``` - 1, 2, 3, 4-7, 8-15, 16-31, 32-127, 128+ -``` - -To some extent, the number of buckets is an implementation artifact: it allows -an in-place mapping of an 8-bit counter generated by the instrumentation to -an 8-position bitmap relied on by the fuzzer executable to keep track of the -already-seen execution counts for each tuple. - -Changes within the range of a single bucket are ignored; transition from one -bucket to another is flagged as an interesting change in program control flow, -and is routed to the evolutionary process outlined in the section below. - -The hit count behavior provides a way to distinguish between potentially -interesting control flow changes, such as a block of code being executed -twice when it was normally hit only once. At the same time, it is fairly -insensitive to empirically less notable changes, such as a loop going from -47 cycles to 48. The counters also provide some degree of "accidental" -immunity against tuple collisions in dense trace maps. - -The execution is policed fairly heavily through memory and execution time -limits; by default, the timeout is set at 5x the initially-calibrated -execution speed, rounded up to 20 ms. The aggressive timeouts are meant to -prevent dramatic fuzzer performance degradation by descending into tarpits -that, say, improve coverage by 1% while being 100x slower; we pragmatically -reject them and hope that the fuzzer will find a less expensive way to reach -the same code. Empirical testing strongly suggests that more generous time -limits are not worth the cost. - -## 3. Evolving the input queue - -Mutated test cases that produced new state transitions within the program are -added to the input queue and used as a starting point for future rounds of -fuzzing. They supplement, but do not automatically replace, existing finds. - -In contrast to more greedy genetic algorithms, this approach allows the tool -to progressively explore various disjoint and possibly mutually incompatible -features of the underlying data format, as shown in this image: - -  - -Several practical examples of the results of this algorithm are discussed -here: - - http://lcamtuf.blogspot.com/2014/11/pulling-jpegs-out-of-thin-air.html - http://lcamtuf.blogspot.com/2014/11/afl-fuzz-nobody-expects-cdata-sections.html - -The synthetic corpus produced by this process is essentially a compact -collection of "hmm, this does something new!" input files, and can be used to -seed any other testing processes down the line (for example, to manually -stress-test resource-intensive desktop apps). - -With this approach, the queue for most targets grows to somewhere between 1k -and 10k entries; approximately 10-30% of this is attributable to the discovery -of new tuples, and the remainder is associated with changes in hit counts. - -The following table compares the relative ability to discover file syntax and -explore program states when using several different approaches to guided -fuzzing. The instrumented target was GNU patch 2.7k.3 compiled with `-O3` and -seeded with a dummy text file; the session consisted of a single pass over the -input queue with afl-fuzz: - -``` - Fuzzer guidance | Blocks | Edges | Edge hit | Highest-coverage - strategy used | reached | reached | cnt var | test case generated - ------------------+---------+---------+----------+--------------------------- - (Initial file) | 156 | 163 | 1.00 | (none) - | | | | - Blind fuzzing S | 182 | 205 | 2.23 | First 2 B of RCS diff - Blind fuzzing L | 228 | 265 | 2.23 | First 4 B of -c mode diff - Block coverage | 855 | 1,130 | 1.57 | Almost-valid RCS diff - Edge coverage | 1,452 | 2,070 | 2.18 | One-chunk -c mode diff - AFL model | 1,765 | 2,597 | 4.99 | Four-chunk -c mode diff -``` - -The first entry for blind fuzzing ("S") corresponds to executing just a single -round of testing; the second set of figures ("L") shows the fuzzer running in a -loop for a number of execution cycles comparable with that of the instrumented -runs, which required more time to fully process the growing queue. - -Roughly similar results have been obtained in a separate experiment where the -fuzzer was modified to compile out all the random fuzzing stages and leave just -a series of rudimentary, sequential operations such as walking bit flips. -Because this mode would be incapable of altering the size of the input file, -the sessions were seeded with a valid unified diff: - -``` - Queue extension | Blocks | Edges | Edge hit | Number of unique - strategy used | reached | reached | cnt var | crashes found - ------------------+---------+---------+----------+------------------ - (Initial file) | 624 | 717 | 1.00 | - - | | | | - Blind fuzzing | 1,101 | 1,409 | 1.60 | 0 - Block coverage | 1,255 | 1,649 | 1.48 | 0 - Edge coverage | 1,259 | 1,734 | 1.72 | 0 - AFL model | 1,452 | 2,040 | 3.16 | 1 -``` - -At noted earlier on, some of the prior work on genetic fuzzing relied on -maintaining a single test case and evolving it to maximize coverage. At least -in the tests described above, this "greedy" approach appears to confer no -substantial benefits over blind fuzzing strategies. - -### 4. Culling the corpus - -The progressive state exploration approach outlined above means that some of -the test cases synthesized later on in the game may have edge coverage that -is a strict superset of the coverage provided by their ancestors. - -To optimize the fuzzing effort, AFL periodically re-evaluates the queue using a -fast algorithm that selects a smaller subset of test cases that still cover -every tuple seen so far, and whose characteristics make them particularly -favorable to the tool. - -The algorithm works by assigning every queue entry a score proportional to its -execution latency and file size; and then selecting lowest-scoring candidates -for each tuple. - -The tuples are then processed sequentially using a simple workflow: - - 1) Find next tuple not yet in the temporary working set, - 2) Locate the winning queue entry for this tuple, - 3) Register *all* tuples present in that entry's trace in the working set, - 4) Go to #1 if there are any missing tuples in the set. - -The generated corpus of "favored" entries is usually 5-10x smaller than the -starting data set. Non-favored entries are not discarded, but they are skipped -with varying probabilities when encountered in the queue: - - - If there are new, yet-to-be-fuzzed favorites present in the queue, 99% - of non-favored entries will be skipped to get to the favored ones. - - If there are no new favorites: - * If the current non-favored entry was fuzzed before, it will be skipped - 95% of the time. - * If it hasn't gone through any fuzzing rounds yet, the odds of skipping - drop down to 75%. - -Based on empirical testing, this provides a reasonable balance between queue -cycling speed and test case diversity. - -Slightly more sophisticated but much slower culling can be performed on input -or output corpora with `afl-cmin`. This tool permanently discards the redundant -entries and produces a smaller corpus suitable for use with `afl-fuzz` or -external tools. - -## 5. Trimming input files - -File size has a dramatic impact on fuzzing performance, both because large -files make the target binary slower, and because they reduce the likelihood -that a mutation would touch important format control structures, rather than -redundant data blocks. This is discussed in more detail in perf_tips.md. - -The possibility that the user will provide a low-quality starting corpus aside, -some types of mutations can have the effect of iteratively increasing the size -of the generated files, so it is important to counter this trend. - -Luckily, the instrumentation feedback provides a simple way to automatically -trim down input files while ensuring that the changes made to the files have no -impact on the execution path. - -The built-in trimmer in afl-fuzz attempts to sequentially remove blocks of data -with variable length and stepover; any deletion that doesn't affect the checksum -of the trace map is committed to disk. The trimmer is not designed to be -particularly thorough; instead, it tries to strike a balance between precision -and the number of `execve()` calls spent on the process, selecting the block size -and stepover to match. The average per-file gains are around 5-20%. - -The standalone `afl-tmin` tool uses a more exhaustive, iterative algorithm, and -also attempts to perform alphabet normalization on the trimmed files. The -operation of `afl-tmin` is as follows. - -First, the tool automatically selects the operating mode. If the initial input -crashes the target binary, afl-tmin will run in non-instrumented mode, simply -keeping any tweaks that produce a simpler file but still crash the target. -The same mode is used for hangs, if `-H` (hang mode) is specified. -If the target is non-crashing, the tool uses an instrumented mode and keeps only -the tweaks that produce exactly the same execution path. - -The actual minimization algorithm is: - - 1) Attempt to zero large blocks of data with large stepovers. Empirically, - this is shown to reduce the number of execs by preempting finer-grained - efforts later on. - 2) Perform a block deletion pass with decreasing block sizes and stepovers, - binary-search-style. - 3) Perform alphabet normalization by counting unique characters and trying - to bulk-replace each with a zero value. - 4) As a last result, perform byte-by-byte normalization on non-zero bytes. - -Instead of zeroing with a 0x00 byte, `afl-tmin` uses the ASCII digit '0'. This -is done because such a modification is much less likely to interfere with -text parsing, so it is more likely to result in successful minimization of -text files. - -The algorithm used here is less involved than some other test case -minimization approaches proposed in academic work, but requires far fewer -executions and tends to produce comparable results in most real-world -applications. - -## 6. Fuzzing strategies - -The feedback provided by the instrumentation makes it easy to understand the -value of various fuzzing strategies and optimize their parameters so that they -work equally well across a wide range of file types. The strategies used by -afl-fuzz are generally format-agnostic and are discussed in more detail here: - - http://lcamtuf.blogspot.com/2014/08/binary-fuzzing-strategies-what-works.html - -It is somewhat notable that especially early on, most of the work done by -`afl-fuzz` is actually highly deterministic, and progresses to random stacked -modifications and test case splicing only at a later stage. The deterministic -strategies include: - - - Sequential bit flips with varying lengths and stepovers, - - Sequential addition and subtraction of small integers, - - Sequential insertion of known interesting integers (`0`, `1`, `INT_MAX`, etc), - -The purpose of opening with deterministic steps is related to their tendency to -produce compact test cases and small diffs between the non-crashing and crashing -inputs. - -With deterministic fuzzing out of the way, the non-deterministic steps include -stacked bit flips, insertions, deletions, arithmetics, and splicing of different -test cases. - -The relative yields and `execve()` costs of all these strategies have been -investigated and are discussed in the aforementioned blog post. - -For the reasons discussed in historical_notes.md (chiefly, performance, -simplicity, and reliability), AFL generally does not try to reason about the -relationship between specific mutations and program states; the fuzzing steps -are nominally blind, and are guided only by the evolutionary design of the -input queue. - -That said, there is one (trivial) exception to this rule: when a new queue -entry goes through the initial set of deterministic fuzzing steps, and tweaks to -some regions in the file are observed to have no effect on the checksum of the -execution path, they may be excluded from the remaining phases of -deterministic fuzzing - and the fuzzer may proceed straight to random tweaks. -Especially for verbose, human-readable data formats, this can reduce the number -of execs by 10-40% or so without an appreciable drop in coverage. In extreme -cases, such as normally block-aligned tar archives, the gains can be as high as -90%. - -Because the underlying "effector maps" are local every queue entry and remain -in force only during deterministic stages that do not alter the size or the -general layout of the underlying file, this mechanism appears to work very -reliably and proved to be simple to implement. - -## 7. Dictionaries - -The feedback provided by the instrumentation makes it easy to automatically -identify syntax tokens in some types of input files, and to detect that certain -combinations of predefined or auto-detected dictionary terms constitute a -valid grammar for the tested parser. - -A discussion of how these features are implemented within afl-fuzz can be found -here: - - http://lcamtuf.blogspot.com/2015/01/afl-fuzz-making-up-grammar-with.html - -In essence, when basic, typically easily-obtained syntax tokens are combined -together in a purely random manner, the instrumentation and the evolutionary -design of the queue together provide a feedback mechanism to differentiate -between meaningless mutations and ones that trigger new behaviors in the -instrumented code - and to incrementally build more complex syntax on top of -this discovery. - -The dictionaries have been shown to enable the fuzzer to rapidly reconstruct -the grammar of highly verbose and complex languages such as JavaScript, SQL, -or XML; several examples of generated SQL statements are given in the blog -post mentioned above. - -Interestingly, the AFL instrumentation also allows the fuzzer to automatically -isolate syntax tokens already present in an input file. It can do so by looking -for run of bytes that, when flipped, produce a consistent change to the -program's execution path; this is suggestive of an underlying atomic comparison -to a predefined value baked into the code. The fuzzer relies on this signal -to build compact "auto dictionaries" that are then used in conjunction with -other fuzzing strategies. - -## 8. De-duping crashes - -De-duplication of crashes is one of the more important problems for any -competent fuzzing tool. Many of the naive approaches run into problems; in -particular, looking just at the faulting address may lead to completely -unrelated issues being clustered together if the fault happens in a common -library function (say, `strcmp`, `strcpy`); while checksumming call stack -backtraces can lead to extreme crash count inflation if the fault can be -reached through a number of different, possibly recursive code paths. - -The solution implemented in `afl-fuzz` considers a crash unique if any of two -conditions are met: - - - The crash trace includes a tuple not seen in any of the previous crashes, - - The crash trace is missing a tuple that was always present in earlier - faults. - -The approach is vulnerable to some path count inflation early on, but exhibits -a very strong self-limiting effect, similar to the execution path analysis -logic that is the cornerstone of `afl-fuzz`. - -## 9. Investigating crashes - -The exploitability of many types of crashes can be ambiguous; afl-fuzz tries -to address this by providing a crash exploration mode where a known-faulting -test case is fuzzed in a manner very similar to the normal operation of the -fuzzer, but with a constraint that causes any non-crashing mutations to be -thrown away. - -A detailed discussion of the value of this approach can be found here: - - http://lcamtuf.blogspot.com/2014/11/afl-fuzz-crash-exploration-mode.html - -The method uses instrumentation feedback to explore the state of the crashing -program to get past the ambiguous faulting condition and then isolate the -newly-found inputs for human review. - -On the subject of crashes, it is worth noting that in contrast to normal -queue entries, crashing inputs are *not* trimmed; they are kept exactly as -discovered to make it easier to compare them to the parent, non-crashing entry -in the queue. That said, `afl-tmin` can be used to shrink them at will. - -## 10 The fork server - -To improve performance, `afl-fuzz` uses a "fork server", where the fuzzed process -goes through `execve()`, linking, and libc initialization only once, and is then -cloned from a stopped process image by leveraging copy-on-write. The -implementation is described in more detail here: - - http://lcamtuf.blogspot.com/2014/10/fuzzing-binaries-without-execve.html - -The fork server is an integral aspect of the injected instrumentation and -simply stops at the first instrumented function to await commands from -`afl-fuzz`. - -With fast targets, the fork server can offer considerable performance gains, -usually between 1.5x and 2x. It is also possible to: - - - Use the fork server in manual ("deferred") mode, skipping over larger, - user-selected chunks of initialization code. It requires very modest - code changes to the targeted program, and With some targets, can - produce 10x+ performance gains. - - Enable "persistent" mode, where a single process is used to try out - multiple inputs, greatly limiting the overhead of repetitive `fork()` - calls. This generally requires some code changes to the targeted program, - but can improve the performance of fast targets by a factor of 5 or more - approximating the benefits of in-process fuzzing jobs while still - maintaining very robust isolation between the fuzzer process and the - targeted binary. - -## 11. Parallelization - -The parallelization mechanism relies on periodically examining the queues -produced by independently-running instances on other CPU cores or on remote -machines, and then selectively pulling in the test cases that, when tried -out locally, produce behaviors not yet seen by the fuzzer at hand. - -This allows for extreme flexibility in fuzzer setup, including running synced -instances against different parsers of a common data format, often with -synergistic effects. - -For more information about this design, see parallel_fuzzing.md. - -## 12. Binary-only instrumentation - -Instrumentation of black-box, binary-only targets is accomplished with the -help of a separately-built version of QEMU in "user emulation" mode. This also -allows the execution of cross-architecture code - say, ARM binaries on x86. - -QEMU uses basic blocks as translation units; the instrumentation is implemented -on top of this and uses a model roughly analogous to the compile-time hooks: - -```c - if (block_address > elf_text_start && block_address < elf_text_end) { - - cur_location = (block_address >> 4) ^ (block_address << 8); - shared_mem[cur_location ^ prev_location]++; - prev_location = cur_location >> 1; - - } -``` - -The shift-and-XOR-based scrambling in the second line is used to mask the -effects of instruction alignment. - -The start-up of binary translators such as QEMU, DynamoRIO, and PIN is fairly -slow; to counter this, the QEMU mode leverages a fork server similar to that -used for compiler-instrumented code, effectively spawning copies of an -already-initialized process paused at `_start`. - -First-time translation of a new basic block also incurs substantial latency. To -eliminate this problem, the AFL fork server is extended by providing a channel -between the running emulator and the parent process. The channel is used -to notify the parent about the addresses of any newly-encountered blocks and to -add them to the translation cache that will be replicated for future child -processes. - -As a result of these two optimizations, the overhead of the QEMU mode is -roughly 2-5x, compared to 100x+ for PIN. - -## 13. The `afl-analyze` tool - -The file format analyzer is a simple extension of the minimization algorithm -discussed earlier on; instead of attempting to remove no-op blocks, the tool -performs a series of walking byte flips and then annotates runs of bytes -in the input file. - -It uses the following classification scheme: - - - "No-op blocks" - segments where bit flips cause no apparent changes to - control flow. Common examples may be comment sections, pixel data within - a bitmap file, etc. - - "Superficial content" - segments where some, but not all, bitflips - produce some control flow changes. Examples may include strings in rich - documents (e.g., XML, RTF). - - "Critical stream" - a sequence of bytes where all bit flips alter control - flow in different but correlated ways. This may be compressed data, - non-atomically compared keywords or magic values, etc. - - "Suspected length field" - small, atomic integer that, when touched in - any way, causes a consistent change to program control flow, suggestive - of a failed length check. - - "Suspected cksum or magic int" - an integer that behaves similarly to a - length field, but has a numerical value that makes the length explanation - unlikely. This is suggestive of a checksum or other "magic" integer. - - "Suspected checksummed block" - a long block of data where any change - always triggers the same new execution path. Likely caused by failing - a checksum or a similar integrity check before any subsequent parsing - takes place. - - "Magic value section" - a generic token where changes cause the type - of binary behavior outlined earlier, but that doesn't meet any of the - other criteria. May be an atomically compared keyword or so. diff --git a/docs/third_party_tools.md b/docs/third_party_tools.md new file mode 100644 index 00000000..92229e84 --- /dev/null +++ b/docs/third_party_tools.md @@ -0,0 +1,57 @@ +# Tools that help fuzzing with AFL++ + +Speeding up fuzzing: +* [libfiowrapper](https://github.com/marekzmyslowski/libfiowrapper) - if the + function you want to fuzz requires loading a file, this allows using the + shared memory test case feature :-) - recommended. + +Minimization of test cases: +* [afl-pytmin](https://github.com/ilsani/afl-pytmin) - a wrapper for afl-tmin + that tries to speed up the process of minimization of a single test case by + using many CPU cores. +* [afl-ddmin-mod](https://github.com/MarkusTeufelberger/afl-ddmin-mod) - a + variation of afl-tmin based on the ddmin algorithm. +* [halfempty](https://github.com/googleprojectzero/halfempty) - is a fast + utility for minimizing test cases by Tavis Ormandy based on parallelization. + +Distributed execution: +* [disfuzz-afl](https://github.com/MartijnB/disfuzz-afl) - distributed fuzzing + for AFL. +* [AFLDFF](https://github.com/quantumvm/AFLDFF) - AFL distributed fuzzing + framework. +* [afl-launch](https://github.com/bnagy/afl-launch) - a tool for the execution + of many AFL instances. +* [afl-mothership](https://github.com/afl-mothership/afl-mothership) - + management and execution of many synchronized AFL fuzzers on AWS cloud. +* [afl-in-the-cloud](https://github.com/abhisek/afl-in-the-cloud) - another + script for running AFL in AWS. + +Deployment, management, monitoring, reporting +* [afl-utils](https://gitlab.com/rc0r/afl-utils) - a set of utilities for + automatic processing/analysis of crashes and reducing the number of test + cases. +* [afl-other-arch](https://github.com/shellphish/afl-other-arch) - is a set of + patches and scripts for easily adding support for various non-x86 + architectures for AFL. +* [afl-trivia](https://github.com/bnagy/afl-trivia) - a few small scripts to + simplify the management of AFL. +* [afl-monitor](https://github.com/reflare/afl-monitor) - a script for + monitoring AFL. +* [afl-manager](https://github.com/zx1340/afl-manager) - a web server on Python + for managing multi-afl. +* [afl-remote](https://github.com/block8437/afl-remote) - a web server for the + remote management of AFL instances. +* [afl-extras](https://github.com/fekir/afl-extras) - shell scripts to + parallelize afl-tmin, startup, and data collection. + +Crash processing +* [afl-crash-analyzer](https://github.com/floyd-fuh/afl-crash-analyzer) - + another crash analyzer for AFL. +* [fuzzer-utils](https://github.com/ThePatrickStar/fuzzer-utils) - a set of + scripts for the analysis of results. +* [atriage](https://github.com/Ayrx/atriage) - a simple triage tool. +* [afl-kit](https://github.com/kcwu/afl-kit) - afl-cmin on Python. +* [AFLize](https://github.com/d33tah/aflize) - a tool that automatically + generates builds of debian packages suitable for AFL. +* [afl-fid](https://github.com/FoRTE-Research/afl-fid) - a set of tools for + working with input data. \ No newline at end of file diff --git a/docs/tutorials.md b/docs/tutorials.md new file mode 100644 index 00000000..ed8a7eec --- /dev/null +++ b/docs/tutorials.md @@ -0,0 +1,30 @@ +# Tutorials + +Here are some good write-ups to show how to effectively use AFL++: + +* [https://aflplus.plus/docs/tutorials/libxml2_tutorial/](https://aflplus.plus/docs/tutorials/libxml2_tutorial/) +* [https://bananamafia.dev/post/gb-fuzz/](https://bananamafia.dev/post/gb-fuzz/) +* [https://securitylab.github.com/research/fuzzing-challenges-solutions-1](https://securitylab.github.com/research/fuzzing-challenges-solutions-1) +* [https://securitylab.github.com/research/fuzzing-software-2](https://securitylab.github.com/research/fuzzing-software-2) +* [https://securitylab.github.com/research/fuzzing-sockets-FTP](https://securitylab.github.com/research/fuzzing-sockets-FTP) +* [https://securitylab.github.com/research/fuzzing-sockets-FreeRDP](https://securitylab.github.com/research/fuzzing-sockets-FreeRDP) +* [https://securitylab.github.com/research/fuzzing-apache-1](https://securitylab.github.com/research/fuzzing-apache-1) + +If you do not want to follow a tutorial but rather try an exercise type of +training, then we can highly recommend the following: + +* [https://github.com/antonio-morales/Fuzzing101](https://github.com/antonio-morales/Fuzzing101) + +If you are interested in fuzzing structured data (where you define what the +structure is), these links have you covered: + +* Superion for AFL++: + [https://github.com/adrian-rt/superion-mutator](https://github.com/adrian-rt/superion-mutator) +* libprotobuf for AFL++: + [https://github.com/P1umer/AFLplusplus-protobuf-mutator](https://github.com/P1umer/AFLplusplus-protobuf-mutator) +* libprotobuf raw: + [https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator](https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator) +* libprotobuf for old AFL++ API: + [https://github.com/thebabush/afl-libprotobuf-mutator](https://github.com/thebabush/afl-libprotobuf-mutator) + +If you find other good ones, please send them to us :-) \ No newline at end of file diff --git a/frida_mode/DEBUGGING.md b/frida_mode/DEBUGGING.md index 69663510..207a48bf 100644 --- a/frida_mode/DEBUGGING.md +++ b/frida_mode/DEBUGGING.md @@ -1,6 +1,8 @@ +# Debugging + If you are using FRIDA mode and have hit some problems, then this guide may help you to diagnose any problems you are encountering. This assumes you have -followed the [osx-lib](#test/osx-lib) example to start fuzzing your target. +followed the [osx-lib](test/osx-lib) example to start fuzzing your target. It should be noted that attempting to debug code using gdb which has been instrumented in FRIDA is unlikely to be successful since the debugger will be @@ -10,69 +12,76 @@ you are very familiar with the implementation of Stalker, the instrumented code generated by FRIDA is likely to be very difficult to follow. For this reason, the following debugging strategies are outlined below. -By convention below all files should be provided with their path (they are +By convention, all files below should be provided with their path (they are omitted for readability) and all items in `<braces>` are placeholders and should be replaced accordingly. -# Select your version +## Select your version + Test with both the `dev` and `stable` branches of AFL++. The `dev` branch should have the very latest version containing any fixes for identified issues. The `stable` branch is updated less frequently, but equally might avoid a problem if a regression has been introduced into the `dev` branch. -# Enable Diagnostic Information -- Run your target specifying the `AFL_DEBUG_CHILD=1` environment variable. This - will print a lot more diagnostic information to the screen when the target - starts up. If you have a simple configuration issue then you will likely see a - warning or error message in the output. +## Enable diagnostic information + +Run your target specifying the `AFL_DEBUG_CHILD=1` environment variable. This +will print a lot more diagnostic information to the screen when the target +starts up. If you have a simple configuration issue, then you will likely see a +warning or error message in the output. + +## Check your test harness -# Check your Test Harness If any of the following steps fail, then there is a problem with your test -harness, or your target library. Since this is running without FRIDA mode or +harness or your target library. Since this is running without FRIDA mode or `afl-fuzz` that greatly reduces the search area for your defect. This is why it is *VERY* important to carry out these basic steps first before taking on the additional complexity of debugging with FRIDA mode or `afl-fuzz`. - Run your harness outside of the fuzzer, passing it a representative seed as it's input `./harness <input>`. -- Pass you harness multiple seeds to check that it is stable when running +- Pass your harness multiple seeds to check that it is stable when running multiple tests as it will when running in fork server mode `./harness <input1> <intput2>`. - Build your test harness with `CFLAGS=-fsanitize=address` and `LDFLAGS=-fsanitize=address`. Then run it again with multiple inputs to check - for errors (note that when fuzzing your harness should not be built with any + for errors (note that when fuzzing, your harness should not be built with any sanitizer options). -# Check the Samples +## Check the samples + FRIDA mode contains a number of different sample targets in the `test` folder. -Have a look throught these and find one which is similar to your real target. +Have a look through these and find one which is similar to your real target. Check whether you have any issues running the sample target and make sure you -compare the command line used to launch the sample with that you are using to -launch your real target very carefully to check for any differences. If possible -start with one of these samples and gradually make changes one at a time -re-testing as you go until you have migrated it to run your own target. +compare the command line used to launch the sample with the one you are using to +launch your real target very carefully to check for any differences. If +possible, start with one of these samples and gradually make changes one at a +time re-testing as you go until you have migrated it to run your own target. -# FRIDA Mode -## Basic -First just try running your target with `LD_PRELOAD=afl-frida-trace.so ./harness - <input>`. An error here means that your defect occurs when running with just - FRIDA mode and isn't related to `afl-fuzz`. +## FRIDA mode + +### Basic + +First, just try running your target with `LD_PRELOAD=afl-frida-trace.so + ./harness <input>`. An error here means that your defect occurs when running + with just FRIDA mode and isn't related to `afl-fuzz`. Now you can try commenting out the implementation of `LLVMFuzzerTestOneInput` so that the harness doesn't actually run your target library. This may also aid in narrowing down the problem. + ```c int LLVMFuzzerTestOneInput(const unsigned char* data, size_t size){ // fpn_crashme(data, size); return 0; } - ``` -## Persistent Mode +### Persistent mode + If your target is ok running in basic mode, you can try running it in persistent mode (if that is the configuration you are having issues with) as follows (again -outside of afl-fuzz). This time you will want to run it inside a debugger so +outside of `afl-fuzz`). This time, you will want to run it inside a debugger so that you can use the debugger to send the `SIGCONT` signals (by continuing) usually sent by `afl-fuzz` on each iteration. @@ -84,13 +93,15 @@ gdb \ --ex 'set environment AFL_FRIDA_PERSISTENT_ADDR=<entry_address>' \ --args ./harness <input> ``` -Note we have to manually set the `__AFL_PERSISTENT` environment variable which -is usually passed by `afl-fuzz`. -Note that setting breakpoints etc is likely to interfere with FRIDA and cause -spurious errors. +Note: +- You have to manually set the `__AFL_PERSISTENT` environment variable which is + usually passed by `afl-fuzz`. +- Setting breakpoints etc. is likely to interfere with FRIDA and cause spurious + errors. If this is successful, you can try additionally loading the hook library: + ```bash gdb \ --ex 'set environment __AFL_PERSISTENT=1' \ @@ -100,6 +111,7 @@ gdb \ --ex 'set environment AFL_FRIDA_PERSISTENT_HOOK=frida_hook.so' --args ./harness <input> ``` + Note that the format of the hook used for FRIDA mode is subtly different to that used when running in QEMU mode as shown below. Thus the DSO used for the hook is not interchangeable. @@ -112,12 +124,14 @@ void afl_persistent_hook(struct x86_64_regs *regs, uint64_t guest_base, uint8_t *input_buf, uint32_t input_buf_len); ``` -## ASAN +### ASAN + It is also possible to enable ASAN (if that is the configuration you are having -issues with) without having to use `afl-fuzz`. This can be done as follows (note -that the name of the asan DSO may need to be changed depending on your -platform). Note that the asan DSO must appear first in the `LD_PRELOAD` -environment variable: +issues with) without having to use `afl-fuzz`. This can be done as follows: + +Note: +- The name of the asan DSO may need to be changed depending on your platform. +- The asan DSO must appear first in the `LD_PRELOAD` environment variable. ```bash LD_PRELOAD=libclang_rt.asan-x86_64.so:afl-frida-trace.so \ @@ -132,29 +146,34 @@ DSO from coverage. Failure to do so will result in ASAN attempting to sanitize itself and as a result detecting failures when it attempts to update the shadow maps. -# Printf +## Printf + If you have an idea of where things are going wrong for you, then don't be scared to add `printf` statements to either AFL++ or FRIDA mode itself to show more diagnostic information. Just be sure to set `AFL_DEBUG=1` and `AFL_DEBUG_CHILD=1` when you are testing it. -# Core Dumps -Lastly, if your defect only occurs when using `afl-fuzz` (e.g. when using -`CMPLOG` which cannot be tested outside of `afl-fuzz` due to it's need for a +## Core dumps + +Lastly, if your defect only occurs when using `afl-fuzz` (e.g., when using +`CMPLOG` which cannot be tested outside of `afl-fuzz` due to its need for a shared memory mapping being created for it to record its data), it is possible to enable the creation of a core dump for post-mortem analysis. -Firstly check your `/proc/sys/kernel/core_pattern` configuration is simply set -to a filename (AFL++ encourages you to set it to the value 'core' in any case -since it doesn't want any handler applications getting in the way). Next set -`ulimit -c unlimited` to remove any size limitations for core files. Lastly, -when you `afl-fuzz` set the environment variable `AFL_DEBUG=1` to enable the -creation of the `core` file. The file should be created in the working directory -of the target application. If there is an existing `core` file aleady there, -then it may not be overwritten. +Firstly, check if your `/proc/sys/kernel/core_pattern` configuration is set to a +filename (AFL++ encourages you to set it to the value `core` in any case since +it doesn't want any handler applications getting in the way). + +Next, set `ulimit -c unlimited` to remove any size limitations for core files. + +Lastly, when you `afl-fuzz`, set the environment variable `AFL_DEBUG=1` to +enable the creation of the `core` file. The file should be created in the +working directory of the target application. If there is an existing `core` file +already there, then it may not be overwritten. + +## Reach out -# Reach out Get in touch on discord and ask for help. The groups are pretty active so someone may well be able to offer some advice. Better still, if you are able to -create a minimal reproducer for your problem it will make it easier to diagnose -the issue. +create a minimal reproducer for your problem, it will make it easier to diagnose +the issue. \ No newline at end of file diff --git a/frida_mode/GNUmakefile b/frida_mode/GNUmakefile index 3e35e2f6..5303fd1a 100644 --- a/frida_mode/GNUmakefile +++ b/frida_mode/GNUmakefile @@ -30,8 +30,7 @@ AFL_CFLAGS:=-Wno-unused-parameter \ LDFLAGS+=-shared \ -lpthread \ - -lresolv \ - -ldl + -lresolv ifdef DEBUG CFLAGS+=-Werror \ @@ -46,6 +45,11 @@ FRIDA_BUILD_DIR:=$(BUILD_DIR)frida/ FRIDA_TRACE:=$(BUILD_DIR)afl-frida-trace.so FRIDA_TRACE_EMBEDDED:=$(BUILD_DIR)afl-frida-trace-embedded +TARGET_CC?=$(CC) +TARGET_CXX?=$(CXX) +HOST_CC?=$(CC) +HOST_CXX?=$(CXX) + ifndef ARCH ARCH=$(shell uname -m) @@ -71,19 +75,45 @@ ifdef DEBUG endif LDFLAGS+= -z noexecstack \ -Wl,--gc-sections \ - -Wl,--exclude-libs,ALL + -Wl,--exclude-libs,ALL \ + -ldl \ + -lrt LDSCRIPT:=-Wl,--version-script=$(PWD)frida.map endif ifeq "$(shell uname)" "Linux" OS:=linux + ifneq "$(findstring musl, $(shell ldd --version 2>&1 | head -n 1))" "" + CFLAGS+= -D__MUSL__ + endif +endif + +ifneq "$(findstring android, $(shell $(CC) --version 2>/dev/null))" "" + OS:=android + ifneq "$(findstring aarch64, $(shell $(CC) --version 2>/dev/null))" "" + ARCH:=arm64 + endif + ifneq "$(findstring arm, $(shell $(CC) --version 2>/dev/null))" "" + ARCH:=arm + endif + ifneq "$(findstring x86_64, $(shell $(CC) --version 2>/dev/null))" "" + ARCH:=x86_64 + endif + ifneq "$(findstring i686, $(shell $(CC) --version 2>/dev/null))" "" + ARCH:=x86 + endif +endif + +ifeq "$(ARCH)" "armhf" + TARGET_CC:=arm-linux-gnueabihf-gcc + TARGET_CXX:=arm-linux-gnueabihf-g++ endif ifndef OS $(error "Operating system unsupported") endif -GUM_DEVKIT_VERSION=15.0.16 +GUM_DEVKIT_VERSION=15.1.13 GUM_DEVKIT_FILENAME=frida-gumjs-devkit-$(GUM_DEVKIT_VERSION)-$(OS)-$(ARCH).tar.xz GUM_DEVKIT_URL="https://github.com/frida/frida/releases/download/$(GUM_DEVKIT_VERSION)/$(GUM_DEVKIT_FILENAME)" @@ -168,7 +198,7 @@ $(GUM_DEVIT_HEADER): $(GUM_DEVKIT_TARBALL) ############################## AFL ############################################# $(AFL_COMPILER_RT_OBJ): $(AFL_COMPILER_RT_SRC) - $(CC) \ + $(TARGET_CC) \ $(CFLAGS) \ $(AFL_CFLAGS) \ -I $(ROOT) \ @@ -177,7 +207,7 @@ $(AFL_COMPILER_RT_OBJ): $(AFL_COMPILER_RT_SRC) -c $< $(AFL_PERFORMANCE_OBJ): $(AFL_PERFORMANCE_SRC) - $(CC) \ + $(TARGET_CC) \ $(CFLAGS) \ $(AFL_CFLAGS) \ -I $(ROOT) \ @@ -188,13 +218,13 @@ $(AFL_PERFORMANCE_OBJ): $(AFL_PERFORMANCE_SRC) ############################### JS ############################################# $(BIN2C): $(BIN2C_SRC) - $(CC) -D_GNU_SOURCE -o $@ $< + $(HOST_CC) -D_GNU_SOURCE -o $@ $< $(JS_SRC): $(JS) $(BIN2C)| $(BUILD_DIR) cd $(JS_DIR) && $(BIN2C) api_js $(JS) $@ $(JS_OBJ): $(JS_SRC) GNUmakefile - $(CC) \ + $(TARGET_CC) \ $(CFLAGS) \ -I $(ROOT)include \ -I $(FRIDA_BUILD_DIR) \ @@ -206,7 +236,7 @@ $(JS_OBJ): $(JS_SRC) GNUmakefile define BUILD_SOURCE $(2): $(1) $(INCLUDES) GNUmakefile | $(OBJ_DIR) - $(CC) \ + $(TARGET_CC) \ $(CFLAGS) \ -I $(ROOT)include \ -I $(FRIDA_BUILD_DIR) \ @@ -220,7 +250,7 @@ $(foreach src,$(SOURCES),$(eval $(call BUILD_SOURCE,$(src),$(OBJ_DIR)$(notdir $( ######################## AFL-FRIDA-TRACE ####################################### $(FRIDA_TRACE): $(GUM_DEVIT_LIBRARY) $(GUM_DEVIT_HEADER) $(OBJS) $(JS_OBJ) $(AFL_COMPILER_RT_OBJ) $(AFL_PERFORMANCE_OBJ) GNUmakefile | $(BUILD_DIR) - $(CXX) \ + $(TARGET_CXX) \ $(OBJS) \ $(JS_OBJ) \ $(GUM_DEVIT_LIBRARY) \ @@ -235,10 +265,10 @@ $(FRIDA_TRACE): $(GUM_DEVIT_LIBRARY) $(GUM_DEVIT_HEADER) $(OBJS) $(JS_OBJ) $(AFL ############################# HOOK ############################################# $(AFLPP_FRIDA_DRIVER_HOOK_OBJ): $(AFLPP_FRIDA_DRIVER_HOOK_SRC) $(GUM_DEVIT_HEADER) | $(BUILD_DIR) - $(CC) $(CFLAGS) $(LDFLAGS) -I $(FRIDA_BUILD_DIR) $< -o $@ + $(TARGET_CC) $(CFLAGS) $(LDFLAGS) -I $(FRIDA_BUILD_DIR) $< -o $@ $(AFLPP_QEMU_DRIVER_HOOK_OBJ): $(AFLPP_QEMU_DRIVER_HOOK_SRC) | $(BUILD_DIR) - $(CC) $(CFLAGS) $(LDFLAGS) $< -o $@ + $(TARGET_CC) $(CFLAGS) $(LDFLAGS) $< -o $@ hook: $(AFLPP_FRIDA_DRIVER_HOOK_OBJ) $(AFLPP_QEMU_DRIVER_HOOK_OBJ) diff --git a/frida_mode/MapDensity.md b/frida_mode/MapDensity.md index f4ae3ace..50f2720f 100644 --- a/frida_mode/MapDensity.md +++ b/frida_mode/MapDensity.md @@ -1,8 +1,9 @@ -# Map Density +# Map density + +## How coverage works -# How Coverage Works The coverage in AFL++ works by assigning each basic block of code a unique ID -and during execution when transitioning between blocks (e.g. by calls or jumps) +and during execution when transitioning between blocks (e.g., by calls or jumps) assigning each of these edges an ID based upon the source and destination block ID. @@ -13,11 +14,12 @@ A single dimensional cumulative byte array is also constructed where each byte again represents an individual edge ID, but this time, the value of the byte represents a range of how many times that edge has been traversed. -```1, 2, 3, 4-7, 8-15, 16-31, 32-127, 128+``` +`1, 2, 3, 4-7, 8-15, 16-31, 32-127, 128+` The theory is that a new path isn't particularly interesting if an edge has been traversed `23` instead of `24` times for example, but is interesting if an edge -has been traversed for the very first time, or the number of times fits within a different bucket. +has been traversed for the very first time or the number of times fits within a +different bucket. After each run, the count of times each edge is hit is compared to the values in the cumulative map and if it is different, then the input is kept as a new seed @@ -27,19 +29,22 @@ This mechanism is described in greater detail in the seminal [paper](https://lcamtuf.coredump.cx/afl/technical_details.txt) on AFL by [lcamtuf](https://github.com/lcamtuf). -# Collisions +## Collisions + In black-box fuzzing, we must assume that control may flow from any block to any -other block, since we don't know any better. Thus for a target with `n` basic +other block, since we don't know any better. Thus, for a target with `n` basic blocks of code, there are `n * n` potential edges. As we can see, even with a small number of edges, a very large map will be required so that we have space to fit them all. Even if our target only had `1024` blocks, this would require a map containing `1048576` entries (or 1Mb in size). -Whilst this may not seem like a lot of memory, it causes problems for two reasons. Firstly, the processing step after each execution must now process much more -data, and secondly a map this size is unlikely to fit within the L2 cache of the processor. Since this is a very hot code path, we are likely to pay a very heavy -performance cost. +Whilst this may not seem like a lot of memory, it causes problems for two +reasons. Firstly, the processing step after each execution must now process much +more data, and secondly, a map this size is unlikely to fit within the L2 cache +of the processor. Since this is a very hot code path, we are likely to pay a +very heavy performance cost. -Therefore, we must accept that not all edges can have a unique and that +Therefore, we must accept that not all edges can have a unique ID and that therefore there will be collisions. This means that if the fuzzer finds a new path by uncovering an edge which was not previously found, but that the same edge ID is used by another edge, then it may go completely unnoticed. This is @@ -47,15 +52,15 @@ obviously undesirable, but equally if our map is too large, then we will not be able to process as many potential inputs in the same time and hence not uncover edges for that reason. Thus a careful trade-off of map size must be made. -# Block & Edge Numbering +## Block & edge numbering + Since the original AFL, blocks and edges have always been numbered in the same -way as we can see from the following C snippet from the whitepaper. +way as we can see from the following C snippet from the whitepaper: ```c - cur_location = (block_address >> 4) ^ (block_address << 8); - shared_mem[cur_location ^ prev_location]++; - prev_location = cur_location >> 1; - +cur_location = (block_address >> 4) ^ (block_address << 8); +shared_mem[cur_location ^ prev_location]++; +prev_location = cur_location >> 1; ``` Each block ID is generated by performing a shift and XOR on its address. Then @@ -63,36 +68,39 @@ the edge ID is calculated as `E = B ^ (B' >> 1)`. Here, we can make two observations. In fact, the edge ID is also masked to ensure it is less than the size of the map being used. -## Block IDs +### Block IDs + Firstly, the block ID doesn't have very good entropy. If we consider the address of the block, then whilst each block has a unique ID, it isn't necessarily very evenly distributed. -We start with a large address, and need to discard a large number of the bits to +We start with a large address and need to discard a large number of the bits to generate a block ID which is within range. But how do we choose the unique bits -of the address verus those which are the same for every block? The high bits of -the address may simply be all `0s` or all `1s` to make the address cannonical, -the middle portion of the address may be the same for all blocks (since if they -are all within the same binary, then they will all be adjacent in memory), and -on some systems, even the low bits may have poor entropy as some use fixed -length aligned instructions. Then we need to consider that a portion of each -binary may contain the `.data` or `.bss` sections and so may not contain any -blocks of code at all. - -## Edge IDs +of the address versus those which are the same for every block? The high bits of +the address may be all `0s` or all `1s` to make the address canonical, the +middle portion of the address may be the same for all blocks (since if they are +all within the same binary, then they will all be adjacent in memory), and on +some systems, even the low bits may have poor entropy as some use fixed length +aligned instructions. Then we need to consider that a portion of each binary may +contain the `.data` or `.bss` sections and so may not contain any blocks of code +at all. + +### Edge IDs + Secondly, we can observe that when we generate an edge ID from the source and destination block IDs, we perform a right shift on the source block ID. Whilst there are good reasons as set out in the whitepaper why such a transform is -applied, in so doing, we dispose of `1` bit of precious entropy in our source +applied, in doing so, we dispose of `1` bit of precious entropy in our source block ID. All together, this means that some edge IDs may be more popular than others. -This means that some portions of the map may be very densly populated with large -numbers of edges, whilst others may be very sparsely populated, or not populated -at all. +This means that some portions of the map may be very densely populated with +large numbers of edges, whilst others may be very sparsely populated, or not +populated at all. -# Improvements -One of the main reaons why this algorithm selected, is performance. All of the +## Improvements + +One of the main reasons why this algorithm selected, is performance. All of the operations are very quick to perform and given we may be carrying this out for every block of code we execute, performance is critical. @@ -106,23 +114,25 @@ only need to generate this ID once per block and so this ID generation no longer needs to be as performant. We can therefore use a hash algorithm to generate this ID and therefore ensure that the block IDs are more evenly distributed. -Edge IDs however, can only be determined at run-time. Since we don't know which +Edge IDs, however, can only be determined at run-time. Since we don't know which blocks a given input will traverse until we run it. However, given our block IDs are now evenly distributed, generating an evenly distributed edge ID becomes -simple. Here, the only change we make is to use a rotate operation rather than -a shift operation so we don't lose a bit of entropy from the source ID. +simple. Here, the only change we make is to use a rotate operation rather than a +shift operation so we don't lose a bit of entropy from the source ID. So our new algorithm becomes: + ```c - cur_location = hash(block_address) - shared_mem[cur_location ^ prev_location]++; - prev_location = rotate(cur_location, 1); +cur_location = hash(block_address) +shared_mem[cur_location ^ prev_location]++; +prev_location = rotate(cur_location, 1); ``` Lastly, in the original design, the `cur_location` was always set to `0`, at the beginning of a run, we instead set the value of `cur_location` to `hash(0)`. -# Parallel Fuzzing +## Parallel fuzzing + Another sub-optimal aspect of the original design is that no matter how many instances of the fuzzer you ran in parallel, each instance numbered each block and so each edge with the same ID. Each instance would therefore find the same @@ -144,4 +154,4 @@ If only a single new edge is found, and the new path is shared with an instance for which that edge collides, that instance may disregard it as irrelevant. In practice, however, the discovery of a single new edge, likely leads to several more edges beneath it also being found and therefore the likelihood of all of -these being collisions is very slim. +these being collisions is very slim. \ No newline at end of file diff --git a/frida_mode/README.md b/frida_mode/README.md index 165f8089..08f6b891 100644 --- a/frida_mode/README.md +++ b/frida_mode/README.md @@ -1,38 +1,34 @@ -# FRIDA MODE +# FRIDA mode The purpose of FRIDA mode is to provide an alternative binary only fuzzer for -AFL just like that provided by QEMU mode. The intention is to provide a very +AFL++ just like that provided by QEMU mode. The intention is to provide a very similar user experience, right down to the options provided through environment variables. -Whilst AFLplusplus already has some support for running on FRIDA [here](https://github.com/AFLplusplus/AFLplusplus/tree/stable/utils/afl_frida) -this requires the code to be fuzzed to be provided as a shared library, it -cannot be used to fuzz executables. Additionally, it requires the user to write -a small harness around their target code of interest. -FRIDA mode instead takes a different approach to avoid these limitations. -In Frida mode binary programs are instrumented, similarly to QEMU mode. +In FRIDA mode, binary programs are instrumented, similarly to QEMU mode. -## Current Progress +## Current progress -As FRIDA mode is new, it is missing a lot of features. The design is such that it -should be possible to add these features in a similar manner to QEMU mode and +As FRIDA mode is new, it is missing a lot of features. The design is such that +it should be possible to add these features in a similar manner to QEMU mode and perhaps leverage some of its design and implementation. - | Feature/Instrumentation | frida-mode | Notes | - | -------------------------|:----------:|:--------------------------------------------:| - | NeverZero | x | | - | Persistent Mode | x | (x86/x64/aarch64 only) | - | LAF-Intel / CompCov | - | (CMPLOG is better 90% of the time) | - | CMPLOG | x | (x86/x64/aarch64 only) | - | Selective Instrumentation| x | | - | Non-Colliding Coverage | - | (Not possible in binary-only instrumentation | - | Ngram prev_loc Coverage | - | | - | Context Coverage | - | | - | Auto Dictionary | - | | - | Snapshot LKM Support | - | | - | In-Memory Test Cases | x | (x86/x64/aarch64 only) | +| Feature/Instrumentation | FRIDA mode | Notes | +| -------------------------|:----------:|:---------------------------------------------:| +| NeverZero | x | | +| Persistent Mode | x | (x86/x64/aarch64 only) | +| LAF-Intel / CompCov | - | (CMPLOG is better 90% of the time) | +| CMPLOG | x | (x86/x64/aarch64 only) | +| Selective Instrumentation| x | | +| Non-Colliding Coverage | - | (not possible in binary-only instrumentation) | +| Ngram prev_loc Coverage | - | | +| Context Coverage | - | | +| Auto Dictionary | - | | +| Snapshot LKM Support | - | | +| In-Memory Test Cases | x | (x86/x64/aarch64 only) | ## Compatibility + Currently FRIDA mode supports Linux and macOS targets on both x86/x64 architecture and aarch64. Later releases may add support for aarch32 and Windows targets as well as embedded linux environments. @@ -41,46 +37,58 @@ FRIDA has been used on various embedded targets using both uClibc and musl C runtime libraries, so porting should be possible. However, the current build system does not support cross compilation. -## Getting Started +## Getting started -To build everything run `make`. To build for x86 run `make 32`. Note that in +To build everything, run `make`. To build for x86, run `make 32`. Note that in x86 bit mode, it is not necessary for afl-fuzz to be built for 32-bit. However, -the shared library for frida_mode must be since it is injected into the target +the shared library for FRIDA mode must be since it is injected into the target process. Various tests can be found in subfolders within the `test/` directory. To use -these, first run `make` to build any dependencies. Then run `make qemu` or -`make frida` to run on either QEMU of FRIDA mode respectively. To run frida -tests in 32-bit mode, run `make ARCH=x86 frida`. When switching between -architectures it may be necessary to run `make clean` first for a given build -target to remove previously generated binaries for a different architecture. +these, first run `make` to build any dependencies. Then run `make qemu` or `make +frida` to run on either QEMU of FRIDA mode respectively. To run frida tests in +32-bit mode, run `make ARCH=x86 frida`. When switching between architectures, it +may be necessary to run `make clean` first for a given build target to remove +previously generated binaries for a different architecture. + +### Android + +In order to build, you need to download the Android SDK: + +[https://developer.android.com/ndk/downloads](https://developer.android.com/ndk/downloads) + +Then creating locally a standalone chain as follows: + +[https://developer.android.com/ndk/guides/standalone_toolchain](https://developer.android.com/ndk/guides/standalone_toolchain) ## Usage -FRIDA mode added some small modifications to `afl-fuzz` and similar tools -in AFLplusplus. The intention was that it behaves identically to QEMU, but it uses -the 'O' switch rather than 'Q'. Whilst the options 'f', 'F', 's' or 'S' may have +FRIDA mode added some small modifications to `afl-fuzz` and similar tools in +AFL++. The intention was that it behaves identically to QEMU, but it uses the +'O' switch rather than 'Q'. Whilst the options 'f', 'F', 's' or 'S' may have made more sense for a mode powered by FRIDA Stalker, they were all taken, so -instead we use 'O' in hommage to the [author](https://github.com/oleavr) of +instead we use 'O' in homage to the [author](https://github.com/oleavr) of FRIDA. Similarly, the intention is to mimic the use of environment variables used by -QEMU where possible (by replacing `s/QEMU/FRIDA/g`). Accordingly, the -following options are currently supported: +QEMU where possible (by replacing `s/QEMU/FRIDA/g`). Accordingly, the following +options are currently supported: -* `AFL_FRIDA_DEBUG_MAPS` - See `AFL_QEMU_DEBUG_MAPS` -* `AFL_FRIDA_EXCLUDE_RANGES` - See `AFL_QEMU_EXCLUDE_RANGES` -* `AFL_FRIDA_INST_RANGES` - See `AFL_QEMU_INST_RANGES` -* `AFL_FRIDA_PERSISTENT_ADDR` - See `AFL_QEMU_PERSISTENT_ADDR` -* `AFL_FRIDA_PERSISTENT_CNT` - See `AFL_QEMU_PERSISTENT_CNT` -* `AFL_FRIDA_PERSISTENT_HOOK` - See `AFL_QEMU_PERSISTENT_HOOK` -* `AFL_FRIDA_PERSISTENT_RET` - See `AFL_QEMU_PERSISTENT_RET` +* `AFL_FRIDA_DEBUG_MAPS` - See `AFL_QEMU_DEBUG_MAPS`. +* `AFL_FRIDA_EXCLUDE_RANGES` - See `AFL_QEMU_EXCLUDE_RANGES`. +* `AFL_FRIDA_INST_RANGES` - See `AFL_QEMU_INST_RANGES`. +* `AFL_FRIDA_PERSISTENT_ADDR` - See `AFL_QEMU_PERSISTENT_ADDR`. +* `AFL_FRIDA_PERSISTENT_CNT` - See `AFL_QEMU_PERSISTENT_CNT`. +* `AFL_FRIDA_PERSISTENT_HOOK` - See `AFL_QEMU_PERSISTENT_HOOK`. +* `AFL_FRIDA_PERSISTENT_RET` - See `AFL_QEMU_PERSISTENT_RET`. To enable the powerful CMPLOG mechanism, set `-c 0` for `afl-fuzz`. ## Scripting -One of the more powerful features of FRIDA mode is it's support for configuration by JavaScript, rather than using environment variables. For details of how this works see [here](Scripting.md). +One of the more powerful features of FRIDA mode is it's support for +configuration by JavaScript, rather than using environment variables. For +details of how this works, see [Scripting.md](Scripting.md). ## Performance @@ -104,16 +112,18 @@ FRIDA mode is supported by using `LD_PRELOAD` (`DYLD_INSERT_LIBRARIES` on macOS) to inject a shared library (`afl-frida-trace.so`) into the target. This shared library is built using the [frida-gum](https://github.com/frida/frida-gum) devkit from the [FRIDA](https://github.com/frida/frida) project. One of the -components of frida-gum is [Stalker](https://medium.com/@oleavr/anatomy-of-a-code-tracer-b081aadb0df8), +components of frida-gum is +[Stalker](https://medium.com/@oleavr/anatomy-of-a-code-tracer-b081aadb0df8), this allows the dynamic instrumentation of running code for AARCH32, AARCH64, x86 and x64 architectures. Implementation details can be found [here](https://frida.re/docs/stalker/). Dynamic instrumentation is used to augment the target application with similar coverage information to that inserted by `afl-gcc` or `afl-clang`. The shared -library is also linked to the `compiler-rt` component of AFLplusplus to feedback -this coverage information to AFL++ and also provide a fork server. It also makes -use of the FRIDA [prefetch](https://github.com/frida/frida-gum/blob/56dd9ba3ee9a5511b4b0c629394bf122775f1ab7/gum/gumstalker.h#L115) +library is also linked to the `compiler-rt` component of AFL++ to feedback this +coverage information to AFL++ and also provide a fork server. It also makes use +of the FRIDA +[prefetch](https://github.com/frida/frida-gum/blob/56dd9ba3ee9a5511b4b0c629394bf122775f1ab7/gum/gumstalker.h#L115) support to feedback instrumented blocks from the child to the parent using a shared memory region to avoid the need to regenerate instrumented blocks on each fork. @@ -131,210 +141,239 @@ instances run CMPLOG mode and instrumentation of the binary is less frequent (only on CMP, SUB and CALL instructions) performance is not quite so critical. ## Advanced configuration options -* `AFL_FRIDA_INST_COVERAGE_FILE` - File to write DynamoRio format coverage -information (e.g. to be loaded within IDA lighthouse). + +* `AFL_FRIDA_DRIVER_NO_HOOK` - See `AFL_QEMU_DRIVER_NO_HOOK`. When using the + QEMU driver to provide a `main` loop for a user provided + `LLVMFuzzerTestOneInput`, this option configures the driver to read input from + `stdin` rather than using in-memory test cases. +* `AFL_FRIDA_INST_COVERAGE_FILE` - File to write DynamoRIO format coverage + information (e.g., to be loaded within IDA lighthouse). * `AFL_FRIDA_INST_DEBUG_FILE` - File to write raw assembly of original blocks -and their instrumented counterparts during block compilation. -``` -*** - -Creating block for 0x7ffff7953313: - 0x7ffff7953313 mov qword ptr [rax], 0 - 0x7ffff795331a add rsp, 8 - 0x7ffff795331e ret - -Generated block 0x7ffff75e98e2 - 0x7ffff75e98e2 mov qword ptr [rax], 0 - 0x7ffff75e98e9 add rsp, 8 - 0x7ffff75e98ed lea rsp, [rsp - 0x80] - 0x7ffff75e98f5 push rcx - 0x7ffff75e98f6 movabs rcx, 0x7ffff795331e - 0x7ffff75e9900 jmp 0x7ffff75e9384 - - -*** -``` + and their instrumented counterparts during block compilation. + + ``` + *** + + Creating block for 0x7ffff7953313: + 0x7ffff7953313 mov qword ptr [rax], 0 + 0x7ffff795331a add rsp, 8 + 0x7ffff795331e ret + + Generated block 0x7ffff75e98e2 + 0x7ffff75e98e2 mov qword ptr [rax], 0 + 0x7ffff75e98e9 add rsp, 8 + 0x7ffff75e98ed lea rsp, [rsp - 0x80] + 0x7ffff75e98f5 push rcx + 0x7ffff75e98f6 movabs rcx, 0x7ffff795331e + 0x7ffff75e9900 jmp 0x7ffff75e9384 + + *** + ``` + * `AFL_FRIDA_INST_JIT` - Enable the instrumentation of Just-In-Time compiled -code. Code is considered to be JIT if the executable segment is not backed by a -file. + code. Code is considered to be JIT if the executable segment is not backed by + a file. * `AFL_FRIDA_INST_NO_OPTIMIZE` - Don't use optimized inline assembly coverage -instrumentation (the default where available). Required to use -`AFL_FRIDA_INST_TRACE`. -* `AFL_FRIDA_INST_NO_PREFETCH` - Disable prefetching. By default the child will -report instrumented blocks back to the parent so that it can also instrument -them and they be inherited by the next child on fork, implies -`AFL_FRIDA_INST_NO_PREFETCH_BACKPATCH`. + instrumentation (the default where available). Required to use + `AFL_FRIDA_INST_TRACE`. +* `AFL_FRIDA_INST_NO_BACKPATCH` - Disable backpatching. At the end of executing + each block, control will return to FRIDA to identify the next block to + execute. +* `AFL_FRIDA_INST_NO_PREFETCH` - Disable prefetching. By default, the child will + report instrumented blocks back to the parent so that it can also instrument + them and they be inherited by the next child on fork, implies + `AFL_FRIDA_INST_NO_PREFETCH_BACKPATCH`. * `AFL_FRIDA_INST_NO_PREFETCH_BACKPATCH` - Disable prefetching of stalker -backpatching information. By default the child will report applied backpatches -to the parent so that they can be applied and then be inherited by the next -child on fork. + backpatching information. By default, the child will report applied + backpatches to the parent so that they can be applied and then be inherited by + the next child on fork. * `AFL_FRIDA_INST_SEED` - Sets the initial seed for the hash function used to -generate block (and hence edge) IDs. Setting this to a constant value may be -useful for debugging purposes, e.g. investigating unstable edges. -* `AFL_FRIDA_INST_TRACE` - Log to stdout the address of executed blocks, -implies `AFL_FRIDA_INST_NO_OPTIMIZE`. + generate block (and hence edge) IDs. Setting this to a constant value may be + useful for debugging purposes, e.g., investigating unstable edges. +* `AFL_FRIDA_INST_TRACE` - Log to stdout the address of executed blocks, implies + `AFL_FRIDA_INST_NO_OPTIMIZE`. * `AFL_FRIDA_INST_TRACE_UNIQUE` - As per `AFL_FRIDA_INST_TRACE`, but each edge -is logged only once, requires `AFL_FRIDA_INST_NO_OPTIMIZE`. -* `AFL_FRIDA_INST_UNSTABLE_COVERAGE_FILE` - File to write DynamoRio format -coverage information for unstable edges (e.g. to be loaded within IDA -lighthouse). + is logged only once, requires `AFL_FRIDA_INST_NO_OPTIMIZE`. +* `AFL_FRIDA_INST_UNSTABLE_COVERAGE_FILE` - File to write DynamoRIO format + coverage information for unstable edges (e.g., to be loaded within IDA + lighthouse). +* `AFL_FRIDA_JS_SCRIPT` - Set the script to be loaded by the FRIDA scripting + engine. See [Scipting.md](Scripting.md) for details. * `AFL_FRIDA_OUTPUT_STDOUT` - Redirect the standard output of the target -application to the named file (supersedes the setting of `AFL_DEBUG_CHILD`) + application to the named file (supersedes the setting of `AFL_DEBUG_CHILD`). * `AFL_FRIDA_OUTPUT_STDERR` - Redirect the standard error of the target -application to the named file (supersedes the setting of `AFL_DEBUG_CHILD`) + application to the named file (supersedes the setting of `AFL_DEBUG_CHILD`). * `AFL_FRIDA_PERSISTENT_DEBUG` - Insert a Breakpoint into the instrumented code -at `AFL_FRIDA_PERSISTENT_HOOK` and `AFL_FRIDA_PERSISTENT_RET` to allow the user -to detect issues in the persistent loop using a debugger. - -``` + at `AFL_FRIDA_PERSISTENT_HOOK` and `AFL_FRIDA_PERSISTENT_RET` to allow the + user to detect issues in the persistent loop using a debugger. + + ``` + gdb \ + --ex 'set environment AFL_FRIDA_PERSISTENT_ADDR=XXXXXXXXXX' \ + --ex 'set environment AFL_FRIDA_PERSISTENT_RET=XXXXXXXXXX' \ + --ex 'set environment AFL_FRIDA_PERSISTENT_DEBUG=1' \ + --ex 'set environment AFL_DEBUG_CHILD=1' \ + --ex 'set environment LD_PRELOAD=afl-frida-trace.so' \ + --args <my-executable> [my arguments] + ``` -gdb \ - --ex 'set environment AFL_FRIDA_PERSISTENT_ADDR=XXXXXXXXXX' \ - --ex 'set environment AFL_FRIDA_PERSISTENT_RET=XXXXXXXXXX' \ - --ex 'set environment AFL_FRIDA_PERSISTENT_DEBUG=1' \ - --ex 'set environment AFL_DEBUG_CHILD=1' \ - --ex 'set environment LD_PRELOAD=afl-frida-trace.so' \ - --args <my-executable> [my arguments] - -``` * `AFL_FRIDA_SECCOMP_FILE` - Write a log of any syscalls made by the target to -the specified file. + the specified file. +* `AFL_FRIDA_STALKER_ADJACENT_BLOCKS` - Configure the number of adjacent blocks + to fetch when generating instrumented code. By fetching blocks in the same + order they appear in the original program, rather than the order of execution + should help reduce locality and adjacency. This includes allowing us to vector + between adjacent blocks using a NOP slide rather than an immediate branch. * `AFL_FRIDA_STALKER_IC_ENTRIES` - Configure the number of inline cache entries -stored along-side branch instructions which provide a cache to avoid having to -call back into FRIDA to find the next block. Default is 32. + stored along-side branch instructions which provide a cache to avoid having to + call back into FRIDA to find the next block. Default is 32. * `AFL_FRIDA_STATS_FILE` - Write statistics information about the code being -instrumented to the given file name. The statistics are written only for the -child process when new block is instrumented (when the -`AFL_FRIDA_STATS_INTERVAL` has expired). Note that simply because a new path is -found does not mean a new block needs to be compiled. It could simply be that -the existing blocks instrumented have been executed in a different order. -``` -stats ------ -Time 2021-07-21 11:45:49 -Elapsed 1 seconds - - -Transitions cumulative delta ------------ ---------- ----- -total 753619 17645 -call_imm 9193 ( 1.22%) 344 ( 1.95%) [ 344/s] -call_reg 0 ( 0.00%) 0 ( 0.00%) [ 0/s] -call_mem 0 ( 0.00%) 0 ( 0.00%) [ 0/s] -ret_slow_path 67974 ( 9.02%) 2988 (16.93%) [ 2988/s] -post_call_invoke 7996 ( 1.06%) 299 ( 1.69%) [ 299/s] -excluded_call_imm 3804 ( 0.50%) 200 ( 1.13%) [ 200/s] -jmp_imm 5445 ( 0.72%) 255 ( 1.45%) [ 255/s] -jmp_reg 42081 ( 5.58%) 1021 ( 5.79%) [ 1021/s] -jmp_mem 578092 (76.71%) 10956 (62.09%) [ 10956/s] -jmp_cond_imm 38951 ( 5.17%) 1579 ( 8.95%) [ 1579/s] -jmp_cond_mem 0 ( 0.00%) 0 ( 0.00%) [ 0/s] -jmp_cond_reg 0 ( 0.00%) 0 ( 0.00%) [ 0/s] -jmp_cond_jcxz 0 ( 0.00%) 0 ( 0.00%) [ 0/s] -jmp_continuation 84 ( 0.01%) 3 ( 0.02%) [ 3/s] - - -Instrumentation ---------------- -Instructions 7907 -Blocks 1764 -Avg Instructions / Block 4 - - -EOB Instructions ----------------- -Total 1763 (22.30%) -Call Immediates 358 ( 4.53%) -Call Immediates Excluded 74 ( 0.94%) -Call Register 0 ( 0.00%) -Call Memory 0 ( 0.00%) -Jump Immediates 176 ( 2.23%) -Jump Register 8 ( 0.10%) -Jump Memory 10 ( 0.13%) -Conditional Jump Immediates 1051 (13.29%) -Conditional Jump CX Immediate 0 ( 0.00%) -Conditional Jump Register 0 ( 0.00%) -Conditional Jump Memory 0 ( 0.00%) -Returns 160 ( 2.02%) - - -Relocated Instructions ----------------------- -Total 232 ( 2.93%) -addsd 2 ( 0.86%) -cmp 46 (19.83%) -comisd 2 ( 0.86%) -divsd 2 ( 0.86%) -divss 2 ( 0.86%) -lea 142 (61.21%) -mov 32 (13.79%) -movsd 2 ( 0.86%) -ucomisd 2 ( 0.86%) -``` -* `AFL_FRIDA_STATS_INTERVAL` - The maximum frequency to output statistics -information. Stats will be written whenever they are updated if the given -interval has elapsed since last time they were written. + instrumented to the given file name. The statistics are written only for the + child process when new block is instrumented (when the + `AFL_FRIDA_STATS_INTERVAL` has expired). Note that just because a new path is + found does not mean a new block needs to be compiled. It could be that the + existing blocks instrumented have been executed in a different order. + + ``` + stats + ----- + Time 2021-07-21 11:45:49 + Elapsed 1 seconds + + + Transitions cumulative delta + ----------- ---------- ----- + total 753619 17645 + call_imm 9193 ( 1.22%) 344 ( 1.95%) [ 344/s] + call_reg 0 ( 0.00%) 0 ( 0.00%) [ 0/s] + call_mem 0 ( 0.00%) 0 ( 0.00%) [ 0/s] + ret_slow_path 67974 ( 9.02%) 2988 (16.93%) [ 2988/s] + post_call_invoke 7996 ( 1.06%) 299 ( 1.69%) [ 299/s] + excluded_call_imm 3804 ( 0.50%) 200 ( 1.13%) [ 200/s] + jmp_imm 5445 ( 0.72%) 255 ( 1.45%) [ 255/s] + jmp_reg 42081 ( 5.58%) 1021 ( 5.79%) [ 1021/s] + jmp_mem 578092 (76.71%) 10956 (62.09%) [ 10956/s] + jmp_cond_imm 38951 ( 5.17%) 1579 ( 8.95%) [ 1579/s] + jmp_cond_mem 0 ( 0.00%) 0 ( 0.00%) [ 0/s] + jmp_cond_reg 0 ( 0.00%) 0 ( 0.00%) [ 0/s] + jmp_cond_jcxz 0 ( 0.00%) 0 ( 0.00%) [ 0/s] + jmp_continuation 84 ( 0.01%) 3 ( 0.02%) [ 3/s] + + + Instrumentation + --------------- + Instructions 7907 + Blocks 1764 + Avg Instructions / Block 4 + + + EOB Instructions + ---------------- + Total 1763 (22.30%) + Call Immediates 358 ( 4.53%) + Call Immediates Excluded 74 ( 0.94%) + Call Register 0 ( 0.00%) + Call Memory 0 ( 0.00%) + Jump Immediates 176 ( 2.23%) + Jump Register 8 ( 0.10%) + Jump Memory 10 ( 0.13%) + Conditional Jump Immediates 1051 (13.29%) + Conditional Jump CX Immediate 0 ( 0.00%) + Conditional Jump Register 0 ( 0.00%) + Conditional Jump Memory 0 ( 0.00%) + Returns 160 ( 2.02%) + + + Relocated Instructions + ---------------------- + Total 232 ( 2.93%) + addsd 2 ( 0.86%) + cmp 46 (19.83%) + comisd 2 ( 0.86%) + divsd 2 ( 0.86%) + divss 2 ( 0.86%) + lea 142 (61.21%) + mov 32 (13.79%) + movsd 2 ( 0.86%) + ucomisd 2 ( 0.86%) + ``` -## FASAN - Frida Address Sanitizer Mode -Frida mode also supports FASAN. The design of this is actually quite simple and +* `AFL_FRIDA_STATS_INTERVAL` - The maximum frequency to output statistics + information. Stats will be written whenever they are updated if the given + interval has elapsed since last time they were written. +* `AFL_FRIDA_TRACEABLE` - Set the child process to be traceable by any process + to aid debugging and overcome the restrictions imposed by YAMA. Supported on + Linux only. Permits a non-root user to use `gcore` or similar to collect a + core dump of the instrumented target. Note that in order to capture the core + dump you must set a sufficient timeout (using `-t`) to avoid `afl-fuzz` + killing the process whilst it is being dumped. + +## FASAN - FRIDA Address Sanitizer mode + +FRIDA mode also supports FASAN. The design of this is actually quite simple and very similar to that used when instrumenting applications compiled from source. -### Address Sanitizer Basics +### Address Sanitizer basics When Address Sanitizer is used to instrument programs built from source, the compiler first adds a dependency (`DT_NEEDED` entry) for the Address Sanitizer -dynamic shared object (DSO). This shared object contains the main logic for Address -Sanitizer, including setting and managing up the shadow memory. It also provides -replacement implementations for a number of functions in standard libraries. - -These replacements include things like `malloc` and `free` which allows for those -allocations to be marked in the shadow memory, but also a number of other fuctions. -Consider `memcpy` for example, this is instrumented to validate the paramters -(test the source and destination buffers against the shadow memory. This is much -easier than instrumenting those standard libraries since, first it would require -you to re-compile them and secondly it would mean that the instrumentation would -be applied at a more expensive granular level. Lastly, load-widening (typically -found in highy optimized code) can also make this instrumentation more difficult. +dynamic shared object (DSO). This shared object contains the main logic for +Address Sanitizer, including setting and managing up the shadow memory. It also +provides replacement implementations for a number of functions in standard +libraries. + +These replacements include things like `malloc` and `free` which allows for +those allocations to be marked in the shadow memory, but also a number of other +functions. Consider `memcpy`, for example. This is instrumented to validate the +parameters (test the source and destination buffers against the shadow memory). +This is much easier than instrumenting those standard libraries, since first, it +would require you to re-compile them and secondly it would mean that the +instrumentation would be applied at a more expensive granular level. Lastly, +load-widening (typically found in highly optimized code) can also make this +instrumentation more difficult. Since the DSO is loaded before all of the standard libraries (in fact it insists on being first), the dynamic loader will use it to resolve imports from other modules which depend on it. -### FASAN Implementation +### FASAN implementation + +FASAN takes a similar approach. It requires the user to add the Address +Sanitizer DSO to the `AFL_PRELOAD` environment variable such that it is loaded +into the target. Again, it must be first in the list. This means that it is not +necessary to instrument the standard libraries to detect when an application has +provided an incorrect argument to `memcpy`, for example. This avoids issues with +load-widening and should also mean a huge improvement in performance. + +FASAN then adds instrumentation for any instructions which use memory operands +and then calls into the `__asan_loadN` and `__asan_storeN` functions provided by +the DSO to validate memory accesses against the shadow memory. + +## Collisions -FASAN takes a similar approach. It requires the user to add the Address Sanitizer -DSO to the `AFL_PRELOAD` environment variable such that it is loaded into the target. -Again, it must be first in the list. This means that it is not necessary to -instrument the standard libraries to detect when an application has provided an -incorrect argument to `memcpy` for example. This avoids issues with load-widening -and should also mean a huge improvement in performance. +FRIDA mode has also introduced some improvements to reduce collisions in the +map. For details, see [MapDensity.md](MapDensity.md). -FASAN then adds instrumentation for any instrucutions which use memory operands and -then calls into the `__asan_loadN` and `__asan_storeN` functions provided by the DSO -to validate memory accesses against the shadow memory. +## OSX library fuzzing -# Collisions -FRIDA mode has also introduced some improvements to reduce collisions in the map. -See [here](MapDensity.md) for details. +An example of how to fuzz a dynamic library on OSX is included, see +[test/osx-lib](test/osx-lib). This requires the use of a simple test harness +executable which will load the library and call a target function within it. The +dependent library can either be loaded in using `dlopen` and `dlsym` in a +function marked `__attribute__((constructor()))` or the test harness can be +linked against it. It is important that the target library is loaded before +execution of `main`, since this is the point where FRIDA mode is initialized. +Otherwise, it will not be possible to configure coverage for the test library +using `AFL_FRIDA_INST_RANGES` or similar. -# OSX Library Fuzzing -An example of how to fuzz a dynamic library on OSX is included [here](test/osx-lib). -This requires the use of a simple test harness executable which will load the -library and call a target function within it. The dependent library can either -be loaded in using `dlopen` and `dlsym` in a function marked -`__attribute__((constructor()))` or the test harness can simply be linked -against it. It is important that the target library is loaded before execution -of `main`, since this is the point where FRIDA mode is initialized. Otherwise, it -will not be possible to configure coverage for the test library using -`AFL_FRIDA_INST_RANGES` or similar. +## Debugging -# Debugging -Please refer to the [debugging](#debugging) guide for assistant should you -encounter problems with FRIDA mode. +Should you encounter problems with FRIDA mode, refer to +[DEBUGGING.md](DEBUGGING.md) for assistance. -## TODO +## To do The next features to be added are Aarch32 support as well as looking at -potential performance improvements. The intention is to achieve feature parity with -QEMU mode in due course. Contributions are welcome, but please get in touch to -ensure that efforts are deconflicted. +potential performance improvements. The intention is to achieve feature parity +with QEMU mode in due course. Contributions are welcome, but please get in touch +to ensure that efforts are deconflicted. \ No newline at end of file diff --git a/frida_mode/Scripting.md b/frida_mode/Scripting.md index f6017fad..ad86fdd3 100644 --- a/frida_mode/Scripting.md +++ b/frida_mode/Scripting.md @@ -1,25 +1,32 @@ # Scripting + FRIDA now supports the ability to configure itself using JavaScript. This allows the user to make use of the convenience of FRIDA's scripting engine (along with it's support for debug symbols and exports) to configure all of the things which were traditionally configured using environment variables. -By default FRIDA mode will look for the file `afl.js` in the current working +By default, FRIDA mode will look for the file `afl.js` in the current working directory of the target. Alternatively, a script file can be configured using the environment variable `AFL_FRIDA_JS_SCRIPT`. -This script can make use of all of the standard [frida api functions](https://frida.re/docs/javascript-api/), but FRIDA mode adds some additional functions to allow -you to interact with FRIDA mode itself. These can all be accessed via the global -`Afl` parameter. e.g. `Afl.print("HELLO WORLD");`, +This script can make use of all of the standard [frida api +functions](https://frida.re/docs/javascript-api/), but FRIDA mode adds some +additional functions to allow you to interact with FRIDA mode itself. These can +all be accessed via the global `Afl` parameter, e.g., `Afl.print("HELLO +WORLD");`. If you encounter a problem with your script, then you should set the environment variable `AFL_DEBUG_CHILD=1` to view any diagnostic information. +## Example -# Example -Most of the time, users will likely be wanting to call the functions which configure an address (e.g. for the entry point, or the persistent address). +Most of the time, users will likely be wanting to call the functions which +configure an address (e.g., for the entry point or the persistent address). -The example below uses the API [`DebugSymbol.fromName()`](https://frida.re/docs/javascript-api/#debugsymbol). Another use API is [`Module.getExportByName()`](https://frida.re/docs/javascript-api/#module). +The example below uses the API +[`DebugSymbol.fromName()`](https://frida.re/docs/javascript-api/#debugsymbol). +Another use API is +[`Module.getExportByName()`](https://frida.re/docs/javascript-api/#module). ```js /* Use Afl.print instead of console.log */ @@ -86,9 +93,9 @@ Afl.done(); Afl.print("done"); ``` -# Stripped Binaries +## Stripped binaries -Lastly, if the binary you attempting to fuzz has no symbol information, and no +Lastly, if the binary you attempting to fuzz has no symbol information and no exports, then the following approach can be used. ```js @@ -98,11 +105,12 @@ const address = module.base.add(0xdeadface); Afl.setPersistentAddress(address); ``` -# Persisent Hook +## Persistent hook + A persistent hook can be implemented using a conventional shared object, sample source code for a hook suitable for the prototype of `LLVMFuzzerTestOneInput` -can be found [here](hook/hook.c). This can be configured using code similar to -the following. +can be found in [hook/](hook/). This can be configured using code similar to the +following. ```js const path = Afl.module.path; @@ -112,7 +120,8 @@ const hook = mod.getExportByName('afl_persistent_hook'); Afl.setPersistentHook(hook); ``` -Alternatively, the hook can be provided by using FRIDAs built in support for `CModule`, powered by TinyCC. +Alternatively, the hook can be provided by using FRIDA's built-in support for +`CModule`, powered by TinyCC. ```js const cm = new CModule(` @@ -134,8 +143,10 @@ const cm = new CModule(` Afl.setPersistentHook(cm.afl_persistent_hook); ``` -# Advanced Persistence +## Advanced persistence + Consider the following target code... + ```c #include <fcntl.h> @@ -246,7 +257,7 @@ FRIDA mode supports the replacement of any function, with an implementation generated by CModule. This allows for a bespoke harness to be written as follows: -``` +```js const slow = DebugSymbol.fromName('slow').address; Afl.print(`slow: ${slow}`); @@ -281,15 +292,96 @@ Afl.done(); Here, we replace the function `slow` with our own code. This code is then selected as the entry point as well as the persistent loop address. -**WARNING** There are two key limitations in replacing a function in this way: -- The function which is to be replaced must not be `main` this is because this -is the point at which FRIDA mode is initialized and at the point the the JS has -been run, the start of the `main` function has already been instrumented and -cached. -- The replacement function must not call itself. e.g. in this example we -couldn't replace `LLVMFuzzerTestOneInput` and call itself. +### Replacing LLVMFuzzerTestOneInput + +The function `LLVMFuzzerTestOneInput` can be replaced just like any other. Also, +any replaced function can also call itself. In the example below, we replace +`LLVMFuzzerTestOneInput` with `My_LLVMFuzzerTestOneInput` which ignores the +parameters `buf` and `len` and then calls the original `LLVMFuzzerTestOneInput` +with the parameters `__afl_fuzz_ptr` and `__afl_fuzz_len`. This allows us to +carry out in-memory fuzzing without the need for any hook function. It should be +noted that the replacement function and the original *CANNOT* share the same +name, since otherwise the `C` code in the `CModule` will not compile due to a +symbol name collision. + +```js +const LLVMFuzzerTestOneInput = DebugSymbol.fromName('LLVMFuzzerTestOneInput').address; +Afl.print(`LLVMFuzzerTestOneInput: ${LLVMFuzzerTestOneInput}`); + +const cm = new CModule(` + + extern unsigned char * __afl_fuzz_ptr; + extern unsigned int * __afl_fuzz_len; + extern void LLVMFuzzerTestOneInput(char *buf, int len); + + void My_LLVMFuzzerTestOneInput(char *buf, int len) { + + LLVMFuzzerTestOneInput(__afl_fuzz_ptr, *__afl_fuzz_len); + + } + `, + { + LLVMFuzzerTestOneInput: LLVMFuzzerTestOneInput, + __afl_fuzz_ptr: Afl.getAflFuzzPtr(), + __afl_fuzz_len: Afl.getAflFuzzLen() + }); + +Afl.setEntryPoint(cm.My_LLVMFuzzerTestOneInput); +Afl.setPersistentAddress(cm.My_LLVMFuzzerTestOneInput); +Afl.setInMemoryFuzzing(); +Interceptor.replace(LLVMFuzzerTestOneInput, cm.My_LLVMFuzzerTestOneInput); +``` + +### Hooking `main` + +Lastly, it should be noted that using FRIDA mode's scripting support to hook the +`main` function is a special case. This is because the `main` function is +already hooked by the FRIDA mode engine itself and hence the function `main` (or +at least the first basic block already been compiled by Stalker ready for +execution). Hence any attempt to use `Interceptor.replace` like in the example +above will not work. Instead the JS bindings provide a function `setJsMainHook` +for just this scenario as demonstrated in the example below. + +```js +const main = DebugSymbol.fromName('main').address; +Afl.print(`main: ${main}`); + +const LLVMFuzzerTestOneInput = DebugSymbol.fromName('LLVMFuzzerTestOneInput').address; +Afl.print(`LLVMFuzzerTestOneInput: ${LLVMFuzzerTestOneInput}`); + +const cm = new CModule(` + + extern unsigned char * __afl_fuzz_ptr; + extern unsigned int * __afl_fuzz_len; + extern void LLVMFuzzerTestOneInput(char *buf, int len); + + int main(int argc, char **argv) { + + LLVMFuzzerTestOneInput(__afl_fuzz_ptr, *__afl_fuzz_len); + + } + `, + { + LLVMFuzzerTestOneInput: LLVMFuzzerTestOneInput, + __afl_fuzz_ptr: Afl.getAflFuzzPtr(), + __afl_fuzz_len: Afl.getAflFuzzLen() + }); + +Afl.setEntryPoint(cm.main); +Afl.setPersistentAddress(cm.main); +Afl.setInMemoryFuzzing(); +Afl.setJsMainHook(cm.main); +``` + +### Library Fuzzing + +It doesn't take too much imagination to see that the above example can be +extended to use FRIDA's `Module.load` API so that the replaced `main` function +can then call an arbitrary function. In this way, if we have a library which we +wish to fuzz rather than an executable, then a surrogate executable can be used. + +## Patching -# Patching Consider the [following](test/js/test2.c) test code... ```c @@ -302,7 +394,7 @@ Consider the [following](test/js/test2.c) test code... Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ #include <fcntl.h> @@ -313,22 +405,22 @@ Consider the [following](test/js/test2.c) test code... #include <unistd.h> const uint32_t crc32_tab[] = { - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, ... - 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d }; uint32_t crc32(const void *buf, size_t size) { - const uint8_t *p = buf; - uint32_t crc; - crc = ~0U; - while (size--) - crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); - return crc ^ ~0U; + const uint8_t *p = buf; + uint32_t crc; + crc = ~0U; + while (size--) + crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); + return crc ^ ~0U; } /* @@ -419,9 +511,9 @@ int main(int argc, char **argv) { ``` There are a couple of obstacles with our target application. Unlike when fuzzing -source code, though, we can't simply edit it and recompile it. The following +source code, though, we can't just edit it and recompile it. The following script shows how we can use the normal functionality of FRIDA to modify any -troublesome behaviour. +troublesome behavior. ```js Afl.print('******************'); @@ -460,8 +552,10 @@ Afl.done(); Afl.print("done"); ``` -# Advanced Patching +## Advanced patching + Consider the following code fragment... + ```c extern void some_boring_bug2(char c); @@ -488,7 +582,7 @@ void LLVMFuzzerTestOneInput(char *buf, int len) { } ``` -Rather than using FRIDAs `Interceptor.replace` or `Interceptor.attach` APIs, it +Rather than using FRIDA's `Interceptor.replace` or `Interceptor.attach` APIs, it is possible to apply much more fine grained modification to the target application by means of using the Stalker APIs. @@ -572,39 +666,43 @@ Afl.setStalkerCallback(cm.js_stalker_callback) Afl.setStdErr("/tmp/stderr.txt"); ``` -Note that you will more likely want to find the -patch address by using: +Note that you will more likely want to find the patch address by using: ```js const module = Process.getModuleByName('target.exe'); /* Hardcoded offset within the target image */ const address = module.base.add(0xdeadface); ``` + OR + ``` const address = DebugSymbol.fromName("my_function").address.add(0xdeadface); ``` + OR + ``` const address = Module.getExportByName(null, "my_function").add(0xdeadface); ``` The function `js_stalker_callback` should return `TRUE` if the original -instruction should be emitted in the instrumented code, or `FALSE` otherwise. -In the example above, we can see it is replaced with a `NOP`. +instruction should be emitted in the instrumented code or `FALSE` otherwise. In +the example above, we can see it is replaced with a `NOP`. Lastly, note that the same callback will be called when compiling instrumented code both in the child of the forkserver (as it is executed) and also in the -parent of the forserver (when prefetching is enabled) so that it can be +parent of the forkserver (when prefetching is enabled) so that it can be inherited by the next forked child. It is **VERY** important that the same -instructions be generated in both the parent and the child, or if prefetching is +instructions be generated in both the parent and the child or if prefetching is disabled that the same instructions are generated every time the block is compiled. Failure to do so will likely lead to bugs which are incredibly difficult to diagnose. The code above only prints the instructions when running in the parent process (the one provided by `Process.id` when the JS script is executed). -# OSX +## OSX + Note that the JavaScript debug symbol api for OSX makes use of the `CoreSymbolication` APIs and as such the `CoreFoundation` module must be loaded into the target to make use of it. This can be done by setting: @@ -614,47 +712,38 @@ AFL_PRELOAD=/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation ``` It should be noted that `CoreSymbolication` API may take a while to initialize -and build its caches. For this reason, it may be nescessary to also increase the +and build its caches. For this reason, it may be necessary to also increase the value of the `-t` flag passed to `afl-fuzz`. -# API +## API + ```js class Afl { - - /** - * Field containing the `Module` object for `afl-frida-trace.so` (the FRIDA mode - * implementation). - */ - public static module: Module = Process.getModuleByName("afl-frida-trace.so"); - /** * This is equivalent to setting a value in `AFL_FRIDA_EXCLUDE_RANGES`, * it takes as arguments a `NativePointer` and a `number`. It can be * called multiple times to exclude several ranges. */ - public static addExcludedRange(addressess: NativePointer, size: number): void { - Afl.jsApiAddExcludeRange(addressess, size); + static addExcludedRange(addressess, size) { + Afl.jsApiAddExcludeRange(addressess, size); } - /** * This is equivalent to setting a value in `AFL_FRIDA_INST_RANGES`, * it takes as arguments a `NativePointer` and a `number`. It can be * called multiple times to include several ranges. */ - public static addIncludedRange(addressess: NativePointer, size: number): void { - Afl.jsApiAddIncludeRange(addressess, size); + static addIncludedRange(addressess, size) { + Afl.jsApiAddIncludeRange(addressess, size); } - /** * This must always be called at the end of your script. This lets * FRIDA mode know that your configuration is finished and that * execution has reached the end of your script. Failure to call * this will result in a fatal error. */ - public static done(): void { - Afl.jsApiDone(); + static done() { + Afl.jsApiDone(); } - /** * This function can be called within your script to cause FRIDA * mode to trigger a fatal error. This is useful if for example you @@ -662,49 +751,48 @@ class Afl { * stop. The user will need to enable `AFL_DEBUG_CHILD=1` to view * this error message. */ - public static error(msg: string): void { - const buf = Memory.allocUtf8String(msg); - Afl.jsApiError(buf); + static error(msg) { + const buf = Memory.allocUtf8String(msg); + Afl.jsApiError(buf); } - /** * Function used to provide access to `__afl_fuzz_ptr`, which contains the length of * fuzzing data when using in-memory test case fuzzing. */ - public static getAflFuzzLen(): NativePointer { - - return Afl.jsApiGetSymbol("__afl_fuzz_len"); + static getAflFuzzLen() { + return Afl.jsApiGetSymbol("__afl_fuzz_len"); } - /** * Function used to provide access to `__afl_fuzz_ptr`, which contains the fuzzing * data when using in-memory test case fuzzing. */ - public static getAflFuzzPtr(): NativePointer { - - return Afl.jsApiGetSymbol("__afl_fuzz_ptr"); + static getAflFuzzPtr() { + return Afl.jsApiGetSymbol("__afl_fuzz_ptr"); } - /** * Print a message to the STDOUT. This should be preferred to * FRIDA's `console.log` since FRIDA will queue it's log messages. * If `console.log` is used in a callback in particular, then there * may no longer be a thread running to service this queue. */ - public static print(msg: string): void { - const STDOUT_FILENO = 2; - const log = `${msg}\n`; - const buf = Memory.allocUtf8String(log); - Afl.jsApiWrite(STDOUT_FILENO, buf, log.length); + static print(msg) { + const STDOUT_FILENO = 2; + const log = `${msg}\n`; + const buf = Memory.allocUtf8String(log); + Afl.jsApiWrite(STDOUT_FILENO, buf, log.length); + } + /** + * See `AFL_FRIDA_INST_NO_BACKPATCH`. + */ + static setBackpatchDisable() { + Afl.jsApiSetBackpatchDisable(); } - /** * See `AFL_FRIDA_DEBUG_MAPS`. */ - public static setDebugMaps(): void { - Afl.jsApiSetDebugMaps(); + static setDebugMaps() { + Afl.jsApiSetDebugMaps(); } - /** * This has the same effect as setting `AFL_ENTRYPOINT`, but has the * convenience of allowing you to use FRIDAs APIs to determine the @@ -713,143 +801,198 @@ class Afl { * function should be called with a `NativePointer` as its * argument. */ - public static setEntryPoint(address: NativePointer): void { - Afl.jsApiSetEntryPoint(address); + static setEntryPoint(address) { + Afl.jsApiSetEntryPoint(address); } - /** * Function used to enable in-memory test cases for fuzzing. */ - public static setInMemoryFuzzing(): void { - Afl.jsApiAflSharedMemFuzzing.writeInt(1); + static setInMemoryFuzzing() { + Afl.jsApiAflSharedMemFuzzing.writeInt(1); + } + /** + * See `AFL_FRIDA_INST_COVERAGE_FILE`. This function takes a single `string` + * as an argument. + */ + static setInstrumentCoverageFile(file) { + const buf = Memory.allocUtf8String(file); + Afl.jsApiSetInstrumentCoverageFile(buf); } - /** * See `AFL_FRIDA_INST_DEBUG_FILE`. This function takes a single `string` as * an argument. */ - public static setInstrumentDebugFile(file: string): void { - const buf = Memory.allocUtf8String(file); - Afl.jsApiSetInstrumentDebugFile(buf); + static setInstrumentDebugFile(file) { + const buf = Memory.allocUtf8String(file); + Afl.jsApiSetInstrumentDebugFile(buf); } - /** * See `AFL_FRIDA_INST_TRACE`. */ - public static setInstrumentEnableTracing(): void { - Afl.jsApiSetInstrumentTrace(); + static setInstrumentEnableTracing() { + Afl.jsApiSetInstrumentTrace(); + } + /** + * See `AFL_FRIDA_INST_JIT`. + */ + static setInstrumentJit() { + Afl.jsApiSetInstrumentJit(); } - /** * See `AFL_INST_LIBS`. */ - public static setInstrumentLibraries(): void { - Afl.jsApiSetInstrumentLibraries(); + static setInstrumentLibraries() { + Afl.jsApiSetInstrumentLibraries(); } - /** * See `AFL_FRIDA_INST_NO_OPTIMIZE` */ - public static setInstrumentNoOptimize(): void { - Afl.jsApiSetInstrumentNoOptimize(); + static setInstrumentNoOptimize() { + Afl.jsApiSetInstrumentNoOptimize(); + } + /* + * See `AFL_FRIDA_INST_SEED` + */ + static setInstrumentSeed(seed) { + Afl.jsApiSetInstrumentSeed(seed); } - /** * See `AFL_FRIDA_INST_TRACE_UNIQUE`. */ - public static setInstrumentTracingUnique(): void { - Afl.jsApiSetInstrumentTraceUnique(); + static setInstrumentTracingUnique() { + Afl.jsApiSetInstrumentTraceUnique(); + } + /** + * See `AFL_FRIDA_INST_UNSTABLE_COVERAGE_FILE`. This function takes a single + * `string` as an argument. + */ + static setInstrumentUnstableCoverageFile(file) { + const buf = Memory.allocUtf8String(file); + Afl.jsApiSetInstrumentUnstableCoverageFile(buf); + } + /* + * Set a callback to be called in place of the usual `main` function. This see + * `Scripting.md` for details. + */ + static setJsMainHook(address) { + Afl.jsApiSetJsMainHook(address); } - /** * This is equivalent to setting `AFL_FRIDA_PERSISTENT_ADDR`, again a * `NativePointer` should be provided as it's argument. */ - public static setPersistentAddress(address: NativePointer): void { - Afl.jsApiSetPersistentAddress(address); + static setPersistentAddress(address) { + Afl.jsApiSetPersistentAddress(address); } - /** * This is equivalent to setting `AFL_FRIDA_PERSISTENT_CNT`, a * `number` should be provided as it's argument. */ - public static setPersistentCount(count: number): void { - Afl.jsApiSetPersistentCount(count); + static setPersistentCount(count) { + Afl.jsApiSetPersistentCount(count); } - /** * See `AFL_FRIDA_PERSISTENT_DEBUG`. */ - public static setPersistentDebug(): void { - Afl.jsApiSetPersistentDebug(); + static setPersistentDebug() { + Afl.jsApiSetPersistentDebug(); } - /** * See `AFL_FRIDA_PERSISTENT_ADDR`. This function takes a NativePointer as an * argument. See above for examples of use. */ - public static setPersistentHook(address: NativePointer): void { - Afl.jsApiSetPersistentHook(address); + static setPersistentHook(address) { + Afl.jsApiSetPersistentHook(address); } - /** * This is equivalent to setting `AFL_FRIDA_PERSISTENT_RET`, again a * `NativePointer` should be provided as it's argument. */ - public static setPersistentReturn(address: NativePointer): void { - Afl.jsApiSetPersistentReturn(address); + static setPersistentReturn(address) { + Afl.jsApiSetPersistentReturn(address); + } + /** + * See `AFL_FRIDA_INST_NO_PREFETCH_BACKPATCH`. + */ + static setPrefetchBackpatchDisable() { + Afl.jsApiSetPrefetchBackpatchDisable(); } - /** * See `AFL_FRIDA_INST_NO_PREFETCH`. */ - public static setPrefetchDisable(): void { - Afl.jsApiSetPrefetchDisable(); + static setPrefetchDisable() { + Afl.jsApiSetPrefetchDisable(); + } + /** + * See `AFL_FRIDA_SECCOMP_FILE`. This function takes a single `string` as + * an argument. + */ + static setSeccompFile(file) { + const buf = Memory.allocUtf8String(file); + Afl.jsApiSetSeccompFile(buf); + } + /** + * See `AFL_FRIDA_STALKER_ADJACENT_BLOCKS`. + */ + static setStalkerAdjacentBlocks(val) { + Afl.jsApiSetStalkerAdjacentBlocks(val); } - /* - * Set a function to be called for each instruction which is instrumented - * by AFL FRIDA mode. + * Set a function to be called for each instruction which is instrumented + * by AFL FRIDA mode. + */ + static setStalkerCallback(callback) { + Afl.jsApiSetStalkerCallback(callback); + } + /** + * See `AFL_FRIDA_STALKER_IC_ENTRIES`. */ - public static setStalkerCallback(callback: NativePointer): void { - Afl.jsApiSetStalkerCallback(callback); + static setStalkerIcEntries(val) { + Afl.jsApiSetStalkerIcEntries(val); } - /** * See `AFL_FRIDA_STATS_FILE`. This function takes a single `string` as * an argument. */ - public static setStatsFile(file: string): void { - const buf = Memory.allocUtf8String(file); - Afl.jsApiSetStatsFile(buf); + static setStatsFile(file) { + const buf = Memory.allocUtf8String(file); + Afl.jsApiSetStatsFile(buf); } - /** * See `AFL_FRIDA_STATS_INTERVAL`. This function takes a `number` as an * argument */ - public static setStatsInterval(interval: number): void { - Afl.jsApiSetStatsInterval(interval); + static setStatsInterval(interval) { + Afl.jsApiSetStatsInterval(interval); } - /** * See `AFL_FRIDA_OUTPUT_STDERR`. This function takes a single `string` as * an argument. */ - public static setStdErr(file: string): void { - const buf = Memory.allocUtf8String(file); - Afl.jsApiSetStdErr(buf); + static setStdErr(file) { + const buf = Memory.allocUtf8String(file); + Afl.jsApiSetStdErr(buf); } - /** * See `AFL_FRIDA_OUTPUT_STDOUT`. This function takes a single `string` as * an argument. */ - public static setStdOut(file: string): void { - const buf = Memory.allocUtf8String(file); - Afl.jsApiSetStdOut(buf); + static setStdOut(file) { + const buf = Memory.allocUtf8String(file); + Afl.jsApiSetStdOut(buf); + } + /** + * See `AFL_FRIDA_TRACEABLE`. + */ + static setTraceable() { + Afl.jsApiSetTraceable(); + } + static jsApiGetFunction(name, retType, argTypes) { + const addr = Afl.module.getExportByName(name); + return new NativeFunction(addr, retType, argTypes); + } + static jsApiGetSymbol(name) { + return Afl.module.getExportByName(name); } - } - -``` +``` \ No newline at end of file diff --git a/frida_mode/frida.map b/frida_mode/frida.map index 0fc48aa6..5276db91 100644 --- a/frida_mode/frida.map +++ b/frida_mode/frida.map @@ -8,6 +8,7 @@ js_api_add_include_range; js_api_done; js_api_error; + js_api_set_backpatch_disable; js_api_set_debug_maps; js_api_set_entrypoint; js_api_set_instrument_coverage_file; @@ -19,6 +20,7 @@ js_api_set_instrument_trace; js_api_set_instrument_trace_unique; js_api_set_instrument_unstable_coverage_file; + js_api_set_js_main_hook; js_api_set_persistent_address; js_api_set_persistent_count; js_api_set_persistent_debug; @@ -28,11 +30,13 @@ js_api_set_prefetch_disable; js_api_set_seccomp_file; js_api_set_stalker_callback; + js_api_set_stalker_adjacent_blocks; js_api_set_stalker_ic_entries; js_api_set_stats_file; js_api_set_stats_interval; js_api_set_stderr; js_api_set_stdout; + js_api_set_traceable; local: *; diff --git a/frida_mode/include/asan.h b/frida_mode/include/asan.h index 67d33591..6745eb02 100644 --- a/frida_mode/include/asan.h +++ b/frida_mode/include/asan.h @@ -9,6 +9,7 @@ void asan_config(void); void asan_init(void); void asan_arch_init(void); void asan_instrument(const cs_insn *instr, GumStalkerIterator *iterator); +void asan_exclude_module_by_symbol(gchar *symbol_name); #endif diff --git a/frida_mode/include/entry.h b/frida_mode/include/entry.h index 3f0a4ecc..edc41467 100644 --- a/frida_mode/include/entry.h +++ b/frida_mode/include/entry.h @@ -4,6 +4,7 @@ #include "frida-gumjs.h" extern guint64 entry_point; +extern gboolean traceable; extern gboolean entry_compiled; extern gboolean entry_run; @@ -15,5 +16,7 @@ void entry_start(void); void entry_prologue(GumStalkerIterator *iterator, GumStalkerOutput *output); +void entry_on_fork(void); + #endif diff --git a/frida_mode/include/instrument.h b/frida_mode/include/instrument.h index 909b2a2c..a5d52616 100644 --- a/frida_mode/include/instrument.h +++ b/frida_mode/include/instrument.h @@ -29,13 +29,15 @@ GumStalkerTransformer *instrument_get_transformer(void); /* Functions to be implemented by the different architectures */ gboolean instrument_is_coverage_optimize_supported(void); +void instrument_coverage_optimize_init(void); void instrument_coverage_optimize(const cs_insn * instr, GumStalkerOutput *output); void instrument_debug_config(void); void instrument_debug_init(void); void instrument_debug_start(uint64_t address, GumStalkerOutput *output); -void instrument_debug_instruction(uint64_t address, uint16_t size); +void instrument_debug_instruction(uint64_t address, uint16_t size, + GumStalkerOutput *output); void instrument_debug_end(GumStalkerOutput *output); void instrument_flush(GumStalkerOutput *output); gpointer instrument_cur(GumStalkerOutput *output); diff --git a/frida_mode/include/js.h b/frida_mode/include/js.h index a5ecb712..39aa0573 100644 --- a/frida_mode/include/js.h +++ b/frida_mode/include/js.h @@ -7,11 +7,14 @@ typedef gboolean (*js_api_stalker_callback_t)(const cs_insn *insn, gboolean begin, gboolean excluded, GumStalkerOutput *output); +typedef int (*js_main_hook_t)(int argc, char **argv, char **envp); + extern unsigned char api_js[]; extern unsigned int api_js_len; extern gboolean js_done; extern js_api_stalker_callback_t js_user_callback; +extern js_main_hook_t js_main_hook; /* Frida Mode */ diff --git a/frida_mode/include/ranges.h b/frida_mode/include/ranges.h index 0220a59d..3bd9eaa6 100644 --- a/frida_mode/include/ranges.h +++ b/frida_mode/include/ranges.h @@ -10,6 +10,8 @@ extern gboolean ranges_inst_jit; void ranges_config(void); void ranges_init(void); +void ranges_print_debug_maps(void); + gboolean range_is_excluded(GumAddress address); void ranges_exclude(); diff --git a/frida_mode/include/seccomp.h b/frida_mode/include/seccomp.h index 2c037ff7..7e8a7d25 100644 --- a/frida_mode/include/seccomp.h +++ b/frida_mode/include/seccomp.h @@ -1,15 +1,95 @@ #ifndef _SECCOMP_H #define _SECCOMP_H -#include <linux/seccomp.h> +#ifndef __APPLE__ -#include "frida-gumjs.h" + #include <stdint.h> + #include <linux/filter.h> -#define SECCOMP_SOCKET_SEND_FD 0x1D3 -#define SECCOMP_SOCKET_RECV_FD 0x1D4 + #include "frida-gumjs.h" -#define SECCOMP_OUTPUT_FILE_FD 0x1D5 -#define SECCOMP_PARENT_EVENT_FD 0x1D6 + /******************************************************************************/ + #define PR_SET_NO_NEW_PRIVS 38 + + #define SECCOMP_SET_MODE_STRICT 0 + #define SECCOMP_SET_MODE_FILTER 1 + #define SECCOMP_GET_ACTION_AVAIL 2 + #define SECCOMP_GET_NOTIF_SIZES 3 + + #define SECCOMP_IOC_MAGIC '!' + #define SECCOMP_IO(nr) _IO(SECCOMP_IOC_MAGIC, nr) + #define SECCOMP_IOR(nr, type) _IOR(SECCOMP_IOC_MAGIC, nr, type) + #define SECCOMP_IOW(nr, type) _IOW(SECCOMP_IOC_MAGIC, nr, type) + #define SECCOMP_IOWR(nr, type) _IOWR(SECCOMP_IOC_MAGIC, nr, type) + + /* Flags for seccomp notification fd ioctl. */ + #define SECCOMP_IOCTL_NOTIF_RECV SECCOMP_IOWR(0, struct seccomp_notif) + #define SECCOMP_IOCTL_NOTIF_SEND SECCOMP_IOWR(1, struct seccomp_notif_resp) + #define SECCOMP_IOCTL_NOTIF_ID_VALID SECCOMP_IOW(2, __u64) + + #define SECCOMP_FILTER_FLAG_NEW_LISTENER (1UL << 3) + #define SECCOMP_RET_ALLOW 0x7fff0000U + #define SECCOMP_RET_USER_NOTIF 0x7fc00000U + + #define SYS_seccomp __NR_seccomp + #ifndef __NR_seccomp + #if defined(__arm__) + #define __NR_seccomp 383 + #elif defined(__aarch64__) + #define __NR_seccomp 277 + #elif defined(__x86_64__) + #define __NR_seccomp 317 + #elif defined(__i386__) + #define __NR_seccomp 354 + #else + #pragma error "Unsupported architecture" + #endif + #endif + + #define SECCOMP_USER_NOTIF_FLAG_CONTINUE (1UL << 0) + +struct seccomp_notif_resp { + + __u64 id; + __s64 val; + __s32 error; + __u32 flags; + +}; + +struct seccomp_data { + + int nr; + __u32 arch; + __u64 instruction_pointer; + __u64 args[6]; + +}; + +struct seccomp_notif { + + __u64 id; + __u32 pid; + __u32 flags; + struct seccomp_data data; + +}; + +struct seccomp_notif_sizes { + + __u16 seccomp_notif; + __u16 seccomp_notif_resp; + __u16 seccomp_data; + +}; + + /******************************************************************************/ + + #define SECCOMP_SOCKET_SEND_FD 0x1D3 + #define SECCOMP_SOCKET_RECV_FD 0x1D4 + + #define SECCOMP_OUTPUT_FILE_FD 0x1D5 + #define SECCOMP_PARENT_EVENT_FD 0x1D6 enum { @@ -319,23 +399,19 @@ enum { }; -extern char *seccomp_filename; - typedef void (*seccomp_child_func_t)(int event_fd, void *ctx); typedef void (*seccomp_filter_callback_t)(struct seccomp_notif * req, struct seccomp_notif_resp *resp, GumReturnAddressArray * frames); -void seccomp_config(void); -void seccomp_init(void); -void seccomp_on_fork(void); -void seccomp_print(char *format, ...); - void seccomp_atomic_set(volatile bool *ptr, bool val); bool seccomp_atomic_try_set(volatile bool *ptr, bool val); void seccomp_atomic_wait(volatile bool *ptr, bool val); +void seccomp_callback_parent(void); +void seccomp_callback_initialize(void); + void seccomp_child_run(seccomp_child_func_t child_func, void *ctx, pid_t *child, int *event_fd); void seccomp_child_wait(int event_fd); @@ -349,6 +425,8 @@ int seccomp_filter_install(pid_t child); void seccomp_filter_child_install(void); void seccomp_filter_run(int fd, seccomp_filter_callback_t callback); +void seccomp_print(char *format, ...); + void seccomp_socket_create(int *sock); void seccomp_socket_send(int sockfd, int fd); int seccomp_socket_recv(int sockfd); @@ -356,4 +434,11 @@ int seccomp_socket_recv(int sockfd); char *seccomp_syscall_lookup(int id); #endif +extern char *seccomp_filename; + +void seccomp_config(void); +void seccomp_init(void); +void seccomp_on_fork(void); + +#endif diff --git a/frida_mode/include/stalker.h b/frida_mode/include/stalker.h index 955f3913..666787e9 100644 --- a/frida_mode/include/stalker.h +++ b/frida_mode/include/stalker.h @@ -3,7 +3,9 @@ #include "frida-gumjs.h" -extern guint stalker_ic_entries; +extern guint stalker_ic_entries; +extern gboolean backpatch_enable; +extern guint stalker_adjacent_blocks; void stalker_config(void); void stalker_init(void); diff --git a/frida_mode/include/util.h b/frida_mode/include/util.h index 525e9d40..4dc05f20 100644 --- a/frida_mode/include/util.h +++ b/frida_mode/include/util.h @@ -3,12 +3,39 @@ #include "frida-gumjs.h" +#include "debug.h" + #define UNUSED_PARAMETER(x) (void)(x) #define IGNORED_RETURN(x) (void)!(x) -guint64 util_read_address(char *key); +guint64 util_read_address(char *key, guint64 default_value); +guint64 util_read_num(char *key, guint64 default_value); +gboolean util_output_enabled(void); +gsize util_rotate(gsize val, gsize shift, gsize size); +gsize util_log2(gsize val); + +#define FOKF(x...) \ + do { \ + \ + if (!util_output_enabled()) { break; } \ + \ + OKF(x); \ + \ + } while (0) + +#define FWARNF(x...) \ + do { \ + \ + WARNF(x); \ + \ + } while (0) -guint64 util_read_num(char *key); +#define FFATAL(x...) \ + do { \ + \ + FATAL(x); \ + \ + } while (0) #endif diff --git a/frida_mode/many-linux/Dockerfile b/frida_mode/many-linux/Dockerfile index 170f0757..6d077dad 100644 --- a/frida_mode/many-linux/Dockerfile +++ b/frida_mode/many-linux/Dockerfile @@ -1,10 +1,6 @@ FROM fridadotre/manylinux-x86_64 -COPY realpath /bin/realpath -RUN chmod +x /bin/realpath - RUN yum -y install xz -RUN yum -y install vim-common WORKDIR /AFLplusplus ENV CFLAGS="\ diff --git a/frida_mode/many-linux/GNUmakefile b/frida_mode/many-linux/GNUmakefile index 03b619f6..9992d4fe 100644 --- a/frida_mode/many-linux/GNUmakefile +++ b/frida_mode/many-linux/GNUmakefile @@ -2,17 +2,22 @@ PWD:=$(shell pwd)/ ROOT:=$(PWD)../../ BUILD_DIR:=$(PWD)build/ -.PHONY: all clean shell +.PHONY: all build docker clean shell -all: - docker build --tag many-afl-frida . +all: docker docker run --rm \ -v $(ROOT):/AFLplusplus \ many-afl-frida \ make -C /AFLplusplus/frida_mode clean all -$(BUILD_DIR): - mkdir -p $@ +build: + docker run --rm \ + -v $(ROOT):/AFLplusplus \ + many-afl-frida \ + make -C /AFLplusplus/frida_mode + +docker: + docker build --tag many-afl-frida . clean: docker images --filter 'dangling=true' -q --no-trunc | xargs -L1 docker rmi --force diff --git a/frida_mode/many-linux/realpath b/frida_mode/many-linux/realpath deleted file mode 100755 index 1fdc49a7..00000000 --- a/frida_mode/many-linux/realpath +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -readlink -f -- "$@" diff --git a/frida_mode/src/asan/asan.c b/frida_mode/src/asan/asan.c index b2e763ca..884bec53 100644 --- a/frida_mode/src/asan/asan.c +++ b/frida_mode/src/asan/asan.c @@ -1,8 +1,8 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "asan.h" +#include "ranges.h" +#include "util.h" static gboolean asan_enabled = FALSE; gboolean asan_initialized = FALSE; @@ -11,12 +11,12 @@ void asan_config(void) { if (getenv("AFL_USE_FASAN") != NULL) { - OKF("Frida ASAN mode enabled"); + FOKF("Frida ASAN mode enabled"); asan_enabled = TRUE; } else { - OKF("Frida ASAN mode disabled"); + FOKF("Frida ASAN mode disabled"); } @@ -33,3 +33,23 @@ void asan_init(void) { } +static gboolean asan_exclude_module(const GumModuleDetails *details, + gpointer user_data) { + + gchar * symbol_name = (gchar *)user_data; + GumAddress address; + + address = gum_module_find_export_by_name(details->name, symbol_name); + if (address == 0) { return TRUE; } + + ranges_add_exclude((GumMemoryRange *)details->range); + return FALSE; + +} + +void asan_exclude_module_by_symbol(gchar *symbol_name) { + + gum_process_enumerate_modules(asan_exclude_module, symbol_name); + +} + diff --git a/frida_mode/src/asan/asan_arm32.c b/frida_mode/src/asan/asan_arm32.c index f5fa4713..21400881 100644 --- a/frida_mode/src/asan/asan_arm32.c +++ b/frida_mode/src/asan/asan_arm32.c @@ -1,7 +1,5 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "asan.h" #include "util.h" @@ -12,7 +10,7 @@ void asan_instrument(const cs_insn *instr, GumStalkerIterator *iterator) { UNUSED_PARAMETER(iterator); if (asan_initialized) { - FATAL("ASAN mode not supported on this architecture"); + FFATAL("ASAN mode not supported on this architecture"); } @@ -20,7 +18,7 @@ void asan_instrument(const cs_insn *instr, GumStalkerIterator *iterator) { void asan_arch_init(void) { - FATAL("ASAN mode not supported on this architecture"); + FFATAL("ASAN mode not supported on this architecture"); } diff --git a/frida_mode/src/asan/asan_arm64.c b/frida_mode/src/asan/asan_arm64.c index 65524e03..b2adfa52 100644 --- a/frida_mode/src/asan/asan_arm64.c +++ b/frida_mode/src/asan/asan_arm64.c @@ -1,8 +1,6 @@ #include <dlfcn.h> #include "frida-gumjs.h" -#include "debug.h" - #include "asan.h" #include "ctx.h" #include "util.h" @@ -86,10 +84,12 @@ void asan_arch_init(void) { asan_storeN = (asan_loadN_t)dlsym(RTLD_DEFAULT, "__asan_storeN"); if (asan_loadN == NULL || asan_storeN == NULL) { - FATAL("Frida ASAN failed to find '__asan_loadN' or '__asan_storeN'"); + FFATAL("Frida ASAN failed to find '__asan_loadN' or '__asan_storeN'"); } + asan_exclude_module_by_symbol("__asan_loadN"); + } #endif diff --git a/frida_mode/src/asan/asan_x64.c b/frida_mode/src/asan/asan_x64.c index 5c12669f..a287ea34 100644 --- a/frida_mode/src/asan/asan_x64.c +++ b/frida_mode/src/asan/asan_x64.c @@ -1,8 +1,6 @@ #include <dlfcn.h> #include "frida-gumjs.h" -#include "debug.h" - #include "asan.h" #include "ctx.h" #include "util.h" @@ -83,10 +81,12 @@ void asan_arch_init(void) { asan_storeN = (asan_loadN_t)dlsym(RTLD_DEFAULT, "__asan_storeN"); if (asan_loadN == NULL || asan_storeN == NULL) { - FATAL("Frida ASAN failed to find '__asan_loadN' or '__asan_storeN'"); + FFATAL("Frida ASAN failed to find '__asan_loadN' or '__asan_storeN'"); } + asan_exclude_module_by_symbol("__asan_loadN"); + } #endif diff --git a/frida_mode/src/asan/asan_x86.c b/frida_mode/src/asan/asan_x86.c index 6d2f9e2b..331d026b 100644 --- a/frida_mode/src/asan/asan_x86.c +++ b/frida_mode/src/asan/asan_x86.c @@ -1,8 +1,6 @@ #include <dlfcn.h> #include "frida-gumjs.h" -#include "debug.h" - #include "asan.h" #include "ctx.h" #include "util.h" @@ -83,10 +81,12 @@ void asan_arch_init(void) { asan_storeN = (asan_loadN_t)dlsym(RTLD_DEFAULT, "__asan_storeN"); if (asan_loadN == NULL || asan_storeN == NULL) { - FATAL("Frida ASAN failed to find '__asan_loadN' or '__asan_storeN'"); + FFATAL("Frida ASAN failed to find '__asan_loadN' or '__asan_storeN'"); } + asan_exclude_module_by_symbol("__asan_loadN"); + } #endif diff --git a/frida_mode/src/cmplog/cmplog.c b/frida_mode/src/cmplog/cmplog.c index ae3116eb..443baa1d 100644 --- a/frida_mode/src/cmplog/cmplog.c +++ b/frida_mode/src/cmplog/cmplog.c @@ -7,8 +7,6 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "util.h" #define DEFAULT_MMAP_MIN_ADDR (32UL << 10) @@ -35,14 +33,28 @@ static gboolean cmplog_range(const GumRangeDetails *details, static gint cmplog_sort(gconstpointer a, gconstpointer b) { - return ((GumMemoryRange *)b)->base_address - - ((GumMemoryRange *)a)->base_address; + GumMemoryRange *ra = (GumMemoryRange *)a; + GumMemoryRange *rb = (GumMemoryRange *)b; + + if (ra->base_address < rb->base_address) { + + return -1; + + } else if (ra->base_address > rb->base_address) { + + return 1; + + } else { + + return 0; + + } } static void cmplog_get_ranges(void) { - OKF("CMPLOG - Collecting ranges"); + FOKF("CMPLOG - Collecting ranges"); cmplog_ranges = g_array_sized_new(false, false, sizeof(GumMemoryRange), 100); gum_process_enumerate_ranges(GUM_PAGE_READ, cmplog_range, cmplog_ranges); @@ -56,7 +68,7 @@ void cmplog_config(void) { void cmplog_init(void) { - OKF("CMPLOG - Enabled [%c]", __afl_cmp_map == NULL ? ' ' : 'X'); + FOKF("CMPLOG - Enabled [%c]", __afl_cmp_map == NULL ? ' ' : 'X'); if (__afl_cmp_map == NULL) { return; } @@ -65,9 +77,9 @@ void cmplog_init(void) { for (guint i = 0; i < cmplog_ranges->len; i++) { GumMemoryRange *range = &g_array_index(cmplog_ranges, GumMemoryRange, i); - OKF("CMPLOG Range - %3u: 0x%016" G_GINT64_MODIFIER - "X - 0x%016" G_GINT64_MODIFIER "X", - i, range->base_address, range->base_address + range->size); + FOKF("CMPLOG Range - %3u: 0x%016" G_GINT64_MODIFIER + "X - 0x%016" G_GINT64_MODIFIER "X", + i, range->base_address, range->base_address + range->size); } @@ -78,14 +90,14 @@ void cmplog_init(void) { hash_yes = g_hash_table_new(g_direct_hash, g_direct_equal); if (hash_yes == NULL) { - FATAL("Failed to g_hash_table_new, errno: %d", errno); + FFATAL("Failed to g_hash_table_new, errno: %d", errno); } hash_no = g_hash_table_new(g_direct_hash, g_direct_equal); if (hash_no == NULL) { - FATAL("Failed to g_hash_table_new, errno: %d", errno); + FFATAL("Failed to g_hash_table_new, errno: %d", errno); } @@ -117,7 +129,7 @@ gboolean cmplog_test_addr(guint64 addr, size_t size) { if (!g_hash_table_add(hash_no, GSIZE_TO_POINTER(addr))) { - FATAL("Failed - g_hash_table_add"); + FFATAL("Failed - g_hash_table_add"); } @@ -127,7 +139,7 @@ gboolean cmplog_test_addr(guint64 addr, size_t size) { if (!g_hash_table_add(hash_yes, GSIZE_TO_POINTER(addr))) { - FATAL("Failed - g_hash_table_add"); + FFATAL("Failed - g_hash_table_add"); } @@ -139,7 +151,7 @@ gboolean cmplog_test_addr(guint64 addr, size_t size) { gboolean cmplog_is_readable(guint64 addr, size_t size) { - if (cmplog_ranges == NULL) FATAL("CMPLOG not initialized"); + if (cmplog_ranges == NULL) FFATAL("CMPLOG not initialized"); /* * The Linux kernel prevents mmap from allocating from the very bottom of the diff --git a/frida_mode/src/cmplog/cmplog_arm32.c b/frida_mode/src/cmplog/cmplog_arm32.c index ac703408..106baa52 100644 --- a/frida_mode/src/cmplog/cmplog_arm32.c +++ b/frida_mode/src/cmplog/cmplog_arm32.c @@ -1,7 +1,5 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "frida_cmplog.h" #include "util.h" @@ -11,7 +9,7 @@ void cmplog_instrument(const cs_insn *instr, GumStalkerIterator *iterator) { UNUSED_PARAMETER(instr); UNUSED_PARAMETER(iterator); if (__afl_cmp_map == NULL) { return; } - FATAL("CMPLOG mode not supported on this architecture"); + FFATAL("CMPLOG mode not supported on this architecture"); } diff --git a/frida_mode/src/cmplog/cmplog_arm64.c b/frida_mode/src/cmplog/cmplog_arm64.c index dd97f38d..515a6256 100644 --- a/frida_mode/src/cmplog/cmplog_arm64.c +++ b/frida_mode/src/cmplog/cmplog_arm64.c @@ -5,6 +5,7 @@ #include "ctx.h" #include "frida_cmplog.h" +#include "instrument.h" #include "util.h" #if defined(__aarch64__) @@ -66,7 +67,7 @@ static gboolean cmplog_read_mem(GumCpuContext *ctx, uint8_t size, *val = *((guint64 *)GSIZE_TO_POINTER(address)); return TRUE; default: - FATAL("Invalid operand size: %d\n", size); + FFATAL("Invalid operand size: %d\n", size); } @@ -88,7 +89,7 @@ static gboolean cmplog_get_operand_value(GumCpuContext *context, case ARM64_OP_MEM: return cmplog_read_mem(context, ctx->size, &ctx->mem, val); default: - FATAL("Invalid operand type: %d\n", ctx->type); + FFATAL("Invalid operand type: %d\n", ctx->type); } @@ -104,30 +105,45 @@ static void cmplog_call_callout(GumCpuContext *context, gpointer user_data) { gsize x0 = ctx_read_reg(context, ARM64_REG_X0); gsize x1 = ctx_read_reg(context, ARM64_REG_X1); - if (((G_MAXULONG - x0) < 32) || ((G_MAXULONG - x1) < 32)) return; + if (((G_MAXULONG - x0) < 31) || ((G_MAXULONG - x1) < 31)) return; - if (!cmplog_is_readable(x0, 32) || !cmplog_is_readable(x1, 32)) return; + if (!cmplog_is_readable(x0, 31) || !cmplog_is_readable(x1, 31)) return; void *ptr1 = GSIZE_TO_POINTER(x0); void *ptr2 = GSIZE_TO_POINTER(x1); - uintptr_t k = address; + guint64 k = instrument_get_offset_hash(GUM_ADDRESS(address)); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) { + + __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; + __afl_cmp_map->headers[k].hits = 0; + + } - __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; + u32 hits = 0; + + if (__afl_cmp_map->headers[k].hits == 0) { + + __afl_cmp_map->headers[k].shape = 30; + + } else { + + hits = __afl_cmp_map->headers[k].hits; + + } - u32 hits = __afl_cmp_map->headers[k].hits; __afl_cmp_map->headers[k].hits = hits + 1; - __afl_cmp_map->headers[k].shape = 31; + __afl_cmp_map->headers[k].shape = 30; hits &= CMP_MAP_RTN_H - 1; + ((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0_len = 31; + ((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1_len = 31; gum_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0, ptr1, - 32); + 31); gum_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1, ptr2, - 32); + 31); } @@ -147,7 +163,7 @@ static void cmplog_instrument_put_operand(cmplog_ctx_t *ctx, gum_memcpy(&ctx->mem, &operand->mem, sizeof(arm64_op_mem)); break; default: - FATAL("Invalid operand type: %d\n", operand->type); + FFATAL("Invalid operand type: %d\n", operand->type); } @@ -193,12 +209,23 @@ static void cmplog_handle_cmp_sub(GumCpuContext *context, gsize operand1, k = (k >> 4) ^ (k << 8); k &= CMP_MAP_W - 1; - __afl_cmp_map->headers[k].type = CMP_TYPE_INS; + if (__afl_cmp_map->headers[k].type != CMP_TYPE_INS) + __afl_cmp_map->headers[k].hits = 0; - u32 hits = __afl_cmp_map->headers[k].hits; - __afl_cmp_map->headers[k].hits = hits + 1; + u32 hits = 0; + + if (__afl_cmp_map->headers[k].hits == 0) { + + __afl_cmp_map->headers[k].type = CMP_TYPE_INS; + __afl_cmp_map->headers[k].shape = (size - 1); + + } else { - __afl_cmp_map->headers[k].shape = (size - 1); + hits = __afl_cmp_map->headers[k].hits; + + } + + __afl_cmp_map->headers[k].hits = hits + 1; hits &= CMP_MAP_H - 1; __afl_cmp_map->log[k][hits].v0 = operand1; diff --git a/frida_mode/src/cmplog/cmplog_x64.c b/frida_mode/src/cmplog/cmplog_x64.c index 0d18767a..7d515336 100644 --- a/frida_mode/src/cmplog/cmplog_x64.c +++ b/frida_mode/src/cmplog/cmplog_x64.c @@ -5,6 +5,7 @@ #include "ctx.h" #include "frida_cmplog.h" +#include "instrument.h" #include "util.h" #if defined(__x86_64__) @@ -61,7 +62,7 @@ static gboolean cmplog_read_mem(GumCpuContext *ctx, uint8_t size, *val = *((guint64 *)GSIZE_TO_POINTER(address)); return TRUE; default: - FATAL("Invalid operand size: %d\n", size); + FFATAL("Invalid operand size: %d\n", size); } @@ -83,7 +84,7 @@ static gboolean cmplog_get_operand_value(GumCpuContext *context, case X86_OP_MEM: return cmplog_read_mem(context, ctx->size, &ctx->mem, val); default: - FATAL("Invalid operand type: %d\n", ctx->type); + FFATAL("Invalid operand type: %d\n", ctx->type); } @@ -99,30 +100,43 @@ static void cmplog_call_callout(GumCpuContext *context, gpointer user_data) { gsize rdi = ctx_read_reg(context, X86_REG_RDI); gsize rsi = ctx_read_reg(context, X86_REG_RSI); - if (((G_MAXULONG - rdi) < 32) || ((G_MAXULONG - rsi) < 32)) return; + if (((G_MAXULONG - rdi) < 31) || ((G_MAXULONG - rsi) < 31)) return; - if (!cmplog_is_readable(rdi, 32) || !cmplog_is_readable(rsi, 32)) return; + if (!cmplog_is_readable(rdi, 31) || !cmplog_is_readable(rsi, 31)) return; void *ptr1 = GSIZE_TO_POINTER(rdi); void *ptr2 = GSIZE_TO_POINTER(rsi); - uintptr_t k = address; + guint64 k = instrument_get_offset_hash(GUM_ADDRESS(address)); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) { - __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; + __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; + __afl_cmp_map->headers[k].hits = 0; - u32 hits = __afl_cmp_map->headers[k].hits; - __afl_cmp_map->headers[k].hits = hits + 1; + } + + u32 hits = 0; + + if (__afl_cmp_map->headers[k].hits == 0) { + + __afl_cmp_map->headers[k].shape = 30; + + } else { + + hits = __afl_cmp_map->headers[k].hits; + + } - __afl_cmp_map->headers[k].shape = 31; + __afl_cmp_map->headers[k].hits = hits + 1; hits &= CMP_MAP_RTN_H - 1; + ((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0_len = 31; + ((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1_len = 31; gum_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0, ptr1, - 32); + 31); gum_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1, ptr2, - 32); + 31); } @@ -143,7 +157,7 @@ static void cmplog_instrument_put_operand(cmplog_ctx_t *ctx, gum_memcpy(&ctx->mem, &operand->mem, sizeof(x86_op_mem)); break; default: - FATAL("Invalid operand type: %d\n", operand->type); + FFATAL("Invalid operand type: %d\n", operand->type); } @@ -179,13 +193,23 @@ static void cmplog_handle_cmp_sub(GumCpuContext *context, gsize operand1, k = (k >> 4) ^ (k << 8); k &= CMP_MAP_W - 7; - __afl_cmp_map->headers[k].type = CMP_TYPE_INS; + if (__afl_cmp_map->headers[k].type != CMP_TYPE_INS) + __afl_cmp_map->headers[k].hits = 0; - u32 hits = __afl_cmp_map->headers[k].hits; - __afl_cmp_map->headers[k].hits = hits + 1; + u32 hits = 0; + + if (__afl_cmp_map->headers[k].hits == 0) { + + __afl_cmp_map->headers[k].type = CMP_TYPE_INS; + __afl_cmp_map->headers[k].shape = (size - 1); - __afl_cmp_map->headers[k].shape = (size - 1); + } else { + hits = __afl_cmp_map->headers[k].hits; + + } + + __afl_cmp_map->headers[k].hits = hits + 1; hits &= CMP_MAP_H - 1; __afl_cmp_map->log[k][hits].v0 = operand1; __afl_cmp_map->log[k][hits].v1 = operand2; diff --git a/frida_mode/src/cmplog/cmplog_x86.c b/frida_mode/src/cmplog/cmplog_x86.c index dd666c34..4a747417 100644 --- a/frida_mode/src/cmplog/cmplog_x86.c +++ b/frida_mode/src/cmplog/cmplog_x86.c @@ -5,6 +5,7 @@ #include "ctx.h" #include "frida_cmplog.h" +#include "instrument.h" #include "util.h" #if defined(__i386__) @@ -58,7 +59,7 @@ static gboolean cmplog_read_mem(GumCpuContext *ctx, uint8_t size, *val = *((guint32 *)GSIZE_TO_POINTER(address)); return TRUE; default: - FATAL("Invalid operand size: %d\n", size); + FFATAL("Invalid operand size: %d\n", size); } @@ -80,7 +81,7 @@ static gboolean cmplog_get_operand_value(GumCpuContext *context, case X86_OP_MEM: return cmplog_read_mem(context, ctx->size, &ctx->mem, val); default: - FATAL("Invalid operand type: %d\n", ctx->type); + FFATAL("Invalid operand type: %d\n", ctx->type); } @@ -104,30 +105,43 @@ static void cmplog_call_callout(GumCpuContext *context, gpointer user_data) { gsize arg1 = esp[0]; gsize arg2 = esp[1]; - if (((G_MAXULONG - arg1) < 32) || ((G_MAXULONG - arg2) < 32)) return; + if (((G_MAXULONG - arg1) < 31) || ((G_MAXULONG - arg2) < 31)) return; - if (!cmplog_is_readable(arg1, 32) || !cmplog_is_readable(arg2, 32)) return; + if (!cmplog_is_readable(arg1, 31) || !cmplog_is_readable(arg2, 31)) return; void *ptr1 = GSIZE_TO_POINTER(arg1); void *ptr2 = GSIZE_TO_POINTER(arg2); - uintptr_t k = address; + guint64 k = instrument_get_offset_hash(GUM_ADDRESS(address)); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) { - __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; + __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; + __afl_cmp_map->headers[k].hits = 0; - u32 hits = __afl_cmp_map->headers[k].hits; - __afl_cmp_map->headers[k].hits = hits + 1; + } + + u32 hits = 0; + + if (__afl_cmp_map->headers[k].hits == 0) { + + __afl_cmp_map->headers[k].shape = 30; + + } else { + + hits = __afl_cmp_map->headers[k].hits; + + } - __afl_cmp_map->headers[k].shape = 31; + __afl_cmp_map->headers[k].hits = hits + 1; hits &= CMP_MAP_RTN_H - 1; + ((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0_len = 31; + ((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1_len = 31; gum_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0, ptr1, - 32); + 31); gum_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1, ptr2, - 32); + 31); } @@ -148,7 +162,7 @@ static void cmplog_instrument_put_operand(cmplog_ctx_t *ctx, gum_memcpy(&ctx->mem, &operand->mem, sizeof(x86_op_mem)); break; default: - FATAL("Invalid operand type: %d\n", operand->type); + FFATAL("Invalid operand type: %d\n", operand->type); } @@ -184,12 +198,23 @@ static void cmplog_handle_cmp_sub(GumCpuContext *context, gsize operand1, k = (k >> 4) ^ (k << 8); k &= CMP_MAP_W - 1; - __afl_cmp_map->headers[k].type = CMP_TYPE_INS; + if (__afl_cmp_map->headers[k].type != CMP_TYPE_INS) + __afl_cmp_map->headers[k].hits = 0; - u32 hits = __afl_cmp_map->headers[k].hits; - __afl_cmp_map->headers[k].hits = hits + 1; + u32 hits = 0; + + if (__afl_cmp_map->headers[k].hits == 0) { + + __afl_cmp_map->headers[k].type = CMP_TYPE_INS; + __afl_cmp_map->headers[k].shape = (size - 1); - __afl_cmp_map->headers[k].shape = (size - 1); + } else { + + hits = __afl_cmp_map->headers[k].hits; + + } + + __afl_cmp_map->headers[k].hits = hits + 1; hits &= CMP_MAP_H - 1; __afl_cmp_map->log[k][hits].v0 = operand1; @@ -203,7 +228,7 @@ static void cmplog_cmp_sub_callout(GumCpuContext *context, gpointer user_data) { gsize operand1; gsize operand2; - if (ctx->operand1.size != ctx->operand2.size) FATAL("Operand size mismatch"); + if (ctx->operand1.size != ctx->operand2.size) FFATAL("Operand size mismatch"); if (!cmplog_get_operand_value(context, &ctx->operand1, &operand1)) { return; } if (!cmplog_get_operand_value(context, &ctx->operand2, &operand2)) { return; } diff --git a/frida_mode/src/ctx/ctx_arm32.c b/frida_mode/src/ctx/ctx_arm32.c index 9fc70fb4..28fc706b 100644 --- a/frida_mode/src/ctx/ctx_arm32.c +++ b/frida_mode/src/ctx/ctx_arm32.c @@ -1,14 +1,13 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "ctx.h" +#include "util.h" #if defined(__arm__) gsize ctx_read_reg(GumArmCpuContext *ctx, arm_reg reg) { - FATAL("ctx_read_reg unimplemented for this architecture"); + FFATAL("ctx_read_reg unimplemented for this architecture"); } diff --git a/frida_mode/src/ctx/ctx_arm64.c b/frida_mode/src/ctx/ctx_arm64.c index a735401b..63b6cf09 100644 --- a/frida_mode/src/ctx/ctx_arm64.c +++ b/frida_mode/src/ctx/ctx_arm64.c @@ -1,8 +1,7 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "ctx.h" +#include "util.h" #if defined(__aarch64__) @@ -174,7 +173,7 @@ gsize ctx_read_reg(GumArm64CpuContext *ctx, arm64_reg reg) { ARM64_REG_64(ARM64_REG_SP, ctx->sp) default: - FATAL("Failed to read register: %d", reg); + FFATAL("Failed to read register: %d", reg); return 0; } @@ -206,7 +205,7 @@ size_t ctx_get_size(const cs_insn *instr, cs_arm64_op *operand) { } mnemonic_len = strlen(instr->mnemonic); - if (mnemonic_len == 0) { FATAL("No mnemonic found"); }; + if (mnemonic_len == 0) { FFATAL("No mnemonic found"); }; char last = instr->mnemonic[mnemonic_len - 1]; switch (last) { @@ -252,14 +251,14 @@ size_t ctx_get_size(const cs_insn *instr, cs_arm64_op *operand) { if (mnemonic_len < 3) { - FATAL("VAS Mnemonic too short: %s\n", instr->mnemonic); + FFATAL("VAS Mnemonic too short: %s\n", instr->mnemonic); } vas_digit = instr->mnemonic[2]; if (vas_digit < '0' || vas_digit > '9') { - FATAL("VAS Mnemonic digit out of range: %s\n", instr->mnemonic); + FFATAL("VAS Mnemonic digit out of range: %s\n", instr->mnemonic); } @@ -293,7 +292,7 @@ size_t ctx_get_size(const cs_insn *instr, cs_arm64_op *operand) { case ARM64_VAS_16B: return 16 * count_byte; default: - FATAL("Unexpected VAS type: %s %d", instr->mnemonic, operand->vas); + FFATAL("Unexpected VAS type: %s %d", instr->mnemonic, operand->vas); } diff --git a/frida_mode/src/ctx/ctx_x64.c b/frida_mode/src/ctx/ctx_x64.c index da5cb13a..02423416 100644 --- a/frida_mode/src/ctx/ctx_x64.c +++ b/frida_mode/src/ctx/ctx_x64.c @@ -1,8 +1,7 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "ctx.h" +#include "util.h" #if defined(__x86_64__) @@ -121,7 +120,7 @@ gsize ctx_read_reg(GumX64CpuContext *ctx, x86_reg reg) { X86_REG_64(X86_REG_RIP, ctx->rip) default: - FATAL("Failed to read register: %d", reg); + FFATAL("Failed to read register: %d", reg); return 0; } diff --git a/frida_mode/src/ctx/ctx_x86.c b/frida_mode/src/ctx/ctx_x86.c index 1a587702..438e1fde 100644 --- a/frida_mode/src/ctx/ctx_x86.c +++ b/frida_mode/src/ctx/ctx_x86.c @@ -1,8 +1,7 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "ctx.h" +#include "util.h" #if defined(__i386__) @@ -72,7 +71,7 @@ gsize ctx_read_reg(GumIA32CpuContext *ctx, x86_reg reg) { X86_REG_32(X86_REG_EIP, ctx->eip) default: - FATAL("Failed to read register: %d", reg); + FFATAL("Failed to read register: %d", reg); return 0; } diff --git a/frida_mode/src/entry.c b/frida_mode/src/entry.c index 186ddd3a..de645fdb 100644 --- a/frida_mode/src/entry.c +++ b/frida_mode/src/entry.c @@ -1,8 +1,10 @@ #include <dlfcn.h> -#include "frida-gumjs.h" +#if defined(__linux__) && !defined(__ANDROID__) + #include <sys/prctl.h> +#endif -#include "debug.h" +#include "frida-gumjs.h" #include "entry.h" #include "instrument.h" @@ -16,33 +18,61 @@ extern void __afl_manual_init(); guint64 entry_point = 0; +gboolean traceable = FALSE; gboolean entry_compiled = FALSE; gboolean entry_run = FALSE; static void entry_launch(void) { - OKF("Entry point reached"); + FOKF("Entry point reached"); __afl_manual_init(); /* Child here */ entry_run = TRUE; + entry_on_fork(); instrument_on_fork(); seccomp_on_fork(); stats_on_fork(); } +#if defined(__linux__) && defined(PR_SET_PTRACER) && !defined(__ANDROID__) +void entry_on_fork(void) { + + if (traceable) { + + if (prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY) < 0) { + + FFATAL("Failed to PR_SET_PTRACER"); + + } + + } + +} + +#else +void entry_on_fork(void) { + + if (traceable) { FWARNF("AFL_FRIDA_TRACEABLE unsupported"); } + +} + +#endif + void entry_config(void) { - entry_point = util_read_address("AFL_ENTRYPOINT"); + entry_point = util_read_address("AFL_ENTRYPOINT", 0); + if (getenv("AFL_FRIDA_TRACEABLE") != NULL) { traceable = TRUE; } } void entry_init(void) { - OKF("entry_point: 0x%016" G_GINT64_MODIFIER "X", entry_point); + FOKF("entry_point: 0x%016" G_GINT64_MODIFIER "X", entry_point); + FOKF("dumpable: [%c]", traceable ? 'X' : ' '); - if (dlopen(NULL, RTLD_NOW) == NULL) { FATAL("Failed to dlopen: %d", errno); } + if (dlopen(NULL, RTLD_NOW) == NULL) { FFATAL("Failed to dlopen: %d", errno); } } @@ -64,7 +94,7 @@ static void entry_callout(GumCpuContext *cpu_context, gpointer user_data) { void entry_prologue(GumStalkerIterator *iterator, GumStalkerOutput *output) { UNUSED_PARAMETER(output); - OKF("AFL_ENTRYPOINT reached"); + FOKF("AFL_ENTRYPOINT reached"); if (persistent_start == 0) { diff --git a/frida_mode/src/instrument/instrument.c b/frida_mode/src/instrument/instrument.c index fd0982f8..8ee21f5b 100644 --- a/frida_mode/src/instrument/instrument.c +++ b/frida_mode/src/instrument/instrument.c @@ -6,7 +6,6 @@ #include "frida-gumjs.h" #include "config.h" -#include "debug.h" #include "hash.h" #include "asan.h" @@ -69,7 +68,8 @@ guint64 instrument_get_offset_hash(GumAddress current_rip) { guint64 area_offset = hash64((unsigned char *)¤t_rip, sizeof(GumAddress), instrument_hash_seed); - return area_offset &= MAP_SIZE - 1; + gsize map_size_pow2 = util_log2(__afl_map_size); + return area_offset &= ((1 << map_size_pow2) - 1); } @@ -135,8 +135,8 @@ __attribute__((hot)) static void on_basic_block(GumCpuContext *context, previous_rip = current_rip; previous_end = current_end; - instrument_previous_pc = ((current_pc & (MAP_SIZE - 1) >> 1)) | - ((current_pc & 0x1) << (MAP_SIZE_POW2 - 1)); + gsize map_size_pow2 = util_log2(__afl_map_size); + instrument_previous_pc = util_rotate(current_pc, 1, map_size_pow2); } @@ -193,7 +193,20 @@ static void instrument_basic_block(GumStalkerIterator *iterator, instrument_debug_start(instr->address, output); instrument_coverage_start(instr->address); +#if defined(__arm__) + if (output->encoding == GUM_INSTRUCTION_SPECIAL) { + + prefetch_write(GSIZE_TO_POINTER(instr->address + 1)); + + } else { + + prefetch_write(GSIZE_TO_POINTER(instr->address)); + + } + +#else prefetch_write(GSIZE_TO_POINTER(instr->address)); +#endif if (likely(!excluded)) { @@ -213,7 +226,7 @@ static void instrument_basic_block(GumStalkerIterator *iterator, } - instrument_debug_instruction(instr->address, instr->size); + instrument_debug_instruction(instr->address, instr->size, output); if (likely(!excluded)) { @@ -246,7 +259,7 @@ void instrument_config(void) { instrument_tracing = (getenv("AFL_FRIDA_INST_TRACE") != NULL); instrument_unique = (getenv("AFL_FRIDA_INST_TRACE_UNIQUE") != NULL); instrument_use_fixed_seed = (getenv("AFL_FRIDA_INST_SEED") != NULL); - instrument_fixed_seed = util_read_num("AFL_FRIDA_INST_SEED"); + instrument_fixed_seed = util_read_num("AFL_FRIDA_INST_SEED", 0); instrument_coverage_unstable_filename = (getenv("AFL_FRIDA_INST_UNSTABLE_COVERAGE_FILE")); @@ -261,14 +274,14 @@ void instrument_init(void) { if (!instrument_is_coverage_optimize_supported()) instrument_optimize = false; - OKF("Instrumentation - optimize [%c]", instrument_optimize ? 'X' : ' '); - OKF("Instrumentation - tracing [%c]", instrument_tracing ? 'X' : ' '); - OKF("Instrumentation - unique [%c]", instrument_unique ? 'X' : ' '); - OKF("Instrumentation - fixed seed [%c] [0x%016" G_GINT64_MODIFIER "x]", - instrument_use_fixed_seed ? 'X' : ' ', instrument_fixed_seed); - OKF("Instrumentation - unstable coverage [%c] [%s]", - instrument_coverage_unstable_filename == NULL ? ' ' : 'X', - instrument_coverage_unstable_filename); + FOKF("Instrumentation - optimize [%c]", instrument_optimize ? 'X' : ' '); + FOKF("Instrumentation - tracing [%c]", instrument_tracing ? 'X' : ' '); + FOKF("Instrumentation - unique [%c]", instrument_unique ? 'X' : ' '); + FOKF("Instrumentation - fixed seed [%c] [0x%016" G_GINT64_MODIFIER "x]", + instrument_use_fixed_seed ? 'X' : ' ', instrument_fixed_seed); + FOKF("Instrumentation - unstable coverage [%c] [%s]", + instrument_coverage_unstable_filename == NULL ? ' ' : 'X', + instrument_coverage_unstable_filename); if (instrument_tracing && instrument_optimize) { @@ -304,7 +317,8 @@ void instrument_init(void) { if (instrument_unique) { - int shm_id = shmget(IPC_PRIVATE, MAP_SIZE, IPC_CREAT | IPC_EXCL | 0600); + int shm_id = + shmget(IPC_PRIVATE, __afl_map_size, IPC_CREAT | IPC_EXCL | 0600); if (shm_id < 0) { FATAL("shm_id < 0 - errno: %d\n", errno); } edges_notified = shmat(shm_id, NULL, 0); @@ -321,7 +335,7 @@ void instrument_init(void) { } /* Clear it, not sure it's necessary, just seems like good practice */ - memset(edges_notified, '\0', MAP_SIZE); + memset(edges_notified, '\0', __afl_map_size); } @@ -341,15 +355,22 @@ void instrument_init(void) { * parallel fuzzing. The seed itself, doesn't have to be random, it * just needs to be different for each instance. */ - instrument_hash_seed = g_get_monotonic_time() ^ - (((guint64)getpid()) << 32) ^ syscall(SYS_gettid); + guint64 tid; +#if defined(__APPLE__) + pthread_threadid_np(NULL, &tid); +#else + tid = syscall(SYS_gettid); +#endif + instrument_hash_seed = + g_get_monotonic_time() ^ (((guint64)getpid()) << 32) ^ tid; } - OKF("Instrumentation - seed [0x%016" G_GINT64_MODIFIER "x]", - instrument_hash_seed); + FOKF("Instrumentation - seed [0x%016" G_GINT64_MODIFIER "x]", + instrument_hash_seed); instrument_hash_zero = instrument_get_offset_hash(0); + instrument_coverage_optimize_init(); instrument_debug_init(); instrument_coverage_init(); asan_init(); diff --git a/frida_mode/src/instrument/instrument_arm32.c b/frida_mode/src/instrument/instrument_arm32.c index 0e15940a..16e8eaab 100644 --- a/frida_mode/src/instrument/instrument_arm32.c +++ b/frida_mode/src/instrument/instrument_arm32.c @@ -1,7 +1,5 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "instrument.h" #include "util.h" @@ -18,13 +16,27 @@ void instrument_coverage_optimize(const cs_insn * instr, UNUSED_PARAMETER(instr); UNUSED_PARAMETER(output); - FATAL("Optimized coverage not supported on this architecture"); + FFATAL("Optimized coverage not supported on this architecture"); + +} + +void instrument_coverage_optimize_init(void) { + + FWARNF("Optimized coverage not supported on this architecture"); } void instrument_flush(GumStalkerOutput *output) { - gum_arm_writer_flush(output->writer.arm); + if (output->encoding == GUM_INSTRUCTION_SPECIAL) { + + gum_thumb_writer_flush(output->writer.thumb); + + } else { + + gum_arm_writer_flush(output->writer.arm); + + } } diff --git a/frida_mode/src/instrument/instrument_arm64.c b/frida_mode/src/instrument/instrument_arm64.c index cf37e048..0f635458 100644 --- a/frida_mode/src/instrument/instrument_arm64.c +++ b/frida_mode/src/instrument/instrument_arm64.c @@ -1,7 +1,6 @@ #include "frida-gumjs.h" #include "config.h" -#include "debug.h" #include "instrument.h" @@ -95,6 +94,10 @@ void instrument_coverage_optimize(const cs_insn * instr, } +void instrument_coverage_optimize_init(void) { + +} + void instrument_flush(GumStalkerOutput *output) { gum_arm64_writer_flush(output->writer.arm64); diff --git a/frida_mode/src/instrument/instrument_coverage.c b/frida_mode/src/instrument/instrument_coverage.c index 46c816bc..c1984eb2 100644 --- a/frida_mode/src/instrument/instrument_coverage.c +++ b/frida_mode/src/instrument/instrument_coverage.c @@ -5,8 +5,6 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "instrument.h" #include "util.h" @@ -239,7 +237,7 @@ static void instrument_coverage_mark(void *key, void *value, void *user_data) { } -static void coverage_write(void *data, size_t size) { +static void coverage_write(int fd, void *data, size_t size) { ssize_t written; size_t remain = size; @@ -247,11 +245,11 @@ static void coverage_write(void *data, size_t size) { for (char *cursor = (char *)data; remain > 0; remain -= written, cursor += written) { - written = write(normal_coverage_fd, cursor, remain); + written = write(fd, cursor, remain); if (written < 0) { - FATAL("Coverage - Failed to write: %s (%d)\n", (char *)data, errno); + FFATAL("Coverage - Failed to write: %s (%d)\n", (char *)data, errno); } @@ -259,7 +257,7 @@ static void coverage_write(void *data, size_t size) { } -static void coverage_format(char *format, ...) { +static void coverage_format(int fd, char *format, ...) { va_list ap; char buffer[4096] = {0}; @@ -274,11 +272,11 @@ static void coverage_format(char *format, ...) { len = strnlen(buffer, sizeof(buffer)); - coverage_write(buffer, len); + coverage_write(fd, buffer, len); } -static void coverage_write_modules(GArray *coverage_modules) { +static void coverage_write_modules(int fd, GArray *coverage_modules) { guint emitted = 0; for (guint i = 0; i < coverage_modules->len; i++) { @@ -287,16 +285,16 @@ static void coverage_write_modules(GArray *coverage_modules) { &g_array_index(coverage_modules, coverage_range_t, i); if (module->count == 0) continue; - coverage_format("%3u, ", emitted); - coverage_format("%016" G_GINT64_MODIFIER "X, ", module->base_address); - coverage_format("%016" G_GINT64_MODIFIER "X, ", module->limit); + coverage_format(fd, "%3u, ", emitted); + coverage_format(fd, "%016" G_GINT64_MODIFIER "X, ", module->base_address); + coverage_format(fd, "%016" G_GINT64_MODIFIER "X, ", module->limit); /* entry */ - coverage_format("%016" G_GINT64_MODIFIER "X, ", 0); + coverage_format(fd, "%016" G_GINT64_MODIFIER "X, ", 0); /* checksum */ - coverage_format("%016" G_GINT64_MODIFIER "X, ", 0); + coverage_format(fd, "%016" G_GINT64_MODIFIER "X, ", 0); /* timestamp */ - coverage_format("%08" G_GINT32_MODIFIER "X, ", 0); - coverage_format("%s\n", module->path); + coverage_format(fd, "%08" G_GINT32_MODIFIER "X, ", 0); + coverage_format(fd, "%s\n", module->path); emitted++; } @@ -306,7 +304,7 @@ static void coverage_write_modules(GArray *coverage_modules) { static void coverage_write_events(void *key, void *value, void *user_data) { UNUSED_PARAMETER(key); - UNUSED_PARAMETER(user_data); + int fd = *((int *)user_data); normal_coverage_data_t *val = (normal_coverage_data_t *)value; if (val->module == NULL) { return; } @@ -319,20 +317,20 @@ static void coverage_write_events(void *key, void *value, void *user_data) { }; - coverage_write(&evt, sizeof(coverage_event_t)); + coverage_write(fd, &evt, sizeof(coverage_event_t)); } -static void coverage_write_header(guint coverage_marked_modules) { +static void coverage_write_header(int fd, guint coverage_marked_modules) { char version[] = "DRCOV VERSION: 2\n"; char flavour[] = "DRCOV FLAVOR: frida\n"; char columns[] = "Columns: id, base, end, entry, checksum, timestamp, path\n"; - coverage_write(version, sizeof(version) - 1); - coverage_write(flavour, sizeof(flavour) - 1); - coverage_format("Module Table: version 2, count %u\n", + coverage_write(fd, version, sizeof(version) - 1); + coverage_write(fd, flavour, sizeof(flavour) - 1); + coverage_format(fd, "Module Table: version 2, count %u\n", coverage_marked_modules); - coverage_write(columns, sizeof(columns) - 1); + coverage_write(fd, columns, sizeof(columns) - 1); } @@ -371,7 +369,7 @@ static void instrument_coverage_normal_run() { if (close(normal_coverage_pipes[STDOUT_FILENO]) != 0) { - FATAL("Failed to close parent read pipe"); + FFATAL("Failed to close parent read pipe"); } @@ -379,7 +377,7 @@ static void instrument_coverage_normal_run() { g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, g_free); if (coverage_hash == NULL) { - FATAL("Failed to g_hash_table_new, errno: %d", errno); + FFATAL("Failed to g_hash_table_new, errno: %d", errno); } @@ -396,7 +394,7 @@ static void instrument_coverage_normal_run() { } - if (bytes != 0) { FATAL("Coverage data truncated"); } + if (bytes != 0) { FFATAL("Coverage data truncated"); } instrument_coverage_print("Coverage - Preparing\n"); @@ -414,10 +412,11 @@ static void instrument_coverage_normal_run() { instrument_coverage_print("Coverage - Marked Modules: %u\n", coverage_marked_modules); - coverage_write_header(coverage_marked_modules); - coverage_write_modules(coverage_modules); - coverage_format("BB Table: %u bbs\n", ctx.count); - g_hash_table_foreach(coverage_hash, coverage_write_events, NULL); + coverage_write_header(normal_coverage_fd, coverage_marked_modules); + coverage_write_modules(normal_coverage_fd, coverage_modules); + coverage_format(normal_coverage_fd, "BB Table: %u bbs\n", ctx.count); + g_hash_table_foreach(coverage_hash, coverage_write_events, + &normal_coverage_fd); g_hash_table_unref(coverage_hash); @@ -435,7 +434,7 @@ static GArray *instrument_coverage_unstable_read_unstable_ids(void) { if (!g_file_get_contents(unstable_coverage_fuzzer_stats, &contents, &length, NULL)) { - FATAL("Failed to read fuzzer_stats"); + FFATAL("Failed to read fuzzer_stats"); } @@ -526,7 +525,7 @@ static GHashTable *instrument_collect_unstable_blocks( GHashTable *child = (GHashTable *)g_hash_table_lookup(unstable_coverage_hash, *id); - if (child == NULL) { FATAL("Failed to find edge ID"); } + if (child == NULL) { FFATAL("Failed to find edge ID"); } GHashTableIter iter = {0}; gpointer value; @@ -565,7 +564,7 @@ static void instrument_coverage_unstable_run(void) { if (close(unstable_coverage_pipes[STDOUT_FILENO]) != 0) { - FATAL("Failed to close parent read pipe"); + FFATAL("Failed to close parent read pipe"); } @@ -573,7 +572,7 @@ static void instrument_coverage_unstable_run(void) { g_direct_hash, g_direct_equal, NULL, (GDestroyNotify)g_hash_table_unref); if (unstable_coverage_hash == NULL) { - FATAL("Failed to g_hash_table_new, errno: %d", errno); + FFATAL("Failed to g_hash_table_new, errno: %d", errno); } @@ -599,7 +598,7 @@ static void instrument_coverage_unstable_run(void) { if (!g_hash_table_insert(unstable_coverage_hash, GSIZE_TO_POINTER(value->edge), hash_value)) { - FATAL("Entry already in hashtable"); + FFATAL("Entry already in hashtable"); } @@ -613,7 +612,7 @@ static void instrument_coverage_unstable_run(void) { } - if (bytes != 0) { FATAL("Unstable coverage data truncated"); } + if (bytes != 0) { FFATAL("Unstable coverage data truncated"); } instrument_coverage_print("Coverage - Preparing\n"); @@ -638,10 +637,11 @@ static void instrument_coverage_unstable_run(void) { instrument_coverage_print("Coverage - Marked Modules: %u\n", coverage_marked_modules); - coverage_write_header(coverage_marked_modules); - coverage_write_modules(coverage_modules); - coverage_format("BB Table: %u bbs\n", ctx.count); - g_hash_table_foreach(unstable_blocks, coverage_write_events, NULL); + coverage_write_header(unstable_coverage_fd, coverage_marked_modules); + coverage_write_modules(unstable_coverage_fd, coverage_modules); + coverage_format(unstable_coverage_fd, "BB Table: %u bbs\n", ctx.count); + g_hash_table_foreach(unstable_blocks, coverage_write_events, + &unstable_coverage_fd); g_hash_table_unref(unstable_blocks); g_array_free(unstable_edge_ids, TRUE); @@ -659,33 +659,33 @@ void instrument_coverage_config(void) { void instrument_coverage_normal_init(void) { - OKF("Coverage - enabled [%c]", - instrument_coverage_filename == NULL ? ' ' : 'X'); + FOKF("Coverage - enabled [%c]", + instrument_coverage_filename == NULL ? ' ' : 'X'); if (instrument_coverage_filename == NULL) { return; } - OKF("Coverage - file [%s]", instrument_coverage_filename); + FOKF("Coverage - file [%s]", instrument_coverage_filename); char *path = g_canonicalize_filename(instrument_coverage_filename, g_get_current_dir()); - OKF("Coverage - path [%s]", path); + FOKF("Coverage - path [%s]", path); normal_coverage_fd = open(path, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); if (normal_coverage_fd < 0) { - FATAL("Failed to open coverage file '%s'", path); + FFATAL("Failed to open coverage file '%s'", path); } g_free(path); - if (pipe(normal_coverage_pipes) != 0) { FATAL("Failed to create pipes"); } + if (pipe(normal_coverage_pipes) != 0) { FFATAL("Failed to create pipes"); } pid_t pid = fork(); - if (pid == -1) { FATAL("Failed to start coverage process"); } + if (pid == -1) { FFATAL("Failed to start coverage process"); } if (pid == 0) { @@ -697,13 +697,13 @@ void instrument_coverage_normal_init(void) { if (close(normal_coverage_fd) < 0) { - FATAL("Failed to close coverage output file"); + FFATAL("Failed to close coverage output file"); } if (close(normal_coverage_pipes[STDIN_FILENO]) != 0) { - FATAL("Failed to close parent read pipe"); + FFATAL("Failed to close parent read pipe"); } @@ -711,15 +711,14 @@ void instrument_coverage_normal_init(void) { void instrument_coverage_unstable_find_output(void) { - pid_t parent = getpid(); gchar *fds_name = g_strdup_printf("/proc/%d/fd/", getppid()); gchar *root = g_file_read_link("/proc/self/root", NULL); - if (root == NULL) { FATAL("Failed to read link"); } + if (root == NULL) { FFATAL("Failed to read link"); } GDir *dir = g_dir_open(fds_name, 0, NULL); - OKF("Coverage Unstable - fds: %s", fds_name); + FOKF("Coverage Unstable - fds: %s", fds_name); for (const gchar *filename = g_dir_read_name(dir); filename != NULL; filename = g_dir_read_name(dir)) { @@ -727,7 +726,7 @@ void instrument_coverage_unstable_find_output(void) { gchar *fullname = g_build_path("/", fds_name, filename, NULL); gchar *link = g_file_read_link(fullname, NULL); - if (link == NULL) { FATAL("Failed to read link: %s", fullname); } + if (link == NULL) { FFATAL("Failed to read link: %s", fullname); } gchar *basename = g_path_get_basename(link); if (g_strcmp0(basename, "default") != 0) { @@ -779,11 +778,11 @@ void instrument_coverage_unstable_find_output(void) { if (unstable_coverage_fuzzer_stats == NULL) { - FATAL("Failed to find fuzzer stats"); + FFATAL("Failed to find fuzzer stats"); } - OKF("Fuzzer stats: %s", unstable_coverage_fuzzer_stats); + FOKF("Fuzzer stats: %s", unstable_coverage_fuzzer_stats); } @@ -794,14 +793,14 @@ void instrument_coverage_unstable_init(void) { char *path = g_canonicalize_filename(instrument_coverage_unstable_filename, g_get_current_dir()); - OKF("Coverage - unstable path [%s]", instrument_coverage_unstable_filename); + FOKF("Coverage - unstable path [%s]", instrument_coverage_unstable_filename); unstable_coverage_fd = open(path, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); if (unstable_coverage_fd < 0) { - FATAL("Failed to open unstable coverage file '%s'", path); + FFATAL("Failed to open unstable coverage file '%s'", path); } @@ -811,12 +810,12 @@ void instrument_coverage_unstable_init(void) { if (pipe(unstable_coverage_pipes) != 0) { - FATAL("Failed to create unstable pipes"); + FFATAL("Failed to create unstable pipes"); } pid_t pid = fork(); - if (pid == -1) { FATAL("Failed to start coverage process"); } + if (pid == -1) { FFATAL("Failed to start coverage process"); } if (pid == 0) { @@ -828,13 +827,13 @@ void instrument_coverage_unstable_init(void) { if (close(unstable_coverage_fd) < 0) { - FATAL("Failed to close unstable coverage output file"); + FFATAL("Failed to close unstable coverage output file"); } if (close(unstable_coverage_pipes[STDIN_FILENO]) != 0) { - FATAL("Failed to close parent read pipe"); + FFATAL("Failed to close parent read pipe"); } @@ -866,7 +865,7 @@ void instrument_coverage_end(uint64_t address) { if (write(normal_coverage_pipes[STDOUT_FILENO], &data, sizeof(normal_coverage_data_t)) != sizeof(normal_coverage_data_t)) { - FATAL("Coverage I/O error"); + FFATAL("Coverage I/O error"); } @@ -889,7 +888,7 @@ void instrument_coverage_unstable(guint64 edge, guint64 previous_rip, sizeof(unstable_coverage_data_t)) != sizeof(unstable_coverage_data_t)) { - FATAL("Unstable coverage I/O error"); + FFATAL("Unstable coverage I/O error"); } diff --git a/frida_mode/src/instrument/instrument_debug.c b/frida_mode/src/instrument/instrument_debug.c index b8cca634..9c95857f 100644 --- a/frida_mode/src/instrument/instrument_debug.c +++ b/frida_mode/src/instrument/instrument_debug.c @@ -5,8 +5,6 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "instrument.h" #include "util.h" @@ -34,18 +32,27 @@ static void instrument_debug(char *format, ...) { } -static void instrument_disasm(guint8 *start, guint8 *end) { +static void instrument_disasm(guint8 *start, guint8 *end, + GumStalkerOutput *output) { csh capstone; cs_err err; + cs_mode mode; uint16_t size; cs_insn *insn; size_t count = 0; size_t i; uint16_t len; + mode = GUM_DEFAULT_CS_MODE | GUM_DEFAULT_CS_ENDIAN; + +#if defined(__arm__) + if (output->encoding == GUM_INSTRUCTION_SPECIAL) { mode |= CS_MODE_THUMB; } +#endif + err = cs_open(GUM_DEFAULT_CS_ARCH, - GUM_DEFAULT_CS_MODE | GUM_DEFAULT_CS_ENDIAN, &capstone); + CS_MODE_THUMB | GUM_DEFAULT_CS_MODE | GUM_DEFAULT_CS_ENDIAN, + &capstone); g_assert(err == CS_ERR_OK); size = GPOINTER_TO_SIZE(end) - GPOINTER_TO_SIZE(start); @@ -89,24 +96,24 @@ void instrument_debug_config(void) { void instrument_debug_init(void) { - OKF("Instrumentation debugging - enabled [%c]", - instrument_debug_filename == NULL ? ' ' : 'X'); + FOKF("Instrumentation debugging - enabled [%c]", + instrument_debug_filename == NULL ? ' ' : 'X'); if (instrument_debug_filename == NULL) { return; } - OKF("Instrumentation debugging - file [%s]", instrument_debug_filename); + FOKF("Instrumentation debugging - file [%s]", instrument_debug_filename); if (instrument_debug_filename == NULL) { return; } char *path = g_canonicalize_filename(instrument_debug_filename, g_get_current_dir()); - OKF("Instrumentation debugging - path [%s]", path); + FOKF("Instrumentation debugging - path [%s]", path); debugging_fd = open(path, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); - if (debugging_fd < 0) { FATAL("Failed to open stats file '%s'", path); } + if (debugging_fd < 0) { FFATAL("Failed to open stats file '%s'", path); } g_free(path); @@ -123,11 +130,12 @@ void instrument_debug_start(uint64_t address, GumStalkerOutput *output) { } -void instrument_debug_instruction(uint64_t address, uint16_t size) { +void instrument_debug_instruction(uint64_t address, uint16_t size, + GumStalkerOutput *output) { if (likely(debugging_fd < 0)) { return; } uint8_t *start = (uint8_t *)GSIZE_TO_POINTER(address); - instrument_disasm(start, start + size); + instrument_disasm(start, start + size, output); } @@ -138,7 +146,7 @@ void instrument_debug_end(GumStalkerOutput *output) { instrument_debug("\nGenerated block %p-%p\n", instrument_gen_start, instrument_gen_end); - instrument_disasm(instrument_gen_start, instrument_gen_end); + instrument_disasm(instrument_gen_start, instrument_gen_end, output); } diff --git a/frida_mode/src/instrument/instrument_x64.c b/frida_mode/src/instrument/instrument_x64.c index fec8afbb..41162f2a 100644 --- a/frida_mode/src/instrument/instrument_x64.c +++ b/frida_mode/src/instrument/instrument_x64.c @@ -1,85 +1,343 @@ +#include <fcntl.h> +#include <stddef.h> +#include <sys/mman.h> +#include <sys/shm.h> + +#if defined(__linux__) + #if !defined(__ANDROID__) + #include <sys/prctl.h> + #include <sys/syscall.h> + #else + #include <linux/ashmem.h> + #endif +#endif + #include "frida-gumjs.h" #include "config.h" #include "instrument.h" +#include "ranges.h" +#include "stalker.h" +#include "util.h" #if defined(__x86_64__) -static GumAddress current_log_impl = GUM_ADDRESS(0); + #ifndef MAP_FIXED_NOREPLACE + #ifdef MAP_EXCL + #define MAP_FIXED_NOREPLACE MAP_EXCL | MAP_FIXED + #else + #define MAP_FIXED_NOREPLACE MAP_FIXED + #endif + #endif -static const guint8 afl_log_code[] = { +static GHashTable *coverage_blocks = NULL; - 0x9c, /* pushfq */ - 0x51, /* push rcx */ - 0x52, /* push rdx */ +gboolean instrument_is_coverage_optimize_supported(void) { - 0x48, 0x8b, 0x0d, 0x26, - 0x00, 0x00, 0x00, /* mov rcx, sym.&previous_pc */ - 0x48, 0x8b, 0x11, /* mov rdx, qword [rcx] */ - 0x48, 0x31, 0xfa, /* xor rdx, rdi */ + return true; - 0x48, 0x03, 0x15, 0x11, - 0x00, 0x00, 0x00, /* add rdx, sym._afl_area_ptr_ptr */ +} - 0x80, 0x02, 0x01, /* add byte ptr [rdx], 1 */ - 0x80, 0x12, 0x00, /* adc byte ptr [rdx], 0 */ - 0x66, 0xd1, 0xcf, /* ror di, 1 */ - 0x48, 0x89, 0x39, /* mov qword [rcx], rdi */ +static gboolean instrument_coverage_in_range(gssize offset) { - 0x5a, /* pop rdx */ - 0x59, /* pop rcx */ - 0x9d, /* popfq */ + return (offset >= G_MININT32 && offset <= G_MAXINT32); - 0xc3, /* ret */ +} - 0x90 + #pragma pack(push, 1) +typedef struct { - /* Read-only data goes here: */ - /* uint8_t* __afl_area_ptr */ - /* uint64_t* &previous_pc */ + // cur_location = (block_address >> 4) ^ (block_address << 8); + // shared_mem[cur_location ^ prev_location]++; + // prev_location = cur_location >> 1; -}; + // mov QWORD PTR [rsp-0x80],rax + // lahf + // mov QWORD PTR [rsp-0x88],rax + // mov QWORD PTR [rsp-0x90],rbx + // mov eax,DWORD PTR [rip+0x333d5a] # 0x7ffff6ff2740 + // mov DWORD PTR [rip+0x333d3c],0x9fbb # 0x7ffff6ff2740 + // xor eax,0x103f77 + // mov bl,BYTE PTR [rax] + // add bl,0x1 + // adc bl,0x0 + // mov BYTE PTR [rax],bl + // mov rbx,QWORD PTR [rsp-0x90] + // mov rax,QWORD PTR [rsp-0x88] + // sahf + // mov rax,QWORD PTR [rsp-0x80] -gboolean instrument_is_coverage_optimize_supported(void) { + uint8_t mov_rax_rsp_88[8]; + uint8_t lahf; + uint8_t mov_rax_rsp_90[8]; + uint8_t mov_rbx_rsp_98[8]; - return true; + uint8_t mov_eax_prev_loc[6]; + uint8_t mov_prev_loc_curr_loc_shr1[10]; + + uint8_t xor_eax_curr_loc[5]; + + uint8_t mov_rbx_ptr_rax[2]; + uint8_t add_bl_1[3]; + uint8_t adc_bl_0[3]; + uint8_t mov_ptr_rax_rbx[2]; + + uint8_t mov_rsp_98_rbx[8]; + uint8_t mov_rsp_90_rax[8]; + uint8_t sahf; + uint8_t mov_rsp_88_rax[8]; + +} afl_log_code_asm_t; + + #pragma pack(pop) + +typedef union { + + afl_log_code_asm_t code; + uint8_t bytes[0]; + +} afl_log_code; + +static const afl_log_code_asm_t template = + { + + .mov_rax_rsp_88 = {0x48, 0x89, 0x84, 0x24, 0x78, 0xFF, 0xFF, 0xFF}, + .lahf = 0x9f, + .mov_rax_rsp_90 = {0x48, 0x89, 0x84, 0x24, 0x70, 0xFF, 0xFF, 0xFF}, + .mov_rbx_rsp_98 = {0x48, 0x89, 0x9C, 0x24, 0x68, 0xFF, 0xFF, 0xFF}, + + .mov_eax_prev_loc = {0x8b, 0x05}, + .mov_prev_loc_curr_loc_shr1 = {0xc7, 0x05}, + + .xor_eax_curr_loc = {0x35}, + .mov_rbx_ptr_rax = {0x8a, 0x18}, + .add_bl_1 = {0x80, 0xc3, 0x01}, + .adc_bl_0 = {0x80, 0xd3, 0x00}, + .mov_ptr_rax_rbx = {0x88, 0x18}, + + .mov_rsp_98_rbx = {0x48, 0x8B, 0x9C, 0x24, 0x68, 0xFF, 0xFF, 0xFF}, + .mov_rsp_90_rax = {0x48, 0x8B, 0x84, 0x24, 0x70, 0xFF, 0xFF, 0xFF}, + .sahf = 0x9e, + .mov_rsp_88_rax = {0x48, 0x8B, 0x84, 0x24, 0x78, 0xFF, 0xFF, 0xFF}, } -static guint8 align_pad[] = {0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90}; +; -static void instrument_coverate_write_function(GumStalkerOutput *output) { +static gboolean instrument_coverage_find_low(const GumRangeDetails *details, + gpointer user_data) { - guint64 misalign = 0; - GumX86Writer *cw = output->writer.x86; + static GumAddress last_limit = (64ULL << 10); + gpointer * address = (gpointer *)user_data; + + if ((details->range->base_address - last_limit) > __afl_map_size) { + + *address = GSIZE_TO_POINTER(last_limit); + return FALSE; + + } + + if (details->range->base_address > ((2ULL << 30) - __afl_map_size)) { + + return FALSE; + + } + + /* + * Align our buffer on a 64k boundary so that the low 16-bits of the address + * are zero, then we can just XOR the base address in, when we XOR with the + * current block ID. + */ + last_limit = GUM_ALIGN_SIZE( + details->range->base_address + details->range->size, (64ULL << 10)); + return TRUE; + +} + +static void instrument_coverage_optimize_map_mmap_anon(gpointer address) { + + __afl_area_ptr = + mmap(address, __afl_map_size, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (__afl_area_ptr != address) { + + FATAL("Failed to map mmap __afl_area_ptr: %d", errno); + + } + +} + +static void instrument_coverage_optimize_map_mmap(char * shm_file_path, + gpointer address) { + + int shm_fd = -1; + + if (munmap(__afl_area_ptr, __afl_map_size) != 0) { + + FATAL("Failed to unmap previous __afl_area_ptr"); + + } + + __afl_area_ptr = NULL; + + #if !defined(__ANDROID__) + shm_fd = shm_open(shm_file_path, O_RDWR, DEFAULT_PERMISSION); + if (shm_fd == -1) { FATAL("shm_open() failed\n"); } + #else + shm_fd = open("/dev/ashmem", O_RDWR); + if (shm_fd == -1) { FATAL("open() failed\n"); } + if (ioctl(shm_fd, ASHMEM_SET_NAME, shm_file_path) == -1) { + + FATAL("ioctl(ASHMEM_SET_NAME) failed"); + + } + + if (ioctl(shm_fd, ASHMEM_SET_SIZE, __afl_map_size) == -1) { + + FATAL("ioctl(ASHMEM_SET_SIZE) failed"); + + } + + #endif + + __afl_area_ptr = mmap(address, __afl_map_size, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_SHARED, shm_fd, 0); + if (__afl_area_ptr != address) { + + FATAL("Failed to map mmap __afl_area_ptr: %d", errno); + + } + + if (close(shm_fd) != 0) { FATAL("Failed to close shm_fd"); } + +} + +static void instrument_coverage_optimize_map_shm(guint64 shm_env_val, + gpointer address) { + + if (shmdt(__afl_area_ptr) != 0) { + + FATAL("Failed to detach previous __afl_area_ptr"); + + } - if (current_log_impl == 0 || - !gum_x86_writer_can_branch_directly_between(cw->pc, current_log_impl) || - !gum_x86_writer_can_branch_directly_between(cw->pc + 128, - current_log_impl)) { + __afl_area_ptr = shmat(shm_env_val, address, 0); + if (__afl_area_ptr != address) { - gconstpointer after_log_impl = cw->code + 1; + FATAL("Failed to map shm __afl_area_ptr: %d", errno); - gum_x86_writer_put_jmp_near_label(cw, after_log_impl); + } + +} + +static void instrument_coverage_switch(GumStalkerObserver *self, + gpointer start_address, + const cs_insn * from_insn, + gpointer * target) { + + UNUSED_PARAMETER(self); + UNUSED_PARAMETER(start_address); + + cs_x86 * x86; + cs_x86_op *op; + if (from_insn == NULL) { return; } + + x86 = &from_insn->detail->x86; + op = x86->operands; + + if (!g_hash_table_contains(coverage_blocks, GSIZE_TO_POINTER(*target))) { + + return; + + } + + switch (from_insn->id) { + + case X86_INS_CALL: + case X86_INS_JMP: + if (x86->op_count != 1) { + + FATAL("Unexpected operand count: %d", x86->op_count); + + } + + if (op[0].type != X86_OP_IMM) { return; } + + break; + case X86_INS_RET: + break; + default: + return; + + } + + *target = (guint8 *)*target + sizeof(afl_log_code); + +} + +void instrument_coverage_optimize_init(void) { + + gpointer low_address = NULL; + + gum_process_enumerate_ranges(GUM_PAGE_NO_ACCESS, instrument_coverage_find_low, + &low_address); + + FOKF("Low address: %p", low_address); + + if (low_address == 0 || + GPOINTER_TO_SIZE(low_address) > ((2UL << 20) - __afl_map_size)) { + + FATAL("Invalid low_address: %p", low_address); + + } + + ranges_print_debug_maps(); + + char *shm_env = getenv(SHM_ENV_VAR); + FOKF("SHM_ENV_VAR: %s", shm_env); + + if (shm_env == NULL) { + + FWARNF("SHM_ENV_VAR not set, using anonymous map for debugging purposes"); - misalign = (cw->pc & 0x7); - if (misalign != 0) { + instrument_coverage_optimize_map_mmap_anon(low_address); - gum_x86_writer_put_bytes(cw, align_pad, 8 - misalign); + } else { + + guint64 shm_env_val = g_ascii_strtoull(shm_env, NULL, 10); + + if (shm_env_val == 0) { + + instrument_coverage_optimize_map_mmap(shm_env, low_address); + + } else { + + instrument_coverage_optimize_map_shm(shm_env_val, low_address); } - current_log_impl = cw->pc; - gum_x86_writer_put_bytes(cw, afl_log_code, sizeof(afl_log_code)); + } + + FOKF("__afl_area_ptr: %p", __afl_area_ptr); + FOKF("instrument_previous_pc: %p", &instrument_previous_pc); + +} - uint64_t *afl_prev_loc_ptr = &instrument_previous_pc; - gum_x86_writer_put_bytes(cw, (const guint8 *)&__afl_area_ptr, - sizeof(__afl_area_ptr)); - gum_x86_writer_put_bytes(cw, (const guint8 *)&afl_prev_loc_ptr, - sizeof(afl_prev_loc_ptr)); +static void instrument_coverage_suppress_init(void) { - gum_x86_writer_put_label(cw, after_log_impl); + static gboolean initialized = false; + if (initialized) { return; } + initialized = true; + + GumStalkerObserver * observer = stalker_get_observer(); + GumStalkerObserverInterface *iface = GUM_STALKER_OBSERVER_GET_IFACE(observer); + iface->switch_callback = instrument_coverage_switch; + + coverage_blocks = g_hash_table_new(g_direct_hash, g_direct_equal); + if (coverage_blocks == NULL) { + + FATAL("Failed to g_hash_table_new, errno: %d", errno); } @@ -88,18 +346,73 @@ static void instrument_coverate_write_function(GumStalkerOutput *output) { void instrument_coverage_optimize(const cs_insn * instr, GumStalkerOutput *output) { + afl_log_code code = {0}; GumX86Writer *cw = output->writer.x86; guint64 area_offset = instrument_get_offset_hash(GUM_ADDRESS(instr->address)); - instrument_coverate_write_function(output); - - gum_x86_writer_put_lea_reg_reg_offset(cw, GUM_REG_RSP, GUM_REG_RSP, - -GUM_RED_ZONE_SIZE); - gum_x86_writer_put_push_reg(cw, GUM_REG_RDI); - gum_x86_writer_put_mov_reg_address(cw, GUM_REG_RDI, area_offset); - gum_x86_writer_put_call_address(cw, current_log_impl); - gum_x86_writer_put_pop_reg(cw, GUM_REG_RDI); - gum_x86_writer_put_lea_reg_reg_offset(cw, GUM_REG_RSP, GUM_REG_RSP, - GUM_RED_ZONE_SIZE); + gsize map_size_pow2; + gsize area_offset_ror; + GumAddress code_addr = 0; + + instrument_coverage_suppress_init(); + + // gum_x86_writer_put_breakpoint(cw); + code_addr = cw->pc; + if (!g_hash_table_add(coverage_blocks, GSIZE_TO_POINTER(cw->code))) { + + FATAL("Failed - g_hash_table_add"); + + } + + code.code = template; + + gssize curr_loc_shr_1_offset = + offsetof(afl_log_code, code.mov_prev_loc_curr_loc_shr1) + + sizeof(code.code.mov_prev_loc_curr_loc_shr1) - sizeof(guint32); + + map_size_pow2 = util_log2(__afl_map_size); + area_offset_ror = util_rotate(area_offset, 1, map_size_pow2); + + *((guint32 *)&code.bytes[curr_loc_shr_1_offset]) = (guint32)(area_offset_ror); + + gssize prev_loc_value = + GPOINTER_TO_SIZE(&instrument_previous_pc) - + (code_addr + offsetof(afl_log_code, code.mov_prev_loc_curr_loc_shr1) + + sizeof(code.code.mov_prev_loc_curr_loc_shr1)); + gssize prev_loc_value_offset = + offsetof(afl_log_code, code.mov_prev_loc_curr_loc_shr1) + + sizeof(code.code.mov_prev_loc_curr_loc_shr1) - sizeof(gint) - + sizeof(guint32); + if (!instrument_coverage_in_range(prev_loc_value)) { + + FATAL("Patch out of range (current_pc_value1): 0x%016lX", prev_loc_value); + + } + + *((gint *)&code.bytes[prev_loc_value_offset]) = (gint)prev_loc_value; + + gssize prev_loc_value2 = + GPOINTER_TO_SIZE(&instrument_previous_pc) - + (code_addr + offsetof(afl_log_code, code.mov_eax_prev_loc) + + sizeof(code.code.mov_eax_prev_loc)); + gssize prev_loc_value_offset2 = + offsetof(afl_log_code, code.mov_eax_prev_loc) + + sizeof(code.code.mov_eax_prev_loc) - sizeof(gint); + if (!instrument_coverage_in_range(prev_loc_value)) { + + FATAL("Patch out of range (current_pc_value1): 0x%016lX", prev_loc_value2); + + } + + *((gint *)&code.bytes[prev_loc_value_offset2]) = (gint)prev_loc_value2; + + gssize xor_curr_loc_offset = offsetof(afl_log_code, code.xor_eax_curr_loc) + + sizeof(code.code.xor_eax_curr_loc) - + sizeof(guint32); + + *((guint32 *)&code.bytes[xor_curr_loc_offset]) = + (guint32)(GPOINTER_TO_SIZE(__afl_area_ptr) | area_offset); + + gum_x86_writer_put_bytes(cw, code.bytes, sizeof(afl_log_code)); } diff --git a/frida_mode/src/instrument/instrument_x86.c b/frida_mode/src/instrument/instrument_x86.c index 7bf48f96..ad837e2d 100644 --- a/frida_mode/src/instrument/instrument_x86.c +++ b/frida_mode/src/instrument/instrument_x86.c @@ -1,69 +1,144 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "instrument.h" +#include "stalker.h" #include "util.h" #if defined(__i386__) -static GumAddress current_log_impl = GUM_ADDRESS(0); +static GHashTable *coverage_blocks = NULL; + + #pragma pack(push, 1) +typedef struct { + + // cur_location = (block_address >> 4) ^ (block_address << 8); + // shared_mem[cur_location ^ prev_location]++; + // prev_location = cur_location >> 1; + + uint8_t mov_eax_esp_4[4]; + uint8_t lahf; + uint8_t mov_eax_esp_8[4]; + uint8_t mov_ebx_esp_c[4]; + + uint8_t mov_eax_prev_loc[5]; + uint8_t mov_prev_loc_curr_loc_shr1[10]; + + uint8_t xor_eax_curr_loc[5]; + uint8_t add_eax_area_ptr[5]; + + uint8_t mov_ebx_ptr_eax[2]; + uint8_t add_bl_1[3]; + uint8_t adc_bl_0[3]; + uint8_t mov_ptr_eax_ebx[2]; -static void instrument_coverage_function(GumX86Writer *cw) { + uint8_t mov_esp_c_ebx[4]; + uint8_t mov_esp_8_eax[4]; + uint8_t sahf; + uint8_t mov_esp_4_eax[4]; - gum_x86_writer_put_pushfx(cw); - gum_x86_writer_put_push_reg(cw, GUM_REG_ECX); - gum_x86_writer_put_push_reg(cw, GUM_REG_EDX); +} afl_log_code_asm_t; - gum_x86_writer_put_mov_reg_address(cw, GUM_REG_ECX, - GUM_ADDRESS(&instrument_previous_pc)); - gum_x86_writer_put_mov_reg_reg_ptr(cw, GUM_REG_EDX, GUM_REG_ECX); - gum_x86_writer_put_xor_reg_reg(cw, GUM_REG_EDX, GUM_REG_EDI); + #pragma pack(pop) - gum_x86_writer_put_add_reg_imm(cw, GUM_REG_EDX, GUM_ADDRESS(__afl_area_ptr)); +typedef union { - /* add byte ptr [edx], 1 */ - uint8_t add_byte_ptr_edx_1[] = {0x80, 0x02, 0x01}; - gum_x86_writer_put_bytes(cw, add_byte_ptr_edx_1, sizeof(add_byte_ptr_edx_1)); + afl_log_code_asm_t code; + uint8_t bytes[0]; - /* adc byte ptr [edx], 0 */ - uint8_t adc_byte_ptr_edx_0[] = {0x80, 0x12, 0x00}; - gum_x86_writer_put_bytes(cw, adc_byte_ptr_edx_0, sizeof(adc_byte_ptr_edx_0)); +} afl_log_code; - uint8_t ror_di_1[] = {0x66, 0xd1, 0xcf}; - gum_x86_writer_put_bytes(cw, ror_di_1, sizeof(ror_di_1)); - gum_x86_writer_put_mov_reg_ptr_reg(cw, GUM_REG_ECX, GUM_REG_EDI); +static const afl_log_code_asm_t template = + { - gum_x86_writer_put_pop_reg(cw, GUM_REG_EDX); - gum_x86_writer_put_pop_reg(cw, GUM_REG_ECX); - gum_x86_writer_put_popfx(cw); - gum_x86_writer_put_ret(cw); + .mov_eax_esp_4 = {0x89, 0x44, 0x24, 0xFC}, + .lahf = 0x9f, + .mov_eax_esp_8 = {0x89, 0x44, 0x24, 0xF8}, + .mov_ebx_esp_c = {0x89, 0x5C, 0x24, 0xF4}, + + .mov_eax_prev_loc = {0xA1}, + .mov_prev_loc_curr_loc_shr1 = {0xc7, 0x05}, + + .xor_eax_curr_loc = {0x35}, + .add_eax_area_ptr = {0x05}, + .mov_ebx_ptr_eax = {0x8a, 0x18}, + .add_bl_1 = {0x80, 0xc3, 0x01}, + .adc_bl_0 = {0x80, 0xd3, 0x00}, + .mov_ptr_eax_ebx = {0x88, 0x18}, + + .mov_esp_c_ebx = {0x8B, 0x5C, 0x24, 0xF4}, + .mov_esp_8_eax = {0x8B, 0x44, 0x24, 0xF8}, + .sahf = 0x9e, + .mov_esp_4_eax = {0x8B, 0x44, 0x24, 0xFC}, } +; + gboolean instrument_is_coverage_optimize_supported(void) { return true; } -static void instrument_coverate_write_function(GumStalkerOutput *output) { +static void instrument_coverage_switch(GumStalkerObserver *self, + gpointer start_address, + const cs_insn * from_insn, + gpointer * target) { - GumX86Writer *cw = output->writer.x86; + UNUSED_PARAMETER(self); + UNUSED_PARAMETER(start_address); + + cs_x86 * x86; + cs_x86_op *op; + if (from_insn == NULL) { return; } + + x86 = &from_insn->detail->x86; + op = x86->operands; + + if (!g_hash_table_contains(coverage_blocks, GSIZE_TO_POINTER(*target))) { + + return; + + } + + switch (from_insn->id) { + + case X86_INS_CALL: + case X86_INS_JMP: + if (x86->op_count != 1) { + + FATAL("Unexpected operand count: %d", x86->op_count); + + } + + if (op[0].type != X86_OP_IMM) { return; } + + break; + case X86_INS_RET: + break; + default: + return; + + } + + *target = (guint8 *)*target + sizeof(afl_log_code); + +} - if (current_log_impl == 0 || - !gum_x86_writer_can_branch_directly_between(cw->pc, current_log_impl) || - !gum_x86_writer_can_branch_directly_between(cw->pc + 128, - current_log_impl)) { +static void instrument_coverage_suppress_init(void) { - gconstpointer after_log_impl = cw->code + 1; + static gboolean initialized = false; + if (initialized) { return; } + initialized = true; - gum_x86_writer_put_jmp_near_label(cw, after_log_impl); + GumStalkerObserver * observer = stalker_get_observer(); + GumStalkerObserverInterface *iface = GUM_STALKER_OBSERVER_GET_IFACE(observer); + iface->switch_callback = instrument_coverage_switch; - current_log_impl = cw->pc; - instrument_coverage_function(cw); + coverage_blocks = g_hash_table_new(g_direct_hash, g_direct_equal); + if (coverage_blocks == NULL) { - gum_x86_writer_put_label(cw, after_log_impl); + FATAL("Failed to g_hash_table_new, errno: %d", errno); } @@ -72,14 +147,65 @@ static void instrument_coverate_write_function(GumStalkerOutput *output) { void instrument_coverage_optimize(const cs_insn * instr, GumStalkerOutput *output) { + afl_log_code code = {0}; GumX86Writer *cw = output->writer.x86; guint64 area_offset = instrument_get_offset_hash(GUM_ADDRESS(instr->address)); - instrument_coverate_write_function(output); + gsize map_size_pow2; + gsize area_offset_ror; + + code.code = template; + + instrument_coverage_suppress_init(); + + // gum_x86_writer_put_breakpoint(cw); + + if (!g_hash_table_add(coverage_blocks, GSIZE_TO_POINTER(cw->code))) { + + FATAL("Failed - g_hash_table_add"); + + } + + gssize prev_loc_value_offset2 = + offsetof(afl_log_code, code.mov_eax_prev_loc) + + sizeof(code.code.mov_eax_prev_loc) - sizeof(gint); + + *((gint *)&code.bytes[prev_loc_value_offset2]) = + (gint)GPOINTER_TO_SIZE(&instrument_previous_pc); + + gssize curr_loc_shr_1_offset = + offsetof(afl_log_code, code.mov_prev_loc_curr_loc_shr1) + + sizeof(code.code.mov_prev_loc_curr_loc_shr1) - sizeof(guint32); + + map_size_pow2 = util_log2(__afl_map_size); + area_offset_ror = util_rotate(area_offset, 1, map_size_pow2); + + *((guint32 *)&code.bytes[curr_loc_shr_1_offset]) = (guint32)(area_offset_ror); + + gssize prev_loc_value_offset = + offsetof(afl_log_code, code.mov_prev_loc_curr_loc_shr1) + + sizeof(code.code.mov_prev_loc_curr_loc_shr1) - sizeof(gint) - + sizeof(guint32); + + *((gint *)&code.bytes[prev_loc_value_offset]) = + (gint)GPOINTER_TO_SIZE(&instrument_previous_pc); + + gssize xor_curr_loc_offset = offsetof(afl_log_code, code.xor_eax_curr_loc) + + sizeof(code.code.xor_eax_curr_loc) - + sizeof(guint32); + + *((guint32 *)&code.bytes[xor_curr_loc_offset]) = (guint32)area_offset; + + gssize add_area_ptr_offset = offsetof(afl_log_code, code.add_eax_area_ptr) + + sizeof(code.code.add_eax_area_ptr) - + sizeof(guint32); + + *((guint32 *)&code.bytes[add_area_ptr_offset]) = (guint32)__afl_area_ptr; + + gum_x86_writer_put_bytes(cw, code.bytes, sizeof(afl_log_code)); + +} - gum_x86_writer_put_push_reg(cw, GUM_REG_EDI); - gum_x86_writer_put_mov_reg_address(cw, GUM_REG_EDI, area_offset); - gum_x86_writer_put_call_address(cw, current_log_impl); - gum_x86_writer_put_pop_reg(cw, GUM_REG_EDI); +void instrument_coverage_optimize_init(void) { } diff --git a/frida_mode/src/intercept.c b/frida_mode/src/intercept.c index ed8d27bd..26d20c50 100644 --- a/frida_mode/src/intercept.c +++ b/frida_mode/src/intercept.c @@ -1,8 +1,7 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "intercept.h" +#include "util.h" void intercept_hook(void *address, gpointer replacement, gpointer user_data) { @@ -10,7 +9,7 @@ void intercept_hook(void *address, gpointer replacement, gpointer user_data) { gum_interceptor_begin_transaction(interceptor); GumReplaceReturn ret = gum_interceptor_replace(interceptor, address, replacement, user_data); - if (ret != GUM_REPLACE_OK) { FATAL("gum_interceptor_attach: %d", ret); } + if (ret != GUM_REPLACE_OK) { FFATAL("gum_interceptor_attach: %d", ret); } gum_interceptor_end_transaction(interceptor); } diff --git a/frida_mode/src/js/api.js b/frida_mode/src/js/api.js index 40bb4a16..215fbdaf 100644 --- a/frida_mode/src/js/api.js +++ b/frida_mode/src/js/api.js @@ -63,6 +63,12 @@ class Afl { Afl.jsApiWrite(STDOUT_FILENO, buf, log.length); } /** + * See `AFL_FRIDA_INST_NO_BACKPATCH`. + */ + static setBackpatchDisable() { + Afl.jsApiSetBackpatchDisable(); + } + /** * See `AFL_FRIDA_DEBUG_MAPS`. */ static setDebugMaps() { @@ -145,6 +151,13 @@ class Afl { const buf = Memory.allocUtf8String(file); Afl.jsApiSetInstrumentUnstableCoverageFile(buf); } + /* + * Set a callback to be called in place of the usual `main` function. This see + * `Scripting.md` for details. + */ + static setJsMainHook(address) { + Afl.jsApiSetJsMainHook(address); + } /** * This is equivalent to setting `AFL_FRIDA_PERSISTENT_ADDR`, again a * `NativePointer` should be provided as it's argument. @@ -199,6 +212,12 @@ class Afl { const buf = Memory.allocUtf8String(file); Afl.jsApiSetSeccompFile(buf); } + /** + * See `AFL_FRIDA_STALKER_ADJACENT_BLOCKS`. + */ + static setStalkerAdjacentBlocks(val) { + Afl.jsApiSetStalkerAdjacentBlocks(val); + } /* * Set a function to be called for each instruction which is instrumented * by AFL FRIDA mode. @@ -243,6 +262,12 @@ class Afl { const buf = Memory.allocUtf8String(file); Afl.jsApiSetStdOut(buf); } + /** + * See `AFL_FRIDA_TRACEABLE`. + */ + static setTraceable() { + Afl.jsApiSetTraceable(); + } static jsApiGetFunction(name, retType, argTypes) { const addr = Afl.module.getExportByName(name); return new NativeFunction(addr, retType, argTypes); @@ -261,6 +286,7 @@ Afl.jsApiAddIncludeRange = Afl.jsApiGetFunction("js_api_add_include_range", "voi Afl.jsApiAflSharedMemFuzzing = Afl.jsApiGetSymbol("__afl_sharedmem_fuzzing"); Afl.jsApiDone = Afl.jsApiGetFunction("js_api_done", "void", []); Afl.jsApiError = Afl.jsApiGetFunction("js_api_error", "void", ["pointer"]); +Afl.jsApiSetBackpatchDisable = Afl.jsApiGetFunction("js_api_set_backpatch_disable", "void", []); Afl.jsApiSetDebugMaps = Afl.jsApiGetFunction("js_api_set_debug_maps", "void", []); Afl.jsApiSetEntryPoint = Afl.jsApiGetFunction("js_api_set_entrypoint", "void", ["pointer"]); Afl.jsApiSetInstrumentCoverageFile = Afl.jsApiGetFunction("js_api_set_instrument_coverage_file", "void", ["pointer"]); @@ -272,6 +298,7 @@ Afl.jsApiSetInstrumentSeed = Afl.jsApiGetFunction("js_api_set_instrument_seed", Afl.jsApiSetInstrumentTrace = Afl.jsApiGetFunction("js_api_set_instrument_trace", "void", []); Afl.jsApiSetInstrumentTraceUnique = Afl.jsApiGetFunction("js_api_set_instrument_trace_unique", "void", []); Afl.jsApiSetInstrumentUnstableCoverageFile = Afl.jsApiGetFunction("js_api_set_instrument_unstable_coverage_file", "void", ["pointer"]); +Afl.jsApiSetJsMainHook = Afl.jsApiGetFunction("js_api_set_js_main_hook", "void", ["pointer"]); Afl.jsApiSetPersistentAddress = Afl.jsApiGetFunction("js_api_set_persistent_address", "void", ["pointer"]); Afl.jsApiSetPersistentCount = Afl.jsApiGetFunction("js_api_set_persistent_count", "void", ["uint64"]); Afl.jsApiSetPersistentDebug = Afl.jsApiGetFunction("js_api_set_persistent_debug", "void", []); @@ -280,12 +307,14 @@ Afl.jsApiSetPersistentReturn = Afl.jsApiGetFunction("js_api_set_persistent_retur Afl.jsApiSetPrefetchBackpatchDisable = Afl.jsApiGetFunction("js_api_set_prefetch_backpatch_disable", "void", []); Afl.jsApiSetPrefetchDisable = Afl.jsApiGetFunction("js_api_set_prefetch_disable", "void", []); Afl.jsApiSetSeccompFile = Afl.jsApiGetFunction("js_api_set_seccomp_file", "void", ["pointer"]); +Afl.jsApiSetStalkerAdjacentBlocks = Afl.jsApiGetFunction("js_api_set_stalker_adjacent_blocks", "void", ["uint32"]); Afl.jsApiSetStalkerCallback = Afl.jsApiGetFunction("js_api_set_stalker_callback", "void", ["pointer"]); Afl.jsApiSetStalkerIcEntries = Afl.jsApiGetFunction("js_api_set_stalker_ic_entries", "void", ["uint32"]); Afl.jsApiSetStatsFile = Afl.jsApiGetFunction("js_api_set_stats_file", "void", ["pointer"]); Afl.jsApiSetStatsInterval = Afl.jsApiGetFunction("js_api_set_stats_interval", "void", ["uint64"]); Afl.jsApiSetStdErr = Afl.jsApiGetFunction("js_api_set_stderr", "void", ["pointer"]); Afl.jsApiSetStdOut = Afl.jsApiGetFunction("js_api_set_stdout", "void", ["pointer"]); +Afl.jsApiSetTraceable = Afl.jsApiGetFunction("js_api_set_traceable", "void", []); Afl.jsApiWrite = new NativeFunction( /* tslint:disable-next-line:no-null-keyword */ Module.getExportByName(null, "write"), "int", ["int", "pointer", "int"]); diff --git a/frida_mode/src/js/js.c b/frida_mode/src/js/js.c index e3cd4933..5f477388 100644 --- a/frida_mode/src/js/js.c +++ b/frida_mode/src/js/js.c @@ -1,14 +1,13 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "js.h" #include "util.h" -static char * js_script = NULL; gboolean js_done = FALSE; js_api_stalker_callback_t js_user_callback = NULL; +js_main_hook_t js_main_hook = NULL; +static char * js_script = NULL; static gchar * filename = "afl.js"; static gchar * contents; static GumScriptBackend * backend; @@ -25,7 +24,7 @@ static void js_msg(GumScript *script, const gchar *message, GBytes *data, UNUSED_PARAMETER(script); UNUSED_PARAMETER(data); UNUSED_PARAMETER(user_data); - OKF("%s", message); + FOKF("%s", message); } @@ -50,14 +49,14 @@ static gchar *js_get_script() { } else { - FATAL("Could not load script file: %s", filename); + FFATAL("Could not load script file: %s", filename); } } else { - OKF("Loaded AFL script: %s, %" G_GSIZE_MODIFIER "d bytes", filename, - length); + FOKF("Loaded AFL script: %s, %" G_GSIZE_MODIFIER "d bytes", filename, + length); gchar *source = g_malloc0(api_js_len + length + 1); memcpy(source, api_js, api_js_len); @@ -75,7 +74,7 @@ static void js_print_script(gchar *source) { for (size_t i = 0; split[i] != NULL; i++) { - OKF("%3" G_GSIZE_MODIFIER "d. %s", i + 1, split[i]); + FOKF("%3" G_GSIZE_MODIFIER "d. %s", i + 1, split[i]); } @@ -89,7 +88,7 @@ static void load_cb(GObject *source_object, GAsyncResult *result, UNUSED_PARAMETER(source_object); UNUSED_PARAMETER(user_data); gum_script_load_finish(script, result); - if (error != NULL) { FATAL("Failed to load script - %s", error->message); } + if (error != NULL) { FFATAL("Failed to load script - %s", error->message); } } @@ -99,7 +98,7 @@ static void create_cb(GObject *source_object, GAsyncResult *result, UNUSED_PARAMETER(source_object); UNUSED_PARAMETER(user_data); script = gum_script_backend_create_finish(backend, result, &error); - if (error != NULL) { FATAL("Failed to create script: %s", error->message); } + if (error != NULL) { FFATAL("Failed to create script: %s", error->message); } gum_script_set_message_handler(script, js_msg, NULL, NULL); @@ -128,7 +127,7 @@ void js_start(void) { while (g_main_context_pending(context)) g_main_context_iteration(context, FALSE); - if (!js_done) { FATAL("Script didn't call Afl.done()"); } + if (!js_done) { FFATAL("Script didn't call Afl.done()"); } } diff --git a/frida_mode/src/js/js_api.c b/frida_mode/src/js/js_api.c index 9dba79aa..5021b531 100644 --- a/frida_mode/src/js/js_api.c +++ b/frida_mode/src/js/js_api.c @@ -1,4 +1,3 @@ -#include "debug.h" #include "entry.h" #include "instrument.h" @@ -12,6 +11,10 @@ #include "stats.h" #include "util.h" +typedef uint8_t u8; + +extern void __afl_set_persistent_mode(u8 mode); + __attribute__((visibility("default"))) void js_api_done() { js_done = TRUE; @@ -20,7 +23,7 @@ __attribute__((visibility("default"))) void js_api_done() { __attribute__((visibility("default"))) void js_api_error(char *msg) { - FATAL("%s", msg); + FFATAL("%s", msg); } @@ -48,6 +51,8 @@ __attribute__((visibility("default"))) void js_api_set_persistent_address( persistent_start = GPOINTER_TO_SIZE(address); + __afl_set_persistent_mode(1); + } __attribute__((visibility("default"))) void js_api_set_persistent_return( @@ -231,3 +236,29 @@ __attribute__((visibility("default"))) void js_api_set_stalker_ic_entries( } +__attribute__((visibility("default"))) void js_api_set_traceable(void) { + + traceable = TRUE; + +} + +__attribute__((visibility("default"))) void js_api_set_backpatch_disable(void) { + + backpatch_enable = FALSE; + +} + +__attribute__((visibility("default"))) void js_api_set_stalker_adjacent_blocks( + guint val) { + + stalker_adjacent_blocks = val; + +} + +__attribute__((visibility("default"))) void js_api_set_js_main_hook( + const js_main_hook_t hook) { + + js_main_hook = hook; + +} + diff --git a/frida_mode/src/lib/lib.c b/frida_mode/src/lib/lib.c index 59a3fcf9..48d2ea2a 100644 --- a/frida_mode/src/lib/lib.c +++ b/frida_mode/src/lib/lib.c @@ -8,9 +8,8 @@ #include "frida-gumjs.h" - #include "debug.h" - #include "lib.h" + #include "util.h" #if defined(__arm__) || defined(__i386__) #define ELFCLASS ELFCLASS32 @@ -55,11 +54,11 @@ static gboolean lib_find_exe(const GumModuleDetails *details, static void lib_validate_hdr(Elf_Ehdr *hdr) { - if (hdr->e_ident[0] != ELFMAG0) FATAL("Invalid e_ident[0]"); - if (hdr->e_ident[1] != ELFMAG1) FATAL("Invalid e_ident[1]"); - if (hdr->e_ident[2] != ELFMAG2) FATAL("Invalid e_ident[2]"); - if (hdr->e_ident[3] != ELFMAG3) FATAL("Invalid e_ident[3]"); - if (hdr->e_ident[4] != ELFCLASS) FATAL("Invalid class"); + if (hdr->e_ident[0] != ELFMAG0) FFATAL("Invalid e_ident[0]"); + if (hdr->e_ident[1] != ELFMAG1) FFATAL("Invalid e_ident[1]"); + if (hdr->e_ident[2] != ELFMAG2) FFATAL("Invalid e_ident[2]"); + if (hdr->e_ident[3] != ELFMAG3) FFATAL("Invalid e_ident[3]"); + if (hdr->e_ident[4] != ELFCLASS) FFATAL("Invalid class"); } @@ -88,18 +87,22 @@ static void lib_read_text_section(lib_details_t *lib_details, Elf_Ehdr *hdr) { } - if (!found_preferred_base) { FATAL("Failed to find preferred load address"); } + if (!found_preferred_base) { + + FFATAL("Failed to find preferred load address"); + + } - OKF("Image preferred load address 0x%016" G_GSIZE_MODIFIER "x", - preferred_base); + FOKF("Image preferred load address 0x%016" G_GSIZE_MODIFIER "x", + preferred_base); shdr = (Elf_Shdr *)((char *)hdr + hdr->e_shoff); shstrtab = &shdr[hdr->e_shstrndx]; shstr = (char *)hdr + shstrtab->sh_offset; - OKF("shdr: %p", shdr); - OKF("shstrtab: %p", shstrtab); - OKF("shstr: %p", shstr); + FOKF("shdr: %p", shdr); + FOKF("shstrtab: %p", shstrtab); + FOKF("shstr: %p", shstr); for (size_t i = 0; i < hdr->e_shnum; i++) { @@ -108,16 +111,16 @@ static void lib_read_text_section(lib_details_t *lib_details, Elf_Ehdr *hdr) { if (curr->sh_name == 0) continue; section_name = &shstr[curr->sh_name]; - OKF("Section: %2" G_GSIZE_MODIFIER "u - base: 0x%016" G_GSIZE_MODIFIER - "X size: 0x%016" G_GSIZE_MODIFIER "X %s", - i, curr->sh_addr, curr->sh_size, section_name); + FOKF("Section: %2" G_GSIZE_MODIFIER "u - base: 0x%016" G_GSIZE_MODIFIER + "X size: 0x%016" G_GSIZE_MODIFIER "X %s", + i, curr->sh_addr, curr->sh_size, section_name); if (memcmp(section_name, text_name, sizeof(text_name)) == 0 && text_base == 0) { text_base = lib_details->base_address + curr->sh_addr - preferred_base; text_limit = text_base + curr->sh_size; - OKF("> text_addr: 0x%016" G_GINT64_MODIFIER "X", text_base); - OKF("> text_limit: 0x%016" G_GINT64_MODIFIER "X", text_limit); + FOKF("> text_addr: 0x%016" G_GINT64_MODIFIER "X", text_base); + FOKF("> text_limit: 0x%016" G_GINT64_MODIFIER "X", text_limit); } @@ -132,16 +135,16 @@ static void lib_get_text_section(lib_details_t *details) { Elf_Ehdr *hdr; fd = open(details->path, O_RDONLY); - if (fd < 0) { FATAL("Failed to open %s", details->path); } + if (fd < 0) { FFATAL("Failed to open %s", details->path); } len = lseek(fd, 0, SEEK_END); - if (len == (off_t)-1) { FATAL("Failed to lseek %s", details->path); } + if (len == (off_t)-1) { FFATAL("Failed to lseek %s", details->path); } - OKF("len: %ld", len); + FOKF("len: %ld", len); hdr = (Elf_Ehdr *)mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); - if (hdr == MAP_FAILED) { FATAL("Failed to map %s", details->path); } + if (hdr == MAP_FAILED) { FFATAL("Failed to map %s", details->path); } lib_validate_hdr(hdr); lib_read_text_section(details, hdr); @@ -159,22 +162,22 @@ void lib_init(void) { lib_details_t lib_details; gum_process_enumerate_modules(lib_find_exe, &lib_details); - OKF("Executable: 0x%016" G_GINT64_MODIFIER "x - %s", lib_details.base_address, - lib_details.path); + FOKF("Executable: 0x%016" G_GINT64_MODIFIER "x - %s", + lib_details.base_address, lib_details.path); lib_get_text_section(&lib_details); } guint64 lib_get_text_base(void) { - if (text_base == 0) FATAL("Lib not initialized"); + if (text_base == 0) FFATAL("Lib not initialized"); return text_base; } guint64 lib_get_text_limit(void) { - if (text_limit == 0) FATAL("Lib not initialized"); + if (text_limit == 0) FFATAL("Lib not initialized"); return text_limit; } diff --git a/frida_mode/src/lib/lib_apple.c b/frida_mode/src/lib/lib_apple.c index 2aa48a13..3bdb8c10 100644 --- a/frida_mode/src/lib/lib_apple.c +++ b/frida_mode/src/lib/lib_apple.c @@ -1,8 +1,6 @@ #ifdef __APPLE__ #include "frida-gumjs.h" - #include "debug.h" - #include "lib.h" #include "util.h" @@ -22,7 +20,7 @@ static gboolean lib_get_main_module(const GumModuleDetails *details, details->path, mach_task_self(), details->range->base_address, GUM_DARWIN_MODULE_FLAGS_NONE, NULL); - OKF("Found main module: %s", module->name); + FOKF("Found main module: %s", module->name); *ret = module; @@ -37,18 +35,18 @@ gboolean lib_get_text_section(const GumDarwinSectionDetails *details, static size_t idx = 0; char text_name[] = "__text"; - OKF("Section: %2lu - base: 0x%016" G_GINT64_MODIFIER - "X size: 0x%016" G_GINT64_MODIFIER "X %s", - idx++, details->vm_address, details->vm_address + details->size, - details->section_name); + FOKF("Section: %2lu - base: 0x%016" G_GINT64_MODIFIER + "X size: 0x%016" G_GINT64_MODIFIER "X %s", + idx++, details->vm_address, details->vm_address + details->size, + details->section_name); if (memcmp(details->section_name, text_name, sizeof(text_name)) == 0 && text_base == 0) { text_base = details->vm_address; text_limit = details->vm_address + details->size; - OKF("> text_addr: 0x%016" G_GINT64_MODIFIER "X", text_base); - OKF("> text_limit: 0x%016" G_GINT64_MODIFIER "X", text_limit); + FOKF("> text_addr: 0x%016" G_GINT64_MODIFIER "X", text_base); + FOKF("> text_limit: 0x%016" G_GINT64_MODIFIER "X", text_limit); } @@ -70,14 +68,14 @@ void lib_init(void) { guint64 lib_get_text_base(void) { - if (text_base == 0) FATAL("Lib not initialized"); + if (text_base == 0) FFATAL("Lib not initialized"); return text_base; } guint64 lib_get_text_limit(void) { - if (text_limit == 0) FATAL("Lib not initialized"); + if (text_limit == 0) FFATAL("Lib not initialized"); return text_limit; } diff --git a/frida_mode/src/main.c b/frida_mode/src/main.c index c0de9c6b..1be63bc4 100644 --- a/frida_mode/src/main.c +++ b/frida_mode/src/main.c @@ -6,6 +6,7 @@ #ifdef __APPLE__ #include <mach/mach.h> #include <mach-o/dyld_images.h> + #include <crt_externs.h> #else #include <sys/wait.h> #include <sys/personality.h> @@ -14,7 +15,6 @@ #include "frida-gumjs.h" #include "config.h" -#include "debug.h" #include "entry.h" #include "instrument.h" @@ -36,13 +36,13 @@ extern mach_port_t mach_task_self(); extern GumAddress gum_darwin_find_entrypoint(mach_port_t task); #else -extern int __libc_start_main(int *(main)(int, char **, char **), int argc, +extern int __libc_start_main(int (*main)(int, char **, char **), int argc, char **ubp_av, void (*init)(void), void (*fini)(void), void (*rtld_fini)(void), void(*stack_end)); #endif -typedef int *(*main_fn_t)(int argc, char **argv, char **envp); +typedef int (*main_fn_t)(int argc, char **argv, char **envp); static main_fn_t main_fn = NULL; @@ -62,7 +62,7 @@ static void on_main_os(int argc, char **argv, char **envp) { /* Personality doesn't affect the current process, it only takes effect on * evec */ int persona = personality(ADDR_NO_RANDOMIZE); - if (persona == -1) { WARNF("Failed to set ADDR_NO_RANDOMIZE: %d", errno); } + if (persona == -1) { FWARNF("Failed to set ADDR_NO_RANDOMIZE: %d", errno); } if ((persona & ADDR_NO_RANDOMIZE) == 0) { execvpe(argv[0], argv, envp); } GumInterceptor *interceptor = gum_interceptor_obtain(); @@ -90,13 +90,14 @@ static void embedded_init(void) { static void afl_print_cmdline(void) { +#if defined(__linux__) char * buffer = g_malloc0(PROC_MAX); gchar *fname = g_strdup_printf("/proc/%d/cmdline", getppid()); int fd = open(fname, O_RDONLY); if (fd < 0) { - WARNF("Failed to open /proc/self/cmdline, errno: (%d)", errno); + FWARNF("Failed to open /proc/self/cmdline, errno: (%d)", errno); return; } @@ -104,7 +105,7 @@ static void afl_print_cmdline(void) { ssize_t bytes_read = read(fd, buffer, PROC_MAX - 1); if (bytes_read < 0) { - FATAL("Failed to read /proc/self/cmdline, errno: (%d)", errno); + FFATAL("Failed to read /proc/self/cmdline, errno: (%d)", errno); } @@ -114,7 +115,7 @@ static void afl_print_cmdline(void) { if (i == 0 || buffer[i - 1] == '\0') { - OKF("AFL - COMMANDLINE: argv[%d] = %s", idx++, &buffer[i]); + FOKF("AFL - COMMANDLINE: argv[%d] = %s", idx++, &buffer[i]); } @@ -123,6 +124,18 @@ static void afl_print_cmdline(void) { close(fd); g_free(fname); g_free(buffer); +#elif defined(__APPLE__) + int idx; + char **argv = *_NSGetArgv(); + int nargv = *_NSGetArgc(); + + for (idx = 0; idx < nargv; idx++) { + + FOKF("AFL - COMMANDLINE: argv[%d] = %s", idx, argv[idx]); + + } + +#endif } @@ -134,7 +147,7 @@ static void afl_print_env(void) { if (fd < 0) { - WARNF("Failed to open /proc/self/cmdline, errno: (%d)", errno); + FWARNF("Failed to open /proc/self/cmdline, errno: (%d)", errno); return; } @@ -142,7 +155,7 @@ static void afl_print_env(void) { ssize_t bytes_read = read(fd, buffer, PROC_MAX - 1); if (bytes_read < 0) { - FATAL("Failed to read /proc/self/cmdline, errno: (%d)", errno); + FFATAL("Failed to read /proc/self/cmdline, errno: (%d)", errno); } @@ -152,7 +165,7 @@ static void afl_print_env(void) { if (i == 0 || buffer[i - 1] == '\0') { - OKF("AFL - ENVIRONMENT %3d: %s", idx++, &buffer[i]); + FOKF("AFL - ENVIRONMENT %3d: %s", idx++, &buffer[i]); } @@ -204,7 +217,9 @@ __attribute__((visibility("default"))) void afl_frida_start(void) { } -static int *on_main(int argc, char **argv, char **envp) { +static int on_main(int argc, char **argv, char **envp) { + + int ret; on_main_os(argc, argv, envp); @@ -212,12 +227,22 @@ static int *on_main(int argc, char **argv, char **envp) { afl_frida_start(); - return main_fn(argc, argv, envp); + if (js_main_hook != NULL) { + + ret = js_main_hook(argc, argv, envp); + + } else { + + ret = main_fn(argc, argv, envp); + + } + + return ret; } #if defined(EMBEDDED) -extern int *main(int argc, char **argv, char **envp); +extern int main(int argc, char **argv, char **envp); static void intercept_main(void) { @@ -230,9 +255,9 @@ static void intercept_main(void) { static void intercept_main(void) { mach_port_t task = mach_task_self(); - OKF("Task Id: %u", task); + FOKF("Task Id: %u", task); GumAddress entry = gum_darwin_find_entrypoint(task); - OKF("Entry Point: 0x%016" G_GINT64_MODIFIER "x", entry); + FOKF("Entry Point: 0x%016" G_GINT64_MODIFIER "x", entry); void *main = GSIZE_TO_POINTER(entry); main_fn = main; intercept_hook(main, on_main, NULL); @@ -240,7 +265,7 @@ static void intercept_main(void) { } #else -static int on_libc_start_main(int *(main)(int, char **, char **), int argc, +static int on_libc_start_main(int (*main)(int, char **, char **), int argc, char **ubp_av, void (*init)(void), void (*fini)(void), void (*rtld_fini)(void), void(*stack_end)) { diff --git a/frida_mode/src/output.c b/frida_mode/src/output.c index e2b744e7..f570fe91 100644 --- a/frida_mode/src/output.c +++ b/frida_mode/src/output.c @@ -4,9 +4,8 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "output.h" +#include "util.h" char *output_stdout = NULL; char *output_stderr = NULL; @@ -19,18 +18,18 @@ static void output_redirect(int fd, char *filename) { path = g_canonicalize_filename(filename, g_get_current_dir()); - OKF("Redirect %d -> '%s'", fd, path); + FOKF("Redirect %d -> '%s'", fd, path); int output_fd = open(path, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); g_free(path); - if (output_fd < 0) { FATAL("Failed to open fd(%d) error %d", fd, errno); } + if (output_fd < 0) { FFATAL("Failed to open fd(%d) error %d", fd, errno); } if (dup2(output_fd, fd) < 0) { - FATAL("Failed to set fd(%d) error %d", fd, errno); + FFATAL("Failed to set fd(%d) error %d", fd, errno); } @@ -47,8 +46,8 @@ void output_config(void) { void output_init(void) { - OKF("Output - StdOut: %s", output_stdout); - OKF("Output - StdErr: %s", output_stderr); + FOKF("Output - StdOut: %s", output_stdout); + FOKF("Output - StdErr: %s", output_stderr); output_redirect(STDOUT_FILENO, output_stdout); output_redirect(STDERR_FILENO, output_stderr); diff --git a/frida_mode/src/persistent/persistent.c b/frida_mode/src/persistent/persistent.c index b2915a2f..817d9925 100644 --- a/frida_mode/src/persistent/persistent.c +++ b/frida_mode/src/persistent/persistent.c @@ -3,7 +3,6 @@ #include "frida-gumjs.h" #include "config.h" -#include "debug.h" #include "entry.h" #include "persistent.h" @@ -23,15 +22,15 @@ gboolean persistent_debug = FALSE; void persistent_config(void) { hook_name = getenv("AFL_FRIDA_PERSISTENT_HOOK"); - persistent_start = util_read_address("AFL_FRIDA_PERSISTENT_ADDR"); - persistent_count = util_read_num("AFL_FRIDA_PERSISTENT_CNT"); - persistent_ret = util_read_address("AFL_FRIDA_PERSISTENT_RET"); + persistent_start = util_read_address("AFL_FRIDA_PERSISTENT_ADDR", 0); + persistent_count = util_read_num("AFL_FRIDA_PERSISTENT_CNT", 0); + persistent_ret = util_read_address("AFL_FRIDA_PERSISTENT_RET", 0); if (getenv("AFL_FRIDA_PERSISTENT_DEBUG") != NULL) { persistent_debug = TRUE; } if (persistent_count != 0 && persistent_start == 0) { - FATAL( + FFATAL( "AFL_FRIDA_PERSISTENT_ADDR must be specified if " "AFL_FRIDA_PERSISTENT_CNT is"); @@ -40,11 +39,11 @@ void persistent_config(void) { if (persistent_start != 0 && persistent_count == 0) persistent_count = 1000; if (persistent_start != 0 && !persistent_is_supported()) - FATAL("Persistent mode not supported on this architecture"); + FFATAL("Persistent mode not supported on this architecture"); if (persistent_ret != 0 && persistent_start == 0) { - FATAL( + FFATAL( "AFL_FRIDA_PERSISTENT_ADDR must be specified if " "AFL_FRIDA_PERSISTENT_RET is"); @@ -54,33 +53,33 @@ void persistent_config(void) { void *hook_obj = dlopen(hook_name, RTLD_NOW); if (hook_obj == NULL) - FATAL("Failed to load AFL_FRIDA_PERSISTENT_HOOK (%s)", hook_name); + FFATAL("Failed to load AFL_FRIDA_PERSISTENT_HOOK (%s)", hook_name); int (*afl_persistent_hook_init_ptr)(void) = dlsym(hook_obj, "afl_persistent_hook_init"); if (afl_persistent_hook_init_ptr == NULL) - FATAL("Failed to find afl_persistent_hook_init in %s", hook_name); + FFATAL("Failed to find afl_persistent_hook_init in %s", hook_name); if (afl_persistent_hook_init_ptr() == 0) - FATAL("afl_persistent_hook_init returned a failure"); + FFATAL("afl_persistent_hook_init returned a failure"); persistent_hook = (afl_persistent_hook_fn)dlsym(hook_obj, "afl_persistent_hook"); if (persistent_hook == NULL) - FATAL("Failed to find afl_persistent_hook in %s", hook_name); + FFATAL("Failed to find afl_persistent_hook in %s", hook_name); } void persistent_init(void) { - OKF("Instrumentation - persistent mode [%c] (0x%016" G_GINT64_MODIFIER "X)", - persistent_start == 0 ? ' ' : 'X', persistent_start); - OKF("Instrumentation - persistent count [%c] (%" G_GINT64_MODIFIER "d)", - persistent_start == 0 ? ' ' : 'X', persistent_count); - OKF("Instrumentation - hook [%s]", hook_name); + FOKF("Instrumentation - persistent mode [%c] (0x%016" G_GINT64_MODIFIER "X)", + persistent_start == 0 ? ' ' : 'X', persistent_start); + FOKF("Instrumentation - persistent count [%c] (%" G_GINT64_MODIFIER "d)", + persistent_start == 0 ? ' ' : 'X', persistent_count); + FOKF("Instrumentation - hook [%s]", hook_name); - OKF("Instrumentation - persistent ret [%c] (0x%016" G_GINT64_MODIFIER "X)", - persistent_ret == 0 ? ' ' : 'X', persistent_ret); + FOKF("Instrumentation - persistent ret [%c] (0x%016" G_GINT64_MODIFIER "X)", + persistent_ret == 0 ? ' ' : 'X', persistent_ret); if (persistent_hook != NULL) { __afl_sharedmem_fuzzing = 1; } @@ -88,7 +87,7 @@ void persistent_init(void) { void persistent_prologue(GumStalkerOutput *output) { - OKF("AFL_FRIDA_PERSISTENT_ADDR reached"); + FOKF("AFL_FRIDA_PERSISTENT_ADDR reached"); entry_compiled = TRUE; ranges_exclude(); stalker_trust(); @@ -98,7 +97,7 @@ void persistent_prologue(GumStalkerOutput *output) { void persistent_epilogue(GumStalkerOutput *output) { - OKF("AFL_FRIDA_PERSISTENT_RET reached"); + FOKF("AFL_FRIDA_PERSISTENT_RET reached"); persistent_epilogue_arch(output); } diff --git a/frida_mode/src/persistent/persistent_arm32.c b/frida_mode/src/persistent/persistent_arm32.c index 769f1505..b4e50897 100644 --- a/frida_mode/src/persistent/persistent_arm32.c +++ b/frida_mode/src/persistent/persistent_arm32.c @@ -1,7 +1,5 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "persistent.h" #include "util.h" @@ -64,14 +62,14 @@ gboolean persistent_is_supported(void) { void persistent_prologue_arch(GumStalkerOutput *output) { UNUSED_PARAMETER(output); - FATAL("Persistent mode not supported on this architecture"); + FFATAL("Persistent mode not supported on this architecture"); } void persistent_epilogue_arch(GumStalkerOutput *output) { UNUSED_PARAMETER(output); - FATAL("Persistent mode not supported on this architecture"); + FFATAL("Persistent mode not supported on this architecture"); } diff --git a/frida_mode/src/persistent/persistent_arm64.c b/frida_mode/src/persistent/persistent_arm64.c index 3cd61cd5..c9159ca1 100644 --- a/frida_mode/src/persistent/persistent_arm64.c +++ b/frida_mode/src/persistent/persistent_arm64.c @@ -2,7 +2,6 @@ #include "frida-gumjs.h" #include "config.h" -#include "debug.h" #include "instrument.h" #include "persistent.h" @@ -325,7 +324,7 @@ void persistent_prologue_arch(GumStalkerOutput *output) { gconstpointer loop = cw->code + 1; - OKF("Persistent loop reached"); + FOKF("Persistent loop reached"); instrument_persitent_save_regs(cw, &saved_regs); diff --git a/frida_mode/src/persistent/persistent_x64.c b/frida_mode/src/persistent/persistent_x64.c index c0bd9a09..8cbde633 100644 --- a/frida_mode/src/persistent/persistent_x64.c +++ b/frida_mode/src/persistent/persistent_x64.c @@ -2,7 +2,6 @@ #include "frida-gumjs.h" #include "config.h" -#include "debug.h" #include "instrument.h" #include "persistent.h" @@ -270,7 +269,7 @@ void persistent_prologue_arch(GumStalkerOutput *output) { gconstpointer loop = cw->code + 1; - OKF("Persistent loop reached"); + FOKF("Persistent loop reached"); /* Pop the return value */ gum_x86_writer_put_lea_reg_reg_offset(cw, GUM_REG_RSP, GUM_REG_RSP, 8); diff --git a/frida_mode/src/persistent/persistent_x86.c b/frida_mode/src/persistent/persistent_x86.c index b911676a..5425b01b 100644 --- a/frida_mode/src/persistent/persistent_x86.c +++ b/frida_mode/src/persistent/persistent_x86.c @@ -1,10 +1,10 @@ #include "frida-gumjs.h" #include "config.h" -#include "debug.h" #include "instrument.h" #include "persistent.h" +#include "util.h" #if defined(__i386__) @@ -210,7 +210,7 @@ void persistent_prologue_arch(GumStalkerOutput *output) { gconstpointer loop = cw->code + 1; - OKF("Persistent loop reached"); + FOKF("Persistent loop reached"); /* Pop the return value */ gum_x86_writer_put_lea_reg_reg_offset(cw, GUM_REG_ESP, GUM_REG_ESP, 4); diff --git a/frida_mode/src/prefetch.c b/frida_mode/src/prefetch.c index 0efbc9bf..8c9ce94d 100644 --- a/frida_mode/src/prefetch.c +++ b/frida_mode/src/prefetch.c @@ -4,8 +4,6 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "entry.h" #include "intercept.h" #include "prefetch.h" @@ -44,8 +42,9 @@ static void gum_afl_stalker_backpatcher_notify(GumStalkerObserver *self, sizeof(prefetch_data->backpatch_data) - prefetch_data->backpatch_size; if (sizeof(gsize) + size > remaining) { return; } - *(gsize *)(&prefetch_data->backpatch_data[prefetch_data->backpatch_size]) = - size; + gsize *dst_backpatch_size = + (gsize *)&prefetch_data->backpatch_data[prefetch_data->backpatch_size]; + *dst_backpatch_size = size; prefetch_data->backpatch_size += sizeof(gsize); memcpy(&prefetch_data->backpatch_data[prefetch_data->backpatch_size], @@ -115,12 +114,13 @@ static void prefetch_read_patches(void) { remaining > sizeof(gsize); remaining = prefetch_data->backpatch_size - offset) { - gsize size = *(gsize *)(&prefetch_data->backpatch_data[offset]); + gsize *src_backpatch_data = (gsize *)&prefetch_data->backpatch_data[offset]; + gsize size = *src_backpatch_data; offset += sizeof(gsize); if (prefetch_data->backpatch_size - offset < size) { - FATAL("Incomplete backpatch entry"); + FFATAL("Incomplete backpatch entry"); } @@ -178,9 +178,9 @@ static void prefetch_hook_fork(void) { void prefetch_init(void) { - OKF("Instrumentation - prefetch [%c]", prefetch_enable ? 'X' : ' '); - OKF("Instrumentation - prefetch_backpatch [%c]", - prefetch_backpatch ? 'X' : ' '); + FOKF("Instrumentation - prefetch [%c]", prefetch_enable ? 'X' : ' '); + FOKF("Instrumentation - prefetch_backpatch [%c]", + prefetch_backpatch ? 'X' : ' '); if (!prefetch_enable) { return; } /* @@ -192,7 +192,7 @@ void prefetch_init(void) { shmget(IPC_PRIVATE, sizeof(prefetch_data_t), IPC_CREAT | IPC_EXCL | 0600); if (prefetch_shm_id < 0) { - FATAL("prefetch_shm_id < 0 - errno: %d\n", errno); + FFATAL("prefetch_shm_id < 0 - errno: %d\n", errno); } @@ -204,7 +204,7 @@ void prefetch_init(void) { */ if (shmctl(prefetch_shm_id, IPC_RMID, NULL) < 0) { - FATAL("shmctl (IPC_RMID) < 0 - errno: %d\n", errno); + FFATAL("shmctl (IPC_RMID) < 0 - errno: %d\n", errno); } diff --git a/frida_mode/src/ranges.c b/frida_mode/src/ranges.c index 5b6eb462..9844c74c 100644 --- a/frida_mode/src/ranges.c +++ b/frida_mode/src/ranges.c @@ -1,7 +1,5 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "lib.h" #include "ranges.h" #include "stalker.h" @@ -37,8 +35,8 @@ static void convert_address_token(gchar *token, GumMemoryRange *range) { if (token_count != 2) { - FATAL("Invalid range (should have two addresses seperated by a '-'): %s\n", - token); + FFATAL("Invalid range (should have two addresses seperated by a '-'): %s\n", + token); } @@ -47,15 +45,15 @@ static void convert_address_token(gchar *token, GumMemoryRange *range) { if (!g_str_has_prefix(from_str, "0x")) { - FATAL("Invalid range: %s - Start address should have 0x prefix: %s\n", - token, from_str); + FFATAL("Invalid range: %s - Start address should have 0x prefix: %s\n", + token, from_str); } if (!g_str_has_prefix(to_str, "0x")) { - FATAL("Invalid range: %s - End address should have 0x prefix: %s\n", token, - to_str); + FFATAL("Invalid range: %s - End address should have 0x prefix: %s\n", token, + to_str); } @@ -66,8 +64,8 @@ static void convert_address_token(gchar *token, GumMemoryRange *range) { if (!g_ascii_isxdigit(*c)) { - FATAL("Invalid range: %s - Start address not formed of hex digits: %s\n", - token, from_str); + FFATAL("Invalid range: %s - Start address not formed of hex digits: %s\n", + token, from_str); } @@ -77,8 +75,8 @@ static void convert_address_token(gchar *token, GumMemoryRange *range) { if (!g_ascii_isxdigit(*c)) { - FATAL("Invalid range: %s - End address not formed of hex digits: %s\n", - token, to_str); + FFATAL("Invalid range: %s - End address not formed of hex digits: %s\n", + token, to_str); } @@ -87,24 +85,25 @@ static void convert_address_token(gchar *token, GumMemoryRange *range) { guint64 from = g_ascii_strtoull(from_str, NULL, 16); if (from == 0) { - FATAL("Invalid range: %s - Start failed hex conversion: %s\n", token, - from_str); + FFATAL("Invalid range: %s - Start failed hex conversion: %s\n", token, + from_str); } guint64 to = g_ascii_strtoull(to_str, NULL, 16); if (to == 0) { - FATAL("Invalid range: %s - End failed hex conversion: %s\n", token, to_str); + FFATAL("Invalid range: %s - End failed hex conversion: %s\n", token, + to_str); } if (from >= to) { - FATAL("Invalid range: %s - Start (0x%016" G_GINT64_MODIFIER - "x) must be less than end " - "(0x%016" G_GINT64_MODIFIER "x)\n", - token, from, to); + FFATAL("Invalid range: %s - Start (0x%016" G_GINT64_MODIFIER + "x) must be less than end " + "(0x%016" G_GINT64_MODIFIER "x)\n", + token, from, to); } @@ -123,10 +122,10 @@ static gboolean convert_name_token_for_module(const GumModuleDetails *details, if (!g_str_has_suffix(details->path, ctx->suffix)) { return true; }; - OKF("Found module - prefix: %s, 0x%016" G_GINT64_MODIFIER - "x-0x%016" G_GINT64_MODIFIER "x %s", - ctx->suffix, details->range->base_address, - details->range->base_address + details->range->size, details->path); + FOKF("Found module - prefix: %s, 0x%016" G_GINT64_MODIFIER + "x-0x%016" G_GINT64_MODIFIER "x %s", + ctx->suffix, details->range->base_address, + details->range->base_address + details->range->size, details->path); *ctx->range = *details->range; ctx->done = true; @@ -140,7 +139,7 @@ static void convert_name_token(gchar *token, GumMemoryRange *range) { convert_name_ctx_t ctx = {.suffix = suffix, .range = range, .done = false}; gum_process_enumerate_modules(convert_name_token_for_module, &ctx); - if (!ctx.done) { FATAL("Failed to resolve module: %s\n", token); } + if (!ctx.done) { FFATAL("Failed to resolve module: %s\n", token); } g_free(suffix); } @@ -159,16 +158,30 @@ static void convert_token(gchar *token, GumMemoryRange *range) { } - OKF("Converted token: %s -> 0x%016" G_GINT64_MODIFIER - "x-0x%016" G_GINT64_MODIFIER "x\n", - token, range->base_address, range->base_address + range->size); + FOKF("Converted token: %s -> 0x%016" G_GINT64_MODIFIER + "x-0x%016" G_GINT64_MODIFIER "x\n", + token, range->base_address, range->base_address + range->size); } gint range_sort(gconstpointer a, gconstpointer b) { - return ((GumMemoryRange *)a)->base_address - - ((GumMemoryRange *)b)->base_address; + GumMemoryRange *ra = (GumMemoryRange *)a; + GumMemoryRange *rb = (GumMemoryRange *)b; + + if (ra->base_address < rb->base_address) { + + return -1; + + } else if (ra->base_address > rb->base_address) { + + return 1; + + } else { + + return 0; + + } } @@ -179,24 +192,24 @@ static gboolean print_ranges_callback(const GumRangeDetails *details, if (details->file == NULL) { - OKF("MAP - 0x%016" G_GINT64_MODIFIER "x - 0x%016" G_GINT64_MODIFIER - "X %c%c%c", - details->range->base_address, - details->range->base_address + details->range->size, - details->protection & GUM_PAGE_READ ? 'R' : '-', - details->protection & GUM_PAGE_WRITE ? 'W' : '-', - details->protection & GUM_PAGE_EXECUTE ? 'X' : '-'); + FOKF("MAP - 0x%016" G_GINT64_MODIFIER "x - 0x%016" G_GINT64_MODIFIER + "X %c%c%c", + details->range->base_address, + details->range->base_address + details->range->size, + details->protection & GUM_PAGE_READ ? 'R' : '-', + details->protection & GUM_PAGE_WRITE ? 'W' : '-', + details->protection & GUM_PAGE_EXECUTE ? 'X' : '-'); } else { - OKF("MAP - 0x%016" G_GINT64_MODIFIER "x - 0x%016" G_GINT64_MODIFIER - "X %c%c%c %s(0x%016" G_GINT64_MODIFIER "x)", - details->range->base_address, - details->range->base_address + details->range->size, - details->protection & GUM_PAGE_READ ? 'R' : '-', - details->protection & GUM_PAGE_WRITE ? 'W' : '-', - details->protection & GUM_PAGE_EXECUTE ? 'X' : '-', details->file->path, - details->file->offset); + FOKF("MAP - 0x%016" G_GINT64_MODIFIER "x - 0x%016" G_GINT64_MODIFIER + "X %c%c%c %s(0x%016" G_GINT64_MODIFIER "x)", + details->range->base_address, + details->range->base_address + details->range->size, + details->protection & GUM_PAGE_READ ? 'R' : '-', + details->protection & GUM_PAGE_WRITE ? 'W' : '-', + details->protection & GUM_PAGE_EXECUTE ? 'X' : '-', + details->file->path, details->file->offset); } @@ -206,14 +219,14 @@ static gboolean print_ranges_callback(const GumRangeDetails *details, static void print_ranges(char *key, GArray *ranges) { - OKF("Range: %s Length: %d", key, ranges->len); + FOKF("Range: %s Length: %d", key, ranges->len); for (guint i = 0; i < ranges->len; i++) { GumMemoryRange *curr = &g_array_index(ranges, GumMemoryRange, i); GumAddress curr_limit = curr->base_address + curr->size; - OKF("Range: %s Idx: %3d - 0x%016" G_GINT64_MODIFIER - "x-0x%016" G_GINT64_MODIFIER "x", - key, i, curr->base_address, curr_limit); + FOKF("Range: %s Idx: %3d - 0x%016" G_GINT64_MODIFIER + "x-0x%016" G_GINT64_MODIFIER "x", + key, i, curr->base_address, curr_limit); } @@ -250,10 +263,10 @@ static void check_for_overlaps(GArray *array) { GumAddress curr_limit = curr->base_address + curr->size; if (prev_limit > curr->base_address) { - FATAL("OVerlapping ranges 0x%016" G_GINT64_MODIFIER - "x-0x%016" G_GINT64_MODIFIER "x 0x%016" G_GINT64_MODIFIER - "x-0x%016" G_GINT64_MODIFIER "x", - prev->base_address, prev_limit, curr->base_address, curr_limit); + FFATAL("Overlapping ranges 0x%016" G_GINT64_MODIFIER + "x-0x%016" G_GINT64_MODIFIER "x 0x%016" G_GINT64_MODIFIER + "x-0x%016" G_GINT64_MODIFIER "x", + prev->base_address, prev_limit, curr->base_address, curr_limit); } @@ -549,18 +562,19 @@ static GArray *merge_ranges(GArray *a) { } +void ranges_print_debug_maps(void) { + + gum_process_enumerate_ranges(GUM_PAGE_NO_ACCESS, print_ranges_callback, NULL); + +} + void ranges_config(void) { if (getenv("AFL_FRIDA_DEBUG_MAPS") != NULL) { ranges_debug_maps = TRUE; } if (getenv("AFL_INST_LIBS") != NULL) { ranges_inst_libs = TRUE; } if (getenv("AFL_FRIDA_INST_JIT") != NULL) { ranges_inst_jit = TRUE; } - if (ranges_debug_maps) { - - gum_process_enumerate_ranges(GUM_PAGE_NO_ACCESS, print_ranges_callback, - NULL); - - } + if (ranges_debug_maps) { ranges_print_debug_maps(); } include_ranges = collect_ranges("AFL_FRIDA_INST_RANGES"); exclude_ranges = collect_ranges("AFL_FRIDA_EXCLUDE_RANGES"); @@ -576,13 +590,13 @@ void ranges_init(void) { GArray * step4; GArray * step5; - OKF("Ranges - Instrument jit [%c]", ranges_inst_jit ? 'X' : ' '); - OKF("Ranges - Instrument libraries [%c]", ranges_inst_libs ? 'X' : ' '); + FOKF("Ranges - Instrument jit [%c]", ranges_inst_jit ? 'X' : ' '); + FOKF("Ranges - Instrument libraries [%c]", ranges_inst_libs ? 'X' : ' '); print_ranges("AFL_FRIDA_INST_RANGES", include_ranges); print_ranges("AFL_FRIDA_EXCLUDE_RANGES", exclude_ranges); - OKF("Ranges - Instrument libraries [%c]", ranges_inst_libs ? 'X' : ' '); + FOKF("Ranges - Instrument libraries [%c]", ranges_inst_libs ? 'X' : ' '); print_ranges("AFL_FRIDA_INST_RANGES", include_ranges); print_ranges("AFL_FRIDA_EXCLUDE_RANGES", exclude_ranges); @@ -659,7 +673,7 @@ void ranges_exclude() { GumMemoryRange *r; GumStalker * stalker = stalker_get(); - OKF("Excluding ranges"); + FOKF("Excluding ranges"); for (guint i = 0; i < ranges->len; i++) { diff --git a/frida_mode/src/seccomp/seccomp.c b/frida_mode/src/seccomp/seccomp.c index 7683cd71..9d8fdd5d 100644 --- a/frida_mode/src/seccomp/seccomp.c +++ b/frida_mode/src/seccomp/seccomp.c @@ -1,123 +1,19 @@ -#include <execinfo.h> -#include <fcntl.h> -#include <linux/seccomp.h> -#include <stdio.h> -#include <unistd.h> - #include "frida-gumjs.h" -#include "debug.h" - #include "seccomp.h" #include "util.h" char *seccomp_filename = NULL; -static void seccomp_vprint(int fd, char *format, va_list ap) { - - char buffer[4096] = {0}; - int len; - - if (vsnprintf(buffer, sizeof(buffer) - 1, format, ap) < 0) { return; } - - len = strnlen(buffer, sizeof(buffer)); - IGNORED_RETURN(write(fd, buffer, len)); - -} - -void seccomp_print(char *format, ...) { - - va_list ap; - va_start(ap, format); - seccomp_vprint(SECCOMP_OUTPUT_FILE_FD, format, ap); - va_end(ap); - -} - -static void seccomp_filter_callback(struct seccomp_notif * req, - struct seccomp_notif_resp *resp, - GumReturnAddressArray * frames) { - - GumDebugSymbolDetails details = {0}; - if (req->data.nr == SYS_OPENAT) { - - seccomp_print("SYS_OPENAT: (%s)\n", (char *)req->data.args[1]); - - } - - seccomp_print( - "\nID (%#llx) for PID %d - %d (%s) [0x%llx 0x%llx 0x%llx 0x%llx 0x%llx " - "0x%llx ]\n", - req->id, req->pid, req->data.nr, seccomp_syscall_lookup(req->data.nr), - req->data.args[0], req->data.args[1], req->data.args[2], - req->data.args[3], req->data.args[4], req->data.args[5]); - - seccomp_print("FRAMES: (%u)\n", frames->len); - char **syms = backtrace_symbols(frames->items, frames->len); - if (syms == NULL) { FATAL("Failed to get symbols"); } - - for (guint i = 0; i < frames->len; i++) { - - if (gum_symbol_details_from_address(frames->items[i], &details)) { - - seccomp_print("\t%3d. %s!%s\n", i, details.module_name, - details.symbol_name); - - } else { - - seccomp_print("\t%3d. %s\n", i, syms[i]); - - } - - } - - free(syms); - - resp->error = 0; - resp->val = 0; - resp->id = req->id; - resp->flags = SECCOMP_USER_NOTIF_FLAG_CONTINUE; - -} - -static void seccomp_child(int signal_parent, void *ctx) { - - int sock_fd = *((int *)ctx); - int fd = seccomp_socket_recv(sock_fd); - - if (close(sock_fd) < 0) { FATAL("child - close"); } - - seccomp_event_signal(signal_parent); - seccomp_filter_child_install(); - seccomp_filter_run(fd, seccomp_filter_callback); - -} - void seccomp_on_fork(void) { - int sock[2] = {-1, -1}; - pid_t child = -1; - int child_fd = -1; - if (seccomp_filename == NULL) { return; } - seccomp_socket_create(sock); - seccomp_child_run(seccomp_child, sock, &child, &child_fd); - - if (dup2(child_fd, SECCOMP_PARENT_EVENT_FD) < 0) { FATAL("dup2"); } - - if (close(child_fd) < 0) { FATAL("seccomp_on_fork - close (1)"); } - - if (close(sock[STDIN_FILENO]) < 0) { FATAL("grandparent - close (2)"); } - - int fd = seccomp_filter_install(child); - seccomp_socket_send(sock[STDOUT_FILENO], fd); - - if (close(sock[STDOUT_FILENO]) < 0) { FATAL("grandparent - close (3)"); } - - if (close(fd) < 0) { FATAL("grandparent - close (4)"); } - - seccomp_child_wait(SECCOMP_PARENT_EVENT_FD); +#ifdef __APPLE__ + FFATAL("Seccomp not supported on OSX"); +#else + seccomp_callback_parent(); +#endif } @@ -129,29 +25,15 @@ void seccomp_config(void) { void seccomp_init(void) { - char *path = NULL; - int fd; - - OKF("Seccomp - file [%s]", seccomp_filename); + FOKF("Seccomp - file [%s]", seccomp_filename); if (seccomp_filename == NULL) { return; } - path = g_canonicalize_filename(seccomp_filename, g_get_current_dir()); - - OKF("Seccomp - path [%s]", path); - - fd = open(path, O_RDWR | O_CREAT | O_TRUNC, - S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); - - if (dup2(fd, SECCOMP_OUTPUT_FILE_FD) < 0) { - - FATAL("Failed to duplicate seccomp output file"); - - } - - if (close(fd) < 0) { FATAL("Failed to close seccomp output file fd"); } - - g_free(path); +#ifdef __APPLE__ + FFATAL("Seccomp not supported on OSX"); +#else + seccomp_callback_initialize(); +#endif } diff --git a/frida_mode/src/seccomp/seccomp_atomic.c b/frida_mode/src/seccomp/seccomp_atomic.c index 1720a726..18cb6724 100644 --- a/frida_mode/src/seccomp/seccomp_atomic.c +++ b/frida_mode/src/seccomp/seccomp_atomic.c @@ -1,13 +1,15 @@ -#include <stdbool.h> -#include <stdio.h> +#if defined(__linux__) && !defined(__ANDROID__) -#include "debug.h" + #include <stdbool.h> + #include <stdio.h> + + #include "util.h" void seccomp_atomic_set(volatile bool *ptr, bool val) { if (!__sync_bool_compare_and_swap(ptr, !val, val)) { - FATAL("Failed to set event"); + FFATAL("Failed to set event"); } @@ -26,3 +28,5 @@ void seccomp_atomic_wait(volatile bool *ptr, bool val) { } +#endif + diff --git a/frida_mode/src/seccomp/seccomp_callback.c b/frida_mode/src/seccomp/seccomp_callback.c new file mode 100644 index 00000000..f7aaf78b --- /dev/null +++ b/frida_mode/src/seccomp/seccomp_callback.c @@ -0,0 +1,145 @@ +#if defined(__linux__) && !defined(__ANDROID__) + + #if !defined(__MUSL__) + #include <execinfo.h> + #endif + #include <fcntl.h> + + #include "seccomp.h" + #include "util.h" + +static void seccomp_callback_filter(struct seccomp_notif * req, + struct seccomp_notif_resp *resp, + GumReturnAddressArray * frames) { + + GumDebugSymbolDetails details = {0}; + if (req->data.nr == SYS_OPENAT) { + + #if UINTPTR_MAX == 0xffffffffffffffffu + seccomp_print("SYS_OPENAT: (%s)\n", (char *)req->data.args[1]); + #endif + #if UINTPTR_MAX == 0xffffffff + seccomp_print("SYS_OPENAT: (%s)\n", (char *)(__u32)req->data.args[1]); + #endif + + } + + seccomp_print( + "\nID (%#llx) for PID %d - %d (%s) [0x%llx 0x%llx 0x%llx 0x%llx 0x%llx " + "0x%llx ]\n", + req->id, req->pid, req->data.nr, seccomp_syscall_lookup(req->data.nr), + req->data.args[0], req->data.args[1], req->data.args[2], + req->data.args[3], req->data.args[4], req->data.args[5]); + + #if !defined(__MUSL__) + seccomp_print("FRAMES: (%u)\n", frames->len); + char **syms = backtrace_symbols(frames->items, frames->len); + if (syms == NULL) { FFATAL("Failed to get symbols"); } + + for (guint i = 0; i < frames->len; i++) { + + if (gum_symbol_details_from_address(frames->items[i], &details)) { + + seccomp_print("\t%3d. %s!%s\n", i, details.module_name, + details.symbol_name); + + } else { + + seccomp_print("\t%3d. %s\n", i, syms[i]); + + } + + } + + free(syms); + #else + void **syms = (void **)__builtin_frame_address(0); + void * framep = __builtin_frame_address(1); + int i = 0; + + syms = framep; + while (syms) { + + framep = *syms; + syms = framep; + + if (!syms) break; + + seccomp_print("\%3d. %s\n", i++, (char *)framep); + + } + + #endif + + resp->error = 0; + resp->val = 0; + resp->id = req->id; + resp->flags = SECCOMP_USER_NOTIF_FLAG_CONTINUE; + +} + +static void seccomp_callback_child(int signal_parent, void *ctx) { + + int sock_fd = *((int *)ctx); + int fd = seccomp_socket_recv(sock_fd); + + if (close(sock_fd) < 0) { FFATAL("child - close"); } + + seccomp_event_signal(signal_parent); + seccomp_filter_child_install(); + seccomp_filter_run(fd, seccomp_callback_filter); + +} + +void seccomp_callback_parent(void) { + + int sock[2] = {-1, -1}; + pid_t child = -1; + int child_fd = -1; + + seccomp_socket_create(sock); + seccomp_child_run(seccomp_callback_child, sock, &child, &child_fd); + + if (dup2(child_fd, SECCOMP_PARENT_EVENT_FD) < 0) { FFATAL("dup2"); } + + if (close(child_fd) < 0) { FFATAL("seccomp_on_fork - close (1)"); } + + if (close(sock[STDIN_FILENO]) < 0) { FFATAL("grandparent - close (2)"); } + + int fd = seccomp_filter_install(child); + seccomp_socket_send(sock[STDOUT_FILENO], fd); + + if (close(sock[STDOUT_FILENO]) < 0) { FFATAL("grandparent - close (3)"); } + + if (close(fd) < 0) { FFATAL("grandparent - close (4)"); } + + seccomp_child_wait(SECCOMP_PARENT_EVENT_FD); + +} + +void seccomp_callback_initialize(void) { + + char *path = NULL; + int fd; + + path = g_canonicalize_filename(seccomp_filename, g_get_current_dir()); + + FOKF("Seccomp - path [%s]", path); + + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); + + if (dup2(fd, SECCOMP_OUTPUT_FILE_FD) < 0) { + + FFATAL("Failed to duplicate seccomp output file"); + + } + + if (close(fd) < 0) { FFATAL("Failed to close seccomp output file fd"); } + + g_free(path); + +} + +#endif + diff --git a/frida_mode/src/seccomp/seccomp_child.c b/frida_mode/src/seccomp/seccomp_child.c index 4d494137..c02ef67c 100644 --- a/frida_mode/src/seccomp/seccomp_child.c +++ b/frida_mode/src/seccomp/seccomp_child.c @@ -1,18 +1,19 @@ -#include <fcntl.h> -#include <sched.h> -#include <signal.h> -#include <stdio.h> -#include <stdlib.h> -#include <sys/mman.h> -#include <sys/prctl.h> -#include <sys/types.h> -#include <unistd.h> +#if defined(__linux__) && !defined(__ANDROID__) -#include "debug.h" + #include <fcntl.h> + #include <sched.h> + #include <signal.h> + #include <stdio.h> + #include <stdlib.h> + #include <sys/mman.h> + #include <sys/prctl.h> + #include <sys/types.h> + #include <unistd.h> -#include "seccomp.h" + #include "seccomp.h" + #include "util.h" -#define SECCOMP_CHILD_STACK_SIZE (1UL << 20) + #define SECCOMP_CHILD_STACK_SIZE (1UL << 20) typedef void (*seccomp_child_func_t)(int event_fd, void *ctx); @@ -49,11 +50,11 @@ void seccomp_child_run(seccomp_child_func_t child_func, void *ctx, pid_t *child, char *stack = (char *)mmap(NULL, SECCOMP_CHILD_STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (stack == MAP_FAILED) { FATAL("mmap"); } + if (stack == MAP_FAILED) { FFATAL("mmap"); } pid_t child_pid = clone(seccomp_child_func, &stack[SECCOMP_CHILD_STACK_SIZE], flags, child_ctx, NULL, NULL, NULL); - if (child_pid < 0) { FATAL("clone"); } + if (child_pid < 0) { FFATAL("clone"); } if (child != NULL) { *child = child_pid; } if (event_fd != NULL) { *event_fd = fd; } @@ -67,3 +68,5 @@ void seccomp_child_wait(int event_fd) { } +#endif + diff --git a/frida_mode/src/seccomp/seccomp_event.c b/frida_mode/src/seccomp/seccomp_event.c index ecb9be32..e6585f1d 100644 --- a/frida_mode/src/seccomp/seccomp_event.c +++ b/frida_mode/src/seccomp/seccomp_event.c @@ -1,16 +1,23 @@ -#include <stdint.h> -#include <stdio.h> -#include <sys/eventfd.h> -#include <unistd.h> +#if defined(__linux__) && !defined(__ANDROID__) -#include "debug.h" + #include <stdint.h> + #include <stdio.h> + #include <sys/syscall.h> + #include <unistd.h> -#include "seccomp.h" + #include "seccomp.h" + #include "util.h" int seccomp_event_create(void) { - int fd = eventfd(0, 0); - if (fd < 0) { FATAL("seccomp_event_create"); } + #ifdef SYS_eventfd + int fd = syscall(SYS_eventfd, 0, 0); + #else + #ifdef SYS_eventfd2 + int fd = syscall(SYS_eventfd2, 0, 0); + #endif + #endif + if (fd < 0) { FFATAL("seccomp_event_create"); } return fd; } @@ -20,7 +27,7 @@ void seccomp_event_signal(int fd) { uint64_t val = 1; if (write(fd, &val, sizeof(uint64_t)) != sizeof(uint64_t)) { - FATAL("seccomp_event_signal"); + FFATAL("seccomp_event_signal"); } @@ -31,7 +38,7 @@ void seccomp_event_wait(int fd) { uint64_t val = 1; if (read(fd, &val, sizeof(uint64_t)) != sizeof(uint64_t)) { - FATAL("seccomp_event_wait"); + FFATAL("seccomp_event_wait"); } @@ -39,7 +46,9 @@ void seccomp_event_wait(int fd) { void seccomp_event_destroy(int fd) { - if (close(fd) < 0) { FATAL("seccomp_event_destroy"); } + if (close(fd) < 0) { FFATAL("seccomp_event_destroy"); } } +#endif + diff --git a/frida_mode/src/seccomp/seccomp_filter.c b/frida_mode/src/seccomp/seccomp_filter.c index c16e7ebd..1d050303 100644 --- a/frida_mode/src/seccomp/seccomp_filter.c +++ b/frida_mode/src/seccomp/seccomp_filter.c @@ -1,27 +1,28 @@ -#include <alloca.h> -#include <errno.h> -#include <execinfo.h> -#include <linux/filter.h> -#include <linux/seccomp.h> -#include <sys/ioctl.h> -#include <sys/prctl.h> -#include <sys/syscall.h> -#include <signal.h> -#include <stdbool.h> -#include <stddef.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> - -#include "debug.h" - -#include "frida-gumjs.h" - -#include "seccomp.h" -#include "util.h" - -#define SECCOMP_FILTER_NUM_FRAMES 512 +#if defined(__linux__) && !defined(__ANDROID__) + + #include <alloca.h> + #include <errno.h> + #if !defined(__MUSL__) + #include <execinfo.h> + #endif + #include <linux/filter.h> + #include <sys/ioctl.h> + #include <sys/prctl.h> + #include <sys/syscall.h> + #include <signal.h> + #include <stdbool.h> + #include <stddef.h> + #include <stdio.h> + #include <stdlib.h> + #include <string.h> + #include <unistd.h> + + #include "frida-gumjs.h" + + #include "seccomp.h" + #include "util.h" + + #define SECCOMP_FILTER_NUM_FRAMES 512 extern void gum_linux_parse_ucontext(const ucontext_t *uc, GumCpuContext *ctx); @@ -71,7 +72,13 @@ static struct sock_filter filter[] = { /* Allow us to make anonymous maps */ BPF_STMT(BPF_LD | BPF_W | BPF_ABS, (offsetof(struct seccomp_data, nr))), + #ifdef __NR_mmap BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_mmap, 0, 3), + #else + #ifdef __NR_mmap2 + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_mmap2, 0, 3), + #endif + #endif BPF_STMT(BPF_LD | BPF_W | BPF_ABS, (offsetof(struct seccomp_data, args[4]))), BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, -1, 0, 1), @@ -127,7 +134,10 @@ static GumBacktracer * seccomp_filter_backtracer = NULL; static void seccomp_filter_child_handler(int sig, siginfo_t *info, void *ucontext) { - GumCpuContext cpu_context; + UNUSED_PARAMETER(sig); + UNUSED_PARAMETER(info); + UNUSED_PARAMETER(ucontext); + if (seccomp_filter_backtracer == NULL) { seccomp_filter_backtracer = gum_backtracer_make_fuzzy(); @@ -150,9 +160,10 @@ static void seccomp_filter_parent_handler(int sig, siginfo_t *info, ucontext_t *uc = (ucontext_t *)ucontext; gum_linux_parse_ucontext(uc, &seccomp_filter_cpu_context); - if (tgkill(seccomp_filter_child, seccomp_filter_child, SIGUSR1) < 0) { + if (syscall(SYS_tgkill, seccomp_filter_child, seccomp_filter_child, SIGUSR1) < + 0) { - FATAL("kill"); + FFATAL("kill"); } @@ -165,7 +176,7 @@ void seccomp_filter_child_install(void) { const struct sigaction sa = {.sa_sigaction = seccomp_filter_child_handler, .sa_flags = SA_SIGINFO | SA_RESTART}; - if (sigaction(SIGUSR1, &sa, NULL) < 0) { FATAL("sigaction"); } + if (sigaction(SIGUSR1, &sa, NULL) < 0) { FFATAL("sigaction"); } } @@ -180,17 +191,17 @@ int seccomp_filter_install(pid_t child) { .len = sizeof(filter) / sizeof(struct sock_filter), .filter = filter}; - if (sigaction(SIGUSR1, &sa, NULL) < 0) { FATAL("sigaction"); } + if (sigaction(SIGUSR1, &sa, NULL) < 0) { FFATAL("sigaction"); } if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) { - FATAL("PR_SET_NO_NEW_PRIVS %d", errno); + FFATAL("PR_SET_NO_NEW_PRIVS %d", errno); } int fd = syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_NEW_LISTENER, &filter_prog); - if (fd < 0) { FATAL("SYS_seccomp %d", fd); } + if (fd < 0) { FFATAL("SYS_seccomp %d", fd); } return fd; @@ -204,19 +215,19 @@ void seccomp_filter_run(int fd, seccomp_filter_callback_t callback) { if (syscall(SYS_seccomp, SECCOMP_GET_NOTIF_SIZES, 0, &sizes) == -1) { - FATAL("seccomp-SECCOMP_GET_NOTIF_SIZES"); + FFATAL("seccomp-SECCOMP_GET_NOTIF_SIZES"); } if (sizes.seccomp_notif != sizeof(struct seccomp_notif)) { - FATAL("size - seccomp_notif"); + FFATAL("size - seccomp_notif"); } if (sizes.seccomp_notif_resp != sizeof(struct seccomp_notif_resp)) { - FATAL("size - seccomp_notif"); + FFATAL("size - seccomp_notif"); } @@ -230,7 +241,7 @@ void seccomp_filter_run(int fd, seccomp_filter_callback_t callback) { if (ioctl(fd, SECCOMP_IOCTL_NOTIF_RECV, req) < 0) { if (errno == EINTR) { continue; } - FATAL("SECCOMP_IOCTL_NOTIF_RECV: %d\n", fd); + FFATAL("SECCOMP_IOCTL_NOTIF_RECV: %d\n", fd); } @@ -240,14 +251,14 @@ void seccomp_filter_run(int fd, seccomp_filter_callback_t callback) { } else { - if (kill(req->pid, SIGUSR1) < 0) { FATAL("kill"); } + if (kill(req->pid, SIGUSR1) < 0) { FFATAL("kill"); } } if (ioctl(fd, SECCOMP_IOCTL_NOTIF_SEND, resp) < 0) { if (errno == ENOENT) { continue; } - OKF("SECCOMP_IOCTL_NOTIF_SEND"); + FOKF("SECCOMP_IOCTL_NOTIF_SEND"); continue; } @@ -256,3 +267,5 @@ void seccomp_filter_run(int fd, seccomp_filter_callback_t callback) { } +#endif + diff --git a/frida_mode/src/seccomp/seccomp_print.c b/frida_mode/src/seccomp/seccomp_print.c new file mode 100644 index 00000000..3cea1239 --- /dev/null +++ b/frida_mode/src/seccomp/seccomp_print.c @@ -0,0 +1,30 @@ +#if defined(__linux__) && !defined(__ANDROID__) + + #include <stdarg.h> + + #include "seccomp.h" + #include "util.h" + +static void seccomp_print_v(int fd, char *format, va_list ap) { + + char buffer[4096] = {0}; + int len; + + if (vsnprintf(buffer, sizeof(buffer) - 1, format, ap) < 0) { return; } + + len = strnlen(buffer, sizeof(buffer)); + IGNORED_RETURN(write(fd, buffer, len)); + +} + +void seccomp_print(char *format, ...) { + + va_list ap; + va_start(ap, format); + seccomp_print_v(SECCOMP_OUTPUT_FILE_FD, format, ap); + va_end(ap); + +} + +#endif + diff --git a/frida_mode/src/seccomp/seccomp_socket.c b/frida_mode/src/seccomp/seccomp_socket.c index ca42e158..a01e88ee 100644 --- a/frida_mode/src/seccomp/seccomp_socket.c +++ b/frida_mode/src/seccomp/seccomp_socket.c @@ -1,11 +1,12 @@ -#include <stdio.h> -#include <string.h> -#include <sys/socket.h> -#include <unistd.h> +#if defined(__linux__) && !defined(__ANDROID__) -#include "debug.h" + #include <stdio.h> + #include <string.h> + #include <sys/socket.h> + #include <unistd.h> -#include "seccomp.h" + #include "seccomp.h" + #include "util.h" union cmsg { @@ -19,31 +20,31 @@ void seccomp_socket_create(int *sock) { int tmp_sock[2] = {-1, -1}; if (socketpair(AF_UNIX, SOCK_STREAM, 0, tmp_sock) < 0) { - FATAL("socketpair"); + FFATAL("socketpair"); } if (dup2(tmp_sock[STDIN_FILENO], SECCOMP_SOCKET_RECV_FD) < 0) { - FATAL("seccomp_socket_create - dup2 (1)"); + FFATAL("seccomp_socket_create - dup2 (1)"); } if (dup2(tmp_sock[STDOUT_FILENO], SECCOMP_SOCKET_SEND_FD) < 0) { - FATAL("seccomp_socket_create - dup2 (1)"); + FFATAL("seccomp_socket_create - dup2 (1)"); } if (close(tmp_sock[STDIN_FILENO]) < 0) { - FATAL("seccomp_socket_create - close (1)"); + FFATAL("seccomp_socket_create - close (1)"); } if (close(tmp_sock[STDOUT_FILENO]) < 0) { - FATAL("seccomp_socket_create - close (2)"); + FFATAL("seccomp_socket_create - close (2)"); } @@ -74,7 +75,7 @@ void seccomp_socket_send(int sockfd, int fd) { memcpy(CMSG_DATA(&control_msg.hdr), &fd, sizeof(int)); - if (sendmsg(sockfd, &message, 0) == -1) { FATAL("sendmsg"); } + if (sendmsg(sockfd, &message, 0) == -1) { FFATAL("sendmsg"); } } @@ -93,23 +94,23 @@ int seccomp_socket_recv(int sockfd) { int fd; - if (recvmsg(sockfd, &message, 0) < 0) { FATAL("recvmsg"); } + if (recvmsg(sockfd, &message, 0) < 0) { FFATAL("recvmsg"); } if (control_msg.hdr.cmsg_len != CMSG_LEN(sizeof(int))) { - FATAL("control_msg.hdr.cmsg_len"); + FFATAL("control_msg.hdr.cmsg_len"); } if (control_msg.hdr.cmsg_level != SOL_SOCKET) { - FATAL("control_msg.hdr.cmsg_level"); + FFATAL("control_msg.hdr.cmsg_level"); } if (control_msg.hdr.cmsg_type != SCM_RIGHTS) { - FATAL("control_msg.hdr.cmsg_type"); + FFATAL("control_msg.hdr.cmsg_type"); } @@ -119,3 +120,5 @@ int seccomp_socket_recv(int sockfd) { } +#endif + diff --git a/frida_mode/src/seccomp/seccomp_syscall.c b/frida_mode/src/seccomp/seccomp_syscall.c index b2c084c8..2eac1af3 100644 --- a/frida_mode/src/seccomp/seccomp_syscall.c +++ b/frida_mode/src/seccomp/seccomp_syscall.c @@ -1,9 +1,10 @@ -#include <limits.h> -#include <stdio.h> +#if defined(__linux__) && !defined(__ANDROID__) -#include "debug.h" + #include <limits.h> + #include <stdio.h> -#include "seccomp.h" + #include "seccomp.h" + #include "util.h" typedef struct { @@ -322,10 +323,10 @@ static syscall_entry_t seccomp_syscall_table[] = { char *seccomp_syscall_lookup(int id) { - if (id < 0) { FATAL("Invalid id: %d", id); } + if (id < 0) { FFATAL("Invalid id: %d", id); } if ((uint32_t)id >= sizeof(seccomp_syscall_table) / sizeof(syscall_entry_t)) { - FATAL("Invalid id: %d", id); + FFATAL("Invalid id: %d", id); } @@ -333,3 +334,5 @@ char *seccomp_syscall_lookup(int id) { } +#endif + diff --git a/frida_mode/src/stalker.c b/frida_mode/src/stalker.c index 814aaeb3..caa16b3f 100644 --- a/frida_mode/src/stalker.c +++ b/frida_mode/src/stalker.c @@ -1,4 +1,3 @@ -#include "debug.h" #include "instrument.h" #include "prefetch.h" @@ -6,7 +5,9 @@ #include "stats.h" #include "util.h" -guint stalker_ic_entries = 0; +guint stalker_ic_entries = 0; +gboolean backpatch_enable = TRUE; +guint stalker_adjacent_blocks = 0; static GumStalker *stalker = NULL; @@ -56,9 +57,14 @@ static void gum_afl_stalker_observer_init(GumAflStalkerObserver *self) { void stalker_config(void) { - if (!gum_stalker_is_supported()) { FATAL("Failed to initialize embedded"); } + if (!gum_stalker_is_supported()) { FFATAL("Failed to initialize embedded"); } - stalker_ic_entries = util_read_num("AFL_FRIDA_STALKER_IC_ENTRIES"); + backpatch_enable = (getenv("AFL_FRIDA_INST_NO_BACKPATCH") == NULL); + + stalker_ic_entries = util_read_num("AFL_FRIDA_STALKER_IC_ENTRIES", 32); + + stalker_adjacent_blocks = + util_read_num("AFL_FRIDA_STALKER_ADJACENT_BLOCKS", 32); observer = g_object_new(GUM_TYPE_AFL_STALKER_OBSERVER, NULL); @@ -87,27 +93,50 @@ static gboolean stalker_exclude_self(const GumRangeDetails *details, void stalker_init(void) { - OKF("Stalker - ic_entries [%u]", stalker_ic_entries); + FOKF("Instrumentation - backpatch [%c]", backpatch_enable ? 'X' : ' '); + + FOKF("Stalker - ic_entries [%u]", stalker_ic_entries); + FOKF("Stalker - adjacent_blocks [%u]", stalker_adjacent_blocks); #if !(defined(__x86_64__) || defined(__i386__)) - if (stalker_ic_entries != 0) { + if (getenv("AFL_FRIDA_STALKER_IC_ENTRIES") != NULL) { + + FFATAL("AFL_FRIDA_STALKER_IC_ENTRIES not supported"); + + } + + if (getenv("AFL_FRIDA_STALKER_ADJACENT_BLOCKS") != NULL) { - FATAL("AFL_FRIDA_STALKER_IC_ENTRIES not supported"); + FFATAL("AFL_FRIDA_STALKER_ADJACENT_BLOCKS not supported"); } #endif - if (stalker_ic_entries == 0) { stalker_ic_entries = 32; } + if (instrument_coverage_filename != NULL) { + + if (getenv("AFL_FRIDA_STALKER_ADJACENT_BLOCKS") != NULL) { + + FFATAL( + "AFL_FRIDA_STALKER_ADJACENT_BLOCKS and AFL_FRIDA_INST_COVERAGE_FILE " + "are incompatible"); + + } else { + + stalker_adjacent_blocks = 0; + + } + + } #if defined(__x86_64__) || defined(__i386__) - stalker = - g_object_new(GUM_TYPE_STALKER, "ic-entries", stalker_ic_entries, NULL); + stalker = g_object_new(GUM_TYPE_STALKER, "ic-entries", stalker_ic_entries, + "adjacent-blocks", stalker_adjacent_blocks, NULL); #else stalker = gum_stalker_new(); #endif - if (stalker == NULL) { FATAL("Failed to initialize stalker"); } + if (stalker == NULL) { FFATAL("Failed to initialize stalker"); } gum_stalker_set_trust_threshold(stalker, -1); @@ -118,7 +147,7 @@ void stalker_init(void) { GumStalker *stalker_get(void) { - if (stalker == NULL) { FATAL("Stalker uninitialized"); } + if (stalker == NULL) { FFATAL("Stalker uninitialized"); } return stalker; } @@ -134,13 +163,13 @@ void stalker_start(void) { void stalker_trust(void) { - gum_stalker_set_trust_threshold(stalker, 0); + if (backpatch_enable) { gum_stalker_set_trust_threshold(stalker, 0); } } GumStalkerObserver *stalker_get_observer(void) { - if (observer == NULL) { FATAL("Stalker not yet initialized"); } + if (observer == NULL) { FFATAL("Stalker not yet initialized"); } return GUM_STALKER_OBSERVER(observer); } diff --git a/frida_mode/src/stats/stats.c b/frida_mode/src/stats/stats.c index 7972b881..83ecf89a 100644 --- a/frida_mode/src/stats/stats.c +++ b/frida_mode/src/stats/stats.c @@ -8,7 +8,6 @@ #include "frida-gumjs.h" #include "config.h" -#include "debug.h" #include "util.h" #include "entry.h" @@ -324,42 +323,42 @@ static void stats_observer_init(GumStalkerObserver *observer) { void stats_config(void) { stats_filename = getenv("AFL_FRIDA_STATS_FILE"); - stats_interval = util_read_num("AFL_FRIDA_STATS_INTERVAL"); + stats_interval = util_read_num("AFL_FRIDA_STATS_INTERVAL", 10); } void stats_init(void) { - OKF("Stats - file [%s]", stats_filename); - OKF("Stats - interval [%" G_GINT64_MODIFIER "u]", stats_interval); + FOKF("Stats - file [%s]", stats_filename); + FOKF("Stats - interval [%" G_GINT64_MODIFIER "u]", stats_interval); - if (stats_interval != 0 && stats_filename == NULL) { + if (getenv("AFL_FRIDA_STATS_INTERVAL") != NULL && + getenv("AFL_FRIDA_STATS_FILE") == NULL) { - FATAL( + FFATAL( "AFL_FRIDA_STATS_FILE must be specified if " "AFL_FRIDA_STATS_INTERVAL is"); } - if (stats_interval == 0) { stats_interval = 10; } stats_interval_us = stats_interval * MICRO_TO_SEC; if (stats_filename == NULL) { return; } char *path = g_canonicalize_filename(stats_filename, g_get_current_dir()); - OKF("Stats - path [%s]", path); + FOKF("Stats - path [%s]", path); stats_fd = open(path, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP); - if (stats_fd < 0) { FATAL("Failed to open stats file '%s'", path); } + if (stats_fd < 0) { FFATAL("Failed to open stats file '%s'", path); } g_free(path); int shm_id = shmget(IPC_PRIVATE, sizeof(stats_data_t), IPC_CREAT | IPC_EXCL | 0600); - if (shm_id < 0) { FATAL("shm_id < 0 - errno: %d\n", errno); } + if (shm_id < 0) { FFATAL("shm_id < 0 - errno: %d\n", errno); } stats_data = shmat(shm_id, NULL, 0); g_assert(stats_data != MAP_FAILED); @@ -372,7 +371,7 @@ void stats_init(void) { */ if (shmctl(shm_id, IPC_RMID, NULL) < 0) { - FATAL("shmctl (IPC_RMID) < 0 - errno: %d\n", errno); + FFATAL("shmctl (IPC_RMID) < 0 - errno: %d\n", errno); } diff --git a/frida_mode/src/stats/stats_arm32.c b/frida_mode/src/stats/stats_arm32.c index 5860d33b..bd652aa3 100644 --- a/frida_mode/src/stats/stats_arm32.c +++ b/frida_mode/src/stats/stats_arm32.c @@ -1,7 +1,5 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "stats.h" #include "util.h" @@ -9,13 +7,13 @@ void starts_arch_init(void) { - FATAL("Stats not supported on this architecture"); + FFATAL("Stats not supported on this architecture"); } void stats_write_arch(stats_data_t *data) { - FATAL("Stats not supported on this architecture"); + FFATAL("Stats not supported on this architecture"); } @@ -23,7 +21,7 @@ void stats_collect_arch(const cs_insn *instr, gboolean begin) { UNUSED_PARAMETER(instr); UNUSED_PARAMETER(begin); - FATAL("Stats not supported on this architecture"); + FFATAL("Stats not supported on this architecture"); } diff --git a/frida_mode/src/stats/stats_arm64.c b/frida_mode/src/stats/stats_arm64.c index 54b3faf1..313ab47a 100644 --- a/frida_mode/src/stats/stats_arm64.c +++ b/frida_mode/src/stats/stats_arm64.c @@ -3,8 +3,6 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "ranges.h" #include "stats.h" #include "util.h" @@ -48,7 +46,7 @@ void starts_arch_init(void) { int shm_id = shmget(IPC_PRIVATE, sizeof(stats_data_arch_t), IPC_CREAT | IPC_EXCL | 0600); - if (shm_id < 0) { FATAL("shm_id < 0 - errno: %d\n", errno); } + if (shm_id < 0) { FFATAL("shm_id < 0 - errno: %d\n", errno); } stats_data_arch = shmat(shm_id, NULL, 0); g_assert(stats_data_arch != MAP_FAILED); @@ -58,7 +56,7 @@ void starts_arch_init(void) { */ if (shmctl(shm_id, IPC_RMID, NULL) < 0) { - FATAL("shmctl (IPC_RMID) < 0 - errno: %d\n", errno); + FFATAL("shmctl (IPC_RMID) < 0 - errno: %d\n", errno); } diff --git a/frida_mode/src/stats/stats_x86_64.c b/frida_mode/src/stats/stats_x86_64.c index ab914951..0bfe3baa 100644 --- a/frida_mode/src/stats/stats_x86_64.c +++ b/frida_mode/src/stats/stats_x86_64.c @@ -3,8 +3,6 @@ #include "frida-gumjs.h" -#include "debug.h" - #include "ranges.h" #include "stats.h" #include "util.h" @@ -50,7 +48,7 @@ void starts_arch_init(void) { int shm_id = shmget(IPC_PRIVATE, sizeof(stats_data_arch_t), IPC_CREAT | IPC_EXCL | 0600); - if (shm_id < 0) { FATAL("shm_id < 0 - errno: %d\n", errno); } + if (shm_id < 0) { FFATAL("shm_id < 0 - errno: %d\n", errno); } stats_data_arch = shmat(shm_id, NULL, 0); g_assert(stats_data_arch != MAP_FAILED); @@ -60,7 +58,7 @@ void starts_arch_init(void) { */ if (shmctl(shm_id, IPC_RMID, NULL) < 0) { - FATAL("shmctl (IPC_RMID) < 0 - errno: %d\n", errno); + FFATAL("shmctl (IPC_RMID) < 0 - errno: %d\n", errno); } @@ -255,8 +253,8 @@ static x86_op_type stats_get_operand_type(const cs_insn *instr) { if (x86->op_count != 1) { - FATAL("Unexpected operand count (%d): %s %s\n", x86->op_count, - instr->mnemonic, instr->op_str); + FFATAL("Unexpected operand count (%d): %s %s\n", x86->op_count, + instr->mnemonic, instr->op_str); } @@ -295,7 +293,7 @@ static void stats_collect_call_arch(const cs_insn *instr) { stats_data_arch->num_call_mem++; break; default: - FATAL("Invalid operand type: %s %s\n", instr->mnemonic, instr->op_str); + FFATAL("Invalid operand type: %s %s\n", instr->mnemonic, instr->op_str); } @@ -316,7 +314,7 @@ static void stats_collect_jump_arch(const cs_insn *instr) { stats_data_arch->num_jmp_mem++; break; default: - FATAL("Invalid operand type: %s %s\n", instr->mnemonic, instr->op_str); + FFATAL("Invalid operand type: %s %s\n", instr->mnemonic, instr->op_str); } @@ -337,7 +335,7 @@ static void stats_collect_jump_cond_arch(const cs_insn *instr) { stats_data_arch->num_jmp_cond_mem++; break; default: - FATAL("Invalid operand type: %s %s\n", instr->mnemonic, instr->op_str); + FFATAL("Invalid operand type: %s %s\n", instr->mnemonic, instr->op_str); } diff --git a/frida_mode/src/util.c b/frida_mode/src/util.c index 09e8a58b..6f52b6cb 100644 --- a/frida_mode/src/util.c +++ b/frida_mode/src/util.c @@ -1,12 +1,11 @@ #include "util.h" -#include "debug.h" - -guint64 util_read_address(char *key) { +guint64 util_read_address(char *key, guint64 default_value) { char *value_str = getenv(key); + char *end_ptr; - if (value_str == NULL) { return 0; } + if (value_str == NULL) { return default_value; } if (!g_str_has_prefix(value_str, "0x")) { @@ -27,8 +26,17 @@ guint64 util_read_address(char *key) { } - guint64 value = g_ascii_strtoull(value_str2, NULL, 16); - if (value == 0) { + errno = 0; + + guint64 value = g_ascii_strtoull(value_str2, &end_ptr, 16); + + if (errno != 0) { + + FATAL("Error (%d) during conversion: %s", errno, value_str); + + } + + if (value == 0 && end_ptr == value_str2) { FATAL("Invalid address failed hex conversion: %s=%s\n", key, value_str2); @@ -38,11 +46,12 @@ guint64 util_read_address(char *key) { } -guint64 util_read_num(char *key) { +guint64 util_read_num(char *key, guint64 default_value) { char *value_str = getenv(key); + char *end_ptr; - if (value_str == NULL) { return 0; } + if (value_str == NULL) { return default_value; } for (char *c = value_str; *c != '\0'; c++) { @@ -55,8 +64,17 @@ guint64 util_read_num(char *key) { } + errno = 0; + guint64 value = g_ascii_strtoull(value_str, NULL, 10); - if (value == 0) { + + if (errno != 0) { + + FATAL("Error (%d) during conversion: %s", errno, value_str); + + } + + if (value == 0 && end_ptr == value_str) { FATAL("Invalid address failed numeric conversion: %s=%s\n", key, value_str); @@ -66,3 +84,40 @@ guint64 util_read_num(char *key) { } +gboolean util_output_enabled(void) { + + static gboolean initialized = FALSE; + static gboolean enabled = TRUE; + + if (!initialized) { + + initialized = TRUE; + if (getenv("AFL_DEBUG_CHILD") == NULL) { enabled = FALSE; } + + } + + return enabled; + +} + +gsize util_rotate(gsize val, gsize shift, gsize size) { + + if (shift == 0) { return val; } + gsize result = ((val >> shift) | (val << (size - shift))); + result = result & ((1 << size) - 1); + return result; + +} + +gsize util_log2(gsize val) { + + for (gsize i = 0; i < 64; i++) { + + if (((gsize)1 << i) == val) { return i; } + + } + + FFATAL("Not a power of two"); + +} + diff --git a/frida_mode/test/bloaty/GNUmakefile b/frida_mode/test/bloaty/GNUmakefile new file mode 100644 index 00000000..7ef4fe81 --- /dev/null +++ b/frida_mode/test/bloaty/GNUmakefile @@ -0,0 +1,114 @@ +PWD:=$(shell pwd)/ +ROOT:=$(PWD)../../../ +BUILD_DIR:=$(PWD)build/ + +AFLPP_FRIDA_DRIVER_HOOK_OBJ=$(ROOT)frida_mode/build/frida_hook.so +AFLPP_QEMU_DRIVER_HOOK_OBJ=$(ROOT)frida_mode/build/qemu_hook.so + +# LIBFUZZER_LIB:=/usr/lib/llvm-12/lib/libFuzzer.a + +BLOATY_GIT_REPO:=https://github.com/google/bloaty.git +BLOATY_DIR:=$(BUILD_DIR)bloaty/ +TEST_BIN:=$(BLOATY_DIR)fuzz_target + +ifeq "$(shell uname)" "Darwin" +TEST_BIN_LDFLAGS:=-undefined dynamic_lookup -Wl,-no_pie +endif + +TEST_DATA_DIR:=$(BUILD_DIR)in/ +TEST_DATA_SRC:=$(BLOATY_DIR)tests/testdata/fuzz_corpus/ +DUMMY_DATA_FILE:=$(BUILD_DIR)default_seed + +FRIDA_OUT:=$(BUILD_DIR)frida-out +QEMU_OUT:=$(BUILD_DIR)qemu-out + +ifndef ARCH + +ARCH=$(shell uname -m) +ifeq "$(ARCH)" "aarch64" + ARCH:=arm64 +endif + +ifeq "$(ARCH)" "i686" + ARCH:=x86 +endif +endif + +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh + +AFL_QEMU_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x4000000000) + +ifeq "$(ARCH)" "aarch64" + AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x0000aaaaaaaaa000) +endif + +ifeq "$(ARCH)" "x86_64" + AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x0000555555554000) +endif + +ifeq "$(ARCH)" "x86" + AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x56555000) +endif + +.PHONY: all clean frida hook + +all: $(TEST_BIN) + make -C $(ROOT)frida_mode/ + +32: + CXXFLAGS="-m32" LDFLAGS="-m32" ARCH="x86" make all + +$(BUILD_DIR): + mkdir -p $@ + +########## BLOATY ####### + +$(BLOATY_DIR): | $(BUILD_DIR) + git clone --depth 1 $(BLOATY_GIT_REPO) $@ + +$(TEST_BIN): $(BLOATY_DIR) + cd $(BLOATY_DIR) && CC=clang CXX=clang++ CCC=clang++ LIB_FUZZING_ENGINE="-fsanitize=fuzzer" cmake -G Ninja -DBUILD_TESTING=false $(BLOATY_DIR) + cd $(BLOATY_DIR) && CC=clang CXX=clang++ CCC=clang++ ninja -j $(shell nproc) + +########## DUMMY ####### + +$(TEST_DATA_DIR): | $(BLOATY_DIR) $(BUILD_DIR) + cp -av $(TEST_DATA_SRC) $@ + +$(DUMMY_DATA_FILE): | $(TEST_DATA_DIR) + dd if=/dev/zero bs=1048576 count=1 of=$@ + +###### TEST DATA ####### + +clean: + rm -rf $(BUILD_DIR) + +frida: $(TEST_BIN) $(AFLPP_FRIDA_DRIVER_HOOK_OBJ) $(TEST_DATA_FILE) $(DUMMY_DATA_FILE) + AFL_FRIDA_PERSISTENT_CNT=1000000 \ + AFL_FRIDA_PERSISTENT_HOOK=$(AFLPP_FRIDA_DRIVER_HOOK_OBJ) \ + AFL_FRIDA_PERSISTENT_ADDR=$(AFL_FRIDA_PERSISTENT_ADDR) \ + AFL_ENTRYPOINT=$(AFL_FRIDA_PERSISTENT_ADDR) \ + $(ROOT)afl-fuzz \ + -i $(TEST_DATA_DIR) \ + -o $(FRIDA_OUT) \ + -m none \ + -d \ + -O \ + -V 30 \ + -- \ + $(TEST_BIN) $(DUMMY_DATA_FILE) + +qemu: $(TEST_BIN) $(AFLPP_QEMU_DRIVER_HOOK_OBJ) $(TEST_DATA_FILE) $(DUMMY_DATA_FILE) + AFL_QEMU_PERSISTENT_CNT=1000000 \ + AFL_QEMU_PERSISTENT_HOOK=$(AFLPP_QEMU_DRIVER_HOOK_OBJ) \ + AFL_QEMU_PERSISTENT_ADDR=$(AFL_QEMU_PERSISTENT_ADDR) \ + AFL_ENTRYPOINT=$(AFL_QEMU_PERSISTENT_ADDR) \ + $(ROOT)afl-fuzz \ + -i $(TEST_DATA_DIR) \ + -o $(QEMU_OUT) \ + -m none \ + -d \ + -Q \ + -V 30 \ + -- \ + $(TEST_BIN) $(DUMMY_DATA_FILE) diff --git a/frida_mode/test/bloaty/Makefile b/frida_mode/test/bloaty/Makefile new file mode 100644 index 00000000..07b139e9 --- /dev/null +++ b/frida_mode/test/bloaty/Makefile @@ -0,0 +1,13 @@ +all: + @echo trying to use GNU make... + @gmake all || echo please install GNUmake + +32: + @echo trying to use GNU make... + @gmake 32 || echo please install GNUmake + +clean: + @gmake clean + +frida: + @gmake frida diff --git a/frida_mode/test/bloaty/get_symbol_addr.py b/frida_mode/test/bloaty/get_symbol_addr.py new file mode 100755 index 00000000..1c46e010 --- /dev/null +++ b/frida_mode/test/bloaty/get_symbol_addr.py @@ -0,0 +1,36 @@ +#!/usr/bin/python3 +import argparse +from elftools.elf.elffile import ELFFile + +def process_file(file, symbol, base): + with open(file, 'rb') as f: + elf = ELFFile(f) + symtab = elf.get_section_by_name('.symtab') + mains = symtab.get_symbol_by_name(symbol) + if len(mains) != 1: + print ("Failed to find main") + return 1 + + main_addr = mains[0]['st_value'] + main = base + main_addr + print ("0x%016x" % main) + return 0 + +def hex_value(x): + return int(x, 16) + +def main(): + parser = argparse.ArgumentParser(description='Process some integers.') + parser.add_argument('-f', '--file', dest='file', type=str, + help='elf file name', required=True) + parser.add_argument('-s', '--symbol', dest='symbol', type=str, + help='symbol name', required=True) + parser.add_argument('-b', '--base', dest='base', type=hex_value, + help='elf base address', required=True) + + args = parser.parse_args() + return process_file (args.file, args.symbol, args.base) + +if __name__ == "__main__": + ret = main() + exit(ret) diff --git a/frida_mode/test/freetype2/GNUmakefile b/frida_mode/test/freetype2/GNUmakefile new file mode 100644 index 00000000..f7a50de2 --- /dev/null +++ b/frida_mode/test/freetype2/GNUmakefile @@ -0,0 +1,192 @@ +PWD:=$(shell pwd)/ +ROOT:=$(PWD)../../../ +BUILD_DIR:=$(PWD)build/ + +AFLPP_FRIDA_DRIVER_HOOK_OBJ=$(ROOT)frida_mode/build/frida_hook.so +AFLPP_QEMU_DRIVER_HOOK_OBJ=$(ROOT)frida_mode/build/qemu_hook.so + +# git clone git://git.sv.nongnu.org/freetype/freetype2.git +# git clone https://github.com/unicode-org/text-rendering-tests.git TRT +# wget https://github.com/libarchive/libarchive/releases/download/v3.4.3/libarchive-3.4.3.tar.xz + +# cp TRT/fonts/TestKERNOne.otf $OUT/seeds/ +# cp TRT/fonts/TestGLYFOne.ttf $OUT/seeds/ + +# $CXX $CXXFLAGS -std=c++11 -I include -I . src/tools/ftfuzzer/ftfuzzer.cc \ +# objs/.libs/libfreetype.a $FUZZER_LIB -L /usr/local/lib -larchive \ +# -o $OUT/ftfuzzer + +LIBARCHIVE_URL:=https://github.com/libarchive/libarchive/releases/download/v3.4.3/libarchive-3.4.3.tar.xz +LIBARCHIVE_BUILD_DIR:=$(BUILD_DIR)libarchive/ +LIBARCHIVE_TARBALL:=$(LIBARCHIVE_BUILD_DIR)libarchive-3.4.3.tar.xz +LIBARCHIVE_DIR:=$(LIBARCHIVE_BUILD_DIR)libarchive-3.4.3/ +LIBARCHIVE_LIB:=$(LIBARCHIVE_DIR).libs/libarchive.a + +FREETYPE2_GIT_REPO:=git://git.sv.nongnu.org/freetype/freetype2.git +FREETYPE2_BUILD_DIR:=$(BUILD_DIR)freetype2/ +FREETYPE2_DIR:=$(FREETYPE2_BUILD_DIR)freetype2/ +FREETYPE2_LIB:=$(FREETYPE2_DIR)objs/.libs/libfreetype.a + +HARNESS_URL:=https://raw.githubusercontent.com/llvm/llvm-project/main/compiler-rt/lib/fuzzer/standalone/StandaloneFuzzTargetMain.c +HARNESS_SRC:=$(BUILD_DIR)StandaloneFuzzTargetMain.c +HARNESS_OBJ:=$(BUILD_DIR)StandaloneFuzzTargetMain.o + +TRT_GIT_REPO:=https://github.com/unicode-org/text-rendering-tests.git +TRT_DIR:=$(BUILD_DIR)TRT/ + +FUZZER_SRC:=$(FREETYPE2_DIR)src/tools/ftfuzzer/ftfuzzer.cc + + +LDFLAGS += -lpthread + +TEST_BIN:=$(BUILD_DIR)test +ifeq "$(shell uname)" "Darwin" +TEST_BIN_LDFLAGS:=-undefined dynamic_lookup -Wl,-no_pie +endif + +TEST_DATA_DIR:=$(BUILD_DIR)in/ +TEST_DATA_FILE:=$(TEST_DATA_DIR)default_seed + +FRIDA_OUT:=$(BUILD_DIR)frida-out +QEMU_OUT:=$(BUILD_DIR)qemu-out + +ifndef ARCH + +ARCH=$(shell uname -m) +ifeq "$(ARCH)" "aarch64" + ARCH:=arm64 +endif + +ifeq "$(ARCH)" "i686" + ARCH:=x86 +endif +endif + +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh + +AFL_QEMU_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x4000000000) + +ifeq "$(ARCH)" "aarch64" + AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x0000aaaaaaaaa000) +endif + +ifeq "$(ARCH)" "x86_64" + AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x0000555555554000) +endif + +ifeq "$(ARCH)" "x86" + AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x56555000) +endif + +.PHONY: all clean frida hook + +all: $(TEST_BIN) + make -C $(ROOT)frida_mode/ + +32: + CFLAGS="-m32" CXXFLAGS="-m32" LDFLAGS="-m32" ARCH="x86" make all + +$(BUILD_DIR): + mkdir -p $@ + +########## LIBARCHIVE ####### + +$(LIBARCHIVE_BUILD_DIR): | $(BUILD_DIR) + mkdir -p $@ + +$(LIBARCHIVE_TARBALL): | $(LIBARCHIVE_BUILD_DIR) + wget -O $@ $(LIBARCHIVE_URL) + +$(LIBARCHIVE_DIR): | $(LIBARCHIVE_TARBALL) + tar Jxvf $(LIBARCHIVE_TARBALL) -C $(LIBARCHIVE_BUILD_DIR) + +$(LIBARCHIVE_DIR)Makefile: | $(LIBARCHIVE_DIR) + cd $(LIBARCHIVE_DIR) && ./configure --disable-shared + +$(LIBARCHIVE_LIB): $(LIBARCHIVE_DIR)Makefile + make -C $(LIBARCHIVE_DIR) clean all + +########## FREETYPE2 ####### + +$(FREETYPE2_BUILD_DIR): | $(BUILD_DIR) + mkdir -p $@ + +$(FREETYPE2_DIR): | $(FREETYPE2_BUILD_DIR) + git clone $(FREETYPE2_GIT_REPO) $@ + git -C $(FREETYPE2_DIR) checkout cd02d359a6d0455e9d16b87bf9665961c4699538 + +$(FREETYPE2_LIB): | $(FREETYPE2_DIR) + cd $(FREETYPE2_DIR) && ./autogen.sh + cd $(FREETYPE2_DIR) && ./configure --with-harfbuzz=no --with-bzip2=no --with-png=no --without-zlib + make -C $(FREETYPE2_DIR) all + +########## HARNESS ####### + +$(HARNESS_SRC): + wget -O $@ $(HARNESS_URL) + +$(HARNESS_OBJ): $(HARNESS_SRC) + $(CC) $(CFLAGS) -o $@ -c $< + +########## TEST ####### + +$(TEST_BIN): $(LIBARCHIVE_LIB) $(FREETYPE2_LIB) $(HARNESS_OBJ) + $(CXX) \ + $(CXXFLAGS) \ + -std=c++11 \ + -I $(FREETYPE2_DIR)include \ + -I $(FREETYPE2_DIR) \ + -I $(LIBARCHIVE_DIR)/libarchive \ + $(FUZZER_SRC) \ + $(FREETYPE2_LIB) \ + $(LIBARCHIVE_LIB) \ + $(HARNESS_OBJ) \ + -o $@ + +########## DUMMY ####### + +$(TRT_DIR): | $(BUILD_DIR) + git clone $(TRT_GIT_REPO) $@ + +$(TEST_DATA_DIR): | $(TRT_DIR) + mkdir -p $@ + cp $(TRT_DIR)fonts/TestKERNOne.otf $@ + cp $(TRT_DIR)fonts/TestGLYFOne.ttf $@ + +$(TEST_DATA_FILE): | $(TEST_DATA_DIR) + dd if=/dev/zero bs=1048576 count=1 of=$@ + +###### TEST DATA ####### + +clean: + rm -rf $(BUILD_DIR) + +frida: $(TEST_BIN) $(AFLPP_FRIDA_DRIVER_HOOK_OBJ) $(TEST_DATA_FILE) + AFL_FRIDA_PERSISTENT_CNT=1000000 \ + AFL_FRIDA_PERSISTENT_HOOK=$(AFLPP_FRIDA_DRIVER_HOOK_OBJ) \ + AFL_FRIDA_PERSISTENT_ADDR=$(AFL_FRIDA_PERSISTENT_ADDR) \ + AFL_ENTRYPOINT=$(AFL_FRIDA_PERSISTENT_ADDR) \ + $(ROOT)afl-fuzz \ + -i $(TEST_DATA_DIR) \ + -o $(FRIDA_OUT) \ + -m none \ + -d \ + -O \ + -V 30 \ + -- \ + $(TEST_BIN) $(TEST_DATA_FILE) + +qemu: $(TEST_BIN) $(AFLPP_QEMU_DRIVER_HOOK_OBJ) $(TEST_DATA_FILE) + AFL_QEMU_PERSISTENT_CNT=1000000 \ + AFL_QEMU_PERSISTENT_HOOK=$(AFLPP_QEMU_DRIVER_HOOK_OBJ) \ + AFL_QEMU_PERSISTENT_ADDR=$(AFL_QEMU_PERSISTENT_ADDR) \ + AFL_ENTRYPOINT=$(AFL_QEMU_PERSISTENT_ADDR) \ + $(ROOT)afl-fuzz \ + -i $(TEST_DATA_DIR) \ + -o $(QEMU_OUT) \ + -m none \ + -d \ + -Q \ + -V 30 \ + -- \ + $(TEST_BIN) $(TEST_DATA_FILE) diff --git a/frida_mode/test/freetype2/Makefile b/frida_mode/test/freetype2/Makefile new file mode 100644 index 00000000..07b139e9 --- /dev/null +++ b/frida_mode/test/freetype2/Makefile @@ -0,0 +1,13 @@ +all: + @echo trying to use GNU make... + @gmake all || echo please install GNUmake + +32: + @echo trying to use GNU make... + @gmake 32 || echo please install GNUmake + +clean: + @gmake clean + +frida: + @gmake frida diff --git a/frida_mode/test/freetype2/get_symbol_addr.py b/frida_mode/test/freetype2/get_symbol_addr.py new file mode 100755 index 00000000..1c46e010 --- /dev/null +++ b/frida_mode/test/freetype2/get_symbol_addr.py @@ -0,0 +1,36 @@ +#!/usr/bin/python3 +import argparse +from elftools.elf.elffile import ELFFile + +def process_file(file, symbol, base): + with open(file, 'rb') as f: + elf = ELFFile(f) + symtab = elf.get_section_by_name('.symtab') + mains = symtab.get_symbol_by_name(symbol) + if len(mains) != 1: + print ("Failed to find main") + return 1 + + main_addr = mains[0]['st_value'] + main = base + main_addr + print ("0x%016x" % main) + return 0 + +def hex_value(x): + return int(x, 16) + +def main(): + parser = argparse.ArgumentParser(description='Process some integers.') + parser.add_argument('-f', '--file', dest='file', type=str, + help='elf file name', required=True) + parser.add_argument('-s', '--symbol', dest='symbol', type=str, + help='symbol name', required=True) + parser.add_argument('-b', '--base', dest='base', type=hex_value, + help='elf base address', required=True) + + args = parser.parse_args() + return process_file (args.file, args.symbol, args.base) + +if __name__ == "__main__": + ret = main() + exit(ret) diff --git a/frida_mode/test/js/GNUmakefile b/frida_mode/test/js/GNUmakefile index aad81d08..c702ad98 100644 --- a/frida_mode/test/js/GNUmakefile +++ b/frida_mode/test/js/GNUmakefile @@ -10,6 +10,7 @@ TESTINSTSRC:=$(PWD)test.c TESTINSTBIN2:=$(BUILD_DIR)test2 TESTINSTSRC2:=$(PWD)test2.c +AFLPP_DRIVER_DUMMY_INPUT:=$(BUILD_DIR)dummy QEMU_OUT:=$(BUILD_DIR)qemu-out FRIDA_OUT:=$(BUILD_DIR)frida-out @@ -40,9 +41,36 @@ $(TESTINSTBIN): $(TESTINSTSRC) | $(BUILD_DIR) $(TESTINSTBIN2): $(TESTINSTSRC2) | $(BUILD_DIR) $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $< +$(AFLPP_DRIVER_DUMMY_INPUT): | $(BUILD_DIR) + dd if=/dev/zero bs=1048576 count=1 of=$@ + clean: rm -rf $(BUILD_DIR) +frida_js_main: $(TESTINSTBIN) $(TEST_DATA_FILE) $(AFLPP_DRIVER_DUMMY_INPUT) + AFL_PRELOAD=$(AFL_PRELOAD) \ + AFL_FRIDA_JS_SCRIPT=main.js \ + $(ROOT)afl-fuzz \ + -D \ + -O \ + -i $(TEST_DATA_DIR) \ + -o $(FRIDA_OUT) \ + -t 10000+ \ + -- \ + $(TESTINSTBIN) $(AFLPP_DRIVER_DUMMY_INPUT) + +frida_js_fuzz: $(TESTINSTBIN) $(TEST_DATA_FILE) $(AFLPP_DRIVER_DUMMY_INPUT) + AFL_PRELOAD=$(AFL_PRELOAD) \ + AFL_FRIDA_JS_SCRIPT=fuzz.js \ + $(ROOT)afl-fuzz \ + -D \ + -O \ + -i $(TEST_DATA_DIR) \ + -o $(FRIDA_OUT) \ + -t 10000+ \ + -- \ + $(TESTINSTBIN) $(AFLPP_DRIVER_DUMMY_INPUT) + frida_js_entry: $(TESTINSTBIN) $(TEST_DATA_FILE) AFL_PRELOAD=$(AFL_PRELOAD) \ AFL_FRIDA_JS_SCRIPT=entry.js \ diff --git a/frida_mode/test/js/fuzz.js b/frida_mode/test/js/fuzz.js new file mode 100644 index 00000000..24eca2b6 --- /dev/null +++ b/frida_mode/test/js/fuzz.js @@ -0,0 +1,41 @@ +Afl.print('******************'); +Afl.print('* AFL FRIDA MODE *'); +Afl.print('******************'); +Afl.print(''); + +Afl.print(`PID: ${Process.id}`); + +const name = Process.enumerateModules()[0].name; +Afl.print(`Name: ${name}`); + +new ModuleMap().values().forEach(m => { + Afl.print(`${m.base}-${m.base.add(m.size)} ${m.name}`); +}); + +const LLVMFuzzerTestOneInput = DebugSymbol.fromName('LLVMFuzzerTestOneInput').address; +Afl.print(`LLVMFuzzerTestOneInput: ${LLVMFuzzerTestOneInput}`); + +const cm = new CModule(` + + extern unsigned char * __afl_fuzz_ptr; + extern unsigned int * __afl_fuzz_len; + extern void LLVMFuzzerTestOneInput(char *buf, int len); + + void My_LLVMFuzzerTestOneInput(char *buf, int len) { + + LLVMFuzzerTestOneInput(__afl_fuzz_ptr, *__afl_fuzz_len); + + } + `, + { + LLVMFuzzerTestOneInput: LLVMFuzzerTestOneInput, + __afl_fuzz_ptr: Afl.getAflFuzzPtr(), + __afl_fuzz_len: Afl.getAflFuzzLen() + }); + +Afl.setEntryPoint(cm.My_LLVMFuzzerTestOneInput); +Afl.setPersistentAddress(cm.My_LLVMFuzzerTestOneInput); +Afl.setInMemoryFuzzing(); +Interceptor.replace(LLVMFuzzerTestOneInput, cm.My_LLVMFuzzerTestOneInput); +Afl.print("done"); +Afl.done(); diff --git a/frida_mode/test/js/main.js b/frida_mode/test/js/main.js new file mode 100644 index 00000000..06306fc4 --- /dev/null +++ b/frida_mode/test/js/main.js @@ -0,0 +1,44 @@ +Afl.print('******************'); +Afl.print('* AFL FRIDA MODE *'); +Afl.print('******************'); +Afl.print(''); + +Afl.print(`PID: ${Process.id}`); + +const name = Process.enumerateModules()[0].name; +Afl.print(`Name: ${name}`); + +new ModuleMap().values().forEach(m => { + Afl.print(`${m.base}-${m.base.add(m.size)} ${m.name}`); +}); + +const main = DebugSymbol.fromName('main').address; +Afl.print(`main: ${main}`); + +const LLVMFuzzerTestOneInput = DebugSymbol.fromName('LLVMFuzzerTestOneInput').address; +Afl.print(`LLVMFuzzerTestOneInput: ${LLVMFuzzerTestOneInput}`); + +const cm = new CModule(` + + extern unsigned char * __afl_fuzz_ptr; + extern unsigned int * __afl_fuzz_len; + extern void LLVMFuzzerTestOneInput(char *buf, int len); + + int main(int argc, char **argv) { + + LLVMFuzzerTestOneInput(__afl_fuzz_ptr, *__afl_fuzz_len); + + } + `, + { + LLVMFuzzerTestOneInput: LLVMFuzzerTestOneInput, + __afl_fuzz_ptr: Afl.getAflFuzzPtr(), + __afl_fuzz_len: Afl.getAflFuzzLen() + }); + +Afl.setEntryPoint(cm.main); +Afl.setPersistentAddress(cm.main); +Afl.setInMemoryFuzzing(); +Afl.setJsMainHook(cm.main); +Afl.print("done"); +Afl.done(); diff --git a/frida_mode/test/libpcap/GNUmakefile b/frida_mode/test/libpcap/GNUmakefile index 6f2b58af..f8dc3db7 100644 --- a/frida_mode/test/libpcap/GNUmakefile +++ b/frida_mode/test/libpcap/GNUmakefile @@ -59,7 +59,7 @@ GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh AFL_QEMU_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x4000000000) -ifeq "$(ARCH)" "aarch64" +ifeq "$(ARCH)" "arm64" AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x0000aaaaaaaaa000) endif diff --git a/frida_mode/test/perf/GNUmakefile b/frida_mode/test/perf/GNUmakefile new file mode 100644 index 00000000..d65aaa6d --- /dev/null +++ b/frida_mode/test/perf/GNUmakefile @@ -0,0 +1,116 @@ +PWD:=$(shell pwd)/ +ROOT:=$(PWD)../../../ +BUILD_DIR:=$(PWD)build/ + +TEST_BIN:=$(BUILD_DIR)perf +TEST_SRC:=$(PWD)perf.c + +TEST_DATA_DIR:=$(BUILD_DIR)in/ +TEST_DATA_FILE:=$(TEST_DATA_DIR)default_seed + +QEMU_OUT:=$(BUILD_DIR)qemu-out +FRIDA_OUT:=$(BUILD_DIR)frida-out + +AFLPP_FRIDA_DRIVER_HOOK_OBJ=$(ROOT)frida_mode/build/frida_hook.so +AFLPP_QEMU_DRIVER_HOOK_OBJ=$(ROOT)frida_mode/build/qemu_hook.so + +ifndef ARCH + +ARCH=$(shell uname -m) +ifeq "$(ARCH)" "aarch64" + ARCH:=arm64 +endif + +ifeq "$(ARCH)" "i686" + ARCH:=x86 +endif +endif + +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh + +AFL_QEMU_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x4000000000) + +ifeq "$(ARCH)" "arm64" + AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x0000aaaaaaaaa000) +endif + +ifeq "$(ARCH)" "x86_64" + AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x0000555555554000) +endif + +ifeq "$(ARCH)" "x86" + AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x56555000) +endif + + +.PHONY: all 32 clean qemu frida + +all: $(TEST_BIN) + make -C $(ROOT)frida_mode/ + +32: + CFLAGS="-m32" LDFLAGS="-m32" ARCH="x86" make all + +$(BUILD_DIR): + mkdir -p $@ + +$(PERF_DATA_DIR): | $(BUILD_DIR) + mkdir -p $@ + +$(PERF_DATA_FILE): | $(PERF_DATA_DIR) + echo -n "000" > $@ + +$(TEST_BIN): $(TEST_SRC) | $(BUILD_DIR) + $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $< + +clean: + rm -rf $(BUILD_DIR) + +$(TEST_DATA_DIR): | $(BUILD_DIR) + mkdir -p $@ + +$(TEST_DATA_FILE): | $(TEST_DATA_DIR) + dd if=/dev/zero bs=1048576 count=1 of=$@ + + +qemu: $(TEST_BIN) $(TEST_DATA_FILE) + AFL_QEMU_PERSISTENT_CNT=1000000 \ + AFL_QEMU_PERSISTENT_HOOK=$(AFLPP_QEMU_DRIVER_HOOK_OBJ) \ + AFL_QEMU_PERSISTENT_ADDR=$(AFL_QEMU_PERSISTENT_ADDR) \ + AFL_ENTRYPOINT=$(AFL_QEMU_PERSISTENT_ADDR) \ + AFL_QEMU_PERSISTENT_GPR=1 \ + $(ROOT)afl-fuzz \ + -D \ + -Q \ + -i $(TEST_DATA_DIR) \ + -o $(QEMU_OUT) \ + -V 10 \ + -- \ + $(TEST_BIN) $(TEST_DATA_FILE) + +frida: $(TEST_BIN) $(TEST_DATA_FILE) + AFL_FRIDA_PERSISTENT_CNT=1000000 \ + AFL_FRIDA_PERSISTENT_HOOK=$(AFLPP_FRIDA_DRIVER_HOOK_OBJ) \ + AFL_FRIDA_PERSISTENT_ADDR=$(AFL_FRIDA_PERSISTENT_ADDR) \ + AFL_ENTRYPOINT=$(AFL_FRIDA_PERSISTENT_ADDR) \ + $(ROOT)afl-fuzz \ + -D \ + -O \ + -i $(TEST_DATA_DIR) \ + -o $(FRIDA_OUT) \ + -V 10 \ + -- \ + $(TEST_BIN) $(TEST_DATA_FILE) + +debug: + echo $(AFL_FRIDA_PERSISTENT_ADDR) + gdb \ + --ex 'set environment LD_PRELOAD=$(ROOT)afl-frida-trace.so' \ + --ex 'set disassembly-flavor intel' \ + --args $(TEST_BIN) $(TEST_DATA_FILE) + +debug: + gdb \ + --ex 'set environment LD_PRELOAD=$(ROOT)afl-frida-trace.so' \ + --ex 'set disassembly-flavor intel' \ + --args $(TEST_BIN) $(TEST_DATA_FILE) diff --git a/frida_mode/test/perf/Makefile b/frida_mode/test/perf/Makefile new file mode 100644 index 00000000..f843af19 --- /dev/null +++ b/frida_mode/test/perf/Makefile @@ -0,0 +1,19 @@ +all: + @echo trying to use GNU make... + @gmake all || echo please install GNUmake + +32: + @echo trying to use GNU make... + @gmake 32 || echo please install GNUmake + +clean: + @gmake clean + +qemu: + @gmake qemu + +frida: + @gmake frida + +debug: + @gmake debug diff --git a/frida_mode/test/perf/perf.c b/frida_mode/test/perf/perf.c new file mode 100644 index 00000000..c5881915 --- /dev/null +++ b/frida_mode/test/perf/perf.c @@ -0,0 +1,105 @@ +/* + american fuzzy lop++ - a trivial program to test the build + -------------------------------------------------------- + Originally written by Michal Zalewski + Copyright 2014 Google Inc. All rights reserved. + Copyright 2019-2020 AFLplusplus Project. All rights reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at: + http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include <fcntl.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +void LLVMFuzzerTestOneInput(char *buf, int len) { + + int ret = 0; + for (int i = 0; i < 1000; i++) { + switch(buf[i]) { + case 'A': ret += 2; break; + case '1': ret += 3; break; + default: ret++; + } + } + printf("ret: %d\n", ret); + +} + +int main(int argc, char **argv) { + + char * file; + int fd = -1; + off_t len; + char * buf = NULL; + size_t n_read; + int result = -1; + + if (argc != 2) { return 1; } + + do { + + file = argv[1]; + + dprintf(STDERR_FILENO, "Running: %s\n", file); + + fd = open(file, O_RDONLY); + if (fd < 0) { + + perror("open"); + break; + + } + + len = lseek(fd, 0, SEEK_END); + if (len < 0) { + + perror("lseek (SEEK_END)"); + break; + + } + + if (lseek(fd, 0, SEEK_SET) != 0) { + + perror("lseek (SEEK_SET)"); + break; + + } + + buf = malloc(len); + if (buf == NULL) { + + perror("malloc"); + break; + + } + + n_read = read(fd, buf, len); + if (n_read != len) { + + perror("read"); + break; + + } + + dprintf(STDERR_FILENO, "Running: %s: (%zd bytes)\n", file, n_read); + + LLVMFuzzerTestOneInput(buf, len); + dprintf(STDERR_FILENO, "Done: %s: (%zd bytes)\n", file, n_read); + + result = 0; + + } while (false); + + if (buf != NULL) { free(buf); } + + if (fd != -1) { close(fd); } + + return result; + +} + diff --git a/frida_mode/test/unstable/GNUmakefile b/frida_mode/test/unstable/GNUmakefile index 0ccc5fb1..54bbe662 100644 --- a/frida_mode/test/unstable/GNUmakefile +++ b/frida_mode/test/unstable/GNUmakefile @@ -86,11 +86,23 @@ frida: $(UNSTABLE_BIN) $(UNSTABLE_DATA_FILE) $(UNSTABLE_BIN) @@ frida_coverage: $(UNSTABLE_BIN) $(UNSTABLE_DATA_FILE) - AFL_DEBUG=1 \ AFL_FRIDA_PERSISTENT_ADDR=$(AFL_FRIDA_PERSISTENT_ADDR) \ AFL_FRIDA_OUTPUT_STDOUT=/tmp/stdout.txt \ AFL_FRIDA_OUTPUT_STDERR=/tmp/stderr.txt \ AFL_FRIDA_INST_COVERAGE_FILE=/tmp/coverage.dat \ + $(ROOT)afl-fuzz \ + -D \ + -O \ + -i $(UNSTABLE_DATA_DIR) \ + -o $(FRIDA_OUT) \ + -- \ + $(UNSTABLE_BIN) @@ + +frida_unstable: $(UNSTABLE_BIN) $(UNSTABLE_DATA_FILE) + AFL_DEBUG=1 \ + AFL_FRIDA_PERSISTENT_ADDR=$(AFL_FRIDA_PERSISTENT_ADDR) \ + AFL_FRIDA_OUTPUT_STDOUT=/tmp/stdout.txt \ + AFL_FRIDA_OUTPUT_STDERR=/tmp/stderr.txt \ AFL_FRIDA_INST_UNSTABLE_COVERAGE_FILE=/tmp/unstable.dat \ $(ROOT)afl-fuzz \ -D \ diff --git a/frida_mode/test/vorbis/GNUmakefile b/frida_mode/test/vorbis/GNUmakefile new file mode 100644 index 00000000..59ae9a59 --- /dev/null +++ b/frida_mode/test/vorbis/GNUmakefile @@ -0,0 +1,200 @@ +PWD:=$(shell pwd)/ +ROOT:=$(PWD)../../../ +BUILD_DIR:=$(PWD)build/ + +AFLPP_FRIDA_DRIVER_HOOK_OBJ=$(ROOT)frida_mode/build/frida_hook.so +AFLPP_QEMU_DRIVER_HOOK_OBJ=$(ROOT)frida_mode/build/qemu_hook.so + +OGG_GIT_REPO:=https://github.com/xiph/ogg.git +OGG_BUILD_DIR:=$(BUILD_DIR)ogg/ +OGG_DIR:=$(OGG_BUILD_DIR)ogg/ +OGG_INSTALL:=$(OGG_BUILD_DIR)install/ +OGG_LIB:=$(OGG_INSTALL)lib/libogg.a + +VORBIS_GIT_REPO:=https://github.com/xiph/vorbis.git +VORBIS_BUILD_DIR:=$(BUILD_DIR)vorbis/ +VORBIS_DIR:=$(VORBIS_BUILD_DIR)vorbis/ +VORBIS_INSTALL:=$(VORBIS_BUILD_DIR)install/ +VORBIS_LIB:=$(VORBIS_INSTALL)lib/libvorbis.a +VORBISFILE_LIB:=$(VORBIS_INSTALL)lib/libvorbisfile.a + +DECODE_URL:=https://raw.githubusercontent.com/google/oss-fuzz/688aadaf44499ddada755562109e5ca5eb3c5662/projects/vorbis/decode_fuzzer.cc +DECODE_SRC:=$(BUILD_DIR)decode_fuzzer.cc +DECODE_OBJ:=$(BUILD_DIR)decode_fuzzer.o + +HARNESS_URL:=https://raw.githubusercontent.com/llvm/llvm-project/main/compiler-rt/lib/fuzzer/standalone/StandaloneFuzzTargetMain.c +HARNESS_SRC:=$(BUILD_DIR)StandaloneFuzzTargetMain.c +HARNESS_OBJ:=$(BUILD_DIR)StandaloneFuzzTargetMain.o + +LDFLAGS += -lpthread + +TEST_BIN:=$(BUILD_DIR)decode_fuzzer +ifeq "$(shell uname)" "Darwin" +TEST_BIN_LDFLAGS:=-undefined dynamic_lookup -Wl,-no_pie +endif + +TEST_DATA_DIR:=$(BUILD_DIR)in/ +TEST_DATA_SRC:=https://github.com/google/fuzzbench/blob/master/benchmarks/vorbis-2017-12-11/seeds/sound.ogg?raw=true +TEST_DATA_FILE:=$(TEST_DATA_DIR)sound.ogg +DUMMY_DATA_FILE:=$(BUILD_DIR)default_seed + +FRIDA_OUT:=$(BUILD_DIR)frida-out +QEMU_OUT:=$(BUILD_DIR)qemu-out + +ifndef ARCH + +ARCH=$(shell uname -m) +ifeq "$(ARCH)" "aarch64" + ARCH:=arm64 +endif + +ifeq "$(ARCH)" "i686" + ARCH:=x86 +endif +endif + +GET_SYMBOL_ADDR:=$(ROOT)frida_mode/util/get_symbol_addr.sh + +AFL_QEMU_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x4000000000) + +ifeq "$(ARCH)" "aarch64" + AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x0000aaaaaaaaa000) +endif + +ifeq "$(ARCH)" "x86_64" + AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x0000555555554000) +endif + +ifeq "$(ARCH)" "x86" + AFL_FRIDA_PERSISTENT_ADDR=$(shell $(GET_SYMBOL_ADDR) $(TEST_BIN) LLVMFuzzerTestOneInput 0x56555000) +endif + +.PHONY: all clean frida hook + +all: $(TEST_BIN) + make -C $(ROOT)frida_mode/ + +32: + CXXFLAGS="-m32" LDFLAGS="-m32" ARCH="x86" make all + +$(BUILD_DIR): + mkdir -p $@ + +########## OGG ####### + +$(OGG_BUILD_DIR): | $(BUILD_DIR) + mkdir -p $@ + +$(OGG_DIR): | $(OGG_BUILD_DIR) + git clone $(OGG_GIT_REPO) $@ + git -C $(OGG_DIR) checkout c8391c2b267a7faf9a09df66b1f7d324e9eb7766 + +$(OGG_LIB): | $(OGG_DIR) + cd $(OGG_DIR) && ./autogen.sh + cd $(OGG_DIR) && ./configure \ + --prefix=$(OGG_INSTALL) \ + --enable-static \ + --disable-shared \ + --disable-crc + make -C $(OGG_DIR) install + +ogg: $(OGG_LIB) + +########## VORBIS ####### + +$(VORBIS_BUILD_DIR): | $(BUILD_DIR) + mkdir -p $@ + +$(VORBIS_DIR): | $(VORBIS_BUILD_DIR) + git clone $(VORBIS_GIT_REPO) $@ + git -C $(VORBIS_DIR) checkout c1c2831fc7306d5fbd7bc800324efd12b28d327f + +$(VORBIS_LIB): $(OGG_LIB) | $(VORBIS_DIR) + cd $(VORBIS_DIR) && ./autogen.sh + cd $(VORBIS_DIR) && ./configure \ + --prefix=$(VORBIS_INSTALL) \ + --enable-static \ + --disable-shared \ + --with-ogg=$(OGG_INSTALL) + make -C $(VORBIS_DIR) install + +vorbis: $(VORBIS_LIB) + +########## HARNESS ####### + +$(DECODE_SRC): + wget -O $@ $(DECODE_URL) + +$(DECODE_OBJ): $(DECODE_SRC) + $(CXX) -o $@ -c $< -I$(VORBIS_DIR)include/ -I$(OGG_DIR)include/ + +decode: $(DECODE_OBJ) + +########## HARNESS ####### + +$(HARNESS_SRC): + wget -O $@ $(HARNESS_URL) + +$(HARNESS_OBJ): $(HARNESS_SRC) + $(CC) -o $@ -c $< + +harness: $(HARNESS_OBJ) + +########## TEST ####### + +$(TEST_BIN): $(VORBIS_LIB) $(OGG_LIB) $(HARNESS_OBJ) $(DECODE_OBJ) + $(CXX) \ + $(CXXFLAGS) \ + -std=c++11 \ + $(DECODE_OBJ) \ + $(HARNESS_OBJ) \ + $(VORBISFILE_LIB) \ + $(VORBIS_LIB) \ + $(OGG_LIB) \ + -o $@ + +########## DUMMY ####### + +$(TEST_DATA_DIR): | $(BUILD_DIR) + mkdir -p $@ + +$(DUMMY_DATA_FILE): | $(TEST_DATA_DIR) + dd if=/dev/zero bs=1048576 count=1 of=$@ + +###### TEST DATA ####### + +$(TEST_DATA_FILE): + wget -O $@ $(TEST_DATA_SRC) + +clean: + rm -rf $(BUILD_DIR) + +frida: $(TEST_BIN) $(AFLPP_FRIDA_DRIVER_HOOK_OBJ) $(TEST_DATA_FILE) $(DUMMY_DATA_FILE) + AFL_FRIDA_PERSISTENT_CNT=1000000 \ + AFL_FRIDA_PERSISTENT_HOOK=$(AFLPP_FRIDA_DRIVER_HOOK_OBJ) \ + AFL_FRIDA_PERSISTENT_ADDR=$(AFL_FRIDA_PERSISTENT_ADDR) \ + AFL_ENTRYPOINT=$(AFL_FRIDA_PERSISTENT_ADDR) \ + $(ROOT)afl-fuzz \ + -i $(TEST_DATA_DIR) \ + -o $(FRIDA_OUT) \ + -m none \ + -d \ + -O \ + -V 30 \ + -- \ + $(TEST_BIN) $(DUMMY_DATA_FILE) + +qemu: $(TEST_BIN) $(AFLPP_QEMU_DRIVER_HOOK_OBJ) $(TEST_DATA_FILE) $(DUMMY_DATA_FILE) + AFL_QEMU_PERSISTENT_CNT=1000000 \ + AFL_QEMU_PERSISTENT_HOOK=$(AFLPP_QEMU_DRIVER_HOOK_OBJ) \ + AFL_QEMU_PERSISTENT_ADDR=$(AFL_QEMU_PERSISTENT_ADDR) \ + AFL_ENTRYPOINT=$(AFL_QEMU_PERSISTENT_ADDR) \ + $(ROOT)afl-fuzz \ + -i $(TEST_DATA_DIR) \ + -o $(QEMU_OUT) \ + -m none \ + -d \ + -Q \ + -V 30 \ + -- \ + $(TEST_BIN) $(DUMMY_DATA_FILE) diff --git a/frida_mode/test/vorbis/Makefile b/frida_mode/test/vorbis/Makefile new file mode 100644 index 00000000..07b139e9 --- /dev/null +++ b/frida_mode/test/vorbis/Makefile @@ -0,0 +1,13 @@ +all: + @echo trying to use GNU make... + @gmake all || echo please install GNUmake + +32: + @echo trying to use GNU make... + @gmake 32 || echo please install GNUmake + +clean: + @gmake clean + +frida: + @gmake frida diff --git a/frida_mode/test/vorbis/get_symbol_addr.py b/frida_mode/test/vorbis/get_symbol_addr.py new file mode 100755 index 00000000..1c46e010 --- /dev/null +++ b/frida_mode/test/vorbis/get_symbol_addr.py @@ -0,0 +1,36 @@ +#!/usr/bin/python3 +import argparse +from elftools.elf.elffile import ELFFile + +def process_file(file, symbol, base): + with open(file, 'rb') as f: + elf = ELFFile(f) + symtab = elf.get_section_by_name('.symtab') + mains = symtab.get_symbol_by_name(symbol) + if len(mains) != 1: + print ("Failed to find main") + return 1 + + main_addr = mains[0]['st_value'] + main = base + main_addr + print ("0x%016x" % main) + return 0 + +def hex_value(x): + return int(x, 16) + +def main(): + parser = argparse.ArgumentParser(description='Process some integers.') + parser.add_argument('-f', '--file', dest='file', type=str, + help='elf file name', required=True) + parser.add_argument('-s', '--symbol', dest='symbol', type=str, + help='symbol name', required=True) + parser.add_argument('-b', '--base', dest='base', type=hex_value, + help='elf base address', required=True) + + args = parser.parse_args() + return process_file (args.file, args.symbol, args.base) + +if __name__ == "__main__": + ret = main() + exit(ret) diff --git a/frida_mode/ts/lib/afl.ts b/frida_mode/ts/lib/afl.ts index 8a1ebf1b..0473cbf6 100644 --- a/frida_mode/ts/lib/afl.ts +++ b/frida_mode/ts/lib/afl.ts @@ -78,6 +78,13 @@ class Afl { } /** + * See `AFL_FRIDA_INST_NO_BACKPATCH`. + */ + public static setBackpatchDisable(): void { + Afl.jsApiSetBackpatchDisable(); + } + + /** * See `AFL_FRIDA_DEBUG_MAPS`. */ public static setDebugMaps(): void { @@ -172,6 +179,14 @@ class Afl { Afl.jsApiSetInstrumentUnstableCoverageFile(buf); } + /* + * Set a callback to be called in place of the usual `main` function. This see + * `Scripting.md` for details. + */ + public static setJsMainHook(address: NativePointer): void { + Afl.jsApiSetJsMainHook(address); + } + /** * This is equivalent to setting `AFL_FRIDA_PERSISTENT_ADDR`, again a * `NativePointer` should be provided as it's argument. @@ -234,6 +249,13 @@ class Afl { Afl.jsApiSetSeccompFile(buf); } + /** + * See `AFL_FRIDA_STALKER_ADJACENT_BLOCKS`. + */ + public static setStalkerAdjacentBlocks(val: number): void { + Afl.jsApiSetStalkerAdjacentBlocks(val); + } + /* * Set a function to be called for each instruction which is instrumented * by AFL FRIDA mode. @@ -284,6 +306,13 @@ class Afl { Afl.jsApiSetStdOut(buf); } + /** + * See `AFL_FRIDA_TRACEABLE`. + */ + public static setTraceable(): void { + Afl.jsApiSetTraceable(); + } + private static readonly jsApiAddExcludeRange = Afl.jsApiGetFunction( "js_api_add_exclude_range", "void", @@ -306,6 +335,11 @@ class Afl { "void", ["pointer"]); + private static readonly jsApiSetBackpatchDisable = Afl.jsApiGetFunction( + "js_api_set_backpatch_disable", + "void", + []); + private static readonly jsApiSetDebugMaps = Afl.jsApiGetFunction( "js_api_set_debug_maps", "void", @@ -361,6 +395,11 @@ class Afl { "void", ["pointer"]); + private static readonly jsApiSetJsMainHook = Afl.jsApiGetFunction( + "js_api_set_js_main_hook", + "void", + ["pointer"]); + private static readonly jsApiSetPersistentAddress = Afl.jsApiGetFunction( "js_api_set_persistent_address", "void", @@ -401,6 +440,11 @@ class Afl { "void", ["pointer"]); + private static readonly jsApiSetStalkerAdjacentBlocks = Afl.jsApiGetFunction( + "js_api_set_stalker_adjacent_blocks", + "void", + ["uint32"]); + private static readonly jsApiSetStalkerCallback = Afl.jsApiGetFunction( "js_api_set_stalker_callback", "void", @@ -431,6 +475,11 @@ class Afl { "void", ["pointer"]); + private static readonly jsApiSetTraceable = Afl.jsApiGetFunction( + "js_api_set_traceable", + "void", + []); + private static readonly jsApiWrite = new NativeFunction( /* tslint:disable-next-line:no-null-keyword */ Module.getExportByName(null, "write"), diff --git a/frida_mode/ub1804/Dockerfile b/frida_mode/ub1804/Dockerfile new file mode 100644 index 00000000..e1f5a339 --- /dev/null +++ b/frida_mode/ub1804/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:xenial + +WORKDIR /AFLplusplus + +RUN apt-get update +RUN apt-get install -y make gcc g++ xz-utils curl wget clang zlib1g-dev \ No newline at end of file diff --git a/frida_mode/ub1804/GNUmakefile b/frida_mode/ub1804/GNUmakefile new file mode 100644 index 00000000..4247916e --- /dev/null +++ b/frida_mode/ub1804/GNUmakefile @@ -0,0 +1,37 @@ +PWD:=$(shell pwd)/ +ROOT:=$(PWD)../../ + +.PHONY: all build docker clean shell test + +all: docker + docker run --rm \ + -v $(ROOT):/AFLplusplus \ + ub1804-afl-frida \ + /bin/sh -c \ + 'make -C /AFLplusplus/ clean all && \ + make -C /AFLplusplus/frida_mode clean all' + +test: + docker run --rm \ + -v $(ROOT):/AFLplusplus \ + ub1804-afl-frida \ + /bin/sh -c \ + 'make -C /AFLplusplus/frida_mode/test/png clean all && \ + make -C /AFLplusplus/frida_mode/test/png/persistent/hook frida' + +build: + docker run --rm \ + -v $(ROOT):/AFLplusplus \ + ub1804-afl-frida \ + /bin/sh -c \ + 'make -C /AFLplusplus/ all && \ + make -C /AFLplusplus/frida_mode all' + +docker: + docker build --tag ub1804-afl-frida . + +clean: + docker images --filter 'dangling=true' -q --no-trunc | xargs -L1 docker rmi --force + +shell: + docker run -ti --rm ub1804-afl-frida /bin/bash diff --git a/frida_mode/ub1804/Makefile b/frida_mode/ub1804/Makefile new file mode 100644 index 00000000..f3c3cd55 --- /dev/null +++ b/frida_mode/ub1804/Makefile @@ -0,0 +1,9 @@ +all: + @echo trying to use GNU make... + @gmake all || echo please install GNUmake + +clean: + @gmake clean + +shell: + @gmake shell diff --git a/include/afl-as.h b/include/afl-as.h index 3c12c68f..2a2e8ad7 100644 --- a/include/afl-as.h +++ b/include/afl-as.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This file houses the assembly-level instrumentation injected into fuzzed programs. The instrumentation stores XORed pairs of data: identifiers of the @@ -396,7 +396,7 @@ static const u8 *main_payload_32 = "\n"; /* The OpenBSD hack is due to lahf and sahf not being recognized by some - versions of binutils: http://marc.info/?l=openbsd-cvs&m=141636589924400 + versions of binutils: https://marc.info/?l=openbsd-cvs&m=141636589924400 The Apple code is a bit different when calling libc functions because they are doing relocations differently from everybody else. We also need diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 4b19e698..f3d6d99d 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to @@ -1130,12 +1130,12 @@ void get_core_count(afl_state_t *); void fix_up_sync(afl_state_t *); void check_asan_opts(afl_state_t *); void check_binary(afl_state_t *, u8 *); -void fix_up_banner(afl_state_t *, u8 *); void check_if_tty(afl_state_t *); void setup_signal_handlers(void); void save_cmdline(afl_state_t *, u32, char **); void read_foreign_testcases(afl_state_t *, int); void write_crash_readme(afl_state_t *afl); +u8 check_if_text_buf(u8 *buf, u32 len); /* CmpLog */ diff --git a/include/afl-prealloc.h b/include/afl-prealloc.h index fa6c9b70..87bbb1cc 100644 --- a/include/afl-prealloc.h +++ b/include/afl-prealloc.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/include/alloc-inl.h b/include/alloc-inl.h index c914da5f..0c540330 100644 --- a/include/alloc-inl.h +++ b/include/alloc-inl.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This allocator is not designed to resist malicious attackers (the canaries are small and predictable), but provides a robust and portable way to detect diff --git a/include/cmplog.h b/include/cmplog.h index 878ed60c..8778a4b6 100644 --- a/include/cmplog.h +++ b/include/cmplog.h @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code to handle the shared memory. This is used by the fuzzer as well the other components like afl-tmin, afl-showmap, etc... @@ -48,7 +48,8 @@ struct cmp_header { unsigned shape : 5; unsigned type : 2; unsigned attribute : 4; - unsigned reserved : 5; + unsigned overflow : 1; + unsigned reserved : 4; } __attribute__((packed)); @@ -59,14 +60,16 @@ struct cmp_operands { u64 v0_128; u64 v1_128; -}; +} __attribute__((packed)); struct cmpfn_operands { - u8 v0[32]; - u8 v1[32]; + u8 v0[31]; + u8 v0_len; + u8 v1[31]; + u8 v1_len; -}; +} __attribute__((packed)); typedef struct cmp_operands cmp_map_list[CMP_MAP_H]; diff --git a/include/common.h b/include/common.h index 7bba9e91..6c8e3b3a 100644 --- a/include/common.h +++ b/include/common.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Gather some functions common to multiple executables @@ -38,6 +38,7 @@ #define STRINGIFY_VAL_SIZE_MAX (16) +u32 check_binary_signatures(u8 *fn); void detect_file_args(char **argv, u8 *prog_in, bool *use_stdin); void print_suggested_envs(char *mispelled_env); void check_environment_vars(char **env); @@ -45,6 +46,7 @@ void check_environment_vars(char **env); char **argv_cpy_dup(int argc, char **argv); void argv_cpy_free(char **argv); +char **get_cs_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv); char **get_qemu_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv); char **get_wine_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv); char * get_afl_env(char *env); diff --git a/include/config.h b/include/config.h index da74989e..b787152f 100644 --- a/include/config.h +++ b/include/config.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ @@ -237,11 +237,11 @@ (note that if this value is changed, several areas in afl-cc.c, afl-fuzz.c and afl-fuzz-state.c have to be changed as well! */ -#define MAX_FILE (1 * 1024 * 1024U) +#define MAX_FILE (1 * 1024 * 1024L) /* The same, for the test case minimizer: */ -#define TMIN_MAX_FILE (10 * 1024 * 1024) +#define TMIN_MAX_FILE (10 * 1024 * 1024L) /* Block normalization steps for afl-tmin: */ @@ -267,8 +267,8 @@ (first value), and to keep in memory as candidates. The latter should be much higher than the former. */ -#define USE_AUTO_EXTRAS 128 -#define MAX_AUTO_EXTRAS (USE_AUTO_EXTRAS * 64) +#define USE_AUTO_EXTRAS 4096 +#define MAX_AUTO_EXTRAS (USE_AUTO_EXTRAS * 8) /* Scaling factor for the effector map used to skip some of the more expensive deterministic steps. The actual divisor is set to diff --git a/include/debug.h b/include/debug.h index f8df5711..feb7f52d 100644 --- a/include/debug.h +++ b/include/debug.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/include/envs.h b/include/envs.h index e3957147..a3ba5e88 100644 --- a/include/envs.h +++ b/include/envs.h @@ -54,10 +54,12 @@ static char *afl_environment_variables[] = { "AFL_FAST_CAL", "AFL_FORCE_UI", "AFL_FRIDA_DEBUG_MAPS", + "AFL_FRIDA_DRIVER_NO_HOOK", "AFL_FRIDA_EXCLUDE_RANGES", "AFL_FRIDA_INST_COVERAGE_FILE", "AFL_FRIDA_INST_DEBUG_FILE", "AFL_FRIDA_INST_JIT", + "AFL_FRIDA_INST_NO_BACKPATCH", "AFL_FRIDA_INST_NO_OPTIMIZE", "AFL_FRIDA_INST_NO_PREFETCH", "AFL_FRIDA_INST_NO_PREFETCH_BACKPATCH", @@ -74,8 +76,11 @@ static char *afl_environment_variables[] = { "AFL_FRIDA_PERSISTENT_DEBUG", "AFL_FRIDA_PERSISTENT_HOOK", "AFL_FRIDA_PERSISTENT_RET", + "AFL_FRIDA_STALKER_IC_ENTRIES", + "AFL_FRIDA_STALKER_ADJACENT_BLOCKS", "AFL_FRIDA_STATS_FILE", "AFL_FRIDA_STATS_INTERVAL", + "AFL_FRIDA_TRACEABLE", "AFL_FUZZER_ARGS", // oss-fuzz "AFL_GDB", "AFL_GCC_ALLOWLIST", @@ -202,6 +207,7 @@ static char *afl_environment_variables[] = { "AFL_USE_MSAN", "AFL_USE_TRACE_PC", "AFL_USE_UBSAN", + "AFL_USE_TSAN", "AFL_USE_CFISAN", "AFL_USE_LSAN", "AFL_WINE_PATH", diff --git a/include/forkserver.h b/include/forkserver.h index c6f7de00..464f208d 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code that implements a forkserver. This is used by the fuzzer as well the other components like afl-tmin. @@ -82,6 +82,8 @@ typedef struct afl_forkserver { bool frida_asan; /* if running with asan in frida mode */ + bool cs_mode; /* if running in CoreSight mode or not */ + bool use_stdin; /* use stdin for sending data */ bool no_unlink; /* do not unlink cur_input */ diff --git a/include/hash.h b/include/hash.h index 9319ab95..9bb34ff8 100644 --- a/include/hash.h +++ b/include/hash.h @@ -21,7 +21,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/include/list.h b/include/list.h index 7ec81cbe..d49e56da 100644 --- a/include/list.h +++ b/include/list.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This allocator is not designed to resist malicious attackers (the canaries are small and predictable), but provides a robust and portable way to detect diff --git a/include/sharedmem.h b/include/sharedmem.h index fdc947f9..93080d0f 100644 --- a/include/sharedmem.h +++ b/include/sharedmem.h @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code to handle the shared memory. This is used by the fuzzer as well the other components like afl-tmin, afl-showmap, etc... diff --git a/include/snapshot-inl.h b/include/snapshot-inl.h index a18187ef..7234bbaa 100644 --- a/include/snapshot-inl.h +++ b/include/snapshot-inl.h @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/include/types.h b/include/types.h index 7b94fb83..bbcc2f81 100644 --- a/include/types.h +++ b/include/types.h @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ @@ -46,6 +46,8 @@ typedef uint128_t u128; #define FS_ERROR_SHM_OPEN 4 #define FS_ERROR_SHMAT 8 #define FS_ERROR_MMAP 16 +#define FS_ERROR_OLD_CMPLOG 32 +#define FS_ERROR_OLD_CMPLOG_QEMU 64 /* Reporting options */ #define FS_OPT_ENABLED 0x80000001 @@ -53,6 +55,7 @@ typedef uint128_t u128; #define FS_OPT_SNAPSHOT 0x20000000 #define FS_OPT_AUTODICT 0x10000000 #define FS_OPT_SHDMEM_FUZZ 0x01000000 +#define FS_OPT_NEWCMPLOG 0x02000000 #define FS_OPT_OLD_AFLPP_WORKAROUND 0x0f000000 // FS_OPT_MAX_MAPSIZE is 8388608 = 0x800000 = 2^23 = 1 << 22 #define FS_OPT_MAX_MAPSIZE ((0x00fffffeU >> 1) + 1) diff --git a/include/xxhash.h b/include/xxhash.h index 006d3f3d..0ca2b852 100644 --- a/include/xxhash.h +++ b/include/xxhash.h @@ -32,7 +32,12 @@ * - xxHash homepage: https://www.xxhash.com * - xxHash source repository: https://github.com/Cyan4973/xxHash */ - +/*! + * @mainpage xxHash + * + * @file xxhash.h + * xxHash prototypes and implementation + */ /* TODO: update */ /* Notice extracted from xxHash homepage: @@ -45,7 +50,7 @@ Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo Name Speed Q.Score Author xxHash 5.4 GB/s 10 CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby +MurmurHash 3a 2.7 GB/s 10 Austin Appleby SpookyHash 2.0 GB/s 10 Bob Jenkins SBox 1.4 GB/s 9 Bret Mulvey Lookup3 1.2 GB/s 9 Bob Jenkins @@ -119,29 +124,78 @@ extern "C" { /* * This part deals with the special case where a unit wants to inline xxHash, - * but "xxhash.h" has previously been included without XXH_INLINE_ALL, such - * as part of some previously included *.h header file. + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. * Without further action, the new include would just be ignored, * and functions would effectively _not_ be inlined (silent failure). * The following macros solve this situation by prefixing all inlined names, * avoiding naming collision with previous inclusions. */ - #ifdef XXH_NAMESPACE - #error "XXH_INLINE_ALL with XXH_NAMESPACE is not supported" - /* - * Note: Alternative: #undef all symbols (it's a pretty large list). - * Without #error: it compiles, but functions are actually not inlined. - */ - #endif +/* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ + #undef XXH_versionNumber +/* XXH32 */ + #undef XXH32 + #undef XXH32_createState + #undef XXH32_freeState + #undef XXH32_reset + #undef XXH32_update + #undef XXH32_digest + #undef XXH32_copyState + #undef XXH32_canonicalFromHash + #undef XXH32_hashFromCanonical +/* XXH64 */ + #undef XXH64 + #undef XXH64_createState + #undef XXH64_freeState + #undef XXH64_reset + #undef XXH64_update + #undef XXH64_digest + #undef XXH64_copyState + #undef XXH64_canonicalFromHash + #undef XXH64_hashFromCanonical +/* XXH3_64bits */ + #undef XXH3_64bits + #undef XXH3_64bits_withSecret + #undef XXH3_64bits_withSeed + #undef XXH3_createState + #undef XXH3_freeState + #undef XXH3_copyState + #undef XXH3_64bits_reset + #undef XXH3_64bits_reset_withSeed + #undef XXH3_64bits_reset_withSecret + #undef XXH3_64bits_update + #undef XXH3_64bits_digest + #undef XXH3_generateSecret +/* XXH3_128bits */ + #undef XXH128 + #undef XXH3_128bits + #undef XXH3_128bits_withSeed + #undef XXH3_128bits_withSecret + #undef XXH3_128bits_reset + #undef XXH3_128bits_reset_withSeed + #undef XXH3_128bits_reset_withSecret + #undef XXH3_128bits_update + #undef XXH3_128bits_digest + #undef XXH128_isEqual + #undef XXH128_cmp + #undef XXH128_canonicalFromHash + #undef XXH128_hashFromCanonical +/* Finally, free the namespace itself */ + #undef XXH_NAMESPACE + +/* employ the namespace for XXH_INLINE_ALL */ #define XXH_NAMESPACE XXH_INLINE_ /* - * Some identifiers (enums, type names) are not symbols, but they must - * still be renamed to avoid redeclaration. + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. * Alternative solution: do not redeclare them. - * However, this requires some #ifdefs, and is a more dispersed action. - * Meanwhile, renaming can be achieved in a single block + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. */ - #define XXH_IPREF(Id) XXH_INLINE_##Id + #define XXH_IPREF(Id) XXH_NAMESPACE##Id #define XXH_OK XXH_IPREF(XXH_OK) #define XXH_ERROR XXH_IPREF(XXH_ERROR) #define XXH_errorcode XXH_IPREF(XXH_errorcode) @@ -166,6 +220,12 @@ extern "C" { #ifndef XXHASH_H_5627135585666179 #define XXHASH_H_5627135585666179 1 + /*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + + */ /* specific declaration modes for Windows */ #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) #if defined(WIN32) && defined(_MSC_VER) && \ @@ -180,19 +240,24 @@ extern "C" { #endif #endif - /*! - * XXH_NAMESPACE, aka Namespace Emulation: - * - * If you want to include _and expose_ xxHash functions from within your own - * library, but also want to avoid symbol collisions with other libraries - * which may also include xxHash, you can use XXH_NAMESPACE to automatically - * prefix any public symbol from xxhash library with the value of - * XXH_NAMESPACE (therefore, avoid empty or numeric values). - * - * Note that no change is required within the calling program as long as it - * includes `xxhash.h`: Regular symbol names will be automatically translated - * by this header. - */ + #ifdef XXH_DOXYGEN + /*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries + * which may also include xxHash, you can use XXH_NAMESPACE to automatically + * prefix any public symbol from xxhash library with the value of + * XXH_NAMESPACE (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically + * translated by this header. + */ + #define XXH_NAMESPACE /* YOUR NAME HERE */ + #undef XXH_NAMESPACE + #endif + #ifdef XXH_NAMESPACE #define XXH_CAT(A, B) A##B #define XXH_NAME2(A, B) XXH_CAT(A, B) @@ -264,10 +329,19 @@ extern "C" { ***************************************/ #define XXH_VERSION_MAJOR 0 #define XXH_VERSION_MINOR 8 - #define XXH_VERSION_RELEASE 0 + #define XXH_VERSION_RELEASE 1 #define XXH_VERSION_NUMBER \ (XXH_VERSION_MAJOR * 100 * 100 + XXH_VERSION_MINOR * 100 + \ XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is only useful when xxHash is compiled as a shared library, as it is + * independent of the version defined in the header. + * + * @return `XXH_VERSION_NUMBER` as of when the libray was compiled. + */ XXH_PUBLIC_API unsigned XXH_versionNumber(void); /* **************************** @@ -279,15 +353,24 @@ typedef enum { XXH_OK = 0, XXH_ERROR } XXH_errorcode; /*-********************************************************************** * 32-bit hash ************************************************************************/ - #if !defined(__VMS) && \ + #if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + + #elif !defined(__VMS) && \ (defined(__cplusplus) || \ (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)) #include <stdint.h> -typedef uint32_t XXH32_hash_t; +typedef uint32_t XXH32_hash_t; + #else #include <limits.h> #if UINT_MAX == 0xFFFFFFFFUL -typedef unsigned int XXH32_hash_t; +typedef unsigned int XXH32_hash_t; #else #if ULONG_MAX == 0xFFFFFFFFUL typedef unsigned long XXH32_hash_t; @@ -298,24 +381,52 @@ typedef unsigned long XXH32_hash_t; #endif /*! - * XXH32(): - * Calculate the 32-bit hash of sequence "length" bytes stored at memory - * address "input". The memory between input & input+length must be valid - * (allocated and read-accessible). "seed" can be used to alter the result - * predictably. Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher - * benchmark): 5.4 GB/s - * - * Note: XXH3 provides competitive speed for both 32-bit and 64-bit systems, - * and offers true 64/128 bit hash results. It provides a superior level of - * dispersion, and greatly reduces the risks of collisions. + * @} + * + * @defgroup xxh32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is considered rather weak by today's standards. + * The @ref xxh3_family provides competitive speed for both 32-bit and 64-bit + * systems, and offers true 64/128 bit hash results. It provides a superior + * level of dispersion, and greatly reduces the risks of collisions. + * + * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families + * @see @ref xxh32_impl for implementation details + * @{ + + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * + * @param input The block of data to be hashed, at least @p length bytes in + * size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit hash value. + * + * @see + * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. */ XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t length, XXH32_hash_t seed); -/******* Streaming *******/ - -/* - * Streaming functions generate the xxHash value from an incrememtal input. +/*! + * Streaming functions generate the xxHash value from an incremental input. * This method is slower than single-call functions, due to state management. * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. * @@ -336,19 +447,125 @@ XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t length, * digest, and generate new hash values later on by invoking `XXH*_digest()`. * * When done, release the state using `XXH*_freeState()`. + * + * Example code for incrementally hashing a file: + * @code{.c} + * #include <stdio.h> + * #include <xxhash.h> + * #define BUFFER_SIZE 256 + * + * // Note: XXH64 and XXH3 use the same interface. + * XXH32_hash_t + * hashFile(FILE* stream) + * { + + * XXH32_state_t* state; + * unsigned char buf[BUFFER_SIZE]; + * size_t amt; + * XXH32_hash_t hash; + * + * state = XXH32_createState(); // Create a state + * assert(state != NULL); // Error check here + * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed + * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { + + * XXH32_update(state, buf, amt); // Hash the file in chunks + * } + * hash = XXH32_digest(state); // Finalize the hash + * XXH32_freeState(state); // Clean up + * return hash; + * } + * @endcode + */ + +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. */ +typedef struct XXH32_state_s XXH32_state_t; -typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * Must be freed with XXH32_freeState(). + * @return An allocated XXH32_state_t on success, `NULL` on failure. + */ XXH_PUBLIC_API XXH32_state_t *XXH32_createState(void); -XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr); -XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t * dst_state, - const XXH32_state_t *src_state); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * Must be allocated with XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref + * XXH32_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t * dst_state, + const XXH32_state_t *src_state); +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in + * size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *statePtr, const void *input, size_t length); -XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *statePtr); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash32 value from that state. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *statePtr); /******* Canonical representation *******/ @@ -373,48 +590,158 @@ XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *statePtr); * canonical format. */ +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ typedef struct { - unsigned char digest[4]; + unsigned char digest[4]; /*!< Hash bytes, big endian */ } XXH32_canonical_t; +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t *dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t *src); + #ifdef __has_attribute + #define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) + #else + #define XXH_HAS_ATTRIBUTE(x) 0 + #endif + + /* C-language Attributes are added in C23. */ + #if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && \ + defined(__has_c_attribute) + #define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) + #else + #define XXH_HAS_C_ATTRIBUTE(x) 0 + #endif + + #if defined(__cplusplus) && defined(__has_cpp_attribute) + #define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) + #else + #define XXH_HAS_CPP_ATTRIBUTE(x) 0 + #endif + + /* + Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' + attribute introduced in CPP17 and C23. CPP17 : + https://en.cppreference.com/w/cpp/language/attributes/fallthrough C23 : + https://en.cppreference.com/w/c/language/attributes/fallthrough + */ + #if XXH_HAS_C_ATTRIBUTE(x) + #define XXH_FALLTHROUGH [[fallthrough]] + #elif XXH_HAS_CPP_ATTRIBUTE(x) + #define XXH_FALLTHROUGH [[fallthrough]] + #elif XXH_HAS_ATTRIBUTE(__fallthrough__) + #define XXH_FALLTHROUGH __attribute__((fallthrough)) + #else + #define XXH_FALLTHROUGH + #endif + +/*! + * @} + * @ingroup public + * @{ + + */ + #ifndef XXH_NO_LONG_LONG /*-********************************************************************** * 64-bit hash ************************************************************************/ - #if !defined(__VMS) && \ + #if defined(XXH_DOXYGEN) /* don't include <stdint.h> */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; + #elif !defined(__VMS) && \ (defined(__cplusplus) || (defined(__STDC_VERSION__) && \ (__STDC_VERSION__ >= 199901L) /* C99 */)) #include <stdint.h> typedef uint64_t XXH64_hash_t; #else + #include <limits.h> + #if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL +/* LP64 ABI says uint64_t is unsigned long */ +typedef unsigned long XXH64_hash_t; + #else /* the following type must have a width of 64-bit */ typedef unsigned long long XXH64_hash_t; + #endif #endif /*! - * XXH64(): - * Returns the 64-bit hash of sequence of length @length stored at memory - * address @input. - * @seed can be used to alter the result predictably. + * @} + * + * @defgroup xxh64_family XXH64 family + * @ingroup public + * @{ + + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. It provides a superior level of + * dispersion, and greatly reduces the risks of collisions. + */ + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. * * This function usually runs faster on 64-bit systems, but slower on 32-bit * systems (see benchmark). * - * Note: XXH3 provides competitive speed for both 32-bit and 64-bit systems, - * and offers true 64/128 bit hash results. It provides a superior level of - * dispersion, and greatly reduces the risks of collisions. + * @param input The block of data to be hashed, at least @p length bytes in + * size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit hash. + * + * @see + * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. */ XXH_PUBLIC_API XXH64_hash_t XXH64(const void *input, size_t length, XXH64_hash_t seed); /******* Streaming *******/ +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ XXH_PUBLIC_API XXH64_state_t *XXH64_createState(void); XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr); @@ -439,12 +766,15 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t *dst, XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t *src); -/*-********************************************************************** - * XXH3 64-bit variant - ************************************************************************/ +/*! + * @} + * ************************************************************************ + * @defgroup xxh3_family XXH3 family + * @ingroup public + * @{ -/* ************************************************************************ - * XXH3 is a new hash algorithm featuring: + * + * XXH3 is a more recent hash algorithm featuring: * - Improved speed for both small and large inputs * - True 64-bit and 128-bit outputs * - SIMD acceleration @@ -454,41 +784,38 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src); * * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html * - * In general, expect XXH3 to run about ~2x faster on large inputs and >3x - * faster on small ones compared to XXH64, though exact differences depend on - * the platform. + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. * - * The algorithm is portable: Like XXH32 and XXH64, it generates the same hash - * on all platforms. - * - * It benefits greatly from SIMD and 64-bit arithmetic, but does not require it. - * - * Almost all 32-bit and 64-bit targets that can run XXH32 smoothly can run - * XXH3 at competitive speeds, even if XXH64 runs slowly. Further details are - * explained in the implementation. + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Any 32-bit and 64-bit targets that can run XXH32 smoothly + * can run XXH3 at competitive speeds, even without vector support. + * Further details are explained in the implementation. * * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, - * ZVector and scalar targets. This can be controlled with the XXH_VECTOR macro. + * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generage exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. * * XXH3 offers 2 variants, _64bits and _128bits. - * When only 64 bits are needed, prefer calling the _64bits variant, as it - * reduces the amount of mixing, resulting in faster speed on small inputs. * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. * It's also generally simpler to manipulate a scalar return type than a struct. * - * The 128-bit version adds additional strength, but it is slightly slower. - * - * Return values of XXH3 and XXH128 are officially finalized starting - * with v0.8.0 and will no longer change in future versions. - * Avoid storing values from before that release in long-term storage. - * - * Results produced by v0.7.x are not comparable with results from v0.7.y. - * However, the API is completely stable, and it can safely be used for - * ephemeral data (local sessions). - * * The API supports one-shot hashing, streaming mode, and custom secrets. */ +/*-********************************************************************** + * XXH3 64-bit variant + ************************************************************************/ + /* XXH3_64bits(): * default 64-bit variant, using default secret and default seed of 0. * It's the fastest variant. */ @@ -504,20 +831,28 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void *data, size_t len); XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void *data, size_t len, XXH64_hash_t seed); - /* - * XXH3_64bits_withSecret(): - * It's possible to provide any blob of bytes as a "secret" to generate the - * hash. This makes it more difficult for an external actor to prepare an - * intentional collision. The main condition is that secretSize *must* be - * large enough (>= XXH3_SECRET_SIZE_MIN). However, the quality of produced - * hash values depends on secret's entropy. Technically, the secret must - * look like a bunch of random bytes. Avoid "trivial" or structured data - * such as repeated sequences or a text document. Whenever unsure about the - * "randomness" of the blob of bytes, consider relabelling it as a "custom - * seed" instead, and employ "XXH3_generateSecret()" (see below) to generate - * a high entropy secret derived from the custom seed. + /*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). */ #define XXH3_SECRET_SIZE_MIN 136 + +/* + * XXH3_64bits_withSecret(): + * It's possible to provide any blob of bytes as a "secret" to generate the + * hash. This makes it more difficult for an external actor to prepare an + * intentional collision. The main condition is that secretSize *must* be large + * enough (>= XXH3_SECRET_SIZE_MIN). However, the quality of produced hash + * values depends on secret's entropy. Technically, the secret must look like a + * bunch of random bytes. Avoid "trivial" or structured data such as repeated + * sequences or a text document. Whenever unsure about the "randomness" of the + * blob of bytes, consider relabelling it as a "custom seed" instead, and employ + * "XXH3_generateSecret()" (see below) to generate a high entropy secret derived + * from the custom seed. + */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *data, size_t len, const void *secret, size_t secretSize); @@ -529,6 +864,12 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *data, size_t len, * As a consequence, streaming is slower than one-shot hashing. * For better performance, prefer one-shot functions whenever applicable. */ + +/*! + * @brief The state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ typedef struct XXH3_state_s XXH3_state_t; XXH_PUBLIC_API XXH3_state_t *XXH3_createState(void); XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr); @@ -572,10 +913,16 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *statePtr); * XXH3 128-bit variant ************************************************************************/ +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ typedef struct { - XXH64_hash_t low64; - XXH64_hash_t high64; + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ } XXH128_hash_t; @@ -649,6 +996,9 @@ XXH128_hashFromCanonical(const XXH128_canonical_t *src); #endif /* XXH_NO_LONG_LONG */ +/*! + * @} + */ #endif /* XXHASH_H_5627135585666179 */ #if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) @@ -660,7 +1010,7 @@ XXH128_hashFromCanonical(const XXH128_canonical_t *src); * These declarations should only be used with static linking. * Never use them in association with dynamic linking! ***************************************************************************** - */ +*/ /* * These definitions are only present to allow static allocation @@ -668,41 +1018,72 @@ XXH128_hashFromCanonical(const XXH128_canonical_t *src); * Never **ever** access their members directly. */ +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ struct XXH32_state_s { - XXH32_hash_t total_len_32; - XXH32_hash_t large_len; - XXH32_hash_t v1; - XXH32_hash_t v2; - XXH32_hash_t v3; - XXH32_hash_t v4; - XXH32_hash_t mem32[4]; - XXH32_hash_t memsize; - XXH32_hash_t - reserved; /* never read nor write, might be removed in a future version */ + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref + total_len_32 overflow) */ + XXH32_hash_t v1; /*!< First accumulator lane */ + XXH32_hash_t v2; /*!< Second accumulator lane */ + XXH32_hash_t v3; /*!< Third accumulator lane */ + XXH32_hash_t v4; /*!< Fourth accumulator lane */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as + unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read or write to it, it may + be removed. */ }; /* typedef'd to XXH32_state_t */ #ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ struct XXH64_state_s { - XXH64_hash_t total_len; - XXH64_hash_t v1; - XXH64_hash_t v2; - XXH64_hash_t v3; - XXH64_hash_t v4; - XXH64_hash_t mem64[4]; - XXH32_hash_t memsize; - XXH32_hash_t reserved32; /* required for padding anyway */ - XXH64_hash_t reserved64; /* never read nor write, might be removed in a future - version */ + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v1; /*!< First accumulator lane */ + XXH64_hash_t v2; /*!< Second accumulator lane */ + XXH64_hash_t v3; /*!< Third accumulator lane */ + XXH64_hash_t v4; /*!< Fourth accumulator lane */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as + unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it, it + may be removed. */ }; /* typedef'd to XXH64_state_t */ - #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11+ */ + #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 \ + */ #include <stdalign.h> #define XXH_ALIGN(n) alignas(n) + #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ + /* In C++ alignas() is a keyword */ + #define XXH_ALIGN(n) alignas(n) #elif defined(__GNUC__) #define XXH_ALIGN(n) __attribute__((aligned(n))) #elif defined(_MSC_VER) @@ -713,39 +1094,94 @@ struct XXH64_state_s { /* Old GCC versions only accept the attribute after the type in structures. */ - #if !(defined(__STDC_VERSION__) && \ - (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + #if !(defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && !(defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ && defined(__GNUC__) #define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) #else #define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type #endif + /*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ #define XXH3_INTERNALBUFFER_SIZE 256 + + /*! + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ #define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ struct XXH3_state_s { XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); - /* used to store a custom secret generated from a seed */ + /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref + * XXH64_state_s */ XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); - XXH32_hash_t bufferedSize; - XXH32_hash_t reserved32; - size_t nbStripesSoFar; - XXH64_hash_t totalLen; - size_t nbStripesPerBlock; - size_t secretLimit; - XXH64_hash_t seed; - XXH64_hash_t reserved64; - const unsigned char *extSecret; /* reference to external secret; - * if == NULL, use .customSecret instead */ + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t reserved32; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see + * XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char *extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ /* note: there may be some padding at the end due to alignment on 64 bytes */ }; /* typedef'd to XXH3_state_t */ #undef XXH_ALIGN_MEMBER - /* When the XXH3_state_t structure is merely emplaced on stack, + /*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, * it should be initialized with XXH3_INITSTATE() or a memset() * in case its first reset uses XXH3_NNbits_reset_withSeed(). * This init can be omitted if the first reset uses default or _withSecret @@ -802,7 +1238,6 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, XXH64_hash_t seed); #endif /* XXH_NO_LONG_LONG */ - #if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) #define XXH_IMPLEMENTATION #endif @@ -844,81 +1279,183 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, /* ************************************* * Tuning parameters ***************************************/ + /*! - * XXH_FORCE_MEMORY_ACCESS: - * By default, access to unaligned memory is controlled by `memcpy()`, which - * is safe and portable. - * - * Unfortunately, on some target/compiler combinations, the generated assembly - * is sub-optimal. + * @defgroup tuning Tuning parameters + * @{ + * - * The below switch allow selection of a different access method - * in the search for improved performance. - * Method 0 (default): - * Use `memcpy()`. Safe and portable. Default. - * Method 1: - * `__attribute__((packed))` statement. It depends on compiler extensions - * and is therefore not portable. - * This method is safe if your compiler supports it, and *generally* as - * fast or faster than `memcpy`. - * Method 2: - * Direct access via cast. This method doesn't depend on the compiler but - * violates the C standard. - * It can generate buggy code on targets which do not support unaligned - * memory accesses. - * But in some circumstances, it's the only known way to get the most - * performance (example: GCC + ARMv6) - * Method 3: - * Byteshift. This can generate the best code on old compilers which don't - * inline small `memcpy()` calls, and it might also be faster on - * big-endian systems which lack a native byteswap instruction. See - * https://stackoverflow.com/a/32095106/646947 for details. Prefer these - * methods in priority order (0 > 1 > 2 > 3) + * Various macros to control xxHash's behavior. */ + #ifdef XXH_DOXYGEN + /*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref xxh32_family and you have a strict C90 + * compiler. + */ + #define XXH_NO_LONG_LONG + #undef XXH_NO_LONG_LONG /* don't actually */ + /*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which + * is safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated + * assembly is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers + * will eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences + * an unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's + * the only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which + * don't inline small `memcpy()` calls, and it might also be faster on + * big-endian systems which lack a native byteswap instruction. However, + * some compilers will emit literal byteshifts even if the target supports + * unaligned access. + * . + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may + * cause another to read garbage data or even crash. + * + * See https://stackoverflow.com/a/32095106/646947 for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ + #define XXH_FORCE_MEMORY_ACCESS 0 + /*! + * @def XXH_ACCEPT_NULL_INPUT_POINTER + * @brief Whether to add explicit `NULL` checks. + * + * If the input pointer is `NULL` and the length is non-zero, xxHash's + * default behavior is to dereference it, triggering a segfault. + * + * When this macro is enabled, xxHash actively checks the input for a null + * pointer. If it is, the result for null input pointers is the same as a + * zero-length input. + */ + #define XXH_ACCEPT_NULL_INPUT_POINTER 0 + /*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs + * (XXH32() and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally + * negligible, but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro + * to 0. Then the code will always use unaligned memory access. Align check + * is automatically disabled on x86, x64 & arm64, which are platforms known + * to offer good unaligned memory accesses performance. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ + #define XXH_FORCE_ALIGN_CHECK 0 + + /*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all + * internal functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary + * which might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to + * performance, depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using + * -fno-inline with GCC or Clang, this will automatically be defined. + */ + #define XXH_NO_INLINE_HINTS 0 + + /*! + * @def XXH_REROLL + * @brief Whether to reroll `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses an unrolled loop + * in the form of a switch statement. + * + * This is not always desirable, as it generates larger code, + * and depending on the architecture, may even be slower + * + * This is automatically defined with `-Os`/`-Oz` on GCC and Clang. + */ + #define XXH_REROLL 0 + + /*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use + * this. + */ + #define XXH_OLD_NAMES + #undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + #endif /* XXH_DOXYGEN */ +/*! + * @} + */ + #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command \ line for example */ - #if !defined(__clang__) && defined(__GNUC__) && \ - defined(__ARM_FEATURE_UNALIGNED) && defined(__ARM_ARCH) && \ - (__ARM_ARCH == 6) - #define XXH_FORCE_MEMORY_ACCESS 2 - #elif !defined(__clang__) && \ - ((defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ - (defined(__GNUC__) && (defined(__ARM_ARCH) && __ARM_ARCH >= 7))) + /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */ + #if !defined(__clang__) && \ + ((defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + (defined(__GNUC__) && \ + ((defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \ + (defined(__mips__) && (__mips <= 5 || __mips_isa_rev < 6) && \ + (!defined(__mips16) || defined(__mips_mips16e2)))))) #define XXH_FORCE_MEMORY_ACCESS 1 #endif #endif - /*! - * XXH_ACCEPT_NULL_INPUT_POINTER: - * If the input pointer is NULL, xxHash's default behavior is to dereference - * it, triggering a segfault. When this macro is enabled, xxHash actively - * checks the input for a null pointer. If it is, the result for null input - * pointers is the same as a zero-length input. - */ #ifndef XXH_ACCEPT_NULL_INPUT_POINTER /* can be defined externally */ #define XXH_ACCEPT_NULL_INPUT_POINTER 0 #endif - /*! - * XXH_FORCE_ALIGN_CHECK: - * This is an important performance trick - * for architectures without decent unaligned memory access performance. - * It checks for input alignment, and when conditions are met, - * uses a "fast path" employing direct 32-bit/64-bit read, - * resulting in _dramatically faster_ read speed. - * - * The check costs one initial branch per hash, which is generally negligible, - * but not zero. Moreover, it's not useful to generate binary for an - * additional code path if memory access uses same instruction for both - * aligned and unaligned adresses. - * - * In these cases, the alignment check can be removed by setting this macro to - * 0. Then the code will always use unaligned memory access. Align check is - * automatically disabled on x86, x64 & arm64, which are platforms known to - * offer good unaligned memory accesses performance. - * - * This option does not affect XXH3 (only XXH32 and XXH64). - */ #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ #if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || \ defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */ @@ -928,25 +1465,6 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, #endif #endif - /*! - * XXH_NO_INLINE_HINTS: - * - * By default, xxHash tries to force the compiler to inline almost all - * internal functions. - * - * This can usually improve performance due to reduced jumping and improved - * constant folding, but significantly increases the size of the binary which - * might not be favorable. - * - * Additionally, sometimes the forced inlining can be detrimental to - * performance, depending on the architecture. - * - * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the - * compiler full control on whether to inline or not. - * - * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using - * -fno-inline with GCC or Clang, this will automatically be defined. - */ #ifndef XXH_NO_INLINE_HINTS #if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ || defined(__NO_INLINE__) /* -O0, -fno-inline */ @@ -956,44 +1474,57 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *data, size_t len, #endif #endif - /*! - * XXH_REROLL: - * Whether to reroll XXH32_finalize, and XXH64_finalize, - * instead of using an unrolled jump table/if statement loop. - * - * This is automatically defined on -Os/-Oz on GCC and Clang. - */ #ifndef XXH_REROLL - #if defined(__OPTIMIZE_SIZE__) + #if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ || \ + (defined(__GNUC__) && !defined(__clang__)) + /* The if/then loop is preferable to switch/case on gcc (on x64) */ #define XXH_REROLL 1 #else #define XXH_REROLL 0 #endif #endif + /*! + * @defgroup impl Implementation + * @{ + + */ + /* ************************************* * Includes & Memory related functions ***************************************/ - /*! + /* * Modify the local functions below should you wish to use * different memory routines for malloc() and free() */ #include <stdlib.h> +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ static void *XXH_malloc(size_t s) { return malloc(s); } +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ static void XXH_free(void *p) { free(p); } - /*! and for memcpy() */ #include <string.h> + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ static void *XXH_memcpy(void *dest, const void *src, size_t size) { return memcpy(dest, src, size); @@ -1037,7 +1568,11 @@ static void *XXH_memcpy(void *dest, const void *src, size_t size) { /* ************************************* * Debug ***************************************/ - /* + /*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * * XXH_DEBUGLEVEL is expected to be defined externally, typically via the * compiler's command line options. The value must be a number. */ @@ -1057,12 +1592,58 @@ static void *XXH_memcpy(void *dest, const void *src, size_t size) { #endif /* note: use after variable declarations */ - #define XXH_STATIC_ASSERT(c) \ - do { \ - \ - enum { XXH_sa = 1 / (int)(!!(c)) }; \ - \ - } while (0) + #ifndef XXH_STATIC_ASSERT + #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ + #include <assert.h> + #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ + do { \ + \ + static_assert((c), m); \ + \ + } while (0) + #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ + #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ + do { \ + \ + static_assert((c), m); \ + \ + } while (0) + #else + #define XXH_STATIC_ASSERT_WITH_MESSAGE(c, m) \ + do { \ + \ + struct xxh_sa { \ + \ + char x[(c) ? 1 : -1]; \ + \ + }; \ + \ + } while (0) + #endif + #define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c), #c) + #endif + + /*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ + #ifdef __GNUC__ + #define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r"(var)) + #else + #define XXH_COMPILER_GUARD(var) ((void)0) + #endif /* ************************************* * Basic Types @@ -1085,6 +1666,56 @@ typedef XXH32_hash_t xxh_u32; /* *** Memory access *** */ +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) /* * Manual byteshift. Best for old compilers which don't inline memcpy. @@ -1146,16 +1777,23 @@ static xxh_u32 XXH_read32(const void *memPtr) { #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ -/* *** Endianess *** */ -typedef enum { XXH_bigEndian = 0, XXH_littleEndian = 1 } XXH_endianess; + /* *** Endianness *** */ /*! - * XXH_CPU_LITTLE_ENDIAN: + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * * Defined to 1 if the target is little endian, or 0 if it is big endian. * It can be defined externally, for example on the compiler command line. * - * If it is not defined, a runtime check (which is usually constant folded) - * is used instead. + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. */ #ifndef XXH_CPU_LITTLE_ENDIAN /* @@ -1170,8 +1808,11 @@ typedef enum { XXH_bigEndian = 0, XXH_littleEndian = 1 } XXH_endianess; (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) #define XXH_CPU_LITTLE_ENDIAN 0 #else -/* - * runtime test, presumed to simplify to a constant by compiler +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. */ static int XXH_isLittleEndian(void) { @@ -1189,7 +1830,7 @@ static int XXH_isLittleEndian(void) { return one.c[0]; } - +\ #define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() #endif #endif @@ -1205,6 +1846,19 @@ static int XXH_isLittleEndian(void) { #define XXH_HAS_BUILTIN(x) 0 #endif + /*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ #if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) && \ XXH_HAS_BUILTIN(__builtin_rotateleft64) #define XXH_rotl32 __builtin_rotateleft32 @@ -1219,6 +1873,14 @@ static int XXH_isLittleEndian(void) { #define XXH_rotl64(x, r) (((x) << (r)) | ((x) >> (64 - (r)))) #endif + /*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ #if defined(_MSC_VER) /* Visual Studio */ #define XXH_swap32 _byteswap_ulong #elif XXH_GCC_VERSION >= 403 @@ -1236,7 +1898,17 @@ static xxh_u32 XXH_swap32(xxh_u32 x) { /* *************************** * Memory reads *****************************/ -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ + +} XXH_alignment; /* * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. @@ -1295,6 +1967,7 @@ XXH_FORCE_INLINE xxh_u32 XXH_readLE32_align(const void * ptr, /* ************************************* * Misc ***************************************/ +/*! @ingroup public */ XXH_PUBLIC_API unsigned XXH_versionNumber(void) { return XXH_VERSION_NUMBER; @@ -1304,16 +1977,19 @@ XXH_PUBLIC_API unsigned XXH_versionNumber(void) { /* ******************************************************************* * 32-bit hash functions *********************************************************************/ -static const xxh_u32 XXH_PRIME32_1 = - 0x9E3779B1U; /* 0b10011110001101110111100110110001 */ -static const xxh_u32 XXH_PRIME32_2 = - 0x85EBCA77U; /* 0b10000101111010111100101001110111 */ -static const xxh_u32 XXH_PRIME32_3 = - 0xC2B2AE3DU; /* 0b11000010101100101010111000111101 */ -static const xxh_u32 XXH_PRIME32_4 = - 0x27D4EB2FU; /* 0b00100111110101001110101100101111 */ -static const xxh_u32 XXH_PRIME32_5 = - 0x165667B1U; /* 0b00010110010101100110011110110001 */ +/*! + * @} + * @defgroup xxh32_impl XXH32 implementation + * @ingroup impl + * @{ + + */ +/* #define instead of static const, to be used as initializers */ + #define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ + #define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ + #define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ + #define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ + #define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ #ifdef XXH_OLD_NAMES #define PRIME32_1 XXH_PRIME32_1 @@ -1323,19 +1999,29 @@ static const xxh_u32 XXH_PRIME32_5 = #define PRIME32_5 XXH_PRIME32_5 #endif +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) { acc += input * XXH_PRIME32_2; acc = XXH_rotl32(acc, 13); acc *= XXH_PRIME32_1; - #if defined(__GNUC__) && defined(__SSE4_1__) && \ + #if (defined(__SSE4_1__) || defined(__aarch64__)) && \ !defined(XXH_ENABLE_AUTOVECTORIZE) /* * UGLY HACK: - * This inline assembly hack forces acc into a normal register. This is the - * only thing that prevents GCC and Clang from autovectorizing the XXH32 - * loop (pragmas and attributes don't work for some resason) without globally - * disabling SSE4.1. + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. * * The reason we want to avoid vectorization is because despite working on * 4 integers at a time, there are multiple factors slowing XXH32 down on @@ -1360,28 +2046,26 @@ static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) { * can load data, while v3 can multiply. SSE forces them to operate * together. * - * How this hack works: - * __asm__("" // Declare an assembly block but don't declare any - * instructions : // However, as an Input/Output Operand, - * "+r" // constrain a read/write operand (+) as a general purpose - * register (r). (acc) // and set acc as the operand - * ); - * - * Because of the 'r', the compiler has promised that seed will be in a - * general purpose register and the '+' says that it will be 'read/write', - * so it has to assume it has changed. It is like volatile without all the - * loads and stores. - * - * Since the argument has to be in a normal register (not an SSE register), - * each time XXH32_round is called, it is impossible to vectorize. + * This is also enabled on AArch64, as Clang autovectorizes it incorrectly + * and it is pointless writing a NEON implementation that is basically the + * same speed as scalar for XXH32. */ - __asm__("" : "+r"(acc)); + XXH_COMPILER_GUARD(acc); #endif return acc; } -/* mix all bits */ +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param h32 The hash to avalanche. + * @return The avalanched hash. + */ static xxh_u32 XXH32_avalanche(xxh_u32 h32) { h32 ^= h32 >> 15; @@ -1395,11 +2079,23 @@ static xxh_u32 XXH32_avalanche(xxh_u32 h32) { #define XXH_get32bits(p) XXH_readLE32_align(p, align) +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param h32 The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + */ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, XXH_alignment align) { - - /* dummy comment */ - +\ #define XXH_PROCESS1 \ do { \ \ @@ -1443,20 +2139,20 @@ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, case 12: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 8: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 4: XXH_PROCESS4; return XXH32_avalanche(h32); case 13: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 9: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 5: XXH_PROCESS4; XXH_PROCESS1; @@ -1464,10 +2160,10 @@ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, case 14: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 10: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 6: XXH_PROCESS4; XXH_PROCESS1; @@ -1476,22 +2172,22 @@ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, case 15: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 11: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 7: XXH_PROCESS4; - /* fallthrough */ + XXH_FALLTHROUGH; case 3: XXH_PROCESS1; - /* fallthrough */ + XXH_FALLTHROUGH; case 2: XXH_PROCESS1; - /* fallthrough */ + XXH_FALLTHROUGH; case 1: XXH_PROCESS1; - /* fallthrough */ + XXH_FALLTHROUGH; case 0: return XXH32_avalanche(h32); @@ -1512,10 +2208,18 @@ static xxh_u32 XXH32_finalize(xxh_u32 h32, const xxh_u8 *ptr, size_t len, #undef XXH_PROCESS4 #endif +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input, len, seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ XXH_FORCE_INLINE xxh_u32 XXH32_endian_align(const xxh_u8 *input, size_t len, xxh_u32 seed, XXH_alignment align) { - const xxh_u8 *bEnd = input + len; + const xxh_u8 *bEnd = input ? input + len : NULL; xxh_u32 h32; #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ @@ -1565,6 +2269,7 @@ XXH_FORCE_INLINE xxh_u32 XXH32_endian_align(const xxh_u8 *input, size_t len, } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t len, XXH32_hash_t seed) { @@ -1574,9 +2279,7 @@ XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t len, XXH32_reset(&state, seed); XXH32_update(&state, (const xxh_u8*)input, len); return XXH32_digest(&state); - #else - if (XXH_FORCE_ALIGN_CHECK) { if ((((size_t)input) & 3) == @@ -1593,13 +2296,16 @@ XXH_PUBLIC_API XXH32_hash_t XXH32(const void *input, size_t len, } /******* Hash streaming *******/ - +/*! + * @ingroup xxh32_family + */ XXH_PUBLIC_API XXH32_state_t *XXH32_createState(void) { return (XXH32_state_t *)XXH_malloc(sizeof(XXH32_state_t)); } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr) { XXH_free(statePtr); @@ -1607,6 +2313,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t *statePtr) { } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t * dstState, const XXH32_state_t *srcState) { @@ -1614,6 +2321,7 @@ XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t * dstState, } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, XXH32_hash_t seed) { @@ -1630,6 +2338,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t *statePtr, } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *state, const void *input, size_t len) { @@ -1719,6 +2428,7 @@ XXH_PUBLIC_API XXH_errorcode XXH32_update(XXH32_state_t *state, } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *state) { xxh_u32 h32; @@ -1743,7 +2453,8 @@ XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t *state) { /******* Canonical representation *******/ -/* +/*! + * @ingroup xxh32_family * The default return values from XXH functions are unsigned 32 and 64 bit * integers. * @@ -1765,6 +2476,7 @@ XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t *dst, } +/*! @ingroup xxh32_family */ XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t *src) { @@ -1777,7 +2489,12 @@ XXH32_hashFromCanonical(const XXH32_canonical_t *src) { /* ******************************************************************* * 64-bit hash functions *********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ /******* Memory access *******/ typedef XXH64_hash_t xxh_u64; @@ -1786,40 +2503,6 @@ typedef XXH64_hash_t xxh_u64; #define U64 xxh_u64 #endif - /*! - * XXH_REROLL_XXH64: - * Whether to reroll the XXH64_finalize() loop. - * - * Just like XXH32, we can unroll the XXH64_finalize() loop. This can be a - * performance gain on 64-bit hosts, as only one jump is required. - * - * However, on 32-bit hosts, because arithmetic needs to be done with two - * 32-bit registers, and 64-bit arithmetic needs to be simulated, it isn't - * beneficial to unroll. The code becomes ridiculously large (the largest - * function in the binary on i386!), and rerolling it saves anywhere from - * 3kB to 20kB. It is also slightly faster because it fits into cache better - * and is more likely to be inlined by the compiler. - * - * If XXH_REROLL is defined, this is ignored and the loop is always - * rerolled. - */ - #ifndef XXH_REROLL_XXH64 - #if (defined(__ILP32__) || \ - defined(_ILP32)) /* ILP32 is often defined on 32-bit GCC family */ \ - || !(defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) /* x86-64 */ \ - || defined(_M_ARM64) || defined(__aarch64__) || \ - defined(__arm64__) /* aarch64 */ \ - || defined(__PPC64__) || defined(__PPC64LE__) || \ - defined(__ppc64__) || defined(__powerpc64__) /* ppc64 */ \ - || defined(__mips64__) || defined(__mips64)) /* mips64 */ \ - || (!defined(SIZE_MAX) || SIZE_MAX < ULLONG_MAX) /* check limits */ - #define XXH_REROLL_XXH64 1 - #else - #define XXH_REROLL_XXH64 0 - #endif - #endif /* !defined(XXH_REROLL_XXH64) */ - #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS == 3)) /* * Manual byteshift. Best for old compilers which don't inline memcpy. @@ -1950,23 +2633,35 @@ XXH_FORCE_INLINE xxh_u64 XXH_readLE64_align(const void * ptr, } -/******* xxh64 *******/ + /******* xxh64 *******/ + /*! + * @} + * @defgroup xxh64_impl XXH64 implementation + * @ingroup impl + * @{ -static const xxh_u64 XXH_PRIME64_1 = - 0x9E3779B185EBCA87ULL; /* 0b1001111000110111011110011011000110000101111010111100101010000111 - */ -static const xxh_u64 XXH_PRIME64_2 = - 0xC2B2AE3D27D4EB4FULL; /* 0b1100001010110010101011100011110100100111110101001110101101001111 - */ -static const xxh_u64 XXH_PRIME64_3 = - 0x165667B19E3779F9ULL; /* 0b0001011001010110011001111011000110011110001101110111100111111001 - */ -static const xxh_u64 XXH_PRIME64_4 = - 0x85EBCA77C2B2AE63ULL; /* 0b1000010111101011110010100111011111000010101100101010111001100011 - */ -static const xxh_u64 XXH_PRIME64_5 = - 0x27D4EB2F165667C5ULL; /* 0b0010011111010100111010110010111100010110010101100110011111000101 - */ + */ + /* #define rather that static const, to be used as initializers */ + #define XXH_PRIME64_1 \ + 0x9E3779B185EBCA87ULL /*!< \ + 0b1001111000110111011110011011000110000101111010111100101010000111 \ + */ + #define XXH_PRIME64_2 \ + 0xC2B2AE3D27D4EB4FULL /*!< \ + 0b1100001010110010101011100011110100100111110101001110101101001111 \ + */ + #define XXH_PRIME64_3 \ + 0x165667B19E3779F9ULL /*!< \ + 0b0001011001010110011001111011000110011110001101110111100111111001 \ + */ + #define XXH_PRIME64_4 \ + 0x85EBCA77C2B2AE63ULL /*!< \ + 0b1000010111101011110010100111011111000010101100101010111001100011 \ + */ + #define XXH_PRIME64_5 \ + 0x27D4EB2F165667C5ULL /*!< \ + 0b0010011111010100111010110010111100010110010101100110011111000101 \ + */ #ifdef XXH_OLD_NAMES #define PRIME64_1 XXH_PRIME64_1 @@ -2010,185 +2705,35 @@ static xxh_u64 XXH64_avalanche(xxh_u64 h64) { static xxh_u64 XXH64_finalize(xxh_u64 h64, const xxh_u8 *ptr, size_t len, XXH_alignment align) { - /* dummy comment */ - - #define XXH_PROCESS1_64 \ - do { \ - \ - h64 ^= (*ptr++) * XXH_PRIME64_5; \ - h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; \ - \ - } while (0) - - #define XXH_PROCESS4_64 \ - do { \ - \ - h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; \ - ptr += 4; \ - h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; \ - \ - } while (0) - - #define XXH_PROCESS8_64 \ - do { \ - \ - xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); \ - ptr += 8; \ - h64 ^= k1; \ - h64 = XXH_rotl64(h64, 27) * XXH_PRIME64_1 + XXH_PRIME64_4; \ - \ - } while (0) - - /* Rerolled version for 32-bit targets is faster and much smaller. */ - if (XXH_REROLL || XXH_REROLL_XXH64) { - - len &= 31; - while (len >= 8) { - - XXH_PROCESS8_64; - len -= 8; - - } - - if (len >= 4) { - - XXH_PROCESS4_64; - len -= 4; - - } - - while (len > 0) { + len &= 31; + while (len >= 8) { - XXH_PROCESS1_64; - --len; + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + h64 ^= k1; + h64 = XXH_rotl64(h64, 27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; - } + } - return XXH64_avalanche(h64); + if (len >= 4) { - } else { + h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; - switch (len & 31) { + } - case 24: - XXH_PROCESS8_64; - /* fallthrough */ - case 16: - XXH_PROCESS8_64; - /* fallthrough */ - case 8: - XXH_PROCESS8_64; - return XXH64_avalanche(h64); - - case 28: - XXH_PROCESS8_64; - /* fallthrough */ - case 20: - XXH_PROCESS8_64; - /* fallthrough */ - case 12: - XXH_PROCESS8_64; - /* fallthrough */ - case 4: - XXH_PROCESS4_64; - return XXH64_avalanche(h64); - - case 25: - XXH_PROCESS8_64; - /* fallthrough */ - case 17: - XXH_PROCESS8_64; - /* fallthrough */ - case 9: - XXH_PROCESS8_64; - XXH_PROCESS1_64; - return XXH64_avalanche(h64); - - case 29: - XXH_PROCESS8_64; - /* fallthrough */ - case 21: - XXH_PROCESS8_64; - /* fallthrough */ - case 13: - XXH_PROCESS8_64; - /* fallthrough */ - case 5: - XXH_PROCESS4_64; - XXH_PROCESS1_64; - return XXH64_avalanche(h64); - - case 26: - XXH_PROCESS8_64; - /* fallthrough */ - case 18: - XXH_PROCESS8_64; - /* fallthrough */ - case 10: - XXH_PROCESS8_64; - XXH_PROCESS1_64; - XXH_PROCESS1_64; - return XXH64_avalanche(h64); - - case 30: - XXH_PROCESS8_64; - /* fallthrough */ - case 22: - XXH_PROCESS8_64; - /* fallthrough */ - case 14: - XXH_PROCESS8_64; - /* fallthrough */ - case 6: - XXH_PROCESS4_64; - XXH_PROCESS1_64; - XXH_PROCESS1_64; - return XXH64_avalanche(h64); - - case 27: - XXH_PROCESS8_64; - /* fallthrough */ - case 19: - XXH_PROCESS8_64; - /* fallthrough */ - case 11: - XXH_PROCESS8_64; - XXH_PROCESS1_64; - XXH_PROCESS1_64; - XXH_PROCESS1_64; - return XXH64_avalanche(h64); - - case 31: - XXH_PROCESS8_64; - /* fallthrough */ - case 23: - XXH_PROCESS8_64; - /* fallthrough */ - case 15: - XXH_PROCESS8_64; - /* fallthrough */ - case 7: - XXH_PROCESS4_64; - /* fallthrough */ - case 3: - XXH_PROCESS1_64; - /* fallthrough */ - case 2: - XXH_PROCESS1_64; - /* fallthrough */ - case 1: - XXH_PROCESS1_64; - /* fallthrough */ - case 0: - return XXH64_avalanche(h64); + while (len > 0) { - } + h64 ^= (*ptr++) * XXH_PRIME64_5; + h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; + --len; } - /* impossible to reach */ - XXH_ASSERT(0); - return 0; /* unreachable, but some compilers complain without it */ + return XXH64_avalanche(h64); } @@ -2205,7 +2750,7 @@ static xxh_u64 XXH64_finalize(xxh_u64 h64, const xxh_u8 *ptr, size_t len, XXH_FORCE_INLINE xxh_u64 XXH64_endian_align(const xxh_u8 *input, size_t len, xxh_u64 seed, XXH_alignment align) { - const xxh_u8 *bEnd = input + len; + const xxh_u8 *bEnd = input ? input + len : NULL; xxh_u64 h64; #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ @@ -2259,6 +2804,7 @@ XXH_FORCE_INLINE xxh_u64 XXH64_endian_align(const xxh_u8 *input, size_t len, } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH64_hash_t XXH64(const void *input, size_t len, XXH64_hash_t seed) { @@ -2268,9 +2814,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH64(const void *input, size_t len, XXH64_reset(&state, seed); XXH64_update(&state, (const xxh_u8*)input, len); return XXH64_digest(&state); - #else - if (XXH_FORCE_ALIGN_CHECK) { if ((((size_t)input) & 7) == @@ -2289,12 +2833,14 @@ XXH_PUBLIC_API XXH64_hash_t XXH64(const void *input, size_t len, /******* Hash Streaming *******/ +/*! @ingroup xxh64_family*/ XXH_PUBLIC_API XXH64_state_t *XXH64_createState(void) { return (XXH64_state_t *)XXH_malloc(sizeof(XXH64_state_t)); } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr) { XXH_free(statePtr); @@ -2302,6 +2848,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t *statePtr) { } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t * dstState, const XXH64_state_t *srcState) { @@ -2309,6 +2856,7 @@ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t * dstState, } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t *statePtr, XXH64_hash_t seed) { @@ -2325,6 +2873,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t *statePtr, } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t *state, const void *input, size_t len) { @@ -2403,6 +2952,7 @@ XXH_PUBLIC_API XXH_errorcode XXH64_update(XXH64_state_t *state, } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t *state) { xxh_u64 h64; @@ -2436,6 +2986,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t *state) { /******* Canonical representation *******/ +/*! @ingroup xxh64_family */ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t *dst, XXH64_hash_t hash) { @@ -2445,6 +2996,7 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t *dst, } +/*! @ingroup xxh64_family */ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t *src) { @@ -2452,380 +3004,452 @@ XXH64_hashFromCanonical(const XXH64_canonical_t *src) { } - /* ********************************************************************* - * XXH3 - * New generation hash designed for speed on small keys and vectorization - ************************************************************************ */ + #ifndef XXH_NO_XXH3 - /* === Compiler specifics === */ + /* ********************************************************************* + * XXH3 + * New generation hash designed for speed on small keys and vectorization + ************************************************************************ */ + /*! + * @} + * @defgroup xxh3_impl XXH3 implementation + * @ingroup impl + * @{ - #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ - #define XXH_RESTRICT restrict - #else - /* Note: it might be useful to define __restrict or __restrict__ for some - * C++ compilers */ - #define XXH_RESTRICT /* disable */ - #endif + */ - #if (defined(__GNUC__) && (__GNUC__ >= 3)) || \ - (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || \ - defined(__clang__) - #define XXH_likely(x) __builtin_expect(x, 1) - #define XXH_unlikely(x) __builtin_expect(x, 0) - #else - #define XXH_likely(x) (x) - #define XXH_unlikely(x) (x) - #endif + /* === Compiler specifics === */ - #if defined(__GNUC__) - #if defined(__AVX2__) - #include <immintrin.h> - #elif defined(__SSE2__) - #include <emmintrin.h> - #elif defined(__ARM_NEON__) || defined(__ARM_NEON) - #define inline __inline__ /* circumvent a clang bug */ - #include <arm_neon.h> - #undef inline + #if ((defined(sun) || defined(__sun)) && \ + __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested \ + with GCC 5.5 */ + #define XXH_RESTRICT /* disable */ + #elif defined(__STDC_VERSION__) && \ + __STDC_VERSION__ >= 199901L /* >= C99 */ + #define XXH_RESTRICT restrict + #else + /* Note: it might be useful to define __restrict or __restrict__ for + * some C++ compilers */ + #define XXH_RESTRICT /* disable */ #endif - #elif defined(_MSC_VER) - #include <intrin.h> - #endif - - /* - * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while - * remaining a true 64-bit/128-bit hash function. - * - * This is done by prioritizing a subset of 64-bit operations that can be - * emulated without too many steps on the average 32-bit machine. - * - * For example, these two lines seem similar, and run equally fast on - * 64-bit: - * - * xxh_u64 x; - * x ^= (x >> 47); // good - * x ^= (x >> 13); // bad - * - * However, to a 32-bit machine, there is a major difference. - * - * x ^= (x >> 47) looks like this: - * - * x.lo ^= (x.hi >> (47 - 32)); - * - * while x ^= (x >> 13) looks like this: - * - * // note: funnel shifts are not usually cheap. - * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); - * x.hi ^= (x.hi >> 13); - * - * The first one is significantly faster than the second, simply because the - * shift is larger than 32. This means: - * - All the bits we need are in the upper 32 bits, so we can ignore the - * lower 32 bits in the shift. - * - The shift result will always fit in the lower 32 bits, and therefore, - * we can ignore the upper 32 bits in the xor. - * - * Thanks to this optimization, XXH3 only requires these features to be - * efficient: - * - * - Usable unaligned access - * - A 32-bit or 64-bit ALU - * - If 32-bit, a decent ADC instruction - * - A 32 or 64-bit multiply with a 64-bit result - * - For the 128-bit variant, a decent byteswap helps short inputs. - * - * The first two are already required by XXH32, and almost all 32-bit and - * 64-bit platforms which can run XXH32 can run XXH3 efficiently. - * - * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one - * notable exception. - * - * First of all, Thumb-1 lacks support for the UMULL instruction which - * performs the important long multiply. This means numerous __aeabi_lmul - * calls. - * - * Second of all, the 8 functional registers are just not enough. - * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic - * need Lo registers, and this shuffling results in thousands more MOVs than - * A32. - * - * A32 and T32 don't have this limitation. They can access all 14 registers, - * do a 32->64 multiply with UMULL, and the flexible operand allowing free - * shifts is helpful, too. - * - * Therefore, we do a quick sanity check. - * - * If compiling Thumb-1 for a target which supports ARM instructions, we - * will emit a warning, as it is not a "sane" platform to compile for. - * - * Usually, if this happens, it is because of an accident and you probably - * need to specify -march, as you likely meant to compile for a newer - * architecture. - * - * Credit: large sections of the vectorial and asm source code paths - * have been contributed by @easyaspi314 - */ - #if defined(__thumb__) && !defined(__thumb2__) && \ - defined(__ARM_ARCH_ISA_ARM) - #warning "XXH3 is highly inefficient without ARM or Thumb-2." - #endif - /* ========================================== - * Vectorization detection - * ========================================== */ - #define XXH_SCALAR 0 /* Portable scalar version */ - #define XXH_SSE2 1 /* SSE2 for Pentium 4 and all x86_64 */ - #define XXH_AVX2 2 /* AVX2 for Haswell and Bulldozer */ - #define XXH_AVX512 3 /* AVX512 for Skylake and Icelake */ - #define XXH_NEON 4 /* NEON for most ARMv7-A and all AArch64 */ - #define XXH_VSX 5 /* VSX and ZVector for POWER8/z13 */ - - #ifndef XXH_VECTOR /* can be defined on command line */ - #if defined(__AVX512F__) - #define XXH_VECTOR XXH_AVX512 - #elif defined(__AVX2__) - #define XXH_VECTOR XXH_AVX2 - #elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) - #define XXH_VECTOR XXH_SSE2 - #elif defined(__GNUC__) /* msvc support maybe later */ \ - && (defined(__ARM_NEON__) || defined(__ARM_NEON)) && \ - (defined(__LITTLE_ENDIAN__) /* We only support little endian NEON */ \ - || (defined(__BYTE_ORDER__) && \ - __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) - #define XXH_VECTOR XXH_NEON - #elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) || \ - (defined(__s390x__) && defined(__VEC__)) && \ - defined(__GNUC__) /* TODO: IBM XL */ - #define XXH_VECTOR XXH_VSX + #if (defined(__GNUC__) && (__GNUC__ >= 3)) || \ + (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || \ + defined(__clang__) + #define XXH_likely(x) __builtin_expect(x, 1) + #define XXH_unlikely(x) __builtin_expect(x, 0) #else - #define XXH_VECTOR XXH_SCALAR + #define XXH_likely(x) (x) + #define XXH_unlikely(x) (x) #endif - #endif - /* - * Controls the alignment of the accumulator, - * for compatibility with aligned vector loads, which are usually faster. - */ - #ifndef XXH_ACC_ALIGN - #if defined(XXH_X86DISPATCH) - #define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ - #elif XXH_VECTOR == XXH_SCALAR /* scalar */ - #define XXH_ACC_ALIGN 8 - #elif XXH_VECTOR == XXH_SSE2 /* sse2 */ - #define XXH_ACC_ALIGN 16 - #elif XXH_VECTOR == XXH_AVX2 /* avx2 */ - #define XXH_ACC_ALIGN 32 - #elif XXH_VECTOR == XXH_NEON /* neon */ - #define XXH_ACC_ALIGN 16 - #elif XXH_VECTOR == XXH_VSX /* vsx */ - #define XXH_ACC_ALIGN 16 - #elif XXH_VECTOR == XXH_AVX512 /* avx512 */ - #define XXH_ACC_ALIGN 64 + #if defined(__GNUC__) + #if defined(__AVX2__) + #include <immintrin.h> + #elif defined(__SSE2__) + #include <emmintrin.h> + #elif defined(__ARM_NEON__) || defined(__ARM_NEON) + #define inline __inline__ /* circumvent a clang bug */ + #include <arm_neon.h> + #undef inline + #endif + #elif defined(_MSC_VER) + #include <intrin.h> #endif - #endif - #if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 || \ - XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 - #define XXH_SEC_ALIGN XXH_ACC_ALIGN - #else - #define XXH_SEC_ALIGN 8 - #endif - - /* - * UGLY HACK: - * GCC usually generates the best code with -O3 for xxHash. - * - * However, when targeting AVX2, it is overzealous in its unrolling - * resulting in code roughly 3/4 the speed of Clang. - * - * There are other issues, such as GCC splitting _mm256_loadu_si256 into - * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which - * only applies to Sandy and Ivy Bridge... which don't even support AVX2. - * - * That is why when compiling the AVX2 version, it is recommended to use - * either -O2 -mavx2 -march=haswell or -O2 -mavx2 - * -mno-avx256-split-unaligned-load for decent performance, or to use Clang - * instead. - * - * Fortunately, we can control the first one with a pragma that forces GCC - * into -O2, but the other one we can't control without "failed to inline - * always inline function due to target mismatch" warnings. - */ - #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ - && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__OPTIMIZE__) && \ - !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ - #pragma GCC push_options - #pragma GCC optimize("-O2") - #endif - - #if XXH_VECTOR == XXH_NEON /* - * NEON's setup for vmlal_u32 is a little more complicated than it is on - * SSE2, AVX2, and VSX. + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. * - * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an - * upcast. + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. * - * To do the same operation, the 128-bit 'Q' register needs to be split - * into two 64-bit 'D' registers, performing this operation:: + * For example, these two lines seem similar, and run equally fast on + * 64-bit: * - * [ a | b ] | - * '---------. .--------' | | x | - * | .---------' '--------. | - * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ] + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad * - * Due to significant changes in aarch64, the fastest method for aarch64 - * is completely different than the fastest method for ARMv7-A. + * However, to a 32-bit machine, there is a major difference. * - * ARMv7-A treats D registers as unions overlaying Q registers, so - * modifying D11 will modify the high half of Q5. This is similar to how - * modifying AH will only affect bits 8-15 of AX on x86. + * x ^= (x >> 47) looks like this: * - * VZIP takes two registers, and puts even lanes in one register and odd - * lanes in the other. + * x.lo ^= (x.hi >> (47 - 32)); * - * On ARMv7-A, this strangely modifies both parameters in place instead of - * taking the usual 3-operand form. + * while x ^= (x >> 13) looks like this: * - * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on - * the lower and upper halves of the Q register to end up with the high - * and low halves where we want - all in one instruction. + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); * - * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], - * d11[1] } + * The first one is significantly faster than the second, simply because + * the shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the + * lower 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and + * therefore, we can ignore the upper 32 bits in the xor. * - * Unfortunately we need inline assembly for this: Instructions modifying - * two registers at once is not possible in GCC or Clang's IR, and they - * have to create a copy. + * Thanks to this optimization, XXH3 only requires these features to be + * efficient: * - * aarch64 requires a different approach. + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. * - * In order to make it easier to write a decent compiler for aarch64, many - * quirks were removed, such as conditional execution. + * The first two are already required by XXH32, and almost all 32-bit and + * 64-bit platforms which can run XXH32 can run XXH3 efficiently. * - * NEON was also affected by this. + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is + * one notable exception. * - * aarch64 cannot access the high bits of a Q-form register, and writes to - * a D-form register zero the high bits, similar to how writes to W-form - * scalar registers (or DWORD registers on x86_64) work. + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. * - * The formerly free vget_high intrinsics now require a vext (with a few - * exceptions) + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic + * need Lo registers, and this shuffling results in thousands more MOVs + * than A32. * - * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the - * equivalent of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to - * only modify one operand. + * A32 and T32 don't have this limitation. They can access all 14 + * registers, do a 32->64 multiply with UMULL, and the flexible operand + * allowing free shifts is helpful, too. * - * The equivalent of the VZIP.32 on the lower and upper halves would be - * this mess: + * Therefore, we do a quick sanity check. * - * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] - * } zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } zip2 v0.2s, - * v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * If compiling Thumb-1 for a target which supports ARM instructions, we + * will emit a warning, as it is not a "sane" platform to compile for. * - * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 - * (SHRN): + * Usually, if this happens, it is because of an accident and you probably + * need to specify -march, as you likely meant to compile for a newer + * architecture. * - * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); - * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); - * - * This is available on ARMv7-A, but is less efficient than a single - * VZIP.32. + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 */ + #if defined(__thumb__) && !defined(__thumb2__) && \ + defined(__ARM_ARCH_ISA_ARM) + #warning "XXH3 is highly inefficient without ARM or Thumb-2." + #endif + + /* ========================================== + * Vectorization detection + * ========================================== */ + + #ifdef XXH_DOXYGEN + /*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the + * best implementation. + */ + #define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * @ref XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + +}; + + /*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment reqired for said + * vector type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ + #define XXH_ACC_ALIGN 8 + #endif + + /* Actual definition */ + #ifndef XXH_DOXYGEN + #define XXH_SCALAR 0 + #define XXH_SSE2 1 + #define XXH_AVX2 2 + #define XXH_AVX512 3 + #define XXH_NEON 4 + #define XXH_VSX 5 + #endif + + #ifndef XXH_VECTOR /* can be defined on command line */ + #if defined(__AVX512F__) + #define XXH_VECTOR XXH_AVX512 + #elif defined(__AVX2__) + #define XXH_VECTOR XXH_AVX2 + #elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) + #define XXH_VECTOR XXH_SSE2 + #elif defined(__GNUC__) /* msvc support maybe later */ \ + && (defined(__ARM_NEON__) || defined(__ARM_NEON)) && \ + (defined( \ + __LITTLE_ENDIAN__) /* We only support little endian NEON */ \ + || (defined(__BYTE_ORDER__) && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + #define XXH_VECTOR XXH_NEON + #elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) || \ + (defined(__s390x__) && defined(__VEC__)) && \ + defined(__GNUC__) /* TODO: IBM XL */ + #define XXH_VECTOR XXH_VSX + #else + #define XXH_VECTOR XXH_SCALAR + #endif + #endif /* - * Function-like macro: - * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t - * &outHi) - * { - - * outLo = (uint32x2_t)(in & 0xFFFFFFFF); - * outHi = (uint32x2_t)(in >> 32); - * in = UNDEFINED; - * } + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. */ - #if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ - && defined(__GNUC__) && !defined(__aarch64__) && !defined(__arm64__) - #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ - do { \ - \ - /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, \ - * %f0 = upper D half */ \ - /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 \ - */ \ - /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 \ - */ \ - __asm__("vzip.32 %e0, %f0" : "+w"(in)); \ - (outLo) = vget_low_u32(vreinterpretq_u32_u64(in)); \ - (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ - \ - } while (0) + #ifndef XXH_ACC_ALIGN + #if defined(XXH_X86DISPATCH) + #define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ + #elif XXH_VECTOR == XXH_SCALAR /* scalar */ + #define XXH_ACC_ALIGN 8 + #elif XXH_VECTOR == XXH_SSE2 /* sse2 */ + #define XXH_ACC_ALIGN 16 + #elif XXH_VECTOR == XXH_AVX2 /* avx2 */ + #define XXH_ACC_ALIGN 32 + #elif XXH_VECTOR == XXH_NEON /* neon */ + #define XXH_ACC_ALIGN 16 + #elif XXH_VECTOR == XXH_VSX /* vsx */ + #define XXH_ACC_ALIGN 16 + #elif XXH_VECTOR == XXH_AVX512 /* avx512 */ + #define XXH_ACC_ALIGN 64 + #endif + #endif + #if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 || \ + XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 + #define XXH_SEC_ALIGN XXH_ACC_ALIGN #else - #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ - do { \ - \ - (outLo) = vmovn_u64(in); \ - (outHi) = vshrn_n_u64((in), 32); \ - \ - } while (0) + #define XXH_SEC_ALIGN 8 + #endif + /* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling + * resulting in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization + * which only applies to Sandy and Ivy Bridge... which don't even support + * AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use + * either -O2 -mavx2 -march=haswell or -O2 -mavx2 + * -mno-avx256-split-unaligned-load for decent performance, or to use + * Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC + * into -O2, but the other one we can't control without "failed to inline + * always inline function due to target mismatch" warnings. + */ + #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && \ + !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ + #pragma GCC push_options + #pragma GCC optimize("-O2") #endif - #endif /* XXH_VECTOR == XXH_NEON */ - /* - * VSX and Z Vector helpers. - * - * This is very messy, and any pull requests to clean this up are welcome. - * - * There are a lot of problems with supporting VSX and s390x, due to - * inconsistent intrinsics, spotty coverage, and multiple endiannesses. - */ - #if XXH_VECTOR == XXH_VSX - #if defined(__s390x__) - #include <s390intrin.h> - #else - /* gcc's altivec.h can have the unwanted consequence to unconditionally - * #define bool, vector, and pixel keywords, - * with bad consequences for programs already using these keywords for - * other purposes. The paragraph defining these macros is skipped when - * __APPLE_ALTIVEC__ is defined. - * __APPLE_ALTIVEC__ is _generally_ defined automatically by the - * compiler, but it seems that, in some cases, it isn't. Force the build - * macro to be defined, so that keywords are not altered. + #if XXH_VECTOR == XXH_NEON + /* + * NEON's setup for vmlal_u32 is a little more complicated than it is on + * SSE2, AVX2, and VSX. + * + * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an + * upcast. + * + * To do the same operation, the 128-bit 'Q' register needs to be split + * into two 64-bit 'D' registers, performing this operation:: + * + * [ a | b ] | + * '---------. .--------' | | x | + * | .---------' '--------. | + * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 + * ] + * + * Due to significant changes in aarch64, the fastest method for aarch64 + * is completely different than the fastest method for ARMv7-A. + * + * ARMv7-A treats D registers as unions overlaying Q registers, so + * modifying D11 will modify the high half of Q5. This is similar to how + * modifying AH will only affect bits 8-15 of AX on x86. + * + * VZIP takes two registers, and puts even lanes in one register and odd + * lanes in the other. + * + * On ARMv7-A, this strangely modifies both parameters in place instead + * of taking the usual 3-operand form. + * + * Therefore, if we want to do this, we can simply use a D-form VZIP.32 + * on the lower and upper halves of the Q register to end up with the + * high and low halves where we want - all in one instruction. + * + * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { + + * d10[1], d11[1] } + * + * Unfortunately we need inline assembly for this: Instructions + * modifying two registers at once is not possible in GCC or Clang's IR, + * and they have to create a copy. + * + * aarch64 requires a different approach. + * + * In order to make it easier to write a decent compiler for aarch64, + * many quirks were removed, such as conditional execution. + * + * NEON was also affected by this. + * + * aarch64 cannot access the high bits of a Q-form register, and writes + * to a D-form register zero the high bits, similar to how writes to + * W-form scalar registers (or DWORD registers on x86_64) work. + * + * The formerly free vget_high intrinsics now require a vext (with a few + * exceptions) + * + * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the + * equivalent of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to + * only modify one operand. + * + * The equivalent of the VZIP.32 on the lower and upper halves would be + * this mess: + * + * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], + * v0[1] } zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } zip2 + * v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * + * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 + * (SHRN): + * + * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); + * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); + * + * This is available on ARMv7-A, but is less efficient than a single + * VZIP.32. */ - #if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) - #define __APPLE_ALTIVEC__ - #endif - #include <altivec.h> - #endif -typedef __vector unsigned long long xxh_u64x2; -typedef __vector unsigned char xxh_u8x16; -typedef __vector unsigned xxh_u32x4; + /*! + * Function-like macro: + * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t + * &outHi) + * { - #ifndef XXH_VSX_BE - #if defined(__BIG_ENDIAN__) || \ - (defined(__BYTE_ORDER__) && \ - __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) - #define XXH_VSX_BE 1 - #elif defined(__VEC_ELEMENT_REG_ORDER__) && \ - __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ - #warning \ - "-maltivec=be is not recommended. Please use native endianness." - #define XXH_VSX_BE 1 + * outLo = (uint32x2_t)(in & 0xFFFFFFFF); + * outHi = (uint32x2_t)(in >> 32); + * in = UNDEFINED; + * } + */ + #if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ + && defined(__GNUC__) && !defined(__aarch64__) && \ + !defined(__arm64__) + #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + \ + /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, \ + * %f0 = upper D half */ \ + /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 \ + */ \ + /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 \ + */ \ + __asm__("vzip.32 %e0, %f0" : "+w"(in)); \ + (outLo) = vget_low_u32(vreinterpretq_u32_u64(in)); \ + (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ + \ + } while (0) #else - #define XXH_VSX_BE 0 + #define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + \ + (outLo) = vmovn_u64(in); \ + (outHi) = vshrn_n_u64((in), 32); \ + \ + } while (0) #endif - #endif /* !defined(XXH_VSX_BE) */ + #endif /* XXH_VECTOR == XXH_NEON */ - #if XXH_VSX_BE - /* A wrapper for POWER9's vec_revb. */ - #if defined(__POWER9_VECTOR__) || \ - (defined(__clang__) && defined(__s390x__)) - #define XXH_vec_revb vec_revb + /* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ + #if XXH_VECTOR == XXH_VSX + #if defined(__s390x__) + #include <s390intrin.h> #else + /* gcc's altivec.h can have the unwanted consequence to + * unconditionally #define bool, vector, and pixel keywords, with bad + * consequences for programs already using these keywords for other + * purposes. The paragraph defining these macros is skipped when + * __APPLE_ALTIVEC__ is defined. + * __APPLE_ALTIVEC__ is _generally_ defined automatically by the + * compiler, but it seems that, in some cases, it isn't. Force the + * build macro to be defined, so that keywords are not altered. + */ + #if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) + #define __APPLE_ALTIVEC__ + #endif + #include <altivec.h> + #endif + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + + #ifndef XXH_VSX_BE + #if defined(__BIG_ENDIAN__) || \ + (defined(__BYTE_ORDER__) && \ + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + #define XXH_VSX_BE 1 + #elif defined(__VEC_ELEMENT_REG_ORDER__) && \ + __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ + #warning \ + "-maltivec=be is not recommended. Please use native endianness." + #define XXH_VSX_BE 1 + #else + #define XXH_VSX_BE 0 + #endif + #endif /* !defined(XXH_VSX_BE) */ + + #if XXH_VSX_BE + #if defined(__POWER9_VECTOR__) || \ + (defined(__clang__) && defined(__s390x__)) + #define XXH_vec_revb vec_revb + #else +/*! + * A polyfill for POWER9's vec_revb(). + */ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) { xxh_u8x16 const vByteSwap = {0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, @@ -2834,40 +3458,40 @@ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) { } - #endif - #endif /* XXH_VSX_BE */ + #endif + #endif /* XXH_VSX_BE */ -/* - * Performs an unaligned load and byte swaps it on big endian. +/*! + * Performs an unaligned vector load and byte swaps it on big endian. */ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) { xxh_u64x2 ret; memcpy(&ret, ptr, sizeof(xxh_u64x2)); - #if XXH_VSX_BE + #if XXH_VSX_BE ret = XXH_vec_revb(ret); - #endif + #endif return ret; } - /* - * vec_mulo and vec_mule are very problematic intrinsics on PowerPC - * - * These intrinsics weren't added until GCC 8, despite existing for a - * while, and they are endian dependent. Also, their meaning swap - * depending on version. - * */ - #if defined(__s390x__) - /* s390x is always big endian, no issue on this platform */ - #define XXH_vec_mulo vec_mulo - #define XXH_vec_mule vec_mule - #elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) - /* Clang has a better way to control this, we can just use the builtin - * which doesn't swap. */ - #define XXH_vec_mulo __builtin_altivec_vmulouw - #define XXH_vec_mule __builtin_altivec_vmuleuw - #else + /* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a + * while, and they are endian dependent. Also, their meaning swap + * depending on version. + * */ + #if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ + #define XXH_vec_mulo vec_mulo + #define XXH_vec_mule vec_mule + #elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) + /* Clang has a better way to control this, we can just use the builtin + * which doesn't swap. */ + #define XXH_vec_mulo __builtin_altivec_vmulouw + #define XXH_vec_mule __builtin_altivec_vmuleuw + #else /* gcc needs inline assembly */ /* Adapted from * https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ @@ -2887,40 +3511,41 @@ XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) { } - #endif /* XXH_vec_mulo, XXH_vec_mule */ - #endif /* XXH_VECTOR == XXH_VSX */ + #endif /* XXH_vec_mulo, XXH_vec_mule */ + #endif /* XXH_VECTOR == XXH_VSX */ - /* prefetch - * can be disabled, by declaring XXH_NO_PREFETCH build macro */ - #if defined(XXH_NO_PREFETCH) - #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ - #else - #if defined(_MSC_VER) && \ - (defined(_M_X64) || \ - defined( \ - _M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ - #include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ - #define XXH_PREFETCH(ptr) _mm_prefetch((const char *)(ptr), _MM_HINT_T0) - #elif defined(__GNUC__) && \ - ((__GNUC__ >= 4) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1))) - #define XXH_PREFETCH(ptr) \ - __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) - #else + /* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ + #if defined(XXH_NO_PREFETCH) #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ - #endif - #endif /* XXH_NO_PREFETCH */ + #else + #if defined(_MSC_VER) && \ + (defined(_M_X64) || \ + defined( \ + _M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ + #include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ + #define XXH_PREFETCH(ptr) \ + _mm_prefetch((const char *)(ptr), _MM_HINT_T0) + #elif defined(__GNUC__) && \ + ((__GNUC__ >= 4) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1))) + #define XXH_PREFETCH(ptr) \ + __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) + #else + #define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ + #endif + #endif /* XXH_NO_PREFETCH */ - /* ========================================== - * XXH3 default settings - * ========================================== */ + /* ========================================== + * XXH3 default settings + * ========================================== */ - #define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + #define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ - #if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) - #error "default keyset is not large enough" - #endif + #if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) + #error "default keyset is not large enough" + #endif -/* Pseudorandom secret taken directly from FARSH */ +/*! Pseudorandom secret taken directly from FARSH. */ XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { @@ -2943,69 +3568,79 @@ static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { }; - #ifdef XXH_OLD_NAMES - #define kSecret XXH3_kSecret - #endif + #ifdef XXH_OLD_NAMES + #define kSecret XXH3_kSecret + #endif - /* - * Calculates a 32-bit to 64-bit long multiply. - * - * Wraps __emulu on MSVC x86 because it tends to call __allmul when it - * doesn't need to (but it shouldn't need to anyways, it is about 7 - * instructions to do a 64x64 multiply...). Since we know that this will - * _always_ emit MULL, we use that instead of the normal method. - * - * If you are compiling for platforms like Thumb-1 and don't have a better - * option, you may also want to write your own long multiply routine here. - * - * XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) - * { + #ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it + * doesn't need to (but it shouldn't need to anyways, it is about 7 instructions + * to do a 64x64 multiply...). Since we know that this will _always_ emit + * `MULL`, we use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better + * option, you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 XXH_mult32to64(xxh_u64 x, xxh_u64 y) { - * return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); - * } - */ - #if defined(_MSC_VER) && defined(_M_IX86) - #include <intrin.h> - #define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) - #else - /* - * Downcast + upcast is usually better than masking on older compilers - * like GCC 4.2 (especially 32-bit ones), all without affecting newer - * compilers. - * - * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both - * operands and perform a full 64x64 multiply -- entirely redundant on - * 32-bit. - */ - #define XXH_mult32to64(x, y) \ - ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) - #endif + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); -/* - * Calculates a 64->128-bit long multiply. +} + + #elif defined(_MSC_VER) && defined(_M_IX86) + #include <intrin.h> + #define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) + #else + /* + * Downcast + upcast is usually better than masking on older compilers + * like GCC 4.2 (especially 32-bit ones), all without affecting newer + * compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both + * operands and perform a full 64x64 multiply -- entirely redundant on + * 32-bit. + */ + #define XXH_mult32to64(x, y) \ + ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) + #endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. * - * Uses __uint128_t and _umul128 if available, otherwise uses a scalar version. + * @param lhs, rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. */ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { - /* - * GCC/Clang __uint128_t method. - * - * On most 64-bit targets, GCC and Clang define a __uint128_t type. - * This is usually the best way as it usually uses a native long 64-bit - * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. - * - * Usually. - * - * Despite being a 32-bit platform, Clang (and emscripten) define this type - * despite not having the arithmetic for it. This results in a laggy - * compiler builtin call which calculates a full 128-bit multiply. - * In that case it is best to use the portable one. - * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 - */ - #if defined(__GNUC__) && !defined(__wasm__) && \ - defined(__SIZEOF_INT128__) || \ - (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this + * type despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ + #if defined(__GNUC__) && !defined(__wasm__) && \ + defined(__SIZEOF_INT128__) || \ + (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; XXH128_hash_t r128; @@ -3013,19 +3648,19 @@ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { r128.high64 = (xxh_u64)(product >> 64); return r128; - /* - * MSVC for x64's _umul128 method. - * - * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 - * *HighProduct); - * - * This compiles to single operand MUL on x64. - */ - #elif defined(_M_X64) || defined(_M_IA64) + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 + * *HighProduct); + * + * This compiles to single operand MUL on x64. + */ + #elif defined(_M_X64) || defined(_M_IA64) - #ifndef _MSC_VER - #pragma intrinsic(_umul128) - #endif + #ifndef _MSC_VER + #pragma intrinsic(_umul128) + #endif xxh_u64 product_high; xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); XXH128_hash_t r128; @@ -3033,7 +3668,7 @@ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { r128.high64 = product_high; return r128; - #else + #else /* * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. * @@ -3093,16 +3728,20 @@ static XXH128_hash_t XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) { r128.low64 = lower; r128.high64 = upper; return r128; - #endif + #endif } -/* - * Does a 64-bit to 128-bit multiply, then XOR folds it. +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. * * The reason for the separate function is to prevent passing too many structs * around by value. This will hopefully inline the multiply, but we don't force * it. + * + * @param lhs, rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() */ static xxh_u64 XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) { @@ -3111,7 +3750,7 @@ static xxh_u64 XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) { } -/* Seems to produce slightly better code on GCC for some reason. */ +/*! Seems to produce slightly better code on GCC for some reason. */ XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) { XXH_ASSERT(0 <= shift && shift < 64); @@ -3216,7 +3855,7 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_4to8_64b(const xxh_u8 *input, size_t len, XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); - XXH_ASSERT(4 <= len && len < 8); + XXH_ASSERT(4 <= len && len <= 8); seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; { @@ -3239,7 +3878,7 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_9to16_64b(const xxh_u8 *input, XXH_ASSERT(input != NULL); XXH_ASSERT(secret != NULL); - XXH_ASSERT(8 <= len && len <= 16); + XXH_ASSERT(9 <= len && len <= 16); { xxh_u64 const bitflip1 = @@ -3306,11 +3945,10 @@ XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8 *XXH_RESTRICT input, const xxh_u8 *XXH_RESTRICT secret, xxh_u64 seed64) { - #if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ - && \ - !defined( \ - XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + #if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like \ + XXH32 hack */ /* * UGLY HACK: * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in @@ -3326,8 +3964,8 @@ XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8 *XXH_RESTRICT input, * GCC generates much better scalar code than Clang for the rest of XXH3, * which is why finding a more optimal codepath is an interest. */ - __asm__("" : "+r"(seed64)); - #endif + XXH_COMPILER_GUARD(seed64); + #endif { xxh_u64 const input_lo = XXH_readLE64(input); @@ -3381,7 +4019,7 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_len_17to128_64b( } - #define XXH3_MIDSIZE_MAX 240 + #define XXH3_MIDSIZE_MAX 240 XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( const xxh_u8 *XXH_RESTRICT input, size_t len, @@ -3391,8 +4029,8 @@ XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( (void)secretSize; XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); - #define XXH3_MIDSIZE_STARTOFFSET 3 - #define XXH3_MIDSIZE_LASTOFFSET 17 + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 { @@ -3407,31 +4045,31 @@ XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( acc = XXH3_avalanche(acc); XXH_ASSERT(nbRounds >= 8); - #if defined(__clang__) /* Clang */ \ - && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ - /* - * UGLY HACK: - * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. - * In everywhere else, it uses scalar code. - * - * For 64->128-bit multiplies, even if the NEON was 100% optimal, it - * would still be slower than UMAAL (see XXH_mult64to128). - * - * Unfortunately, Clang doesn't handle the long multiplies properly and - * converts them to the nonexistent "vmulq_u64" intrinsic, which is then - * scalarized into an ugly mess of VMOV.32 instructions. - * - * This mess is difficult to avoid without turning autovectorization - * off completely, but they are usually relatively minor and/or not - * worth it to fix. - * - * This loop is the easiest to fix, as unlike XXH32, this pragma - * _actually works_ because it is a loop vectorization instead of an - * SLP vectorization. - */ - #pragma clang loop vectorize(disable) - #endif + #if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) + #endif for (i = 8; i < nbRounds; i++) { acc += @@ -3450,17 +4088,17 @@ XXH_NO_INLINE XXH64_hash_t XXH3_len_129to240_64b( } - /* ======= Long Keys ======= */ + /* ======= Long Keys ======= */ - #define XXH_STRIPE_LEN 64 - #define XXH_SECRET_CONSUME_RATE \ - 8 /* nb of secret bytes consumed at each accumulation */ - #define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + #define XXH_STRIPE_LEN 64 + #define XXH_SECRET_CONSUME_RATE \ + 8 /* nb of secret bytes consumed at each accumulation */ + #define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) - #ifdef XXH_OLD_NAMES - #define STRIPE_LEN XXH_STRIPE_LEN - #define ACC_NB XXH_ACC_NB - #endif + #ifdef XXH_OLD_NAMES + #define STRIPE_LEN XXH_STRIPE_LEN + #define ACC_NB XXH_ACC_NB + #endif XXH_FORCE_INLINE void XXH_writeLE64(void *dst, xxh_u64 v64) { @@ -3469,56 +4107,58 @@ XXH_FORCE_INLINE void XXH_writeLE64(void *dst, xxh_u64 v64) { } - /* Several intrinsic functions below are supposed to accept __int64 as - * argument, as documented in - * https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . However, - * several environments do not define __int64 type, requiring a workaround. - */ - #if !defined(__VMS) && \ - (defined(__cplusplus) || (defined(__STDC_VERSION__) && \ - (__STDC_VERSION__ >= 199901L) /* C99 */)) + /* Several intrinsic functions below are supposed to accept __int64 as + * argument, as documented in + * https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ + #if !defined(__VMS) && \ + (defined(__cplusplus) || (defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 199901L) /* C99 */)) typedef int64_t xxh_i64; - #else + #else /* the following type must have a width of 64-bit */ typedef long long xxh_i64; - #endif + #endif - /* - * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the - * most optimized. - * - * It is a hardened version of UMAC, based off of FARSH's implementation. - * - * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD - * implementations, and it is ridiculously fast. - * - * We harden it by mixing the original input to the accumulators as well as - * the product. - * - * This means that in the (relatively likely) case of a multiply by zero, the - * original input is preserved. - * - * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve - * cross-pollination, as otherwise the upper and lower halves would be - * essentially independent. - * - * This doesn't matter on 64-bit hashes since they all get merged together in - * the end, so we skip the extra step. - * - * Both XXH3_64bits and XXH3_128bits use this subroutine. - */ + /* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the + * most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as + * the product. + * + * This means that in the (relatively likely) case of a multiply by zero, + * the original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together + * in the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ - #if (XXH_VECTOR == XXH_AVX512) || defined(XXH_X86DISPATCH) + #if (XXH_VECTOR == XXH_AVX512) || \ + (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) - #ifndef XXH_TARGET_AVX512 - #define XXH_TARGET_AVX512 /* disable attribute target */ - #endif + #ifndef XXH_TARGET_AVX512 + #define XXH_TARGET_AVX512 /* disable attribute target */ + #endif XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_accumulate_512_avx512( void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, const void *XXH_RESTRICT secret) { - XXH_ALIGN(64) __m512i *const xacc = (__m512i *)acc; + __m512i *const xacc = (__m512i *)acc; XXH_ASSERT((((size_t)acc) & 63) == 0); XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); @@ -3576,8 +4216,8 @@ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_scrambleAcc_avx512( XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); { - XXH_ALIGN(64) __m512i *const xacc = (__m512i *)acc; - const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + __m512i *const xacc = (__m512i *)acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); /* xacc[0] ^= (xacc[0] >> 47) */ __m512i const acc_vec = *xacc; @@ -3609,19 +4249,21 @@ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_initCustomSecret_avx512( int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); __m512i const seed = _mm512_mask_set1_epi64( - _mm512_set1_epi64((xxh_i64)seed64), 0xAA, -(xxh_i64)seed64); + _mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64)); - XXH_ALIGN(64) const __m512i *const src = (const __m512i *)XXH3_kSecret; - XXH_ALIGN(64) __m512i *const dest = (__m512i *)customSecret; - int i; + const __m512i *const src = (const __m512i *)((const void *)XXH3_kSecret); + __m512i *const dest = (__m512i *)customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); for (i = 0; i < nbRounds; ++i) { /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void - * const*', this will warn "discards ‘const’ qualifier". */ + * const*', this will warn "discards 'const' qualifier". */ union { - XXH_ALIGN(64) const __m512i *cp; - XXH_ALIGN(64) void *p; + const __m512i *cp; + void * p; } remote_const_void; @@ -3635,13 +4277,14 @@ XXH_FORCE_INLINE XXH_TARGET_AVX512 void XXH3_initCustomSecret_avx512( } - #endif + #endif - #if (XXH_VECTOR == XXH_AVX2) || defined(XXH_X86DISPATCH) + #if (XXH_VECTOR == XXH_AVX2) || \ + (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) - #ifndef XXH_TARGET_AVX2 - #define XXH_TARGET_AVX2 /* disable attribute target */ - #endif + #ifndef XXH_TARGET_AVX2 + #define XXH_TARGET_AVX2 /* disable attribute target */ + #endif XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_accumulate_512_avx2( void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, @@ -3650,7 +4293,7 @@ XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_accumulate_512_avx2( XXH_ASSERT((((size_t)acc) & 31) == 0); { - XXH_ALIGN(32) __m256i *const xacc = (__m256i *)acc; + __m256i *const xacc = (__m256i *)acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ @@ -3692,7 +4335,7 @@ XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_scrambleAcc_avx2( XXH_ASSERT((((size_t)acc) & 31) == 0); { - XXH_ALIGN(32) __m256i *const xacc = (__m256i *)acc; + __m256i *const xacc = (__m256i *)acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ const __m256i *const xsecret = (const __m256i *)secret; @@ -3732,24 +4375,23 @@ XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2( XXH_PREFETCH(customSecret); { - __m256i const seed = _mm256_set_epi64x(-(xxh_i64)seed64, (xxh_i64)seed64, - -(xxh_i64)seed64, (xxh_i64)seed64); + __m256i const seed = + _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, + (xxh_i64)(0U - seed64), (xxh_i64)seed64); - XXH_ALIGN(64) const __m256i *const src = (const __m256i *)XXH3_kSecret; - XXH_ALIGN(64) __m256i * dest = (__m256i *)customSecret; + const __m256i *const src = (const __m256i *)((const void *)XXH3_kSecret); + __m256i * dest = (__m256i *)customSecret; - #if defined(__GNUC__) || defined(__clang__) + #if defined(__GNUC__) || defined(__clang__) /* * On GCC & Clang, marking 'dest' as modified will cause the compiler: * - do not extract the secret from sse registers in the internal loop * - use less common registers, and avoid pushing these reg into stack - * The asm hack causes Clang to assume that XXH3_kSecretPtr aliases with - * customSecret, and on aarch64, this prevented LDP from merging two - * loads together for free. Putting the loads together before the stores - * properly generates LDP. */ - __asm__("" : "+r"(dest)); - #endif + XXH_COMPILER_GUARD(dest); + #endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); /* GCC -O2 need unroll loop manually */ dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src + 0), seed); @@ -3763,13 +4405,14 @@ XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2( } - #endif + #endif - #if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + /* x86dispatch always generates SSE2 */ + #if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) - #ifndef XXH_TARGET_SSE2 - #define XXH_TARGET_SSE2 /* disable attribute target */ - #endif + #ifndef XXH_TARGET_SSE2 + #define XXH_TARGET_SSE2 /* disable attribute target */ + #endif XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_accumulate_512_sse2( void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, @@ -3779,7 +4422,7 @@ XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_accumulate_512_sse2( XXH_ASSERT((((size_t)acc) & 15) == 0); { - XXH_ALIGN(16) __m128i *const xacc = (__m128i *)acc; + __m128i *const xacc = (__m128i *)acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ const __m128i *const xinput = (const __m128i *)input; @@ -3820,7 +4463,7 @@ XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_scrambleAcc_sse2( XXH_ASSERT((((size_t)acc) & 15) == 0); { - XXH_ALIGN(16) __m128i *const xacc = (__m128i *)acc; + __m128i *const xacc = (__m128i *)acc; /* Unaligned. This is mainly for pointer arithmetic, and because * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ const __m128i *const xsecret = (const __m128i *)secret; @@ -3859,30 +4502,34 @@ XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2( int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); - #if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 - // MSVC 32bit mode does not support _mm_set_epi64x before 2015 + #if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ XXH_ALIGN(16) - const xxh_i64 seed64x2[2] = {(xxh_i64)seed64, -(xxh_i64)seed64}; + const xxh_i64 seed64x2[2] = {(xxh_i64)seed64, (xxh_i64)(0U - seed64)}; __m128i const seed = _mm_load_si128((__m128i const *)seed64x2); - #else - __m128i const seed = _mm_set_epi64x(-(xxh_i64)seed64, (xxh_i64)seed64); - #endif + #else + __m128i const seed = + _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); + #endif int i; - XXH_ALIGN(64) const float *const src = (float const *)XXH3_kSecret; - XXH_ALIGN(XXH_SEC_ALIGN) __m128i *dest = (__m128i *)customSecret; - #if defined(__GNUC__) || defined(__clang__) + const void *const src16 = XXH3_kSecret; + __m128i * dst16 = (__m128i *)customSecret; + #if defined(__GNUC__) || defined(__clang__) /* * On GCC & Clang, marking 'dest' as modified will cause the compiler: * - do not extract the secret from sse registers in the internal loop * - use less common registers, and avoid pushing these reg into stack */ - __asm__("" : "+r"(dest)); - #endif + XXH_COMPILER_GUARD(dst16); + #endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); for (i = 0; i < nbRounds; ++i) { - dest[i] = _mm_add_epi64(_mm_castps_si128(_mm_load_ps(src + i * 4)), seed); + dst16[i] = + _mm_add_epi64(_mm_load_si128((const __m128i *)src16 + i), seed); } @@ -3890,9 +4537,9 @@ XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2( } - #endif + #endif - #if (XXH_VECTOR == XXH_NEON) + #if (XXH_VECTOR == XXH_NEON) XXH_FORCE_INLINE void XXH3_accumulate_512_neon( void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, @@ -3901,7 +4548,7 @@ XXH_FORCE_INLINE void XXH3_accumulate_512_neon( XXH_ASSERT((((size_t)acc) & 15) == 0); { - XXH_ALIGN(16) uint64x2_t *const xacc = (uint64x2_t *)acc; + uint64x2_t *const xacc = (uint64x2_t *)acc; /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ uint8_t const *const xinput = (const uint8_t *)input; @@ -3996,9 +4643,9 @@ XXH_FORCE_INLINE void XXH3_scrambleAcc_neon(void *XXH_RESTRICT acc, } - #endif + #endif - #if (XXH_VECTOR == XXH_VSX) + #if (XXH_VECTOR == XXH_VSX) XXH_FORCE_INLINE void XXH3_accumulate_512_vsx(void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, @@ -4025,12 +4672,12 @@ XXH_FORCE_INLINE void XXH3_accumulate_512_vsx(void *XXH_RESTRICT acc, xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); xacc[i] += product; - /* swap high and low halves */ - #ifdef __s390x__ + /* swap high and low halves */ + #ifdef __s390x__ xacc[i] += vec_permi(data_vec, data_vec, 2); - #else + #else xacc[i] += vec_xxpermdi(data_vec, data_vec, 2); - #endif + #endif } @@ -4075,7 +4722,7 @@ XXH_FORCE_INLINE void XXH3_scrambleAcc_vsx(void *XXH_RESTRICT acc, } - #endif + #endif /* scalar variants - universal */ @@ -4083,7 +4730,6 @@ XXH_FORCE_INLINE void XXH3_accumulate_512_scalar( void *XXH_RESTRICT acc, const void *XXH_RESTRICT input, const void *XXH_RESTRICT secret) { - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 *const xacc = (xxh_u64 *)acc; /* presumed aligned */ const xxh_u8 *const xinput = (const xxh_u8 *)input; /* no alignment restriction */ @@ -4105,7 +4751,6 @@ XXH_FORCE_INLINE void XXH3_accumulate_512_scalar( XXH_FORCE_INLINE void XXH3_scrambleAcc_scalar(void *XXH_RESTRICT acc, const void *XXH_RESTRICT secret) { - XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 *const xacc = (xxh_u64 *)acc; /* presumed aligned */ const xxh_u8 *const xsecret = (const xxh_u8 *)secret; /* no alignment restriction */ @@ -4135,7 +4780,7 @@ XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar( const xxh_u8 *kSecretPtr = XXH3_kSecret; XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); - #if defined(__clang__) && defined(__aarch64__) + #if defined(__clang__) && defined(__aarch64__) /* * UGLY HACK: * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are @@ -4164,8 +4809,8 @@ XXH_FORCE_INLINE void XXH3_initCustomSecret_scalar( * without hack: 2654.4 MB/s * with hack: 3202.9 MB/s */ - __asm__("" : "+r"(kSecretPtr)); - #endif + XXH_COMPILER_GUARD(kSecretPtr); + #endif /* * Note: in debug mode, this overrides the asm optimization * and Clang will emit MOVK chains again. @@ -4200,55 +4845,55 @@ typedef void (*XXH3_f_accumulate_512)(void *XXH_RESTRICT, const void *, typedef void (*XXH3_f_scrambleAcc)(void *XXH_RESTRICT, const void *); typedef void (*XXH3_f_initCustomSecret)(void *XXH_RESTRICT, xxh_u64); - #if (XXH_VECTOR == XXH_AVX512) + #if (XXH_VECTOR == XXH_AVX512) - #define XXH3_accumulate_512 XXH3_accumulate_512_avx512 - #define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 - #define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + #define XXH3_accumulate_512 XXH3_accumulate_512_avx512 + #define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 + #define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 - #elif (XXH_VECTOR == XXH_AVX2) + #elif (XXH_VECTOR == XXH_AVX2) - #define XXH3_accumulate_512 XXH3_accumulate_512_avx2 - #define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 - #define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + #define XXH3_accumulate_512 XXH3_accumulate_512_avx2 + #define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 + #define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 - #elif (XXH_VECTOR == XXH_SSE2) + #elif (XXH_VECTOR == XXH_SSE2) - #define XXH3_accumulate_512 XXH3_accumulate_512_sse2 - #define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 - #define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + #define XXH3_accumulate_512 XXH3_accumulate_512_sse2 + #define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 + #define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 - #elif (XXH_VECTOR == XXH_NEON) + #elif (XXH_VECTOR == XXH_NEON) - #define XXH3_accumulate_512 XXH3_accumulate_512_neon - #define XXH3_scrambleAcc XXH3_scrambleAcc_neon - #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #define XXH3_accumulate_512 XXH3_accumulate_512_neon + #define XXH3_scrambleAcc XXH3_scrambleAcc_neon + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - #elif (XXH_VECTOR == XXH_VSX) + #elif (XXH_VECTOR == XXH_VSX) - #define XXH3_accumulate_512 XXH3_accumulate_512_vsx - #define XXH3_scrambleAcc XXH3_scrambleAcc_vsx - #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #define XXH3_accumulate_512 XXH3_accumulate_512_vsx + #define XXH3_scrambleAcc XXH3_scrambleAcc_vsx + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - #else /* scalar */ + #else /* scalar */ - #define XXH3_accumulate_512 XXH3_accumulate_512_scalar - #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar - #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + #define XXH3_accumulate_512 XXH3_accumulate_512_scalar + #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar + #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar - #endif + #endif - #ifndef XXH_PREFETCH_DIST - #ifdef __clang__ - #define XXH_PREFETCH_DIST 320 - #else - #if (XXH_VECTOR == XXH_AVX512) - #define XXH_PREFETCH_DIST 512 + #ifndef XXH_PREFETCH_DIST + #ifdef __clang__ + #define XXH_PREFETCH_DIST 320 #else - #define XXH_PREFETCH_DIST 384 - #endif - #endif /* __clang__ */ - #endif /* XXH_PREFETCH_DIST */ + #if (XXH_VECTOR == XXH_AVX512) + #define XXH_PREFETCH_DIST 512 + #else + #define XXH_PREFETCH_DIST 384 + #endif + #endif /* __clang__ */ + #endif /* XXH_PREFETCH_DIST */ /* * XXH3_accumulate() @@ -4308,8 +4953,9 @@ XXH_FORCE_INLINE void XXH3_hashLong_internal_loop( { const xxh_u8 *const p = input + len - XXH_STRIPE_LEN; - #define XXH_SECRET_LASTACC_START \ - 7 /* not aligned on 8, last secret is different from acc & scrambler */ + #define XXH_SECRET_LASTACC_START \ + 7 /* not aligned on 8, last secret is different from acc & scrambler \ + */ f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); @@ -4337,10 +4983,10 @@ static XXH64_hash_t XXH3_mergeAccs(const xxh_u64 *XXH_RESTRICT acc, for (i = 0; i < 4; i++) { result64 += XXH3_mix2Accs(acc + 2 * i, secret + 16 * i); - #if defined(__clang__) /* Clang */ \ - && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ - && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ - && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + #if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ /* * UGLY HACK: * Prevent autovectorization on Clang ARMv7-a. Exact same problem as @@ -4349,8 +4995,8 @@ static XXH64_hash_t XXH3_mergeAccs(const xxh_u64 *XXH_RESTRICT acc, * without hack: 2063.7 MB/s * with hack: 2560.7 MB/s */ - __asm__("" : "+r"(result64)); - #endif + XXH_COMPILER_GUARD(result64); + #endif } @@ -4358,13 +5004,13 @@ static XXH64_hash_t XXH3_mergeAccs(const xxh_u64 *XXH_RESTRICT acc, } - #define XXH3_INIT_ACC \ - { \ - \ - XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ - XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 \ - \ - } + #define XXH3_INIT_ACC \ + { \ + \ + XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 \ + \ + } XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_internal( const void *XXH_RESTRICT input, size_t len, const void *XXH_RESTRICT secret, @@ -4379,9 +5025,9 @@ XXH_FORCE_INLINE XXH64_hash_t XXH3_hashLong_64b_internal( /* converge into final hash */ XXH_STATIC_ASSERT(sizeof(acc) == 64); - /* do not align on 8, so that the secret is different from the accumulator - */ - #define XXH_SECRET_MERGEACCS_START 11 + /* do not align on 8, so that the secret is different from the accumulator + */ + #define XXH_SECRET_MERGEACCS_START 11 XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); return XXH3_mergeAccs(acc, (const xxh_u8 *)secret + XXH_SECRET_MERGEACCS_START, @@ -4501,6 +5147,7 @@ XXH3_64bits_internal(const void *XXH_RESTRICT input, size_t len, /* === Public entry point === */ +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void *input, size_t len) { return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), @@ -4508,6 +5155,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void *input, size_t len) { } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *input, size_t len, const void *secret, @@ -4518,6 +5166,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void *input, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void *input, size_t len, XXH64_hash_t seed) { @@ -4603,6 +5252,7 @@ static void XXH_alignedFree(void *p) { } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH3_state_t *XXH3_createState(void) { XXH3_state_t *const state = @@ -4613,6 +5263,7 @@ XXH_PUBLIC_API XXH3_state_t *XXH3_createState(void) { } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr) { XXH_alignedFree(statePtr); @@ -4620,6 +5271,7 @@ XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t *statePtr) { } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t * dst_state, const XXH3_state_t *src_state) { @@ -4627,9 +5279,8 @@ XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t * dst_state, } -static void XXH3_64bits_reset_internal(XXH3_state_t *statePtr, - XXH64_hash_t seed, const void *secret, - size_t secretSize) { +static void XXH3_reset_internal(XXH3_state_t *statePtr, XXH64_hash_t seed, + const void *secret, size_t secretSize) { size_t const initStart = offsetof(XXH3_state_t, bufferedSize); size_t const initLength = @@ -4654,26 +5305,28 @@ static void XXH3_64bits_reset_internal(XXH3_state_t *statePtr, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t *statePtr) { if (statePtr == NULL) return XXH_ERROR; - XXH3_64bits_reset_internal(statePtr, 0, XXH3_kSecret, - XXH_SECRET_DEFAULT_SIZE); + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret( XXH3_state_t *statePtr, const void *secret, size_t secretSize) { if (statePtr == NULL) return XXH_ERROR; - XXH3_64bits_reset_internal(statePtr, 0, secret, secretSize); + XXH3_reset_internal(statePtr, 0, secret, secretSize); if (secret == NULL) return XXH_ERROR; if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; return XXH_OK; } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t *statePtr, XXH64_hash_t seed) { @@ -4681,7 +5334,7 @@ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t *statePtr, if (seed == 0) return XXH3_64bits_reset(statePtr); if (seed != statePtr->seed) XXH3_initCustomSecret(statePtr->customSecret, seed); - XXH3_64bits_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } @@ -4733,12 +5386,12 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, XXH3_f_scrambleAcc f_scramble) { if (input == NULL) - #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ - (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) + #if defined(XXH_ACCEPT_NULL_INPUT_POINTER) && \ + (XXH_ACCEPT_NULL_INPUT_POINTER >= 1) return XXH_OK; - #else + #else return XXH_ERROR; - #endif + #endif { @@ -4747,6 +5400,7 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, (state->extSecret == NULL) ? state->customSecret : state->extSecret; state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) { /* fill in tmp buffer */ @@ -4756,10 +5410,10 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, } - /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ - #define XXH3_INTERNALBUFFER_STRIPES \ - (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + #define XXH3_INTERNALBUFFER_STRIPES \ + (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ @@ -4783,7 +5437,7 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, XXH_ASSERT(input < bEnd); /* Consume input by a multiple of internal buffer size */ - if (input + XXH3_INTERNALBUFFER_SIZE < bEnd) { + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { const xxh_u8 *const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; do { @@ -4814,6 +5468,7 @@ XXH_FORCE_INLINE XXH_errorcode XXH3_update(XXH3_state_t *state, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update(XXH3_state_t *state, const void *input, size_t len) { @@ -4859,6 +5514,7 @@ XXH_FORCE_INLINE void XXH3_digest_long(XXH64_hash_t * acc, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *state) { const unsigned char *const secret = @@ -4881,8 +5537,9 @@ XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *state) { } - #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) +/*! @ingroup xxh3_family */ XXH_PUBLIC_API void XXH3_generateSecret(void * secretBuffer, const void *customSeed, size_t customSeedSize) { @@ -5398,6 +6055,7 @@ XXH3_128bits_internal(const void *input, size_t len, XXH64_hash_t seed64, /* === Public XXH128 API === */ +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void *input, size_t len) { return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, @@ -5406,6 +6064,7 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void *input, size_t len) { } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void *input, size_t len, const void *secret, @@ -5416,6 +6075,7 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void *input, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void * input, size_t len, XXH64_hash_t seed) { @@ -5426,6 +6086,7 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void * input, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *input, size_t len, XXH64_hash_t seed) { @@ -5437,37 +6098,31 @@ XXH_PUBLIC_API XXH128_hash_t XXH128(const void *input, size_t len, /* * All the functions are actually the same as for 64-bit streaming variant. - * The only difference is the finalizatiom routine. + * The only difference is the finalization routine. */ -static void XXH3_128bits_reset_internal(XXH3_state_t *statePtr, - XXH64_hash_t seed, const void *secret, - size_t secretSize) { - - XXH3_64bits_reset_internal(statePtr, seed, secret, secretSize); - -} - +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t *statePtr) { if (statePtr == NULL) return XXH_ERROR; - XXH3_128bits_reset_internal(statePtr, 0, XXH3_kSecret, - XXH_SECRET_DEFAULT_SIZE); + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret( XXH3_state_t *statePtr, const void *secret, size_t secretSize) { if (statePtr == NULL) return XXH_ERROR; - XXH3_128bits_reset_internal(statePtr, 0, secret, secretSize); + XXH3_reset_internal(statePtr, 0, secret, secretSize); if (secret == NULL) return XXH_ERROR; if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; return XXH_OK; } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t *statePtr, XXH64_hash_t seed) { @@ -5475,11 +6130,12 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t *statePtr, if (seed == 0) return XXH3_128bits_reset(statePtr); if (seed != statePtr->seed) XXH3_initCustomSecret(statePtr->customSecret, seed); - XXH3_128bits_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); return XXH_OK; } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update(XXH3_state_t *state, const void * input, size_t len) { @@ -5489,6 +6145,7 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update(XXH3_state_t *state, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest(const XXH3_state_t *state) { const unsigned char *const secret = @@ -5524,11 +6181,12 @@ XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest(const XXH3_state_t *state) { } - /* 128-bit utility functions */ + /* 128-bit utility functions */ - #include <string.h> /* memcmp, memcpy */ + #include <string.h> /* memcmp, memcpy */ /* return : 1 is equal, 0 if different */ +/*! @ingroup xxh3_family */ XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) { /* note : XXH128_hash_t is compact, it has no padding byte */ @@ -5540,6 +6198,7 @@ XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) { * return : >0 if *h128_1 > *h128_2 * <0 if *h128_1 < *h128_2 * =0 if *h128_1 == *h128_2 */ +/*! @ingroup xxh3_family */ XXH_PUBLIC_API int XXH128_cmp(const void *h128_1, const void *h128_2) { XXH128_hash_t const h1 = *(const XXH128_hash_t *)h128_1; @@ -5552,6 +6211,7 @@ XXH_PUBLIC_API int XXH128_cmp(const void *h128_1, const void *h128_2) { } /*====== Canonical representation ======*/ +/*! @ingroup xxh3_family */ XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t *dst, XXH128_hash_t hash) { @@ -5568,6 +6228,7 @@ XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t *dst, } +/*! @ingroup xxh3_family */ XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t *src) { @@ -5578,16 +6239,21 @@ XXH128_hashFromCanonical(const XXH128_canonical_t *src) { } - /* Pop our optimization override from above */ - #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ - && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ - && defined(__OPTIMIZE__) && \ - !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ - #pragma GCC pop_options - #endif + /* Pop our optimization override from above */ + #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && \ + !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ + #pragma GCC pop_options + #endif - #endif /* XXH_NO_LONG_LONG */ + #endif /* XXH_NO_LONG_LONG */ + + #endif /* XXH_NO_XXH3 */ +/*! + * @} + */ #endif /* XXH_IMPLEMENTATION */ #if defined(__cplusplus) diff --git a/instrumentation/README.cmplog.md b/instrumentation/README.cmplog.md index a796c7a7..146b4620 100644 --- a/instrumentation/README.cmplog.md +++ b/instrumentation/README.cmplog.md @@ -1,11 +1,12 @@ # CmpLog instrumentation -The CmpLog instrumentation enables logging of comparison operands in a -shared memory. +The CmpLog instrumentation enables logging of comparison operands in a shared +memory. -These values can be used by various mutators built on top of it. -At the moment we support the RedQueen mutator (input-2-state instructions only), -for details see [the RedQueen paper](https://www.syssec.ruhr-uni-bochum.de/media/emma/veroeffentlichungen/2018/12/17/NDSS19-Redqueen.pdf). +These values can be used by various mutators built on top of it. At the moment, +we support the RedQueen mutator (input-2-state instructions only), for details +see +[the RedQueen paper](https://www.syssec.ruhr-uni-bochum.de/media/emma/veroeffentlichungen/2018/12/17/NDSS19-Redqueen.pdf). ## Build @@ -14,7 +15,8 @@ program. The first version is built using the regular AFL++ instrumentation. -The second one, the CmpLog binary, is built with setting AFL_LLVM_CMPLOG during the compilation. +The second one, the CmpLog binary, is built with setting AFL_LLVM_CMPLOG during +the compilation. For example: @@ -32,8 +34,8 @@ unset AFL_LLVM_CMPLOG ## Use -AFL++ has the new `-c` option that needs to be used to specify the CmpLog binary (the second -build). +AFL++ has the new `-c` option that needs to be used to specify the CmpLog binary +(the second build). For example: @@ -41,4 +43,4 @@ For example: afl-fuzz -i input -o output -c ./program.cmplog -m none -- ./program.afl @@ ``` -Be sure to use `-m none` because CmpLog can map a lot of pages. +Be sure to use `-m none` because CmpLog can map a lot of pages. \ No newline at end of file diff --git a/instrumentation/README.ctx.md b/instrumentation/README.ctx.md deleted file mode 100644 index 335e9921..00000000 --- a/instrumentation/README.ctx.md +++ /dev/null @@ -1,38 +0,0 @@ -# AFL Context Sensitive Branch Coverage - -## What is this? - -This is an LLVM-based implementation of the context sensitive branch coverage. - -Basically every function gets its own ID and, every time when an edge is logged, -all the IDs in the callstack are hashed and combined with the edge transition -hash to augment the classic edge coverage with the information about the -calling context. - -So if both function A and function B call a function C, the coverage -collected in C will be different. - -In math the coverage is collected as follows: -`map[current_location_ID ^ previous_location_ID >> 1 ^ hash_callstack_IDs] += 1` - -The callstack hash is produced XOR-ing the function IDs to avoid explosion with -recursive functions. - -## Usage - -Set the `AFL_LLVM_INSTRUMENT=CTX` or `AFL_LLVM_CTX=1` environment variable. - -It is highly recommended to increase the MAP_SIZE_POW2 definition in -config.h to at least 18 and maybe up to 20 for this as otherwise too -many map collisions occur. - -## Caller Branch Coverage - -If the context sensitive coverage introduces too may collisions and becoming -detrimental, the user can choose to augment edge coverage with just the -called function ID, instead of the entire callstack hash. - -In math the coverage is collected as follows: -`map[current_location_ID ^ previous_location_ID >> 1 ^ previous_callee_ID] += 1` - -Set the `AFL_LLVM_INSTRUMENT=CALLER` or `AFL_LLVM_CALLER=1` environment variable. diff --git a/instrumentation/README.gcc_plugin.md b/instrumentation/README.gcc_plugin.md index 230ceb73..ef38662b 100644 --- a/instrumentation/README.gcc_plugin.md +++ b/instrumentation/README.gcc_plugin.md @@ -1,64 +1,68 @@ # GCC-based instrumentation for afl-fuzz -See [../README.md](../README.md) for the general instruction manual. -See [README.llvm.md](README.llvm.md) for the LLVM-based instrumentation. +For the general instruction manual, see [../README.md](../README.md). For the +LLVM-based instrumentation, see [README.llvm.md](README.llvm.md). This document describes how to build and use `afl-gcc-fast` and `afl-g++-fast`, which instrument the target with the help of gcc plugins. -TLDR: - * check the version of your gcc compiler: `gcc --version` - * `apt-get install gcc-VERSION-plugin-dev` or similar to install headers for gcc plugins - * `gcc` and `g++` must match the gcc-VERSION you installed headers for. You can set `AFL_CC`/`AFL_CXX` - to point to these! - * `make` - * just use `afl-gcc-fast`/`afl-g++-fast` normally like you would do with `afl-clang-fast` +TL;DR: +* Check the version of your gcc compiler: `gcc --version` +* `apt-get install gcc-VERSION-plugin-dev` or similar to install headers for gcc + plugins. +* `gcc` and `g++` must match the gcc-VERSION you installed headers for. You can + set `AFL_CC`/`AFL_CXX` to point to these! +* `make` +* Just use `afl-gcc-fast`/`afl-g++-fast` normally like you would do with + `afl-clang-fast`. ## 1) Introduction -The code in this directory allows to instrument programs for AFL using -true compiler-level instrumentation, instead of the more crude -assembly-level rewriting approach taken by afl-gcc and afl-clang. This has -several interesting properties: +The code in this directory allows to instrument programs for AFL++ using true +compiler-level instrumentation, instead of the more crude assembly-level +rewriting approach taken by afl-gcc and afl-clang. This has several interesting +properties: - - The compiler can make many optimizations that are hard to pull off when - manually inserting assembly. As a result, some slow, CPU-bound programs will - run up to around faster. +- The compiler can make many optimizations that are hard to pull off when + manually inserting assembly. As a result, some slow, CPU-bound programs will + run up to around faster. - The gains are less pronounced for fast binaries, where the speed is limited - chiefly by the cost of creating new processes. In such cases, the gain will - probably stay within 10%. + The gains are less pronounced for fast binaries, where the speed is limited + chiefly by the cost of creating new processes. In such cases, the gain will + probably stay within 10%. - - The instrumentation is CPU-independent. At least in principle, you should - be able to rely on it to fuzz programs on non-x86 architectures (after - building `afl-fuzz` with `AFL_NOX86=1`). +- The instrumentation is CPU-independent. At least in principle, you should be + able to rely on it to fuzz programs on non-x86 architectures (after building + `afl-fuzz` with `AFL_NOX86=1`). - - Because the feature relies on the internals of GCC, it is gcc-specific - and will *not* work with LLVM (see [README.llvm.md](README.llvm.md) for an alternative). +- Because the feature relies on the internals of GCC, it is gcc-specific and + will *not* work with LLVM (see [README.llvm.md](README.llvm.md) for an + alternative). Once this implementation is shown to be sufficiently robust and portable, it -will probably replace afl-gcc. For now, it can be built separately and -co-exists with the original code. +will probably replace afl-gcc. For now, it can be built separately and co-exists +with the original code. The idea and much of the implementation comes from Laszlo Szekeres. ## 2) How to use -In order to leverage this mechanism, you need to have modern enough GCC -(>= version 4.5.0) and the plugin development headers installed on your system. That +In order to leverage this mechanism, you need to have modern enough GCC (>= +version 4.5.0) and the plugin development headers installed on your system. That should be all you need. On Debian machines, these headers can be acquired by installing the `gcc-VERSION-plugin-dev` packages. To build the instrumentation itself, type `make`. This will generate binaries -called `afl-gcc-fast` and `afl-g++-fast` in the parent directory. +called `afl-gcc-fast` and `afl-g++-fast` in the parent directory. -The gcc and g++ compiler links have to point to gcc-VERSION - or set these -by pointing the environment variables `AFL_CC`/`AFL_CXX` to them. -If the `CC`/`CXX` environment variables have been set, those compilers will be -preferred over those from the `AFL_CC`/`AFL_CXX` settings. +The gcc and g++ compiler links have to point to gcc-VERSION - or set these by +pointing the environment variables `AFL_CC`/`AFL_CXX` to them. If the `CC`/`CXX` +environment variables have been set, those compilers will be preferred over +those from the `AFL_CC`/`AFL_CXX` settings. Once this is done, you can instrument third-party code in a way similar to the -standard operating mode of AFL, e.g.: +standard operating mode of AFL++, e.g.: + ``` CC=/path/to/afl/afl-gcc-fast CXX=/path/to/afl/afl-g++-fast @@ -66,15 +70,15 @@ standard operating mode of AFL, e.g.: ./configure [...options...] make ``` + Note: We also used `CXX` to set the C++ compiler to `afl-g++-fast` for C++ code. The tool honors roughly the same environmental variables as `afl-gcc` (see -[env_variables.md](../docs/env_variables.md). This includes `AFL_INST_RATIO`, -`AFL_USE_ASAN`, `AFL_HARDEN`, and `AFL_DONT_OPTIMIZE`. +[docs/env_variables.md](../docs/env_variables.md). This includes +`AFL_INST_RATIO`, `AFL_USE_ASAN`, `AFL_HARDEN`, and `AFL_DONT_OPTIMIZE`. -Note: if you want the GCC plugin to be installed on your system for all -users, you need to build it before issuing 'make install' in the parent -directory. +Note: if you want the GCC plugin to be installed on your system for all users, +you need to build it before issuing 'make install' in the parent directory. ## 3) Gotchas, feedback, bugs @@ -83,93 +87,15 @@ reports to afl@aflplus.plus. ## 4) Bonus feature #1: deferred initialization -AFL tries to optimize performance by executing the targeted binary just once, -stopping it just before main(), and then cloning this "main" process to get -a steady supply of targets to fuzz. - -Although this approach eliminates much of the OS-, linker- and libc-level -costs of executing the program, it does not always help with binaries that -perform other time-consuming initialization steps - say, parsing a large config -file before getting to the fuzzed data. - -In such cases, it's beneficial to initialize the forkserver a bit later, once -most of the initialization work is already done, but before the binary attempts -to read the fuzzed input and parse it; in some cases, this can offer a 10x+ -performance gain. You can implement delayed initialization in GCC mode in a -fairly simple way. - -First, locate a suitable location in the code where the delayed cloning can -take place. This needs to be done with *extreme* care to avoid breaking the -binary. In particular, the program will probably malfunction if you select -a location after: - - - The creation of any vital threads or child processes - since the forkserver - can't clone them easily. - - - The initialization of timers via setitimer() or equivalent calls. - - - The creation of temporary files, network sockets, offset-sensitive file - descriptors, and similar shared-state resources - but only provided that - their state meaningfully influences the behavior of the program later on. - - - Any access to the fuzzed input, including reading the metadata about its - size. - -With the location selected, add this code in the appropriate spot: - -``` -#ifdef __AFL_HAVE_MANUAL_CONTROL - __AFL_INIT(); -#endif -``` - -You don't need the #ifdef guards, but they will make the program still work as -usual when compiled with a compiler other than afl-gcc-fast/afl-clang-fast. - -Finally, recompile the program with afl-gcc-fast (afl-gcc or afl-clang will -*not* generate a deferred-initialization binary) - and you should be all set! +See +[README.persistent_mode.md#3) Deferred initialization](README.persistent_mode.md#3-deferred-initialization). ## 5) Bonus feature #2: persistent mode -Some libraries provide APIs that are stateless, or whose state can be reset in -between processing different input files. When such a reset is performed, a -single long-lived process can be reused to try out multiple test cases, -eliminating the need for repeated `fork()` calls and the associated OS overhead. - -The basic structure of the program that does this would be: - -``` - while (__AFL_LOOP(1000)) { - - /* Read input data. */ - /* Call library code to be fuzzed. */ - /* Reset state. */ - - } - - /* Exit normally */ -``` - -The numerical value specified within the loop controls the maximum number -of iterations before AFL will restart the process from scratch. This minimizes -the impact of memory leaks and similar glitches; 1000 is a good starting point. - -A more detailed template is shown in ../utils/persistent_mode/. -Similarly to the previous mode, the feature works only with afl-gcc-fast or -afl-clang-fast; #ifdef guards can be used to suppress it when using other -compilers. - -Note that as with the previous mode, the feature is easy to misuse; if you -do not reset the critical state fully, you may end up with false positives or -waste a whole lot of CPU power doing nothing useful at all. Be particularly -wary of memory leaks and the state of file descriptors. - -When running in this mode, the execution paths will inherently vary a bit -depending on whether the input loop is being entered for the first time or -executed again. To avoid spurious warnings, the feature implies -`AFL_NO_VAR_CHECK` and hides the "variable path" warnings in the UI. +See +[README.persistent_mode.md#4) Persistent mode](README.persistent_mode.md#4-persistent-mode). ## 6) Bonus feature #3: selective instrumentation -It can be more effective to fuzzing to only instrument parts of the code. -For details see [README.instrument_list.md](README.instrument_list.md). +It can be more effective to fuzzing to only instrument parts of the code. For +details, see [README.instrument_list.md](README.instrument_list.md). \ No newline at end of file diff --git a/instrumentation/README.instrument_list.md b/instrumentation/README.instrument_list.md index 7db9c055..3ed64807 100644 --- a/instrumentation/README.instrument_list.md +++ b/instrumentation/README.instrument_list.md @@ -1,80 +1,84 @@ # Using AFL++ with partial instrumentation - This file describes two different mechanisms to selectively instrument - only specific parts in the target. +This file describes two different mechanisms to selectively instrument only +specific parts in the target. - Both mechanisms work for LLVM and GCC_PLUGIN, but not for afl-clang/afl-gcc. +Both mechanisms work for LLVM and GCC_PLUGIN, but not for afl-clang/afl-gcc. ## 1) Description and purpose When building and testing complex programs where only a part of the program is -the fuzzing target, it often helps to only instrument the necessary parts of -the program, leaving the rest uninstrumented. This helps to focus the fuzzer -on the important parts of the program, avoiding undesired noise and -disturbance by uninteresting code being exercised. +the fuzzing target, it often helps to only instrument the necessary parts of the +program, leaving the rest uninstrumented. This helps to focus the fuzzer on the +important parts of the program, avoiding undesired noise and disturbance by +uninteresting code being exercised. For this purpose, "partial instrumentation" support is provided by AFL++ that allows to specify what should be instrumented and what not. -Both mechanisms can be used together. +Both mechanisms for partial instrumentation can be used together. ## 2) Selective instrumentation with __AFL_COVERAGE_... directives -In this mechanism the selective instrumentation is done in the source code. +In this mechanism, the selective instrumentation is done in the source code. -After the includes a special define has to be made, eg.: +After the includes, a special define has to be made, e.g.: ``` #include <stdio.h> #include <stdint.h> // ... - + __AFL_COVERAGE(); // <- required for this feature to work ``` -If you want to disable the coverage at startup until you specify coverage -should be started, then add `__AFL_COVERAGE_START_OFF();` at that position. +If you want to disable the coverage at startup until you specify coverage should +be started, then add `__AFL_COVERAGE_START_OFF();` at that position. -From here on out you have the following macros available that you can use -in any function where you want: +From here on out, you have the following macros available that you can use in +any function where you want: - * `__AFL_COVERAGE_ON();` - enable coverage from this point onwards - * `__AFL_COVERAGE_OFF();` - disable coverage from this point onwards - * `__AFL_COVERAGE_DISCARD();` - reset all coverage gathered until this point - * `__AFL_COVERAGE_SKIP();` - mark this test case as unimportant. Whatever happens, afl-fuzz will ignore it. +* `__AFL_COVERAGE_ON();` - Enable coverage from this point onwards. +* `__AFL_COVERAGE_OFF();` - Disable coverage from this point onwards. +* `__AFL_COVERAGE_DISCARD();` - Reset all coverage gathered until this point. +* `__AFL_COVERAGE_SKIP();` - Mark this test case as unimportant. Whatever + happens, afl-fuzz will ignore it. -A special function is `__afl_coverage_interesting`. -To use this, you must define `void __afl_coverage_interesting(u8 val, u32 id);`. -Then you can use this function globally, where the `val` parameter can be set -by you, the `id` parameter is for afl-fuzz and will be overwritten. -Note that useful parameters for `val` are: 1, 2, 3, 4, 8, 16, 32, 64, 128. -A value of e.g. 33 will be seen as 32 for coverage purposes. +A special function is `__afl_coverage_interesting`. To use this, you must define +`void __afl_coverage_interesting(u8 val, u32 id);`. Then you can use this +function globally, where the `val` parameter can be set by you, the `id` +parameter is for afl-fuzz and will be overwritten. Note that useful parameters +for `val` are: 1, 2, 3, 4, 8, 16, 32, 64, 128. A value of, e.g., 33 will be seen +as 32 for coverage purposes. ## 3) Selective instrumentation with AFL_LLVM_ALLOWLIST/AFL_LLVM_DENYLIST -This feature is equivalent to llvm 12 sancov feature and allows to specify -on a filename and/or function name level to instrument these or skip them. +This feature is equivalent to llvm 12 sancov feature and allows to specify on a +filename and/or function name level to instrument these or skip them. ### 3a) How to use the partial instrumentation mode In order to build with partial instrumentation, you need to build with -afl-clang-fast/afl-clang-fast++ or afl-clang-lto/afl-clang-lto++. -The only required change is that you need to set either the environment variable -AFL_LLVM_ALLOWLIST or AFL_LLVM_DENYLIST set with a filename. +afl-clang-fast/afl-clang-fast++ or afl-clang-lto/afl-clang-lto++. The only +required change is that you need to set either the environment variable +`AFL_LLVM_ALLOWLIST` or `AFL_LLVM_DENYLIST` set with a filename. That file should contain the file names or functions that are to be instrumented -(AFL_LLVM_ALLOWLIST) or are specifically NOT to be instrumented (AFL_LLVM_DENYLIST). +(`AFL_LLVM_ALLOWLIST`) or are specifically NOT to be instrumented +(`AFL_LLVM_DENYLIST`). + +GCC_PLUGIN: you can use either `AFL_LLVM_ALLOWLIST` or `AFL_GCC_ALLOWLIST` (or +the same for `_DENYLIST`), both work. -GCC_PLUGIN: you can use either AFL_LLVM_ALLOWLIST or AFL_GCC_ALLOWLIST (or the -same for _DENYLIST), both work. +For matching to succeed, the function/file name that is being compiled must end +in the function/file name entry contained in this instrument file list. That is +to avoid breaking the match when absolute paths are used during compilation. -For matching to succeed, the function/file name that is being compiled must end in the -function/file name entry contained in this instrument file list. That is to avoid -breaking the match when absolute paths are used during compilation. +**NOTE:** In builds with optimization enabled, functions might be inlined and +would not match! -**NOTE:** In builds with optimization enabled, functions might be inlined and would not match! +For example, if your source tree looks like this: -For example if your source tree looks like this: ``` project/ project/feature_a/a1.cpp @@ -83,36 +87,45 @@ project/feature_b/b1.cpp project/feature_b/b2.cpp ``` -and you only want to test feature_a, then create an "instrument file list" file containing: +And you only want to test feature_a, then create an "instrument file list" file +containing: + ``` feature_a/a1.cpp feature_a/a2.cpp ``` -However if the "instrument file list" file contains only this, it works as well: +However, if the "instrument file list" file contains only this, it works as +well: + ``` a1.cpp a2.cpp ``` -but it might lead to files being unwantedly instrumented if the same filename + +But it might lead to files being unwantedly instrumented if the same filename exists somewhere else in the project directories. -You can also specify function names. Note that for C++ the function names -must be mangled to match! `nm` can print these names. +You can also specify function names. Note that for C++ the function names must +be mangled to match! `nm` can print these names. + +AFL++ is able to identify whether an entry is a filename or a function. However, +if you want to be sure (and compliant to the sancov allow/blocklist format), you +can specify source file entries like this: -AFL++ is able to identify whether an entry is a filename or a function. -However if you want to be sure (and compliant to the sancov allow/blocklist -format), you can specify source file entries like this: ``` src: *malloc.c ``` -and function entries like this: + +And function entries like this: + ``` fun: MallocFoo ``` + Note that whitespace is ignored and comments (`# foo`) are supported. ### 3b) UNIX-style pattern matching You can add UNIX-style pattern matching in the "instrument file list" entries. -See `man fnmatch` for the syntax. We do not set any of the `fnmatch` flags. +See `man fnmatch` for the syntax. Do not set any of the `fnmatch` flags. \ No newline at end of file diff --git a/instrumentation/README.laf-intel.md b/instrumentation/README.laf-intel.md index 229807e8..06e653ea 100644 --- a/instrumentation/README.laf-intel.md +++ b/instrumentation/README.laf-intel.md @@ -2,20 +2,17 @@ ## Introduction -This originally is the work of an individual nicknamed laf-intel. -His blog [Circumventing Fuzzing Roadblocks with Compiler Transformations] -(https://lafintel.wordpress.com/) and gitlab repo [laf-llvm-pass] -(https://gitlab.com/laf-intel/laf-llvm-pass/) -describe some code transformations that -help AFL++ to enter conditional blocks, where conditions consist of -comparisons of large values. +This originally is the work of an individual nicknamed laf-intel. His blog +[Circumventing Fuzzing Roadblocks with Compiler Transformations](https://lafintel.wordpress.com/) +and GitLab repo [laf-llvm-pass](https://gitlab.com/laf-intel/laf-llvm-pass/) +describe some code transformations that help AFL++ to enter conditional blocks, +where conditions consist of comparisons of large values. ## Usage -By default these passes will not run when you compile programs using -afl-clang-fast. Hence, you can use AFL as usual. -To enable the passes you must set environment variables before you -compile the target project. +By default, these passes will not run when you compile programs using +afl-clang-fast. Hence, you can use AFL++ as usual. To enable the passes, you +must set environment variables before you compile the target project. The following options exist: @@ -25,32 +22,30 @@ Enables the split-switches pass. `export AFL_LLVM_LAF_TRANSFORM_COMPARES=1` -Enables the transform-compares pass (strcmp, memcmp, strncmp, -strcasecmp, strncasecmp). +Enables the transform-compares pass (strcmp, memcmp, strncmp, strcasecmp, +strncasecmp). `export AFL_LLVM_LAF_SPLIT_COMPARES=1` -Enables the split-compares pass. -By default it will +Enables the split-compares pass. By default, it will 1. simplify operators >= (and <=) into chains of > (<) and == comparisons -2. change signed integer comparisons to a chain of sign-only comparison -and unsigned integer comparisons -3. split all unsigned integer comparisons with bit widths of -64, 32 or 16 bits to chains of 8 bits comparisons. - -You can change the behaviour of the last step by setting -`export AFL_LLVM_LAF_SPLIT_COMPARES_BITW=<bit_width>`, where -bit_width may be 64, 32 or 16. For example, a bit_width of 16 -would split larger comparisons down to 16 bit comparisons. - -A new experimental feature is splitting floating point comparisons into a -series of sign, exponent and mantissa comparisons followed by splitting each -of them into 8 bit comparisons when necessary. -It is activated with the `AFL_LLVM_LAF_SPLIT_FLOATS` setting. -Please note that full IEEE 754 functionality is not preserved, that is -values of nan and infinity will probably behave differently. - -Note that setting this automatically activates `AFL_LLVM_LAF_SPLIT_COMPARES` - -You can also set `AFL_LLVM_LAF_ALL` and have all of the above enabled :-) - +2. change signed integer comparisons to a chain of sign-only comparison and + unsigned integer comparisons +3. split all unsigned integer comparisons with bit widths of 64, 32, or 16 bits + to chains of 8 bits comparisons. + +You can change the behavior of the last step by setting `export +AFL_LLVM_LAF_SPLIT_COMPARES_BITW=<bit_width>`, where bit_width may be 64, 32, or +16. For example, a bit_width of 16 would split larger comparisons down to 16 bit +comparisons. + +A new experimental feature is splitting floating point comparisons into a series +of sign, exponent and mantissa comparisons followed by splitting each of them +into 8 bit comparisons when necessary. It is activated with the +`AFL_LLVM_LAF_SPLIT_FLOATS` setting. Note that full IEEE 754 functionality is +not preserved, that is values of nan and infinity will probably behave +differently. + +Note that setting this automatically activates `AFL_LLVM_LAF_SPLIT_COMPARES`. + +You can also set `AFL_LLVM_LAF_ALL` and have all of the above enabled. :-) \ No newline at end of file diff --git a/instrumentation/README.llvm.md b/instrumentation/README.llvm.md index 6e210a7c..d220e52c 100644 --- a/instrumentation/README.llvm.md +++ b/instrumentation/README.llvm.md @@ -1,72 +1,79 @@ # Fast LLVM-based instrumentation for afl-fuzz - (See [../README.md](../README.md) for the general instruction manual.) +For the general instruction manual, see [../README.md](../README.md). - (See [README.gcc_plugin.md](../README.gcc_plugin.md) for the GCC-based instrumentation.) +For the GCC-based instrumentation, see +[README.gcc_plugin.md](README.gcc_plugin.md). ## 1) Introduction ! llvm_mode works with llvm versions 3.8 up to 13 ! -The code in this directory allows you to instrument programs for AFL using -true compiler-level instrumentation, instead of the more crude -assembly-level rewriting approach taken by afl-gcc and afl-clang. This has -several interesting properties: +The code in this directory allows you to instrument programs for AFL++ using +true compiler-level instrumentation, instead of the more crude assembly-level +rewriting approach taken by afl-gcc and afl-clang. This has several interesting +properties: - - The compiler can make many optimizations that are hard to pull off when - manually inserting assembly. As a result, some slow, CPU-bound programs will - run up to around 2x faster. +- The compiler can make many optimizations that are hard to pull off when + manually inserting assembly. As a result, some slow, CPU-bound programs will + run up to around 2x faster. - The gains are less pronounced for fast binaries, where the speed is limited - chiefly by the cost of creating new processes. In such cases, the gain will - probably stay within 10%. + The gains are less pronounced for fast binaries, where the speed is limited + chiefly by the cost of creating new processes. In such cases, the gain will + probably stay within 10%. - - The instrumentation is CPU-independent. At least in principle, you should - be able to rely on it to fuzz programs on non-x86 architectures (after - building afl-fuzz with AFL_NO_X86=1). +- The instrumentation is CPU-independent. At least in principle, you should be + able to rely on it to fuzz programs on non-x86 architectures (after building + afl-fuzz with AFL_NO_X86=1). - - The instrumentation can cope a bit better with multi-threaded targets. +- The instrumentation can cope a bit better with multi-threaded targets. - - Because the feature relies on the internals of LLVM, it is clang-specific - and will *not* work with GCC (see ../gcc_plugin/ for an alternative once - it is available). +- Because the feature relies on the internals of LLVM, it is clang-specific and + will *not* work with GCC (see ../gcc_plugin/ for an alternative once it is + available). Once this implementation is shown to be sufficiently robust and portable, it will probably replace afl-clang. For now, it can be built separately and co-exists with the original code. -The idea and much of the intial implementation came from Laszlo Szekeres. +The idea and much of the initial implementation came from Laszlo Szekeres. ## 2a) How to use this - short -Set the `LLVM_CONFIG` variable to the clang version you want to use, e.g. +Set the `LLVM_CONFIG` variable to the clang version you want to use, e.g.: + ``` LLVM_CONFIG=llvm-config-9 make ``` + In case you have your own compiled llvm version specify the full path: + ``` LLVM_CONFIG=~/llvm-project/build/bin/llvm-config make ``` + If you try to use a new llvm version on an old Linux this can fail because of old c++ libraries. In this case usually switching to gcc/g++ to compile llvm_mode will work: + ``` LLVM_CONFIG=llvm-config-7 REAL_CC=gcc REAL_CXX=g++ make ``` -It is highly recommended to use the newest clang version you can put your -hands on :) + +It is highly recommended to use the newest clang version you can put your hands +on :) Then look at [README.persistent_mode.md](README.persistent_mode.md). ## 2b) How to use this - long In order to leverage this mechanism, you need to have clang installed on your -system. You should also make sure that the llvm-config tool is in your path -(or pointed to via LLVM_CONFIG in the environment). +system. You should also make sure that the llvm-config tool is in your path (or +pointed to via LLVM_CONFIG in the environment). -Note that if you have several LLVM versions installed, pointing LLVM_CONFIG -to the version you want to use will switch compiling to this specific -version - if you installation is set up correctly :-) +Note that if you have several LLVM versions installed, pointing LLVM_CONFIG to +the version you want to use will switch compiling to this specific version - if +you installation is set up correctly :-) Unfortunately, some systems that do have clang come without llvm-config or the LLVM development headers; one example of this is FreeBSD. FreeBSD users will @@ -75,15 +82,15 @@ load modules (you'll see "Service unavailable" when loading afl-llvm-pass.so). To solve all your problems, you can grab pre-built binaries for your OS from: - http://llvm.org/releases/download.html +[https://llvm.org/releases/download.html](https://llvm.org/releases/download.html) ...and then put the bin/ directory from the tarball at the beginning of your $PATH when compiling the feature and building packages later on. You don't need to be root for that. -To build the instrumentation itself, type 'make'. This will generate binaries -called afl-clang-fast and afl-clang-fast++ in the parent directory. Once this -is done, you can instrument third-party code in a way similar to the standard +To build the instrumentation itself, type `make`. This will generate binaries +called afl-clang-fast and afl-clang-fast++ in the parent directory. Once this is +done, you can instrument third-party code in a way similar to the standard operating mode of AFL, e.g.: ``` @@ -93,81 +100,179 @@ operating mode of AFL, e.g.: Be sure to also include CXX set to afl-clang-fast++ for C++ code. -Note that afl-clang-fast/afl-clang-fast++ are just pointers to afl-cc. -You can also use afl-cc/afl-c++ and instead direct it to use LLVM -instrumentation by either setting `AFL_CC_COMPILER=LLVM` or pass the parameter -`--afl-llvm` via CFLAGS/CXXFLAGS/CPPFLAGS. +Note that afl-clang-fast/afl-clang-fast++ are just pointers to afl-cc. You can +also use afl-cc/afl-c++ and instead direct it to use LLVM instrumentation by +either setting `AFL_CC_COMPILER=LLVM` or pass the parameter `--afl-llvm` via +CFLAGS/CXXFLAGS/CPPFLAGS. The tool honors roughly the same environmental variables as afl-gcc (see -[docs/env_variables.md](../docs/env_variables.md)). This includes AFL_USE_ASAN, -AFL_HARDEN, and AFL_DONT_OPTIMIZE. However AFL_INST_RATIO is not honored -as it does not serve a good purpose with the more effective PCGUARD analysis. +[docs/env_variables.md](../docs/env_variables.md)). This includes +`AFL_USE_ASAN`, `AFL_HARDEN`, and `AFL_DONT_OPTIMIZE`. However, `AFL_INST_RATIO` +is not honored as it does not serve a good purpose with the more effective +PCGUARD analysis. ## 3) Options -Several options are present to make llvm_mode faster or help it rearrange -the code to make afl-fuzz path discovery easier. +Several options are present to make llvm_mode faster or help it rearrange the +code to make afl-fuzz path discovery easier. -If you need just to instrument specific parts of the code, you can the instrument file list -which C/C++ files to actually instrument. See [README.instrument_list.md](README.instrument_list.md) +If you need just to instrument specific parts of the code, you can the +instrument file list which C/C++ files to actually instrument. See +[README.instrument_list.md](README.instrument_list.md) -For splitting memcmp, strncmp, etc. please see [README.laf-intel.md](README.laf-intel.md) +For splitting memcmp, strncmp, etc., see +[README.laf-intel.md](README.laf-intel.md). Then there are different ways of instrumenting the target: -1. An better instrumentation strategy uses LTO and link time -instrumentation. Note that not all targets can compile in this mode, however -if it works it is the best option you can use. -Simply use afl-clang-lto/afl-clang-lto++ to use this option. -See [README.lto.md](README.lto.md) +1. A better instrumentation strategy uses LTO and link time instrumentation. + Note that not all targets can compile in this mode, however, if it works it + is the best option you can use. To go with this option, use + afl-clang-lto/afl-clang-lto++. See [README.lto.md](README.lto.md). -2. Alternativly you can choose a completely different coverage method: +2. Alternatively you can choose a completely different coverage method: -2a. N-GRAM coverage - which combines the previous visited edges with the -current one. This explodes the map but on the other hand has proven to be -effective for fuzzing. -See [README.ngram.md](README.ngram.md) +2a. N-GRAM coverage - which combines the previous visited edges with the current + one. This explodes the map but on the other hand has proven to be effective + for fuzzing. See + [7) AFL++ N-Gram Branch Coverage](#7-afl-n-gram-branch-coverage). 2b. Context sensitive coverage - which combines the visited edges with an -individual caller ID (the function that called the current one) -[README.ctx.md](README.ctx.md) + individual caller ID (the function that called the current one). See + [6) AFL++ Context Sensitive Branch Coverage](#6-afl-context-sensitive-branch-coverage). -Then - additionally to one of the instrumentation options above - there is -a very effective new instrumentation option called CmpLog as an alternative to -laf-intel that allow AFL++ to apply mutations similar to Redqueen. -See [README.cmplog.md](README.cmplog.md) +Then - additionally to one of the instrumentation options above - there is a +very effective new instrumentation option called CmpLog as an alternative to +laf-intel that allow AFL++ to apply mutations similar to Redqueen. See +[README.cmplog.md](README.cmplog.md). -Finally if your llvm version is 8 or lower, you can activate a mode that -prevents that a counter overflow result in a 0 value. This is good for -path discovery, but the llvm implementation for x86 for this functionality -is not optimal and was only fixed in llvm 9. -You can set this with AFL_LLVM_NOT_ZERO=1 -See [README.neverzero.md](README.neverzero.md) +Finally, if your llvm version is 8 or lower, you can activate a mode that +prevents that a counter overflow result in a 0 value. This is good for path +discovery, but the llvm implementation for x86 for this functionality is not +optimal and was only fixed in llvm 9. You can set this with AFL_LLVM_NOT_ZERO=1. -Support for thread safe counters has been added for all modes. -Activate it with `AFL_LLVM_THREADSAFE_INST=1`. The tradeoff is better precision -in multi threaded apps for a slightly higher instrumentation overhead. -This also disables the nozero counter default for performance reasons. +Support for thread safe counters has been added for all modes. Activate it with +`AFL_LLVM_THREADSAFE_INST=1`. The tradeoff is better precision in multi threaded +apps for a slightly higher instrumentation overhead. This also disables the +nozero counter default for performance reasons. -## 4) Snapshot feature +## 4) deferred initialization, persistent mode, shared memory fuzzing -To speed up fuzzing you can use a linux loadable kernel module which enables -a snapshot feature. -See [README.snapshot.md](README.snapshot.md) +This is the most powerful and effective fuzzing you can do. For a full +explanation, see [README.persistent_mode.md](README.persistent_mode.md). -## 5) Gotchas, feedback, bugs +## 5) Bonus feature: 'dict2file' pass -This is an early-stage mechanism, so field reports are welcome. You can send bug -reports to <afl-users@googlegroups.com>. +Just specify `AFL_LLVM_DICT2FILE=/absolute/path/file.txt` and during compilation +all constant string compare parameters will be written to this file to be used +with afl-fuzz' `-x` option. -## 6) deferred initialization, persistent mode, shared memory fuzzing +## 6) AFL++ Context Sensitive Branch Coverage -This is the most powerful and effective fuzzing you can do. -Please see [README.persistent_mode.md](README.persistent_mode.md) for a -full explanation. +### What is this? -## 7) Bonus feature: 'dict2file' pass +This is an LLVM-based implementation of the context sensitive branch coverage. -Just specify `AFL_LLVM_DICT2FILE=/absolute/path/file.txt` and during compilation -all constant string compare parameters will be written to this file to be -used with afl-fuzz' `-x` option. +Basically every function gets its own ID and, every time when an edge is logged, +all the IDs in the callstack are hashed and combined with the edge transition +hash to augment the classic edge coverage with the information about the calling +context. + +So if both function A and function B call a function C, the coverage collected +in C will be different. + +In math the coverage is collected as follows: `map[current_location_ID ^ +previous_location_ID >> 1 ^ hash_callstack_IDs] += 1` + +The callstack hash is produced XOR-ing the function IDs to avoid explosion with +recursive functions. + +### Usage + +Set the `AFL_LLVM_INSTRUMENT=CTX` or `AFL_LLVM_CTX=1` environment variable. + +It is highly recommended to increase the MAP_SIZE_POW2 definition in config.h to +at least 18 and maybe up to 20 for this as otherwise too many map collisions +occur. + +### Caller Branch Coverage + +If the context sensitive coverage introduces too may collisions and becoming +detrimental, the user can choose to augment edge coverage with just the called +function ID, instead of the entire callstack hash. + +In math the coverage is collected as follows: `map[current_location_ID ^ +previous_location_ID >> 1 ^ previous_callee_ID] += 1` + +Set the `AFL_LLVM_INSTRUMENT=CALLER` or `AFL_LLVM_CALLER=1` environment +variable. + +## 7) AFL++ N-Gram Branch Coverage + +### Source + +This is an LLVM-based implementation of the n-gram branch coverage proposed in +the paper +["Be Sensitive and Collaborative: Analyzing Impact of Coverage Metrics in Greybox Fuzzing"](https://www.usenix.org/system/files/raid2019-wang-jinghan.pdf) +by Jinghan Wang, et. al. + +Note that the original implementation (available +[here](https://github.com/bitsecurerlab/afl-sensitive)) is built on top of AFL's +QEMU mode. This is essentially a port that uses LLVM vectorized instructions +(available from llvm versions 4.0.1 and higher) to achieve the same results when +compiling source code. + +In math the branch coverage is performed as follows: `map[current_location ^ +prev_location[0] >> 1 ^ prev_location[1] >> 1 ^ ... up to n-1`] += 1` + +### Usage + +The size of `n` (i.e., the number of branches to remember) is an option that is +specified either in the `AFL_LLVM_INSTRUMENT=NGRAM-{value}` or the +`AFL_LLVM_NGRAM_SIZE` environment variable. Good values are 2, 4, or 8, valid +are 2-16. + +It is highly recommended to increase the MAP_SIZE_POW2 definition in config.h to +at least 18 and maybe up to 20 for this as otherwise too many map collisions +occur. + +## 8) NeverZero counters + +In larger, complex, or reiterative programs, the byte sized counters that +collect the edge coverage can easily fill up and wrap around. This is not that +much of an issue - unless, by chance, it wraps just to a value of zero when the +program execution ends. In this case, afl-fuzz is not able to see that the edge +has been accessed and will ignore it. + +NeverZero prevents this behavior. If a counter wraps, it jumps over the value 0 +directly to a 1. This improves path discovery (by a very small amount) at a very +low cost (one instruction per edge). + +(The alternative of saturated counters has been tested also and proved to be +inferior in terms of path discovery.) + +This is implemented in afl-gcc and afl-gcc-fast, however, for llvm_mode this is +optional if multithread safe counters are selected or the llvm version is below +9 - as there are severe performance costs in these cases. + +If you want to enable this for llvm versions below 9 or thread safe counters, +then set + +``` +export AFL_LLVM_NOT_ZERO=1 +``` + +In case you are on llvm 9 or greater and you do not want this behavior, then you +can set: + +``` +AFL_LLVM_SKIP_NEVERZERO=1 +``` + +If the target does not have extensive loops or functions that are called a lot, +then this can give a small performance boost. + +Please note that the default counter implementations are not thread safe! + +Support for thread safe counters in mode LLVM CLASSIC can be activated with +setting `AFL_LLVM_THREADSAFE_INST=1`. \ No newline at end of file diff --git a/instrumentation/README.lto.md b/instrumentation/README.lto.md index 6174cdc0..a20175b1 100644 --- a/instrumentation/README.lto.md +++ b/instrumentation/README.lto.md @@ -1,55 +1,56 @@ # afl-clang-lto - collision free instrumentation at link time -## TLDR; +## TL;DR: -This version requires a current llvm 11+ compiled from the github master. +This version requires a current llvm 11+ compiled from the GitHub master. 1. Use afl-clang-lto/afl-clang-lto++ because it is faster and gives better - coverage than anything else that is out there in the AFL world + coverage than anything else that is out there in the AFL world. -2. You can use it together with llvm_mode: laf-intel and the instrument file listing - features and can be combined with cmplog/Redqueen +2. You can use it together with llvm_mode: laf-intel and the instrument file + listing features and can be combined with cmplog/Redqueen. -3. It only works with llvm 11+ +3. It only works with llvm 11+. -4. AUTODICTIONARY feature! see below +4. AUTODICTIONARY feature (see below)! -5. If any problems arise be sure to set `AR=llvm-ar RANLIB=llvm-ranlib`. - Some targets might need `LD=afl-clang-lto` and others `LD=afl-ld-lto`. +5. If any problems arise, be sure to set `AR=llvm-ar RANLIB=llvm-ranlib`. Some + targets might need `LD=afl-clang-lto` and others `LD=afl-ld-lto`. ## Introduction and problem description -A big issue with how AFL/AFL++ works is that the basic block IDs that are -set during compilation are random - and hence naturally the larger the number -of instrumented locations, the higher the number of edge collisions are in the -map. This can result in not discovering new paths and therefore degrade the +A big issue with how AFL++ works is that the basic block IDs that are set during +compilation are random - and hence naturally the larger the number of +instrumented locations, the higher the number of edge collisions are in the map. +This can result in not discovering new paths and therefore degrade the efficiency of the fuzzing process. -*This issue is underestimated in the fuzzing community!* -With a 2^16 = 64kb standard map at already 256 instrumented blocks there is -on average one collision. On average a target has 10.000 to 50.000 -instrumented blocks hence the real collisions are between 750-18.000! +*This issue is underestimated in the fuzzing community!* With a 2^16 = 64kb +standard map at already 256 instrumented blocks, there is on average one +collision. On average, a target has 10.000 to 50.000 instrumented blocks, hence +the real collisions are between 750-18.000! -To reach a solution that prevents any collisions took several approaches -and many dead ends until we got to this: +To reach a solution that prevents any collisions took several approaches and +many dead ends until we got to this: - * We instrument at link time when we have all files pre-compiled - * To instrument at link time we compile in LTO (link time optimization) mode - * Our compiler (afl-clang-lto/afl-clang-lto++) takes care of setting the - correct LTO options and runs our own afl-ld linker instead of the system - linker - * The LLVM linker collects all LTO files to link and instruments them so that - we have non-colliding edge overage - * We use a new (for afl) edge coverage - which is the same as in llvm - -fsanitize=coverage edge coverage mode :) +* We instrument at link time when we have all files pre-compiled. +* To instrument at link time, we compile in LTO (link time optimization) mode. +* Our compiler (afl-clang-lto/afl-clang-lto++) takes care of setting the correct + LTO options and runs our own afl-ld linker instead of the system linker. +* The LLVM linker collects all LTO files to link and instruments them so that we + have non-colliding edge overage. +* We use a new (for afl) edge coverage - which is the same as in llvm + -fsanitize=coverage edge coverage mode. :) The result: - * 10-25% speed gain compared to llvm_mode - * guaranteed non-colliding edge coverage :-) - * The compile time especially for binaries to an instrumented library can be - much longer + +* 10-25% speed gain compared to llvm_mode +* guaranteed non-colliding edge coverage :-) +* The compile time, especially for binaries to an instrumented library, can be + much longer. Example build output from a libtiff build: + ``` libtool: link: afl-clang-lto -g -O2 -Wall -W -o thumbnail thumbnail.o ../libtiff/.libs/libtiff.a ../port/.libs/libport.a -llzma -ljbig -ljpeg -lz -lm afl-clang-lto++2.63d by Marc "vanHauser" Heuse <mh@mh-sec.de> in mode LTO @@ -62,21 +63,24 @@ AUTODICTIONARY: 11 strings found ### Installing llvm version 11 or 12 -llvm 11 or even 12 should be available in all current Linux repositories. -If you use an outdated Linux distribution read the next section. +llvm 11 or even 12 should be available in all current Linux repositories. If you +use an outdated Linux distribution, read the next section. ### Installing llvm from the llvm repository (version 12+) Installing the llvm snapshot builds is easy and mostly painless: -In the follow line change `NAME` for your Debian or Ubuntu release name -(e.g. buster, focal, eon, etc.): +In the following line, change `NAME` for your Debian or Ubuntu release name +(e.g., buster, focal, eon, etc.): + ``` echo deb http://apt.llvm.org/NAME/ llvm-toolchain-NAME NAME >> /etc/apt/sources.list ``` -then add the pgp key of llvm and install the packages: + +Then add the pgp key of llvm and install the packages: + ``` -wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - +wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - apt-get update && apt-get upgrade -y apt-get install -y clang-12 clang-tools-12 libc++1-12 libc++-12-dev \ libc++abi1-12 libc++abi-12-dev libclang1-12 libclang-12-dev \ @@ -87,7 +91,8 @@ apt-get install -y clang-12 clang-tools-12 libc++1-12 libc++-12-dev \ ### Building llvm yourself (version 12+) -Building llvm from github takes quite some long time and is not painless: +Building llvm from GitHub takes quite some time and is not painless: + ```sh sudo apt install binutils-dev # this is *essential*! git clone --depth=1 https://github.com/llvm/llvm-project @@ -126,10 +131,12 @@ sudo make install Just use afl-clang-lto like you did with afl-clang-fast or afl-gcc. -Also the instrument file listing (AFL_LLVM_ALLOWLIST/AFL_LLVM_DENYLIST -> [README.instrument_list.md](README.instrument_list.md)) and -laf-intel/compcov (AFL_LLVM_LAF_* -> [README.laf-intel.md](README.laf-intel.md)) work. +Also, the instrument file listing (AFL_LLVM_ALLOWLIST/AFL_LLVM_DENYLIST -> +[README.instrument_list.md](README.instrument_list.md)) and laf-intel/compcov +(AFL_LLVM_LAF_* -> [README.laf-intel.md](README.laf-intel.md)) work. Example: + ``` CC=afl-clang-lto CXX=afl-clang-lto++ RANLIB=llvm-ranlib AR=llvm-ar ./configure make @@ -143,51 +150,48 @@ NOTE: some targets also need to set the linker, try both `afl-clang-lto` and Note: this is highly discouraged! Try to compile to static libraries with afl-clang-lto instead of shared libraries! -To make instrumented shared libraries work with afl-clang-lto you have to do +To make instrumented shared libraries work with afl-clang-lto, you have to do quite some extra steps. -Every shared library you want to instrument has to be individually compiled. -The environment variable `AFL_LLVM_LTO_DONTWRITEID=1` has to be set during -compilation. -Additionally the environment variable `AFL_LLVM_LTO_STARTID` has to be set to -the added edge count values of all previous compiled instrumented shared -libraries for that target. -E.g. for the first shared library this would be `AFL_LLVM_LTO_STARTID=0` and -afl-clang-lto will then report how many edges have been instrumented (let's say -it reported 1000 instrumented edges). -The second shared library then has to be set to that value +Every shared library you want to instrument has to be individually compiled. The +environment variable `AFL_LLVM_LTO_DONTWRITEID=1` has to be set during +compilation. Additionally, the environment variable `AFL_LLVM_LTO_STARTID` has +to be set to the added edge count values of all previous compiled instrumented +shared libraries for that target. E.g., for the first shared library this would +be `AFL_LLVM_LTO_STARTID=0` and afl-clang-lto will then report how many edges +have been instrumented (let's say it reported 1000 instrumented edges). The +second shared library then has to be set to that value (`AFL_LLVM_LTO_STARTID=1000` in our example), for the third to all previous counts added, etc. -The final program compilation step then may *not* have `AFL_LLVM_LTO_DONTWRITEID` -set, and `AFL_LLVM_LTO_STARTID` must be set to all edge counts added of all shared -libraries it will be linked to. +The final program compilation step then may *not* have +`AFL_LLVM_LTO_DONTWRITEID` set, and `AFL_LLVM_LTO_STARTID` must be set to all +edge counts added of all shared libraries it will be linked to. -This is quite some hands-on work, so better stay away from instrumenting -shared libraries :-) +This is quite some hands-on work, so better stay away from instrumenting shared +libraries. :-) ## AUTODICTIONARY feature While compiling, a dictionary based on string comparisons is automatically -generated and put into the target binary. This dictionary is transfered to afl-fuzz -on start. This improves coverage statistically by 5-10% :) +generated and put into the target binary. This dictionary is transferred to +afl-fuzz on start. This improves coverage statistically by 5-10%. :) -Note that if for any reason you do not want to use the autodictionary feature +Note that if for any reason you do not want to use the autodictionary feature, then just set the environment variable `AFL_NO_AUTODICT` when starting afl-fuzz. ## Fixed memory map To speed up fuzzing a little bit more, it is possible to set a fixed shared -memory map. -Recommended is the value 0x10000. +memory map. Recommended is the value 0x10000. -In most cases this will work without any problems. However if a target uses -early constructors, ifuncs or a deferred forkserver this can crash the target. +In most cases, this will work without any problems. However, if a target uses +early constructors, ifuncs, or a deferred forkserver, this can crash the target. -Also on unusual operating systems/processors/kernels or weird libraries the +Also, on unusual operating systems/processors/kernels or weird libraries the recommended 0x10000 address might not work, so then change the fixed address. -To enable this feature set AFL_LLVM_MAP_ADDR with the address. +To enable this feature, set `AFL_LLVM_MAP_ADDR` with the address. ## Document edge IDs @@ -198,7 +202,7 @@ bytes or which functions were touched by an input. ## Solving difficult targets Some targets are difficult because the configure script does unusual stuff that -is unexpected for afl. See the next chapter `Potential issues` for how to solve +is unexpected for afl. See the next section `Potential issues` for how to solve these. ### Example: ffmpeg @@ -206,143 +210,155 @@ these. An example of a hard to solve target is ffmpeg. Here is how to successfully instrument it: -1. Get and extract the current ffmpeg and change to its directory +1. Get and extract the current ffmpeg and change to its directory. 2. Running configure with --cc=clang fails and various other items will fail when compiling, so we have to trick configure: -``` -./configure --enable-lto --disable-shared --disable-inline-asm -``` - -3. Now the configuration is done - and we edit the settings in `./ffbuild/config.mak` - (-: the original line, +: what to change it into): -``` --CC=gcc -+CC=afl-clang-lto --CXX=g++ -+CXX=afl-clang-lto++ --AS=gcc -+AS=llvm-as --LD=gcc -+LD=afl-clang-lto++ --DEPCC=gcc -+DEPCC=afl-clang-lto --DEPAS=gcc -+DEPAS=afl-clang-lto++ --AR=ar -+AR=llvm-ar --AR_CMD=ar -+AR_CMD=llvm-ar --NM_CMD=nm -g -+NM_CMD=llvm-nm -g --RANLIB=ranlib -D -+RANLIB=llvm-ranlib -D -``` - -4. Then type make, wait for a long time and you are done :) + ``` + ./configure --enable-lto --disable-shared --disable-inline-asm + ``` + +3. Now the configuration is done - and we edit the settings in + `./ffbuild/config.mak` (-: the original line, +: what to change it into): + + ``` + -CC=gcc + +CC=afl-clang-lto + -CXX=g++ + +CXX=afl-clang-lto++ + -AS=gcc + +AS=llvm-as + -LD=gcc + +LD=afl-clang-lto++ + -DEPCC=gcc + +DEPCC=afl-clang-lto + -DEPAS=gcc + +DEPAS=afl-clang-lto++ + -AR=ar + +AR=llvm-ar + -AR_CMD=ar + +AR_CMD=llvm-ar + -NM_CMD=nm -g + +NM_CMD=llvm-nm -g + -RANLIB=ranlib -D + +RANLIB=llvm-ranlib -D + ``` + +4. Then type make, wait for a long time, and you are done. :) ### Example: WebKit jsc Building jsc is difficult as the build script has bugs. -1. checkout Webkit: -``` -svn checkout https://svn.webkit.org/repository/webkit/trunk WebKit -cd WebKit -``` +1. Checkout Webkit: + + ``` + svn checkout https://svn.webkit.org/repository/webkit/trunk WebKit + cd WebKit + ``` 2. Fix the build environment: -``` -mkdir -p WebKitBuild/Release -cd WebKitBuild/Release -ln -s ../../../../../usr/bin/llvm-ar-12 llvm-ar-12 -ln -s ../../../../../usr/bin/llvm-ranlib-12 llvm-ranlib-12 -cd ../.. -``` -3. Build :) + ``` + mkdir -p WebKitBuild/Release + cd WebKitBuild/Release + ln -s ../../../../../usr/bin/llvm-ar-12 llvm-ar-12 + ln -s ../../../../../usr/bin/llvm-ranlib-12 llvm-ranlib-12 + cd ../.. + ``` -``` -Tools/Scripts/build-jsc --jsc-only --cli --cmakeargs="-DCMAKE_AR='llvm-ar-12' -DCMAKE_RANLIB='llvm-ranlib-12' -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_CC_FLAGS='-O3 -lrt' -DCMAKE_CXX_FLAGS='-O3 -lrt' -DIMPORTED_LOCATION='/lib/x86_64-linux-gnu/' -DCMAKE_CC=afl-clang-lto -DCMAKE_CXX=afl-clang-lto++ -DENABLE_STATIC_JSC=ON" -``` +3. Build. :) + + ``` + Tools/Scripts/build-jsc --jsc-only --cli --cmakeargs="-DCMAKE_AR='llvm-ar-12' -DCMAKE_RANLIB='llvm-ranlib-12' -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_CC_FLAGS='-O3 -lrt' -DCMAKE_CXX_FLAGS='-O3 -lrt' -DIMPORTED_LOCATION='/lib/x86_64-linux-gnu/' -DCMAKE_CC=afl-clang-lto -DCMAKE_CXX=afl-clang-lto++ -DENABLE_STATIC_JSC=ON" + ``` ## Potential issues -### compiling libraries fails +### Compiling libraries fails If you see this message: + ``` /bin/ld: libfoo.a: error adding symbols: archive has no index; run ranlib to add one ``` -This is because usually gnu gcc ranlib is being called which cannot deal with clang LTO files. -The solution is simple: when you ./configure you also have to set RANLIB=llvm-ranlib and AR=llvm-ar + +This is because usually gnu gcc ranlib is being called which cannot deal with +clang LTO files. The solution is simple: when you `./configure`, you also have +to set `RANLIB=llvm-ranlib` and `AR=llvm-ar`. Solution: + ``` AR=llvm-ar RANLIB=llvm-ranlib CC=afl-clang-lto CXX=afl-clang-lto++ ./configure --disable-shared ``` -and on some targets you have to set AR=/RANLIB= even for make as the configure script does not save it. -Other targets ignore environment variables and need the parameters set via -`./configure --cc=... --cxx= --ranlib= ...` etc. (I am looking at you ffmpeg!). +And on some targets you have to set `AR=/RANLIB=` even for `make` as the +configure script does not save it. Other targets ignore environment variables +and need the parameters set via `./configure --cc=... --cxx= --ranlib= ...` etc. +(I am looking at you ffmpeg!) + +If you see this message: -If you see this message ``` assembler command failed ... ``` -then try setting `llvm-as` for configure: + +Then try setting `llvm-as` for configure: + ``` AS=llvm-as ... ``` -### compiling programs still fail +### Compiling programs still fail afl-clang-lto is still work in progress. Known issues: - * Anything that llvm 11+ cannot compile, afl-clang-lto cannot compile either - obviously - * Anything that does not compile with LTO, afl-clang-lto cannot compile either - obviously +* Anything that llvm 11+ cannot compile, afl-clang-lto cannot compile either - + obviously. +* Anything that does not compile with LTO, afl-clang-lto cannot compile either - + obviously. -Hence if building a target with afl-clang-lto fails try to build it with llvm12 -and LTO enabled (`CC=clang-12` `CXX=clang++-12` `CFLAGS=-flto=full` and -`CXXFLAGS=-flto=full`). +Hence, if building a target with afl-clang-lto fails, try to build it with +llvm12 and LTO enabled (`CC=clang-12`, `CXX=clang++-12`, `CFLAGS=-flto=full`, +and `CXXFLAGS=-flto=full`). -If this succeeeds then there is an issue with afl-clang-lto. Please report at -[https://github.com/AFLplusplus/AFLplusplus/issues/226](https://github.com/AFLplusplus/AFLplusplus/issues/226) +If this succeeds, then there is an issue with afl-clang-lto. Please report at +[https://github.com/AFLplusplus/AFLplusplus/issues/226](https://github.com/AFLplusplus/AFLplusplus/issues/226). -Even some targets where clang-12 fails can be build if the fail is just in +Even some targets where clang-12 fails can be built if the fail is just in `./configure`, see `Solving difficult targets` above. ## History -This was originally envisioned by hexcoder- in Summer 2019, however we saw no -way to create a pass that is run at link time - although there is a option -for this in the PassManager: EP_FullLinkTimeOptimizationLast -("Fun" info - nobody knows what this is doing. And the developer who -implemented this didn't respond to emails.) - -In December then came the idea to implement this as a pass that is run via -the llvm "opt" program, which is performed via an own linker that afterwards -calls the real linker. -This was first implemented in January and work ... kinda. -The LTO time instrumentation worked, however "how" the basic blocks were -instrumented was a problem, as reducing duplicates turned out to be very, -very difficult with a program that has so many paths and therefore so many -dependencies. A lot of strategies were implemented - and failed. -And then sat solvers were tried, but with over 10.000 variables that turned -out to be a dead-end too. +This was originally envisioned by hexcoder- in Summer 2019. However, we saw no +way to create a pass that is run at link time - although there is a option for +this in the PassManager: EP_FullLinkTimeOptimizationLast. ("Fun" info - nobody +knows what this is doing. And the developer who implemented this didn't respond +to emails.) + +In December then came the idea to implement this as a pass that is run via the +llvm "opt" program, which is performed via an own linker that afterwards calls +the real linker. This was first implemented in January and work ... kinda. The +LTO time instrumentation worked, however, "how" the basic blocks were +instrumented was a problem, as reducing duplicates turned out to be very, very +difficult with a program that has so many paths and therefore so many +dependencies. A lot of strategies were implemented - and failed. And then sat +solvers were tried, but with over 10.000 variables that turned out to be a +dead-end too. The final idea to solve this came from domenukk who proposed to insert a block -into an edge and then just use incremental counters ... and this worked! -After some trials and errors to implement this vanhauser-thc found out that -there is actually an llvm function for this: SplitEdge() :-) +into an edge and then just use incremental counters ... and this worked! After +some trials and errors to implement this vanhauser-thc found out that there is +actually an llvm function for this: SplitEdge() :-) -Still more problems came up though as this only works without bugs from -llvm 9 onwards, and with high optimization the link optimization ruins -the instrumented control flow graph. +Still more problems came up though as this only works without bugs from llvm 9 +onwards, and with high optimization the link optimization ruins the instrumented +control flow graph. -This is all now fixed with llvm 11+. The llvm's own linker is now able to -load passes and this bypasses all problems we had. +This is all now fixed with llvm 11+. The llvm's own linker is now able to load +passes and this bypasses all problems we had. -Happy end :) +Happy end :) \ No newline at end of file diff --git a/instrumentation/README.neverzero.md b/instrumentation/README.neverzero.md deleted file mode 100644 index 9bcae324..00000000 --- a/instrumentation/README.neverzero.md +++ /dev/null @@ -1,41 +0,0 @@ -# NeverZero counters for LLVM instrumentation - -## Usage - -In larger, complex or reiterative programs the byte sized counters that collect -the edge coverage can easily fill up and wrap around. -This is not that much of an issue - unless by chance it wraps just to a value -of zero when the program execution ends. -In this case afl-fuzz is not able to see that the edge has been accessed and -will ignore it. - -NeverZero prevents this behaviour. If a counter wraps, it jumps over the value -0 directly to a 1. This improves path discovery (by a very little amount) -at a very little cost (one instruction per edge). - -(The alternative of saturated counters has been tested also and proved to be -inferior in terms of path discovery.) - -This is implemented in afl-gcc and afl-gcc-fast, however for llvm_mode this is -optional if multithread safe counters are selected or the llvm version is below -9 - as there are severe performance costs in these cases. - -If you want to enable this for llvm versions below 9 or thread safe counters -then set - -``` -export AFL_LLVM_NOT_ZERO=1 -``` - -In case you are on llvm 9 or greater and you do not want this behaviour then -you can set: -``` -AFL_LLVM_SKIP_NEVERZERO=1 -``` -If the target does not have extensive loops or functions that are called -a lot then this can give a small performance boost. - -Please note that the default counter implementations are not thread safe! - -Support for thread safe counters in mode LLVM CLASSIC can be activated with setting -`AFL_LLVM_THREADSAFE_INST=1`. \ No newline at end of file diff --git a/instrumentation/README.ngram.md b/instrumentation/README.ngram.md deleted file mode 100644 index da61ef32..00000000 --- a/instrumentation/README.ngram.md +++ /dev/null @@ -1,28 +0,0 @@ -# AFL N-Gram Branch Coverage - -## Source - -This is an LLVM-based implementation of the n-gram branch coverage proposed in -the paper ["Be Sensitive and Collaborative: Analzying Impact of Coverage Metrics -in Greybox Fuzzing"](https://www.usenix.org/system/files/raid2019-wang-jinghan.pdf), -by Jinghan Wang, et. al. - -Note that the original implementation (available -[here](https://github.com/bitsecurerlab/afl-sensitive)) -is built on top of AFL's QEMU mode. -This is essentially a port that uses LLVM vectorized instructions (available from -llvm versions 4.0.1 and higher) to achieve the same results when compiling source code. - -In math the branch coverage is performed as follows: -`map[current_location ^ prev_location[0] >> 1 ^ prev_location[1] >> 1 ^ ... up to n-1`] += 1` - -## Usage - -The size of `n` (i.e., the number of branches to remember) is an option -that is specified either in the `AFL_LLVM_INSTRUMENT=NGRAM-{value}` or the -`AFL_LLVM_NGRAM_SIZE` environment variable. -Good values are 2, 4 or 8, valid are 2-16. - -It is highly recommended to increase the MAP_SIZE_POW2 definition in -config.h to at least 18 and maybe up to 20 for this as otherwise too -many map collisions occur. diff --git a/instrumentation/README.out_of_line.md b/instrumentation/README.out_of_line.md deleted file mode 100644 index 346fe98d..00000000 --- a/instrumentation/README.out_of_line.md +++ /dev/null @@ -1,19 +0,0 @@ -## Using AFL++ without inlined instrumentation - - This file describes how you can disable inlining of instrumentation. - - -By default, the GCC plugin will duplicate the effects of calling -`__afl_trace` (see `afl-gcc-rt.o.c`) in instrumented code, instead of -issuing function calls. - -The calls are presumed to be slower, more so because the rt file -itself is not optimized by the compiler. - -Setting `AFL_GCC_OUT_OF_LINE=1` in the environment while compiling code -with the plugin will disable this inlining, issuing calls to the -unoptimized runtime instead. - -You probably don't want to do this, but it might be useful in certain -AFL debugging scenarios, and it might work as a fallback in case -something goes wrong with the inlined instrumentation. diff --git a/instrumentation/README.persistent_mode.md b/instrumentation/README.persistent_mode.md index c6ba2103..14e59f4a 100644 --- a/instrumentation/README.persistent_mode.md +++ b/instrumentation/README.persistent_mode.md @@ -3,23 +3,23 @@ ## 1) Introduction In persistent mode, AFL++ fuzzes a target multiple times in a single forked -process, instead of forking a new process for each fuzz execution. -This is the most effective way to fuzz, as the speed can easily be x10 or x20 -times faster without any disadvanges. -*All professional fuzzing uses this mode.* +process, instead of forking a new process for each fuzz execution. This is the +most effective way to fuzz, as the speed can easily be x10 or x20 times faster +without any disadvantages. *All professional fuzzing uses this mode.* Persistent mode requires that the target can be called in one or more functions, and that it's state can be completely reset so that multiple calls can be performed without resource leaks, and that earlier runs will have no impact on -future runs (an indicator for this is the `stability` value in the `afl-fuzz` -UI, if this decreases to lower values in persistent mode compared to -non-persistent mode, that the fuzz target keeps state). +future runs. An indicator for this is the `stability` value in the `afl-fuzz` +UI. If this decreases to lower values in persistent mode compared to +non-persistent mode, then the fuzz target keeps state. Examples can be found in [utils/persistent_mode](../utils/persistent_mode). -## 2) TLDR; +## 2) TL;DR: Example `fuzz_target.c`: + ```c #include "what_you_need_for_your_target.h" @@ -27,7 +27,7 @@ __AFL_FUZZ_INIT(); main() { - // anything else here, eg. command line arguments, initialization, etc. + // anything else here, e.g. command line arguments, initialization, etc. #ifdef __AFL_HAVE_MANUAL_CONTROL __AFL_INIT(); @@ -54,14 +54,16 @@ main() { } ``` + And then compile: + ``` afl-clang-fast -o fuzz_target fuzz_target.c -lwhat_you_need_for_your_target ``` -And that is it! -The speed increase is usually x10 to x20. -If you want to be able to compile the target without afl-clang-fast/lto then +And that is it! The speed increase is usually x10 to x20. + +If you want to be able to compile the target without afl-clang-fast/lto, then add this just after the includes: ```c @@ -72,20 +74,20 @@ add this just after the includes: #define __AFL_FUZZ_TESTCASE_BUF fuzz_buf #define __AFL_FUZZ_INIT() void sync(void); #define __AFL_LOOP(x) ((fuzz_len = read(0, fuzz_buf, sizeof(fuzz_buf))) > 0 ? 1 : 0) - #define __AFL_INIT() sync() + #define __AFL_INIT() sync() #endif ``` ## 3) Deferred initialization -AFL tries to optimize performance by executing the targeted binary just once, -stopping it just before `main()`, and then cloning this "main" process to get -a steady supply of targets to fuzz. +AFL++ tries to optimize performance by executing the targeted binary just once, +stopping it just before `main()`, and then cloning this "main" process to get a +steady supply of targets to fuzz. -Although this approach eliminates much of the OS-, linker- and libc-level -costs of executing the program, it does not always help with binaries that -perform other time-consuming initialization steps - say, parsing a large config -file before getting to the fuzzed data. +Although this approach eliminates much of the OS-, linker- and libc-level costs +of executing the program, it does not always help with binaries that perform +other time-consuming initialization steps - say, parsing a large config file +before getting to the fuzzed data. In such cases, it's beneficial to initialize the forkserver a bit later, once most of the initialization work is already done, but before the binary attempts @@ -93,22 +95,21 @@ to read the fuzzed input and parse it; in some cases, this can offer a 10x+ performance gain. You can implement delayed initialization in LLVM mode in a fairly simple way. -First, find a suitable location in the code where the delayed cloning can -take place. This needs to be done with *extreme* care to avoid breaking the -binary. In particular, the program will probably malfunction if you select -a location after: +First, find a suitable location in the code where the delayed cloning can take +place. This needs to be done with *extreme* care to avoid breaking the binary. +In particular, the program will probably malfunction if you select a location +after: - - The creation of any vital threads or child processes - since the forkserver - can't clone them easily. +- The creation of any vital threads or child processes - since the forkserver + can't clone them easily. - - The initialization of timers via `setitimer()` or equivalent calls. +- The initialization of timers via `setitimer()` or equivalent calls. - - The creation of temporary files, network sockets, offset-sensitive file - descriptors, and similar shared-state resources - but only provided that - their state meaningfully influences the behavior of the program later on. +- The creation of temporary files, network sockets, offset-sensitive file + descriptors, and similar shared-state resources - but only provided that their + state meaningfully influences the behavior of the program later on. - - Any access to the fuzzed input, including reading the metadata about its - size. +- Any access to the fuzzed input, including reading the metadata about its size. With the location selected, add this code in the appropriate spot: @@ -126,13 +127,12 @@ Finally, recompile the program with afl-clang-fast/afl-clang-lto/afl-gcc-fast (afl-gcc or afl-clang will *not* generate a deferred-initialization binary) - and you should be all set! - ## 4) Persistent mode Some libraries provide APIs that are stateless, or whose state can be reset in between processing different input files. When such a reset is performed, a single long-lived process can be reused to try out multiple test cases, -eliminating the need for repeated fork() calls and the associated OS overhead. +eliminating the need for repeated `fork()` calls and the associated OS overhead. The basic structure of the program that does this would be: @@ -145,34 +145,34 @@ The basic structure of the program that does this would be: } - /* Exit normally */ + /* Exit normally. */ ``` -The numerical value specified within the loop controls the maximum number -of iterations before AFL will restart the process from scratch. This minimizes +The numerical value specified within the loop controls the maximum number of +iterations before AFL++ will restart the process from scratch. This minimizes the impact of memory leaks and similar glitches; 1000 is a good starting point, -and going much higher increases the likelihood of hiccups without giving you -any real performance benefits. +and going much higher increases the likelihood of hiccups without giving you any +real performance benefits. -A more detailed template is shown in `../utils/persistent_mode/.` -Similarly to the previous mode, the feature works only with afl-clang-fast; -`#ifdef` guards can be used to suppress it when using other compilers. +A more detailed template is shown in +[utils/persistent_mode](../utils/persistent_mode). Similarly to the deferred +initialization, the feature works only with afl-clang-fast; `#ifdef` guards can +be used to suppress it when using other compilers. -Note that as with the previous mode, the feature is easy to misuse; if you -do not fully reset the critical state, you may end up with false positives or -waste a whole lot of CPU power doing nothing useful at all. Be particularly +Note that as with the deferred initialization, the feature is easy to misuse; if +you do not fully reset the critical state, you may end up with false positives +or waste a whole lot of CPU power doing nothing useful at all. Be particularly wary of memory leaks and of the state of file descriptors. -PS. Because there are task switches still involved, the mode isn't as fast as -"pure" in-process fuzzing offered, say, by LLVM's LibFuzzer; but it is a lot -faster than the normal `fork()` model, and compared to in-process fuzzing, -should be a lot more robust. +When running in this mode, the execution paths will inherently vary a bit +depending on whether the input loop is being entered for the first time or +executed again. ## 5) Shared memory fuzzing -You can speed up the fuzzing process even more by receiving the fuzzing data -via shared memory instead of stdin or files. -This is a further speed multiplier of about 2x. +You can speed up the fuzzing process even more by receiving the fuzzing data via +shared memory instead of stdin or files. This is a further speed multiplier of +about 2x. Setting this up is very easy: @@ -181,14 +181,18 @@ After the includes set the following macro: ```c __AFL_FUZZ_INIT(); ``` -Directly at the start of main - or if you are using the deferred forkserver -with `__AFL_INIT()` then *after* `__AFL_INIT()` : + +Directly at the start of main - or if you are using the deferred forkserver with +`__AFL_INIT()`, then *after* `__AFL_INIT()`: + ```c unsigned char *buf = __AFL_FUZZ_TESTCASE_BUF; ``` Then as first line after the `__AFL_LOOP` while loop: + ```c int len = __AFL_FUZZ_TESTCASE_LEN; ``` -and that is all! + +And that is all! \ No newline at end of file diff --git a/instrumentation/README.snapshot.md b/instrumentation/README.snapshot.md deleted file mode 100644 index c794c2fd..00000000 --- a/instrumentation/README.snapshot.md +++ /dev/null @@ -1,18 +0,0 @@ -# AFL++ snapshot feature - -**NOTE:** the snapshot lkm is currently not supported and needs a maintainer :-) - -Snapshotting is a feature that makes a snapshot from a process and then -restores its state, which is faster then forking it again. - -All targets compiled with llvm_mode are automatically enabled for the -snapshot feature. - -To use the snapshot feature for fuzzing compile and load this kernel -module: [https://github.com/AFLplusplus/AFL-Snapshot-LKM](https://github.com/AFLplusplus/AFL-Snapshot-LKM) - -Note that is has little value for persistent (__AFL_LOOP) fuzzing. - -## Notes - -Snapshot does not work with multithreaded targets yet. Still in WIP, it is now usable only for single threaded applications. diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index e06f8b93..8d7f0c80 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -235,6 +235,8 @@ class ModuleSanitizerCoverage { uint32_t autodictionary = 1; uint32_t inst = 0; uint32_t afl_global_id = 0; + uint32_t unhandled = 0; + uint32_t select_cnt = 0; uint64_t map_addr = 0; const char * skip_nozero = NULL; const char * use_threadsafe_counters = nullptr; @@ -250,7 +252,7 @@ class ModuleSanitizerCoverage { Module * Mo = NULL; GlobalVariable * AFLMapPtr = NULL; Value * MapPtrFixed = NULL; - FILE * documentFile = NULL; + std::ofstream dFile; size_t found = 0; // afl++ END @@ -446,8 +448,8 @@ bool ModuleSanitizerCoverage::instrumentModule( if ((ptr = getenv("AFL_LLVM_DOCUMENT_IDS")) != NULL) { - if ((documentFile = fopen(ptr, "a")) == NULL) - WARNF("Cannot access document file %s", ptr); + dFile.open(ptr, std::ofstream::out | std::ofstream::app); + if (dFile.is_open()) WARNF("Cannot access document file %s", ptr); } @@ -619,7 +621,6 @@ bool ModuleSanitizerCoverage::instrumentModule( bool isStrncasecmp = true; bool isIntMemcpy = true; bool isStdString = true; - bool addedNull = false; size_t optLen = 0; Function *Callee = callInst->getCalledFunction(); @@ -799,7 +800,6 @@ bool ModuleSanitizerCoverage::instrumentModule( if (literalLength + 1 == optLength) { Str2.append("\0", 1); // add null byte - // addedNull = true; } @@ -907,8 +907,8 @@ bool ModuleSanitizerCoverage::instrumentModule( if (optLen < 2) { continue; } if (literalLength + 1 == optLen) { // add null byte + thestring.append("\0", 1); - addedNull = true; } @@ -920,14 +920,18 @@ bool ModuleSanitizerCoverage::instrumentModule( // was not already added if (!isMemcmp) { - if (addedNull == false && thestring[optLen - 1] != '\0') { + /* + if (addedNull == false && thestring[optLen - 1] != + '\0') { - thestring.append("\0", 1); // add null byte - optLen++; + thestring.append("\0", 1); // add null byte + optLen++; - } + } - if (!isStdString) { + */ + if (!isStdString && + thestring.find('\0', 0) != std::string::npos) { // ensure we do not have garbage size_t offset = thestring.find('\0', 0); @@ -1003,12 +1007,7 @@ bool ModuleSanitizerCoverage::instrumentModule( instrumentFunction(F, DTCallback, PDTCallback); // afl++ START - if (documentFile) { - - fclose(documentFile); - documentFile = NULL; - - } + if (dFile.is_open()) dFile.close(); if (!getenv("AFL_LLVM_LTO_DONTWRITEID") || dictionary.size() || map_addr) { @@ -1045,8 +1044,7 @@ bool ModuleSanitizerCoverage::instrumentModule( M, Int64Tyi, true, GlobalValue::ExternalLinkage, 0, "__afl_map_addr"); ConstantInt *MapAddr = ConstantInt::get(Int64Tyi, map_addr); StoreInst * StoreMapAddr = IRB.CreateStore(MapAddr, AFLMapAddrFixed); - StoreMapAddr->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(Ctx, None)); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(StoreMapAddr); } @@ -1054,22 +1052,20 @@ bool ModuleSanitizerCoverage::instrumentModule( uint32_t write_loc = afl_global_id; - if (afl_global_id % 8) write_loc = (((afl_global_id + 8) >> 3) << 3); + write_loc = (((afl_global_id + 8) >> 3) << 3); GlobalVariable *AFLFinalLoc = new GlobalVariable(M, Int32Tyi, true, GlobalValue::ExternalLinkage, 0, "__afl_final_loc"); ConstantInt *const_loc = ConstantInt::get(Int32Tyi, write_loc); StoreInst * StoreFinalLoc = IRB.CreateStore(const_loc, AFLFinalLoc); - StoreFinalLoc->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(Ctx, None)); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(StoreFinalLoc); } if (dictionary.size()) { size_t memlen = 0, count = 0, offset = 0; - char * ptr; // sort and unique the dictionary std::sort(dictionary.begin(), dictionary.end()); @@ -1089,13 +1085,7 @@ bool ModuleSanitizerCoverage::instrumentModule( if (count) { - if ((ptr = (char *)malloc(memlen + count)) == NULL) { - - fprintf(stderr, "Error: malloc for %lu bytes failed!\n", - memlen + count); - exit(-1); - - } + auto ptrhld = std::unique_ptr<char[]>(new char[memlen + count]); count = 0; @@ -1103,8 +1093,8 @@ bool ModuleSanitizerCoverage::instrumentModule( if (offset + token.length() < 0xfffff0 && count < MAX_AUTO_EXTRAS) { - ptr[offset++] = (uint8_t)token.length(); - memcpy(ptr + offset, token.c_str(), token.length()); + ptrhld.get()[offset++] = (uint8_t)token.length(); + memcpy(ptrhld.get() + offset, token.c_str(), token.length()); offset += token.length(); count++; @@ -1117,17 +1107,16 @@ bool ModuleSanitizerCoverage::instrumentModule( 0, "__afl_dictionary_len"); ConstantInt *const_len = ConstantInt::get(Int32Tyi, offset); StoreInst *StoreDictLen = IRB.CreateStore(const_len, AFLDictionaryLen); - StoreDictLen->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(Ctx, None)); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(StoreDictLen); ArrayType *ArrayTy = ArrayType::get(IntegerType::get(Ctx, 8), offset); GlobalVariable *AFLInternalDictionary = new GlobalVariable( M, ArrayTy, true, GlobalValue::ExternalLinkage, ConstantDataArray::get(Ctx, - *(new ArrayRef<char>((char *)ptr, offset))), + *(new ArrayRef<char>(ptrhld.get(), offset))), "__afl_internal_dictionary"); AFLInternalDictionary->setInitializer(ConstantDataArray::get( - Ctx, *(new ArrayRef<char>((char *)ptr, offset)))); + Ctx, *(new ArrayRef<char>(ptrhld.get(), offset)))); AFLInternalDictionary->setConstant(true); GlobalVariable *AFLDictionary = new GlobalVariable( @@ -1138,8 +1127,7 @@ bool ModuleSanitizerCoverage::instrumentModule( Value *AFLDictPtr = IRB.CreatePointerCast(AFLDictOff, PointerType::get(Int8Tyi, 0)); StoreInst *StoreDict = IRB.CreateStore(AFLDictPtr, AFLDictionary); - StoreDict->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(Ctx, None)); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(StoreDict); } @@ -1156,15 +1144,16 @@ bool ModuleSanitizerCoverage::instrumentModule( else { char modeline[100]; - snprintf(modeline, sizeof(modeline), "%s%s%s%s%s", + snprintf(modeline, sizeof(modeline), "%s%s%s%s%s%s", getenv("AFL_HARDEN") ? "hardened" : "non-hardened", getenv("AFL_USE_ASAN") ? ", ASAN" : "", getenv("AFL_USE_MSAN") ? ", MSAN" : "", + getenv("AFL_USE_TSAN") ? ", TSAN" : "", getenv("AFL_USE_CFISAN") ? ", CFISAN" : "", getenv("AFL_USE_UBSAN") ? ", UBSAN" : ""); - OKF("Instrumented %u locations with no collisions (on average %llu " - "collisions would be in afl-gcc/vanilla AFL) (%s mode).", - inst, calculateCollisions(inst), modeline); + OKF("Instrumented %u locations (%u selects) without collisions (%llu " + "collisions have been avoided) (%s mode).", + inst, select_cnt, calculateCollisions(inst), modeline); } @@ -1286,6 +1275,7 @@ void ModuleSanitizerCoverage::instrumentFunction( const DominatorTree * DT = DTCallback(F); const PostDominatorTree *PDT = PDTCallback(F); bool IsLeafFunc = true; + uint32_t skip_next = 0, local_selects = 0; for (auto &BB : F) { @@ -1299,10 +1289,164 @@ void ModuleSanitizerCoverage::instrumentFunction( if (!Callee) continue; if (callInst->getCallingConv() != llvm::CallingConv::C) continue; StringRef FuncName = Callee->getName(); + if (!FuncName.compare(StringRef("dlopen")) || + !FuncName.compare(StringRef("_dlopen"))) { + + fprintf(stderr, + "WARNING: dlopen() detected. To have coverage for a library " + "that your target dlopen()'s this must either happen before " + "__AFL_INIT() or you must use AFL_PRELOAD to preload all " + "dlopen()'ed libraries!\n"); + continue; + + } + if (FuncName.compare(StringRef("__afl_coverage_interesting"))) continue; Value *val = ConstantInt::get(Int32Ty, ++afl_global_id); callInst->setOperand(1, val); + ++inst; + + } + + SelectInst *selectInst = nullptr; + + /* + std::string errMsg; + raw_string_ostream os(errMsg); + IN.print(os); + fprintf(stderr, "X(%u): %s\n", skip_next, os.str().c_str()); + */ + if (!skip_next && (selectInst = dyn_cast<SelectInst>(&IN))) { + + uint32_t vector_cnt = 0; + Value * condition = selectInst->getCondition(); + Value * result; + auto t = condition->getType(); + IRBuilder<> IRB(selectInst->getNextNode()); + + ++select_cnt; + + if (t->getTypeID() == llvm::Type::IntegerTyID) { + + Value *val1 = ConstantInt::get(Int32Ty, ++afl_global_id); + Value *val2 = ConstantInt::get(Int32Ty, ++afl_global_id); + result = IRB.CreateSelect(condition, val1, val2); + skip_next = 1; + inst += 2; + + } else + +#if LLVM_VERSION_MAJOR >= 14 + if (t->getTypeID() == llvm::Type::FixedVectorTyID) { + + FixedVectorType *tt = dyn_cast<FixedVectorType>(t); + if (tt) { + + uint32_t elements = tt->getElementCount().getFixedValue(); + vector_cnt = elements; + inst += vector_cnt * 2; + if (elements) { + + FixedVectorType *GuardPtr1 = + FixedVectorType::get(Int32Ty, elements); + FixedVectorType *GuardPtr2 = + FixedVectorType::get(Int32Ty, elements); + Value *x, *y; + + Value *val1 = ConstantInt::get(Int32Ty, ++afl_global_id); + Value *val2 = ConstantInt::get(Int32Ty, ++afl_global_id); + x = IRB.CreateInsertElement(GuardPtr1, val1, (uint64_t)0); + y = IRB.CreateInsertElement(GuardPtr2, val2, (uint64_t)0); + + for (uint64_t i = 1; i < elements; i++) { + + val1 = ConstantInt::get(Int32Ty, ++afl_global_id); + val2 = ConstantInt::get(Int32Ty, ++afl_global_id); + x = IRB.CreateInsertElement(GuardPtr1, val1, i); + y = IRB.CreateInsertElement(GuardPtr2, val2, i); + + } + + result = IRB.CreateSelect(condition, x, y); + skip_next = 1; + + } + + } + + } else + +#endif + { + + unhandled++; + continue; + + } + + local_selects++; + uint32_t vector_cur = 0; + /* Load SHM pointer */ + LoadInst *MapPtr = IRB.CreateLoad(AFLMapPtr); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(MapPtr); + + while (1) { + + /* Get CurLoc */ + Value *MapPtrIdx = nullptr; + + /* Load counter for CurLoc */ + if (!vector_cnt) { + + MapPtrIdx = IRB.CreateGEP(MapPtr, result); + + } else { + + auto element = IRB.CreateExtractElement(result, vector_cur++); + MapPtrIdx = IRB.CreateGEP(MapPtr, element); + + } + + if (use_threadsafe_counters) { + + IRB.CreateAtomicRMW(llvm::AtomicRMWInst::BinOp::Add, MapPtrIdx, One, +#if LLVM_VERSION_MAJOR >= 13 + llvm::MaybeAlign(1), +#endif + llvm::AtomicOrdering::Monotonic); + + } else { + + LoadInst *Counter = IRB.CreateLoad(MapPtrIdx); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(Counter); + + /* Update bitmap */ + + Value *Incr = IRB.CreateAdd(Counter, One); + + if (skip_nozero == NULL) { + + auto cf = IRB.CreateICmpEQ(Incr, Zero); + auto carry = IRB.CreateZExt(cf, Int8Ty); + Incr = IRB.CreateAdd(Incr, carry); + + } + + auto nosan = IRB.CreateStore(Incr, MapPtrIdx); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(nosan); + + } + + if (!vector_cnt || vector_cnt == vector_cur) { break; } + + } + + skip_next = 1; + + } else { + + skip_next = 0; } @@ -1336,7 +1480,7 @@ GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection( *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, Constant::getNullValue(ArrayTy), "__sancov_gen_"); -#if LLVM_VERSION_MAJOR > 12 +#if LLVM_VERSION_MAJOR >= 13 if (TargetTriple.supportsCOMDAT() && (TargetTriple.isOSBinFormatELF() || !F.isInterposable())) if (auto Comdat = getOrCreateFunctionComdat(F, TargetTriple)) @@ -1496,10 +1640,10 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, if (Options.TracePC) { IRB.CreateCall(SanCovTracePC) -#if LLVM_VERSION_MAJOR < 12 - ->cannotMerge(); // gets the PC using GET_CALLER_PC. -#else +#if LLVM_VERSION_MAJOR >= 12 ->setCannotMerge(); // gets the PC using GET_CALLER_PC. +#else + ->cannotMerge(); // gets the PC using GET_CALLER_PC. #endif } @@ -1509,12 +1653,12 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, // afl++ START ++afl_global_id; - if (documentFile) { + if (dFile.is_open()) { unsigned long long int moduleID = (((unsigned long long int)(rand() & 0xffffffff)) << 32) | getpid(); - fprintf(documentFile, "ModuleID=%llu Function=%s edgeID=%u\n", moduleID, - F.getName().str().c_str(), afl_global_id); + dFile << "ModuleID=" << moduleID << " Function=" << F.getName().str() + << " edgeID=" << afl_global_id << "\n"; } @@ -1533,8 +1677,7 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, } else { LoadInst *MapPtr = IRB.CreateLoad(AFLMapPtr); - MapPtr->setMetadata(Mo->getMDKindID("nosanitize"), - MDNode::get(*Ct, None)); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(MapPtr); MapPtrIdx = IRB.CreateGEP(MapPtr, CurLoc); } @@ -1551,8 +1694,7 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, } else { LoadInst *Counter = IRB.CreateLoad(MapPtrIdx); - Counter->setMetadata(Mo->getMDKindID("nosanitize"), - MDNode::get(*Ct, None)); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(Counter); Value *Incr = IRB.CreateAdd(Counter, One); @@ -1564,8 +1706,8 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, } - IRB.CreateStore(Incr, MapPtrIdx) - ->setMetadata(Mo->getMDKindID("nosanitize"), MDNode::get(*Ct, None)); + auto nosan = IRB.CreateStore(Incr, MapPtrIdx); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(nosan); } diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 48ad2d02..d5746cc7 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -36,7 +36,8 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/SpecialCaseList.h" -#if LLVM_MAJOR > 10 || (LLVM_MAJOR == 10 && LLVM_MINOR > 0) +#if LLVM_VERSION_MAJOR >= 11 || \ + (LLVM_VERSION_MAJOR == 10 && LLVM_VERSION_MINOR >= 1) #include "llvm/Support/VirtualFileSystem.h" #endif #include "llvm/Support/raw_ostream.h" @@ -127,7 +128,7 @@ class ModuleSanitizerCoverage { public: ModuleSanitizerCoverage( const SanitizerCoverageOptions &Options = SanitizerCoverageOptions() -#if LLVM_MAJOR > 10 +#if (LLVM_VERSION_MAJOR >= 11) , const SpecialCaseList *Allowlist = nullptr, const SpecialCaseList *Blocklist = nullptr @@ -203,7 +204,7 @@ class ModuleSanitizerCoverage { SanitizerCoverageOptions Options; - uint32_t instr = 0; + uint32_t instr = 0, selects = 0, unhandled = 0; GlobalVariable *AFLMapPtr = NULL; ConstantInt * One = NULL; ConstantInt * Zero = NULL; @@ -215,7 +216,7 @@ class ModuleSanitizerCoverageLegacyPass : public ModulePass { public: ModuleSanitizerCoverageLegacyPass( const SanitizerCoverageOptions &Options = SanitizerCoverageOptions() -#if LLVM_VERSION_MAJOR > 10 +#if LLVM_VERSION_MAJOR >= 11 , const std::vector<std::string> &AllowlistFiles = std::vector<std::string>(), @@ -233,7 +234,7 @@ class ModuleSanitizerCoverageLegacyPass : public ModulePass { bool runOnModule(Module &M) override { ModuleSanitizerCoverage ModuleSancov(Options -#if LLVM_MAJOR > 10 +#if (LLVM_VERSION_MAJOR >= 11) , Allowlist.get(), Blocklist.get() #endif @@ -283,7 +284,7 @@ PreservedAnalyses ModuleSanitizerCoveragePass::run(Module & M, ModuleAnalysisManager &MAM) { ModuleSanitizerCoverage ModuleSancov(Options -#if LLVM_MAJOR > 10 +#if (LLVM_VERSION_MAJOR >= 11) , Allowlist.get(), Blocklist.get() #endif @@ -547,14 +548,16 @@ bool ModuleSanitizerCoverage::instrumentModule( else { char modeline[100]; - snprintf(modeline, sizeof(modeline), "%s%s%s%s%s", + snprintf(modeline, sizeof(modeline), "%s%s%s%s%s%s", getenv("AFL_HARDEN") ? "hardened" : "non-hardened", getenv("AFL_USE_ASAN") ? ", ASAN" : "", getenv("AFL_USE_MSAN") ? ", MSAN" : "", + getenv("AFL_USE_TSAN") ? ", TSAN" : "", getenv("AFL_USE_CFISAN") ? ", CFISAN" : "", getenv("AFL_USE_UBSAN") ? ", UBSAN" : ""); - OKF("Instrumented %u locations with no collisions (%s mode).", instr, - modeline); + OKF("Instrumented %u locations with no collisions (%s mode) of which are " + "%u handled and %u unhandled selects.", + instr, modeline, selects, unhandled); } @@ -747,7 +750,7 @@ GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection( *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, Constant::getNullValue(ArrayTy), "__sancov_gen_"); -#if LLVM_VERSION_MAJOR > 12 +#if LLVM_VERSION_MAJOR >= 13 if (TargetTriple.supportsCOMDAT() && (TargetTriple.isOSBinFormatELF() || !F.isInterposable())) if (auto Comdat = getOrCreateFunctionComdat(F, TargetTriple)) @@ -760,7 +763,8 @@ GlobalVariable *ModuleSanitizerCoverage::CreateFunctionLocalArrayInSection( #endif Array->setSection(getSectionName(Section)); -#if LLVM_MAJOR > 10 || (LLVM_MAJOR == 10 && LLVM_MINOR > 0) +#if (LLVM_VERSION_MAJOR >= 11) || \ + (LLVM_VERSION_MAJOR == 10 && LLVM_VERSION_MINOR >= 1) Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedSize())); #else Array->setAlignment(Align(4)); // cheating @@ -833,9 +837,8 @@ bool ModuleSanitizerCoverage::InjectCoverage(Function & F, ArrayRef<BasicBlock *> AllBlocks, bool IsLeafFunc) { - if (AllBlocks.empty()) return false; + uint32_t cnt_cov = 0, cnt_sel = 0, cnt_sel_inc = 0; - uint32_t special = 0; for (auto &BB : F) { for (auto &IN : BB) { @@ -848,11 +851,51 @@ bool ModuleSanitizerCoverage::InjectCoverage(Function & F, if (!Callee) continue; if (callInst->getCallingConv() != llvm::CallingConv::C) continue; StringRef FuncName = Callee->getName(); + if (!FuncName.compare(StringRef("dlopen")) || + !FuncName.compare(StringRef("_dlopen"))) { + + fprintf(stderr, + "WARNING: dlopen() detected. To have coverage for a library " + "that your target dlopen()'s this must either happen before " + "__AFL_INIT() or you must use AFL_PRELOAD to preload all " + "dlopen()'ed libraries!\n"); + continue; + + } + if (FuncName.compare(StringRef("__afl_coverage_interesting"))) continue; - uint32_t id = 1 + instr + (uint32_t)AllBlocks.size() + special++; - Value * val = ConstantInt::get(Int32Ty, id); - callInst->setOperand(1, val); + cnt_cov++; + + } + + SelectInst *selectInst = nullptr; + + if ((selectInst = dyn_cast<SelectInst>(&IN))) { + + Value *c = selectInst->getCondition(); + auto t = c->getType(); + if (t->getTypeID() == llvm::Type::IntegerTyID) { + + cnt_sel++; + cnt_sel_inc += 2; + + } + +#if (LLVM_VERSION_MAJOR >= 12) + else if (t->getTypeID() == llvm::Type::FixedVectorTyID) { + + FixedVectorType *tt = dyn_cast<FixedVectorType>(t); + if (tt) { + + cnt_sel++; + cnt_sel_inc += tt->getElementCount().getKnownMinValue(); + + } + + } + +#endif } @@ -860,11 +903,256 @@ bool ModuleSanitizerCoverage::InjectCoverage(Function & F, } - CreateFunctionLocalArrays(F, AllBlocks, special); - for (size_t i = 0, N = AllBlocks.size(); i < N; i++) - InjectCoverageAtBlock(F, *AllBlocks[i], i, IsLeafFunc); + /* Create PCGUARD array */ + CreateFunctionLocalArrays(F, AllBlocks, cnt_cov + cnt_sel_inc); + selects += cnt_sel; + + uint32_t special = 0, local_selects = 0, skip_next = 0; + + for (auto &BB : F) { + + for (auto &IN : BB) { + + CallInst *callInst = nullptr; + + /* + std::string errMsg; + raw_string_ostream os(errMsg); + IN.print(os); + fprintf(stderr, "X: %s\n", os.str().c_str()); + */ + if ((callInst = dyn_cast<CallInst>(&IN))) { + + Function *Callee = callInst->getCalledFunction(); + if (!Callee) continue; + if (callInst->getCallingConv() != llvm::CallingConv::C) continue; + StringRef FuncName = Callee->getName(); + if (FuncName.compare(StringRef("__afl_coverage_interesting"))) continue; + + IRBuilder<> IRB(callInst); + + Value *GuardPtr = IRB.CreateIntToPtr( + IRB.CreateAdd( + IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), + ConstantInt::get(IntptrTy, (++special + AllBlocks.size()) * 4)), + Int32PtrTy); + + LoadInst *Idx = IRB.CreateLoad(GuardPtr); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(Idx); + + callInst->setOperand(1, Idx); + + } + + SelectInst *selectInst = nullptr; + + if (!skip_next && (selectInst = dyn_cast<SelectInst>(&IN))) { + + uint32_t vector_cnt = 0; + Value * condition = selectInst->getCondition(); + Value * result; + auto t = condition->getType(); + IRBuilder<> IRB(selectInst->getNextNode()); + + if (t->getTypeID() == llvm::Type::IntegerTyID) { + + auto GuardPtr1 = IRB.CreateIntToPtr( + IRB.CreateAdd( + IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), + ConstantInt::get( + IntptrTy, + (cnt_cov + ++local_selects + AllBlocks.size()) * 4)), + Int32PtrTy); + + auto GuardPtr2 = IRB.CreateIntToPtr( + IRB.CreateAdd( + IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), + ConstantInt::get( + IntptrTy, + (cnt_cov + ++local_selects + AllBlocks.size()) * 4)), + Int32PtrTy); + + result = IRB.CreateSelect(condition, GuardPtr1, GuardPtr2); + + } else + +#if LLVM_VERSION_MAJOR >= 14 + if (t->getTypeID() == llvm::Type::FixedVectorTyID) { + + FixedVectorType *tt = dyn_cast<FixedVectorType>(t); + if (tt) { + + uint32_t elements = tt->getElementCount().getFixedValue(); + vector_cnt = elements; + if (elements) { + + FixedVectorType *GuardPtr1 = + FixedVectorType::get(Int32PtrTy, elements); + FixedVectorType *GuardPtr2 = + FixedVectorType::get(Int32PtrTy, elements); + Value *x, *y; + + Value *val1 = IRB.CreateIntToPtr( + IRB.CreateAdd( + IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), + ConstantInt::get( + IntptrTy, + (cnt_cov + ++local_selects + AllBlocks.size()) * 4)), + Int32PtrTy); + x = IRB.CreateInsertElement(GuardPtr1, val1, (uint64_t)0); + + Value *val2 = IRB.CreateIntToPtr( + IRB.CreateAdd( + IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), + ConstantInt::get( + IntptrTy, + (cnt_cov + ++local_selects + AllBlocks.size()) * 4)), + Int32PtrTy); + y = IRB.CreateInsertElement(GuardPtr2, val2, (uint64_t)0); + + for (uint64_t i = 1; i < elements; i++) { + + val1 = IRB.CreateIntToPtr( + IRB.CreateAdd( + IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), + ConstantInt::get(IntptrTy, (cnt_cov + ++local_selects + + AllBlocks.size()) * + 4)), + Int32PtrTy); + x = IRB.CreateInsertElement(x, val1, i); + + val2 = IRB.CreateIntToPtr( + IRB.CreateAdd( + IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), + ConstantInt::get(IntptrTy, (cnt_cov + ++local_selects + + AllBlocks.size()) * + 4)), + Int32PtrTy); + y = IRB.CreateInsertElement(y, val2, i); + + } + + /* + std::string errMsg; + raw_string_ostream os(errMsg); + x->print(os); + fprintf(stderr, "X: %s\n", os.str().c_str()); + */ + result = IRB.CreateSelect(condition, x, y); + + } + + } + + } else + +#endif + { + + unhandled++; + continue; - instr += special; + } + + local_selects++; + uint32_t vector_cur = 0; + + /* Load SHM pointer */ + + LoadInst *MapPtr = IRB.CreateLoad(AFLMapPtr); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(MapPtr); + + /* + std::string errMsg; + raw_string_ostream os(errMsg); + result->print(os); + fprintf(stderr, "X: %s\n", os.str().c_str()); + */ + + while (1) { + + /* Get CurLoc */ + LoadInst *CurLoc = nullptr; + Value * MapPtrIdx = nullptr; + + /* Load counter for CurLoc */ + if (!vector_cnt) { + + CurLoc = IRB.CreateLoad(result); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(CurLoc); + MapPtrIdx = IRB.CreateGEP(MapPtr, CurLoc); + + } else { + + auto element = IRB.CreateExtractElement(result, vector_cur++); + auto elementptr = IRB.CreateIntToPtr(element, Int32PtrTy); + auto elementld = IRB.CreateLoad(elementptr); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(elementld); + MapPtrIdx = IRB.CreateGEP(MapPtr, elementld); + + } + + if (use_threadsafe_counters) { + + IRB.CreateAtomicRMW(llvm::AtomicRMWInst::BinOp::Add, MapPtrIdx, One, +#if LLVM_VERSION_MAJOR >= 13 + llvm::MaybeAlign(1), +#endif + llvm::AtomicOrdering::Monotonic); + + } else { + + LoadInst *Counter = IRB.CreateLoad(MapPtrIdx); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(Counter); + + /* Update bitmap */ + + Value *Incr = IRB.CreateAdd(Counter, One); + + if (skip_nozero == NULL) { + + auto cf = IRB.CreateICmpEQ(Incr, Zero); + auto carry = IRB.CreateZExt(cf, Int8Ty); + Incr = IRB.CreateAdd(Incr, carry); + + } + + StoreInst *StoreCtx = IRB.CreateStore(Incr, MapPtrIdx); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(StoreCtx); + + } + + if (!vector_cnt) { + + vector_cnt = 2; + break; + + } else if (vector_cnt == vector_cur) { + + break; + + } + + } + + skip_next = 1; + instr += vector_cnt; + + } else { + + skip_next = 0; + + } + + } + + } + + if (AllBlocks.empty() && !special && !local_selects) return false; + + if (!AllBlocks.empty()) + for (size_t i = 0, N = AllBlocks.size(); i < N; i++) + InjectCoverageAtBlock(F, *AllBlocks[i], i, IsLeafFunc); return true; @@ -881,8 +1169,6 @@ void ModuleSanitizerCoverage::InjectCoverageForIndirectCalls( Function &F, ArrayRef<Instruction *> IndirCalls) { if (IndirCalls.empty()) return; - assert(Options.TracePC || Options.TracePCGuard || - Options.Inline8bitCounters /*|| Options.InlineBoolFlag*/); for (auto I : IndirCalls) { IRBuilder<> IRB(I); @@ -1062,10 +1348,12 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, Int32PtrTy); LoadInst *CurLoc = IRB.CreateLoad(GuardPtr); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(CurLoc); /* Load SHM pointer */ LoadInst *MapPtr = IRB.CreateLoad(AFLMapPtr); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(MapPtr); /* Load counter for CurLoc */ @@ -1082,6 +1370,8 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, } else { LoadInst *Counter = IRB.CreateLoad(MapPtrIdx); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(Counter); + /* Update bitmap */ Value *Incr = IRB.CreateAdd(Counter, One); @@ -1094,7 +1384,8 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB, } - IRB.CreateStore(Incr, MapPtrIdx); + StoreInst *StoreCtx = IRB.CreateStore(Incr, MapPtrIdx); + ModuleSanitizerCoverage::SetNoSanitizeMetadata(StoreCtx); } @@ -1207,7 +1498,7 @@ INITIALIZE_PASS_END(ModuleSanitizerCoverageLegacyPass, "sancov", ModulePass *llvm::createModuleSanitizerCoverageLegacyPassPass( const SanitizerCoverageOptions &Options -#if LLVM_MAJOR > 10 +#if (LLVM_VERSION_MAJOR >= 11) , const std::vector<std::string> &AllowlistFiles, const std::vector<std::string> &BlocklistFiles @@ -1215,7 +1506,7 @@ ModulePass *llvm::createModuleSanitizerCoverageLegacyPassPass( ) { return new ModuleSanitizerCoverageLegacyPass(Options -#if LLVM_MAJOR > 10 +#if (LLVM_VERSION_MAJOR >= 11) , AllowlistFiles, BlocklistFiles #endif diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 9acab4e7..20f325f3 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -9,7 +9,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ @@ -22,6 +22,10 @@ #include "cmplog.h" #include "llvm-alternative-coverage.h" +#define XXH_INLINE_ALL +#include "xxhash.h" +#undef XXH_INLINE_ALL + #include <stdio.h> #include <stdlib.h> #include <signal.h> @@ -154,6 +158,8 @@ static void at_exit(int signal) { } +#define default_hash(a, b) XXH3_64bits(a, b) + /* Uninspired gcc plugin instrumentation */ void __afl_trace(const u32 x) { @@ -269,6 +275,8 @@ static void __afl_map_shm(void) { char *id_str = getenv(SHM_ENV_VAR); + if (__afl_final_loc) { ++__afl_final_loc; } // as we count starting 0 + if (__afl_final_loc) { __afl_map_size = __afl_final_loc; @@ -664,12 +672,12 @@ static void __afl_start_snapshots(void) { u8 child_stopped = 0; - void (*old_sigchld_handler)(int) = 0; // = signal(SIGCHLD, SIG_DFL); + void (*old_sigchld_handler)(int) = signal(SIGCHLD, SIG_DFL); /* Phone home and tell the parent that we're OK. If parent isn't there, assume we're not running in forkserver mode and just execute program. */ - status |= (FS_OPT_ENABLED | FS_OPT_SNAPSHOT); + status |= (FS_OPT_ENABLED | FS_OPT_SNAPSHOT | FS_OPT_NEWCMPLOG); if (__afl_sharedmem_fuzzing != 0) status |= FS_OPT_SHDMEM_FUZZ; if (__afl_map_size <= FS_OPT_MAX_MAPSIZE) status |= (FS_OPT_SET_MAPSIZE(__afl_map_size) | FS_OPT_MAPSIZE); @@ -920,7 +928,7 @@ static void __afl_start_forkserver(void) { u8 child_stopped = 0; - void (*old_sigchld_handler)(int) = 0; // = signal(SIGCHLD, SIG_DFL); + void (*old_sigchld_handler)(int) = signal(SIGCHLD, SIG_DFL); if (__afl_map_size <= FS_OPT_MAX_MAPSIZE) { @@ -935,7 +943,12 @@ static void __afl_start_forkserver(void) { } if (__afl_sharedmem_fuzzing != 0) { status_for_fsrv |= FS_OPT_SHDMEM_FUZZ; } - if (status_for_fsrv) { status_for_fsrv |= (FS_OPT_ENABLED); } + if (status_for_fsrv) { + + status_for_fsrv |= (FS_OPT_ENABLED | FS_OPT_NEWCMPLOG); + + } + memcpy(tmp, &status_for_fsrv, 4); /* Phone home and tell the parent that we're OK. If parent isn't there, @@ -1404,6 +1417,18 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { if (start == stop || *start) return; + // If a dlopen of an instrumented library happens after the forkserver then + // we have a problem as we cannot increase the coverage map anymore. + if (__afl_already_initialized_forkserver) { + + fprintf(stderr, + "[-] FATAL: forkserver is already up, but an instrumented dlopen() " + "library loaded afterwards. You must AFL_PRELOAD such libraries to " + "be able to fuzz them or LD_PRELOAD to run outside of afl-fuzz.\n"); + abort(); + + } + x = getenv("AFL_INST_RATIO"); if (x) inst_ratio = (u32)atoi(x); @@ -1416,6 +1441,7 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { /* instrumented code is loaded *after* our forkserver is up. this is a problem. We cannot prevent collisions then :( */ + /* if (__afl_already_initialized_forkserver && __afl_final_loc + 1 + stop - start > __afl_map_size) { @@ -1448,6 +1474,8 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { } + */ + /* Make sure that the first element in the range is always set - we use that to avoid duplicate calls (which can happen as an artifact of the underlying implementation in LLVM). */ @@ -1499,8 +1527,7 @@ void __cmplog_ins_hook1(uint8_t arg1, uint8_t arg2, uint8_t attr) { if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1530,8 +1557,7 @@ void __cmplog_ins_hook2(uint16_t arg1, uint16_t arg2, uint8_t attr) { if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1569,8 +1595,7 @@ void __cmplog_ins_hook4(uint32_t arg1, uint32_t arg2, uint8_t attr) { if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1608,8 +1633,7 @@ void __cmplog_ins_hook8(uint64_t arg1, uint64_t arg2, uint8_t attr) { if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1652,8 +1676,7 @@ void __cmplog_ins_hookN(uint128_t arg1, uint128_t arg2, uint8_t attr, if (unlikely(!__afl_cmp_map || arg1 == arg2)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1696,8 +1719,7 @@ void __cmplog_ins_hook16(uint128_t arg1, uint128_t arg2, uint8_t attr) { if (likely(!__afl_cmp_map)) return; uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1802,8 +1824,8 @@ void __sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases) { for (uint64_t i = 0; i < cases[0]; i++) { uintptr_t k = (uintptr_t)__builtin_return_address(0) + i; - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & + (CMP_MAP_W - 1)); u32 hits; @@ -1880,11 +1902,108 @@ static int area_is_valid(void *ptr, size_t len) { } +/* hook for string with length functions, eg. strncmp, strncasecmp etc. + Note that we ignore the len parameter and take longer strings if present. */ +void __cmplog_rtn_hook_strn(u8 *ptr1, u8 *ptr2, u64 len) { + + // fprintf(stderr, "RTN1 %p %p %u\n", ptr1, ptr2, len); + if (likely(!__afl_cmp_map)) return; + if (unlikely(!len)) return; + int len0 = MIN(len, 31); + int len1 = strnlen(ptr1, len0); + if (len1 < 31) len1 = area_is_valid(ptr1, len1 + 1); + int len2 = strnlen(ptr2, len0); + if (len2 < 31) len2 = area_is_valid(ptr1, len2 + 1); + int l = MAX(len1, len2); + if (l < 2) return; + + uintptr_t k = (uintptr_t)__builtin_return_address(0); + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); + + u32 hits; + + if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) { + + __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; + __afl_cmp_map->headers[k].hits = 1; + __afl_cmp_map->headers[k].shape = l - 1; + hits = 0; + + } else { + + hits = __afl_cmp_map->headers[k].hits++; + + if (__afl_cmp_map->headers[k].shape < l) { + + __afl_cmp_map->headers[k].shape = l - 1; + + } + + } + + struct cmpfn_operands *cmpfn = (struct cmpfn_operands *)__afl_cmp_map->log[k]; + hits &= CMP_MAP_RTN_H - 1; + + cmpfn[hits].v0_len = 0x80 + l; + cmpfn[hits].v1_len = 0x80 + l; + __builtin_memcpy(cmpfn[hits].v0, ptr1, len1); + __builtin_memcpy(cmpfn[hits].v1, ptr2, len2); + // fprintf(stderr, "RTN3\n"); + +} + +/* hook for string functions, eg. strcmp, strcasecmp etc. */ +void __cmplog_rtn_hook_str(u8 *ptr1, u8 *ptr2) { + + // fprintf(stderr, "RTN1 %p %p\n", ptr1, ptr2); + if (likely(!__afl_cmp_map)) return; + if (unlikely(!ptr1 || !ptr2)) return; + int len1 = strnlen(ptr1, 30) + 1; + int len2 = strnlen(ptr2, 30) + 1; + int l = MAX(len1, len2); + if (l < 3) return; + + uintptr_t k = (uintptr_t)__builtin_return_address(0); + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); + + u32 hits; + + if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) { + + __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; + __afl_cmp_map->headers[k].hits = 1; + __afl_cmp_map->headers[k].shape = l - 1; + hits = 0; + + } else { + + hits = __afl_cmp_map->headers[k].hits++; + + if (__afl_cmp_map->headers[k].shape < l) { + + __afl_cmp_map->headers[k].shape = l - 1; + + } + + } + + struct cmpfn_operands *cmpfn = (struct cmpfn_operands *)__afl_cmp_map->log[k]; + hits &= CMP_MAP_RTN_H - 1; + + cmpfn[hits].v0_len = 0x80 + len1; + cmpfn[hits].v1_len = 0x80 + len2; + __builtin_memcpy(cmpfn[hits].v0, ptr1, len1); + __builtin_memcpy(cmpfn[hits].v1, ptr2, len2); + // fprintf(stderr, "RTN3\n"); + +} + +/* hook function for all other func(ptr, ptr, ...) variants */ void __cmplog_rtn_hook(u8 *ptr1, u8 *ptr2) { /* u32 i; - if (area_is_valid(ptr1, 32) <= 0 || area_is_valid(ptr2, 32) <= 0) return; + if (area_is_valid(ptr1, 31) <= 0 || area_is_valid(ptr2, 31) <= 0) return; fprintf(stderr, "rtn arg0="); for (i = 0; i < 32; i++) fprintf(stderr, "%02x", ptr1[i]); @@ -1894,18 +2013,17 @@ void __cmplog_rtn_hook(u8 *ptr1, u8 *ptr2) { fprintf(stderr, "\n"); */ - if (likely(!__afl_cmp_map)) return; // fprintf(stderr, "RTN1 %p %p\n", ptr1, ptr2); + if (likely(!__afl_cmp_map)) return; int l1, l2; - if ((l1 = area_is_valid(ptr1, 32)) <= 0 || - (l2 = area_is_valid(ptr2, 32)) <= 0) + if ((l1 = area_is_valid(ptr1, 31)) <= 0 || + (l2 = area_is_valid(ptr2, 31)) <= 0) return; - int len = MIN(l1, l2); + int len = MIN(31, MIN(l1, l2)); // fprintf(stderr, "RTN2 %u\n", len); uintptr_t k = (uintptr_t)__builtin_return_address(0); - k = (k >> 4) ^ (k << 8); - k &= CMP_MAP_W - 1; + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); u32 hits; @@ -1928,15 +2046,83 @@ void __cmplog_rtn_hook(u8 *ptr1, u8 *ptr2) { } + struct cmpfn_operands *cmpfn = (struct cmpfn_operands *)__afl_cmp_map->log[k]; hits &= CMP_MAP_RTN_H - 1; - __builtin_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v0, - ptr1, len); - __builtin_memcpy(((struct cmpfn_operands *)__afl_cmp_map->log[k])[hits].v1, - ptr2, len); + + cmpfn[hits].v0_len = len; + cmpfn[hits].v1_len = len; + __builtin_memcpy(cmpfn[hits].v0, ptr1, len); + __builtin_memcpy(cmpfn[hits].v1, ptr2, len); // fprintf(stderr, "RTN3\n"); } +/* hook for func(ptr, ptr, len, ...) looking functions. + Note that for the time being we ignore len as this could be wrong + information and pass it on to the standard binary rtn hook */ +void __cmplog_rtn_hook_n(u8 *ptr1, u8 *ptr2, u64 len) { + + (void)(len); + __cmplog_rtn_hook(ptr1, ptr2); + +#if 0 + /* + u32 i; + if (area_is_valid(ptr1, 31) <= 0 || area_is_valid(ptr2, 31) <= 0) return; + fprintf(stderr, "rtn_n len=%u arg0=", len); + for (i = 0; i < len; i++) + fprintf(stderr, "%02x", ptr1[i]); + fprintf(stderr, " arg1="); + for (i = 0; i < len; i++) + fprintf(stderr, "%02x", ptr2[i]); + fprintf(stderr, "\n"); + */ + + // fprintf(stderr, "RTN1 %p %p %u\n", ptr1, ptr2, len); + if (likely(!__afl_cmp_map)) return; + if (unlikely(!len)) return; + int l = MIN(31, len); + + if ((l = area_is_valid(ptr1, l)) <= 0 || (l = area_is_valid(ptr2, l)) <= 0) + return; + + // fprintf(stderr, "RTN2 %u\n", l); + uintptr_t k = (uintptr_t)__builtin_return_address(0); + k = (uintptr_t)(default_hash((u8 *)&k, sizeof(uintptr_t)) & (CMP_MAP_W - 1)); + + u32 hits; + + if (__afl_cmp_map->headers[k].type != CMP_TYPE_RTN) { + + __afl_cmp_map->headers[k].type = CMP_TYPE_RTN; + __afl_cmp_map->headers[k].hits = 1; + __afl_cmp_map->headers[k].shape = l - 1; + hits = 0; + + } else { + + hits = __afl_cmp_map->headers[k].hits++; + + if (__afl_cmp_map->headers[k].shape < l) { + + __afl_cmp_map->headers[k].shape = l - 1; + + } + + } + + struct cmpfn_operands *cmpfn = (struct cmpfn_operands *)__afl_cmp_map->log[k]; + hits &= CMP_MAP_RTN_H - 1; + + cmpfn[hits].v0_len = l; + cmpfn[hits].v1_len = l; + __builtin_memcpy(cmpfn[hits].v0, ptr1, l); + __builtin_memcpy(cmpfn[hits].v1, ptr2, l); + // fprintf(stderr, "RTN3\n"); +#endif + +} + // gcc libstdc++ // _ZNKSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEE7compareEPKc static u8 *get_gcc_stdstring(u8 *string) { @@ -2084,5 +2270,11 @@ void __afl_coverage_interesting(u8 val, u32 id) { } +void __afl_set_persistent_mode(u8 mode) { + + is_persistent = mode; + +} + #undef write_error diff --git a/instrumentation/afl-gcc-pass.so.cc b/instrumentation/afl-gcc-pass.so.cc index 3b7eb878..df2b6f2a 100644 --- a/instrumentation/afl-gcc-pass.so.cc +++ b/instrumentation/afl-gcc-pass.so.cc @@ -30,7 +30,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. + along with this program. If not, see <https://www.gnu.org/licenses/>. */ diff --git a/instrumentation/afl-llvm-common.cc b/instrumentation/afl-llvm-common.cc index 3239ea91..e5e367a7 100644 --- a/instrumentation/afl-llvm-common.cc +++ b/instrumentation/afl-llvm-common.cc @@ -281,7 +281,7 @@ void scanForDangerousFunctions(llvm::Module *M) { if (!M) return; -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 9) for (GlobalIFunc &IF : M->ifuncs()) { diff --git a/instrumentation/afl-llvm-dict2file.so.cc b/instrumentation/afl-llvm-dict2file.so.cc index 4622e488..bf07a154 100644 --- a/instrumentation/afl-llvm-dict2file.so.cc +++ b/instrumentation/afl-llvm-dict2file.so.cc @@ -10,7 +10,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This library is plugged into LLVM when invoking clang through afl-clang-lto. @@ -66,6 +66,9 @@ namespace { class AFLdict2filePass : public ModulePass { + std::ofstream of; + void dict2file(u8 *, u32); + public: static char ID; @@ -81,7 +84,7 @@ class AFLdict2filePass : public ModulePass { } // namespace -void dict2file(int fd, u8 *mem, u32 len) { +void AFLdict2filePass::dict2file(u8 *mem, u32 len) { u32 i, j, binary = 0; char line[MAX_AUTO_EXTRA * 8], tmp[8]; @@ -113,9 +116,8 @@ void dict2file(int fd, u8 *mem, u32 len) { line[j] = 0; strcat(line, "\"\n"); - if (write(fd, line, strlen(line)) <= 0) - PFATAL("Could not write to dictionary file"); - fsync(fd); + of << line; + of.flush(); if (!be_quiet) fprintf(stderr, "Found dictionary token: %s", line); @@ -125,7 +127,7 @@ bool AFLdict2filePass::runOnModule(Module &M) { DenseMap<Value *, std::string *> valueMap; char * ptr; - int fd, found = 0; + int found = 0; /* Show a banner */ setvbuf(stdout, NULL, _IONBF, 0); @@ -146,8 +148,8 @@ bool AFLdict2filePass::runOnModule(Module &M) { if (!ptr || *ptr != '/') FATAL("AFL_LLVM_DICT2FILE is not set to an absolute path: %s", ptr); - if ((fd = open(ptr, O_WRONLY | O_APPEND | O_CREAT | O_DSYNC, 0644)) < 0) - PFATAL("Could not open/create %s.", ptr); + of.open(ptr, std::ofstream::out | std::ofstream::app); + if (!of.is_open()) PFATAL("Could not open/create %s.", ptr); /* Instrument all the things! */ @@ -264,11 +266,11 @@ bool AFLdict2filePass::runOnModule(Module &M) { } - dict2file(fd, (u8 *)&val, len); + dict2file((u8 *)&val, len); found++; if (val2) { - dict2file(fd, (u8 *)&val2, len); + dict2file((u8 *)&val2, len); found++; } @@ -289,7 +291,6 @@ bool AFLdict2filePass::runOnModule(Module &M) { bool isIntMemcpy = true; bool isStdString = true; bool isStrstr = true; - bool addedNull = false; size_t optLen = 0; Function *Callee = callInst->getCalledFunction(); @@ -588,8 +589,8 @@ bool AFLdict2filePass::runOnModule(Module &M) { if (optLen < 2) { continue; } if (literalLength + 1 == optLen) { // add null byte + thestring.append("\0", 1); - addedNull = true; } @@ -601,14 +602,17 @@ bool AFLdict2filePass::runOnModule(Module &M) { // was not already added if (!isMemcmp) { - if (addedNull == false && thestring[optLen - 1] != '\0') { + /* + if (addedNull == false && thestring[optLen - 1] != '\0') + { - thestring.append("\0", 1); // add null byte - optLen++; + thestring.append("\0", 1); // add null byte + optLen++; - } + } - if (!isStdString) { + */ + if (!isStdString && thestring.find('\0', 0) != std::string::npos) { // ensure we do not have garbage size_t offset = thestring.find('\0', 0); @@ -630,7 +634,7 @@ bool AFLdict2filePass::runOnModule(Module &M) { ptr = (char *)thestring.c_str(); - dict2file(fd, (u8 *)ptr, optLen); + dict2file((u8 *)ptr, optLen); found++; } @@ -641,7 +645,7 @@ bool AFLdict2filePass::runOnModule(Module &M) { } - close(fd); + of.close(); /* Say something nice. */ diff --git a/instrumentation/afl-llvm-lto-instrumentation.so.cc b/instrumentation/afl-llvm-lto-instrumentation.so.cc deleted file mode 100644 index e300044c..00000000 --- a/instrumentation/afl-llvm-lto-instrumentation.so.cc +++ /dev/null @@ -1,1118 +0,0 @@ -/* - american fuzzy lop++ - LLVM LTO instrumentation pass - ---------------------------------------------------- - - Written by Marc Heuse <mh@mh-sec.de> - - Copyright 2019-2020 AFLplusplus Project. All rights reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - - This library is plugged into LLVM when invoking clang through afl-clang-lto. - - */ - -#define AFL_LLVM_PASS - -#include "config.h" -#include "debug.h" - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <string.h> -#include <sys/time.h> - -#include <list> -#include <string> -#include <fstream> -#include <set> -#include <iostream> - -#include "llvm/Config/llvm-config.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/IR/IRBuilder.h" -#include "llvm/IR/LegacyPassManager.h" -#include "llvm/IR/BasicBlock.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/DebugInfo.h" -#include "llvm/IR/CFG.h" -#include "llvm/IR/Verifier.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" -#include "llvm/Transforms/Utils/BasicBlockUtils.h" -#include "llvm/Analysis/LoopInfo.h" -#include "llvm/Analysis/MemorySSAUpdater.h" -#include "llvm/Analysis/ValueTracking.h" -#include "llvm/Pass.h" -#include "llvm/IR/Constants.h" - -#include "afl-llvm-common.h" - -using namespace llvm; - -namespace { - -class AFLLTOPass : public ModulePass { - - public: - static char ID; - - AFLLTOPass() : ModulePass(ID) { - - char *ptr; - - if (getenv("AFL_DEBUG")) debug = 1; - if ((ptr = getenv("AFL_LLVM_LTO_STARTID")) != NULL) - if ((afl_global_id = (uint32_t)atoi(ptr)) < 0 || - afl_global_id >= MAP_SIZE) - FATAL("AFL_LLVM_LTO_STARTID value of \"%s\" is not between 0 and %u\n", - ptr, MAP_SIZE - 1); - - skip_nozero = getenv("AFL_LLVM_SKIP_NEVERZERO"); - - } - - void getAnalysisUsage(AnalysisUsage &AU) const override { - - ModulePass::getAnalysisUsage(AU); - AU.addRequired<DominatorTreeWrapperPass>(); - AU.addRequired<LoopInfoWrapperPass>(); - - } - - bool runOnModule(Module &M) override; - - protected: - uint32_t afl_global_id = 1, autodictionary = 1; - uint32_t function_minimum_size = 1; - uint32_t inst_blocks = 0, inst_funcs = 0, total_instr = 0; - unsigned long long int map_addr = 0x10000; - const char * skip_nozero = NULL; - const char * use_threadsafe_counters = nullptr; - -}; - -} // namespace - -bool AFLLTOPass::runOnModule(Module &M) { - - LLVMContext & C = M.getContext(); - std::vector<std::string> dictionary; - // std::vector<CallInst *> calls; - DenseMap<Value *, std::string *> valueMap; - std::vector<BasicBlock *> BlockList; - char * ptr; - FILE * documentFile = NULL; - size_t found = 0; - - srand((unsigned int)time(NULL)); - - unsigned long long int moduleID = - (((unsigned long long int)(rand() & 0xffffffff)) << 32) | getpid(); - - IntegerType *Int8Ty = IntegerType::getInt8Ty(C); - IntegerType *Int32Ty = IntegerType::getInt32Ty(C); - IntegerType *Int64Ty = IntegerType::getInt64Ty(C); - - /* Show a banner */ - setvbuf(stdout, NULL, _IONBF, 0); - - if ((isatty(2) && !getenv("AFL_QUIET")) || debug) { - - SAYF(cCYA "afl-llvm-lto" VERSION cRST - " by Marc \"vanHauser\" Heuse <mh@mh-sec.de>\n"); - - } else - - be_quiet = 1; - - use_threadsafe_counters = getenv("AFL_LLVM_THREADSAFE_INST"); - - if ((ptr = getenv("AFL_LLVM_DOCUMENT_IDS")) != NULL) { - - if ((documentFile = fopen(ptr, "a")) == NULL) - WARNF("Cannot access document file %s", ptr); - - } - - // we make this the default as the fixed map has problems with - // defered forkserver, early constructors, ifuncs and maybe more - /*if (getenv("AFL_LLVM_MAP_DYNAMIC"))*/ - map_addr = 0; - - if ((ptr = getenv("AFL_LLVM_MAP_ADDR"))) { - - uint64_t val; - if (!*ptr || !strcmp(ptr, "0") || !strcmp(ptr, "0x0")) { - - map_addr = 0; - - } else if (getenv("AFL_LLVM_MAP_DYNAMIC")) { - - FATAL( - "AFL_LLVM_MAP_ADDR and AFL_LLVM_MAP_DYNAMIC cannot be used together"); - - } else if (strncmp(ptr, "0x", 2) != 0) { - - map_addr = 0x10000; // the default - - } else { - - val = strtoull(ptr, NULL, 16); - if (val < 0x100 || val > 0xffffffff00000000) { - - FATAL( - "AFL_LLVM_MAP_ADDR must be a value between 0x100 and " - "0xffffffff00000000"); - - } - - map_addr = val; - - } - - } - - if (debug) { fprintf(stderr, "map address is 0x%llx\n", map_addr); } - - /* Get/set the globals for the SHM region. */ - - GlobalVariable *AFLMapPtr = NULL; - Value * MapPtrFixed = NULL; - - if (!map_addr) { - - AFLMapPtr = - new GlobalVariable(M, PointerType::get(Int8Ty, 0), false, - GlobalValue::ExternalLinkage, 0, "__afl_area_ptr"); - - } else { - - ConstantInt *MapAddr = ConstantInt::get(Int64Ty, map_addr); - MapPtrFixed = - ConstantExpr::getIntToPtr(MapAddr, PointerType::getUnqual(Int8Ty)); - - } - - ConstantInt *Zero = ConstantInt::get(Int8Ty, 0); - ConstantInt *One = ConstantInt::get(Int8Ty, 1); - - // This dumps all inialized global strings - might be useful in the future - /* - for (auto G=M.getGlobalList().begin(); G!=M.getGlobalList().end(); G++) { - - GlobalVariable &GV=*G; - if (!GV.getName().str().empty()) { - - fprintf(stderr, "Global Variable: %s", GV.getName().str().c_str()); - if (GV.hasInitializer()) - if (auto *Val = dyn_cast<ConstantDataArray>(GV.getInitializer())) - fprintf(stderr, " Value: \"%s\"", Val->getAsString().str().c_str()); - fprintf(stderr, "\n"); - - } - - } - - */ - - scanForDangerousFunctions(&M); - - /* Instrument all the things! */ - - int inst_blocks = 0; - - for (auto &F : M) { - - /*For debugging - AttributeSet X = F.getAttributes().getFnAttributes(); - fprintf(stderr, "DEBUG: Module %s Function %s attributes %u\n", - M.getName().str().c_str(), F.getName().str().c_str(), - X.getNumAttributes()); - */ - - if (F.size() < function_minimum_size) continue; - if (isIgnoreFunction(&F)) continue; - - // the instrument file list check - AttributeList Attrs = F.getAttributes(); - if (Attrs.hasAttribute(-1, StringRef("skipinstrument"))) { - - if (debug) - fprintf(stderr, - "DEBUG: Function %s is not in a source file that was specified " - "in the instrument file list\n", - F.getName().str().c_str()); - continue; - - } - - std::vector<BasicBlock *> InsBlocks; - - if (autodictionary) { - - /* Some implementation notes. - * - * We try to handle 3 cases: - * - memcmp("foo", arg, 3) <- literal string - * - static char globalvar[] = "foo"; - * memcmp(globalvar, arg, 3) <- global variable - * - char localvar[] = "foo"; - * memcmp(locallvar, arg, 3) <- local variable - * - * The local variable case is the hardest. We can only detect that - * case if there is no reassignment or change in the variable. - * And it might not work across llvm version. - * What we do is hooking the initializer function for local variables - * (llvm.memcpy.p0i8.p0i8.i64) and note the string and the assigned - * variable. And if that variable is then used in a compare function - * we use that noted string. - * This seems not to work for tokens that have a size <= 4 :-( - * - * - if the compared length is smaller than the string length we - * save the full string. This is likely better for fuzzing but - * might be wrong in a few cases depending on optimizers - * - * - not using StringRef because there is a bug in the llvm 11 - * checkout I am using which sometimes points to wrong strings - * - * Over and out. Took me a full day. damn. mh/vh - */ - - for (auto &BB : F) { - - for (auto &IN : BB) { - - CallInst *callInst = nullptr; - CmpInst * cmpInst = nullptr; - - if ((cmpInst = dyn_cast<CmpInst>(&IN))) { - - Value * op = cmpInst->getOperand(1); - ConstantInt *ilen = dyn_cast<ConstantInt>(op); - - if (ilen && ilen->uge(0xffffffffffffffff) == false) { - - u64 val2 = 0, val = ilen->getZExtValue(); - u32 len = 0; - if (val > 0x10000 && val < 0xffffffff) len = 4; - if (val > 0x100000001 && val < 0xffffffffffffffff) len = 8; - - if (len) { - - auto c = cmpInst->getPredicate(); - - switch (c) { - - case CmpInst::FCMP_OGT: // fall through - case CmpInst::FCMP_OLE: // fall through - case CmpInst::ICMP_SLE: // fall through - case CmpInst::ICMP_SGT: - - // signed comparison and it is a negative constant - if ((len == 4 && (val & 80000000)) || - (len == 8 && (val & 8000000000000000))) { - - if ((val & 0xffff) != 1) val2 = val - 1; - break; - - } - - // fall through - - case CmpInst::FCMP_UGT: // fall through - case CmpInst::FCMP_ULE: // fall through - case CmpInst::ICMP_UGT: // fall through - case CmpInst::ICMP_ULE: - if ((val & 0xffff) != 0xfffe) val2 = val + 1; - break; - - case CmpInst::FCMP_OLT: // fall through - case CmpInst::FCMP_OGE: // fall through - case CmpInst::ICMP_SLT: // fall through - case CmpInst::ICMP_SGE: - - // signed comparison and it is a negative constant - if ((len == 4 && (val & 80000000)) || - (len == 8 && (val & 8000000000000000))) { - - if ((val & 0xffff) != 1) val2 = val - 1; - break; - - } - - // fall through - - case CmpInst::FCMP_ULT: // fall through - case CmpInst::FCMP_UGE: // fall through - case CmpInst::ICMP_ULT: // fall through - case CmpInst::ICMP_UGE: - if ((val & 0xffff) != 1) val2 = val - 1; - break; - - default: - val2 = 0; - - } - - dictionary.push_back(std::string((char *)&val, len)); - found++; - - if (val2) { - - dictionary.push_back(std::string((char *)&val2, len)); - found++; - - } - - } - - } - - } - - if ((callInst = dyn_cast<CallInst>(&IN))) { - - bool isStrcmp = true; - bool isMemcmp = true; - bool isStrncmp = true; - bool isStrcasecmp = true; - bool isStrncasecmp = true; - bool isIntMemcpy = true; - bool isStdString = true; - bool addedNull = false; - size_t optLen = 0; - - Function *Callee = callInst->getCalledFunction(); - if (!Callee) continue; - if (callInst->getCallingConv() != llvm::CallingConv::C) continue; - std::string FuncName = Callee->getName().str(); - - isStrcmp &= (!FuncName.compare("strcmp") || - !FuncName.compare("xmlStrcmp") || - !FuncName.compare("xmlStrEqual") || - !FuncName.compare("g_strcmp0") || - !FuncName.compare("curl_strequal") || - !FuncName.compare("strcsequal")); - isMemcmp &= - (!FuncName.compare("memcmp") || !FuncName.compare("bcmp") || - !FuncName.compare("CRYPTO_memcmp") || - !FuncName.compare("OPENSSL_memcmp") || - !FuncName.compare("memcmp_const_time") || - !FuncName.compare("memcmpct")); - isStrncmp &= (!FuncName.compare("strncmp") || - !FuncName.compare("xmlStrncmp") || - !FuncName.compare("curl_strnequal")); - isStrcasecmp &= (!FuncName.compare("strcasecmp") || - !FuncName.compare("stricmp") || - !FuncName.compare("ap_cstr_casecmp") || - !FuncName.compare("OPENSSL_strcasecmp") || - !FuncName.compare("xmlStrcasecmp") || - !FuncName.compare("g_strcasecmp") || - !FuncName.compare("g_ascii_strcasecmp") || - !FuncName.compare("Curl_strcasecompare") || - !FuncName.compare("Curl_safe_strcasecompare") || - !FuncName.compare("cmsstrcasecmp")); - isStrncasecmp &= (!FuncName.compare("strncasecmp") || - !FuncName.compare("strnicmp") || - !FuncName.compare("ap_cstr_casecmpn") || - !FuncName.compare("OPENSSL_strncasecmp") || - !FuncName.compare("xmlStrncasecmp") || - !FuncName.compare("g_ascii_strncasecmp") || - !FuncName.compare("Curl_strncasecompare") || - !FuncName.compare("g_strncasecmp")); - isIntMemcpy &= !FuncName.compare("llvm.memcpy.p0i8.p0i8.i64"); - isStdString &= - ((FuncName.find("basic_string") != std::string::npos && - FuncName.find("compare") != std::string::npos) || - (FuncName.find("basic_string") != std::string::npos && - FuncName.find("find") != std::string::npos)); - - /* we do something different here, putting this BB and the - successors in a block map */ - if (!FuncName.compare("__afl_persistent_loop")) { - - BlockList.push_back(&BB); - /* - for (succ_iterator SI = succ_begin(&BB), SE = - succ_end(&BB); SI != SE; ++SI) { - - BasicBlock *succ = *SI; - BlockList.push_back(succ); - - } - - */ - - } - - if (!isStrcmp && !isMemcmp && !isStrncmp && !isStrcasecmp && - !isStrncasecmp && !isIntMemcpy && !isStdString) - continue; - - /* Verify the strcmp/memcmp/strncmp/strcasecmp/strncasecmp function - * prototype */ - FunctionType *FT = Callee->getFunctionType(); - - isStrcmp &= FT->getNumParams() == 2 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()); - isStrcasecmp &= FT->getNumParams() == 2 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()); - isMemcmp &= FT->getNumParams() == 3 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0)->isPointerTy() && - FT->getParamType(1)->isPointerTy() && - FT->getParamType(2)->isIntegerTy(); - isStrncmp &= FT->getNumParams() == 3 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()) && - FT->getParamType(2)->isIntegerTy(); - isStrncasecmp &= FT->getNumParams() == 3 && - FT->getReturnType()->isIntegerTy(32) && - FT->getParamType(0) == FT->getParamType(1) && - FT->getParamType(0) == - IntegerType::getInt8PtrTy(M.getContext()) && - FT->getParamType(2)->isIntegerTy(); - isStdString &= FT->getNumParams() >= 2 && - FT->getParamType(0)->isPointerTy() && - FT->getParamType(1)->isPointerTy(); - - if (!isStrcmp && !isMemcmp && !isStrncmp && !isStrcasecmp && - !isStrncasecmp && !isIntMemcpy && !isStdString) - continue; - - /* is a str{n,}{case,}cmp/memcmp, check if we have - * str{case,}cmp(x, "const") or str{case,}cmp("const", x) - * strn{case,}cmp(x, "const", ..) or strn{case,}cmp("const", x, ..) - * memcmp(x, "const", ..) or memcmp("const", x, ..) */ - Value *Str1P = callInst->getArgOperand(0), - *Str2P = callInst->getArgOperand(1); - std::string Str1, Str2; - StringRef TmpStr; - bool HasStr1; - getConstantStringInfo(Str1P, TmpStr); - if (TmpStr.empty()) { - - HasStr1 = false; - - } else { - - HasStr1 = true; - Str1 = TmpStr.str(); - - } - - bool HasStr2; - getConstantStringInfo(Str2P, TmpStr); - if (TmpStr.empty()) { - - HasStr2 = false; - - } else { - - HasStr2 = true; - Str2 = TmpStr.str(); - - } - - if (debug) - fprintf(stderr, "F:%s %p(%s)->\"%s\"(%s) %p(%s)->\"%s\"(%s)\n", - FuncName.c_str(), Str1P, Str1P->getName().str().c_str(), - Str1.c_str(), HasStr1 == true ? "true" : "false", Str2P, - Str2P->getName().str().c_str(), Str2.c_str(), - HasStr2 == true ? "true" : "false"); - - // we handle the 2nd parameter first because of llvm memcpy - if (!HasStr2) { - - auto *Ptr = dyn_cast<ConstantExpr>(Str2P); - if (Ptr && Ptr->isGEPWithNoNotionalOverIndexing()) { - - if (auto *Var = dyn_cast<GlobalVariable>(Ptr->getOperand(0))) { - - if (Var->hasInitializer()) { - - if (auto *Array = dyn_cast<ConstantDataArray>( - Var->getInitializer())) { - - HasStr2 = true; - Str2 = Array->getRawDataValues().str(); - - } - - } - - } - - } - - } - - // for the internal memcpy routine we only care for the second - // parameter and are not reporting anything. - if (isIntMemcpy == true) { - - if (HasStr2 == true) { - - Value * op2 = callInst->getArgOperand(2); - ConstantInt *ilen = dyn_cast<ConstantInt>(op2); - if (ilen) { - - uint64_t literalLength = Str2.size(); - uint64_t optLength = ilen->getZExtValue(); - if (optLength > literalLength + 1) { - - optLength = Str2.length() + 1; - - } - - if (literalLength + 1 == optLength) { - - Str2.append("\0", 1); // add null byte - // addedNull = true; - - } - - } - - valueMap[Str1P] = new std::string(Str2); - - if (debug) - fprintf(stderr, "Saved: %s for %p\n", Str2.c_str(), Str1P); - continue; - - } - - continue; - - } - - // Neither a literal nor a global variable? - // maybe it is a local variable that we saved - if (!HasStr2) { - - std::string *strng = valueMap[Str2P]; - if (strng && !strng->empty()) { - - Str2 = *strng; - HasStr2 = true; - if (debug) - fprintf(stderr, "Filled2: %s for %p\n", strng->c_str(), - Str2P); - - } - - } - - if (!HasStr1) { - - auto Ptr = dyn_cast<ConstantExpr>(Str1P); - - if (Ptr && Ptr->isGEPWithNoNotionalOverIndexing()) { - - if (auto *Var = dyn_cast<GlobalVariable>(Ptr->getOperand(0))) { - - if (Var->hasInitializer()) { - - if (auto *Array = dyn_cast<ConstantDataArray>( - Var->getInitializer())) { - - HasStr1 = true; - Str1 = Array->getRawDataValues().str(); - - } - - } - - } - - } - - } - - // Neither a literal nor a global variable? - // maybe it is a local variable that we saved - if (!HasStr1) { - - std::string *strng = valueMap[Str1P]; - if (strng && !strng->empty()) { - - Str1 = *strng; - HasStr1 = true; - if (debug) - fprintf(stderr, "Filled1: %s for %p\n", strng->c_str(), - Str1P); - - } - - } - - /* handle cases of one string is const, one string is variable */ - if (!(HasStr1 ^ HasStr2)) continue; - - std::string thestring; - - if (HasStr1) - thestring = Str1; - else - thestring = Str2; - - optLen = thestring.length(); - if (optLen < 2 || (optLen == 2 && !thestring[1])) { continue; } - - if (isMemcmp || isStrncmp || isStrncasecmp) { - - Value * op2 = callInst->getArgOperand(2); - ConstantInt *ilen = dyn_cast<ConstantInt>(op2); - - if (ilen) { - - uint64_t literalLength = optLen; - optLen = ilen->getZExtValue(); - if (optLen > literalLength + 1) { optLen = literalLength + 1; } - if (optLen < 2) { continue; } - if (literalLength + 1 == optLen) { // add null byte - thestring.append("\0", 1); - addedNull = true; - - } - - } - - } - - // add null byte if this is a string compare function and a null - // was not already added - if (!isMemcmp) { - - if (addedNull == false && thestring[optLen - 1] != '\0') { - - thestring.append("\0", 1); // add null byte - optLen++; - - } - - if (!isStdString) { - - // ensure we do not have garbage - size_t offset = thestring.find('\0', 0); - if (offset + 1 < optLen) optLen = offset + 1; - thestring = thestring.substr(0, optLen); - - } - - } - - if (!be_quiet) { - - fprintf(stderr, "%s: length %zu/%zu \"", FuncName.c_str(), optLen, - thestring.length()); - for (uint8_t i = 0; i < thestring.length(); i++) { - - uint8_t c = thestring[i]; - if (c <= 32 || c >= 127) - fprintf(stderr, "\\x%02x", c); - else - fprintf(stderr, "%c", c); - - } - - fprintf(stderr, "\"\n"); - - } - - // we take the longer string, even if the compare was to a - // shorter part. Note that depending on the optimizer of the - // compiler this can be wrong, but it is more likely that this - // is helping the fuzzer - if (optLen != thestring.length()) optLen = thestring.length(); - if (optLen > MAX_AUTO_EXTRA) optLen = MAX_AUTO_EXTRA; - if (optLen < MIN_AUTO_EXTRA) // too short? skip - continue; - - dictionary.push_back(thestring.substr(0, optLen)); - - } - - } - - } - - } - - for (auto &BB : F) { - - if (F.size() == 1) { - - InsBlocks.push_back(&BB); - continue; - - } - - uint32_t succ = 0; - for (succ_iterator SI = succ_begin(&BB), SE = succ_end(&BB); SI != SE; - ++SI) - if ((*SI)->size() > 0) succ++; - if (succ < 2) // no need to instrument - continue; - - if (BlockList.size()) { - - int skip = 0; - for (uint32_t k = 0; k < BlockList.size(); k++) { - - if (&BB == BlockList[k]) { - - if (debug) - fprintf(stderr, - "DEBUG: Function %s skipping BB with/after __afl_loop\n", - F.getName().str().c_str()); - skip = 1; - - } - - } - - if (skip) continue; - - } - - InsBlocks.push_back(&BB); - - } - - if (InsBlocks.size() > 0) { - - uint32_t i = InsBlocks.size(); - - do { - - --i; - BasicBlock * newBB = NULL; - BasicBlock * origBB = &(*InsBlocks[i]); - std::vector<BasicBlock *> Successors; - Instruction * TI = origBB->getTerminator(); - uint32_t fs = origBB->getParent()->size(); - uint32_t countto; - - for (succ_iterator SI = succ_begin(origBB), SE = succ_end(origBB); - SI != SE; ++SI) { - - BasicBlock *succ = *SI; - Successors.push_back(succ); - - } - - if (fs == 1) { - - newBB = origBB; - countto = 1; - - } else { - - if (TI == NULL || TI->getNumSuccessors() < 2) continue; - countto = Successors.size(); - - } - - // if (Successors.size() != TI->getNumSuccessors()) - // FATAL("Different successor numbers %lu <-> %u\n", Successors.size(), - // TI->getNumSuccessors()); - - for (uint32_t j = 0; j < countto; j++) { - - if (fs != 1) newBB = llvm::SplitEdge(origBB, Successors[j]); - - if (!newBB) { - - if (!be_quiet) WARNF("Split failed!"); - continue; - - } - - if (documentFile) { - - fprintf(documentFile, "ModuleID=%llu Function=%s edgeID=%u\n", - moduleID, F.getName().str().c_str(), afl_global_id); - - } - - BasicBlock::iterator IP = newBB->getFirstInsertionPt(); - IRBuilder<> IRB(&(*IP)); - - /* Set the ID of the inserted basic block */ - - ConstantInt *CurLoc = ConstantInt::get(Int32Ty, afl_global_id++); - - /* Load SHM pointer */ - - Value *MapPtrIdx; - - if (map_addr) { - - MapPtrIdx = IRB.CreateGEP(MapPtrFixed, CurLoc); - - } else { - - LoadInst *MapPtr = IRB.CreateLoad(AFLMapPtr); - MapPtr->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - MapPtrIdx = IRB.CreateGEP(MapPtr, CurLoc); - - } - - /* Update bitmap */ - - if (use_threadsafe_counters) { - - IRB.CreateAtomicRMW(llvm::AtomicRMWInst::BinOp::Add, MapPtrIdx, One, -#if LLVM_VERSION_MAJOR >= 13 - llvm::MaybeAlign(1), -#endif - llvm::AtomicOrdering::Monotonic); - - } else { - - LoadInst *Counter = IRB.CreateLoad(MapPtrIdx); - Counter->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - - Value *Incr = IRB.CreateAdd(Counter, One); - - if (skip_nozero == NULL) { - - auto cf = IRB.CreateICmpEQ(Incr, Zero); - auto carry = IRB.CreateZExt(cf, Int8Ty); - Incr = IRB.CreateAdd(Incr, carry); - - } - - IRB.CreateStore(Incr, MapPtrIdx) - ->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - - } - - // done :) - - inst_blocks++; - - } - - } while (i > 0); - - } - - } - - if (documentFile) fclose(documentFile); - documentFile = NULL; - - // save highest location ID to global variable - // do this after each function to fail faster - if (!be_quiet && afl_global_id > MAP_SIZE && - afl_global_id > FS_OPT_MAX_MAPSIZE) { - - uint32_t pow2map = 1, map = afl_global_id; - while ((map = map >> 1)) - pow2map++; - WARNF( - "We have %u blocks to instrument but the map size is only %u. Either " - "edit config.h and set MAP_SIZE_POW2 from %d to %u, then recompile " - "afl-fuzz and llvm_mode and then make this target - or set " - "AFL_MAP_SIZE with at least size %u when running afl-fuzz with this " - "target.", - afl_global_id, MAP_SIZE, MAP_SIZE_POW2, pow2map, afl_global_id); - - } - - if (!getenv("AFL_LLVM_LTO_DONTWRITEID") || dictionary.size() || map_addr) { - - // yes we could create our own function, insert it into ctors ... - // but this would be a pain in the butt ... so we use afl-llvm-rt-lto.o - - Function *f = M.getFunction("__afl_auto_init_globals"); - - if (!f) { - - fprintf(stderr, - "Error: init function could not be found (this should not " - "happen)\n"); - exit(-1); - - } - - BasicBlock *bb = &f->getEntryBlock(); - if (!bb) { - - fprintf(stderr, - "Error: init function does not have an EntryBlock (this should " - "not happen)\n"); - exit(-1); - - } - - BasicBlock::iterator IP = bb->getFirstInsertionPt(); - IRBuilder<> IRB(&(*IP)); - - if (map_addr) { - - GlobalVariable *AFLMapAddrFixed = new GlobalVariable( - M, Int64Ty, true, GlobalValue::ExternalLinkage, 0, "__afl_map_addr"); - ConstantInt *MapAddr = ConstantInt::get(Int64Ty, map_addr); - StoreInst * StoreMapAddr = IRB.CreateStore(MapAddr, AFLMapAddrFixed); - StoreMapAddr->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - - } - - if (getenv("AFL_LLVM_LTO_DONTWRITEID") == NULL) { - - uint32_t write_loc = (((afl_global_id + 63) >> 6) << 6); - - GlobalVariable *AFLFinalLoc = new GlobalVariable( - M, Int32Ty, true, GlobalValue::ExternalLinkage, 0, "__afl_final_loc"); - ConstantInt *const_loc = ConstantInt::get(Int32Ty, write_loc); - StoreInst * StoreFinalLoc = IRB.CreateStore(const_loc, AFLFinalLoc); - StoreFinalLoc->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - - } - - if (dictionary.size()) { - - size_t memlen = 0, count = 0; - - // sort and unique the dictionary - std::sort(dictionary.begin(), dictionary.end()); - auto last = std::unique(dictionary.begin(), dictionary.end()); - dictionary.erase(last, dictionary.end()); - - for (auto token : dictionary) { - - memlen += token.length(); - count++; - - } - - if (!be_quiet) - printf("AUTODICTIONARY: %zu string%s found\n", count, - count == 1 ? "" : "s"); - - if (count) { - - if ((ptr = (char *)malloc(memlen + count)) == NULL) { - - fprintf(stderr, "Error: malloc for %zu bytes failed!\n", - memlen + count); - exit(-1); - - } - - count = 0; - - size_t offset = 0; - for (auto token : dictionary) { - - if (offset + token.length() < 0xfffff0 && count < MAX_AUTO_EXTRAS) { - - ptr[offset++] = (uint8_t)token.length(); - memcpy(ptr + offset, token.c_str(), token.length()); - offset += token.length(); - count++; - - } - - } - - GlobalVariable *AFLDictionaryLen = - new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage, - 0, "__afl_dictionary_len"); - ConstantInt *const_len = ConstantInt::get(Int32Ty, offset); - StoreInst *StoreDictLen = IRB.CreateStore(const_len, AFLDictionaryLen); - StoreDictLen->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - - ArrayType *ArrayTy = ArrayType::get(IntegerType::get(C, 8), offset); - GlobalVariable *AFLInternalDictionary = new GlobalVariable( - M, ArrayTy, true, GlobalValue::ExternalLinkage, - ConstantDataArray::get(C, - *(new ArrayRef<char>((char *)ptr, offset))), - "__afl_internal_dictionary"); - AFLInternalDictionary->setInitializer(ConstantDataArray::get( - C, *(new ArrayRef<char>((char *)ptr, offset)))); - AFLInternalDictionary->setConstant(true); - - GlobalVariable *AFLDictionary = new GlobalVariable( - M, PointerType::get(Int8Ty, 0), false, GlobalValue::ExternalLinkage, - 0, "__afl_dictionary"); - - Value *AFLDictOff = IRB.CreateGEP(AFLInternalDictionary, Zero); - Value *AFLDictPtr = - IRB.CreatePointerCast(AFLDictOff, PointerType::get(Int8Ty, 0)); - StoreInst *StoreDict = IRB.CreateStore(AFLDictPtr, AFLDictionary); - StoreDict->setMetadata(M.getMDKindID("nosanitize"), - MDNode::get(C, None)); - - } - - } - - } - - /* Say something nice. */ - - if (!be_quiet) { - - if (!inst_blocks) - WARNF("No instrumentation targets found."); - else { - - char modeline[100]; - snprintf(modeline, sizeof(modeline), "%s%s%s%s%s", - getenv("AFL_HARDEN") ? "hardened" : "non-hardened", - getenv("AFL_USE_ASAN") ? ", ASAN" : "", - getenv("AFL_USE_MSAN") ? ", MSAN" : "", - getenv("AFL_USE_CFISAN") ? ", CFISAN" : "", - getenv("AFL_USE_UBSAN") ? ", UBSAN" : ""); - OKF("Instrumented %d locations with no collisions (on average %llu " - "collisions would be in afl-gcc/vanilla AFL) (%s mode).", - inst_blocks, calculateCollisions(inst_blocks), modeline); - - } - - } - - return true; - -} - -char AFLLTOPass::ID = 0; - -static void registerAFLLTOPass(const PassManagerBuilder &, - legacy::PassManagerBase &PM) { - - PM.add(new AFLLTOPass()); - -} - -static RegisterPass<AFLLTOPass> X("afl-lto", "afl++ LTO instrumentation pass", - false, false); - -static RegisterStandardPasses RegisterAFLLTOPass( - PassManagerBuilder::EP_FullLinkTimeOptimizationLast, registerAFLLTOPass); - diff --git a/instrumentation/afl-llvm-lto-instrumentlist.so.cc b/instrumentation/afl-llvm-lto-instrumentlist.so.cc index 416dbb88..906af879 100644 --- a/instrumentation/afl-llvm-lto-instrumentlist.so.cc +++ b/instrumentation/afl-llvm-lto-instrumentlist.so.cc @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This library is plugged into LLVM when invoking clang through afl-clang-fast. It tells the compiler to add code roughly equivalent to the bits discussed @@ -116,10 +116,15 @@ bool AFLcheckIfInstrument::runOnModule(Module &M) { auto & Ctx = F.getContext(); AttributeList Attrs = F.getAttributes(); - AttrBuilder NewAttrs; +#if LLVM_VERSION_MAJOR >= 14 + AttributeList NewAttrs = Attrs.addFnAttribute(Ctx, "skipinstrument"); + F.setAttributes(NewAttrs); +#else + AttrBuilder NewAttrs; NewAttrs.addAttribute("skipinstrument"); F.setAttributes( Attrs.addAttributes(Ctx, AttributeList::FunctionIndex, NewAttrs)); +#endif } diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc index ecf28f31..8e22fde8 100644 --- a/instrumentation/afl-llvm-pass.so.cc +++ b/instrumentation/afl-llvm-pass.so.cc @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This library is plugged into LLVM when invoking clang through afl-clang-fast. It tells the compiler to add code roughly equivalent to the bits discussed @@ -52,7 +52,7 @@ typedef long double max_align_t; #include "llvm/Support/MathExtras.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/DebugInfo.h" #include "llvm/IR/CFG.h" @@ -114,7 +114,7 @@ uint64_t PowerOf2Ceil(unsigned in) { #endif /* #if LLVM_VERSION_STRING >= "4.0.1" */ -#if LLVM_VERSION_MAJOR > 4 || \ +#if LLVM_VERSION_MAJOR >= 5 || \ (LLVM_VERSION_MAJOR == 4 && LLVM_VERSION_PATCH >= 1) #define AFL_HAVE_VECTOR_INTRINSICS 1 #endif @@ -662,22 +662,7 @@ bool AFLCoverage::runOnModule(Module &M) { /* Update bitmap */ if (use_threadsafe_counters) { /* Atomic */ - /* - #if LLVM_VERSION_MAJOR < 9 - if (neverZero_counters_str != - NULL) { // with llvm 9 we make this the default as the bug - in llvm - // is then fixed - #else - if (!skip_nozero) { - - #endif - // register MapPtrIdx in a todo list - todo.push_back(MapPtrIdx); - - } else { - - */ + IRB.CreateAtomicRMW(llvm::AtomicRMWInst::BinOp::Add, MapPtrIdx, One, #if LLVM_VERSION_MAJOR >= 13 llvm::MaybeAlign(1), @@ -696,13 +681,12 @@ bool AFLCoverage::runOnModule(Module &M) { Value *Incr = IRB.CreateAdd(Counter, One); -#if LLVM_VERSION_MAJOR < 9 - if (neverZero_counters_str != - NULL) { // with llvm 9 we make this the default as the bug in llvm - // is then fixed -#else +#if LLVM_VERSION_MAJOR >= 9 if (!skip_nozero) { +#else + if (neverZero_counters_str != NULL) { + #endif /* hexcoder: Realize a counter that skips zero during overflow. * Once this counter reaches its maximum value, it next increments to @@ -956,11 +940,12 @@ bool AFLCoverage::runOnModule(Module &M) { else { char modeline[100]; - snprintf(modeline, sizeof(modeline), "%s%s%s%s%s", + snprintf(modeline, sizeof(modeline), "%s%s%s%s%s%s", getenv("AFL_HARDEN") ? "hardened" : "non-hardened", getenv("AFL_USE_ASAN") ? ", ASAN" : "", getenv("AFL_USE_MSAN") ? ", MSAN" : "", getenv("AFL_USE_CFISAN") ? ", CFISAN" : "", + getenv("AFL_USE_TSAN") ? ", TSAN" : "", getenv("AFL_USE_UBSAN") ? ", UBSAN" : ""); OKF("Instrumented %d locations (%s mode, ratio %u%%).", inst_blocks, modeline, inst_ratio); diff --git a/instrumentation/afl-llvm-rt-lto.o.c b/instrumentation/afl-llvm-rt-lto.o.c index e53785ff..eb346157 100644 --- a/instrumentation/afl-llvm-rt-lto.o.c +++ b/instrumentation/afl-llvm-rt-lto.o.c @@ -6,7 +6,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ diff --git a/instrumentation/cmplog-instructions-pass.cc b/instrumentation/cmplog-instructions-pass.cc index 0562c5b2..07f80b2c 100644 --- a/instrumentation/cmplog-instructions-pass.cc +++ b/instrumentation/cmplog-instructions-pass.cc @@ -11,7 +11,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ @@ -37,7 +37,7 @@ #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/Verifier.h" #include "llvm/IR/DebugInfo.h" @@ -66,11 +66,11 @@ class CmpLogInstructions : public ModulePass { bool runOnModule(Module &M) override; -#if LLVM_VERSION_MAJOR < 4 - const char *getPassName() const override { +#if LLVM_VERSION_MAJOR >= 4 + StringRef getPassName() const override { #else - StringRef getPassName() const override { + const char *getPassName() const override { #endif return "cmplog instructions"; @@ -113,10 +113,10 @@ bool CmpLogInstructions::hookInstrs(Module &M) { IntegerType *Int64Ty = IntegerType::getInt64Ty(C); IntegerType *Int128Ty = IntegerType::getInt128Ty(C); -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c1 = M.getOrInsertFunction("__cmplog_ins_hook1", VoidTy, Int8Ty, Int8Ty, Int8Ty @@ -125,16 +125,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns1 = cast<Function>(c1); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns1 = c1; +#else + Function *cmplogHookIns1 = cast<Function>(c1); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c2 = M.getOrInsertFunction("__cmplog_ins_hook2", VoidTy, Int16Ty, Int16Ty, Int8Ty @@ -143,16 +143,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns2 = cast<Function>(c2); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns2 = c2; +#else + Function *cmplogHookIns2 = cast<Function>(c2); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c4 = M.getOrInsertFunction("__cmplog_ins_hook4", VoidTy, Int32Ty, Int32Ty, Int8Ty @@ -161,16 +161,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns4 = cast<Function>(c4); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns4 = c4; +#else + Function *cmplogHookIns4 = cast<Function>(c4); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c8 = M.getOrInsertFunction("__cmplog_ins_hook8", VoidTy, Int64Ty, Int64Ty, Int8Ty @@ -179,16 +179,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns8 = cast<Function>(c8); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns8 = c8; +#else + Function *cmplogHookIns8 = cast<Function>(c8); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c16 = M.getOrInsertFunction("__cmplog_ins_hook16", VoidTy, Int128Ty, Int128Ty, Int8Ty @@ -203,10 +203,10 @@ bool CmpLogInstructions::hookInstrs(Module &M) { FunctionCallee cmplogHookIns16 = c16; #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif cN = M.getOrInsertFunction("__cmplog_ins_hookN", VoidTy, Int128Ty, Int128Ty, Int8Ty, Int8Ty @@ -215,10 +215,10 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookInsN = cast<Function>(cN); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookInsN = cN; +#else + Function *cmplogHookInsN = cast<Function>(cN); #endif GlobalVariable *AFLCmplogPtr = M.getNamedGlobal("__afl_cmp_map"); @@ -274,14 +274,15 @@ bool CmpLogInstructions::hookInstrs(Module &M) { Value *op0 = selectcmpInst->getOperand(0); Value *op1 = selectcmpInst->getOperand(1); + Value *op0_saved = op0, *op1_saved = op1; + auto ty0 = op0->getType(); + auto ty1 = op1->getType(); - IntegerType * intTyOp0 = NULL; - IntegerType * intTyOp1 = NULL; - unsigned max_size = 0, cast_size = 0; - unsigned char attr = 0; - std::vector<Value *> args; - - CmpInst *cmpInst = dyn_cast<CmpInst>(selectcmpInst); + IntegerType *intTyOp0 = NULL; + IntegerType *intTyOp1 = NULL; + unsigned max_size = 0, cast_size = 0; + unsigned attr = 0, vector_cnt = 0; + CmpInst * cmpInst = dyn_cast<CmpInst>(selectcmpInst); if (!cmpInst) { continue; } @@ -327,7 +328,23 @@ bool CmpLogInstructions::hookInstrs(Module &M) { if (selectcmpInst->getOpcode() == Instruction::FCmp) { - auto ty0 = op0->getType(); + if (ty0->isVectorTy()) { + + VectorType *tt = dyn_cast<VectorType>(ty0); + if (!tt) { + + fprintf(stderr, "Warning: cmplog cmp vector is not a vector!\n"); + continue; + + } + +#if (LLVM_VERSION_MAJOR >= 12) + vector_cnt = tt->getElementCount().getKnownMinValue(); + ty0 = tt->getElementType(); +#endif + + } + if (ty0->isHalfTy() #if LLVM_VERSION_MAJOR >= 11 || ty0->isBFloatTy() @@ -342,13 +359,35 @@ bool CmpLogInstructions::hookInstrs(Module &M) { max_size = 80; else if (ty0->isFP128Ty() || ty0->isPPC_FP128Ty()) max_size = 128; +#if (LLVM_VERSION_MAJOR >= 12) + else if (ty0->getTypeID() != llvm::Type::PointerTyID && !be_quiet) + fprintf(stderr, "Warning: unsupported cmp type for cmplog: %u!\n", + ty0->getTypeID()); +#endif attr += 8; } else { - intTyOp0 = dyn_cast<IntegerType>(op0->getType()); - intTyOp1 = dyn_cast<IntegerType>(op1->getType()); + if (ty0->isVectorTy()) { + +#if (LLVM_VERSION_MAJOR >= 12) + VectorType *tt = dyn_cast<VectorType>(ty0); + if (!tt) { + + fprintf(stderr, "Warning: cmplog cmp vector is not a vector!\n"); + continue; + + } + + vector_cnt = tt->getElementCount().getKnownMinValue(); + ty1 = ty0 = tt->getElementType(); +#endif + + } + + intTyOp0 = dyn_cast<IntegerType>(ty0); + intTyOp1 = dyn_cast<IntegerType>(ty1); if (intTyOp0 && intTyOp1) { @@ -356,11 +395,28 @@ bool CmpLogInstructions::hookInstrs(Module &M) { ? intTyOp0->getBitWidth() : intTyOp1->getBitWidth(); + } else { + +#if (LLVM_VERSION_MAJOR >= 12) + if (ty0->getTypeID() != llvm::Type::PointerTyID && !be_quiet) { + + fprintf(stderr, "Warning: unsupported cmp type for cmplog: %u\n", + ty0->getTypeID()); + + } + +#endif + } } - if (!max_size || max_size < 16) { continue; } + if (!max_size || max_size < 16) { + + // fprintf(stderr, "too small\n"); + continue; + + } if (max_size % 8) { max_size = (((max_size / 8) + 1) * 8); } @@ -393,67 +449,110 @@ bool CmpLogInstructions::hookInstrs(Module &M) { } - // errs() << "[CMPLOG] cmp " << *cmpInst << "(in function " << - // cmpInst->getFunction()->getName() << ")\n"; + uint64_t cur = 0, last_val0 = 0, last_val1 = 0, cur_val; - // first bitcast to integer type of the same bitsize as the original - // type (this is a nop, if already integer) - Value *op0_i = IRB.CreateBitCast( - op0, IntegerType::get(C, op0->getType()->getPrimitiveSizeInBits())); - // then create a int cast, which does zext, trunc or bitcast. In our case - // usually zext to the next larger supported type (this is a nop if - // already the right type) - Value *V0 = - IRB.CreateIntCast(op0_i, IntegerType::get(C, cast_size), false); - args.push_back(V0); - Value *op1_i = IRB.CreateBitCast( - op1, IntegerType::get(C, op1->getType()->getPrimitiveSizeInBits())); - Value *V1 = - IRB.CreateIntCast(op1_i, IntegerType::get(C, cast_size), false); - args.push_back(V1); + while (1) { - // errs() << "[CMPLOG] casted parameters:\n0: " << *V0 << "\n1: " << *V1 - // << "\n"; + std::vector<Value *> args; + uint32_t skip = 0; - ConstantInt *attribute = ConstantInt::get(Int8Ty, attr); - args.push_back(attribute); + if (vector_cnt) { - if (cast_size != max_size) { + op0 = IRB.CreateExtractElement(op0_saved, cur); + op1 = IRB.CreateExtractElement(op1_saved, cur); + ConstantInt *i0 = dyn_cast<ConstantInt>(op0); + ConstantInt *i1 = dyn_cast<ConstantInt>(op1); + if (i0 && i0->uge(0xffffffffffffffff) == false) { - ConstantInt *bitsize = ConstantInt::get(Int8Ty, (max_size / 8) - 1); - args.push_back(bitsize); + cur_val = i0->getZExtValue(); + if (last_val0 && last_val0 == cur_val) { skip = 1; } + last_val0 = cur_val; - } + } - // fprintf(stderr, "_ExtInt(%u) castTo %u with attr %u didcast %u\n", - // max_size, cast_size, attr); + if (i1 && i1->uge(0xffffffffffffffff) == false) { - switch (cast_size) { + cur_val = i1->getZExtValue(); + if (last_val1 && last_val1 == cur_val) { skip = 1; } + last_val1 = cur_val; - case 8: - IRB.CreateCall(cmplogHookIns1, args); - break; - case 16: - IRB.CreateCall(cmplogHookIns2, args); - break; - case 32: - IRB.CreateCall(cmplogHookIns4, args); - break; - case 64: - IRB.CreateCall(cmplogHookIns8, args); - break; - case 128: - if (max_size == 128) { + } + + } + + if (!skip) { + + // errs() << "[CMPLOG] cmp " << *cmpInst << "(in function " << + // cmpInst->getFunction()->getName() << ")\n"; + + // first bitcast to integer type of the same bitsize as the original + // type (this is a nop, if already integer) + Value *op0_i = IRB.CreateBitCast( + op0, IntegerType::get(C, ty0->getPrimitiveSizeInBits())); + // then create a int cast, which does zext, trunc or bitcast. In our + // case usually zext to the next larger supported type (this is a nop + // if already the right type) + Value *V0 = + IRB.CreateIntCast(op0_i, IntegerType::get(C, cast_size), false); + args.push_back(V0); + Value *op1_i = IRB.CreateBitCast( + op1, IntegerType::get(C, ty1->getPrimitiveSizeInBits())); + Value *V1 = + IRB.CreateIntCast(op1_i, IntegerType::get(C, cast_size), false); + args.push_back(V1); - IRB.CreateCall(cmplogHookIns16, args); + // errs() << "[CMPLOG] casted parameters:\n0: " << *V0 << "\n1: " << + // *V1 + // << "\n"; - } else { + ConstantInt *attribute = ConstantInt::get(Int8Ty, attr); + args.push_back(attribute); - IRB.CreateCall(cmplogHookInsN, args); + if (cast_size != max_size) { + + ConstantInt *bitsize = ConstantInt::get(Int8Ty, (max_size / 8) - 1); + args.push_back(bitsize); } - break; + // fprintf(stderr, "_ExtInt(%u) castTo %u with attr %u didcast %u\n", + // max_size, cast_size, attr); + + switch (cast_size) { + + case 8: + IRB.CreateCall(cmplogHookIns1, args); + break; + case 16: + IRB.CreateCall(cmplogHookIns2, args); + break; + case 32: + IRB.CreateCall(cmplogHookIns4, args); + break; + case 64: + IRB.CreateCall(cmplogHookIns8, args); + break; + case 128: + if (max_size == 128) { + + IRB.CreateCall(cmplogHookIns16, args); + + } else { + + IRB.CreateCall(cmplogHookInsN, args); + + } + + break; + + } + + } + + /* else fprintf(stderr, "skipped\n"); */ + + ++cur; + if (cur >= vector_cnt) { break; } } diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index 1e2610f2..0565875e 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -11,7 +11,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ @@ -36,7 +36,7 @@ #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/Verifier.h" #include "llvm/IR/DebugInfo.h" @@ -65,11 +65,11 @@ class CmpLogRoutines : public ModulePass { bool runOnModule(Module &M) override; -#if LLVM_VERSION_MAJOR < 4 - const char *getPassName() const override { +#if LLVM_VERSION_MAJOR >= 4 + StringRef getPassName() const override { #else - StringRef getPassName() const override { + const char *getPassName() const override { #endif return "cmplog routines"; @@ -87,18 +87,20 @@ char CmpLogRoutines::ID = 0; bool CmpLogRoutines::hookRtns(Module &M) { - std::vector<CallInst *> calls, llvmStdStd, llvmStdC, gccStdStd, gccStdC; - LLVMContext & C = M.getContext(); + std::vector<CallInst *> calls, llvmStdStd, llvmStdC, gccStdStd, gccStdC, + Memcmp, Strcmp, Strncmp; + LLVMContext &C = M.getContext(); Type *VoidTy = Type::getVoidTy(C); // PointerType *VoidPtrTy = PointerType::get(VoidTy, 0); IntegerType *Int8Ty = IntegerType::getInt8Ty(C); + IntegerType *Int64Ty = IntegerType::getInt64Ty(C); PointerType *i8PtrTy = PointerType::get(Int8Ty, 0); -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c = M.getOrInsertFunction("__cmplog_rtn_hook", VoidTy, i8PtrTy, i8PtrTy #if LLVM_VERSION_MAJOR < 5 @@ -106,16 +108,16 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookFn = cast<Function>(c); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookFn = c; +#else + Function *cmplogHookFn = cast<Function>(c); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c1 = M.getOrInsertFunction("__cmplog_rtn_llvm_stdstring_stdstring", VoidTy, i8PtrTy, i8PtrTy @@ -124,16 +126,16 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogLlvmStdStd = cast<Function>(c1); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogLlvmStdStd = c1; +#else + Function *cmplogLlvmStdStd = cast<Function>(c1); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c2 = M.getOrInsertFunction("__cmplog_rtn_llvm_stdstring_cstring", VoidTy, i8PtrTy, i8PtrTy @@ -142,16 +144,16 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogLlvmStdC = cast<Function>(c2); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogLlvmStdC = c2; +#else + Function *cmplogLlvmStdC = cast<Function>(c2); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c3 = M.getOrInsertFunction("__cmplog_rtn_gcc_stdstring_stdstring", VoidTy, i8PtrTy, i8PtrTy @@ -160,16 +162,16 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogGccStdStd = cast<Function>(c3); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogGccStdStd = c3; +#else + Function *cmplogGccStdStd = cast<Function>(c3); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c4 = M.getOrInsertFunction("__cmplog_rtn_gcc_stdstring_cstring", VoidTy, i8PtrTy, i8PtrTy @@ -178,10 +180,64 @@ bool CmpLogRoutines::hookRtns(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee cmplogGccStdC = c4; +#else Function *cmplogGccStdC = cast<Function>(c4); +#endif + +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee #else - FunctionCallee cmplogGccStdC = c4; + Constant * +#endif + c5 = M.getOrInsertFunction("__cmplog_rtn_hook_n", VoidTy, i8PtrTy, + i8PtrTy, Int64Ty +#if LLVM_VERSION_MAJOR < 5 + , + NULL +#endif + ); +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee cmplogHookFnN = c5; +#else + Function *cmplogHookFnN = cast<Function>(c5); +#endif + +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee +#else + Constant * +#endif + c6 = M.getOrInsertFunction("__cmplog_rtn_hook_strn", VoidTy, i8PtrTy, + i8PtrTy, Int64Ty +#if LLVM_VERSION_MAJOR < 5 + , + NULL +#endif + ); +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee cmplogHookFnStrN = c6; +#else + Function *cmplogHookFnStrN = cast<Function>(c6); +#endif + +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee +#else + Constant * +#endif + c7 = M.getOrInsertFunction("__cmplog_rtn_hook_str", VoidTy, i8PtrTy, + i8PtrTy +#if LLVM_VERSION_MAJOR < 5 + , + NULL +#endif + ); +#if LLVM_VERSION_MAJOR >= 9 + FunctionCallee cmplogHookFnStr = c7; +#else + Function *cmplogHookFnStr = cast<Function>(c7); #endif GlobalVariable *AFLCmplogPtr = M.getNamedGlobal("__afl_cmp_map"); @@ -214,12 +270,93 @@ bool CmpLogRoutines::hookRtns(Module &M) { if (callInst->getCallingConv() != llvm::CallingConv::C) continue; FunctionType *FT = Callee->getFunctionType(); + std::string FuncName = Callee->getName().str(); bool isPtrRtn = FT->getNumParams() >= 2 && !FT->getReturnType()->isVoidTy() && FT->getParamType(0) == FT->getParamType(1) && FT->getParamType(0)->isPointerTy(); + bool isPtrRtnN = FT->getNumParams() >= 3 && + !FT->getReturnType()->isVoidTy() && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0)->isPointerTy() && + FT->getParamType(2)->isIntegerTy(); + if (isPtrRtnN) { + + auto intTyOp = + dyn_cast<IntegerType>(callInst->getArgOperand(2)->getType()); + if (intTyOp) { + + if (intTyOp->getBitWidth() != 32 && + intTyOp->getBitWidth() != 64) { + + isPtrRtnN = false; + + } + + } + + } + + bool isMemcmp = + (!FuncName.compare("memcmp") || !FuncName.compare("bcmp") || + !FuncName.compare("CRYPTO_memcmp") || + !FuncName.compare("OPENSSL_memcmp") || + !FuncName.compare("memcmp_const_time") || + !FuncName.compare("memcmpct")); + isMemcmp &= FT->getNumParams() == 3 && + FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0)->isPointerTy() && + FT->getParamType(1)->isPointerTy() && + FT->getParamType(2)->isIntegerTy(); + + bool isStrcmp = + (!FuncName.compare("strcmp") || !FuncName.compare("xmlStrcmp") || + !FuncName.compare("xmlStrEqual") || + !FuncName.compare("g_strcmp0") || + !FuncName.compare("curl_strequal") || + !FuncName.compare("strcsequal") || + !FuncName.compare("strcasecmp") || + !FuncName.compare("stricmp") || + !FuncName.compare("ap_cstr_casecmp") || + !FuncName.compare("OPENSSL_strcasecmp") || + !FuncName.compare("xmlStrcasecmp") || + !FuncName.compare("g_strcasecmp") || + !FuncName.compare("g_ascii_strcasecmp") || + !FuncName.compare("Curl_strcasecompare") || + !FuncName.compare("Curl_safe_strcasecompare") || + !FuncName.compare("cmsstrcasecmp") || + !FuncName.compare("strstr") || + !FuncName.compare("g_strstr_len") || + !FuncName.compare("ap_strcasestr") || + !FuncName.compare("xmlStrstr") || + !FuncName.compare("xmlStrcasestr") || + !FuncName.compare("g_str_has_prefix") || + !FuncName.compare("g_str_has_suffix")); + isStrcmp &= + FT->getNumParams() == 2 && FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == IntegerType::getInt8PtrTy(M.getContext()); + + bool isStrncmp = (!FuncName.compare("strncmp") || + !FuncName.compare("xmlStrncmp") || + !FuncName.compare("curl_strnequal") || + !FuncName.compare("strncasecmp") || + !FuncName.compare("strnicmp") || + !FuncName.compare("ap_cstr_casecmpn") || + !FuncName.compare("OPENSSL_strncasecmp") || + !FuncName.compare("xmlStrncasecmp") || + !FuncName.compare("g_ascii_strncasecmp") || + !FuncName.compare("Curl_strncasecompare") || + !FuncName.compare("g_strncasecmp")); + isStrncmp &= FT->getNumParams() == 3 && + FT->getReturnType()->isIntegerTy(32) && + FT->getParamType(0) == FT->getParamType(1) && + FT->getParamType(0) == + IntegerType::getInt8PtrTy(M.getContext()) && + FT->getParamType(2)->isIntegerTy(); + bool isGccStdStringStdString = Callee->getName().find("__is_charIT_EE7__value") != std::string::npos && @@ -267,13 +404,19 @@ bool CmpLogRoutines::hookRtns(Module &M) { */ if (isGccStdStringCString || isGccStdStringStdString || - isLlvmStdStringStdString || isLlvmStdStringCString) { + isLlvmStdStringStdString || isLlvmStdStringCString || isMemcmp || + isStrcmp || isStrncmp) { - isPtrRtn = false; + isPtrRtnN = isPtrRtn = false; } + if (isPtrRtnN) { isPtrRtn = false; } + if (isPtrRtn) { calls.push_back(callInst); } + if (isMemcmp || isPtrRtnN) { Memcmp.push_back(callInst); } + if (isStrcmp) { Strcmp.push_back(callInst); } + if (isStrncmp) { Strncmp.push_back(callInst); } if (isGccStdStringStdString) { gccStdStd.push_back(callInst); } if (isGccStdStringCString) { gccStdC.push_back(callInst); } if (isLlvmStdStringStdString) { llvmStdStd.push_back(callInst); } @@ -288,7 +431,8 @@ bool CmpLogRoutines::hookRtns(Module &M) { } if (!calls.size() && !gccStdStd.size() && !gccStdC.size() && - !llvmStdStd.size() && !llvmStdC.size()) + !llvmStdStd.size() && !llvmStdC.size() && !Memcmp.size() && + Strcmp.size() && Strncmp.size()) return false; /* @@ -323,6 +467,96 @@ bool CmpLogRoutines::hookRtns(Module &M) { } + for (auto &callInst : Memcmp) { + + Value *v1P = callInst->getArgOperand(0), *v2P = callInst->getArgOperand(1), + *v3P = callInst->getArgOperand(2); + + IRBuilder<> IRB2(callInst->getParent()); + IRB2.SetInsertPoint(callInst); + + LoadInst *CmpPtr = IRB2.CreateLoad(AFLCmplogPtr); + CmpPtr->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + auto is_not_null = IRB2.CreateICmpNE(CmpPtr, Null); + auto ThenTerm = SplitBlockAndInsertIfThen(is_not_null, callInst, false); + + IRBuilder<> IRB(ThenTerm); + + std::vector<Value *> args; + Value * v1Pcasted = IRB.CreatePointerCast(v1P, i8PtrTy); + Value * v2Pcasted = IRB.CreatePointerCast(v2P, i8PtrTy); + Value * v3Pbitcast = IRB.CreateBitCast( + v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); + Value *v3Pcasted = + IRB.CreateIntCast(v3Pbitcast, IntegerType::get(C, 64), false); + args.push_back(v1Pcasted); + args.push_back(v2Pcasted); + args.push_back(v3Pcasted); + + IRB.CreateCall(cmplogHookFnN, args); + + // errs() << callInst->getCalledFunction()->getName() << "\n"; + + } + + for (auto &callInst : Strcmp) { + + Value *v1P = callInst->getArgOperand(0), *v2P = callInst->getArgOperand(1); + + IRBuilder<> IRB2(callInst->getParent()); + IRB2.SetInsertPoint(callInst); + + LoadInst *CmpPtr = IRB2.CreateLoad(AFLCmplogPtr); + CmpPtr->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + auto is_not_null = IRB2.CreateICmpNE(CmpPtr, Null); + auto ThenTerm = SplitBlockAndInsertIfThen(is_not_null, callInst, false); + + IRBuilder<> IRB(ThenTerm); + + std::vector<Value *> args; + Value * v1Pcasted = IRB.CreatePointerCast(v1P, i8PtrTy); + Value * v2Pcasted = IRB.CreatePointerCast(v2P, i8PtrTy); + args.push_back(v1Pcasted); + args.push_back(v2Pcasted); + + IRB.CreateCall(cmplogHookFnStr, args); + + // errs() << callInst->getCalledFunction()->getName() << "\n"; + + } + + for (auto &callInst : Strncmp) { + + Value *v1P = callInst->getArgOperand(0), *v2P = callInst->getArgOperand(1), + *v3P = callInst->getArgOperand(2); + + IRBuilder<> IRB2(callInst->getParent()); + IRB2.SetInsertPoint(callInst); + + LoadInst *CmpPtr = IRB2.CreateLoad(AFLCmplogPtr); + CmpPtr->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); + auto is_not_null = IRB2.CreateICmpNE(CmpPtr, Null); + auto ThenTerm = SplitBlockAndInsertIfThen(is_not_null, callInst, false); + + IRBuilder<> IRB(ThenTerm); + + std::vector<Value *> args; + Value * v1Pcasted = IRB.CreatePointerCast(v1P, i8PtrTy); + Value * v2Pcasted = IRB.CreatePointerCast(v2P, i8PtrTy); + Value * v3Pbitcast = IRB.CreateBitCast( + v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); + Value *v3Pcasted = + IRB.CreateIntCast(v3Pbitcast, IntegerType::get(C, 64), false); + args.push_back(v1Pcasted); + args.push_back(v2Pcasted); + args.push_back(v3Pcasted); + + IRB.CreateCall(cmplogHookFnStrN, args); + + // errs() << callInst->getCalledFunction()->getName() << "\n"; + + } + for (auto &callInst : gccStdStd) { Value *v1P = callInst->getArgOperand(0), *v2P = callInst->getArgOperand(1); diff --git a/instrumentation/cmplog-switches-pass.cc b/instrumentation/cmplog-switches-pass.cc index c42d44fe..bcd5f8bd 100644 --- a/instrumentation/cmplog-switches-pass.cc +++ b/instrumentation/cmplog-switches-pass.cc @@ -11,7 +11,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ @@ -37,7 +37,7 @@ #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/Verifier.h" #include "llvm/IR/DebugInfo.h" @@ -112,10 +112,10 @@ bool CmpLogInstructions::hookInstrs(Module &M) { IntegerType *Int32Ty = IntegerType::getInt32Ty(C); IntegerType *Int64Ty = IntegerType::getInt64Ty(C); -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c1 = M.getOrInsertFunction("__cmplog_ins_hook1", VoidTy, Int8Ty, Int8Ty, Int8Ty @@ -124,16 +124,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns1 = cast<Function>(c1); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns1 = c1; +#else + Function *cmplogHookIns1 = cast<Function>(c1); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c2 = M.getOrInsertFunction("__cmplog_ins_hook2", VoidTy, Int16Ty, Int16Ty, Int8Ty @@ -142,16 +142,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns2 = cast<Function>(c2); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns2 = c2; +#else + Function *cmplogHookIns2 = cast<Function>(c2); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c4 = M.getOrInsertFunction("__cmplog_ins_hook4", VoidTy, Int32Ty, Int32Ty, Int8Ty @@ -160,16 +160,16 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns4 = cast<Function>(c4); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns4 = c4; +#else + Function *cmplogHookIns4 = cast<Function>(c4); #endif -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c8 = M.getOrInsertFunction("__cmplog_ins_hook8", VoidTy, Int64Ty, Int64Ty, Int8Ty @@ -178,10 +178,10 @@ bool CmpLogInstructions::hookInstrs(Module &M) { NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - Function *cmplogHookIns8 = cast<Function>(c8); -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee cmplogHookIns8 = c8; +#else + Function *cmplogHookIns8 = cast<Function>(c8); #endif GlobalVariable *AFLCmplogPtr = M.getNamedGlobal("__afl_cmp_map"); diff --git a/instrumentation/compare-transform-pass.so.cc b/instrumentation/compare-transform-pass.so.cc index 288e8282..ef3bd66b 100644 --- a/instrumentation/compare-transform-pass.so.cc +++ b/instrumentation/compare-transform-pass.so.cc @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -35,7 +35,7 @@ #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/Verifier.h" #include "llvm/IR/DebugInfo.h" @@ -64,11 +64,11 @@ class CompareTransform : public ModulePass { bool runOnModule(Module &M) override; -#if LLVM_VERSION_MAJOR < 4 - const char *getPassName() const override { +#if LLVM_VERSION_MAJOR >= 4 + StringRef getPassName() const override { #else - StringRef getPassName() const override { + const char *getPassName() const override { #endif return "transforms compare functions"; @@ -100,17 +100,17 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, IntegerType * Int32Ty = IntegerType::getInt32Ty(C); IntegerType * Int64Ty = IntegerType::getInt64Ty(C); -#if LLVM_VERSION_MAJOR < 9 - Function *tolowerFn; -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee tolowerFn; +#else + Function * tolowerFn; #endif { -#if LLVM_VERSION_MAJOR < 9 - Constant * -#else +#if LLVM_VERSION_MAJOR >= 9 FunctionCallee +#else + Constant * #endif c = M.getOrInsertFunction("tolower", Int32Ty, Int32Ty #if LLVM_VERSION_MAJOR < 5 @@ -118,10 +118,10 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, NULL #endif ); -#if LLVM_VERSION_MAJOR < 9 - tolowerFn = cast<Function>(c); -#else +#if LLVM_VERSION_MAJOR >= 9 tolowerFn = c; +#else + tolowerFn = cast<Function>(c); #endif } @@ -445,6 +445,10 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, } + // the following is in general OK, but strncmp is sometimes used in binary + // data structures and this can result in crashes :( so it is commented out + /* + // add null termination character implicit in c strings if (!isMemcmp && TmpConstStr[TmpConstStr.length() - 1]) { @@ -452,10 +456,12 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, } + */ + // in the unusual case the const str has embedded null // characters, the string comparison functions should terminate // at the first null - if (!isMemcmp) { + if (!isMemcmp && TmpConstStr.find('\0') != std::string::npos) { TmpConstStr.assign(TmpConstStr, 0, TmpConstStr.find('\0') + 1); @@ -490,10 +496,10 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, PHINode *PN = PHINode::Create( Int32Ty, (next_lenchk_bb ? 2 : 1) * unrollLen + 1, "cmp_phi"); -#if LLVM_VERSION_MAJOR < 8 - TerminatorInst *term = bb->getTerminator(); -#else +#if LLVM_VERSION_MAJOR >= 8 Instruction *term = bb->getTerminator(); +#else + TerminatorInst *term = bb->getTerminator(); #endif BranchInst::Create(next_lenchk_bb ? next_lenchk_bb : next_cmp_bb, bb); term->eraseFromParent(); diff --git a/instrumentation/split-compares-pass.so.cc b/instrumentation/split-compares-pass.so.cc index 13f45b69..95485be9 100644 --- a/instrumentation/split-compares-pass.so.cc +++ b/instrumentation/split-compares-pass.so.cc @@ -6,7 +6,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -34,7 +34,7 @@ #include "llvm/IR/Module.h" #include "llvm/IR/IRBuilder.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/Verifier.h" #include "llvm/IR/DebugInfo.h" @@ -578,16 +578,16 @@ bool SplitComparesTransform::splitCompare(CmpInst *cmp_inst, Module &M, /* dependent on the cmp of the high parts go to the end or go on with * the comparison */ - auto term = bb->getTerminator(); - BranchInst *br = nullptr; + auto term = bb->getTerminator(); + if (pred == CmpInst::ICMP_EQ) { - br = BranchInst::Create(cmp_low_bb, end_bb, icmp_high, bb); + BranchInst::Create(cmp_low_bb, end_bb, icmp_high, bb); } else { - /* CmpInst::ICMP_NE */ - br = BranchInst::Create(end_bb, cmp_low_bb, icmp_high, bb); + // CmpInst::ICMP_NE + BranchInst::Create(end_bb, cmp_low_bb, icmp_high, bb); } @@ -675,7 +675,7 @@ bool SplitComparesTransform::splitCompare(CmpInst *cmp_inst, Module &M, ReplaceInstWithInst(cmp_inst->getParent()->getInstList(), ii, PN); // We split the comparison into low and high. If this isn't our target - // bitwidth we recursivly split the low and high parts again until we have + // bitwidth we recursively split the low and high parts again until we have // target bitwidth. if ((bitw / 2) > target_bitwidth) { @@ -796,7 +796,7 @@ size_t SplitComparesTransform::splitFPCompares(Module &M) { LLVMContext &C = M.getContext(); -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 7) const DataLayout &dl = M.getDataLayout(); @@ -1398,7 +1398,7 @@ bool SplitComparesTransform::runOnModule(Module &M) { bool brokenDebug = false; if (verifyModule(M, &errs() -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 9) , &brokenDebug // 9th May 2016 diff --git a/instrumentation/split-switches-pass.so.cc b/instrumentation/split-switches-pass.so.cc index 82f198aa..c0fa7c9c 100644 --- a/instrumentation/split-switches-pass.so.cc +++ b/instrumentation/split-switches-pass.so.cc @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -37,7 +37,7 @@ #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/IRBuilder.h" -#if LLVM_VERSION_MAJOR > 3 || \ +#if LLVM_VERSION_MAJOR >= 4 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) #include "llvm/IR/Verifier.h" #include "llvm/IR/DebugInfo.h" @@ -369,10 +369,10 @@ bool SplitSwitchesTransform::splitSwitches(Module &M) { CaseVector Cases; for (SwitchInst::CaseIt i = SI->case_begin(), e = SI->case_end(); i != e; ++i) -#if LLVM_VERSION_MAJOR < 5 - Cases.push_back(CaseExpr(i.getCaseValue(), i.getCaseSuccessor())); -#else +#if LLVM_VERSION_MAJOR >= 5 Cases.push_back(CaseExpr(i->getCaseValue(), i->getCaseSuccessor())); +#else + Cases.push_back(CaseExpr(i.getCaseValue(), i.getCaseSuccessor())); #endif /* bugfix thanks to pbst * round up bytesChecked (in case getBitWidth() % 8 != 0) */ diff --git a/qemu_mode/QEMUAFL_VERSION b/qemu_mode/QEMUAFL_VERSION index 215826cc..680c04d6 100644 --- a/qemu_mode/QEMUAFL_VERSION +++ b/qemu_mode/QEMUAFL_VERSION @@ -1 +1 @@ -a6758d1cc3 +002e473939 diff --git a/qemu_mode/README.md b/qemu_mode/README.md index d28479d9..8e04cbf9 100644 --- a/qemu_mode/README.md +++ b/qemu_mode/README.md @@ -1,184 +1,195 @@ # High-performance binary-only instrumentation for afl-fuzz - (See ../README.md for the general instruction manual.) +For the general instruction manual, see [README.md](../README.md). ## 1) Introduction The code in this directory allows you to build a standalone feature that leverages the QEMU "user emulation" mode and allows callers to obtain -instrumentation output for black-box, closed-source binaries. This mechanism -can be then used by afl-fuzz to stress-test targets that couldn't be built -with afl-gcc. +instrumentation output for black-box, closed-source binaries. This mechanism can +be then used by afl-fuzz to stress-test targets that couldn't be built with +afl-cc. -The usual performance cost is 2-5x, which is considerably better than -seen so far in experiments with tools such as DynamoRIO and PIN. +The usual performance cost is 2-5x, which is considerably better than seen so +far in experiments with tools such as DynamoRIO and PIN. -The idea and much of the initial implementation comes from Andrew Griffiths. -The actual implementation on current QEMU (shipped as qemuafl) is from -Andrea Fioraldi. Special thanks to abiondo that re-enabled TCG chaining. +The idea and much of the initial implementation comes from Andrew Griffiths. The +actual implementation on current QEMU (shipped as qemuafl) is from Andrea +Fioraldi. Special thanks to abiondo that re-enabled TCG chaining. -## 2) How to use qemu_mode +## 2) How to use QEMU mode -The feature is implemented with a patched QEMU. The simplest way -to build it is to run ./build_qemu_support.sh. The script will download, -configure, and compile the QEMU binary for you. +The feature is implemented with a patched QEMU. The simplest way to build it is +to run ./build_qemu_support.sh. The script will download, configure, and compile +the QEMU binary for you. -QEMU is a big project, so this will take a while, and you may have to -resolve a couple of dependencies (most notably, you will definitely need -libtool and glib2-devel). +QEMU is a big project, so this will take a while, and you may have to resolve a +couple of dependencies (most notably, you will definitely need libtool and +glib2-devel). Once the binaries are compiled, you can leverage the QEMU tool by calling -afl-fuzz and all the related utilities with -Q in the command line. +afl-fuzz and all the related utilities with `-Q` in the command line. -Note that QEMU requires a generous memory limit to run; somewhere around -200 MB is a good starting point, but considerably more may be needed for -more complex programs. The default -m limit will be automatically bumped up -to 200 MB when specifying -Q to afl-fuzz; be careful when overriding this. +Note that QEMU requires a generous memory limit to run; somewhere around 200 MB +is a good starting point, but considerably more may be needed for more complex +programs. The default `-m` limit will be automatically bumped up to 200 MB when +specifying `-Q` to afl-fuzz; be careful when overriding this. -In principle, if you set CPU_TARGET before calling ./build_qemu_support.sh, -you should get a build capable of running non-native binaries (say, you -can try CPU_TARGET=arm). This is also necessary for running 32-bit binaries -on a 64-bit system (CPU_TARGET=i386). If you're trying to run QEMU on a -different architecture you can also set HOST to the cross-compiler prefix -to use (for example HOST=arm-linux-gnueabi to use arm-linux-gnueabi-gcc). +In principle, if you set `CPU_TARGET` before calling ./build_qemu_support.sh, +you should get a build capable of running non-native binaries (say, you can try +`CPU_TARGET=arm`). This is also necessary for running 32-bit binaries on a +64-bit system (`CPU_TARGET=i386`). If you're trying to run QEMU on a different +architecture, you can also set `HOST` to the cross-compiler prefix to use (for +example `HOST=arm-linux-gnueabi` to use arm-linux-gnueabi-gcc). -You can also compile statically-linked binaries by setting STATIC=1. This -can be useful when compiling QEMU on a different system than the one you're -planning to run the fuzzer on and is most often used with the HOST variable. +You can also compile statically-linked binaries by setting `STATIC=1`. This can +be useful when compiling QEMU on a different system than the one you're planning +to run the fuzzer on and is most often used with the `HOST` variable. -Note: when targetting the i386 architecture, on some binaries the forkserver -handshake may fail due to the lack of reserved memory. Fix it with +Note: when targeting the i386 architecture, on some binaries the forkserver +handshake may fail due to the lack of reserved memory. Fix it with: +``` export QEMU_RESERVED_VA=0x1000000 +``` -Note: if you want the QEMU helper to be installed on your system for all -users, you need to build it before issuing 'make install' in the parent -directory. +Note: if you want the QEMU helper to be installed on your system for all users, +you need to build it before issuing `make install` in the parent directory. -If you want to specify a different path for libraries (e.g. to run an arm64 -binary on x86_64) use QEMU_LD_PREFIX. +If you want to specify a different path for libraries (e.g., to run an arm64 +binary on x86_64) use `QEMU_LD_PREFIX`. ## 3) Deferred initialization -As for LLVM mode (refer to its README.md for mode details) QEMU mode supports -the deferred initialization. +As for LLVM mode (refer to +[instrumentation/README.llvm.md](../instrumentation/README.llvm.md) for mode +details), QEMU mode supports the deferred initialization. -This can be enabled setting the environment variable AFL_ENTRYPOINT which allows -to move the forkserver to a different part, e.g. just before the file is -opened (e.g. way after command line parsing and config file loading, etc.) +This can be enabled by setting the environment variable `AFL_ENTRYPOINT` which +allows to move the forkserver to a different part, e.g., just before the file is +opened (e.g., way after command line parsing and config file loading, etc.) which can be a huge speed improvement. ## 4) Persistent mode -AFL++'s QEMU mode now supports also persistent mode for x86, x86_64, arm -and aarch64 targets. -This increases the speed by several factors, however it is a bit of work to set -up - but worth the effort. +AFL++'s QEMU mode now supports also persistent mode for x86, x86_64, arm, and +aarch64 targets. This increases the speed by several factors, however, it is a +bit of work to set up - but worth the effort. -Please see the extra documentation for it: [README.persistent.md](README.persistent.md) +For more information, see [README.persistent.md](README.persistent.md). ## 5) Snapshot mode As an extension to persistent mode, qemuafl can snapshot and restore the memory -state and brk(). Details are in the persistent mode readme. +state and brk(). For details, see [README.persistent.md](README.persistent.md). -The env var that enables the ready to use snapshot mode is AFL_QEMU_SNAPSHOT and -takes a hex address as a value that is the snapshot entrypoint. +The environment variable that enables the ready to use snapshot mode is +`AFL_QEMU_SNAPSHOT` and takes a hex address as a value that is the snapshot +entry point. -Snapshot mode can work restoring all the writeable pages, that is typically slower than -fork() mode but, on the other hand, it can scale better with multicore. -If the AFL++ Snapshot kernel module is loaded, qemuafl will use it and, in this -case, the speed is better than fork() and also the scaling capabilities. +Snapshot mode can work restoring all the writeable pages, that is typically +slower than fork() mode but, on the other hand, it can scale better with +multicore. If the AFL++ snapshot kernel module is loaded, qemuafl will use it +and, in this case, the speed is better than fork() and also the scaling +capabilities. ## 6) Partial instrumentation You can tell QEMU to instrument only a part of the address space. -Just set AFL_QEMU_INST_RANGES=A,B,C... +Just set `AFL_QEMU_INST_RANGES=A,B,C...`. -The format of the items in the list is either a range of addresses like 0x123-0x321 -or a module name like module.so (that is matched in the mapped object filename). +The format of the items in the list is either a range of addresses like +0x123-0x321 or a module name like module.so (that is matched in the mapped +object filename). -Alternatively you can tell QEMU to ignore part of an address space for instrumentation. +Alternatively, you can tell QEMU to ignore part of an address space for +instrumentation. -Just set AFL_QEMU_EXCLUDE_RANGES=A,B,C... +Just set `AFL_QEMU_EXCLUDE_RANGES=A,B,C...`. -The format of the items on the list is the same as for AFL_QEMU_INST_RANGES, and excluding ranges -takes priority over any included ranges or AFL_INST_LIBS. +The format of the items on the list is the same as for `AFL_QEMU_INST_RANGES` +and excluding ranges takes priority over any included ranges or `AFL_INST_LIBS`. ## 7) CompareCoverage CompareCoverage is a sub-instrumentation with effects similar to laf-intel. -You have to set `AFL_PRELOAD=/path/to/libcompcov.so` together with -setting the AFL_COMPCOV_LEVEL you want to enable it. +You have to set `AFL_PRELOAD=/path/to/libcompcov.so` together with setting the +`AFL_COMPCOV_LEVEL` you want to enable it. -AFL_COMPCOV_LEVEL=1 is to instrument comparisons with only immediate -values / read-only memory. +`AFL_COMPCOV_LEVEL=1` is to instrument comparisons with only immediate +values/read-only memory. -AFL_COMPCOV_LEVEL=2 instruments all comparison instructions and memory +`AFL_COMPCOV_LEVEL=2` instruments all comparison instructions and memory comparison functions when libcompcov is preloaded. -AFL_COMPCOV_LEVEL=3 has the same effects of AFL_COMPCOV_LEVEL=2 but enables +`AFL_COMPCOV_LEVEL=3` has the same effects of `AFL_COMPCOV_LEVEL=2` but enables also the instrumentation of the floating-point comparisons on x86 and x86_64 (experimental). -Integer comparison instructions are currently instrumented only -on the x86, x86_64, arm and aarch64 targets. +Integer comparison instructions are currently instrumented only on the x86, +x86_64, arm, and aarch64 targets. Recommended, but not as good as CMPLOG mode (see below). ## 8) CMPLOG mode -Another new feature is CMPLOG, which is based on the redqueen project. -Here all immediates in CMP instructions are learned and put into a dynamic -dictionary and applied to all locations in the input that reached that -CMP, trying to solve and pass it. -This is a very effective feature and it is available for x86, x86_64, arm -and aarch64. +Another new feature is CMPLOG, which is based on the redqueen project. Here all +immediates in CMP instructions are learned and put into a dynamic dictionary and +applied to all locations in the input that reached that CMP, trying to solve and +pass it. This is a very effective feature and it is available for x86, x86_64, +arm, and aarch64. -To enable it you must pass on the command line of afl-fuzz: - -c /path/to/your/target +To enable it, you must pass on the command line of afl-fuzz: + +``` +-c /path/to/your/target +``` ## 9) Wine mode -AFL++ QEMU can use Wine to fuzz Win32 PE binaries. Use the -W flag of afl-fuzz. +AFL++ QEMU can use Wine to fuzz Win32 PE binaries. Use the `-W` flag of +afl-fuzz. -Note that some binaries require user interaction with the GUI and must be patched. +Note that some binaries require user interaction with the GUI and must be +patched. -For examples look [here](https://github.com/andreafioraldi/WineAFLplusplusDEMO). +For examples, look +[here](https://github.com/andreafioraldi/WineAFLplusplusDEMO). ## 10) Notes on linking -The feature is supported only on Linux. Supporting BSD may amount to porting -the changes made to linux-user/elfload.c and applying them to -bsd-user/elfload.c, but I have not looked into this yet. +The feature is supported only on Linux. Supporting BSD may amount to porting the +changes made to linux-user/elfload.c and applying them to bsd-user/elfload.c, +but I have not looked into this yet. The instrumentation follows only the .text section of the first ELF binary encountered in the linking process. It does not trace shared libraries. In practice, this means two things: - - Any libraries you want to analyze *must* be linked statically into the - executed ELF file (this will usually be the case for closed-source - apps). +- Any libraries you want to analyze *must* be linked statically into the + executed ELF file (this will usually be the case for closed-source apps). - - Standard C libraries and other stuff that is wasteful to instrument - should be linked dynamically - otherwise, AFL will have no way to avoid - peeking into them. +- Standard C libraries and other stuff that is wasteful to instrument should be + linked dynamically - otherwise, AFL++ will have no way to avoid peeking into + them. -Setting AFL_INST_LIBS=1 can be used to circumvent the .text detection logic +Setting `AFL_INST_LIBS=1` can be used to circumvent the .text detection logic and instrument every basic block encountered. ## 11) Benchmarking If you want to compare the performance of the QEMU instrumentation with that of -afl-gcc compiled code against the same target, you need to build the +afl-clang-fast compiled code against the same target, you need to build the non-instrumented binary with the same optimization flags that are normally -injected by afl-gcc, and make sure that the bits to be tested are statically -linked into the binary. A common way to do this would be: +injected by afl-clang-fast, and make sure that the bits to be tested are +statically linked into the binary. A common way to do this would be: +``` CFLAGS="-O3 -funroll-loops" ./configure --disable-shared make clean all +``` Comparative measurements of execution speed or instrumentation coverage will be fairly meaningless if the optimization levels or instrumentation scopes don't @@ -186,36 +197,37 @@ match. ## 12) Other features -With `AFL_QEMU_FORCE_DFL` you force QEMU to ignore the registered signal +With `AFL_QEMU_FORCE_DFL`, you force QEMU to ignore the registered signal handlers of the target. ## 13) Gotchas, feedback, bugs If you need to fix up checksums or do other cleanups on mutated test cases, see -`afl_custom_post_process` in custom_mutators/examples/example.c for a viable solution. +`afl_custom_post_process` in custom_mutators/examples/example.c for a viable +solution. -Do not mix QEMU mode with ASAN, MSAN, or the likes; QEMU doesn't appreciate -the "shadow VM" trick employed by the sanitizers and will probably just -run out of memory. +Do not mix QEMU mode with ASAN, MSAN, or the likes; QEMU doesn't appreciate the +"shadow VM" trick employed by the sanitizers and will probably just run out of +memory. Compared to fully-fledged virtualization, the user emulation mode is *NOT* a security boundary. The binaries can freely interact with the host OS. If you somehow need to fuzz an untrusted binary, put everything in a sandbox first. -QEMU does not necessarily support all CPU or hardware features that your -target program may be utilizing. In particular, it does not appear to have -full support for AVX2 / FMA3. Using binaries for older CPUs, or recompiling them -with -march=core2, can help. +QEMU does not necessarily support all CPU or hardware features that your target +program may be utilizing. In particular, it does not appear to have full support +for AVX2/FMA3. Using binaries for older CPUs or recompiling them with +`-march=core2`, can help. Beyond that, this is an early-stage mechanism, so fields reports are welcome. You can send them to <afl-users@googlegroups.com>. ## 14) Alternatives: static rewriting -Statically rewriting binaries just once, instead of attempting to translate -them at run time, can be a faster alternative. That said, static rewriting is -fraught with peril, because it depends on being able to properly and fully model -program control flow without actually executing each and every code path. +Statically rewriting binaries just once, instead of attempting to translate them +at run time, can be a faster alternative. That said, static rewriting is fraught +with peril, because it depends on being able to properly and fully model program +control flow without actually executing each and every code path. -Checkout the "Fuzzing binary-only targets" section in our main README.md and -the docs/binaryonly_fuzzing.md document for more information and hints. +For more information and hints, check out +[docs/fuzzing_binary-only_targets.md](../docs/fuzzing_binary-only_targets.md). \ No newline at end of file diff --git a/qemu_mode/README.persistent.md b/qemu_mode/README.persistent.md index 2ca5c873..ab45860d 100644 --- a/qemu_mode/README.persistent.md +++ b/qemu_mode/README.persistent.md @@ -2,118 +2,118 @@ ## 1) Introduction -Persistent mode lets you fuzz your target persistently between two -addresses - without forking for every fuzzing attempt. -This increases the speed by a factor between x2 and x5, hence it is -very, very valuable. +Persistent mode lets you fuzz your target persistently between two addresses - +without forking for every fuzzing attempt. This increases the speed by a factor +between x2 and x5, hence it is very, very valuable. -The persistent mode is currently only available for x86/x86_64, arm -and aarch64 targets. +The persistent mode is currently only available for x86/x86_64, arm, and aarch64 +targets. ## 2) How use the persistent mode ### 2.1) The START address -The start of the persistent loop has to be set with env var AFL_QEMU_PERSISTENT_ADDR. +The start of the persistent loop has to be set with environment variable +`AFL_QEMU_PERSISTENT_ADDR`. -This address can be the address of whatever instruction. -Setting this address to the start of a function makes the usage simple. -If the address is however within a function, either RET, OFFSET or EXITS -(see below in 2.2, 2.3, 2.6) have to be set. -This address (as well as the RET address, see below) has to be defined in -hexadecimal with the 0x prefix or as a decimal value. +This address can be the address of whatever instruction. Setting this address to +the start of a function makes the usage simple. If the address is, however, +within a function, either RET, OFFSET, or EXITS (see below in 2.2, 2.3, 2.6) +have to be set. This address (as well as the RET address, see below) has to be +defined in hexadecimal with the 0x prefix or as a decimal value. If both RET and EXITS are not set, QEMU will assume that START points to a -function and will patch the return address (on stack or in the link register) -to return to START (like WinAFL). +function and will patch the return address (on stack or in the link register) to +return to START (like WinAFL). -*Note:* If the target is compiled with position independant code (PIE/PIC) -qemu loads these to a specific base address. -For 64 bit you have to add 0x4000000000 (9 zeroes) and for 32 bit 0x40000000 -(7 zeroes) to the address. -On strange setups the base address set by QEMU for PIE executable may change, -you can check it printing the process map using -`AFL_QEMU_DEBUG_MAPS=1 afl-qemu-trace TARGET-BINARY` +*Note:* If the target is compiled with position independent code (PIE/PIC) qemu +loads these to a specific base address. For 64 bit you have to add 0x4000000000 +(9 zeroes) and for 32 bit 0x40000000 (7 zeroes) to the address. On strange +setups the base address set by QEMU for PIE executable may change. You can check +it printing the process map using `AFL_QEMU_DEBUG_MAPS=1 afl-qemu-trace +TARGET-BINARY`. If this address is not valid, afl-fuzz will error during startup with the message that the forkserver was not found. ### 2.2) The RET address -The RET address is the last instruction of the persistent loop. -The emulator will emit a jump to START when translating the instruction at RET. -It is optional, and only needed if the return should not be -at the end of the function to which the START address points into, but earlier. +The RET address is the last instruction of the persistent loop. The emulator +will emit a jump to START when translating the instruction at RET. It is +optional and only needed if the return should not be at the end of the function +to which the START address points into, but earlier. -It is defined by setting AFL_QEMU_PERSISTENT_RET, and too 0x4000000000 has to -be set if the target is position independant. +It is defined by setting `AFL_QEMU_PERSISTENT_RET`, and too 0x4000000000 has to +be set if the target is position independent. ### 2.3) The OFFSET This option is valid only for x86/x86_64 only, arm/aarch64 do not save the return address on stack. -If the START address is *not* the beginning of a function, and *no* RET has -been set (so the end of the loop will be at the end of the function but START -will not be at the beginning of it), we need an offset from the ESP pointer -to locate the return address to patch. +If the START address is *not* the beginning of a function, and *no* RET has been +set (so the end of the loop will be at the end of the function but START will +not be at the beginning of it), we need an offset from the ESP pointer to locate +the return address to patch. The value by which the ESP pointer has to be corrected has to be set in the -variable AFL_QEMU_PERSISTENT_RETADDR_OFFSET. +variable `AFL_QEMU_PERSISTENT_RETADDR_OFFSET`. Now to get this value right here is some help: -1. use gdb on the target -2. set a breakpoint to "main" (this is required for PIE/PIC binaries so the - addresses are set up) -3. "run" the target with a valid commandline -4. set a breakpoint to the function in which START is contained -5. set a breakpoint to your START address -6. "continue" to the function start breakpoint -6. print the ESP value with `print $esp` and take note of it -7. "continue" the target until the second breakpoint -8. again print the ESP value -9. calculate the difference between the two values - and this is the offset +1. Use gdb on the target. +2. Set a breakpoint to "main" (this is required for PIE/PIC binaries so the + addresses are set up). +3. "run" the target with a valid commandline. +4. Set a breakpoint to the function in which START is contained. +5. Set a breakpoint to your START address. +6. "continue" to the function start breakpoint. +7. Print the ESP value with `print $esp` and take note of it. +8. "continue" the target until the second breakpoint. +9. Again print the ESP value. +10. Calculate the difference between the two values - and this is the offset. ### 2.4) Resetting the register state It is very, very likely you need to restore the general purpose registers state when starting a new loop. Because of this 99% of the time you should set +``` AFL_QEMU_PERSISTENT_GPR=1 +``` -An example is when you want to use main() as persistent START: +An example is when you want to use `main()` as persistent START: ```c int main(int argc, char **argv) { if (argc < 2) return 1; - + // do stuff } ``` -If you don't save and restore the registers in x86_64, the parameter `argc` -will be lost at the second execution of the loop. +If you don't save and restore the registers in x86_64, the parameter `argc` will +be lost at the second execution of the loop. ### 2.5) Resetting the memory state This option restores the memory state using the AFL++ Snapshot LKM if loaded. Otherwise, all the writeable pages are restored. -To enable this option, set AFL_QEMU_PERSISTENT_MEM=1. +To enable this option, set `AFL_QEMU_PERSISTENT_MEM=1`. ### 2.6) Reset on exit() The user can force QEMU to set the program counter to START instead of executing the exit_group syscall and exit the program. -The env variable is AFL_QEMU_PERSISTENT_EXITS. +The environment variable is `AFL_QEMU_PERSISTENT_EXITS`. ### 2.7) Snapshot -AFL_QEMU_SNAPSHOT=address is just a "syntactical sugar" env variable that is equivalent to -the following set of variables: +`AFL_QEMU_SNAPSHOT=address` is just a "syntactical sugar" environment variable +that is equivalent to the following set of variables: ``` AFL_QEMU_PERSISTENT_ADDR=address @@ -127,26 +127,27 @@ AFL_QEMU_PERSISTENT_EXITS=1 ### 3.1) Loop counter value The more stable your loop in the target, the longer you can run it, the more -unstable it is the lower the loop count should be. A low value would be 100, -the maximum value should be 10000. The default is 1000. -This value can be set with AFL_QEMU_PERSISTENT_CNT +unstable it is the lower the loop count should be. A low value would be 100, the +maximum value should be 10000. The default is 1000. This value can be set with +`AFL_QEMU_PERSISTENT_CNT`. -This is the same concept as in the llvm_mode persistent mode with __AFL_LOOP(). +This is the same concept as in the llvm_mode persistent mode with +`__AFL_LOOP()`. ### 3.2) A hook for in-memory fuzzing -You can increase the speed of the persistent mode even more by bypassing all -the reading of the fuzzing input via a file by reading directly into the -memory address space of the target process. +You can increase the speed of the persistent mode even more by bypassing all the +reading of the fuzzing input via a file by reading directly into the memory +address space of the target process. All this needs is that the START address has a register that can reach the -memory buffer or that the memory buffer is at a known location. You probably need -the value of the size of the buffer (maybe it is in a register when START is -hit). +memory buffer or that the memory buffer is at a known location. You probably +need the value of the size of the buffer (maybe it is in a register when START +is hit). -The persistent hook will execute a function on every persistent iteration -(at the start START) defined in a shared object specified with -AFL_QEMU_PERSISTENT_HOOK=/path/to/hook.so. +The persistent hook will execute a function on every persistent iteration (at +the start START) defined in a shared object specified with +`AFL_QEMU_PERSISTENT_HOOK=/path/to/hook.so`. The signature is: @@ -157,8 +158,8 @@ void afl_persistent_hook(struct ARCH_regs *regs, uint32_t input_buf_len); ``` -Where ARCH is one of x86, x86_64, arm or arm64. -You have to include `path/to/qemuafl/qemuafl/api.h`. +Where ARCH is one of x86, x86_64, arm or arm64. You have to include +`path/to/qemuafl/qemuafl/api.h`. In this hook, you can inspect and change the saved GPR state at START. @@ -168,8 +169,8 @@ with: `int afl_persistent_hook_init(void);` If this routine returns true, the shared mem fuzzing feature of AFL++ is used -and so the input_buf variables of the hook becomes meaningful. Otherwise, -you have to read the input from a file like stdin. +and so the input_buf variables of the hook becomes meaningful. Otherwise, you +have to read the input from a file like stdin. -An example that you can use with little modification for your target can -be found here: [utils/qemu_persistent_hook](../utils/qemu_persistent_hook) +An example that you can use with little modification for your target can be +found here: [utils/qemu_persistent_hook](../utils/qemu_persistent_hook) \ No newline at end of file diff --git a/qemu_mode/README.wine.md b/qemu_mode/README.wine.md index 567901cd..ee1ef58a 100644 --- a/qemu_mode/README.wine.md +++ b/qemu_mode/README.wine.md @@ -1,21 +1,23 @@ # How to troubleshoot AFL++'s wine mode ## 1) Debugging -To turn on wine debugging use the `WINEDEBUG` environment variable, -e.g. `WINEDEBUG=+timestamp,+tid,+loaddll`. + +To turn on wine debugging, use the `WINEDEBUG` environment variable, e.g., +`WINEDEBUG=+timestamp,+tid,+loaddll`. ## 2) LoadLibraryA workaround -The forked process fails to load libraries loaded via `LoadLibrary` -if the load happens after the entry point (error code: 87). To resolve -this issue, one needs to load any external libraries before the fork happens. -An early DLL load can be achieved by adding the DLL name into the `Import Directory` -in the PE file. Such an entry can be added manually in any PE editor. +The forked process fails to load libraries loaded via `LoadLibrary` if the load +happens after the entry point (error code: 87). To resolve this issue, one needs +to load any external libraries before the fork happens. + +An early DLL load can be achieved by adding the DLL name into the `Import +Directory` in the PE file. Such an entry can be added manually in any PE editor. -Alternativly, one can generate a `.lib` file from the DLL exports and link -them together with the harness to create an entry in the `Import Directory`. -Use `dumpbin /exports <filename>.dll` to extract the exports and paste the -exported function names into a `.def` file. Use `lib /def:<deffile> /OUT:<libfile>` -to generate a `.lib` and add the library to the linker options. Once the usage of -an export is detected (`__declspec(dllimport)`), the -linker adds the early DLL load. \ No newline at end of file +Alternatively, one can generate a `.lib` file from the DLL exports and link them +together with the harness to create an entry in the `Import Directory`. Use +`dumpbin /exports <filename>.dll` to extract the exports and paste the exported +function names into a `.def` file. Use `lib /def:<deffile> /OUT:<libfile>` to +generate a `.lib` and add the library to the linker options. Once the usage of +an export is detected (`__declspec(dllimport)`), the linker adds the early DLL +load. \ No newline at end of file diff --git a/qemu_mode/build_qemu_support.sh b/qemu_mode/build_qemu_support.sh index 84f144be..71453a71 100755 --- a/qemu_mode/build_qemu_support.sh +++ b/qemu_mode/build_qemu_support.sh @@ -19,7 +19,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # This script downloads, patches, and builds a version of QEMU with # minor tweaks to allow non-instrumented binaries to be run under diff --git a/qemu_mode/libcompcov/README.md b/qemu_mode/libcompcov/README.md index fca20a69..bb010d8f 100644 --- a/qemu_mode/libcompcov/README.md +++ b/qemu_mode/libcompcov/README.md @@ -18,12 +18,12 @@ and this module is not capable to log the coverage in this case. If you have the source code of the fuzzing target you should nto use this library and QEMU but build it with afl-clang-fast and the laf-intel options. -To use this library make sure to preload it with AFL_PRELOAD. +To use this library, make sure to preload it with AFL_PRELOAD. ``` export AFL_PRELOAD=/path/to/libcompcov.so export AFL_COMPCOV_LEVEL=1 - + afl-fuzz -Q -i input -o output <your options> -- <target args> ``` @@ -31,7 +31,7 @@ The AFL_COMPCOV_LEVEL tells to QEMU and libcompcov how to log comaprisons. Level 1 logs just comparison with immediates / read-only memory and level 2 logs all the comparisons. -The library make use of https://github.com/ouadev/proc_maps_parser and so it is -Linux specific. However this is not a strict dependency, other UNIX operating -systems can be supported simply replacing the code related to the -/proc/self/maps parsing. +The library makes use of https://github.com/ouadev/proc_maps_parser and so it is +Linux specific. However, this is not a strict dependency, other UNIX operating +systems can be supported by replacing the code related to the /proc/self/maps +parsing. \ No newline at end of file diff --git a/qemu_mode/libcompcov/libcompcov.so.c b/qemu_mode/libcompcov/libcompcov.so.c index 4fc84e62..eba3d80a 100644 --- a/qemu_mode/libcompcov/libcompcov.so.c +++ b/qemu_mode/libcompcov/libcompcov.so.c @@ -41,6 +41,13 @@ #error "Sorry, this library is Linux-specific for now!" #endif /* !__linux__ */ +#ifndef likely + #define likely(x) __builtin_expect((!!(x)), 1) +#endif +#ifndef unlikely + #define unlikely(x) __builtin_expect((!!(x)), 0) +#endif + /* Change this value to tune the compare coverage */ #define MAX_CMP_LENGTH 32 @@ -199,6 +206,7 @@ static u8 __compcov_is_in_bound(const void *ptr) { int strcmp(const char *str1, const char *str2) { + if (unlikely(!__libc_strcmp)) { __libc_strcmp = dlsym(RTLD_NEXT, "strcmp"); } void *retaddr = __builtin_return_address(0); if (__compcov_is_in_bound(retaddr) && @@ -227,6 +235,12 @@ int strcmp(const char *str1, const char *str2) { int strncmp(const char *str1, const char *str2, size_t len) { + if (unlikely(!__libc_strncmp)) { + + __libc_strncmp = dlsym(RTLD_NEXT, "strncmp"); + + } + void *retaddr = __builtin_return_address(0); if (__compcov_is_in_bound(retaddr) && @@ -256,6 +270,12 @@ int strncmp(const char *str1, const char *str2, size_t len) { int strcasecmp(const char *str1, const char *str2) { + if (unlikely(!__libc_strcasecmp)) { + + __libc_strncasecmp = dlsym(RTLD_NEXT, "strcasecmp"); + + } + void *retaddr = __builtin_return_address(0); if (__compcov_is_in_bound(retaddr) && @@ -286,6 +306,12 @@ int strcasecmp(const char *str1, const char *str2) { int strncasecmp(const char *str1, const char *str2, size_t len) { + if (unlikely(!__libc_strncasecmp)) { + + __libc_strncasecmp = dlsym(RTLD_NEXT, "strncasecmp"); + + } + void *retaddr = __builtin_return_address(0); if (__compcov_is_in_bound(retaddr) && @@ -317,6 +343,7 @@ int strncasecmp(const char *str1, const char *str2, size_t len) { int memcmp(const void *mem1, const void *mem2, size_t len) { + if (unlikely(!__libc_memcmp)) { __libc_memcmp = dlsym(RTLD_NEXT, "memcmp"); } void *retaddr = __builtin_return_address(0); if (__compcov_is_in_bound(retaddr) && diff --git a/qemu_mode/libqasan/README.md b/qemu_mode/libqasan/README.md index 4a241233..41195933 100644 --- a/qemu_mode/libqasan/README.md +++ b/qemu_mode/libqasan/README.md @@ -9,7 +9,7 @@ and this runtime is injected via LD_PRELOAD (so works just for dynamically linked binaries). The usage is super simple, just set the env var `AFL_USE_QASAN=1` when fuzzing -in qemu mode (-Q). afl-fuzz will automatically set AFL_PRELOAD to load this +in QEMU mode (-Q). afl-fuzz will automatically set AFL_PRELOAD to load this library and enable the QASan instrumentation in afl-qemu-trace. For debugging purposes, we still suggest to run the original QASan as the @@ -19,7 +19,7 @@ finding capabilities during fuzzing) is WIP. ### When should I use QASan? If your target binary is PIC x86_64, you should also give a try to -[retrowrite](https://github.com/HexHive/retrowrite) for static rewriting. +[RetroWrite](https://github.com/HexHive/retrowrite) for static rewriting. If it fails, or if your binary is for another architecture, or you want to use persistent and snapshot mode, AFL++ QASan mode is what you want/have to use. diff --git a/qemu_mode/qemuafl b/qemu_mode/qemuafl -Subproject a6758d1cc3e4dde88fca3f0b3a903581b7c8b2e +Subproject 002e473939a350854d56f67ce7b2e2d9706b8bc diff --git a/src/afl-analyze.c b/src/afl-analyze.c index e19df3ce..ac5a324c 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 A nifty utility that grabs an input file and takes a stab at explaining its structure by observing how changes to it affect the execution path. @@ -77,6 +77,7 @@ static volatile u8 stop_soon; /* Ctrl-C pressed? */ static u8 *target_path; static u8 frida_mode; static u8 qemu_mode; +static u8 cs_mode; static u32 map_size = MAP_SIZE; static afl_forkserver_t fsrv = {0}; /* The forkserver */ @@ -120,6 +121,17 @@ static u8 count_class_lookup[256] = { #undef TIMES8 #undef TIMES4 +static void kill_child() { + + if (fsrv.child_pid > 0) { + + kill(fsrv.child_pid, fsrv.kill_signal); + fsrv.child_pid = -1; + + } + +} + static void classify_counts(u8 *mem) { u32 i = map_size; @@ -184,7 +196,7 @@ static void read_initial_file(void) { if (st.st_size >= TMIN_MAX_FILE) { - FATAL("Input file is too large (%u MB max)", TMIN_MAX_FILE / 1024 / 1024); + FATAL("Input file is too large (%ld MB max)", TMIN_MAX_FILE / 1024 / 1024); } @@ -779,6 +791,8 @@ static void set_up_environment(char **argv) { } else { + /* CoreSight mode uses the default behavior. */ + setenv("LD_PRELOAD", getenv("AFL_PRELOAD"), 1); setenv("DYLD_INSERT_LIBRARIES", getenv("AFL_PRELOAD"), 1); @@ -834,11 +848,17 @@ static void usage(u8 *argv0) { " -f file - input file read by the tested program (stdin)\n" " -t msec - timeout for each run (%u ms)\n" " -m megs - memory limit for child process (%u MB)\n" +#if defined(__linux__) && defined(__aarch64__) + " -A - use binary-only instrumentation (ARM CoreSight mode)\n" +#endif " -O - use binary-only instrumentation (FRIDA mode)\n" +#if defined(__linux__) " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use unicorn-based instrumentation (Unicorn mode)\n" " -W - use qemu-based instrumentation with Wine (Wine " - "mode)\n\n" + "mode)\n" +#endif + "\n" "Analysis settings:\n" @@ -879,7 +899,7 @@ int main(int argc, char **argv_orig, char **envp) { afl_fsrv_init(&fsrv); - while ((opt = getopt(argc, argv, "+i:f:m:t:eOQUWh")) > 0) { + while ((opt = getopt(argc, argv, "+i:f:m:t:eAOQUWh")) > 0) { switch (opt) { @@ -978,13 +998,25 @@ int main(int argc, char **argv_orig, char **envp) { break; + case 'A': /* CoreSight mode */ + +#if !defined(__aarch64__) || !defined(__linux__) + FATAL("-A option is not supported on this platform"); +#endif + + if (cs_mode) { FATAL("Multiple -A options not supported"); } + + cs_mode = 1; + fsrv.cs_mode = cs_mode; + break; + case 'O': /* FRIDA mode */ if (frida_mode) { FATAL("Multiple -O options not supported"); } frida_mode = 1; fsrv.frida_mode = frida_mode; - setenv("AFL_FRIDA_INST_SEED", "0x0", 1); + setenv("AFL_FRIDA_INST_SEED", "1", 1); break; @@ -1053,6 +1085,7 @@ int main(int argc, char **argv_orig, char **envp) { fsrv.target_path = find_binary(argv[optind]); fsrv.trace_bits = afl_shm_init(&shm, map_size, 0); detect_file_args(argv + optind, fsrv.out_file, &use_stdin); + signal(SIGALRM, kill_child); if (qemu_mode) { @@ -1068,6 +1101,10 @@ int main(int argc, char **argv_orig, char **envp) { } + } else if (cs_mode) { + + use_argv = get_cs_argv(argv[0], &target_path, argc - optind, argv + optind); + } else { use_argv = argv + optind; @@ -1093,6 +1130,7 @@ int main(int argc, char **argv_orig, char **envp) { parse_afl_kill_signal_env(getenv("AFL_KILL_SIGNAL"), SIGKILL); read_initial_file(); + (void)check_binary_signatures(fsrv.target_path); ACTF("Performing dry run (mem limit = %llu MB, timeout = %u ms%s)...", mem_limit, exec_tmout, edges_only ? ", edges only" : ""); diff --git a/src/afl-as.c b/src/afl-as.c index 7119d630..b644b82a 100644 --- a/src/afl-as.c +++ b/src/afl-as.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 The sole purpose of this wrapper is to preprocess assembly files generated by GCC / clang and inject the instrumentation bits included from afl-as.h. It @@ -101,7 +101,7 @@ static void edit_params(int argc, char **argv) { /* On MacOS X, the Xcode cctool 'as' driver is a bit stale and does not work with the code generated by newer versions of clang that are hand-built - by the user. See the thread here: http://goo.gl/HBWDtn. + by the user. See the thread here: https://goo.gl/HBWDtn. To work around this, when using clang and running without AFL_AS specified, we will actually call 'clang -c' instead of 'as -q' to @@ -517,10 +517,11 @@ static void add_instrumentation(void) { } else { char modeline[100]; - snprintf(modeline, sizeof(modeline), "%s%s%s%s%s", + snprintf(modeline, sizeof(modeline), "%s%s%s%s%s%s", getenv("AFL_HARDEN") ? "hardened" : "non-hardened", getenv("AFL_USE_ASAN") ? ", ASAN" : "", getenv("AFL_USE_MSAN") ? ", MSAN" : "", + getenv("AFL_USE_TSAN") ? ", TSAN" : "", getenv("AFL_USE_UBSAN") ? ", UBSAN" : "", getenv("AFL_USE_LSAN") ? ", LSAN" : ""); diff --git a/src/afl-cc.c b/src/afl-cc.c index e49addc4..1448d8ae 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -11,7 +11,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ @@ -423,6 +423,8 @@ static void edit_params(u32 argc, char **argv, char **envp) { char *fplugin_arg = alloc_printf("-fplugin=%s/afl-gcc-pass.so", obj_path); cc_params[cc_par_cnt++] = fplugin_arg; + cc_params[cc_par_cnt++] = "-fno-if-conversion"; + cc_params[cc_par_cnt++] = "-fno-if-conversion2"; } @@ -553,7 +555,8 @@ static void edit_params(u32 argc, char **argv, char **envp) { if (lto_mode && !have_c) { - u8 *ld_path = strdup(AFL_REAL_LD); + u8 *ld_path = NULL; + if (getenv("AFL_REAL_LD")) { ld_path = strdup(getenv("AFL_REAL_LD")); } if (!ld_path || !*ld_path) { ld_path = strdup("ld.lld"); } if (!ld_path) { PFATAL("Could not allocate mem for ld_path"); } #if defined(AFL_CLANG_LDPATH) && LLVM_MAJOR >= 12 @@ -564,22 +567,15 @@ static void edit_params(u32 argc, char **argv, char **envp) { free(ld_path); cc_params[cc_par_cnt++] = "-Wl,--allow-multiple-definition"; - - if (instrument_mode == INSTRUMENT_CFG || - instrument_mode == INSTRUMENT_PCGUARD) - cc_params[cc_par_cnt++] = alloc_printf( - "-Wl,-mllvm=-load=%s/SanitizerCoverageLTO.so", obj_path); - else - - cc_params[cc_par_cnt++] = alloc_printf( - "-Wl,-mllvm=-load=%s/afl-llvm-lto-instrumentation.so", obj_path); + cc_params[cc_par_cnt++] = + alloc_printf("-Wl,-mllvm=-load=%s/SanitizerCoverageLTO.so", obj_path); cc_params[cc_par_cnt++] = lto_flag; } else { if (instrument_mode == INSTRUMENT_PCGUARD) { -#if LLVM_MAJOR > 10 || (LLVM_MAJOR == 10 && LLVM_MINOR > 0) +#if LLVM_MAJOR >= 11 || (LLVM_MAJOR == 10 && LLVM_MINOR >= 1) #if defined __ANDROID__ || ANDROID cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; instrument_mode = INSTRUMENT_LLVMNATIVE; @@ -735,6 +731,14 @@ static void edit_params(u32 argc, char **argv, char **envp) { } + if ((compiler_mode == GCC || compiler_mode == GCC_PLUGIN) && + !strncmp(cur, "-stdlib=", 8)) { + + if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } + continue; + + } + if ((!strncmp(cur, "-fsanitize=fuzzer-", strlen("-fsanitize=fuzzer-")) || !strncmp(cur, "-fsanitize-coverage", strlen("-fsanitize-coverage"))) && (strncmp(cur, "sanitize-coverage-allow", @@ -847,6 +851,14 @@ static void edit_params(u32 argc, char **argv, char **envp) { cc_params[cc_par_cnt++] = "-fsanitize=undefined"; cc_params[cc_par_cnt++] = "-fsanitize-undefined-trap-on-error"; cc_params[cc_par_cnt++] = "-fno-sanitize-recover=all"; + cc_params[cc_par_cnt++] = "-fno-omit-frame-pointer"; + + } + + if (getenv("AFL_USE_TSAN")) { + + cc_params[cc_par_cnt++] = "-fsanitize=thread"; + cc_params[cc_par_cnt++] = "-fno-omit-frame-pointer"; } @@ -1007,7 +1019,11 @@ static void edit_params(u32 argc, char **argv, char **envp) { } // prevent unnecessary build errors - cc_params[cc_par_cnt++] = "-Wno-unused-command-line-argument"; + if (compiler_mode != GCC_PLUGIN && compiler_mode != GCC) { + + cc_params[cc_par_cnt++] = "-Wno-unused-command-line-argument"; + + } if (preprocessor_only || have_c) { @@ -1152,7 +1168,7 @@ int main(int argc, char **argv, char **envp) { } -#if (LLVM_MAJOR > 2) +#if (LLVM_MAJOR >= 3) if ((ptr = find_object("SanitizerCoverageLTO.so", argv[0])) != NULL) { @@ -1181,7 +1197,7 @@ int main(int argc, char **argv, char **envp) { } -#if (LLVM_MAJOR > 2) +#if (LLVM_MAJOR >= 3) if (strncmp(callname, "afl-clang-fast", 14) == 0) { @@ -1709,8 +1725,8 @@ int main(int argc, char **argv, char **envp) { compiler_mode == LTO ? " [SELECTED]" : "", have_llvm ? "AVAILABLE" : "unavailable!", compiler_mode == LLVM ? " [SELECTED]" : "", - LLVM_MAJOR > 6 ? "DEFAULT" : " ", - LLVM_MAJOR > 6 ? " " : "DEFAULT", + LLVM_MAJOR >= 7 ? "DEFAULT" : " ", + LLVM_MAJOR >= 7 ? " " : "DEFAULT", have_gcc_plugin ? "AVAILABLE" : "unavailable!", compiler_mode == GCC_PLUGIN ? " [SELECTED]" : "", have_gcc ? "AVAILABLE" : "unavailable!", @@ -1800,6 +1816,7 @@ int main(int argc, char **argv, char **envp) { " AFL_USE_CFISAN: activate control flow sanitizer\n" " AFL_USE_MSAN: activate memory sanitizer\n" " AFL_USE_UBSAN: activate undefined behaviour sanitizer\n" + " AFL_USE_TSAN: activate thread sanitizer\n" " AFL_USE_LSAN: activate leak-checker sanitizer\n"); if (have_gcc_plugin) @@ -1810,12 +1827,12 @@ int main(int argc, char **argv, char **envp) { " AFL_GCC_INSTRUMENT_FILE: enable selective instrumentation by " "filename\n"); -#if LLVM_MAJOR < 9 +#if LLVM_MAJOR >= 9 #define COUNTER_BEHAVIOUR \ - " AFL_LLVM_NOT_ZERO: use cycling trace counters that skip zero\n" + " AFL_LLVM_SKIP_NEVERZERO: do not skip zero on trace counters\n" #else #define COUNTER_BEHAVIOUR \ - " AFL_LLVM_SKIP_NEVERZERO: do not skip zero on trace counters\n" + " AFL_LLVM_NOT_ZERO: use cycling trace counters that skip zero\n" #endif if (have_llvm) SAYF( @@ -1889,7 +1906,7 @@ int main(int argc, char **argv, char **envp) { "consult the README.md, especially section 3.1 about instrumenting " "targets.\n\n"); -#if (LLVM_MAJOR > 2) +#if (LLVM_MAJOR >= 3) if (have_lto) SAYF("afl-cc LTO with ld=%s %s\n", AFL_REAL_LD, AFL_CLANG_FLTO); if (have_llvm) @@ -1951,9 +1968,7 @@ int main(int argc, char **argv, char **envp) { if (instrument_mode == 0 && compiler_mode < GCC_PLUGIN) { -#if LLVM_MAJOR <= 6 - instrument_mode = INSTRUMENT_AFL; -#else +#if LLVM_MAJOR >= 7 #if LLVM_MAJOR < 11 && (LLVM_MAJOR < 10 || LLVM_MINOR < 1) if (have_instr_env) { @@ -1968,6 +1983,8 @@ int main(int argc, char **argv, char **envp) { #endif instrument_mode = INSTRUMENT_PCGUARD; +#else + instrument_mode = INSTRUMENT_AFL; #endif } @@ -2034,7 +2051,7 @@ int main(int argc, char **argv, char **envp) { if ((isatty(2) && !be_quiet) || debug) { SAYF(cCYA - "afl-cc " VERSION cRST + "afl-cc" VERSION cRST " by Michal Zalewski, Laszlo Szekeres, Marc Heuse - mode: %s-%s\n", compiler_mode_string[compiler_mode], ptr); diff --git a/src/afl-common.c b/src/afl-common.c index 9ca2b3e8..6c2d0753 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Gather some functions common to multiple executables @@ -25,8 +25,12 @@ #include <stdlib.h> #include <stdio.h> +#define _GNU_SOURCE +#define __USE_GNU +#include <string.h> #include <strings.h> #include <math.h> +#include <sys/mman.h> #include "debug.h" #include "alloc-inl.h" @@ -51,6 +55,66 @@ u8 last_intr = 0; #define AFL_PATH "/usr/local/lib/afl/" #endif +u32 check_binary_signatures(u8 *fn) { + + int ret = 0, fd = open(fn, O_RDONLY); + if (fd < 0) { PFATAL("Unable to open '%s'", fn); } + struct stat st; + if (fstat(fd, &st) < 0) { PFATAL("Unable to fstat '%s'", fn); } + u32 f_len = st.st_size; + u8 *f_data = mmap(0, f_len, PROT_READ, MAP_PRIVATE, fd, 0); + if (f_data == MAP_FAILED) { PFATAL("Unable to mmap file '%s'", fn); } + close(fd); + + if (memmem(f_data, f_len, PERSIST_SIG, strlen(PERSIST_SIG) + 1)) { + + if (!be_quiet) { OKF(cPIN "Persistent mode binary detected."); } + setenv(PERSIST_ENV_VAR, "1", 1); + ret = 1; + + } else if (getenv("AFL_PERSISTENT")) { + + if (!be_quiet) { + + WARNF("AFL_PERSISTENT is no longer supported and may misbehave!"); + + } + + } else if (getenv("AFL_FRIDA_PERSISTENT_ADDR")) { + + if (!be_quiet) { + + OKF("FRIDA Persistent mode configuration options detected."); + + } + + setenv(PERSIST_ENV_VAR, "1", 1); + ret = 1; + + } + + if (memmem(f_data, f_len, DEFER_SIG, strlen(DEFER_SIG) + 1)) { + + if (!be_quiet) { OKF(cPIN "Deferred forkserver binary detected."); } + setenv(DEFER_ENV_VAR, "1", 1); + ret += 2; + + } else if (getenv("AFL_DEFER_FORKSRV")) { + + if (!be_quiet) { + + WARNF("AFL_DEFER_FORKSRV is no longer supported and may misbehave!"); + + } + + } + + if (munmap(f_data, f_len)) { PFATAL("unmap() failed"); } + + return ret; + +} + void detect_file_args(char **argv, u8 *prog_in, bool *use_stdin) { u32 i = 0; @@ -140,6 +204,35 @@ void argv_cpy_free(char **argv) { } +/* Rewrite argv for CoreSight process tracer. */ + +char **get_cs_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv) { + + if (unlikely(getenv("AFL_CS_CUSTOM_BIN"))) { + + WARNF( + "AFL_CS_CUSTOM_BIN is enabled. " + "You must run your target under afl-cs-proxy on your own!"); + return argv; + + } + + char **new_argv = ck_alloc(sizeof(char *) * (argc + 4)); + if (unlikely(!new_argv)) { FATAL("Illegal amount of arguments specified"); } + + memcpy(&new_argv[3], &argv[1], (int)(sizeof(char *)) * (argc - 1)); + new_argv[argc + 3] = NULL; + + new_argv[2] = *target_path_p; + new_argv[1] = "--"; + + /* Now we need to actually find the cs-proxy binary to put in argv[0]. */ + + *target_path_p = new_argv[0] = find_afl_binary(own_loc, "afl-cs-proxy"); + return new_argv; + +} + /* Rewrite argv for QEMU. */ char **get_qemu_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv) { @@ -153,11 +246,10 @@ char **get_qemu_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv) { } - char **new_argv = ck_alloc(sizeof(char *) * (argc + 4)); + char **new_argv = ck_alloc(sizeof(char *) * (argc + 3)); if (unlikely(!new_argv)) { FATAL("Illegal amount of arguments specified"); } memcpy(&new_argv[3], &argv[1], (int)(sizeof(char *)) * (argc - 1)); - new_argv[argc + 3] = NULL; new_argv[2] = *target_path_p; new_argv[1] = "--"; @@ -173,11 +265,10 @@ char **get_qemu_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv) { char **get_wine_argv(u8 *own_loc, u8 **target_path_p, int argc, char **argv) { - char **new_argv = ck_alloc(sizeof(char *) * (argc + 3)); + char **new_argv = ck_alloc(sizeof(char *) * (argc + 2)); if (unlikely(!new_argv)) { FATAL("Illegal amount of arguments specified"); } memcpy(&new_argv[2], &argv[1], (int)(sizeof(char *)) * (argc - 1)); - new_argv[argc + 2] = NULL; new_argv[1] = *target_path_p; diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index c8c94c08..b871ea8c 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -19,7 +19,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code that implements a forkserver. This is used by the fuzzer as well the other components like afl-tmin. @@ -342,6 +342,16 @@ static void report_error_and_exit(int error) { "the fuzzing target reports that the mmap() call to the shared " "memory failed."); break; + case FS_ERROR_OLD_CMPLOG: + FATAL( + "the -c cmplog target was instrumented with an too old afl++ " + "version, you need to recompile it."); + break; + case FS_ERROR_OLD_CMPLOG_QEMU: + FATAL( + "The AFL++ QEMU/FRIDA loaders are from an older version, for -c you " + "need to recompile it.\n"); + break; default: FATAL("unknown error code %d from fuzzing target!", error); @@ -351,7 +361,7 @@ static void report_error_and_exit(int error) { /* Spins up fork server. The idea is explained here: - http://lcamtuf.blogspot.com/2014/10/fuzzing-binaries-without-execve.html + https://lcamtuf.blogspot.com/2014/10/fuzzing-binaries-without-execve.html In essence, the instrumentation allows us to skip execve(), and just keep cloning a stopped child. So, we just execute once, and then send commands @@ -603,19 +613,31 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, /* Wait for the fork server to come up, but don't wait too long. */ rlen = 0; - if (fsrv->exec_tmout) { + if (fsrv->init_tmout) { u32 time_ms = read_s32_timed(fsrv->fsrv_st_fd, &status, fsrv->init_tmout, stop_soon_p); if (!time_ms) { - kill(fsrv->fsrv_pid, fsrv->kill_signal); + s32 tmp_pid = fsrv->fsrv_pid; + if (tmp_pid > 0) { + + kill(tmp_pid, fsrv->kill_signal); + fsrv->fsrv_pid = -1; + + } } else if (time_ms > fsrv->init_tmout) { fsrv->last_run_timed_out = 1; - kill(fsrv->fsrv_pid, fsrv->kill_signal); + s32 tmp_pid = fsrv->fsrv_pid; + if (tmp_pid > 0) { + + kill(tmp_pid, fsrv->kill_signal); + fsrv->fsrv_pid = -1; + + } } else { @@ -651,6 +673,20 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if ((status & FS_OPT_OLD_AFLPP_WORKAROUND) == FS_OPT_OLD_AFLPP_WORKAROUND) status = (status & 0xf0ffffff); + if ((status & FS_OPT_NEWCMPLOG) == 0 && fsrv->cmplog_binary) { + + if (fsrv->qemu_mode || fsrv->frida_mode) { + + report_error_and_exit(FS_ERROR_OLD_CMPLOG_QEMU); + + } else { + + report_error_and_exit(FS_ERROR_OLD_CMPLOG); + + } + + } + if ((status & FS_OPT_SNAPSHOT) == FS_OPT_SNAPSHOT) { fsrv->snapshot = 1; @@ -905,8 +941,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, MSG_ULIMIT_USAGE " /path/to/fuzzed_app )\n\n" - " Tip: you can use http://jwilk.net/software/recidivm to " - "quickly\n" + " Tip: you can use https://jwilk.net/software/recidivm to\n" " estimate the required amount of virtual memory for the " "binary.\n\n" @@ -1005,7 +1040,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, MSG_ULIMIT_USAGE " /path/to/fuzzed_app )\n\n" - " Tip: you can use http://jwilk.net/software/recidivm to quickly\n" + " Tip: you can use https://jwilk.net/software/recidivm to\n" " estimate the required amount of virtual memory for the " "binary.\n\n" @@ -1248,7 +1283,14 @@ fsrv_run_result_t afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, /* If there was no response from forkserver after timeout seconds, we kill the child. The forkserver should inform us afterwards */ - kill(fsrv->child_pid, fsrv->kill_signal); + s32 tmp_pid = fsrv->child_pid; + if (tmp_pid > 0) { + + kill(tmp_pid, fsrv->kill_signal); + fsrv->child_pid = -1; + + } + fsrv->last_run_timed_out = 1; if (read(fsrv->fsrv_st_fd, &fsrv->child_status, 4) < 4) { exec_ms = 0; } @@ -1282,7 +1324,7 @@ fsrv_run_result_t afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, } - if (!WIFSTOPPED(fsrv->child_status)) { fsrv->child_pid = 0; } + if (!WIFSTOPPED(fsrv->child_status)) { fsrv->child_pid = -1; } fsrv->total_execs++; diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 0ae4d607..fa413dcf 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to @@ -58,7 +58,7 @@ void write_bitmap(afl_state_t *afl) { u32 count_bits(afl_state_t *afl, u8 *mem) { u32 *ptr = (u32 *)mem; - u32 i = (afl->fsrv.map_size >> 2); + u32 i = ((afl->fsrv.real_map_size + 3) >> 2); u32 ret = 0; while (i--) { @@ -68,7 +68,7 @@ u32 count_bits(afl_state_t *afl, u8 *mem) { /* This gets called on the inverse, virgin bitmap; optimize for sparse data. */ - if (v == 0xffffffff) { + if (likely(v == 0xffffffff)) { ret += 32; continue; @@ -92,14 +92,14 @@ u32 count_bits(afl_state_t *afl, u8 *mem) { u32 count_bytes(afl_state_t *afl, u8 *mem) { u32 *ptr = (u32 *)mem; - u32 i = (afl->fsrv.map_size >> 2); + u32 i = ((afl->fsrv.real_map_size + 3) >> 2); u32 ret = 0; while (i--) { u32 v = *(ptr++); - if (!v) { continue; } + if (likely(!v)) { continue; } if (v & 0x000000ffU) { ++ret; } if (v & 0x0000ff00U) { ++ret; } if (v & 0x00ff0000U) { ++ret; } @@ -117,7 +117,7 @@ u32 count_bytes(afl_state_t *afl, u8 *mem) { u32 count_non_255_bytes(afl_state_t *afl, u8 *mem) { u32 *ptr = (u32 *)mem; - u32 i = (afl->fsrv.map_size >> 2); + u32 i = ((afl->fsrv.real_map_size + 3) >> 2); u32 ret = 0; while (i--) { @@ -127,7 +127,7 @@ u32 count_non_255_bytes(afl_state_t *afl, u8 *mem) { /* This is called on the virgin bitmap, so optimize for the most likely case. */ - if (v == 0xffffffffU) { continue; } + if (likely(v == 0xffffffffU)) { continue; } if ((v & 0x000000ffU) != 0x000000ffU) { ++ret; } if ((v & 0x0000ff00U) != 0x0000ff00U) { ++ret; } if ((v & 0x00ff0000U) != 0x00ff0000U) { ++ret; } @@ -216,14 +216,14 @@ inline u8 has_new_bits(afl_state_t *afl, u8 *virgin_map) { u64 *current = (u64 *)afl->fsrv.trace_bits; u64 *virgin = (u64 *)virgin_map; - u32 i = (afl->fsrv.map_size >> 3); + u32 i = ((afl->fsrv.real_map_size + 7) >> 3); #else u32 *current = (u32 *)afl->fsrv.trace_bits; u32 *virgin = (u32 *)virgin_map; - u32 i = (afl->fsrv.map_size >> 2); + u32 i = ((afl->fsrv.real_map_size + 3) >> 2); #endif /* ^WORD_SIZE_64 */ @@ -317,8 +317,9 @@ u8 *describe_op(afl_state_t *afl, u8 new_bits, size_t max_description_len) { } - sprintf(ret + strlen(ret), ",time:%llu", - get_cur_time() + afl->prev_run_time - afl->start_time); + sprintf(ret + strlen(ret), ",time:%llu,execs:%llu", + get_cur_time() + afl->prev_run_time - afl->start_time, + afl->fsrv.total_execs); if (afl->current_custom_fuzz && afl->current_custom_fuzz->afl_custom_describe) { @@ -451,14 +452,12 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { if (unlikely(len == 0)) { return 0; } + u8 fn[PATH_MAX]; u8 *queue_fn = ""; - u8 new_bits = '\0'; + u8 new_bits = 0, keeping = 0, res, classified = 0; s32 fd; - u8 keeping = 0, res, classified = 0; u64 cksum = 0; - u8 fn[PATH_MAX]; - /* Update path frequency. */ /* Generating a hash on every input is super expensive. Bad idea and should diff --git a/src/afl-fuzz-cmplog.c b/src/afl-fuzz-cmplog.c index c2e9c80f..6fc926f0 100644 --- a/src/afl-fuzz-cmplog.c +++ b/src/afl-fuzz-cmplog.c @@ -17,7 +17,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code to handle the shared memory. This is used by the fuzzer as well the other components like afl-tmin, afl-showmap, etc... diff --git a/src/afl-fuzz-extras.c b/src/afl-fuzz-extras.c index 584241d4..0f0fe331 100644 --- a/src/afl-fuzz-extras.c +++ b/src/afl-fuzz-extras.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 9bb25785..4c030c0a 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to @@ -974,8 +974,7 @@ void perform_dry_run(afl_state_t *afl) { MSG_ULIMIT_USAGE " /path/to/binary [...] <testcase )\n\n" - " Tip: you can use http://jwilk.net/software/recidivm to " - "quickly\n" + " Tip: you can use https://jwilk.net/software/recidivm to\n" " estimate the required amount of virtual memory for the " "binary. Also,\n" " if you are using ASAN, set '-m 0'.\n\n" @@ -1325,8 +1324,8 @@ void pivot_inputs(afl_state_t *afl) { } - nfn = alloc_printf("%s/queue/id:%06u,time:0,orig:%s", afl->out_dir, id, - use_name); + nfn = alloc_printf("%s/queue/id:%06u,time:0,execs:%llu,orig:%s", + afl->out_dir, id, afl->fsrv.total_execs, use_name); #else @@ -2645,6 +2644,7 @@ void check_binary(afl_state_t *afl, u8 *fname) { if (afl->afl_env.afl_skip_bin_check || afl->use_wine || afl->unicorn_mode || (afl->fsrv.qemu_mode && getenv("AFL_QEMU_CUSTOM_BIN")) || + (afl->fsrv.cs_mode && getenv("AFL_CS_CUSTOM_BIN")) || afl->non_instrumented_mode) { return; @@ -2721,7 +2721,7 @@ void check_binary(afl_state_t *afl, u8 *fname) { #endif /* ^!__APPLE__ */ if (!afl->fsrv.qemu_mode && !afl->fsrv.frida_mode && !afl->unicorn_mode && - !afl->non_instrumented_mode && + !afl->fsrv.cs_mode && !afl->non_instrumented_mode && !memmem(f_data, f_len, SHM_ENV_VAR, strlen(SHM_ENV_VAR) + 1)) { SAYF("\n" cLRD "[-] " cRST @@ -2752,7 +2752,7 @@ void check_binary(afl_state_t *afl, u8 *fname) { } - if ((afl->fsrv.qemu_mode || afl->fsrv.frida_mode) && + if ((afl->fsrv.cs_mode || afl->fsrv.qemu_mode || afl->fsrv.frida_mode) && memmem(f_data, f_len, SHM_ENV_VAR, strlen(SHM_ENV_VAR) + 1)) { SAYF("\n" cLRD "[-] " cRST @@ -2815,43 +2815,6 @@ void check_binary(afl_state_t *afl, u8 *fname) { } -/* Trim and possibly create a banner for the run. */ - -void fix_up_banner(afl_state_t *afl, u8 *name) { - - if (!afl->use_banner) { - - if (afl->sync_id) { - - afl->use_banner = afl->sync_id; - - } else { - - u8 *trim = strrchr(name, '/'); - if (!trim) { - - afl->use_banner = name; - - } else { - - afl->use_banner = trim + 1; - - } - - } - - } - - if (strlen(afl->use_banner) > 32) { - - u8 *tmp = ck_alloc(36); - sprintf(tmp, "%.32s...", afl->use_banner); - afl->use_banner = tmp; - - } - -} - /* Check if we're on TTY. */ void check_if_tty(afl_state_t *afl) { diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 5332b9fe..e0dfd6b0 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to @@ -255,6 +255,7 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { mutator->afl_custom_init_trim = dlsym(dh, "afl_custom_init_trim"); if (!mutator->afl_custom_init_trim) { + notrim = 1; ACTF("optional symbol 'afl_custom_init_trim' not found."); } @@ -263,6 +264,7 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { mutator->afl_custom_trim = dlsym(dh, "afl_custom_trim"); if (!mutator->afl_custom_trim) { + notrim = 1; ACTF("optional symbol 'afl_custom_trim' not found."); } @@ -271,6 +273,7 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { mutator->afl_custom_post_trim = dlsym(dh, "afl_custom_post_trim"); if (!mutator->afl_custom_post_trim) { + notrim = 1; ACTF("optional symbol 'afl_custom_post_trim' not found."); } diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 17749601..f4d3b77f 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to @@ -448,11 +448,11 @@ u8 fuzz_one_original(afl_state_t *afl) { ACTF( "Fuzzing test case #%u (%u total, %llu uniq crashes found, " - "perf_score=%0.0f, exec_us=%llu, hits=%u, map=%u)...", + "perf_score=%0.0f, exec_us=%llu, hits=%u, map=%u, ascii=%u)...", afl->current_entry, afl->queued_paths, afl->unique_crashes, afl->queue_cur->perf_score, afl->queue_cur->exec_us, likely(afl->n_fuzz) ? afl->n_fuzz[afl->queue_cur->n_fuzz_entry] : 0, - afl->queue_cur->bitmap_size); + afl->queue_cur->bitmap_size, afl->queue_cur->is_ascii); fflush(stdout); } @@ -2003,11 +2003,16 @@ havoc_stage: where we take the input file and make random stacked tweaks. */ #define MAX_HAVOC_ENTRY 59 /* 55 to 60 */ +#define MUTATE_ASCII_DICT 64 u32 r_max, r; r_max = (MAX_HAVOC_ENTRY + 1) + (afl->extras_cnt ? 4 : 0) + - (afl->a_extras_cnt ? 4 : 0); + (afl->a_extras_cnt + ? (unlikely(afl->cmplog_binary && afl->queue_cur->is_ascii) + ? MUTATE_ASCII_DICT + : 4) + : 0); if (unlikely(afl->expand_havoc && afl->ready_for_splicing_count > 1)) { @@ -2592,7 +2597,15 @@ havoc_stage: if (afl->a_extras_cnt) { - if (r < 2) { + u32 r_cmp = 2; + + if (unlikely(afl->cmplog_binary && afl->queue_cur->is_ascii)) { + + r_cmp = MUTATE_ASCII_DICT >> 1; + + } + + if (r < r_cmp) { /* Use the dictionary. */ @@ -2612,7 +2625,7 @@ havoc_stage: break; - } else if (r < 4) { + } else if (r < (r_cmp << 1)) { u32 use_extra = rand_below(afl, afl->a_extras_cnt); u32 extra_len = afl->a_extras[use_extra].len; @@ -2641,7 +2654,7 @@ havoc_stage: } else { - r -= 4; + r -= (r_cmp << 1); } diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index 065977c0..6484768b 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 16af2c6b..1523d556 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -14,7 +14,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to @@ -315,7 +315,96 @@ void mark_as_redundant(afl_state_t *afl, struct queue_entry *q, u8 state) { } -/* check if ascii or UTF-8 */ +/* check if pointer is ascii or UTF-8 */ + +u8 check_if_text_buf(u8 *buf, u32 len) { + + u32 offset = 0, ascii = 0, utf8 = 0; + + while (offset < len) { + + // ASCII: <= 0x7F to allow ASCII control characters + if ((buf[offset + 0] == 0x09 || buf[offset + 0] == 0x0A || + buf[offset + 0] == 0x0D || + (0x20 <= buf[offset + 0] && buf[offset + 0] <= 0x7E))) { + + offset++; + utf8++; + ascii++; + continue; + + } + + if (isascii((int)buf[offset]) || isprint((int)buf[offset])) { + + ascii++; + // we continue though as it can also be a valid utf8 + + } + + // non-overlong 2-byte + if (len - offset > 1 && + ((0xC2 <= buf[offset + 0] && buf[offset + 0] <= 0xDF) && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF))) { + + offset += 2; + utf8++; + continue; + + } + + // excluding overlongs + if ((len - offset > 2) && + ((buf[offset + 0] == 0xE0 && + (0xA0 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) && + (0x80 <= buf[offset + 2] && + buf[offset + 2] <= 0xBF)) || // straight 3-byte + (((0xE1 <= buf[offset + 0] && buf[offset + 0] <= 0xEC) || + buf[offset + 0] == 0xEE || buf[offset + 0] == 0xEF) && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) && + (0x80 <= buf[offset + 2] && + buf[offset + 2] <= 0xBF)) || // excluding surrogates + (buf[offset + 0] == 0xED && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0x9F) && + (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF)))) { + + offset += 3; + utf8++; + continue; + + } + + // planes 1-3 + if ((len - offset > 3) && + ((buf[offset + 0] == 0xF0 && + (0x90 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) && + (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF) && + (0x80 <= buf[offset + 3] && + buf[offset + 3] <= 0xBF)) || // planes 4-15 + ((0xF1 <= buf[offset + 0] && buf[offset + 0] <= 0xF3) && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0xBF) && + (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF) && + (0x80 <= buf[offset + 3] && buf[offset + 3] <= 0xBF)) || // plane 16 + (buf[offset + 0] == 0xF4 && + (0x80 <= buf[offset + 1] && buf[offset + 1] <= 0x8F) && + (0x80 <= buf[offset + 2] && buf[offset + 2] <= 0xBF) && + (0x80 <= buf[offset + 3] && buf[offset + 3] <= 0xBF)))) { + + offset += 4; + utf8++; + continue; + + } + + offset++; + + } + + return (utf8 > ascii ? utf8 : ascii); + +} + +/* check if queue entry is ascii or UTF-8 */ static u8 check_if_text(afl_state_t *afl, struct queue_entry *q) { diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 268f726c..0a6e5eee 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -17,7 +17,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code to handle the shared memory. This is used by the fuzzer as well the other components like afl-tmin, afl-showmap, etc... @@ -45,6 +45,23 @@ enum { }; +// add to dictionary enum +// DEFAULT = 1, notTXT = 2, FOUND = 4, notSAME = 8 +enum { + + DICT_ADD_NEVER = 0, + DICT_ADD_NOTFOUND_SAME_TXT = 1, + DICT_ADD_NOTFOUND_SAME = 3, + DICT_ADD_FOUND_SAME_TXT = 5, + DICT_ADD_FOUND_SAME = 7, + DICT_ADD_NOTFOUND_TXT = 9, + DICT_ADD_NOTFOUND = 11, + DICT_ADD_FOUND_TXT = 13, + DICT_ADD_FOUND = 15, + DICT_ADD_ANY = DICT_ADD_FOUND + +}; + // CMPLOG LVL enum { @@ -54,6 +71,8 @@ enum { }; +#define DICT_ADD_STRATEGY DICT_ADD_FOUND_SAME + struct range { u32 start; @@ -64,6 +83,10 @@ struct range { }; +static u32 hshape; +static u64 screen_update; +static u64 last_update; + static struct range *add_range(struct range *ranges, u32 start, u32 end) { struct range *r = ck_alloc_nozero(sizeof(struct range)); @@ -252,7 +275,6 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len, u64 start_time = get_cur_time(); #endif - u32 screen_update; u64 orig_hit_cnt, new_hit_cnt, exec_cksum; orig_hit_cnt = afl->queued_paths + afl->unique_crashes; @@ -261,24 +283,6 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len, afl->stage_max = (len << 1); afl->stage_cur = 0; - if (likely(afl->queue_cur->exec_us)) { - - if (likely((100000 / 2) >= afl->queue_cur->exec_us)) { - - screen_update = 100000 / afl->queue_cur->exec_us; - - } else { - - screen_update = 1; - - } - - } else { - - screen_update = 100000; - - } - // in colorization we do not classify counts, hence we have to calculate // the original checksum. if (unlikely(get_exec_checksum(afl, buf, len, &exec_cksum))) { @@ -348,7 +352,7 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len, } - if (++afl->stage_cur % screen_update) { show_stats(afl); }; + if (++afl->stage_cur % screen_update == 0) { show_stats(afl); }; } @@ -440,10 +444,10 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len, fprintf( f, "Colorization: fname=%s len=%u ms=%llu result=%u execs=%u found=%llu " - "taint=%u\n", + "taint=%u ascii=%u auto_extra_before=%u\n", afl->queue_cur->fname, len, get_cur_time() - start_time, afl->queue_cur->colorized, afl->stage_cur, new_hit_cnt - orig_hit_cnt, - positions); + positions, afl->queue_cur->is_ascii ? 1 : 0, afl->a_extras_cnt); #ifndef _DEBUG if (afl->not_on_tty) { fclose(f); } @@ -759,11 +763,18 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u32 its_len = MIN(len - idx, taint_len); + if (afl->fsrv.total_execs - last_update > screen_update) { + + show_stats(afl); + last_update = afl->fsrv.total_execs; + + } + // fprintf(stderr, // "Encode: %llx->%llx into %llx(<-%llx) at idx=%u " // "taint_len=%u shape=%u attr=%u\n", // o_pattern, pattern, repl, changed_val, idx, taint_len, - // h->shape + 1, attr); + // hshape, attr); //#ifdef CMPLOG_SOLVE_TRANSFORM // reverse atoi()/strnu?toll() is expensive, so we only to it in lvl 3 @@ -845,7 +856,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u64 b_val, o_b_val, mask; u8 bytes; - switch (SHAPE_BYTES(h->shape)) { + switch (hshape) { case 0: case 1: @@ -924,7 +935,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, s64 diff = pattern - b_val; s64 o_diff = o_pattern - o_b_val; /* fprintf(stderr, "DIFF1 idx=%03u shape=%02u %llx-%llx=%lx\n", idx, - h->shape + 1, o_pattern, o_b_val, o_diff); + hshape, o_pattern, o_b_val, o_diff); fprintf(stderr, "DIFF1 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff); */ if (diff == o_diff && diff) { @@ -953,7 +964,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, s64 o_diff = o_pattern ^ o_b_val; /* fprintf(stderr, "DIFF2 idx=%03u shape=%02u %llx-%llx=%lx\n", - idx, h->shape + 1, o_pattern, o_b_val, o_diff); + idx, hshape, o_pattern, o_b_val, o_diff); fprintf(stderr, "DIFF2 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff); */ @@ -1002,7 +1013,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } /* fprintf(stderr, "DIFF3 idx=%03u shape=%02u %llx-%llx=%lx\n", - idx, h->shape + 1, o_pattern, o_b_val, o_diff); + idx, hshape, o_pattern, o_b_val, o_diff); fprintf(stderr, "DIFF3 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff); */ @@ -1051,7 +1062,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } /* fprintf(stderr, "DIFF4 idx=%03u shape=%02u %llx-%llx=%lx\n", - idx, h->shape + 1, o_pattern, o_b_val, o_diff); + idx, hshape, o_pattern, o_b_val, o_diff); fprintf(stderr, "DIFF4 %016llx %llx-%llx=%lx\n", repl, pattern, b_val, diff); */ @@ -1089,7 +1100,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, if ((lvl & LVL1) || attr >= IS_FP_MOD) { - if (SHAPE_BYTES(h->shape) >= 8 && *status != 1) { + if (hshape >= 8 && *status != 1) { // if (its_len >= 8) // fprintf(stderr, @@ -1132,7 +1143,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - if (SHAPE_BYTES(h->shape) >= 4 && *status != 1) { + if (hshape >= 4 && *status != 1) { // if (its_len >= 4 && (attr <= 1 || attr >= 8)) // fprintf(stderr, @@ -1173,7 +1184,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - if (SHAPE_BYTES(h->shape) >= 2 && *status != 1) { + if (hshape >= 2 && *status != 1) { if (its_len >= 2 && ((*buf_16 == (u16)pattern && *o_buf_16 == (u16)o_pattern) || @@ -1244,11 +1255,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - if (!(attr & (IS_GREATER | IS_LESSER)) || SHAPE_BYTES(h->shape) < 4) { - - return 0; - - } + if (!(attr & (IS_GREATER | IS_LESSER)) || hshape < 4) { return 0; } // transform >= to < and <= to > if ((attr & IS_EQUAL) && (attr & (IS_GREATER | IS_LESSER))) { @@ -1272,7 +1279,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, if (attr & IS_GREATER) { - if (SHAPE_BYTES(h->shape) == 4 && its_len >= 4) { + if (hshape == 4 && its_len >= 4) { float *f = (float *)&repl; float g = *f; @@ -1280,7 +1287,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u32 *r = (u32 *)&g; repl_new = (u32)*r; - } else if (SHAPE_BYTES(h->shape) == 8 && its_len >= 8) { + } else if (hshape == 8 && its_len >= 8) { double *f = (double *)&repl; double g = *f; @@ -1307,7 +1314,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } else { - if (SHAPE_BYTES(h->shape) == 4) { + if (hshape == 4) { float *f = (float *)&repl; float g = *f; @@ -1315,7 +1322,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u32 *r = (u32 *)&g; repl_new = (u32)*r; - } else if (SHAPE_BYTES(h->shape) == 8) { + } else if (hshape == 8) { double *f = (double *)&repl; double g = *f; @@ -1342,7 +1349,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } // transform double to float, llvm likes to do that internally ... - if (SHAPE_BYTES(h->shape) == 8 && its_len >= 4) { + if (hshape == 8 && its_len >= 4) { double *f = (double *)&repl; float g = (float)*f; @@ -1353,7 +1360,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, memcpy(((char *)&repl_new) + 4, (char *)&g, 4); #endif changed_val = repl_new; - h->shape = 3; // modify shape + hshape = 4; // modify shape // fprintf(stderr, "DOUBLE2FLOAT %llx\n", repl_new); @@ -1361,12 +1368,12 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, afl, h, pattern, repl_new, o_pattern, changed_val, 16, idx, taint_len, orig_buf, buf, cbuf, len, 1, lvl, status))) { - h->shape = 7; // recover shape + hshape = 8; // recover shape return 1; } - h->shape = 7; // recover shape + hshape = 8; // recover shape } @@ -1421,6 +1428,13 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, u32 taint_len, u8 *orig_buf, u8 *buf, u8 *cbuf, u32 len, u8 do_reverse, u8 lvl, u8 *status) { + if (afl->fsrv.total_execs - last_update > screen_update) { + + show_stats(afl); + last_update = afl->fsrv.total_execs; + + } + u8 *ptr = (u8 *)&buf[idx]; u8 *o_ptr = (u8 *)&orig_buf[idx]; u8 *p = (u8 *)&pattern; @@ -1428,52 +1442,51 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, u8 *r = (u8 *)&repl; u8 backup[16]; u32 its_len = MIN(len - idx, taint_len); - u32 shape = h->shape + 1; #if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) size_t off = 0; #else - size_t off = 16 - shape; + size_t off = 16 - hshape; #endif - if (its_len >= shape) { + if (its_len >= hshape) { #ifdef _DEBUG fprintf(stderr, "TestUN: %u>=%u (len=%u idx=%u attr=%u off=%lu) (%u) ", - its_len, shape, len, idx, attr, off, do_reverse); + its_len, hshape, len, idx, attr, off, do_reverse); u32 i; u8 *o_r = (u8 *)&changed_val; - for (i = 0; i < shape; i++) + for (i = 0; i < hshape; i++) fprintf(stderr, "%02x", ptr[i]); fprintf(stderr, "=="); - for (i = 0; i < shape; i++) + for (i = 0; i < hshape; i++) fprintf(stderr, "%02x", p[off + i]); fprintf(stderr, " "); - for (i = 0; i < shape; i++) + for (i = 0; i < hshape; i++) fprintf(stderr, "%02x", o_ptr[i]); fprintf(stderr, "=="); - for (i = 0; i < shape; i++) + for (i = 0; i < hshape; i++) fprintf(stderr, "%02x", o_p[off + i]); fprintf(stderr, " <= "); - for (i = 0; i < shape; i++) + for (i = 0; i < hshape; i++) fprintf(stderr, "%02x", r[off + i]); fprintf(stderr, "<-"); - for (i = 0; i < shape; i++) + for (i = 0; i < hshape; i++) fprintf(stderr, "%02x", o_r[off + i]); fprintf(stderr, "\n"); #endif - if (!memcmp(ptr, p + off, shape) && !memcmp(o_ptr, o_p + off, shape)) { + if (!memcmp(ptr, p + off, hshape) && !memcmp(o_ptr, o_p + off, hshape)) { - memcpy(backup, ptr, shape); - memcpy(ptr, r + off, shape); + memcpy(backup, ptr, hshape); + memcpy(ptr, r + off, hshape); if (unlikely(its_fuzz(afl, buf, len, status))) { return 1; } #ifdef CMPLOG_COMBINE - if (*status == 1) { memcpy(cbuf + idx, r, shape); } + if (*status == 1) { memcpy(cbuf + idx, r, hshape); } #endif - memcpy(ptr, backup, shape); + memcpy(ptr, backup, hshape); #ifdef _DEBUG fprintf(stderr, "Status=%u\n", *status); @@ -1485,10 +1498,10 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, if (do_reverse && *status != 1) { if (unlikely(cmp_extend_encodingN( - afl, h, SWAPN(pattern, (shape << 3)), SWAPN(repl, (shape << 3)), - SWAPN(o_pattern, (shape << 3)), SWAPN(changed_val, (shape << 3)), - attr, idx, taint_len, orig_buf, buf, cbuf, len, 0, lvl, - status))) { + afl, h, SWAPN(pattern, (hshape << 3)), SWAPN(repl, (hshape << 3)), + SWAPN(o_pattern, (hshape << 3)), + SWAPN(changed_val, (hshape << 3)), attr, idx, taint_len, orig_buf, + buf, cbuf, len, 0, lvl, status))) { return 1; @@ -1615,6 +1628,8 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, u8 s_v0_inc = 1, s_v1_inc = 1; u8 s_v0_dec = 1, s_v1_dec = 1; + hshape = SHAPE_BYTES(h->shape); + if (h->hits > CMP_MAP_H) { loggeds = CMP_MAP_H; @@ -1626,7 +1641,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } #ifdef WORD_SIZE_64 - switch (SHAPE_BYTES(h->shape)) { + switch (hshape) { case 1: case 2: @@ -1669,7 +1684,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, for (j = 0; j < i; ++j) { if (afl->shm.cmp_map->log[key][j].v0 == o->v0 && - afl->shm.cmp_map->log[key][i].v1 == o->v1) { + afl->shm.cmp_map->log[key][j].v1 == o->v1) { goto cmp_fuzz_next_iter; @@ -1679,8 +1694,7 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, #ifdef _DEBUG fprintf(stderr, "Handling: %llx->%llx vs %llx->%llx attr=%u shape=%u\n", - orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute, - SHAPE_BYTES(h->shape)); + orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute, hshape); #endif t = taint; @@ -1830,27 +1844,41 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, "END: %llx->%llx vs %llx->%llx attr=%u i=%u found=%u " "isN=%u size=%u\n", orig_o->v0, o->v0, orig_o->v1, o->v1, h->attribute, i, found_one, - is_n, SHAPE_BYTES(h->shape)); + is_n, hshape); #endif - // If failed, add to dictionary - if (!found_one) { + // we only learn 16 bit + + if (hshape > 1) { - if (afl->pass_stats[key].total == 0) { + if (!found_one || afl->queue_cur->is_ascii) { #ifdef WORD_SIZE_64 if (unlikely(is_n)) { - try_to_add_to_dictN(afl, s128_v0, SHAPE_BYTES(h->shape)); - try_to_add_to_dictN(afl, s128_v1, SHAPE_BYTES(h->shape)); + if (!found_one || + check_if_text_buf((u8 *)&s128_v0, SHAPE_BYTES(h->shape)) == + SHAPE_BYTES(h->shape)) + try_to_add_to_dictN(afl, s128_v0, SHAPE_BYTES(h->shape)); + if (!found_one || + check_if_text_buf((u8 *)&s128_v1, SHAPE_BYTES(h->shape)) == + SHAPE_BYTES(h->shape)) + try_to_add_to_dictN(afl, s128_v1, SHAPE_BYTES(h->shape)); } else #endif { - try_to_add_to_dict(afl, o->v0, SHAPE_BYTES(h->shape)); - try_to_add_to_dict(afl, o->v1, SHAPE_BYTES(h->shape)); + if (!memcmp((u8 *)&o->v0, (u8 *)&orig_o->v0, SHAPE_BYTES(h->shape)) && + (!found_one || + check_if_text_buf((u8 *)&o->v0, SHAPE_BYTES(h->shape)) == + SHAPE_BYTES(h->shape))) + try_to_add_to_dict(afl, o->v0, SHAPE_BYTES(h->shape)); + if (!memcmp((u8 *)&o->v1, (u8 *)&orig_o->v1, SHAPE_BYTES(h->shape)) && + (!found_one || + check_if_text_buf((u8 *)&o->v1, SHAPE_BYTES(h->shape)) == + SHAPE_BYTES(h->shape))) + try_to_add_to_dict(afl, o->v1, SHAPE_BYTES(h->shape)); } @@ -1882,8 +1910,9 @@ static u8 cmp_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } -static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl, - u8 *o_pattern, u8 *changed_val, u8 plen, u32 idx, +static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, + struct cmpfn_operands *o, + struct cmpfn_operands *orig_o, u32 idx, u32 taint_len, u8 *orig_buf, u8 *buf, u8 *cbuf, u32 len, u8 lvl, u8 *status) { @@ -1894,9 +1923,60 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl, // (void)(changed_val); //#endif + if (afl->fsrv.total_execs - last_update > screen_update) { + + show_stats(afl); + last_update = afl->fsrv.total_execs; + + } + + u8 *pattern, *repl, *o_pattern, *changed_val; + u8 l0, l1, ol0, ol1; + + if (entry == 0) { + + pattern = o->v0; + repl = o->v1; + o_pattern = orig_o->v0; + changed_val = orig_o->v1; + l0 = o->v0_len; + ol0 = orig_o->v0_len; + l1 = o->v1_len; + ol1 = orig_o->v1_len; + + } else { + + pattern = o->v1; + repl = o->v0; + o_pattern = orig_o->v1; + changed_val = orig_o->v0; + l0 = o->v1_len; + ol0 = orig_o->v1_len; + l1 = o->v0_len; + ol1 = orig_o->v0_len; + + } + + if (l0 >= 0x80 || ol0 >= 0x80) { + + l0 -= 0x80; + l1 -= 0x80; + ol0 -= 0x80; + ol1 -= 0x80; + + } + + if (l0 == 0 || l1 == 0 || ol0 == 0 || ol1 == 0 || l0 > 31 || l1 > 31 || + ol0 > 31 || ol1 > 31) { + + l0 = l1 = ol0 = ol1 = hshape; + + } + + u8 lmax = MAX(l0, ol0); u8 save[40]; u32 saved_idx = idx, pre, from = 0, to = 0, i, j; - u32 its_len = MIN((u32)plen, len - idx); + u32 its_len = MIN(MIN(lmax, hshape), len - idx); its_len = MIN(its_len, taint_len); u32 saved_its_len = its_len; @@ -1912,7 +1992,8 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl, (void)(j); #ifdef _DEBUG - fprintf(stderr, "RTN T idx=%u lvl=%02x ", idx, lvl); + fprintf(stderr, "RTN T idx=%u lvl=%02x is_txt=%u shape=%u/%u ", idx, lvl, + o->v0_len >= 0x80 ? 1 : 0, hshape, l0); for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_buf[idx + j]); fprintf(stderr, " -> "); @@ -1972,10 +2053,10 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 *pattern, u8 *repl, } - //#ifdef CMPLOG_SOLVE_TRANSFORM - if (*status == 1) return 0; + // transform solving + if (afl->cmplog_enable_transform && (lvl & LVL3)) { u32 toupper = 0, tolower = 0, xor = 0, arith = 0, tohex = 0, fromhex = 0; @@ -2322,6 +2403,8 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, u32 i, j, idx, have_taint = 1, taint_len, loggeds; u8 status = 0, found_one = 0; + hshape = SHAPE_BYTES(h->shape); + if (h->hits > CMP_MAP_RTN_H) { loggeds = CMP_MAP_RTN_H; @@ -2353,18 +2436,22 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } /* - struct cmp_header *hh = &afl->orig_cmp_map->headers[key]; - fprintf(stderr, "RTN N hits=%u id=%u shape=%u attr=%u v0=", h->hits, - h->id, h->shape, h->attribute); - for (j = 0; j < 8; j++) fprintf(stderr, "%02x", o->v0[j]); - fprintf(stderr, " v1="); - for (j = 0; j < 8; j++) fprintf(stderr, "%02x", o->v1[j]); - fprintf(stderr, "\nRTN O hits=%u id=%u shape=%u attr=%u o0=", - hh->hits, hh->id, hh->shape, hh->attribute); - for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_o->v0[j]); - fprintf(stderr, " o1="); - for (j = 0; j < 8; j++) fprintf(stderr, "%02x", orig_o->v1[j]); - fprintf(stderr, "\n"); + struct cmp_header *hh = &afl->orig_cmp_map->headers[key]; + fprintf(stderr, "RTN N hits=%u id=%u shape=%u attr=%u v0=", h->hits, h->id, + hshape, h->attribute); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", o->v0[j]); + fprintf(stderr, " v1="); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", o->v1[j]); + fprintf(stderr, "\nRTN O hits=%u id=%u shape=%u attr=%u o0=", hh->hits, + hh->id, hshape, hh->attribute); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", orig_o->v0[j]); + fprintf(stderr, " o1="); + for (j = 0; j < 8; j++) + fprintf(stderr, "%02x", orig_o->v1[j]); + fprintf(stderr, "\n"); */ t = taint; @@ -2400,25 +2487,24 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, #ifdef _DEBUG int w; - fprintf(stderr, "key=%u idx=%u len=%u o0=", key, idx, - SHAPE_BYTES(h->shape)); - for (w = 0; w < SHAPE_BYTES(h->shape); ++w) + fprintf(stderr, "key=%u idx=%u len=%u o0=", key, idx, hshape); + for (w = 0; w < hshape; ++w) fprintf(stderr, "%02x", orig_o->v0[w]); fprintf(stderr, " v0="); - for (w = 0; w < SHAPE_BYTES(h->shape); ++w) + for (w = 0; w < hshape; ++w) fprintf(stderr, "%02x", o->v0[w]); fprintf(stderr, " o1="); - for (w = 0; w < SHAPE_BYTES(h->shape); ++w) + for (w = 0; w < hshape; ++w) fprintf(stderr, "%02x", orig_o->v1[w]); fprintf(stderr, " v1="); - for (w = 0; w < SHAPE_BYTES(h->shape); ++w) + for (w = 0; w < hshape; ++w) fprintf(stderr, "%02x", o->v1[w]); fprintf(stderr, "\n"); #endif - if (unlikely(rtn_extend_encoding( - afl, o->v0, o->v1, orig_o->v0, orig_o->v1, SHAPE_BYTES(h->shape), - idx, taint_len, orig_buf, buf, cbuf, len, lvl, &status))) { + if (unlikely(rtn_extend_encoding(afl, 0, o, orig_o, idx, taint_len, + orig_buf, buf, cbuf, len, lvl, + &status))) { return 1; @@ -2433,9 +2519,9 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, status = 0; - if (unlikely(rtn_extend_encoding( - afl, o->v1, o->v0, orig_o->v1, orig_o->v0, SHAPE_BYTES(h->shape), - idx, taint_len, orig_buf, buf, cbuf, len, lvl, &status))) { + if (unlikely(rtn_extend_encoding(afl, 1, o, orig_o, idx, taint_len, + orig_buf, buf, cbuf, len, lvl, + &status))) { return 1; @@ -2450,16 +2536,42 @@ static u8 rtn_fuzz(afl_state_t *afl, u32 key, u8 *orig_buf, u8 *buf, u8 *cbuf, } - // If failed, add to dictionary - if (!found_one && (lvl & LVL1)) { + // if (unlikely(!afl->pass_stats[key].total)) { + + if ((!found_one && (lvl & LVL1)) || afl->queue_cur->is_ascii) { + + // if (unlikely(!afl->pass_stats[key].total)) { + + u32 shape_len = SHAPE_BYTES(h->shape); + u32 v0_len = shape_len, v1_len = shape_len; + if (afl->queue_cur->is_ascii || + check_if_text_buf((u8 *)&o->v0, shape_len) == shape_len) { + + if (strlen(o->v0)) v0_len = strlen(o->v0); + + } - if (unlikely(!afl->pass_stats[key].total)) { + if (afl->queue_cur->is_ascii || + check_if_text_buf((u8 *)&o->v1, shape_len) == shape_len) { - maybe_add_auto(afl, o->v0, SHAPE_BYTES(h->shape)); - maybe_add_auto(afl, o->v1, SHAPE_BYTES(h->shape)); + if (strlen(o->v1)) v1_len = strlen(o->v1); } + // fprintf(stderr, "SHOULD: found:%u ascii:%u text?%u:%u %u:%s %u:%s \n", + // found_one, afl->queue_cur->is_ascii, check_if_text_buf((u8 *)&o->v0, + // shape_len), check_if_text_buf((u8 *)&o->v1, shape_len), v0_len, + // o->v0, v1_len, o->v1); + + if (!memcmp(o->v0, orig_o->v0, v0_len) || + (!found_one || check_if_text_buf((u8 *)&o->v0, v0_len) == v0_len)) + maybe_add_auto(afl, o->v0, v0_len); + if (!memcmp(o->v1, orig_o->v1, v1_len) || + (!found_one || check_if_text_buf((u8 *)&o->v1, v1_len) == v1_len)) + maybe_add_auto(afl, o->v1, v1_len); + + //} + } rtn_fuzz_next_iter: @@ -2492,6 +2604,23 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { } struct tainted *taint = NULL; + if (likely(afl->queue_cur->exec_us)) { + + if (likely((100000 / 2) >= afl->queue_cur->exec_us)) { + + screen_update = 100000 / afl->queue_cur->exec_us; + + } else { + + screen_update = 1; + + } + + } else { + + screen_update = 100000; + + } if (!afl->queue_cur->taint || !afl->queue_cur->cmplog_colorinput) { @@ -2592,8 +2721,6 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { u64 orig_hit_cnt, new_hit_cnt; u64 orig_execs = afl->fsrv.total_execs; orig_hit_cnt = afl->queued_paths + afl->unique_crashes; - u64 screen_update = 100000 / afl->queue_cur->exec_us, - execs = afl->fsrv.total_execs; afl->stage_name = "input-to-state"; afl->stage_short = "its"; @@ -2630,11 +2757,13 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { if (afl->shm.cmp_map->headers[k].type == CMP_TYPE_INS) { + // fprintf(stderr, "INS %u\n", k); afl->stage_max += MIN((u32)(afl->shm.cmp_map->headers[k].hits), (u32)CMP_MAP_H); } else { + // fprintf(stderr, "RTN %u\n", k); afl->stage_max += MIN((u32)(afl->shm.cmp_map->headers[k].hits), (u32)CMP_MAP_RTN_H); @@ -2673,13 +2802,6 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { } - if (afl->fsrv.total_execs - execs > screen_update) { - - execs = afl->fsrv.total_execs; - show_stats(afl); - - } - } r = 0; @@ -2795,9 +2917,10 @@ exit_its: if (f) { fprintf(f, - "Cmplog: fname=%s len=%u ms=%llu result=%u finds=%llu entries=%u\n", + "Cmplog: fname=%s len=%u ms=%llu result=%u finds=%llu entries=%u " + "auto_extra_after=%u\n", afl->queue_cur->fname, len, get_cur_time() - start_time, r, - new_hit_cnt - orig_hit_cnt, cmp_locations); + new_hit_cnt - orig_hit_cnt, cmp_locations, afl->a_extras_cnt); #ifndef _DEBUG if (afl->not_on_tty) { fclose(f); } diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 4173f4e1..2789b56f 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -16,7 +16,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to @@ -291,8 +291,6 @@ static void write_with_gap(afl_state_t *afl, u8 *mem, u32 len, u32 skip_at, u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem, u32 handicap, u8 from_queue) { - if (unlikely(afl->shm.cmplog_mode)) { q->exec_cksum = 0; } - u8 fault = 0, new_bits = 0, var_detected = 0, hnb = 0, first_run = (q->exec_cksum == 0); u64 start_us, stop_us, diff_us; @@ -300,6 +298,8 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem, u32 use_tmout = afl->fsrv.exec_tmout; u8 *old_sn = afl->stage_name; + if (unlikely(afl->shm.cmplog_mode)) { q->exec_cksum = 0; } + /* Be a bit more generous about timeouts when resuming sessions, or when trying to calibrate already-added finds. This helps avoid trouble due to intermittent latency. */ @@ -343,6 +343,32 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem, } + /* we need a dummy run if this is LTO + cmplog */ + if (unlikely(afl->shm.cmplog_mode)) { + + write_to_testcase(afl, use_mem, q->len); + + fault = fuzz_run_target(afl, &afl->fsrv, use_tmout); + + /* afl->stop_soon is set by the handler for Ctrl+C. When it's pressed, + we want to bail out quickly. */ + + if (afl->stop_soon || fault != afl->crash_mode) { goto abort_calibration; } + + if (!afl->non_instrumented_mode && !afl->stage_cur && + !count_bytes(afl, afl->fsrv.trace_bits)) { + + fault = FSRV_RUN_NOINST; + goto abort_calibration; + + } + +#ifdef INTROSPECTION + if (unlikely(!q->bitsmap_size)) q->bitsmap_size = afl->bitsmap_size; +#endif + + } + if (q->exec_cksum) { memcpy(afl->first_trace, afl->fsrv.trace_bits, afl->fsrv.map_size); diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index 24ccc108..737a49a7 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index eb1fe2d9..152bebe9 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to @@ -278,13 +278,14 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, "total_edges : %u\n" "var_byte_count : %u\n" "havoc_expansion : %u\n" + "auto_dict_entries : %u\n" "testcache_size : %llu\n" "testcache_count : %u\n" "testcache_evict : %u\n" "afl_banner : %s\n" "afl_version : " VERSION "\n" - "target_mode : %s%s%s%s%s%s%s%s%s\n" + "target_mode : %s%s%s%s%s%s%s%s%s%s\n" "command_line : %s\n", (afl->start_time - afl->prev_run_time) / 1000, cur_time / 1000, (afl->prev_run_time + cur_time - afl->start_time) / 1000, @@ -316,16 +317,17 @@ void write_stats_file(afl_state_t *afl, u32 t_bytes, double bitmap_cvg, -1, #endif t_bytes, afl->fsrv.real_map_size, afl->var_byte_count, - afl->expand_havoc, afl->q_testcase_cache_size, + afl->expand_havoc, afl->a_extras_cnt, afl->q_testcase_cache_size, afl->q_testcase_cache_count, afl->q_testcase_evictions, afl->use_banner, afl->unicorn_mode ? "unicorn" : "", afl->fsrv.qemu_mode ? "qemu " : "", + afl->fsrv.cs_mode ? "coresight" : "", afl->non_instrumented_mode ? " non_instrumented " : "", afl->no_forkserver ? "no_fsrv " : "", afl->crash_mode ? "crash " : "", afl->persistent_mode ? "persistent " : "", afl->shmem_testcase_mode ? "shmem_testcase " : "", afl->deferred_mode ? "deferred " : "", - (afl->unicorn_mode || afl->fsrv.qemu_mode || + (afl->unicorn_mode || afl->fsrv.qemu_mode || afl->fsrv.cs_mode || afl->non_instrumented_mode || afl->no_forkserver || afl->crash_mode || afl->persistent_mode || afl->deferred_mode) ? "" @@ -441,9 +443,10 @@ void show_stats(afl_state_t *afl) { u64 cur_ms; u32 t_bytes, t_bits; - u32 banner_len, banner_pad; - u8 tmp[256]; - u8 time_tmp[64]; + static u8 banner[128]; + u32 banner_len, banner_pad; + u8 tmp[256]; + u8 time_tmp[64]; u8 val_buf[8][STRINGIFY_VAL_SIZE_MAX]; #define IB(i) (val_buf[(i)]) @@ -540,9 +543,11 @@ void show_stats(afl_state_t *afl) { FATAL( "Incorrect fuzzing setup detected. Your target seems to have loaded " - "incorrectly instrumented shared libraries. If you use LTO mode " + "incorrectly instrumented shared libraries (%u of %u/%u). If you use " + "LTO mode " "please see instrumentation/README.lto.md. To ignore this problem " - "and continue fuzzing just set 'AFL_IGNORE_PROBLEMS=1'.\n"); + "and continue fuzzing just set 'AFL_IGNORE_PROBLEMS=1'.\n", + t_bytes, afl->fsrv.real_map_size, afl->fsrv.map_size); } @@ -560,8 +565,9 @@ void show_stats(afl_state_t *afl) { /* Roughly every minute, update fuzzer stats and save auto tokens. */ - if (unlikely(afl->force_ui_update || - cur_ms - afl->stats_last_stats_ms > STATS_UPDATE_SEC * 1000)) { + if (unlikely(!afl->non_instrumented_mode && + (afl->force_ui_update || + cur_ms - afl->stats_last_stats_ms > STATS_UPDATE_SEC * 1000))) { afl->stats_last_stats_ms = cur_ms; write_stats_file(afl, t_bytes, t_byte_ratio, stab_ratio, @@ -655,26 +661,34 @@ void show_stats(afl_state_t *afl) { } /* Let's start by drawing a centered banner. */ + if (unlikely(!banner[0])) { - banner_len = (afl->crash_mode ? 24 : 22) + strlen(VERSION) + - strlen(afl->use_banner) + strlen(afl->power_name) + 3 + 5; - banner_pad = (79 - banner_len) / 2; - memset(tmp, ' ', banner_pad); + char *si = ""; + if (afl->sync_id) { si = afl->sync_id; } + memset(banner, 0, sizeof(banner)); + banner_len = (afl->crash_mode ? 20 : 18) + strlen(VERSION) + strlen(si) + + strlen(afl->power_name) + 4 + 6; -#ifdef HAVE_AFFINITY - sprintf( - tmp + banner_pad, - "%s " cLCY VERSION cLGN " (%s) " cPIN "[%s]" cBLU " {%d}", - afl->crash_mode ? cPIN "peruvian were-rabbit" : cYEL "american fuzzy lop", - afl->use_banner, afl->power_name, afl->cpu_aff); -#else - sprintf( - tmp + banner_pad, "%s " cLCY VERSION cLGN " (%s) " cPIN "[%s]", - afl->crash_mode ? cPIN "peruvian were-rabbit" : cYEL "american fuzzy lop", - afl->use_banner, afl->power_name); -#endif /* HAVE_AFFINITY */ + if (strlen(afl->use_banner) + banner_len > 75) { + + afl->use_banner += (strlen(afl->use_banner) + banner_len) - 76; + memset(afl->use_banner, '.', 3); + + } + + banner_len += strlen(afl->use_banner); + banner_pad = (79 - banner_len) / 2; + memset(banner, ' ', banner_pad); + + sprintf(banner + banner_pad, + "%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s]", + afl->crash_mode ? cPIN "peruvian were-rabbit" + : cYEL "american fuzzy lop", + si, afl->use_banner, afl->power_name); + + } - SAYF("\n%s\n", tmp); + SAYF("\n%s\n", banner); /* "Handy" shortcuts for drawing boxes... */ @@ -1227,7 +1241,9 @@ void show_init_stats(afl_state_t *afl) { // SAYF("\n"); - if (avg_us > ((afl->fsrv.qemu_mode || afl->unicorn_mode) ? 50000 : 10000)) { + if (avg_us > ((afl->fsrv.cs_mode || afl->fsrv.qemu_mode || afl->unicorn_mode) + ? 50000 + : 10000)) { WARNF(cLRD "The target binary is pretty slow! See %s/perf_tips.md.", doc_path); diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 8ffc0e77..195366bd 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This is the real deal: the program takes an instrumented binary and attempts a variety of basic fuzzing tricks, paying close attention to @@ -113,11 +113,17 @@ static void usage(u8 *argv0, int more_help) { "maximum.\n" " -m megs - memory limit for child process (%u MB, 0 = no limit " "[default])\n" +#if defined(__linux__) && defined(__aarch64__) + " -A - use binary-only instrumentation (ARM CoreSight mode)\n" +#endif " -O - use binary-only instrumentation (FRIDA mode)\n" +#if defined(__linux__) " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use unicorn-based instrumentation (Unicorn mode)\n" " -W - use qemu-based instrumentation with Wine (Wine " - "mode)\n\n" + "mode)\n" +#endif + "\n" "Mutator settings:\n" " -D - enable deterministic fuzzing (once per queue entry)\n" @@ -434,7 +440,8 @@ int main(int argc, char **argv_orig, char **envp) { while ((opt = getopt( argc, argv, - "+b:B:c:CdDe:E:hi:I:f:F:l:L:m:M:nNOo:p:RQs:S:t:T:UV:Wx:Z")) > 0) { + "+Ab:B:c:CdDe:E:hi:I:f:F:l:L:m:M:nNOo:p:RQs:S:t:T:UV:Wx:Z")) > + 0) { switch (opt) { @@ -563,6 +570,12 @@ int main(int argc, char **argv_orig, char **envp) { } + if (afl->fsrv.cs_mode) { + + FATAL("-M is not supported in ARM CoreSight mode"); + + } + if (afl->sync_id) { FATAL("Multiple -S or -M options not supported"); } /* sanity check for argument: should not begin with '-' (possible @@ -609,6 +622,12 @@ int main(int argc, char **argv_orig, char **envp) { } + if (afl->fsrv.cs_mode) { + + FATAL("-S is not supported in ARM CoreSight mode"); + + } + if (afl->sync_id) { FATAL("Multiple -S or -M options not supported"); } /* sanity check for argument: should not begin with '-' (possible @@ -825,6 +844,24 @@ int main(int argc, char **argv_orig, char **envp) { afl->use_banner = optarg; break; + case 'A': /* CoreSight mode */ + + #if !defined(__aarch64__) || !defined(__linux__) + FATAL("-A option is not supported on this platform"); + #endif + + if (afl->is_main_node || afl->is_secondary_node) { + + FATAL("ARM CoreSight mode is not supported with -M / -S"); + + } + + if (afl->fsrv.cs_mode) { FATAL("Multiple -A options not supported"); } + + afl->fsrv.cs_mode = 1; + + break; + case 'O': /* FRIDA mode */ if (afl->fsrv.frida_mode) { @@ -1189,7 +1226,17 @@ int main(int argc, char **argv_orig, char **envp) { } - if (afl->sync_id) { fix_up_sync(afl); } + if (afl->sync_id) { + + if (strlen(afl->sync_id) > 24) { + + FATAL("sync_id max length is 24 characters"); + + } + + fix_up_sync(afl); + + } if (!strcmp(afl->in_dir, afl->out_dir)) { @@ -1202,6 +1249,7 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->crash_mode) { FATAL("-C and -n are mutually exclusive"); } if (afl->fsrv.frida_mode) { FATAL("-O and -n are mutually exclusive"); } if (afl->fsrv.qemu_mode) { FATAL("-Q and -n are mutually exclusive"); } + if (afl->fsrv.cs_mode) { FATAL("-A and -n are mutually exclusive"); } if (afl->unicorn_mode) { FATAL("-U and -n are mutually exclusive"); } } @@ -1218,6 +1266,8 @@ int main(int argc, char **argv_orig, char **envp) { if (unlikely(afl->afl_env.afl_statsd)) { statsd_setup_format(afl); } + if (!afl->use_banner) { afl->use_banner = argv[optind]; } + if (strchr(argv[optind], '/') == NULL && !afl->unicorn_mode) { WARNF(cLRD @@ -1348,7 +1398,7 @@ int main(int argc, char **argv_orig, char **envp) { } else if (afl->q_testcase_max_cache_size < 2 * MAX_FILE) { - FATAL("AFL_TESTCACHE_SIZE must be set to %u or more, or 0 to disable", + FATAL("AFL_TESTCACHE_SIZE must be set to %ld or more, or 0 to disable", (2 * MAX_FILE) % 1048576 == 0 ? (2 * MAX_FILE) / 1048576 : 1 + ((2 * MAX_FILE) / 1048576)); @@ -1446,6 +1496,8 @@ int main(int argc, char **argv_orig, char **envp) { } else { + /* CoreSight mode uses the default behavior. */ + setenv("LD_PRELOAD", getenv("AFL_PRELOAD"), 1); setenv("DYLD_INSERT_LIBRARIES", getenv("AFL_PRELOAD"), 1); @@ -1486,9 +1538,6 @@ int main(int argc, char **argv_orig, char **envp) { } save_cmdline(afl, argc, argv); - - fix_up_banner(afl, argv[optind]); - check_if_tty(afl); if (afl->afl_env.afl_force_ui) { afl->not_on_tty = 0; } @@ -1642,7 +1691,7 @@ int main(int argc, char **argv_orig, char **envp) { } - if (!afl->fsrv.qemu_mode && !afl->fsrv.frida_mode && + if (!afl->fsrv.qemu_mode && !afl->fsrv.frida_mode && !afl->fsrv.cs_mode && !afl->non_instrumented_mode) { check_binary(afl, afl->cmplog_binary); @@ -1688,6 +1737,11 @@ int main(int argc, char **argv_orig, char **envp) { } + } else if (afl->fsrv.cs_mode) { + + use_argv = get_cs_argv(argv[0], &afl->fsrv.target_path, argc - optind, + argv + optind); + } else { use_argv = argv + optind; @@ -1695,9 +1749,9 @@ int main(int argc, char **argv_orig, char **envp) { } if (afl->non_instrumented_mode || afl->fsrv.qemu_mode || - afl->fsrv.frida_mode || afl->unicorn_mode) { + afl->fsrv.frida_mode || afl->fsrv.cs_mode || afl->unicorn_mode) { - map_size = afl->fsrv.map_size = MAP_SIZE; + map_size = afl->fsrv.real_map_size = afl->fsrv.map_size = MAP_SIZE; afl->virgin_bits = ck_realloc(afl->virgin_bits, map_size); afl->virgin_tmout = ck_realloc(afl->virgin_tmout, map_size); afl->virgin_crash = ck_realloc(afl->virgin_crash, map_size); @@ -1715,7 +1769,7 @@ int main(int argc, char **argv_orig, char **envp) { afl_shm_init(&afl->shm, afl->fsrv.map_size, afl->non_instrumented_mode); if (!afl->non_instrumented_mode && !afl->fsrv.qemu_mode && - !afl->unicorn_mode && !afl->fsrv.frida_mode && + !afl->unicorn_mode && !afl->fsrv.frida_mode && !afl->fsrv.cs_mode && !afl->afl_env.afl_skip_bin_check) { if (map_size <= DEFAULT_SHMEM_SIZE) { @@ -1768,6 +1822,7 @@ int main(int argc, char **argv_orig, char **envp) { afl_fsrv_init_dup(&afl->cmplog_fsrv, &afl->fsrv); // TODO: this is semi-nice afl->cmplog_fsrv.trace_bits = afl->fsrv.trace_bits; + afl->cmplog_fsrv.cs_mode = afl->fsrv.cs_mode; afl->cmplog_fsrv.qemu_mode = afl->fsrv.qemu_mode; afl->cmplog_fsrv.frida_mode = afl->fsrv.frida_mode; afl->cmplog_fsrv.cmplog_binary = afl->cmplog_binary; @@ -1776,7 +1831,7 @@ int main(int argc, char **argv_orig, char **envp) { if ((map_size <= DEFAULT_SHMEM_SIZE || afl->cmplog_fsrv.map_size < map_size) && !afl->non_instrumented_mode && !afl->fsrv.qemu_mode && - !afl->fsrv.frida_mode && !afl->unicorn_mode && + !afl->fsrv.frida_mode && !afl->unicorn_mode && !afl->fsrv.cs_mode && !afl->afl_env.afl_skip_bin_check) { afl->cmplog_fsrv.map_size = MAX(map_size, (u32)DEFAULT_SHMEM_SIZE); @@ -1918,7 +1973,7 @@ int main(int argc, char **argv_orig, char **envp) { } - write_stats_file(afl, 0, 0, 0, 0); + if (!afl->non_instrumented_mode) { write_stats_file(afl, 0, 0, 0, 0); } maybe_update_plot_file(afl, 0, 0, 0); save_auto(afl); @@ -2226,13 +2281,12 @@ int main(int argc, char **argv_orig, char **envp) { } - write_bitmap(afl); - save_auto(afl); - stop_fuzzing: afl->force_ui_update = 1; // ensure the screen is reprinted show_stats(afl); // print the screen one last time + write_bitmap(afl); + save_auto(afl); SAYF(CURSOR_SHOW cLRD "\n\n+++ Testing aborted %s +++\n" cRST, afl->stop_soon == 2 ? "programmatically" : "by user"); @@ -2261,6 +2315,20 @@ stop_fuzzing: } + if (afl->not_on_tty) { + + u32 t_bytes = count_non_255_bytes(afl, afl->virgin_bits); + u8 time_tmp[64]; + u_stringify_time_diff(time_tmp, get_cur_time(), afl->start_time); + ACTF( + "Statistics: %u new paths found, %.02f%% coverage achieved, %llu " + "crashes found, %llu timeouts found, total runtime %s", + afl->queued_discovered, + ((double)t_bytes * 100) / afl->fsrv.real_map_size, afl->unique_crashes, + afl->unique_hangs, time_tmp); + + } + #ifdef PROFILING SAYF(cYEL "[!] " cRST "Profiling information: %llu ms total work, %llu ns/run\n", diff --git a/src/afl-gotcpu.c b/src/afl-gotcpu.c index ac002a93..f8466680 100644 --- a/src/afl-gotcpu.c +++ b/src/afl-gotcpu.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 This tool provides a fairly accurate measurement of CPU preemption rate. It is meant to complement the quick-and-dirty load average widget shown diff --git a/src/afl-ld-lto.c b/src/afl-ld-lto.c index 1ce97649..1dcdb176 100644 --- a/src/afl-ld-lto.c +++ b/src/afl-ld-lto.c @@ -15,7 +15,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 The sole purpose of this wrapper is to preprocess clang LTO files when linking with lld and performing the instrumentation on the whole program. diff --git a/src/afl-performance.c b/src/afl-performance.c index 89b170eb..04507410 100644 --- a/src/afl-performance.c +++ b/src/afl-performance.c @@ -5,7 +5,7 @@ and related and neighboring rights to this software to the public domain worldwide. This software is distributed without any warranty. - See <http://creativecommons.org/publicdomain/zero/1.0/>. + See <https://creativecommons.org/publicdomain/zero/1.0/>. This is xoshiro256++ 1.0, one of our all-purpose, rock-solid generators. It has excellent (sub-ns) speed, a state (256 bits) that is large @@ -90,7 +90,8 @@ inline u32 hash32(u8 *key, u32 len, u32 seed) { #endif - return (u32)XXH64(key, len, seed); + (void)seed; + return (u32)XXH3_64bits(key, len); } @@ -102,7 +103,8 @@ inline u64 hash64(u8 *key, u32 len, u64 seed) { #endif - return XXH64(key, len, seed); + (void)seed; + return XXH3_64bits(key, len); } diff --git a/src/afl-sharedmem.c b/src/afl-sharedmem.c index 22fe5a62..7fb8f821 100644 --- a/src/afl-sharedmem.c +++ b/src/afl-sharedmem.c @@ -17,7 +17,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Shared code to handle the shared memory. This is used by the fuzzer as well the other components like afl-tmin, afl-showmap, etc... diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 9122cd25..8cddcb32 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 A very simple tool that runs the targeted binary and displays the contents of the trace bitmap in a human-readable form. Useful in @@ -77,7 +77,7 @@ static u32 tcnt, highest; /* tuple content information */ static u32 in_len; /* Input data length */ -static u32 map_size = MAP_SIZE; +static u32 map_size = MAP_SIZE, timed_out = 0; static bool quiet_mode, /* Hide non-essential messages? */ edges_only, /* Ignore hit counts? */ @@ -146,6 +146,18 @@ static const u8 count_class_binary[256] = { #undef TIMES8 #undef TIMES4 +static void kill_child() { + + timed_out = 1; + if (fsrv->child_pid > 0) { + + kill(fsrv->child_pid, fsrv->kill_signal); + fsrv->child_pid = -1; + + } + +} + static void classify_counts(afl_forkserver_t *fsrv) { u8 * mem = fsrv->trace_bits; @@ -242,9 +254,14 @@ static u32 write_results_to_file(afl_forkserver_t *fsrv, u8 *outfile) { if (cmin_mode && (fsrv->last_run_timed_out || (!caa && child_crashed != cco))) { - // create empty file to prevent error messages in afl-cmin - fd = open(outfile, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION); - close(fd); + if (strcmp(outfile, "-")) { + + // create empty file to prevent error messages in afl-cmin + fd = open(outfile, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION); + close(fd); + + } + return ret; } @@ -357,9 +374,10 @@ static void showmap_run_target_forkserver(afl_forkserver_t *fsrv, u8 *mem, if (!quiet_mode) { - if (fsrv->last_run_timed_out) { + if (timed_out || fsrv->last_run_timed_out) { SAYF(cLRD "\n+++ Program timed off +++\n" cRST); + timed_out = 0; } else if (stop_soon) { @@ -413,7 +431,7 @@ static u32 read_file(u8 *in_file) { if (!be_quiet && !quiet_mode) { - WARNF("Input file '%s' is too large, only reading %u bytes.", in_file, + WARNF("Input file '%s' is too large, only reading %ld bytes.", in_file, MAX_FILE); } @@ -521,6 +539,8 @@ static void showmap_run_target(afl_forkserver_t *fsrv, char **argv) { } + signal(SIGALRM, kill_child); + setitimer(ITIMER_REAL, &it, NULL); if (waitpid(fsrv->child_pid, &status, 0) <= 0) { FATAL("waitpid() failed"); } @@ -563,9 +583,10 @@ static void showmap_run_target(afl_forkserver_t *fsrv, char **argv) { if (!quiet_mode) { - if (fsrv->last_run_timed_out) { + if (timed_out || fsrv->last_run_timed_out) { SAYF(cLRD "\n+++ Program timed off +++\n" cRST); + timed_out = 0; } else if (stop_soon) { @@ -669,6 +690,8 @@ static void set_up_environment(afl_forkserver_t *fsrv, char **argv) { } else { + /* CoreSight mode uses the default behavior. */ + setenv("LD_PRELOAD", getenv("AFL_PRELOAD"), 1); setenv("DYLD_INSERT_LIBRARIES", getenv("AFL_PRELOAD"), 1); @@ -821,12 +844,18 @@ static void usage(u8 *argv0) { "Execution control settings:\n" " -t msec - timeout for each run (none)\n" " -m megs - memory limit for child process (%u MB)\n" +#if defined(__linux__) && defined(__aarch64__) + " -A - use binary-only instrumentation (ARM CoreSight mode)\n" +#endif " -O - use binary-only instrumentation (FRIDA mode)\n" +#if defined(__linux__) " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use Unicorn-based instrumentation (Unicorn mode)\n" " -W - use qemu-based instrumentation with Wine (Wine mode)\n" " (Not necessary, here for consistency with other afl-* " - "tools)\n\n" + "tools)\n" +#endif + "\n" "Other settings:\n" " -i dir - process all files below this directory, must be combined " "with -o.\n" @@ -896,7 +925,7 @@ int main(int argc, char **argv_orig, char **envp) { if (getenv("AFL_QUIET") != NULL) { be_quiet = true; } - while ((opt = getopt(argc, argv, "+i:o:f:m:t:A:eqCZOQUWbcrsh")) > 0) { + while ((opt = getopt(argc, argv, "+i:o:f:m:t:AeqCZOH:QUWbcrsh")) > 0) { switch (opt) { @@ -1025,7 +1054,7 @@ int main(int argc, char **argv_orig, char **envp) { quiet_mode = true; break; - case 'A': + case 'H': /* Another afl-cmin specific feature. */ at_file = optarg; break; @@ -1035,10 +1064,23 @@ int main(int argc, char **argv_orig, char **envp) { if (fsrv->frida_mode) { FATAL("Multiple -O options not supported"); } fsrv->frida_mode = true; - setenv("AFL_FRIDA_INST_SEED", "0x0", 1); + setenv("AFL_FRIDA_INST_SEED", "1", 1); break; + /* FIXME: We want to use -P for consistency, but it is already unsed for + * undocumenetd feature "Another afl-cmin specific feature." */ + case 'A': /* CoreSight mode */ + +#if !defined(__aarch64__) || !defined(__linux__) + FATAL("-A option is not supported on this platform"); +#endif + + if (fsrv->cs_mode) { FATAL("Multiple -A options not supported"); } + + fsrv->cs_mode = true; + break; + case 'Q': if (fsrv->qemu_mode) { FATAL("Multiple -Q options not supported"); } @@ -1183,12 +1225,19 @@ int main(int argc, char **argv_orig, char **envp) { } + } else if (fsrv->cs_mode) { + + use_argv = + get_cs_argv(argv[0], &fsrv->target_path, argc - optind, argv + optind); + } else { use_argv = argv + optind; } + if (in_dir) { (void)check_binary_signatures(fsrv->target_path); } + shm_fuzz = ck_alloc(sizeof(sharedmem_t)); /* initialize cmplog_mode */ @@ -1207,7 +1256,7 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->shmem_fuzz_len = (u32 *)map; fsrv->shmem_fuzz = map + sizeof(u32); - if (!fsrv->qemu_mode && !unicorn_mode) { + if (!fsrv->cs_mode && !fsrv->qemu_mode && !unicorn_mode) { u32 save_be_quiet = be_quiet; be_quiet = !debug; @@ -1386,9 +1435,9 @@ int main(int argc, char **argv_orig, char **envp) { if (!quiet_mode || collect_coverage) { if (!tcnt && !have_coverage) { FATAL("No instrumentation detected" cRST); } - OKF("Captured %u tuples (highest value %u, total values %llu) in " - "'%s'." cRST, - tcnt, highest, total, out_file); + OKF("Captured %u tuples (map size %u, highest value %u, total values %llu) " + "in '%s'." cRST, + tcnt, fsrv->real_map_size, highest, total, out_file); if (collect_coverage) OKF("A coverage of %u edges were achieved out of %u existing (%.02f%%) " "with %llu input files.", diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 792770e0..89546c45 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -18,7 +18,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 A simple test case minimizer that takes an input file and tries to remove as much data as possible while keeping the binary in a crashing state @@ -120,6 +120,17 @@ static const u8 count_class_lookup[256] = { #undef TIMES8 #undef TIMES4 +static void kill_child() { + + if (fsrv->child_pid > 0) { + + kill(fsrv->child_pid, fsrv->kill_signal); + fsrv->child_pid = -1; + + } + +} + static sharedmem_t *deinit_shmem(afl_forkserver_t *fsrv, sharedmem_t * shm_fuzz) { @@ -221,7 +232,7 @@ static void read_initial_file(void) { if (st.st_size >= TMIN_MAX_FILE) { - FATAL("Input file is too large (%u MB max)", TMIN_MAX_FILE / 1024 / 1024); + FATAL("Input file is too large (%ld MB max)", TMIN_MAX_FILE / 1024 / 1024); } @@ -797,6 +808,8 @@ static void set_up_environment(afl_forkserver_t *fsrv, char **argv) { } else { + /* CoreSight mode uses the default behavior. */ + setenv("LD_PRELOAD", getenv("AFL_PRELOAD"), 1); setenv("DYLD_INSERT_LIBRARIES", getenv("AFL_PRELOAD"), 1); @@ -853,13 +866,19 @@ static void usage(u8 *argv0) { " -f file - input file read by the tested program (stdin)\n" " -t msec - timeout for each run (%u ms)\n" " -m megs - memory limit for child process (%u MB)\n" +#if defined(__linux__) && defined(__aarch64__) + " -A - use binary-only instrumentation (ARM CoreSight mode)\n" +#endif " -O - use binary-only instrumentation (FRIDA mode)\n" +#if defined(__linux__) " -Q - use binary-only instrumentation (QEMU mode)\n" " -U - use unicorn-based instrumentation (Unicorn mode)\n" " -W - use qemu-based instrumentation with Wine (Wine " "mode)\n" " (Not necessary, here for consistency with other afl-* " - "tools)\n\n" + "tools)\n" +#endif + "\n" "Minimization settings:\n" @@ -910,7 +929,7 @@ int main(int argc, char **argv_orig, char **envp) { SAYF(cCYA "afl-tmin" VERSION cRST " by Michal Zalewski\n"); - while ((opt = getopt(argc, argv, "+i:o:f:m:t:B:xeOQUWHh")) > 0) { + while ((opt = getopt(argc, argv, "+i:o:f:m:t:B:xeAOQUWHh")) > 0) { switch (opt) { @@ -1022,12 +1041,23 @@ int main(int argc, char **argv_orig, char **envp) { break; + case 'A': /* CoreSight mode */ + +#if !defined(__aarch64__) || !defined(__linux__) + FATAL("-A option is not supported on this platform"); +#endif + + if (fsrv->cs_mode) { FATAL("Multiple -A options not supported"); } + + fsrv->cs_mode = 1; + break; + case 'O': /* FRIDA mode */ if (fsrv->frida_mode) { FATAL("Multiple -O options not supported"); } fsrv->frida_mode = 1; - setenv("AFL_FRIDA_INST_SEED", "0x0", 1); + setenv("AFL_FRIDA_INST_SEED", "1", 1); break; @@ -1125,6 +1155,7 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->target_path = find_binary(argv[optind]); fsrv->trace_bits = afl_shm_init(&shm, map_size, 0); detect_file_args(argv + optind, out_file, &fsrv->use_stdin); + signal(SIGALRM, kill_child); if (fsrv->qemu_mode) { @@ -1140,6 +1171,11 @@ int main(int argc, char **argv_orig, char **envp) { } + } else if (fsrv->cs_mode) { + + use_argv = + get_cs_argv(argv[0], &fsrv->target_path, argc - optind, argv + optind); + } else { use_argv = argv + optind; @@ -1209,6 +1245,7 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->shmem_fuzz = map + sizeof(u32); read_initial_file(); + (void)check_binary_signatures(fsrv->target_path); if (!fsrv->qemu_mode && !unicorn_mode) { diff --git a/test-instr.c b/test-instr.c index 13d4eb93..eaae50ef 100644 --- a/test-instr.c +++ b/test-instr.c @@ -7,7 +7,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 */ #include <stdio.h> diff --git a/test/test-cmplog.c b/test/test-cmplog.c index b077e3ab..262df6bd 100644 --- a/test/test-cmplog.c +++ b/test/test-cmplog.c @@ -1,15 +1,13 @@ #include <stdio.h> #include <string.h> +#include <stdint.h> #include <stdarg.h> #include <stdlib.h> #include <stdint.h> #include <unistd.h> -int main(int argc, char *argv[]) { - char buf[1024]; - ssize_t i; - if ((i = read(0, buf, sizeof(buf) - 1)) < 24) return 0; - buf[i] = 0; +int LLVMFuzzerTestOneInput(const uint8_t *buf, size_t i) { + if (i < 24) return 0; if (buf[0] != 'A') return 0; if (buf[1] != 'B') return 0; if (buf[2] != 'C') return 0; @@ -18,6 +16,17 @@ int main(int argc, char *argv[]) { if (strncmp(buf + 12, "IJKL", 4) == 0 && strcmp(buf + 16, "DEADBEEF") == 0) abort(); return 0; - } +#ifdef __AFL_COMPILER +int main(int argc, char *argv[]) { + unsigned char buf[1024]; + ssize_t i; + while(__AFL_LOOP(1000)) { + i = read(0, (char*)buf, sizeof(buf) - 1); + if (i > 0) buf[i] = 0; + LLVMFuzzerTestOneInput(buf, i); + } + return 0; +} +#endif diff --git a/test/test-dlopen.c b/test/test-dlopen.c index d08d9092..b81bab13 100644 --- a/test/test-dlopen.c +++ b/test/test-dlopen.c @@ -5,7 +5,13 @@ int main(int argc, char **argv) { - if (!getenv("TEST_DLOPEN_TARGET")) return 1; + if (!getenv("TEST_DLOPEN_TARGET")) { + + fprintf(stderr, "Error: TEST_DLOPEN_TARGET not set!\n"); + return 1; + + } + void *lib = dlopen(getenv("TEST_DLOPEN_TARGET"), RTLD_LAZY); if (!lib) { @@ -15,8 +21,18 @@ int main(int argc, char **argv) { } int (*func)(int, char **) = dlsym(lib, "main_exported"); - if (!func) return 3; + if (!func) { + + fprintf(stderr, "Error: main_exported not found!\n"); + return 3; + + } + + // must use deferred forkserver as otherwise afl++ instrumentation aborts + // because all dlopen() of instrumented libs must be before the forkserver + __AFL_INIT(); + fprintf(stderr, "Running main_exported\n"); return func(argc, argv); } diff --git a/test/test-pre.sh b/test/test-pre.sh index 7819da47..e12d95be 100755 --- a/test/test-pre.sh +++ b/test/test-pre.sh @@ -88,6 +88,8 @@ unset AFL_QEMU_PERSISTENT_GPR unset AFL_QEMU_PERSISTENT_RET unset AFL_QEMU_PERSISTENT_HOOK unset AFL_QEMU_PERSISTENT_CNT +unset AFL_QEMU_PERSISTENT_MEM +unset AFL_QEMU_PERSISTENT_EXITS unset AFL_CUSTOM_MUTATOR_LIBRARY unset AFL_PYTHON_MODULE unset AFL_PRELOAD diff --git a/testcases/README.md b/testcases/README.md index ef38d3c4..a2f74d68 100644 --- a/testcases/README.md +++ b/testcases/README.md @@ -1,4 +1,4 @@ -# AFL starting test cases +# AFL++ starting test cases (See [../README.md](../README.md) for the general instruction manual.) diff --git a/unicorn_mode/README.md b/unicorn_mode/README.md index d2b7d16f..ee4a7b22 100644 --- a/unicorn_mode/README.md +++ b/unicorn_mode/README.md @@ -8,10 +8,11 @@ The CompareCoverage and NeverZero counters features are by Andrea Fioraldi <andr ## 1) Introduction -The code in ./unicorn_mode allows you to build the (Unicorn Engine)[https://github.com/unicorn-engine/unicorn] with AFL support. +The code in ./unicorn_mode allows you to build the +(Unicorn Engine)[https://github.com/unicorn-engine/unicorn] with AFL++ support. This means, you can run anything that can be emulated in unicorn and obtain instrumentation -output for black-box, closed-source binary code snippets. This mechanism -can be then used by afl-fuzz to stress-test targets that couldn't be built +output for black-box, closed-source binary code snippets. This mechanism +can be then used by afl-fuzz to stress-test targets that couldn't be built with afl-cc or used in QEMU mode. There is a significant performance penalty compared to native AFL, @@ -25,7 +26,7 @@ For some pointers for more advanced emulation, take a look at [BaseSAFE](https:/ ### Building AFL++'s Unicorn Mode First, make AFL++ as usual. -Once that completes successfully you need to build and add in the Unicorn Mode +Once that completes successfully you need to build and add in the Unicorn Mode features: ``` @@ -33,10 +34,10 @@ cd unicorn_mode ./build_unicorn_support.sh ``` -NOTE: This script checks out a Unicorn Engine fork as submodule that has been tested -and is stable-ish, based on the unicorn engine `next` branch. +NOTE: This script checks out a Unicorn Engine fork as submodule that has been tested +and is stable-ish, based on the unicorn engine `next` branch. -Building Unicorn will take a little bit (~5-10 minutes). Once it completes +Building Unicorn will take a little bit (~5-10 minutes). Once it completes it automatically compiles a sample application and verifies that it works. ### Fuzzing with Unicorn Mode @@ -46,25 +47,24 @@ To use unicorn-mode effectively you need to prepare the following: * Relevant binary code to be fuzzed * Knowledge of the memory map and good starting state * Folder containing sample inputs to start fuzzing with - + Same ideas as any other AFL inputs - + Quality/speed of results will depend greatly on the quality of starting + + Same ideas as any other AFL++ inputs + + Quality/speed of results will depend greatly on the quality of starting samples + See AFL's guidance on how to create a sample corpus * Unicornafl-based test harness in Rust, C, or Python, which: + Adds memory map regions - + Loads binary code into memory + + Loads binary code into memory + Calls uc.afl_fuzz() / uc.afl_start_forkserver + Loads and verifies data to fuzz from a command-line specified file - + AFL will provide mutated inputs by changing the file passed to + + AFL++ will provide mutated inputs by changing the file passed to the test harness + Presumably the data to be fuzzed is at a fixed buffer address - + If input constraints (size, invalid bytes, etc.) are known they - should be checked in the place_input handler. If a constraint - fails, just return false from the handler. AFL will treat the input as - 'uninteresting' and move on. + + If input constraints (size, invalid bytes, etc.) are known they + should be checked in the place_input handler. If a constraint + fails, just return false from the handler. AFL++ will treat the input as 'uninteresting' and move on. + Sets up registers and memory state for beginning of test + Emulates the interesting code from beginning to end - + If a crash is detected, the test harness must 'crash' by + + If a crash is detected, the test harness must 'crash' by throwing a signal (SIGSEGV, SIGKILL, SIGABORT, etc.), or indicate a crash in the crash validation callback. Once you have all those things ready to go you just need to run afl-fuzz in @@ -77,14 +77,13 @@ afl-fuzz -U -m none -i /path/to/inputs -o /path/to/results -- ./test_harness @@ The normal afl-fuzz command line format applies to everything here. Refer to AFL's main documentation for more info about how to use afl-fuzz effectively. -For a much clearer vision of what all of this looks like, please refer to the -sample provided in the 'unicorn_mode/samples' directory. There is also a blog -post that uses slightly older concepts, but describes the general ideas, at: +For a much clearer vision of what all of this looks like, refer to the sample +provided in the 'unicorn_mode/samples' directory. There is also a blog post that +uses slightly older concepts, but describes the general ideas, at: [https://medium.com/@njvoss299/afl-unicorn-fuzzing-arbitrary-binary-code-563ca28936bf](https://medium.com/@njvoss299/afl-unicorn-fuzzing-arbitrary-binary-code-563ca28936bf) - -The ['helper_scripts'](./helper_scripts) directory also contains several helper scripts that allow you +The ['helper_scripts'](./helper_scripts) directory also contains several helper scripts that allow you to dump context from a running process, load it, and hook heap allocations. For details on how to use this check out the follow-up blog post to the one linked above. @@ -105,8 +104,8 @@ Comparison instructions are currently instrumented only for the x86, x86_64 and ## 4) Gotchas, feedback, bugs -Running the build script builds Unicornafl and its python bindings and installs -them on your system. +Running the build script builds Unicornafl and its python bindings and installs +them on your system. This installation will leave any existing Unicorn installations untouched. If you want to use unicornafl instead of unicorn in a script, replace all `unicorn` imports with `unicornafl` inputs, everything else should "just work". diff --git a/unicorn_mode/UNICORNAFL_VERSION b/unicorn_mode/UNICORNAFL_VERSION index 0db54339..dbe3999f 100644 --- a/unicorn_mode/UNICORNAFL_VERSION +++ b/unicorn_mode/UNICORNAFL_VERSION @@ -1 +1 @@ -c0e03d2c +9df92d6868e8b219886e4b7458e5e134c48ff2c9 diff --git a/unicorn_mode/build_unicorn_support.sh b/unicorn_mode/build_unicorn_support.sh index 6c376f8d..f9c0be7f 100755 --- a/unicorn_mode/build_unicorn_support.sh +++ b/unicorn_mode/build_unicorn_support.sh @@ -20,7 +20,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # This script downloads, patches, and builds a version of Unicorn with # minor tweaks to allow Unicorn-emulated binaries to be run under diff --git a/unicorn_mode/samples/c/COMPILE.md b/unicorn_mode/samples/c/COMPILE.md index 7da140f7..e5265071 100644 --- a/unicorn_mode/samples/c/COMPILE.md +++ b/unicorn_mode/samples/c/COMPILE.md @@ -6,6 +6,7 @@ This shows a simple harness for unicornafl in C The target can be built using the `make` command. Just make sure you have built unicorn support first: + ```bash cd /path/to/afl/unicorn_mode ./build_unicorn_support.sh @@ -19,4 +20,4 @@ was built in case you want to rebuild it or recompile it for any reason. The pre-built binary (persistent_target_x86_64) was built using -g -O0 in gcc. -We then load the binary and execute the main function directly. +Then load the binary and execute the main function directly. diff --git a/unicorn_mode/samples/persistent/COMPILE.md b/unicorn_mode/samples/persistent/COMPILE.md index 111dfc54..5e607aef 100644 --- a/unicorn_mode/samples/persistent/COMPILE.md +++ b/unicorn_mode/samples/persistent/COMPILE.md @@ -1,13 +1,16 @@ # C Sample This shows a simple persistent harness for unicornafl in C. -In contrast to the normal c harness, this harness manually resets the unicorn state on each new input. -Thanks to this, we can rerun the testcase in unicorn multiple times, without the need to fork again. +In contrast to the normal c harness, this harness manually resets the unicorn +state on each new input. +Thanks to this, you can rerun the test case in unicorn multiple times, without +the need to fork again. ## Compiling sample.c The target can be built using the `make` command. Just make sure you have built unicorn support first: + ```bash cd /path/to/afl/unicorn_mode ./build_unicorn_support.sh @@ -19,6 +22,7 @@ You don't need to compile persistent_target.c since a X86_64 binary version is pre-built and shipped in this sample folder. This file documents how the binary was built in case you want to rebuild it or recompile it for any reason. -The pre-built binary (persistent_target_x86_64.bin) was built using -g -O0 in gcc. +The pre-built binary (persistent_target_x86_64.bin) was built using -g -O0 in +gcc. -We then load the binary and we execute the main function directly. +Then load the binary and execute the main function directly. \ No newline at end of file diff --git a/unicorn_mode/samples/speedtest/README.md b/unicorn_mode/samples/speedtest/README.md index 3c1184a2..bd5ba8d0 100644 --- a/unicorn_mode/samples/speedtest/README.md +++ b/unicorn_mode/samples/speedtest/README.md @@ -35,7 +35,6 @@ cd python TODO: add results here. - ## Compiling speedtest_target.c You shouldn't need to compile simple_target.c since a X86_64 binary version is @@ -44,22 +43,30 @@ was built in case you want to rebuild it or recompile it for any reason. The pre-built binary (simple_target_x86_64.bin) was built using -g -O0 in gcc. -We then load the binary and execute the main function directly. +Then load the binary and execute the main function directly. + +## Addresses for the harness -## Addresses for the harness: To find the address (in hex) of main, run: + ```bash objdump -M intel -D target | grep '<main>:' | cut -d" " -f1 ``` + To find all call sites to magicfn, run: + ```bash objdump -M intel -D target | grep '<magicfn>$' | cut -d":" -f1 ``` + For malloc callsites: + ```bash objdump -M intel -D target | grep '<malloc@plt>$' | cut -d":" -f1 ``` + And free callsites: + ```bash objdump -M intel -D target | grep '<free@plt>$' | cut -d":" -f1 -``` +``` \ No newline at end of file diff --git a/unicorn_mode/samples/speedtest/rust/Cargo.toml b/unicorn_mode/samples/speedtest/rust/Cargo.toml index c19ee0a1..9b81be0b 100644 --- a/unicorn_mode/samples/speedtest/rust/Cargo.toml +++ b/unicorn_mode/samples/speedtest/rust/Cargo.toml @@ -11,5 +11,5 @@ panic = "abort" [dependencies] unicornafl = { path = "../../../unicornafl/bindings/rust/", version="1.0.0" } -capstone="0.6.0" -libc="0.2.66" \ No newline at end of file +capstone="0.10.0" +libc="0.2.66" diff --git a/unicorn_mode/samples/speedtest/rust/src/main.rs b/unicorn_mode/samples/speedtest/rust/src/main.rs index 105ba4b4..cded1a3c 100644 --- a/unicorn_mode/samples/speedtest/rust/src/main.rs +++ b/unicorn_mode/samples/speedtest/rust/src/main.rs @@ -11,12 +11,13 @@ use std::{ }; use unicornafl::{ + afl::afl_fuzz, unicorn_const::{uc_error, Arch, Mode, Permission}, - RegisterX86::{self, *}, - Unicorn, UnicornHandle, + RegisterX86::*, + Unicorn, }; -const BINARY: &str = &"../target"; +const BINARY: &str = "../target"; // Memory map for the code to be tested // Arbitrary address where code to test will be loaded @@ -47,7 +48,7 @@ fn read_file(filename: &str) -> Result<Vec<u8>, io::Error> { fn parse_locs(loc_name: &str) -> Result<Vec<u64>, io::Error> { let contents = &read_file(&format!("../target.offsets.{}", loc_name))?; //println!("Read: {:?}", contents); - Ok(str_from_u8_unchecked(&contents) + Ok(str_from_u8_unchecked(contents) .split('\n') .map(|x| { //println!("Trying to convert {}", &x[2..]); @@ -87,8 +88,7 @@ fn main() { } fn fuzz(input_file: &str) -> Result<(), uc_error> { - let mut unicorn = Unicorn::new(Arch::X86, Mode::MODE_64, 0)?; - let mut uc: UnicornHandle<'_, _> = unicorn.borrow(); + let mut uc = Unicorn::new(Arch::X86, Mode::MODE_64)?; let binary = read_file(BINARY).unwrap_or_else(|_| panic!("Could not read modem image: {}", BINARY)); @@ -105,7 +105,7 @@ fn fuzz(input_file: &str) -> Result<(), uc_error> { // Set the program counter to the start of the code let main_locs = parse_locs("main").unwrap(); //println!("Entry Point: {:x}", main_locs[0]); - uc.reg_write(RegisterX86::RIP as i32, main_locs[0])?; + uc.reg_write(RIP, main_locs[0])?; // Setup the stack. uc.mem_map( @@ -114,14 +114,14 @@ fn fuzz(input_file: &str) -> Result<(), uc_error> { Permission::READ | Permission::WRITE, )?; // Setup the stack pointer, but allocate two pointers for the pointers to input. - uc.reg_write(RSP as i32, STACK_ADDRESS + STACK_SIZE - 16)?; + uc.reg_write(RSP, STACK_ADDRESS + STACK_SIZE - 16)?; // Setup our input space, and push the pointer to it in the function params uc.mem_map(INPUT_ADDRESS, INPUT_MAX as usize, Permission::READ)?; // We have argc = 2 - uc.reg_write(RDI as i32, 2)?; + uc.reg_write(RDI, 2)?; // RSI points to our little 2 QWORD space at the beginning of the stack... - uc.reg_write(RSI as i32, STACK_ADDRESS + STACK_SIZE - 16)?; + uc.reg_write(RSI, STACK_ADDRESS + STACK_SIZE - 16)?; // ... which points to the Input. Write the ptr to mem in little endian. uc.mem_write( STACK_ADDRESS + STACK_SIZE - 16, @@ -133,13 +133,13 @@ fn fuzz(input_file: &str) -> Result<(), uc_error> { let already_allocated_malloc = already_allocated.clone(); // We use a very simple malloc/free stub here, // that only works for exactly one allocation at a time. - let hook_malloc = move |mut uc: UnicornHandle<'_, _>, addr: u64, size: u32| { + let hook_malloc = move |uc: &mut Unicorn<'_, _>, addr: u64, size: u32| { if already_allocated_malloc.get() { println!("Double malloc, not supported right now!"); abort(); } // read the first param - let malloc_size = uc.reg_read(RDI as i32).unwrap(); + let malloc_size = uc.reg_read(RDI).unwrap(); if malloc_size > HEAP_SIZE_MAX { println!( "Tried to allocate {} bytes, but we may only allocate up to {}", @@ -147,20 +147,20 @@ fn fuzz(input_file: &str) -> Result<(), uc_error> { ); abort(); } - uc.reg_write(RAX as i32, HEAP_ADDRESS).unwrap(); - uc.reg_write(RIP as i32, addr + size as u64).unwrap(); + uc.reg_write(RAX, HEAP_ADDRESS).unwrap(); + uc.reg_write(RIP, addr + size as u64).unwrap(); already_allocated_malloc.set(true); }; let already_allocated_free = already_allocated; // No real free, just set the "used"-flag to false. - let hook_free = move |mut uc: UnicornHandle<'_, _>, addr, size| { + let hook_free = move |uc: &mut Unicorn<'_, _>, addr, size| { if already_allocated_free.get() { println!("Double free detected. Real bug?"); abort(); } // read the first param - let free_ptr = uc.reg_read(RDI as i32).unwrap(); + let free_ptr = uc.reg_read(RDI).unwrap(); if free_ptr != HEAP_ADDRESS { println!( "Tried to free wrong mem region {:x} at code loc {:x}", @@ -168,7 +168,7 @@ fn fuzz(input_file: &str) -> Result<(), uc_error> { ); abort(); } - uc.reg_write(RIP as i32, addr + size as u64).unwrap(); + uc.reg_write(RIP, addr + size as u64).unwrap(); already_allocated_free.set(false); }; @@ -177,8 +177,8 @@ fn fuzz(input_file: &str) -> Result<(), uc_error> { */ // This is a fancy print function that we're just going to skip for fuzzing. - let hook_magicfn = move |mut uc: UnicornHandle<'_, _>, addr, size| { - uc.reg_write(RIP as i32, addr + size as u64).unwrap(); + let hook_magicfn = move |uc: &mut Unicorn<'_, _>, addr, size| { + uc.reg_write(RIP, addr + size as u64).unwrap(); }; for addr in parse_locs("malloc").unwrap() { @@ -195,7 +195,7 @@ fn fuzz(input_file: &str) -> Result<(), uc_error> { } let place_input_callback = - |uc: &mut UnicornHandle<'_, _>, afl_input: &mut [u8], _persistent_round| { + |uc: &mut Unicorn<'_, _>, afl_input: &mut [u8], _persistent_round| { // apply constraints to the mutated input if afl_input.len() > INPUT_MAX as usize { //println!("Skipping testcase with leng {}", afl_input.len()); @@ -209,13 +209,12 @@ fn fuzz(input_file: &str) -> Result<(), uc_error> { // return true if the last run should be counted as crash let crash_validation_callback = - |_uc: &mut UnicornHandle<'_, _>, result, _input: &[u8], _persistent_round| { - result != uc_error::OK - }; + |_uc: &mut Unicorn<'_, _>, result, _input: &[u8], _persistent_round| result != uc_error::OK; let end_addrs = parse_locs("main_ends").unwrap(); - let ret = uc.afl_fuzz( + let ret = afl_fuzz( + &mut uc, input_file, place_input_callback, &end_addrs, diff --git a/unicorn_mode/unicornafl b/unicorn_mode/unicornafl -Subproject 019b871539fe9ed3f41d882385a8b02c243d49a +Subproject d4915053d477dd827b3fe4b494173d3fbf9f456 diff --git a/unicorn_mode/update_uc_ref.sh b/unicorn_mode/update_uc_ref.sh index 7c1c7778..6e809a7b 100755 --- a/unicorn_mode/update_uc_ref.sh +++ b/unicorn_mode/update_uc_ref.sh @@ -21,10 +21,10 @@ fi git submodule init && git submodule update unicornafl || exit 1 cd ./unicornafl || exit 1 -git fetch origin dev 1>/dev/null || exit 1 +git fetch origin uc1 1>/dev/null || exit 1 git stash 1>/dev/null 2>/dev/null git stash drop 1>/dev/null 2>/dev/null -git checkout dev +git checkout uc1 if [ -z "$NEW_VERSION" ]; then # No version provided, take HEAD. diff --git a/utils/README.md b/utils/README.md index b8df0b47..debc86e8 100644 --- a/utils/README.md +++ b/utils/README.md @@ -13,10 +13,10 @@ Here's a quick overview of the stuff you can find in this directory: - afl_proxy - skeleton file example to show how to fuzz something where you gather coverage data via - different means, e.g. hw debugger + different means, e.g., hw debugger - afl_untracer - fuzz binary-only libraries much faster but with - less coverage than qemu_mode + less coverage than QEMU mode - analysis_scripts - random -o out analysis scripts @@ -48,7 +48,7 @@ Here's a quick overview of the stuff you can find in this directory: - defork - intercept fork() in targets - distributed_fuzzing - a sample script for synchronizing fuzzer instances - across multiple machines (see parallel_fuzzing.md). + across multiple machines. - libdislocator - like ASAN but lightweight. diff --git a/utils/afl_network_proxy/README.md b/utils/afl_network_proxy/README.md index 05659c45..c34463e2 100644 --- a/utils/afl_network_proxy/README.md +++ b/utils/afl_network_proxy/README.md @@ -1,12 +1,13 @@ # afl-network-proxy -If you want to run afl-fuzz over the network than this is what you need :) +If you want to run afl-fuzz over the network, then this is what you need. :) Note that the impact on fuzzing speed will be huge, expect a loss of 90%. ## When to use this 1. when you have to fuzz a target that has to run on a system that cannot - contain the fuzzing output (e.g. /tmp too small and file system is read-only) + contain the fuzzing output (e.g., /tmp too small and file system is + read-only) 2. when the target instantly reboots on crashes 3. ... any other reason you would need this @@ -28,32 +29,34 @@ For most targets this hurts performance though so it is disabled by default. Run `afl-network-server` with your target with the -m and -t values you need. Important is the -i parameter which is the TCP port to listen on. e.g.: + ``` afl-network-server -i 1111 -m 25M -t 1000 -- /bin/target -f @@ ``` ### on the (afl-fuzz) main node -Just run afl-fuzz with your normal options, however the target should be +Just run afl-fuzz with your normal options, however, the target should be `afl-network-client` with the IP and PORT of the `afl-network-server` and increase the -t value: + ``` afl-fuzz -i in -o out -t 2000+ -- afl-network-client TARGET-IP 1111 ``` -Note the '+' on the -t parameter value. The afl-network-server will take -care of proper timeouts hence afl-fuzz should not. The '+' increases the -timeout and the value itself should be 500-1000 higher than the one on -afl-network-server. + +Note the '+' on the -t parameter value. The afl-network-server will take care of +proper timeouts hence afl-fuzz should not. The '+' increases the timeout and the +value itself should be 500-1000 higher than the one on afl-network-server. ### networking The TARGET can be an IPv4 or IPv6 address, or a host name that resolves to either. Note that also the outgoing interface can be specified with a '%' for -`afl-network-client`, e.g. `fe80::1234%eth0`. +`afl-network-client`, e.g., `fe80::1234%eth0`. Also make sure your default TCP window size is larger than your MAP_SIZE (130kb is a good value). -On Linux that is the middle value of `/proc/sys/net/ipv4/tcp_rmem` +On Linux that is the middle value of `/proc/sys/net/ipv4/tcp_rmem` ## how to compile and install diff --git a/utils/afl_untracer/README.md b/utils/afl_untracer/README.md index ada0c916..da0e0c77 100644 --- a/utils/afl_untracer/README.md +++ b/utils/afl_untracer/README.md @@ -5,9 +5,9 @@ afl-untracer is an example skeleton file which can easily be used to fuzz a closed source library. -It requires less memory and is x3-5 faster than qemu_mode however it is way -more course grained and does not provide interesting features like compcov -or cmplog. +It requires less memory and is x3-5 faster than QEMU mode, however, it is way +more course grained and does not provide interesting features like compcov or +cmplog. Supported is so far Intel (i386/x86_64) and AARCH64. @@ -15,7 +15,7 @@ Supported is so far Intel (i386/x86_64) and AARCH64. ### Modify afl-untracer.c -Read and modify afl-untracer.c then `make`. +Read and modify afl-untracer.c, then `make`. To adapt afl-untracer.c to your needs, read the header of the file and then search and edit the `STEP 1`, `STEP 2` and `STEP 3` locations. @@ -28,28 +28,34 @@ To generate the `patches.txt` file for your target library use the The patches.txt file has to be pointed to by `AFL_UNTRACER_FILE`. To easily run the scripts without needing to run the GUI with Ghidra: + ``` /opt/ghidra/support/analyzeHeadless /tmp/ tmp$$ -import libtestinstr.so -postscript ./ghidra_get_patchpoints.java rm -rf /tmp/tmp$$ ``` + The file is created at `~/Desktop/patches.txt` ### Fuzzing Example (after modifying afl-untracer.c to your needs, compiling and creating patches.txt): + ``` LD_LIBRARY_PATH=/path/to/target/library AFL_UNTRACER_FILE=./patches.txt afl-fuzz -i in -o out -- ./afl-untracer ``` + (or even remote via afl-network-proxy). ### Testing and debugging For testing/debugging you can try: + ``` make DEBUG=1 AFL_UNTRACER_FILE=./patches.txt AFL_DEBUG=1 gdb ./afl-untracer ``` + and then you can easily set breakpoints to "breakpoint" and "fuzz". # Background @@ -57,4 +63,4 @@ and then you can easily set breakpoints to "breakpoint" and "fuzz". This idea is based on [UnTracer](https://github.com/FoRTE-Research/UnTracer-AFL) and modified by [Trapfuzz](https://github.com/googleprojectzero/p0tools/tree/master/TrapFuzz). This implementation is slower because the traps are not patched out with each -run, but on the other hand gives much better coverage information. +run, but on the other hand gives much better coverage information. \ No newline at end of file diff --git a/utils/aflpp_driver/README.md b/utils/aflpp_driver/README.md index 30e2412f..c547aaea 100644 --- a/utils/aflpp_driver/README.md +++ b/utils/aflpp_driver/README.md @@ -5,39 +5,39 @@ aflpp_driver is used to compile directly libfuzzer `LLVMFuzzerTestOneInput()` targets. -Just do `afl-clang-fast++ -o fuzz fuzzer_harness.cc libAFLDriver.a [plus required linking]`. +Just do `afl-clang-fast++ -o fuzz fuzzer_harness.cc libAFLDriver.a [plus +required linking]`. -You can also sneakily do this little trick: -If this is the clang compile command to build for libfuzzer: - `clang++ -o fuzz -fsanitize=fuzzer fuzzer_harness.cc -lfoo` -then just switch `clang++` with `afl-clang-fast++` and our compiler will +You can also sneakily do this little trick: If this is the clang compile command +to build for libfuzzer: `clang++ -o fuzz -fsanitize=fuzzer fuzzer_harness.cc +-lfoo`, then just switch `clang++` with `afl-clang-fast++` and our compiler will magically insert libAFLDriver.a :) -To use shared-memory testcases, you need nothing to do. -To use stdin testcases give `-` as the only command line parameter. -To use file input testcases give `@@` as the only command line parameter. +To use shared-memory test cases, you need nothing to do. To use stdin test +cases, give `-` as the only command line parameter. To use file input test +cases, give `@@` as the only command line parameter. -IMPORTANT: if you use `afl-cmin` or `afl-cmin.bash` then either pass `-` -or `@@` as command line parameters. +IMPORTANT: if you use `afl-cmin` or `afl-cmin.bash`, then either pass `-` or +`@@` as command line parameters. ## aflpp_qemu_driver -Note that you can use the driver too for frida_mode (`-O`). +Note that you can use the driver too for FRIDA mode (`-O`). aflpp_qemu_driver is used for libfuzzer `LLVMFuzzerTestOneInput()` targets that -are to be fuzzed in qemu_mode. So we compile them with clang/clang++, without +are to be fuzzed in QEMU mode. So compile them with clang/clang++, without -fsantize=fuzzer or afl-clang-fast, and link in libAFLQemuDriver.a: `clang++ -o fuzz fuzzer_harness.cc libAFLQemuDriver.a [plus required linking]`. - Then just do (where the name of the binary is `fuzz`): + ``` AFL_QEMU_PERSISTENT_ADDR=0x$(nm fuzz | grep "T LLVMFuzzerTestOneInput" | awk '{print $1}') AFL_QEMU_PERSISTENT_HOOK=/path/to/aflpp_qemu_driver_hook.so afl-fuzz -Q ... -- ./fuzz` ``` if you use afl-cmin or `afl-showmap -C` with the aflpp_qemu_driver you need to -set the set same AFL_QEMU_... (or AFL_FRIDA_...) environment variables. -If you want to use afl-showmap (without -C) or afl-cmin.bash then you may not -set these environment variables and rather set `AFL_QEMU_DRIVER_NO_HOOK=1`. +set the set same AFL_QEMU_... (or AFL_FRIDA_...) environment variables. If you +want to use afl-showmap (without -C) or afl-cmin.bash, then you may not set +these environment variables and rather set `AFL_QEMU_DRIVER_NO_HOOK=1`. \ No newline at end of file diff --git a/utils/aflpp_driver/aflpp_qemu_driver.c b/utils/aflpp_driver/aflpp_qemu_driver.c index 99a4c9a8..e47df1e6 100644 --- a/utils/aflpp_driver/aflpp_qemu_driver.c +++ b/utils/aflpp_driver/aflpp_qemu_driver.c @@ -22,7 +22,7 @@ int main(int argc, char **argv) { if (LLVMFuzzerInitialize) LLVMFuzzerInitialize(&argc, &argv); // Do any other expensive one-time initialization here. - if (getenv("AFL_QEMU_DRIVER_NO_HOOK")) { + if (getenv("AFL_QEMU_DRIVER_NO_HOOK") || getenv("AFL_FRIDA_DRIVER_NO_HOOK")) { afl_qemu_driver_stdin_input(); diff --git a/utils/argv_fuzzing/README.md b/utils/argv_fuzzing/README.md index 192d72f7..e9224995 100644 --- a/utils/argv_fuzzing/README.md +++ b/utils/argv_fuzzing/README.md @@ -1,6 +1,6 @@ # argvfuzz -AFL supports fuzzing file inputs or stdin. When source is available, +AFL++ supports fuzzing file inputs or stdin. When source is available, `argv-fuzz-inl.h` can be used to change `main()` to build argv from stdin. `argvfuzz` tries to provide the same functionality for binaries. When loaded @@ -13,4 +13,4 @@ A few conditions need to be fulfilled for this mechanism to work correctly: 2. If the target binary does not use the default libc's `_start` implementation (crt1.o), the hook may not run. 3. The hook will replace argv with pointers to `.data` of `argvfuzz.so`. If the - target binary expects argv to be living on the stack, things may go wrong. + target binary expects argv to be living on the stack, things may go wrong. \ No newline at end of file diff --git a/utils/autodict_ql/build-codeql.sh b/utils/autodict_ql/build-codeql.sh index 6ae4b362..65ca838e 100644 --- a/utils/autodict_ql/build-codeql.sh +++ b/utils/autodict_ql/build-codeql.sh @@ -7,7 +7,7 @@ mkdir codeql-home cd codeql-home git clone https://github.com/github/codeql.git codeql-repo git clone https://github.com/github/codeql-go.git -wget https://github.com/github/codeql-cli-binaries/releases/download/v2.4.6/codeql-linux64.zip +wget https://github.com/github/codeql-cli-binaries/releases/download/v2.7.3/codeql-linux64.zip unzip codeql-linux64.zip mv codeql codeql-cli export "PATH=~/codeql-home/codeql-cli/:$PATH" diff --git a/utils/autodict_ql/readme.md b/utils/autodict_ql/readme.md index 9170f552..f61026b7 100644 --- a/utils/autodict_ql/readme.md +++ b/utils/autodict_ql/readme.md @@ -2,21 +2,35 @@ ## What is this? -`Autodict-QL` is a plugin system that enables fast generation of Tokens/Dictionaries in a handy way that can be manipulated by the user (unlike The LLVM Passes that are hard to modify). This means that autodict-ql is a scriptable feature which basically uses CodeQL (a powerful semantic code analysis engine) to fetch information from a code base. +`Autodict-QL` is a plugin system that enables fast generation of +Tokens/Dictionaries in a handy way that can be manipulated by the user (unlike +The LLVM Passes that are hard to modify). This means that autodict-ql is a +scriptable feature which basically uses CodeQL (a powerful semantic code +analysis engine) to fetch information from a code base. -Tokens are useful when you perform fuzzing on different parsers. The AFL++ `-x` switch enables the usage of dictionaries through your fuzzing campaign. If you are not familiar with Dictionaries in fuzzing, take a look [here](https://github.com/AFLplusplus/AFLplusplus/tree/stable/dictionaries) . +Tokens are useful when you perform fuzzing on different parsers. The AFL++ `-x` +switch enables the usage of dictionaries through your fuzzing campaign. If you +are not familiar with Dictionaries in fuzzing, take a look +[here](https://github.com/AFLplusplus/AFLplusplus/tree/stable/dictionaries). +## Why CodeQL? -## Why CodeQL ? -We basically developed this plugin on top of the CodeQL engine because it gives the user scripting features, it's easier and it's independent of the LLVM system. This means that a user can write his CodeQL scripts or modify the current scripts to improve or change the token generation algorithms based on different program analysis concepts. - +We basically developed this plugin on top of the CodeQL engine because it gives +the user scripting features, it's easier and it's independent of the LLVM +system. This means that a user can write his CodeQL scripts or modify the +current scripts to improve or change the token generation algorithms based on +different program analysis concepts. ## CodeQL scripts -Currently, we pushed some scripts as defaults for Token generation. In addition, we provide every CodeQL script as an standalone script because it's easier to modify or test. -Currently we provided the following CodeQL scripts : +Currently, we pushed some scripts as defaults for Token generation. In addition, +we provide every CodeQL script as an standalone script because it's easier to +modify or test. + +Currently we provided the following CodeQL scripts: -`strcmp-str.ql` is used to extract strings that are related to the `strcmp` function. +`strcmp-str.ql` is used to extract strings that are related to the `strcmp` +function. `strncmp-str.ql` is used to extract the strings from the `strncmp` function. @@ -24,77 +38,110 @@ Currently we provided the following CodeQL scripts : `litool.ql` extracts Magic numbers as Hexadecimal format. -`strtool.ql` extracts strings with uses of a regex and dataflow concept to capture the string comparison functions. If `strcmp` is rewritten in a project as Mystrcmp or something like strmycmp, then this script can catch the arguments and these are valuable tokens. - -You can write other CodeQL scripts to extract possible effective tokens if you think they can be useful. +`strtool.ql` extracts strings with uses of a regex and dataflow concept to +capture the string comparison functions. If `strcmp` is rewritten in a project +as Mystrcmp or something like strmycmp, then this script can catch the arguments +and these are valuable tokens. +You can write other CodeQL scripts to extract possible effective tokens if you +think they can be useful. ## Usage -Before you proceed to installation make sure that you have the following packages by installing them : -```shell -sudo apt install build-essential libtool-bin python3-dev python3 automake git vim wget -y -``` -The usage of Autodict-QL is pretty easy. But let's describe it as: +Before you proceed to installation make sure that you have the following +packages by installing them: -1. First of all, you need to have CodeQL installed on the system. we make this possible with `build-codeql.sh` bash script. This script will install CodeQL completety and will set the required environment variables for your system. -Do the following : ```shell -# chmod +x codeql-build.sh -# ./codeql-build.sh -# source ~/.bashrc -# codeql +sudo apt install build-essential libtool-bin python3-dev python3 automake git vim wget -y ``` -Then you should get: -```shell -Usage: codeql <command> <argument>... -Create and query CodeQL databases, or work with the QL language. - -GitHub makes this program freely available for the analysis of open-source software and certain other uses, but it is -not itself free software. Type codeql --license to see the license terms. - - --license Show the license terms for the CodeQL toolchain. -Common options: - -h, --help Show this help text. - -v, --verbose Incrementally increase the number of progress messages printed. - -q, --quiet Incrementally decrease the number of progress messages printed. -Some advanced options have been hidden; try --help -v for a fuller view. -Commands: - query Compile and execute QL code. - bqrs Get information from .bqrs files. - database Create, analyze and process CodeQL databases. - dataset [Plumbing] Work with raw QL datasets. - test Execute QL unit tests. - resolve [Deep plumbing] Helper commands to resolve disk locations etc. - execute [Deep plumbing] Low-level commands that need special JVM options. - version Show the version of the CodeQL toolchain. - generate Generate formatted QL documentation. - github Commands useful for interacting with the GitHub API through CodeQL. -``` +The usage of Autodict-QL is pretty easy. But let's describe it as: -2. Compile your project with CodeQL: For using the Autodict-QL plugin, you need to compile the source of the target you want to fuzz with CodeQL. This is not something hard. - - First you need to create a CodeQL database of the project codebase, suppose we want to compile `libxml` with codeql. Go to libxml and issue the following commands: - - `./configure --disable-shared` - - `codeql create database libxml-db --language=cpp --command=make` - - Now you have the CodeQL database of the project :-) -3. The final step is to update the CodeQL database you created in step 2 (Suppose we are in `aflplusplus/utils/autodict_ql/` directory): - - `codeql database upgrade /home/user/libxml/libxml-db` +1. First of all, you need to have CodeQL installed on the system. We make this + possible with `build-codeql.sh` bash script. This script will install CodeQL + completety and will set the required environment variables for your system. + Do the following: + + ```shell + # chmod +x codeql-build.sh + # ./codeql-build.sh + # source ~/.bashrc + # codeql + ``` + + Then you should get: + + ```shell + Usage: codeql <command> <argument>... + Create and query CodeQL databases, or work with the QL language. + + GitHub makes this program freely available for the analysis of open-source software and certain other uses, but it is + not itself free software. Type codeql --license to see the license terms. + + --license Show the license terms for the CodeQL toolchain. + Common options: + -h, --help Show this help text. + -v, --verbose Incrementally increase the number of progress messages printed. + -q, --quiet Incrementally decrease the number of progress messages printed. + Some advanced options have been hidden; try --help -v for a fuller view. + Commands: + query Compile and execute QL code. + bqrs Get information from .bqrs files. + database Create, analyze and process CodeQL databases. + dataset [Plumbing] Work with raw QL datasets. + test Execute QL unit tests. + resolve [Deep plumbing] Helper commands to resolve disk locations etc. + execute [Deep plumbing] Low-level commands that need special JVM options. + version Show the version of the CodeQL toolchain. + generate Generate formatted QL documentation. + github Commands useful for interacting with the GitHub API through CodeQL. + ``` + +2. Compile your project with CodeQL: For using the Autodict-QL plugin, you need + to compile the source of the target you want to fuzz with CodeQL. This is not + something hard. + - First you need to create a CodeQL database of the project codebase, suppose + we want to compile `libxml` with codeql. Go to libxml and issue the + following commands: + - `./configure --disable-shared` + - `codeql create database libxml-db --language=cpp --command=make` + - Now you have the CodeQL database of the project :-) +3. The final step is to update the CodeQL database you created in step 2 + (Suppose we are in `aflplusplus/utils/autodict_ql/` directory): + - `codeql database upgrade /home/user/libxml/libxml-db` 4. Everything is set! Now you should issue the following to get the tokens: - - `python3 autodict-ql.py [CURRECT_DIR] [CODEQL_DATABASE_PATH] [TOKEN_PATH]` - - example : `python3 /home/user/AFLplusplus/utils/autodict_ql/autodict-ql.py $PWD /home/user/libxml/libxml-db tokens` - - This will create the final `tokens` dir for you and you are done, then pass the tokens path to AFL++'s `-x` flag. -5. Done! - + - `python3 autodict-ql.py [CURRECT_DIR] [CODEQL_DATABASE_PATH] [TOKEN_PATH]` + - example: `python3 /home/user/AFLplusplus/utils/autodict_ql/autodict-ql.py + $PWD /home/user/libxml/libxml-db tokens` + - This will create the final `tokens` dir for you and you are done, then + pass the tokens path to AFL++'s `-x` flag. +5. Done! ## More on dictionaries and tokens -Core developer of the AFL++ project Marc Heuse also developed a similar tool named `dict2file` which is a LLVM pass which can automatically extract useful tokens, in addition with LTO instrumentation mode, this dict2file is automatically generates token extraction. `Autodict-QL` plugin gives you scripting capability and you can do whatever you want to extract from the Codebase and it's up to you. In addition it's independent from LLVM system. -On the other hand, you can also use Google dictionaries which have been made public in May 2020, but the problem of using Google dictionaries is that they are limited to specific file formats and specifications. For example, for testing binutils and ELF file format or AVI in FFMPEG, there are no prebuilt dictionaries, so it is highly recommended to use `Autodict-QL` or `Dict2File` features to automatically generate dictionaries based on the target. - -I've personally prefered to use `Autodict-QL` or `dict2file` rather than Google dictionaries or any other manually generated dictionaries as `Autodict-QL` and `dict2file` are working based on the target. -In overall, fuzzing with dictionaries and well-generated tokens will give better results. - -There are 2 important points to remember : -- If you combine `Autodict-QL` with AFL++ cmplog, you will get much better code coverage and hence better chances to discover new bugs. -- Do not forget to set `AFL_MAX_DET_EXTRAS` at least to the number of generated dictionaries. If you forget to set this environment variable, then AFL++ uses just 200 tokens and use the rest of them only probabilistically. So this will guarantee that your tokens will be used by AFL++. +Core developer of the AFL++ project Marc Heuse also developed a similar tool +named `dict2file` which is a LLVM pass which can automatically extract useful +tokens, in addition with LTO instrumentation mode, this dict2file is +automatically generates token extraction. `Autodict-QL` plugin gives you +scripting capability and you can do whatever you want to extract from the +Codebase and it's up to you. In addition it's independent from LLVM system. On +the other hand, you can also use Google dictionaries which have been made public +in May 2020, but the problem of using Google dictionaries is that they are +limited to specific file formats and specifications. For example, for testing +binutils and ELF file format or AVI in FFMPEG, there are no pre-built +dictionaries, so it is highly recommended to use `Autodict-QL` or `Dict2File` +features to automatically generate dictionaries based on the target. + +I've personally preferred to use `Autodict-QL` or `dict2file` rather than Google +dictionaries or any other manually generated dictionaries as `Autodict-QL` and +`dict2file` are working based on the target. In overall, fuzzing with +dictionaries and well-generated tokens will give better results. + +There are 2 important points to remember: + +- If you combine `Autodict-QL` with AFL++ cmplog, you will get much better code + coverage and hence better chances to discover new bugs. +- Do not forget to set `AFL_MAX_DET_EXTRAS` at least to the number of generated + dictionaries. If you forget to set this environment variable, then AFL++ uses + just 200 tokens and use the rest of them only probabilistically. So this will + guarantee that your tokens will be used by AFL++. \ No newline at end of file diff --git a/utils/libdislocator/README.md b/utils/libdislocator/README.md index d0340af0..7150c205 100644 --- a/utils/libdislocator/README.md +++ b/utils/libdislocator/README.md @@ -10,8 +10,8 @@ heap-related security bugs in several ways: subsequent PROT_NONE page, causing most off-by-one reads and writes to immediately segfault, - - It adds a canary immediately below the allocated buffer, to catch writes - to negative offsets (won't catch reads, though), + - It adds a canary immediately below the allocated buffer, to catch writes to + negative offsets (won't catch reads, though), - It sets the memory returned by malloc() to garbage values, improving the odds of crashing when the target accesses uninitialized data, @@ -19,37 +19,36 @@ heap-related security bugs in several ways: - It sets freed memory to PROT_NONE and does not actually reuse it, causing most use-after-free bugs to segfault right away, - - It forces all realloc() calls to return a new address - and sets - PROT_NONE on the original block. This catches use-after-realloc bugs, + - It forces all realloc() calls to return a new address - and sets PROT_NONE + on the original block. This catches use-after-realloc bugs, - - It checks for calloc() overflows and can cause soft or hard failures - of alloc requests past a configurable memory limit (AFL_LD_LIMIT_MB, + - It checks for calloc() overflows and can cause soft or hard failures of + alloc requests past a configurable memory limit (AFL_LD_LIMIT_MB, AFL_LD_HARD_FAIL). - Optionally, in platforms supporting it, huge pages can be used by passing USEHUGEPAGE=1 to make. - - - Size alignment to `max_align_t` can be enforced with AFL_ALIGNED_ALLOC=1. - In this case, a tail canary is inserted in the padding bytes at the end - of the allocated zone. This reduce the ability of libdislocator to detect + + - Size alignment to `max_align_t` can be enforced with AFL_ALIGNED_ALLOC=1. In + this case, a tail canary is inserted in the padding bytes at the end of the + allocated zone. This reduce the ability of libdislocator to detect off-by-one bugs but also it make slibdislocator compliant to the C standard. Basically, it is inspired by some of the non-default options available for the OpenBSD allocator - see malloc.conf(5) on that platform for reference. It is -also somewhat similar to several other debugging libraries, such as gmalloc -and DUMA - but is simple, plug-and-play, and designed specifically for fuzzing -jobs. +also somewhat similar to several other debugging libraries, such as gmalloc and +DUMA - but is simple, plug-and-play, and designed specifically for fuzzing jobs. Note that it does nothing for stack-based memory handling errors. The -fstack-protector-all setting for GCC / clang, enabled when using AFL_HARDEN, can catch some subset of that. The allocator is slow and memory-intensive (even the tiniest allocation uses up -4 kB of physical memory and 8 kB of virtual mem), making it completely unsuitable -for "production" uses; but it can be faster and more hassle-free than ASAN / MSAN -when fuzzing small, self-contained binaries. +4 kB of physical memory and 8 kB of virtual mem), making it completely +unsuitable for "production" uses; but it can be faster and more hassle-free than +ASAN / MSAN when fuzzing small, self-contained binaries. -To use this library, run AFL like so: +To use this library, run AFL++ like so: ``` AFL_PRELOAD=/path/to/libdislocator.so ./afl-fuzz [...other params...] @@ -62,7 +61,7 @@ Similarly to afl-tmin, the library is not "proprietary" and can be used with other fuzzers or testing tools without the need for any code tweaks. It does not require AFL-instrumented binaries to work. -Note that the AFL_PRELOAD approach (which AFL internally maps to LD_PRELOAD or +Note that the AFL_PRELOAD approach (which AFL++ internally maps to LD_PRELOAD or DYLD_INSERT_LIBRARIES, depending on the OS) works only if the target binary is dynamically linked. Otherwise, attempting to use the library will have no -effect. +effect. \ No newline at end of file diff --git a/utils/libtokencap/README.md b/utils/libtokencap/README.md index a39ed3a5..343fcce0 100644 --- a/utils/libtokencap/README.md +++ b/utils/libtokencap/README.md @@ -40,10 +40,10 @@ when using afl-gcc. This setting specifically adds the following flags: -fno-builtin-strcasestr ``` -The next step is simply loading this library via LD_PRELOAD. The optimal usage -pattern is to allow afl-fuzz to fuzz normally for a while and build up a corpus, -and then fire off the target binary, with libtokencap.so loaded, on every file -found by AFL in that earlier run. This demonstrates the basic principle: +The next step is to load this library via LD_PRELOAD. The optimal usage pattern +is to allow afl-fuzz to fuzz normally for a while and build up a corpus, and +then fire off the target binary, with libtokencap.so loaded, on every file found +by AFL++ in that earlier run. This demonstrates the basic principle: ``` export AFL_TOKEN_FILE=$PWD/temp_output.txt diff --git a/utils/libtokencap/libtokencap.so.c b/utils/libtokencap/libtokencap.so.c index 3629e804..2b1e3903 100644 --- a/utils/libtokencap/libtokencap.so.c +++ b/utils/libtokencap/libtokencap.so.c @@ -33,6 +33,8 @@ #include "../types.h" #include "../config.h" +#include "debug.h" + #if !defined __linux__ && !defined __APPLE__ && !defined __FreeBSD__ && \ !defined __OpenBSD__ && !defined __NetBSD__ && !defined __DragonFly__ && \ !defined(__HAIKU__) && !defined(__sun) @@ -169,7 +171,7 @@ static void __tokencap_load_mappings(void) { int mib[] = {CTL_VM, VM_PROC, VM_PROC_MAP, __tokencap_pid, sizeof(struct kinfo_vmentry)}; #endif - char * buf, *low, *high; + char *buf, *low, *high; size_t miblen = sizeof(mib) / sizeof(mib[0]); size_t len; @@ -343,6 +345,12 @@ static void __tokencap_dump(const u8 *ptr, size_t len, u8 is_text) { wrt_ok &= (pos == write(__tokencap_out_file, buf, pos)); wrt_ok &= (2 == write(__tokencap_out_file, "\"\n", 2)); + if (!wrt_ok) { + + DEBUGF("%s", "writing to the token file failed\n"); + + } + } /* Replacements for strcmp(), memcmp(), and so on. Note that these will be used diff --git a/utils/optimin/README.md b/utils/optimin/README.md index c6f2af06..340022b8 100644 --- a/utils/optimin/README.md +++ b/utils/optimin/README.md @@ -73,11 +73,11 @@ and `WEIGHT_N` is an integer weight. ## Further Details and Citation -For more details, please see the paper [Seed Selection for Successful -Fuzzing](https://dl.acm.org/doi/10.1145/3460319.3464795). If you use OptiMin in -your research, please cite this paper. +For more details, see the paper +[Seed Selection for Successful Fuzzing](https://dl.acm.org/doi/10.1145/3460319.3464795). +If you use OptiMin in your research, please cite this paper. -Bibtex: +BibTeX: ```bibtex @inproceedings{Herrera:2021:FuzzSeedSelection, @@ -91,4 +91,4 @@ Bibtex: location = {Virtual, Denmark}, publisher = {Association for Computing Machinery}, } -``` +``` \ No newline at end of file diff --git a/utils/optimin/build_optimin.sh b/utils/optimin/build_optimin.sh index 9480f966..51d1bd26 100755 --- a/utils/optimin/build_optimin.sh +++ b/utils/optimin/build_optimin.sh @@ -102,7 +102,7 @@ else CNT=1 while [ '!' -d EvalMaxSAT -a "$CNT" -lt 4 ]; do echo "Trying to clone EvalMaxSAT (attempt $CNT/3)" - git clone "$GRAMMAR_REPO" + git clone "$EVALMAXSAT_REPO" CNT=`expr "$CNT" + 1` done } diff --git a/utils/optimin/src/CMakeLists.txt b/utils/optimin/src/CMakeLists.txt index f31ceeaf..693f63f2 100644 --- a/utils/optimin/src/CMakeLists.txt +++ b/utils/optimin/src/CMakeLists.txt @@ -1,4 +1,5 @@ add_executable(optimin OptiMin.cpp) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") foreach(LIB MaLib EvalMaxSAT glucose) target_include_directories(optimin PRIVATE diff --git a/utils/qbdi_mode/README.md b/utils/qbdi_mode/README.md index 8b768906..08558017 100755 --- a/utils/qbdi_mode/README.md +++ b/utils/qbdi_mode/README.md @@ -2,14 +2,13 @@ NOTE: this code is outdated and first would need to be adapted to the current AFL++ versions. -Try frida_mode or fpicker [https://github.com/ttdennis/fpicker/](https://github.com/ttdennis/fpicker/) first, maybe they suite your need. +Try FRIDA mode or fpicker [https://github.com/ttdennis/fpicker/](https://github.com/ttdennis/fpicker/) first, maybe they suite your need. ## 1) Introduction The code in ./qbdi_mode allows you to build a standalone feature that using the QBDI framework to fuzz android native library. - ## 2) Build First download the Android NDK @@ -34,7 +33,9 @@ For x86 standalone-toolchain ./build/tools/make_standalone_toolchain.py --arch x86 --api 21 --install-dir ../android-standalone-toolchain-x86 ``` -In alternative you can also use the prebuilt toolchain, in that case make sure to set the proper CC and CXX env variables because there are many different compilers for each API version in the prebuilt toolchain. +In alternative you can also use the pre-built toolchain, in that case make sure +to set the proper CC and CXX environment variables because there are many +different compilers for each API version in the pre-built toolchain. For example: @@ -51,6 +52,7 @@ https://qbdi.quarkslab.com/ ``` For Android x86_64 + ``` https://github.com/QBDI/QBDI/releases/download/v0.7.0/QBDI-0.7.0-android-X86_64.tar.gz ``` @@ -64,7 +66,7 @@ cd android-qbdi-sdk-x86_64/ tar xvf QBDI-0.7.0-android-X86_64.tar.gz ``` -Now set the `STANDALONE_TOOLCHAIN_PATH` to the path of standalone-toolchain +Now set the `STANDALONE_TOOLCHAIN_PATH` to the path of standalone-toolchain ``` export STANDALONE_TOOLCHAIN_PATH=/home/hac425/workspace/android-standalone-toolchain-x86_64 @@ -84,7 +86,6 @@ Then run the build.sh this could build the afl-fuzz and also the qbdi template for android x86_64 - ### Example The demo-so.c is an vulnerable library, it has a function for test @@ -127,9 +128,10 @@ int target_func(char *buf, int size) { } ``` -This could be build to `libdemo.so`. +This could be built to `libdemo.so`. + +Then load the library in template.cpp and find the `target` function address: -Then we should load the library in template.cpp and find the `target` function address. ```c void *handle = dlopen(lib_path, RTLD_LAZY); .......................................... @@ -138,7 +140,7 @@ Then we should load the library in template.cpp and find the `target` function a p_target_func = (target_func)dlsym(handle, "target_func"); ``` -then we read the data from file and call the function in `fuzz_func` +Then read the data from file and call the function in `fuzz_func`: ```c QBDI_NOINLINE int fuzz_func() { @@ -157,6 +159,7 @@ QBDI_NOINLINE int fuzz_func() { ``` Just compile it + ``` ./build.sh x86_64 ``` @@ -173,6 +176,7 @@ adb push ../../android-standalone-toolchain-x86_64/sysroot/usr/lib/x86_64-linux- ``` In android adb shell, run the loader to test if it runs + ``` cd /data/local/tmp export LD_LIBRARY_PATH=/data/local/tmp @@ -199,5 +203,4 @@ Now run `afl-fuzz` to fuzz the demo library ./afl-fuzz -i in -o out -- ./loader /data/local/tmp/libdemo.so @@ ``` - - + \ No newline at end of file diff --git a/utils/qemu_persistent_hook/README.md b/utils/qemu_persistent_hook/README.md index 3f908c22..3bbaef6b 100644 --- a/utils/qemu_persistent_hook/README.md +++ b/utils/qemu_persistent_hook/README.md @@ -16,4 +16,4 @@ mkdir in echo 0000 > in/in ../../afl-fuzz -Q -i in -o out -- ./test -``` +``` \ No newline at end of file diff --git a/utils/socket_fuzzing/README.md b/utils/socket_fuzzing/README.md index 84398a71..2805fa22 100644 --- a/utils/socket_fuzzing/README.md +++ b/utils/socket_fuzzing/README.md @@ -6,6 +6,6 @@ for sending input to stdin which the target binary will think is coming from a network socket. This is desock_dup.c from the amazing preeny project -https://github.com/zardus/preeny +[https://github.com/zardus/preeny](https://github.com/zardus/preeny) It is packaged in AFL++ to have it at hand if needed |